From b8435ba8e67b2cd8273103eeea81adc551aa4316 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 25 Jun 2026 15:00:05 -0700 Subject: [PATCH 1/4] fix(cost): preserve Anthropic server_tool_use web search usage in cost tracking Anthropic /v1/messages responses report built-in web search usage under usage.server_tool_use.web_search_requests, but the sync cost path reconstructs an OpenAI-shape Usage that drops server_tool_use and validates the response through AnthropicResponse, which previously stripped the field. Either path could leave the web-search fee uncounted. AnthropicResponseUsageBlock now allows extra fields so model_validate/model_dump keeps server_tool_use, and the built-in tool cost tracker reads the web search count straight off the raw Anthropic response dict when the reconstructed Usage lacks it, synthesizing a ServerToolUse without mutating the caller's Usage. --- .../llm_cost_calc/tool_call_cost_tracking.py | 76 ++++++++++++++++++- litellm/types/llms/anthropic.py | 2 + .../test_tool_call_cost_tracking.py | 72 ++++++++++++++++++ 3 files changed, 148 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index 413ddb71bf8..7910474dcf4 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -4,6 +4,8 @@ from typing import Any, Dict, List, Literal, Optional, Tuple +from pydantic import BaseModel, ValidationError + import litellm from litellm.constants import OPENAI_FILE_SEARCH_COST_PER_1K_CALLS from litellm.litellm_core_utils.llm_cost_calc.utils import _get_web_search_requests @@ -17,11 +19,24 @@ ModelInfo, ModelResponse, SearchContextCostPerQuery, + ServerToolUse, StandardBuiltInToolsParams, Usage, ) +class _AnthropicServerToolUseProbe(BaseModel): + web_search_requests: Optional[int] = None + + +class _AnthropicUsageProbe(BaseModel): + server_tool_use: Optional[_AnthropicServerToolUseProbe] = None + + +class _AnthropicResponseProbe(BaseModel): + usage: Optional[_AnthropicUsageProbe] = None + + class StandardBuiltInToolCostTracking: """ Helper class for tracking the cost of built-in tools @@ -58,6 +73,7 @@ def get_cost_for_built_in_tools( custom_llm_provider=custom_llm_provider, usage=usage, standard_built_in_tools_params=standard_built_in_tools_params, + response_object=response_object, ) # Handle file search @@ -83,6 +99,7 @@ def _handle_web_search_cost( custom_llm_provider: Optional[str], usage: Optional[Usage], standard_built_in_tools_params: StandardBuiltInToolsParams, + response_object: object = None, ) -> float: """Handle web search cost calculation.""" from litellm.llms import get_cost_for_web_search_request @@ -94,14 +111,20 @@ def _handle_web_search_cost( if custom_llm_provider is None and model_info is not None: custom_llm_provider = model_info["litellm_provider"] + resolved_usage = ( + StandardBuiltInToolCostTracking._usage_with_anthropic_web_search( + usage=usage, response_object=response_object + ) + ) + if ( model_info is not None - and usage is not None + and resolved_usage is not None and custom_llm_provider is not None ): result = get_cost_for_web_search_request( custom_llm_provider=custom_llm_provider, - usage=usage, + usage=resolved_usage, model_info=model_info, ) if result is not None: @@ -301,6 +324,48 @@ def _safe_convert_to_int(value: Any) -> Optional[int]: return None return None + @staticmethod + def _anthropic_web_search_count(response_object: object) -> Optional[int]: + """Read usage.server_tool_use.web_search_requests from a raw Anthropic + /v1/messages response dict, returning None when absent.""" + if not isinstance(response_object, dict): + return None + try: + probe = _AnthropicResponseProbe.model_validate(response_object) + except ValidationError: + return None + if probe.usage is None or probe.usage.server_tool_use is None: + return None + return probe.usage.server_tool_use.web_search_requests + + @staticmethod + def _usage_with_anthropic_web_search( + usage: Optional[Usage], response_object: object + ) -> Optional[Usage]: + """Return a Usage carrying server_tool_use.web_search_requests sourced from a + raw Anthropic /v1/messages response dict when the reconstructed Usage dropped + it. The original Usage is returned unchanged when it already exposes the field + or the response is not an Anthropic dict.""" + if usage is None: + return None + if ( + _get_web_search_requests(getattr(usage, "server_tool_use", None)) + is not None + ): + return usage + web_search_requests = ( + StandardBuiltInToolCostTracking._anthropic_web_search_count(response_object) + ) + if web_search_requests is None: + return usage + return usage.model_copy( + update={ + "server_tool_use": ServerToolUse( + web_search_requests=web_search_requests + ) + } + ) + @staticmethod def response_object_includes_web_search_call( response_object: Any, usage: Optional[Usage] = None @@ -311,9 +376,16 @@ def response_object_includes_web_search_call( This covers: - Chat Completion Response (ModelResponse) - ResponsesAPIResponse (streaming + non-streaming) + - Anthropic /v1/messages raw response dict """ from litellm.types.utils import PromptTokensDetailsWrapper + if ( + StandardBuiltInToolCostTracking._anthropic_web_search_count(response_object) + is not None + ): + return True + if isinstance(response_object, ModelResponse): # chat completions only include url_citation annotations when a web search call is made has_url_citations = ( diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index a4a059dc88a..561d3692ee2 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -625,6 +625,8 @@ class AnthropicResponseContentBlockRedactedThinking(BaseModel): class AnthropicResponseUsageBlock(BaseModel): + model_config = ConfigDict(extra="allow") + input_tokens: int output_tokens: int diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index c43291566b6..0bbace96114 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -152,6 +152,78 @@ def test_get_cost_for_anthropic_web_search_with_server_tool_use_dict(): ) +def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_drops_server_tool_use(): + """ + Regression: on the Anthropic /v1/messages sync cost path the response is the raw + Anthropic dict while the reconstructed OpenAI-shape Usage drops server_tool_use. + The web-search fee must still be charged by reading the count off the raw dict, + and the passed-in Usage must not be mutated. + """ + from litellm.types.utils import Usage + + model = "claude-3-7-sonnet-20250219" + web_search_requests = 3 + raw_response = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "server_tool_use": {"web_search_requests": web_search_requests}, + }, + } + usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) + assert getattr(usage, "server_tool_use", None) is None + + cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=model, + usage=usage, + response_object=raw_response, + custom_llm_provider="anthropic", + standard_built_in_tools_params=None, + ) + + per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"][ + "search_context_size_medium" + ] + assert cost == per_query_cost * web_search_requests + assert cost > 0.0 + assert getattr(usage, "server_tool_use", None) is None + + +def test_anthropic_response_usage_block_preserves_server_tool_use(): + """ + Regression: AnthropicResponse.model_validate(...).model_dump() must keep + server_tool_use so the /v1/messages logging fallback does not strip the + web-search usage before cost tracking sees it. + """ + from litellm.types.llms.anthropic import AnthropicResponse + + raw_response = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-3-7-sonnet-20250219", + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "server_tool_use": {"web_search_requests": 2}, + }, + } + + dumped_usage = AnthropicResponse.model_validate(raw_response).model_dump()["usage"] + + assert dumped_usage["server_tool_use"] == {"web_search_requests": 2} + + @pytest.mark.parametrize( "model", ["gemini/gemini-2.0-flash-001", "gemini-2.0-flash-001"] ) From d48a84e0d936307819872f3587fca66a8e5a0a54 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 25 Jun 2026 15:48:08 -0700 Subject: [PATCH 2/4] fix(lint): use PEP 604 unions in anthropic web search probes to satisfy strict-rule budget --- .../llm_cost_calc/tool_call_cost_tracking.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index 7910474dcf4..71536e1051d 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -26,15 +26,15 @@ class _AnthropicServerToolUseProbe(BaseModel): - web_search_requests: Optional[int] = None + web_search_requests: int | None = None class _AnthropicUsageProbe(BaseModel): - server_tool_use: Optional[_AnthropicServerToolUseProbe] = None + server_tool_use: _AnthropicServerToolUseProbe | None = None class _AnthropicResponseProbe(BaseModel): - usage: Optional[_AnthropicUsageProbe] = None + usage: _AnthropicUsageProbe | None = None class StandardBuiltInToolCostTracking: @@ -325,7 +325,7 @@ def _safe_convert_to_int(value: Any) -> Optional[int]: return None @staticmethod - def _anthropic_web_search_count(response_object: object) -> Optional[int]: + def _anthropic_web_search_count(response_object: object) -> int | None: """Read usage.server_tool_use.web_search_requests from a raw Anthropic /v1/messages response dict, returning None when absent.""" if not isinstance(response_object, dict): @@ -340,8 +340,8 @@ def _anthropic_web_search_count(response_object: object) -> Optional[int]: @staticmethod def _usage_with_anthropic_web_search( - usage: Optional[Usage], response_object: object - ) -> Optional[Usage]: + usage: Usage | None, response_object: object + ) -> Usage | None: """Return a Usage carrying server_tool_use.web_search_requests sourced from a raw Anthropic /v1/messages response dict when the reconstructed Usage dropped it. The original Usage is returned unchanged when it already exposes the field From 2833b1262e5717da6c26e7842f748f626cfe899c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 26 Jun 2026 00:25:58 +0000 Subject: [PATCH 3/4] refactor(cost): move Anthropic web search response parsing into llms/anthropic Relocate the raw /v1/messages web-search-count probe out of the shared built-in tool cost tracker into litellm/llms/anthropic/cost_calculation.py, next to get_cost_for_anthropic_web_search, so provider-specific response parsing lives under llms/. The core cost tracker now delegates to get_anthropic_web_search_requests_from_response and keeps only the generic Usage/ServerToolUse orchestration. --- .../llm_cost_calc/tool_call_cost_tracking.py | 44 +++++-------------- litellm/llms/anthropic/cost_calculation.py | 30 +++++++++++++ 2 files changed, 40 insertions(+), 34 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index 71536e1051d..b3d50279461 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -4,8 +4,6 @@ from typing import Any, Dict, List, Literal, Optional, Tuple -from pydantic import BaseModel, ValidationError - import litellm from litellm.constants import OPENAI_FILE_SEARCH_COST_PER_1K_CALLS from litellm.litellm_core_utils.llm_cost_calc.utils import _get_web_search_requests @@ -25,18 +23,6 @@ ) -class _AnthropicServerToolUseProbe(BaseModel): - web_search_requests: int | None = None - - -class _AnthropicUsageProbe(BaseModel): - server_tool_use: _AnthropicServerToolUseProbe | None = None - - -class _AnthropicResponseProbe(BaseModel): - usage: _AnthropicUsageProbe | None = None - - class StandardBuiltInToolCostTracking: """ Helper class for tracking the cost of built-in tools @@ -324,20 +310,6 @@ def _safe_convert_to_int(value: Any) -> Optional[int]: return None return None - @staticmethod - def _anthropic_web_search_count(response_object: object) -> int | None: - """Read usage.server_tool_use.web_search_requests from a raw Anthropic - /v1/messages response dict, returning None when absent.""" - if not isinstance(response_object, dict): - return None - try: - probe = _AnthropicResponseProbe.model_validate(response_object) - except ValidationError: - return None - if probe.usage is None or probe.usage.server_tool_use is None: - return None - return probe.usage.server_tool_use.web_search_requests - @staticmethod def _usage_with_anthropic_web_search( usage: Usage | None, response_object: object @@ -346,6 +318,10 @@ def _usage_with_anthropic_web_search( raw Anthropic /v1/messages response dict when the reconstructed Usage dropped it. The original Usage is returned unchanged when it already exposes the field or the response is not an Anthropic dict.""" + from litellm.llms.anthropic.cost_calculation import ( + get_anthropic_web_search_requests_from_response, + ) + if usage is None: return None if ( @@ -353,8 +329,8 @@ def _usage_with_anthropic_web_search( is not None ): return usage - web_search_requests = ( - StandardBuiltInToolCostTracking._anthropic_web_search_count(response_object) + web_search_requests = get_anthropic_web_search_requests_from_response( + response_object ) if web_search_requests is None: return usage @@ -378,12 +354,12 @@ def response_object_includes_web_search_call( - ResponsesAPIResponse (streaming + non-streaming) - Anthropic /v1/messages raw response dict """ + from litellm.llms.anthropic.cost_calculation import ( + get_anthropic_web_search_requests_from_response, + ) from litellm.types.utils import PromptTokensDetailsWrapper - if ( - StandardBuiltInToolCostTracking._anthropic_web_search_count(response_object) - is not None - ): + if get_anthropic_web_search_requests_from_response(response_object) is not None: return True if isinstance(response_object, ModelResponse): diff --git a/litellm/llms/anthropic/cost_calculation.py b/litellm/llms/anthropic/cost_calculation.py index 44081ea9e79..fc34938b281 100644 --- a/litellm/llms/anthropic/cost_calculation.py +++ b/litellm/llms/anthropic/cost_calculation.py @@ -5,6 +5,8 @@ from typing import TYPE_CHECKING, Optional, Tuple +from pydantic import BaseModel, ValidationError + from litellm.litellm_core_utils.llm_cost_calc.utils import ( _get_token_base_cost, _get_web_search_requests, @@ -111,6 +113,34 @@ def cost_per_token( return prompt_cost, completion_cost +class _AnthropicServerToolUseProbe(BaseModel): + web_search_requests: int | None = None + + +class _AnthropicUsageProbe(BaseModel): + server_tool_use: _AnthropicServerToolUseProbe | None = None + + +class _AnthropicResponseProbe(BaseModel): + usage: _AnthropicUsageProbe | None = None + + +def get_anthropic_web_search_requests_from_response( + response_object: object, +) -> int | None: + """Read usage.server_tool_use.web_search_requests from a raw Anthropic + /v1/messages response dict, returning None when absent.""" + if not isinstance(response_object, dict): + return None + try: + probe = _AnthropicResponseProbe.model_validate(response_object) + except ValidationError: + return None + if probe.usage is None or probe.usage.server_tool_use is None: + return None + return probe.usage.server_tool_use.web_search_requests + + def get_cost_for_anthropic_web_search( model_info: Optional["ModelInfo"] = None, usage: Optional["Usage"] = None, From d282dd23687ebba830af25850fb0dadbe9942b12 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 26 Jun 2026 00:41:28 +0000 Subject: [PATCH 4/4] fix(cost): price Anthropic web search when only the raw response carries the count response_object_includes_web_search_call enters the web search branch as soon as the raw Anthropic dict reports usage.server_tool_use.web_search_requests, but _usage_with_anthropic_web_search bailed when the caller did not also pass a Usage object. _handle_web_search_cost then skipped the per-request anthropic path and fell back to the flat search_context_size_medium tier, charging a fixed fee instead of per_query x count (or zero when the count is zero). Synthesize a Usage from the raw dict when no Usage is supplied so count-based pricing runs uniformly regardless of how the response reaches the tracker. --- .../llm_cost_calc/tool_call_cost_tracking.py | 19 ++--- .../test_tool_call_cost_tracking.py | 70 +++++++++++++++++++ 2 files changed, 77 insertions(+), 12 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index b3d50279461..365efe7903c 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -316,15 +316,13 @@ def _usage_with_anthropic_web_search( ) -> Usage | None: """Return a Usage carrying server_tool_use.web_search_requests sourced from a raw Anthropic /v1/messages response dict when the reconstructed Usage dropped - it. The original Usage is returned unchanged when it already exposes the field - or the response is not an Anthropic dict.""" + it (or was never supplied). The original Usage is returned unchanged when it + already exposes the field or the response is not an Anthropic dict.""" from litellm.llms.anthropic.cost_calculation import ( get_anthropic_web_search_requests_from_response, ) - if usage is None: - return None - if ( + if usage is not None and ( _get_web_search_requests(getattr(usage, "server_tool_use", None)) is not None ): @@ -334,13 +332,10 @@ def _usage_with_anthropic_web_search( ) if web_search_requests is None: return usage - return usage.model_copy( - update={ - "server_tool_use": ServerToolUse( - web_search_requests=web_search_requests - ) - } - ) + server_tool_use = ServerToolUse(web_search_requests=web_search_requests) + if usage is None: + return Usage(server_tool_use=server_tool_use) + return usage.model_copy(update={"server_tool_use": server_tool_use}) @staticmethod def response_object_includes_web_search_call( diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index 0bbace96114..fff81aaf7b3 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -196,6 +196,76 @@ def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_drops_serve assert getattr(usage, "server_tool_use", None) is None +def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_is_none(): + """ + Regression: when a caller hands the cost tracker a raw Anthropic dict without a + parallel Usage object, the web-search fee must still be priced per request from + usage.server_tool_use.web_search_requests on the dict instead of falling back to + the flat search_context_size_medium tier. + """ + model = "claude-3-7-sonnet-20250219" + web_search_requests = 4 + raw_response = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "server_tool_use": {"web_search_requests": web_search_requests}, + }, + } + + cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=model, + usage=None, + response_object=raw_response, + custom_llm_provider="anthropic", + standard_built_in_tools_params=None, + ) + + per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"][ + "search_context_size_medium" + ] + assert cost == per_query_cost * web_search_requests + + +def test_anthropic_web_search_zero_requests_from_raw_response_charges_zero(): + """ + Regression: a raw Anthropic dict reporting zero web search requests must price + the call at zero rather than charging the default medium-tier fee. + """ + model = "claude-3-7-sonnet-20250219" + raw_response = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "server_tool_use": {"web_search_requests": 0}, + }, + } + + cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=model, + usage=None, + response_object=raw_response, + custom_llm_provider="anthropic", + standard_built_in_tools_params=None, + ) + + assert cost == 0.0 + + def test_anthropic_response_usage_block_preserves_server_tool_use(): """ Regression: AnthropicResponse.model_validate(...).model_dump() must keep