diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index 413ddb71bf8..365efe7903c 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -17,6 +17,7 @@ ModelInfo, ModelResponse, SearchContextCostPerQuery, + ServerToolUse, StandardBuiltInToolsParams, Usage, ) @@ -58,6 +59,7 @@ def get_cost_for_built_in_tools( custom_llm_provider=custom_llm_provider, usage=usage, standard_built_in_tools_params=standard_built_in_tools_params, + response_object=response_object, ) # Handle file search @@ -83,6 +85,7 @@ def _handle_web_search_cost( custom_llm_provider: Optional[str], usage: Optional[Usage], standard_built_in_tools_params: StandardBuiltInToolsParams, + response_object: object = None, ) -> float: """Handle web search cost calculation.""" from litellm.llms import get_cost_for_web_search_request @@ -94,14 +97,20 @@ def _handle_web_search_cost( if custom_llm_provider is None and model_info is not None: custom_llm_provider = model_info["litellm_provider"] + resolved_usage = ( + StandardBuiltInToolCostTracking._usage_with_anthropic_web_search( + usage=usage, response_object=response_object + ) + ) + if ( model_info is not None - and usage is not None + and resolved_usage is not None and custom_llm_provider is not None ): result = get_cost_for_web_search_request( custom_llm_provider=custom_llm_provider, - usage=usage, + usage=resolved_usage, model_info=model_info, ) if result is not None: @@ -301,6 +310,33 @@ def _safe_convert_to_int(value: Any) -> Optional[int]: return None return None + @staticmethod + def _usage_with_anthropic_web_search( + usage: Usage | None, response_object: object + ) -> Usage | None: + """Return a Usage carrying server_tool_use.web_search_requests sourced from a + raw Anthropic /v1/messages response dict when the reconstructed Usage dropped + it (or was never supplied). The original Usage is returned unchanged when it + already exposes the field or the response is not an Anthropic dict.""" + from litellm.llms.anthropic.cost_calculation import ( + get_anthropic_web_search_requests_from_response, + ) + + if usage is not None and ( + _get_web_search_requests(getattr(usage, "server_tool_use", None)) + is not None + ): + return usage + web_search_requests = get_anthropic_web_search_requests_from_response( + response_object + ) + if web_search_requests is None: + return usage + server_tool_use = ServerToolUse(web_search_requests=web_search_requests) + if usage is None: + return Usage(server_tool_use=server_tool_use) + return usage.model_copy(update={"server_tool_use": server_tool_use}) + @staticmethod def response_object_includes_web_search_call( response_object: Any, usage: Optional[Usage] = None @@ -311,9 +347,16 @@ def response_object_includes_web_search_call( This covers: - Chat Completion Response (ModelResponse) - ResponsesAPIResponse (streaming + non-streaming) + - Anthropic /v1/messages raw response dict """ + from litellm.llms.anthropic.cost_calculation import ( + get_anthropic_web_search_requests_from_response, + ) from litellm.types.utils import PromptTokensDetailsWrapper + if get_anthropic_web_search_requests_from_response(response_object) is not None: + return True + if isinstance(response_object, ModelResponse): # chat completions only include url_citation annotations when a web search call is made has_url_citations = ( diff --git a/litellm/llms/anthropic/cost_calculation.py b/litellm/llms/anthropic/cost_calculation.py index 44081ea9e79..fc34938b281 100644 --- a/litellm/llms/anthropic/cost_calculation.py +++ b/litellm/llms/anthropic/cost_calculation.py @@ -5,6 +5,8 @@ from typing import TYPE_CHECKING, Optional, Tuple +from pydantic import BaseModel, ValidationError + from litellm.litellm_core_utils.llm_cost_calc.utils import ( _get_token_base_cost, _get_web_search_requests, @@ -111,6 +113,34 @@ def cost_per_token( return prompt_cost, completion_cost +class _AnthropicServerToolUseProbe(BaseModel): + web_search_requests: int | None = None + + +class _AnthropicUsageProbe(BaseModel): + server_tool_use: _AnthropicServerToolUseProbe | None = None + + +class _AnthropicResponseProbe(BaseModel): + usage: _AnthropicUsageProbe | None = None + + +def get_anthropic_web_search_requests_from_response( + response_object: object, +) -> int | None: + """Read usage.server_tool_use.web_search_requests from a raw Anthropic + /v1/messages response dict, returning None when absent.""" + if not isinstance(response_object, dict): + return None + try: + probe = _AnthropicResponseProbe.model_validate(response_object) + except ValidationError: + return None + if probe.usage is None or probe.usage.server_tool_use is None: + return None + return probe.usage.server_tool_use.web_search_requests + + def get_cost_for_anthropic_web_search( model_info: Optional["ModelInfo"] = None, usage: Optional["Usage"] = None, diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index a4a059dc88a..561d3692ee2 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -625,6 +625,8 @@ class AnthropicResponseContentBlockRedactedThinking(BaseModel): class AnthropicResponseUsageBlock(BaseModel): + model_config = ConfigDict(extra="allow") + input_tokens: int output_tokens: int diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index c43291566b6..fff81aaf7b3 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -152,6 +152,148 @@ def test_get_cost_for_anthropic_web_search_with_server_tool_use_dict(): ) +def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_drops_server_tool_use(): + """ + Regression: on the Anthropic /v1/messages sync cost path the response is the raw + Anthropic dict while the reconstructed OpenAI-shape Usage drops server_tool_use. + The web-search fee must still be charged by reading the count off the raw dict, + and the passed-in Usage must not be mutated. + """ + from litellm.types.utils import Usage + + model = "claude-3-7-sonnet-20250219" + web_search_requests = 3 + raw_response = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "server_tool_use": {"web_search_requests": web_search_requests}, + }, + } + usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) + assert getattr(usage, "server_tool_use", None) is None + + cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=model, + usage=usage, + response_object=raw_response, + custom_llm_provider="anthropic", + standard_built_in_tools_params=None, + ) + + per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"][ + "search_context_size_medium" + ] + assert cost == per_query_cost * web_search_requests + assert cost > 0.0 + assert getattr(usage, "server_tool_use", None) is None + + +def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_is_none(): + """ + Regression: when a caller hands the cost tracker a raw Anthropic dict without a + parallel Usage object, the web-search fee must still be priced per request from + usage.server_tool_use.web_search_requests on the dict instead of falling back to + the flat search_context_size_medium tier. + """ + model = "claude-3-7-sonnet-20250219" + web_search_requests = 4 + raw_response = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "server_tool_use": {"web_search_requests": web_search_requests}, + }, + } + + cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=model, + usage=None, + response_object=raw_response, + custom_llm_provider="anthropic", + standard_built_in_tools_params=None, + ) + + per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"][ + "search_context_size_medium" + ] + assert cost == per_query_cost * web_search_requests + + +def test_anthropic_web_search_zero_requests_from_raw_response_charges_zero(): + """ + Regression: a raw Anthropic dict reporting zero web search requests must price + the call at zero rather than charging the default medium-tier fee. + """ + model = "claude-3-7-sonnet-20250219" + raw_response = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "server_tool_use": {"web_search_requests": 0}, + }, + } + + cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=model, + usage=None, + response_object=raw_response, + custom_llm_provider="anthropic", + standard_built_in_tools_params=None, + ) + + assert cost == 0.0 + + +def test_anthropic_response_usage_block_preserves_server_tool_use(): + """ + Regression: AnthropicResponse.model_validate(...).model_dump() must keep + server_tool_use so the /v1/messages logging fallback does not strip the + web-search usage before cost tracking sees it. + """ + from litellm.types.llms.anthropic import AnthropicResponse + + raw_response = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-3-7-sonnet-20250219", + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "server_tool_use": {"web_search_requests": 2}, + }, + } + + dumped_usage = AnthropicResponse.model_validate(raw_response).model_dump()["usage"] + + assert dumped_usage["server_tool_use"] == {"web_search_requests": 2} + + @pytest.mark.parametrize( "model", ["gemini/gemini-2.0-flash-001", "gemini-2.0-flash-001"] )