diff --git a/test-quality-budget.json b/test-quality-budget.json index 1613c8c75cbf..b317dd1b4f18 100644 --- a/test-quality-budget.json +++ b/test-quality-budget.json @@ -6,13 +6,13 @@ "limit": 742 }, "TQ003": { - "limit": 1078 + "limit": 1076 }, "TQ004": { - "limit": 768 + "limit": 721 }, "TQ005": { - "limit": 2832 + "limit": 2784 }, "TQ006": { "limit": 34 diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index f66056a54e24..582d8f6ea87f 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1,6 +1,4 @@ import json -import os -import sys import pytest from fastapi.testclient import TestClient @@ -28,10 +26,6 @@ StandardBuiltInToolsParams, ) -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path - from litellm.litellm_core_utils.llm_cost_calc.utils import ( PromptTokensDetailsResult, TokenTypeCostBreakdown, @@ -44,13 +38,11 @@ from litellm.types.utils import CacheCreationTokenDetails, Usage -def test_reasoning_tokens_no_price_set(): +def test_reasoning_tokens_no_price_set(_local_model_cost_map): # Use o1 - o1-mini was deprecated/renamed; o1 has same reasoning-token semantics # (no separate output_cost_per_reasoning_token, so all completion tokens use output_cost_per_token) model = "o1" custom_llm_provider = "openai" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] usage = Usage( completion_tokens=1578, @@ -87,11 +79,9 @@ def test_reasoning_tokens_no_price_set(): ) -def test_reasoning_tokens_gemini(): +def test_reasoning_tokens_gemini(_local_model_cost_map): model = "gemini-2.5-flash" custom_llm_provider = "gemini" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") usage = Usage( completion_tokens=1578, @@ -132,12 +122,10 @@ def test_reasoning_tokens_gemini(): ) -def test_reasoning_tokens_gemini_3_1_flash_lite(): +def test_reasoning_tokens_gemini_3_1_flash_lite(_local_model_cost_map): """Test cost calculation for gemini-3.1-flash-lite-preview with reasoning tokens""" model = "gemini-3.1-flash-lite-preview" custom_llm_provider = "gemini" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") usage = Usage( completion_tokens=1000, @@ -270,11 +258,9 @@ def test_image_tokens_fallback_to_base_cost(): assert round(completion_cost, 12) == round(expected_completion_cost, 12) -def test_video_output_tokens_gemini_omni_flash_preview(): +def test_video_output_tokens_gemini_omni_flash_preview(_local_model_cost_map): """Video output tokens are billed at output_cost_per_video_token, not the text rate and not zero.""" model = "gemini-omni-flash-preview" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") text_tokens = 100 video_tokens = 46336 @@ -310,11 +296,9 @@ def test_video_output_tokens_gemini_omni_flash_preview(): ) -def test_video_input_tokens_gemini_omni_flash_preview(): +def test_video_input_tokens_gemini_omni_flash_preview(_local_model_cost_map): """Video input tokens are billed at the standard input rate instead of being dropped.""" model = "gemini-omni-flash-preview" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") usage = Usage( completion_tokens=10, @@ -369,12 +353,10 @@ def test_video_tokens_fallback_to_base_cost(): assert round(completion_cost, 12) == round((600 + 1120) * 2e-6, 12) -def test_generic_cost_per_token_above_200k_tokens(): +def test_generic_cost_per_token_above_200k_tokens(_local_model_cost_map): # gemini-2.5-pro-exp-03-25 was removed; gemini-2.5-pro has same above-200k pricing model = "gemini-2.5-pro" custom_llm_provider = "vertex_ai" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] prompt_tokens = 220 * 1e6 @@ -420,12 +402,10 @@ def test_get_token_base_cost_picks_highest_crossed_tier(): assert prompt_base_cost == 9e-6 -def test_generic_cost_per_token_gpt54_above_272k_tokens(): +def test_generic_cost_per_token_gpt54_above_272k_tokens(_local_model_cost_map): """GPT-5.4/5.4-pro: prompts >272K input tokens priced at 2x input, 1.5x output.""" model = "gpt-5.4" custom_llm_provider = "openai" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] prompt_tokens = 273000 # Above 272K threshold @@ -450,12 +430,10 @@ def test_generic_cost_per_token_gpt54_above_272k_tokens(): assert round(completion_cost, 10) == round(expected_completion, 10) -def test_generic_cost_per_token_minimax_m3_above_512k_tokens(): +def test_generic_cost_per_token_minimax_m3_above_512k_tokens(_local_model_cost_map): """MiniMax-M3: prompts >512K input tokens priced at 2x input, output, and cache read.""" model = "minimax/MiniMax-M3" custom_llm_provider = "minimax" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] prompt_tokens = 600000 @@ -493,10 +471,11 @@ def test_generic_cost_per_token_minimax_m3_above_512k_tokens(): "bedrock_mantle/openai.gpt-5.6-luna", ], ) -def test_generic_cost_per_token_bedrock_mantle_gpt56_long_context(model): +def test_generic_cost_per_token_bedrock_mantle_gpt56_long_context( + model, + _local_model_cost_map, +): """Bedrock GPT-5.6 supports a 1M context window, billed at the long-context rates above 272K.""" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] assert model_cost_map["max_input_tokens"] == 1000000 @@ -827,12 +806,10 @@ def test_generic_cost_per_token_tiered_pricing_bills_reasoning_at_tier_rate(): litellm.model_cost.pop(model, None) -def test_generic_cost_per_token_gpt55(): +def test_generic_cost_per_token_gpt55(_local_model_cost_map): """gpt-5.5: base pricing — $5/1M input, $30/1M output, $0.50/1M cached input.""" model = "gpt-5.5" custom_llm_provider = "openai" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] @@ -867,12 +844,10 @@ def test_generic_cost_per_token_gpt55(): ) -def test_generic_cost_per_token_gpt55_pro(): +def test_generic_cost_per_token_gpt55_pro(_local_model_cost_map): """gpt-5.5-pro: responses-only model — $30/1M input, $180/1M output, $3/1M cached input.""" model = "gpt-5.5-pro" custom_llm_provider = "openai" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] @@ -920,15 +895,18 @@ def test_generic_cost_per_token_gpt55_pro(): ], ) def test_generic_cost_per_token_gpt56( - model, input_cost, output_cost, cache_read_cost, cache_write_cost + model, + input_cost, + output_cost, + cache_read_cost, + cache_write_cost, + _local_model_cost_map, ): """gpt-5.6 (sol/terra/luna): base pricing + new cache-write cost. Cache writes are billed at 1.25x the uncached input rate for this family. """ custom_llm_provider = "openai" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] @@ -975,7 +953,10 @@ def test_generic_cost_per_token_gpt56( ], ) def test_generic_cost_per_token_gpt56_flex_above_272k( - model, flex_long_input_cost, flex_long_output_cost + model, + flex_long_input_cost, + flex_long_output_cost, + _local_model_cost_map, ): """A >272K flex request bills the flex long-context rate, not the standard one. @@ -983,8 +964,6 @@ def test_generic_cost_per_token_gpt56_flex_above_272k( ``*_above_272k_tokens_flex`` keys these requests silently fell back to the standard long-context price, billing 2x what OpenAI charges. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") prompt_tokens = 300000 completion_tokens = 1000 @@ -1024,10 +1003,13 @@ def test_generic_cost_per_token_gpt56_flex_above_272k( ], ) def test_generic_cost_per_token_gpt56_terra_cache_costs_by_tier_and_context( - service_tier, prompt_tokens, input_rate, cache_write_rate, cache_read_rate + service_tier, + prompt_tokens, + input_rate, + cache_write_rate, + cache_read_rate, + _local_model_cost_map, ): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") cached_tokens = 50000 cache_write_tokens = 40000 @@ -1116,13 +1098,15 @@ def test_generic_cost_per_token_gpt56_cyber( ], ) def test_generic_cost_per_token_azure_gpt56( - model, input_cost, output_cost, cache_read_cost + model, + input_cost, + output_cost, + cache_read_cost, + _local_model_cost_map, ): """Azure gpt-5.6 (global + us/eu regional): pricing mirrors the openai family for global deployments and carries the standard 10% regional uplift. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] assert model_cost_map["litellm_provider"] == "azure" @@ -1164,7 +1148,11 @@ def test_generic_cost_per_token_azure_gpt56( ], ) def test_gpt55_reasoning_effort_flags_match_live_openai_api( - model, expected_none, expected_xhigh, expected_minimal + model, + expected_none, + expected_xhigh, + expected_minimal, + _local_model_cost_map, ): """Pin reasoning_effort capability flags to OpenAI's actual API contract. @@ -1172,8 +1160,6 @@ def test_gpt55_reasoning_effort_flags_match_live_openai_api( ``Unsupported value: 'reasoning_effort' does not support 'minimal' with this model``. gpt-5.5-pro additionally rejects 'none' and 'low'. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") m = litellm.model_cost[model] assert ( @@ -1195,7 +1181,9 @@ def test_gpt55_reasoning_effort_flags_match_live_openai_api( ], ) def test_gpt55_dated_variants_match_base_reasoning_effort_capabilities( - base_model, dated_model + base_model, + dated_model, + _local_model_cost_map, ): """Dated snapshots must carry the same reasoning_effort capability flags as their non-dated counterparts. @@ -1206,8 +1194,6 @@ def test_gpt55_dated_variants_match_base_reasoning_effort_capabilities( behavior between ``gpt-5.5`` and ``gpt-5.5-2026-04-23``. Pinning to a dated variant must never lose capabilities relative to the base alias. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") base = litellm.model_cost[base_model] dated = litellm.model_cost[dated_model] @@ -1235,7 +1221,12 @@ def test_gpt55_dated_variants_match_base_reasoning_effort_capabilities( ], ) def test_azure_gpt55_entries_present_with_correct_pricing( - model, expected_mode, expected_input, expected_output, expected_cache_read + model, + expected_mode, + expected_input, + expected_output, + expected_cache_read, + _local_model_cost_map, ): """Day-0 Azure entries for GPT-5.5 mirror the OpenAI pricing structure. @@ -1243,8 +1234,6 @@ def test_azure_gpt55_entries_present_with_correct_pricing( on 2026-04-24): $5/$30 input/output per 1M for chat, $30/$180 for pro. Cache discount is 10% of input. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") m = litellm.model_cost[model] assert m["litellm_provider"] == "azure" @@ -1270,11 +1259,13 @@ def test_azure_gpt55_entries_present_with_correct_pricing( ], ) def test_azure_gpt55_reasoning_effort_flags_match_live_openai_api( - model, expected_none, expected_minimal, expected_xhigh + model, + expected_none, + expected_minimal, + expected_xhigh, + _local_model_cost_map, ): """Azure entries pin reasoning_effort flags to OpenAI's actual API contract.""" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") m = litellm.model_cost[model] assert m.get("supports_none_reasoning_effort") is expected_none @@ -1654,11 +1645,9 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details(): assert round(result, 6) == round(expected, 6) -def test_service_tier_flex_pricing(): +def test_service_tier_flex_pricing(_local_model_cost_map): """Test that flex service tier uses correct pricing (approximately 50% of standard).""" # Set up environment for local model cost map - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") # Test with gpt-5-nano which has flex pricing model = "gpt-5-nano" @@ -1711,11 +1700,9 @@ def test_service_tier_flex_pricing(): ), f"Flex total cost mismatch: {flex_total} vs {expected_flex_total}" -def test_service_tier_default_pricing(): +def test_service_tier_default_pricing(_local_model_cost_map): """Test that when no service tier is provided, standard pricing is used.""" # Set up environment for local model cost map - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") # Test with gpt-5-nano model = "gpt-5-nano" @@ -1762,11 +1749,9 @@ def test_service_tier_default_pricing(): ), f"Standard completion cost mismatch: {default_cost[1]} vs {expected_standard_completion}" -def test_service_tier_fallback_pricing(): +def test_service_tier_fallback_pricing(_local_model_cost_map): """Test that when service tier is provided but model doesn't have those keys, it falls back to standard pricing.""" # Set up environment for local model cost map - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") # Test with gpt-4 which doesn't have flex pricing keys model = "gpt-4" @@ -1874,15 +1859,13 @@ def test_service_tier_ultrafast_pricing(): assert completion_cost == pytest.approx(400 * 3e-04) -def test_service_tier_ultrafast_fallback_pricing(): +def test_service_tier_ultrafast_fallback_pricing(_local_model_cost_map): """Without *_ultrafast keys an ultrafast request bills the standard rate, not zero. Guards the suffix fallback in _get_cost_per_unit: "_fast" is a substring of "_ultrafast", so a shortest-first suffix match would strip the wrong suffix and price the request at 0. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) @@ -1911,7 +1894,10 @@ def test_service_tier_ultrafast_fallback_pricing(): "gemini-3.1-flash-image-preview", ], ) -def test_gemini_image_generation_cost_with_zero_text_tokens(model: str): +def test_gemini_image_generation_cost_with_zero_text_tokens( + model: str, + _local_model_cost_map, +): """ Test that image_tokens are correctly costed when text_tokens=0. @@ -1921,8 +1907,6 @@ def test_gemini_image_generation_cost_with_zero_text_tokens(model: str): https://github.com/BerriAI/litellm/issues/17410 """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") custom_llm_provider = "vertex_ai" @@ -1977,13 +1961,13 @@ def test_gemini_image_generation_cost_with_zero_text_tokens(model: str): ), f"Expected completion cost ${expected_completion_cost:.6f}, got ${completion_cost:.6f}" -def test_vertex_image_generation_cost_prefers_token_usage_metadata(): +def test_vertex_image_generation_cost_prefers_token_usage_metadata( + _local_model_cost_map, +): """ When usage metadata exists on image responses, Vertex image generation cost should be calculated from token pricing, not flat output_cost_per_image. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini-3.1-flash-image-preview" model_info = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai") @@ -2022,13 +2006,13 @@ def test_vertex_image_generation_cost_prefers_token_usage_metadata(): assert cost != len(image_response.data) * model_info["output_cost_per_image"] -def test_vertex_image_generation_cost_falls_back_to_flat_image_pricing(): +def test_vertex_image_generation_cost_falls_back_to_flat_image_pricing( + _local_model_cost_map, +): """ Without usage metadata, Vertex image generation cost should fall back to output_cost_per_image * number_of_images. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini-3.1-flash-image-preview" model_info = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai") @@ -2046,13 +2030,13 @@ def test_vertex_image_generation_cost_falls_back_to_flat_image_pricing(): assert round(cost, 10) == round(expected_cost, 10) -def test_gemini_image_generation_cost_prefers_token_usage_metadata(): +def test_gemini_image_generation_cost_prefers_token_usage_metadata( + _local_model_cost_map, +): """ When usage metadata exists on image responses, Gemini image generation cost should be calculated from token pricing, not flat output_cost_per_image. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini/gemini-3-pro-image-preview" model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") @@ -2091,13 +2075,13 @@ def test_gemini_image_generation_cost_prefers_token_usage_metadata(): assert cost != len(image_response.data) * model_info["output_cost_per_image"] -def test_gemini_image_generation_cost_falls_back_to_flat_image_pricing(): +def test_gemini_image_generation_cost_falls_back_to_flat_image_pricing( + _local_model_cost_map, +): """ Without usage metadata, Gemini image generation cost should fall back to output_cost_per_image * number_of_images. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini/gemini-3-pro-image-preview" model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") @@ -2194,7 +2178,7 @@ def test_reasoning_tokens_without_text_tokens_gpt5_nano(): ), "Bug detected: Cost calculation is using only reasoning_tokens instead of all completion_tokens!" -def test_image_count_prevents_text_tokens_fallback(): +def test_image_count_prevents_text_tokens_fallback(_local_model_cost_map): """ Test that the text_tokens fallback in generic_cost_per_token does not override text_tokens=0 when image_count > 0. @@ -2203,8 +2187,6 @@ def test_image_count_prevents_text_tokens_fallback(): When image_count > 0, text_tokens=0 is intentional (image-only request), not "text_tokens not set by provider." """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") # Simulate Nova image-only embedding: prompt_tokens estimated from # embedding dimensions (768 for 3072-dim), image_count=1 @@ -2239,19 +2221,9 @@ def test_image_count_prevents_text_tokens_fallback(): @pytest.fixture -def _local_model_cost_map(): - prev_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") - prev_model_cost = litellm.model_cost - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - try: - yield - finally: - litellm.model_cost = prev_model_cost - if prev_env is None: - os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) - else: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = prev_env +def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) @pytest.mark.parametrize("model", ["gpt-5.4", "gpt-realtime-2.1", "gpt-realtime-2.1-mini"]) @@ -2586,7 +2558,11 @@ def test_threshold_keys_exclude_service_tier_variants(): ], ) def test_token_type_cost_breakdown_is_provider_agnostic( - model, custom_llm_provider, reasoning_tokens, cached_tokens + model, + custom_llm_provider, + reasoning_tokens, + cached_tokens, + _local_model_cost_map, ): """ Reasoning and cache-read costs must be surfaced for every provider that reports @@ -2597,8 +2573,6 @@ def test_token_type_cost_breakdown_is_provider_agnostic( there - not the top-level cache_read_input_tokens attribute the old breakdown code relied on - is what makes Vertex/OpenAI/Azure cache costs show up at all. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") usage = Usage( prompt_tokens=1000, @@ -2629,10 +2603,8 @@ def test_token_type_cost_breakdown_is_provider_agnostic( assert breakdown.cache_read_cost == pytest.approx(cached_tokens * cache_read_rate) -def test_token_type_cost_breakdown_matches_real_gemini_numbers(): +def test_token_type_cost_breakdown_matches_real_gemini_numbers(_local_model_cost_map): """Hard-coded against the exact gemini-2.5-flash response that exposed the gap.""" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") usage = Usage( prompt_tokens=209, @@ -2655,9 +2627,9 @@ def test_token_type_cost_breakdown_matches_real_gemini_numbers(): assert breakdown.cache_creation_cost == 0.0 -def test_token_type_cost_breakdown_xai_at_exactly_200k_uses_higher_tier_rates(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_token_type_cost_breakdown_xai_at_exactly_200k_uses_higher_tier_rates( + _local_model_cost_map, +): usage = Usage( prompt_tokens=200_000, @@ -2679,9 +2651,9 @@ def test_token_type_cost_breakdown_xai_at_exactly_200k_uses_higher_tier_rates(): assert breakdown.cache_read_cost == pytest.approx(50_000 * 4e-07) -def test_token_type_cost_breakdown_xai_just_below_200k_uses_base_tier_rates(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_token_type_cost_breakdown_xai_just_below_200k_uses_base_tier_rates( + _local_model_cost_map, +): usage = Usage( prompt_tokens=199_999, @@ -2703,14 +2675,14 @@ def test_token_type_cost_breakdown_xai_just_below_200k_uses_base_tier_rates(): assert breakdown.cache_read_cost == pytest.approx(50_000 * 2e-07) -def test_token_type_cost_breakdown_includes_cache_creation_from_top_level_usage(): +def test_token_type_cost_breakdown_includes_cache_creation_from_top_level_usage( + _local_model_cost_map, +): """ Bedrock/Anthropic report cache tokens as top-level usage fields; the Usage constructor maps them onto prompt_tokens_details, so the breakdown must still pick up both cache-read and cache-creation costs. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "anthropic.claude-3-5-haiku-20241022-v1:0" usage = Usage( @@ -2734,14 +2706,12 @@ def test_token_type_cost_breakdown_includes_cache_creation_from_top_level_usage( ) -def test_token_type_cost_breakdown_reads_cache_write_tokens(): +def test_token_type_cost_breakdown_reads_cache_write_tokens(_local_model_cost_map): """ Some OpenAI-compatible providers (e.g. kimi-k2) report cache-write tokens under `cache_write_tokens` rather than `cache_creation_tokens`. The breakdown must read it the same way the total-cost normalization does, so the two agree. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "anthropic.claude-3-5-haiku-20241022-v1:0" usage = Usage( @@ -2762,7 +2732,9 @@ def test_token_type_cost_breakdown_reads_cache_write_tokens(): ) -def test_generic_cost_per_token_openai_cache_write_tokens_gpt_5_6(): +def test_generic_cost_per_token_openai_cache_write_tokens_gpt_5_6( + _local_model_cost_map, +): """ Regression: OpenAI gpt-5.6 reports cache-write tokens under prompt_tokens_details.cache_write_tokens (not the Anthropic cache_creation_tokens @@ -2770,8 +2742,6 @@ def test_generic_cost_per_token_openai_cache_write_tokens_gpt_5_6(): input rate. Customer report: cache creation tokens were never counted for the GPT-5.6 series, so cost was undercounted on cache-write requests. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gpt-5.6" usage = Usage( @@ -2793,14 +2763,14 @@ def test_generic_cost_per_token_openai_cache_write_tokens_gpt_5_6(): assert prompt_cost > 1000 * info["input_cost_per_token"] -def test_generic_cost_per_token_backs_out_cache_write_tokens_from_text_tokens(): +def test_generic_cost_per_token_backs_out_cache_write_tokens_from_text_tokens( + _local_model_cost_map, +): """ Regression for #34801: when a provider reports text_tokens covering the whole prompt alongside cache-write tokens (and no cache reads), the cache-write tokens must be backed out of the text total instead of being billed twice. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gpt-5.6" usage = Usage( @@ -2819,15 +2789,13 @@ def test_generic_cost_per_token_backs_out_cache_write_tokens_from_text_tokens(): assert prompt_cost == pytest.approx(expected_prompt) -def test_token_type_cost_breakdown_reconciles_with_generic_total(): +def test_token_type_cost_breakdown_reconciles_with_generic_total(_local_model_cost_map): """ Both-ways check: the reasoning subset must sum with the remaining (text) output cost to exactly the completion total, and the cache-read subset with the remaining input cost to exactly the prompt total, as computed by generic_cost_per_token. A mismatch here would mean the breakdown misrepresents what was actually billed. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini-2.5-flash" custom_llm_provider = "vertex_ai" @@ -2860,9 +2828,7 @@ def test_token_type_cost_breakdown_reconciles_with_generic_total(): assert text_input_cost + breakdown.cache_read_cost == pytest.approx(prompt_cost) -def test_token_type_cost_breakdown_zero_without_special_tokens(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_token_type_cost_breakdown_zero_without_special_tokens(_local_model_cost_map): usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) breakdown = get_token_type_cost_breakdown( @@ -2900,7 +2866,10 @@ def test_token_type_cost_breakdown_zero_without_special_tokens(): ], ) def test_token_type_cost_breakdown_openai_responses_api_cache_write_read( - raw_usage, expect_read, expect_write + raw_usage, + expect_read, + expect_write, + _local_model_cost_map, ): """Regression for #34309: OpenAI Responses API reports cache tokens under input_tokens_details.{cached_tokens, cache_write_tokens}, not the Anthropic-style @@ -2908,8 +2877,6 @@ def test_token_type_cost_breakdown_openai_responses_api_cache_write_read( cache_read_cost / cache_creation_cost from the transformed usage.""" from litellm.responses.utils import ResponseAPILoggingUtils - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gpt-5.6" usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(raw_usage) @@ -2950,15 +2917,13 @@ def test_token_type_cost_breakdown_handles_unknown_model_gracefully(): ) -def test_token_type_cost_breakdown_applies_regional_uplift(): +def test_token_type_cost_breakdown_applies_regional_uplift(_local_model_cost_map): """ Regional OpenAI hosts (eu./us.) apply a flat uplift to every token cost. The per-type breakdown must apply the same uplift via data_residency so it stays reconciled with the uplifted input_cost/output_cost totals, instead of being logged at the base rate. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gpt-5.4" custom_llm_provider = "openai" @@ -3006,15 +2971,15 @@ def test_token_type_cost_breakdown_applies_regional_uplift(): assert text_input_cost + eu.cache_read_cost == pytest.approx(prompt_cost) -def test_token_type_cost_breakdown_applies_vertex_regional_uplift(): +def test_token_type_cost_breakdown_applies_vertex_regional_uplift( + _local_model_cost_map, +): """ Non-global Vertex endpoints apply a flat 1.1x uplift to every token cost. The per-type breakdown must apply the same uplift via vertex_location so it stays reconciled with the uplifted input_cost/output_cost totals, instead of being logged at the global rate. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "claude-haiku-4-5@20251001" custom_llm_provider = "vertex_ai" @@ -3070,7 +3035,7 @@ def test_token_type_cost_breakdown_applies_anthropic_geo_multiplier(monkeypatch) ) monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) model = "claude-test-geo-breakdown-model" litellm.register_model( @@ -3191,9 +3156,13 @@ def test_image_response_input_image_tokens_priced_at_image_rate(details_as_dict) @pytest.mark.parametrize("model,input_cost,output_cost,cache_read_cost", GEMINI_DAY0_LAUNCH_PRICING) -def test_gemini_36_flash_and_35_flash_lite_launch_pricing(model, input_cost, output_cost, cache_read_cost): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_gemini_36_flash_and_35_flash_lite_launch_pricing( + model, + input_cost, + output_cost, + cache_read_cost, + _local_model_cost_map, +): model_cost_map = litellm.model_cost[model] assert model_cost_map["input_cost_per_token"] == input_cost @@ -3206,9 +3175,7 @@ def test_gemini_36_flash_and_35_flash_lite_launch_pricing(model, input_cost, out assert model_cost_map["max_input_tokens"] == 1048576 -def test_generic_cost_per_token_gemini_36_flash(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_generic_cost_per_token_gemini_36_flash(_local_model_cost_map): usage = Usage( prompt_tokens=1000, @@ -3274,9 +3241,7 @@ def test_gemini_36_flash_batch_introductory_pricing(model, _local_model_cost_map assert model_cost_map["output_cost_per_token_batches"] == 1.875e-06 -def test_generic_cost_per_token_gemini_35_flash_lite(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_generic_cost_per_token_gemini_35_flash_lite(_local_model_cost_map): usage = Usage( prompt_tokens=1000,