From 7c1c800ae6ae2ebd94f933ade46923fa6abfc45a Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 21 Aug 2026 05:50:10 -0700 Subject: [PATCH] test: route the cost-calc tests through the local-cost-map fixture 45 tests in test_llm_cost_calc_utils.py set LITELLM_LOCAL_MODEL_COST_MAP and swapped litellm.model_cost inline and never put either back, so anything that ran after them in the same process read the checked-in snapshot instead of the live map. The file already had a _local_model_cost_map fixture doing this correctly for the data-residency tests; those 45 now request it too. The fixture itself moves from a hand-rolled try/finally to monkeypatch, which also restores when setup raises rather than only when the test body finishes. Dropping the sys.path.insert takes the last reader of os and sys with it. It resolved "../../.." against the working directory, so it only ever pointed at the repo root when pytest happened to be invoked from this test's own folder. --- test-quality-budget.json | 6 +- .../llm_cost_calc/test_llm_cost_calc_utils.py | 275 ++++++++---------- 2 files changed, 123 insertions(+), 158 deletions(-) diff --git a/test-quality-budget.json b/test-quality-budget.json index 1613c8c75cbf..b317dd1b4f18 100644 --- a/test-quality-budget.json +++ b/test-quality-budget.json @@ -6,13 +6,13 @@ "limit": 742 }, "TQ003": { - "limit": 1078 + "limit": 1076 }, "TQ004": { - "limit": 768 + "limit": 721 }, "TQ005": { - "limit": 2832 + "limit": 2784 }, "TQ006": { "limit": 34 diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index f66056a54e24..582d8f6ea87f 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1,6 +1,4 @@ import json -import os -import sys import pytest from fastapi.testclient import TestClient @@ -28,10 +26,6 @@ StandardBuiltInToolsParams, ) -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path - from litellm.litellm_core_utils.llm_cost_calc.utils import ( PromptTokensDetailsResult, TokenTypeCostBreakdown, @@ -44,13 +38,11 @@ from litellm.types.utils import CacheCreationTokenDetails, Usage -def test_reasoning_tokens_no_price_set(): +def test_reasoning_tokens_no_price_set(_local_model_cost_map): # Use o1 - o1-mini was deprecated/renamed; o1 has same reasoning-token semantics # (no separate output_cost_per_reasoning_token, so all completion tokens use output_cost_per_token) model = "o1" custom_llm_provider = "openai" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] usage = Usage( completion_tokens=1578, @@ -87,11 +79,9 @@ def test_reasoning_tokens_no_price_set(): ) -def test_reasoning_tokens_gemini(): +def test_reasoning_tokens_gemini(_local_model_cost_map): model = "gemini-2.5-flash" custom_llm_provider = "gemini" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") usage = Usage( completion_tokens=1578, @@ -132,12 +122,10 @@ def test_reasoning_tokens_gemini(): ) -def test_reasoning_tokens_gemini_3_1_flash_lite(): +def test_reasoning_tokens_gemini_3_1_flash_lite(_local_model_cost_map): """Test cost calculation for gemini-3.1-flash-lite-preview with reasoning tokens""" model = "gemini-3.1-flash-lite-preview" custom_llm_provider = "gemini" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") usage = Usage( completion_tokens=1000, @@ -270,11 +258,9 @@ def test_image_tokens_fallback_to_base_cost(): assert round(completion_cost, 12) == round(expected_completion_cost, 12) -def test_video_output_tokens_gemini_omni_flash_preview(): +def test_video_output_tokens_gemini_omni_flash_preview(_local_model_cost_map): """Video output tokens are billed at output_cost_per_video_token, not the text rate and not zero.""" model = "gemini-omni-flash-preview" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") text_tokens = 100 video_tokens = 46336 @@ -310,11 +296,9 @@ def test_video_output_tokens_gemini_omni_flash_preview(): ) -def test_video_input_tokens_gemini_omni_flash_preview(): +def test_video_input_tokens_gemini_omni_flash_preview(_local_model_cost_map): """Video input tokens are billed at the standard input rate instead of being dropped.""" model = "gemini-omni-flash-preview" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") usage = Usage( completion_tokens=10, @@ -369,12 +353,10 @@ def test_video_tokens_fallback_to_base_cost(): assert round(completion_cost, 12) == round((600 + 1120) * 2e-6, 12) -def test_generic_cost_per_token_above_200k_tokens(): +def test_generic_cost_per_token_above_200k_tokens(_local_model_cost_map): # gemini-2.5-pro-exp-03-25 was removed; gemini-2.5-pro has same above-200k pricing model = "gemini-2.5-pro" custom_llm_provider = "vertex_ai" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] prompt_tokens = 220 * 1e6 @@ -420,12 +402,10 @@ def test_get_token_base_cost_picks_highest_crossed_tier(): assert prompt_base_cost == 9e-6 -def test_generic_cost_per_token_gpt54_above_272k_tokens(): +def test_generic_cost_per_token_gpt54_above_272k_tokens(_local_model_cost_map): """GPT-5.4/5.4-pro: prompts >272K input tokens priced at 2x input, 1.5x output.""" model = "gpt-5.4" custom_llm_provider = "openai" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] prompt_tokens = 273000 # Above 272K threshold @@ -450,12 +430,10 @@ def test_generic_cost_per_token_gpt54_above_272k_tokens(): assert round(completion_cost, 10) == round(expected_completion, 10) -def test_generic_cost_per_token_minimax_m3_above_512k_tokens(): +def test_generic_cost_per_token_minimax_m3_above_512k_tokens(_local_model_cost_map): """MiniMax-M3: prompts >512K input tokens priced at 2x input, output, and cache read.""" model = "minimax/MiniMax-M3" custom_llm_provider = "minimax" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] prompt_tokens = 600000 @@ -493,10 +471,11 @@ def test_generic_cost_per_token_minimax_m3_above_512k_tokens(): "bedrock_mantle/openai.gpt-5.6-luna", ], ) -def test_generic_cost_per_token_bedrock_mantle_gpt56_long_context(model): +def test_generic_cost_per_token_bedrock_mantle_gpt56_long_context( + model, + _local_model_cost_map, +): """Bedrock GPT-5.6 supports a 1M context window, billed at the long-context rates above 272K.""" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] assert model_cost_map["max_input_tokens"] == 1000000 @@ -827,12 +806,10 @@ def test_generic_cost_per_token_tiered_pricing_bills_reasoning_at_tier_rate(): litellm.model_cost.pop(model, None) -def test_generic_cost_per_token_gpt55(): +def test_generic_cost_per_token_gpt55(_local_model_cost_map): """gpt-5.5: base pricing — $5/1M input, $30/1M output, $0.50/1M cached input.""" model = "gpt-5.5" custom_llm_provider = "openai" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] @@ -867,12 +844,10 @@ def test_generic_cost_per_token_gpt55(): ) -def test_generic_cost_per_token_gpt55_pro(): +def test_generic_cost_per_token_gpt55_pro(_local_model_cost_map): """gpt-5.5-pro: responses-only model — $30/1M input, $180/1M output, $3/1M cached input.""" model = "gpt-5.5-pro" custom_llm_provider = "openai" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] @@ -920,15 +895,18 @@ def test_generic_cost_per_token_gpt55_pro(): ], ) def test_generic_cost_per_token_gpt56( - model, input_cost, output_cost, cache_read_cost, cache_write_cost + model, + input_cost, + output_cost, + cache_read_cost, + cache_write_cost, + _local_model_cost_map, ): """gpt-5.6 (sol/terra/luna): base pricing + new cache-write cost. Cache writes are billed at 1.25x the uncached input rate for this family. """ custom_llm_provider = "openai" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] @@ -975,7 +953,10 @@ def test_generic_cost_per_token_gpt56( ], ) def test_generic_cost_per_token_gpt56_flex_above_272k( - model, flex_long_input_cost, flex_long_output_cost + model, + flex_long_input_cost, + flex_long_output_cost, + _local_model_cost_map, ): """A >272K flex request bills the flex long-context rate, not the standard one. @@ -983,8 +964,6 @@ def test_generic_cost_per_token_gpt56_flex_above_272k( ``*_above_272k_tokens_flex`` keys these requests silently fell back to the standard long-context price, billing 2x what OpenAI charges. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") prompt_tokens = 300000 completion_tokens = 1000 @@ -1024,10 +1003,13 @@ def test_generic_cost_per_token_gpt56_flex_above_272k( ], ) def test_generic_cost_per_token_gpt56_terra_cache_costs_by_tier_and_context( - service_tier, prompt_tokens, input_rate, cache_write_rate, cache_read_rate + service_tier, + prompt_tokens, + input_rate, + cache_write_rate, + cache_read_rate, + _local_model_cost_map, ): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") cached_tokens = 50000 cache_write_tokens = 40000 @@ -1116,13 +1098,15 @@ def test_generic_cost_per_token_gpt56_cyber( ], ) def test_generic_cost_per_token_azure_gpt56( - model, input_cost, output_cost, cache_read_cost + model, + input_cost, + output_cost, + cache_read_cost, + _local_model_cost_map, ): """Azure gpt-5.6 (global + us/eu regional): pricing mirrors the openai family for global deployments and carries the standard 10% regional uplift. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] assert model_cost_map["litellm_provider"] == "azure" @@ -1164,7 +1148,11 @@ def test_generic_cost_per_token_azure_gpt56( ], ) def test_gpt55_reasoning_effort_flags_match_live_openai_api( - model, expected_none, expected_xhigh, expected_minimal + model, + expected_none, + expected_xhigh, + expected_minimal, + _local_model_cost_map, ): """Pin reasoning_effort capability flags to OpenAI's actual API contract. @@ -1172,8 +1160,6 @@ def test_gpt55_reasoning_effort_flags_match_live_openai_api( ``Unsupported value: 'reasoning_effort' does not support 'minimal' with this model``. gpt-5.5-pro additionally rejects 'none' and 'low'. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") m = litellm.model_cost[model] assert ( @@ -1195,7 +1181,9 @@ def test_gpt55_reasoning_effort_flags_match_live_openai_api( ], ) def test_gpt55_dated_variants_match_base_reasoning_effort_capabilities( - base_model, dated_model + base_model, + dated_model, + _local_model_cost_map, ): """Dated snapshots must carry the same reasoning_effort capability flags as their non-dated counterparts. @@ -1206,8 +1194,6 @@ def test_gpt55_dated_variants_match_base_reasoning_effort_capabilities( behavior between ``gpt-5.5`` and ``gpt-5.5-2026-04-23``. Pinning to a dated variant must never lose capabilities relative to the base alias. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") base = litellm.model_cost[base_model] dated = litellm.model_cost[dated_model] @@ -1235,7 +1221,12 @@ def test_gpt55_dated_variants_match_base_reasoning_effort_capabilities( ], ) def test_azure_gpt55_entries_present_with_correct_pricing( - model, expected_mode, expected_input, expected_output, expected_cache_read + model, + expected_mode, + expected_input, + expected_output, + expected_cache_read, + _local_model_cost_map, ): """Day-0 Azure entries for GPT-5.5 mirror the OpenAI pricing structure. @@ -1243,8 +1234,6 @@ def test_azure_gpt55_entries_present_with_correct_pricing( on 2026-04-24): $5/$30 input/output per 1M for chat, $30/$180 for pro. Cache discount is 10% of input. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") m = litellm.model_cost[model] assert m["litellm_provider"] == "azure" @@ -1270,11 +1259,13 @@ def test_azure_gpt55_entries_present_with_correct_pricing( ], ) def test_azure_gpt55_reasoning_effort_flags_match_live_openai_api( - model, expected_none, expected_minimal, expected_xhigh + model, + expected_none, + expected_minimal, + expected_xhigh, + _local_model_cost_map, ): """Azure entries pin reasoning_effort flags to OpenAI's actual API contract.""" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") m = litellm.model_cost[model] assert m.get("supports_none_reasoning_effort") is expected_none @@ -1654,11 +1645,9 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details(): assert round(result, 6) == round(expected, 6) -def test_service_tier_flex_pricing(): +def test_service_tier_flex_pricing(_local_model_cost_map): """Test that flex service tier uses correct pricing (approximately 50% of standard).""" # Set up environment for local model cost map - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") # Test with gpt-5-nano which has flex pricing model = "gpt-5-nano" @@ -1711,11 +1700,9 @@ def test_service_tier_flex_pricing(): ), f"Flex total cost mismatch: {flex_total} vs {expected_flex_total}" -def test_service_tier_default_pricing(): +def test_service_tier_default_pricing(_local_model_cost_map): """Test that when no service tier is provided, standard pricing is used.""" # Set up environment for local model cost map - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") # Test with gpt-5-nano model = "gpt-5-nano" @@ -1762,11 +1749,9 @@ def test_service_tier_default_pricing(): ), f"Standard completion cost mismatch: {default_cost[1]} vs {expected_standard_completion}" -def test_service_tier_fallback_pricing(): +def test_service_tier_fallback_pricing(_local_model_cost_map): """Test that when service tier is provided but model doesn't have those keys, it falls back to standard pricing.""" # Set up environment for local model cost map - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") # Test with gpt-4 which doesn't have flex pricing keys model = "gpt-4" @@ -1874,15 +1859,13 @@ def test_service_tier_ultrafast_pricing(): assert completion_cost == pytest.approx(400 * 3e-04) -def test_service_tier_ultrafast_fallback_pricing(): +def test_service_tier_ultrafast_fallback_pricing(_local_model_cost_map): """Without *_ultrafast keys an ultrafast request bills the standard rate, not zero. Guards the suffix fallback in _get_cost_per_unit: "_fast" is a substring of "_ultrafast", so a shortest-first suffix match would strip the wrong suffix and price the request at 0. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) @@ -1911,7 +1894,10 @@ def test_service_tier_ultrafast_fallback_pricing(): "gemini-3.1-flash-image-preview", ], ) -def test_gemini_image_generation_cost_with_zero_text_tokens(model: str): +def test_gemini_image_generation_cost_with_zero_text_tokens( + model: str, + _local_model_cost_map, +): """ Test that image_tokens are correctly costed when text_tokens=0. @@ -1921,8 +1907,6 @@ def test_gemini_image_generation_cost_with_zero_text_tokens(model: str): https://github.com/BerriAI/litellm/issues/17410 """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") custom_llm_provider = "vertex_ai" @@ -1977,13 +1961,13 @@ def test_gemini_image_generation_cost_with_zero_text_tokens(model: str): ), f"Expected completion cost ${expected_completion_cost:.6f}, got ${completion_cost:.6f}" -def test_vertex_image_generation_cost_prefers_token_usage_metadata(): +def test_vertex_image_generation_cost_prefers_token_usage_metadata( + _local_model_cost_map, +): """ When usage metadata exists on image responses, Vertex image generation cost should be calculated from token pricing, not flat output_cost_per_image. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini-3.1-flash-image-preview" model_info = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai") @@ -2022,13 +2006,13 @@ def test_vertex_image_generation_cost_prefers_token_usage_metadata(): assert cost != len(image_response.data) * model_info["output_cost_per_image"] -def test_vertex_image_generation_cost_falls_back_to_flat_image_pricing(): +def test_vertex_image_generation_cost_falls_back_to_flat_image_pricing( + _local_model_cost_map, +): """ Without usage metadata, Vertex image generation cost should fall back to output_cost_per_image * number_of_images. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini-3.1-flash-image-preview" model_info = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai") @@ -2046,13 +2030,13 @@ def test_vertex_image_generation_cost_falls_back_to_flat_image_pricing(): assert round(cost, 10) == round(expected_cost, 10) -def test_gemini_image_generation_cost_prefers_token_usage_metadata(): +def test_gemini_image_generation_cost_prefers_token_usage_metadata( + _local_model_cost_map, +): """ When usage metadata exists on image responses, Gemini image generation cost should be calculated from token pricing, not flat output_cost_per_image. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini/gemini-3-pro-image-preview" model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") @@ -2091,13 +2075,13 @@ def test_gemini_image_generation_cost_prefers_token_usage_metadata(): assert cost != len(image_response.data) * model_info["output_cost_per_image"] -def test_gemini_image_generation_cost_falls_back_to_flat_image_pricing(): +def test_gemini_image_generation_cost_falls_back_to_flat_image_pricing( + _local_model_cost_map, +): """ Without usage metadata, Gemini image generation cost should fall back to output_cost_per_image * number_of_images. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini/gemini-3-pro-image-preview" model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") @@ -2194,7 +2178,7 @@ def test_reasoning_tokens_without_text_tokens_gpt5_nano(): ), "Bug detected: Cost calculation is using only reasoning_tokens instead of all completion_tokens!" -def test_image_count_prevents_text_tokens_fallback(): +def test_image_count_prevents_text_tokens_fallback(_local_model_cost_map): """ Test that the text_tokens fallback in generic_cost_per_token does not override text_tokens=0 when image_count > 0. @@ -2203,8 +2187,6 @@ def test_image_count_prevents_text_tokens_fallback(): When image_count > 0, text_tokens=0 is intentional (image-only request), not "text_tokens not set by provider." """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") # Simulate Nova image-only embedding: prompt_tokens estimated from # embedding dimensions (768 for 3072-dim), image_count=1 @@ -2239,19 +2221,9 @@ def test_image_count_prevents_text_tokens_fallback(): @pytest.fixture -def _local_model_cost_map(): - prev_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") - prev_model_cost = litellm.model_cost - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - try: - yield - finally: - litellm.model_cost = prev_model_cost - if prev_env is None: - os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) - else: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = prev_env +def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) @pytest.mark.parametrize("model", ["gpt-5.4", "gpt-realtime-2.1", "gpt-realtime-2.1-mini"]) @@ -2586,7 +2558,11 @@ def test_threshold_keys_exclude_service_tier_variants(): ], ) def test_token_type_cost_breakdown_is_provider_agnostic( - model, custom_llm_provider, reasoning_tokens, cached_tokens + model, + custom_llm_provider, + reasoning_tokens, + cached_tokens, + _local_model_cost_map, ): """ Reasoning and cache-read costs must be surfaced for every provider that reports @@ -2597,8 +2573,6 @@ def test_token_type_cost_breakdown_is_provider_agnostic( there - not the top-level cache_read_input_tokens attribute the old breakdown code relied on - is what makes Vertex/OpenAI/Azure cache costs show up at all. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") usage = Usage( prompt_tokens=1000, @@ -2629,10 +2603,8 @@ def test_token_type_cost_breakdown_is_provider_agnostic( assert breakdown.cache_read_cost == pytest.approx(cached_tokens * cache_read_rate) -def test_token_type_cost_breakdown_matches_real_gemini_numbers(): +def test_token_type_cost_breakdown_matches_real_gemini_numbers(_local_model_cost_map): """Hard-coded against the exact gemini-2.5-flash response that exposed the gap.""" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") usage = Usage( prompt_tokens=209, @@ -2655,9 +2627,9 @@ def test_token_type_cost_breakdown_matches_real_gemini_numbers(): assert breakdown.cache_creation_cost == 0.0 -def test_token_type_cost_breakdown_xai_at_exactly_200k_uses_higher_tier_rates(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_token_type_cost_breakdown_xai_at_exactly_200k_uses_higher_tier_rates( + _local_model_cost_map, +): usage = Usage( prompt_tokens=200_000, @@ -2679,9 +2651,9 @@ def test_token_type_cost_breakdown_xai_at_exactly_200k_uses_higher_tier_rates(): assert breakdown.cache_read_cost == pytest.approx(50_000 * 4e-07) -def test_token_type_cost_breakdown_xai_just_below_200k_uses_base_tier_rates(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_token_type_cost_breakdown_xai_just_below_200k_uses_base_tier_rates( + _local_model_cost_map, +): usage = Usage( prompt_tokens=199_999, @@ -2703,14 +2675,14 @@ def test_token_type_cost_breakdown_xai_just_below_200k_uses_base_tier_rates(): assert breakdown.cache_read_cost == pytest.approx(50_000 * 2e-07) -def test_token_type_cost_breakdown_includes_cache_creation_from_top_level_usage(): +def test_token_type_cost_breakdown_includes_cache_creation_from_top_level_usage( + _local_model_cost_map, +): """ Bedrock/Anthropic report cache tokens as top-level usage fields; the Usage constructor maps them onto prompt_tokens_details, so the breakdown must still pick up both cache-read and cache-creation costs. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "anthropic.claude-3-5-haiku-20241022-v1:0" usage = Usage( @@ -2734,14 +2706,12 @@ def test_token_type_cost_breakdown_includes_cache_creation_from_top_level_usage( ) -def test_token_type_cost_breakdown_reads_cache_write_tokens(): +def test_token_type_cost_breakdown_reads_cache_write_tokens(_local_model_cost_map): """ Some OpenAI-compatible providers (e.g. kimi-k2) report cache-write tokens under `cache_write_tokens` rather than `cache_creation_tokens`. The breakdown must read it the same way the total-cost normalization does, so the two agree. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "anthropic.claude-3-5-haiku-20241022-v1:0" usage = Usage( @@ -2762,7 +2732,9 @@ def test_token_type_cost_breakdown_reads_cache_write_tokens(): ) -def test_generic_cost_per_token_openai_cache_write_tokens_gpt_5_6(): +def test_generic_cost_per_token_openai_cache_write_tokens_gpt_5_6( + _local_model_cost_map, +): """ Regression: OpenAI gpt-5.6 reports cache-write tokens under prompt_tokens_details.cache_write_tokens (not the Anthropic cache_creation_tokens @@ -2770,8 +2742,6 @@ def test_generic_cost_per_token_openai_cache_write_tokens_gpt_5_6(): input rate. Customer report: cache creation tokens were never counted for the GPT-5.6 series, so cost was undercounted on cache-write requests. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gpt-5.6" usage = Usage( @@ -2793,14 +2763,14 @@ def test_generic_cost_per_token_openai_cache_write_tokens_gpt_5_6(): assert prompt_cost > 1000 * info["input_cost_per_token"] -def test_generic_cost_per_token_backs_out_cache_write_tokens_from_text_tokens(): +def test_generic_cost_per_token_backs_out_cache_write_tokens_from_text_tokens( + _local_model_cost_map, +): """ Regression for #34801: when a provider reports text_tokens covering the whole prompt alongside cache-write tokens (and no cache reads), the cache-write tokens must be backed out of the text total instead of being billed twice. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gpt-5.6" usage = Usage( @@ -2819,15 +2789,13 @@ def test_generic_cost_per_token_backs_out_cache_write_tokens_from_text_tokens(): assert prompt_cost == pytest.approx(expected_prompt) -def test_token_type_cost_breakdown_reconciles_with_generic_total(): +def test_token_type_cost_breakdown_reconciles_with_generic_total(_local_model_cost_map): """ Both-ways check: the reasoning subset must sum with the remaining (text) output cost to exactly the completion total, and the cache-read subset with the remaining input cost to exactly the prompt total, as computed by generic_cost_per_token. A mismatch here would mean the breakdown misrepresents what was actually billed. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini-2.5-flash" custom_llm_provider = "vertex_ai" @@ -2860,9 +2828,7 @@ def test_token_type_cost_breakdown_reconciles_with_generic_total(): assert text_input_cost + breakdown.cache_read_cost == pytest.approx(prompt_cost) -def test_token_type_cost_breakdown_zero_without_special_tokens(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_token_type_cost_breakdown_zero_without_special_tokens(_local_model_cost_map): usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) breakdown = get_token_type_cost_breakdown( @@ -2900,7 +2866,10 @@ def test_token_type_cost_breakdown_zero_without_special_tokens(): ], ) def test_token_type_cost_breakdown_openai_responses_api_cache_write_read( - raw_usage, expect_read, expect_write + raw_usage, + expect_read, + expect_write, + _local_model_cost_map, ): """Regression for #34309: OpenAI Responses API reports cache tokens under input_tokens_details.{cached_tokens, cache_write_tokens}, not the Anthropic-style @@ -2908,8 +2877,6 @@ def test_token_type_cost_breakdown_openai_responses_api_cache_write_read( cache_read_cost / cache_creation_cost from the transformed usage.""" from litellm.responses.utils import ResponseAPILoggingUtils - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gpt-5.6" usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(raw_usage) @@ -2950,15 +2917,13 @@ def test_token_type_cost_breakdown_handles_unknown_model_gracefully(): ) -def test_token_type_cost_breakdown_applies_regional_uplift(): +def test_token_type_cost_breakdown_applies_regional_uplift(_local_model_cost_map): """ Regional OpenAI hosts (eu./us.) apply a flat uplift to every token cost. The per-type breakdown must apply the same uplift via data_residency so it stays reconciled with the uplifted input_cost/output_cost totals, instead of being logged at the base rate. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gpt-5.4" custom_llm_provider = "openai" @@ -3006,15 +2971,15 @@ def test_token_type_cost_breakdown_applies_regional_uplift(): assert text_input_cost + eu.cache_read_cost == pytest.approx(prompt_cost) -def test_token_type_cost_breakdown_applies_vertex_regional_uplift(): +def test_token_type_cost_breakdown_applies_vertex_regional_uplift( + _local_model_cost_map, +): """ Non-global Vertex endpoints apply a flat 1.1x uplift to every token cost. The per-type breakdown must apply the same uplift via vertex_location so it stays reconciled with the uplifted input_cost/output_cost totals, instead of being logged at the global rate. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "claude-haiku-4-5@20251001" custom_llm_provider = "vertex_ai" @@ -3070,7 +3035,7 @@ def test_token_type_cost_breakdown_applies_anthropic_geo_multiplier(monkeypatch) ) monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) model = "claude-test-geo-breakdown-model" litellm.register_model( @@ -3191,9 +3156,13 @@ def test_image_response_input_image_tokens_priced_at_image_rate(details_as_dict) @pytest.mark.parametrize("model,input_cost,output_cost,cache_read_cost", GEMINI_DAY0_LAUNCH_PRICING) -def test_gemini_36_flash_and_35_flash_lite_launch_pricing(model, input_cost, output_cost, cache_read_cost): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_gemini_36_flash_and_35_flash_lite_launch_pricing( + model, + input_cost, + output_cost, + cache_read_cost, + _local_model_cost_map, +): model_cost_map = litellm.model_cost[model] assert model_cost_map["input_cost_per_token"] == input_cost @@ -3206,9 +3175,7 @@ def test_gemini_36_flash_and_35_flash_lite_launch_pricing(model, input_cost, out assert model_cost_map["max_input_tokens"] == 1048576 -def test_generic_cost_per_token_gemini_36_flash(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_generic_cost_per_token_gemini_36_flash(_local_model_cost_map): usage = Usage( prompt_tokens=1000, @@ -3274,9 +3241,7 @@ def test_gemini_36_flash_batch_introductory_pricing(model, _local_model_cost_map assert model_cost_map["output_cost_per_token_batches"] == 1.875e-06 -def test_generic_cost_per_token_gemini_35_flash_lite(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_generic_cost_per_token_gemini_35_flash_lite(_local_model_cost_map): usage = Usage( prompt_tokens=1000,