diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index da420ab05787..2a01135216d3 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -299,6 +299,45 @@ def _get_additional_costs( return None +def _recorded_cache_storage_token_hours(litellm_logging_obj: LitellmLoggingObject | None) -> object: + return litellm_logging_obj.context_cache_storage_token_hours if litellm_logging_obj else 0.0 + + +def _context_cache_storage_cost( + litellm_logging_obj: LitellmLoggingObject | None, + *, + requested_model: str | None, + completion_response: object | None, + base_model: str | None, + custom_llm_provider: str | None, + router_model_id: str | None, + region_name: str | None, +) -> float: + token_hours: Final = _recorded_cache_storage_token_hours(litellm_logging_obj) + if not isinstance(token_hours, int | float) or token_hours <= 0: + return 0.0 + underlying_model: Final = select_model_name_for_cost_calc( + model=requested_model, + completion_response=completion_response, + base_model=base_model, + custom_llm_provider=custom_llm_provider, + region_name=region_name, + ) + rate: Final = next( + ( + price + for source in ( + _deployment_model_info(litellm_logging_obj, custom_pricing=True, router_model_id=router_model_id), + _cost_map_model_info(underlying_model, custom_llm_provider) if underlying_model else None, + ) + if source is not None + and isinstance(price := source.get("cache_storage_cost_per_token_per_hour"), int | float) + ), + 0.0, + ) + return token_hours * rate + + def _transcription_usage_has_token_details( usage_block: Usage | None, ) -> bool: @@ -1482,6 +1521,7 @@ def completion_cost( region_name=region_name, ) + requested_model: Final = model potential_model_names: Final = [ selected_model, _get_response_model(completion_response), @@ -1871,6 +1911,20 @@ def completion_cost( ) else: additional_costs = None + _storage_response: ModelResponse | None = ( + completion_response if isinstance(completion_response, ModelResponse) else None + ) + _storage_provider: str | None = custom_llm_provider if isinstance(custom_llm_provider, str) else None + _storage_region: str | None = region_name if isinstance(region_name, str) else None + cache_storage_cost: float = _context_cache_storage_cost( + litellm_logging_obj, + requested_model=requested_model, + completion_response=_storage_response, + base_model=base_model, + custom_llm_provider=_storage_provider, + router_model_id=router_model_id, + region_name=_storage_region, + ) _final_cost = prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar cost_for_built_in_tools = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( @@ -1883,6 +1937,12 @@ def completion_cost( _final_cost += cost_for_built_in_tools if additional_costs: _final_cost += sum(additional_costs.values()) + if cache_storage_cost > 0: + _final_cost += cache_storage_cost + if additional_costs is None: + additional_costs = {"cache_storage_cost": cache_storage_cost} + else: + additional_costs["cache_storage_cost"] = cache_storage_cost original_cost = _final_cost if litellm.cost_discount_config: diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 715e492b159e..f034107b8864 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -575,6 +575,10 @@ class Logging(LiteLLMLoggingBaseClass): classifier_input: Mapping[str, JsonValue] | None = None baseline_cache_context: "BaselineCacheContext | None" = None baseline_observation: "CapturedBaselineObservation | None" = None + context_cache_storage_token_hours: float = 0.0 + + def record_context_cache_storage(self, token_hours: float) -> None: + self.context_cache_storage_token_hours = token_hours @property def _defer_async_logging(self) -> bool: diff --git a/litellm/llms/vertex_ai/context_caching/storage_cost.py b/litellm/llms/vertex_ai/context_caching/storage_cost.py new file mode 100644 index 000000000000..7cb74296f94f --- /dev/null +++ b/litellm/llms/vertex_ai/context_caching/storage_cost.py @@ -0,0 +1,56 @@ +from datetime import datetime +from typing import Final + +from pydantic import BaseModel, ConfigDict, ValidationError + +DEFAULT_CONTEXT_CACHE_TTL_SECONDS: Final = 3600.0 +_SECONDS_PER_HOUR: Final = 3600.0 + + +class _CachedContentUsage(BaseModel): + model_config = ConfigDict(frozen=True) + + totalTokenCount: int = 0 + + +class _CreatedCachedContent(BaseModel): + model_config = ConfigDict(frozen=True) + + usageMetadata: _CachedContentUsage = _CachedContentUsage() + createTime: datetime | None = None + expireTime: datetime | None = None + + +def _reported_lifetime_seconds(created: _CreatedCachedContent) -> float | None: + if created.createTime is None or created.expireTime is None: + return None + seconds: Final = (created.expireTime - created.createTime).total_seconds() + return seconds if seconds > 0 else None + + +def _requested_ttl_seconds(requested_ttl: str | None) -> float | None: + if requested_ttl is None or not requested_ttl.endswith("s"): + return None + try: + seconds: Final = float(requested_ttl[:-1]) + except ValueError: + return None + return seconds if seconds > 0 else None + + +def context_cache_storage_token_hours(created_cached_content: object, requested_ttl: str | None) -> float: + """Token-hours a newly created cachedContents resource is billed for over its whole TTL. + + The lifetime comes from the create response (expireTime - createTime), then the TTL that was + requested, then the provider default of one hour. + """ + try: + created: Final = _CreatedCachedContent.model_validate(created_cached_content) + except ValidationError: + return 0.0 + lifetime_seconds: Final = ( + _reported_lifetime_seconds(created) + or _requested_ttl_seconds(requested_ttl) + or DEFAULT_CONTEXT_CACHE_TTL_SECONDS + ) + return created.usageMetadata.totalTokenCount * lifetime_seconds / _SECONDS_PER_HOUR diff --git a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py index 2d35dd9b480f..7e33bde0c09b 100644 --- a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py @@ -21,6 +21,7 @@ from ..common_utils import VertexAIError, get_vertex_base_url from ..vertex_llm_base import VertexBase +from .storage_cost import context_cache_storage_token_hours from .transformation import ( cached_messages_end_on_supported_turn, separate_cached_messages, @@ -425,6 +426,9 @@ def check_and_create_cache( raise VertexAIError(status_code=408, message="Timeout error occurred.") raw_response_cached: Final = response.json() + logging_obj.record_context_cache_storage( + context_cache_storage_token_hours(raw_response_cached, cached_content_request_body.get("ttl")) + ) cached_content_response_obj: Final = VertexAICachedContentResponseObject( name=raw_response_cached.get("name"), model=raw_response_cached.get("model") ) @@ -581,6 +585,9 @@ async def async_check_and_create_cache( raise VertexAIError(status_code=408, message="Timeout error occurred.") raw_response_cached: Final = response.json() + logging_obj.record_context_cache_storage( + context_cache_storage_token_hours(raw_response_cached, cached_content_request_body.get("ttl")) + ) cached_content_response_obj: Final = VertexAICachedContentResponseObject( name=raw_response_cached.get("name"), model=raw_response_cached.get("model") ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f34606ce1272..ecdee276b4d6 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1,5 +1,6 @@ { "sample_spec": { + "cache_storage_cost_per_token_per_hour": 0.0, "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, "computer_use_output_cost_per_1k_tokens": 0.0, @@ -27091,6 +27092,7 @@ "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, "litellm_provider": "vertex_ai-language-models", @@ -27442,6 +27444,7 @@ }, "gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 2.5e-07, "litellm_provider": "vertex_ai-language-models", @@ -27496,6 +27499,7 @@ "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, @@ -27557,6 +27561,7 @@ "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, @@ -27651,6 +27656,7 @@ "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 1.8e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_audio_token": 3e-07, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-language-models", @@ -27939,6 +27945,7 @@ "cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07, "cache_read_input_token_cost_flex": 1.25e-07, "cache_read_input_token_cost_priority": 2.25e-07, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "vertex_ai-language-models", @@ -28041,6 +28048,7 @@ "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_flex": 2e-07, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28152,6 +28160,7 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28202,6 +28211,7 @@ "input_cost_per_audio_token_priority": 1.8e-06, "output_cost_per_token_priority": 5.4e-06, "cache_read_input_token_cost_priority": 9e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28274,7 +28284,8 @@ "output_cost_per_token_batches": 4.5e-06, "input_cost_per_token_flex": 7.5e-07, "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08 + "cache_read_input_token_cost_flex": 7.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06 }, "vertex_ai/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, @@ -28326,6 +28337,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2026-11-19", "search_context_cost_per_query": { "search_context_size_low": 0.014, @@ -28386,6 +28398,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2027-01-28", "search_context_cost_per_query": { "search_context_size_low": 0.014, @@ -28446,6 +28459,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28459,6 +28473,7 @@ "cache_read_input_token_cost_batches": 7.5e-08, "cache_read_input_token_cost_flex": 7.5e-08, "cache_read_input_token_cost_priority": 2.7e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_token": 1.5e-06, "input_cost_per_token_batches": 7.5e-07, "input_cost_per_token_flex": 7.5e-07, @@ -28592,6 +28607,7 @@ "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_flex": 2e-07, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30449,6 +30465,7 @@ "input_cost_per_audio_token_priority": 1.8e-06, "output_cost_per_token_priority": 5.4e-06, "cache_read_input_token_cost_priority": 9e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30552,7 +30569,8 @@ "output_cost_per_token_batches": 4.5e-06, "input_cost_per_token_flex": 7.5e-07, "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08 + "cache_read_input_token_cost_flex": 7.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06 }, "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, @@ -30604,6 +30622,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2026-11-19", "search_context_cost_per_query": { "search_context_size_low": 0.014, @@ -30664,6 +30683,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2027-01-28", "search_context_cost_per_query": { "search_context_size_low": 0.014, @@ -30724,6 +30744,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30737,6 +30758,7 @@ "cache_read_input_token_cost_batches": 7.5e-08, "cache_read_input_token_cost_flex": 7.5e-08, "cache_read_input_token_cost_priority": 2.7e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_token": 1.5e-06, "input_cost_per_token_batches": 7.5e-07, "input_cost_per_token_flex": 7.5e-07, @@ -50905,6 +50927,7 @@ }, "vertex_ai/gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 2.5e-07, "litellm_provider": "vertex_ai-language-models", @@ -50959,6 +50982,7 @@ "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, @@ -51021,6 +51045,7 @@ "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, diff --git a/litellm/types/router.py b/litellm/types/router.py index a66c4571b393..4da354fd8182 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -274,6 +274,7 @@ class ModelInfo(MirroredPricingParams): cost_per_ptu_per_hour: float | None = None ptu_effective_from: datetime.datetime | None = None ptu_effective_to: datetime.datetime | None = None + cache_storage_cost_per_token_per_hour: float | None = None # when tag-based routing's "!" or "&" constraints eliminate every deployment # in this model group, fall back to the default-tagged pool instead of diff --git a/litellm/types/utils.py b/litellm/types/utils.py index a99c22e9479f..352b8ad4cb38 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -324,6 +324,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_creation_input_token_cost_above_200k_tokens_batches: ReadOnly[float | None] cache_creation_input_token_cost_above_100k_tokens_batches: ReadOnly[float | None] cache_creation_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None] + cache_storage_cost_per_token_per_hour: ReadOnly[float | None] # Smallest prefix this model will actually cache, whatever caching mechanism its provider uses. # Absent means the provider-agnostic default applies; see MINIMUM_PROMPT_CACHE_TOKEN_COUNT. prompt_cache_min_tokens: int | None @@ -3945,12 +3946,15 @@ def strip_custom_pricing_fields(cls, model_info: dict[str, Any]) -> dict[str, An DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"}) +DEPLOYMENT_SCOPED_SEPARATE_CHARGE_FIELDS: Final[frozenset[str]] = frozenset({"cache_storage_cost_per_token_per_hour"}) + DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset({"supports_regex_lookaround"}) SHARED_BACKEND_MODEL_INFO_FIELDS: Final[frozenset[str]] = ( frozenset(ModelInfoBase.__required_keys__ | ModelInfoBase.__optional_keys__) - frozenset(CustomPricingLiteLLMParams.model_fields) - DEPLOYMENT_SCOPED_PRICING_FIELDS + - DEPLOYMENT_SCOPED_SEPARATE_CHARGE_FIELDS - DEPLOYMENT_SCOPED_CAPABILITY_FIELDS ) diff --git a/litellm/utils.py b/litellm/utils.py index 89544cf31e2a..8a22f29f72aa 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6238,6 +6238,7 @@ def get_model_info_helper( cache_creation_input_token_cost_above_1hr=_model_info.get( "cache_creation_input_token_cost_above_1hr", None ), + cache_storage_cost_per_token_per_hour=_model_info.get("cache_storage_cost_per_token_per_hour"), off_peak_pricing=_model_info.get("off_peak_pricing", None), input_cost_per_character=_model_info.get("input_cost_per_character", None), input_cost_per_token_above_128k_tokens=_model_info.get("input_cost_per_token_above_128k_tokens", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f34606ce1272..ecdee276b4d6 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1,5 +1,6 @@ { "sample_spec": { + "cache_storage_cost_per_token_per_hour": 0.0, "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, "computer_use_output_cost_per_1k_tokens": 0.0, @@ -27091,6 +27092,7 @@ "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, "litellm_provider": "vertex_ai-language-models", @@ -27442,6 +27444,7 @@ }, "gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 2.5e-07, "litellm_provider": "vertex_ai-language-models", @@ -27496,6 +27499,7 @@ "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, @@ -27557,6 +27561,7 @@ "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, @@ -27651,6 +27656,7 @@ "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 1.8e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_audio_token": 3e-07, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-language-models", @@ -27939,6 +27945,7 @@ "cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07, "cache_read_input_token_cost_flex": 1.25e-07, "cache_read_input_token_cost_priority": 2.25e-07, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "vertex_ai-language-models", @@ -28041,6 +28048,7 @@ "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_flex": 2e-07, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28152,6 +28160,7 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28202,6 +28211,7 @@ "input_cost_per_audio_token_priority": 1.8e-06, "output_cost_per_token_priority": 5.4e-06, "cache_read_input_token_cost_priority": 9e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28274,7 +28284,8 @@ "output_cost_per_token_batches": 4.5e-06, "input_cost_per_token_flex": 7.5e-07, "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08 + "cache_read_input_token_cost_flex": 7.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06 }, "vertex_ai/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, @@ -28326,6 +28337,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2026-11-19", "search_context_cost_per_query": { "search_context_size_low": 0.014, @@ -28386,6 +28398,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2027-01-28", "search_context_cost_per_query": { "search_context_size_low": 0.014, @@ -28446,6 +28459,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28459,6 +28473,7 @@ "cache_read_input_token_cost_batches": 7.5e-08, "cache_read_input_token_cost_flex": 7.5e-08, "cache_read_input_token_cost_priority": 2.7e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_token": 1.5e-06, "input_cost_per_token_batches": 7.5e-07, "input_cost_per_token_flex": 7.5e-07, @@ -28592,6 +28607,7 @@ "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_flex": 2e-07, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30449,6 +30465,7 @@ "input_cost_per_audio_token_priority": 1.8e-06, "output_cost_per_token_priority": 5.4e-06, "cache_read_input_token_cost_priority": 9e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30552,7 +30569,8 @@ "output_cost_per_token_batches": 4.5e-06, "input_cost_per_token_flex": 7.5e-07, "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08 + "cache_read_input_token_cost_flex": 7.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06 }, "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, @@ -30604,6 +30622,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2026-11-19", "search_context_cost_per_query": { "search_context_size_low": 0.014, @@ -30664,6 +30683,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2027-01-28", "search_context_cost_per_query": { "search_context_size_low": 0.014, @@ -30724,6 +30744,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30737,6 +30758,7 @@ "cache_read_input_token_cost_batches": 7.5e-08, "cache_read_input_token_cost_flex": 7.5e-08, "cache_read_input_token_cost_priority": 2.7e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_token": 1.5e-06, "input_cost_per_token_batches": 7.5e-07, "input_cost_per_token_flex": 7.5e-07, @@ -50905,6 +50927,7 @@ }, "vertex_ai/gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 2.5e-07, "litellm_provider": "vertex_ai-language-models", @@ -50959,6 +50982,7 @@ "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, @@ -51021,6 +51045,7 @@ "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index e51ae0453ea5..8e942d2af98e 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -281,6 +281,10 @@ "type": "number", "minimum": 0 }, + "cache_storage_cost_per_token_per_hour": { + "type": "number", + "minimum": 0 + }, "citation_cost_per_token": { "type": "number", "minimum": 0 diff --git a/tests/unit/llms/vertex_ai/context_caching/test_storage_cost.py b/tests/unit/llms/vertex_ai/context_caching/test_storage_cost.py new file mode 100644 index 000000000000..5fd3bbad9cc3 --- /dev/null +++ b/tests/unit/llms/vertex_ai/context_caching/test_storage_cost.py @@ -0,0 +1,51 @@ +from typing import Final + +import pytest + +from litellm.llms.vertex_ai.context_caching.storage_cost import ( + context_cache_storage_token_hours, +) + + +@pytest.mark.parametrize( + "created, requested_ttl, expected", + [ + ( + { + "usageMetadata": {"totalTokenCount": 5000}, + "createTime": "2026-01-01T00:00:00.123456789Z", + "expireTime": "2026-01-01T02:00:00.123456789Z", + }, + "60s", + 10000.0, + ), + ({"usageMetadata": {"totalTokenCount": 5000}}, "1800s", 2500.0), + ({"usageMetadata": {"totalTokenCount": 5000}}, "90.5s", 5000 * 90.5 / 3600), + ( + {"usageMetadata": {"totalTokenCount": 4000}, "expireTime": "2026-01-01T02:00:00Z"}, + "900s", + 1000.0, + ), + ({"name": "cachedContents/1"}, "3600s", 0.0), + ({"usageMetadata": {"totalTokenCount": "many"}}, "3600s", 0.0), + ({"usageMetadata": {"totalTokenCount": 5000}, "expireTime": "not-a-time"}, "3600s", 0.0), + ("not-an-object", "3600s", 0.0), + ], +) +def test_token_hours_follow_reported_lifetime_then_requested_ttl( + created: object, requested_ttl: str | None, expected: float +) -> None: + assert context_cache_storage_token_hours(created, requested_ttl) == pytest.approx(expected) + + +def test_token_hours_default_to_one_hour_without_lifetime_or_ttl() -> None: + # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/context-cache/context-cache-create + # "The default expiration time of a context cache is 60 minutes after it's created" (read 2026-10-07) + assert context_cache_storage_token_hours({"usageMetadata": {"totalTokenCount": 5000}}, None) == 5000.0 + + +@pytest.mark.parametrize("requested_ttl", ["", "s", "abc", "10m", "-5s"]) +def test_unparseable_requested_ttl_falls_back_to_default_lifetime(requested_ttl: str) -> None: + created: Final = {"usageMetadata": {"totalTokenCount": 3600}} + + assert context_cache_storage_token_hours(created, requested_ttl) == context_cache_storage_token_hours(created, None) diff --git a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py index 67d78d6030e8..bf9f73aa08e2 100644 --- a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py +++ b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py @@ -1,8 +1,10 @@ -from typing import Final +from collections.abc import Mapping +from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +from typing_extensions import ReadOnly, TypedDict import litellm @@ -12,6 +14,7 @@ from litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching import ( ContextCachingEndpoints, ) +from litellm.types.llms.openai import AllMessageValues @pytest.fixture @@ -2289,3 +2292,144 @@ def test_vertex_ai_global_still_uses_global_host(self): assert url.startswith("https://aiplatform.googleapis.com/") assert "/locations/global/cachedContents" in url + + +_STORAGE_TEST_MODEL: Final = "gemini-2.5-pro" + + +_STORAGE_TEST_CACHE_NAME: Final = "cachedContents/storage-cost" + + +def _storage_logging_obj() -> Logging: + logging_obj: Final = Logging( + model=_STORAGE_TEST_MODEL, + messages=[], + stream=False, + call_type="acompletion", + start_time=None, + litellm_call_id="context-cache-storage", + function_id="context-cache-storage", + ) + logging_obj.update_environment_variables( + model=_STORAGE_TEST_MODEL, litellm_params={}, optional_params={}, custom_llm_provider="gemini" + ) + return logging_obj + + +def _storage_messages() -> list[AllMessageValues]: + return [ + { + "role": "system", + "content": [{"type": "text", "text": "policy " * 6000, "cache_control": {"type": "ephemeral"}}], + }, + {"role": "user", "content": "Hello"}, + ] + + +def _json_response(method: str, payload: Mapping[str, object]) -> httpx.Response: + return httpx.Response(200, json=payload, request=httpx.Request(method, "https://example.invalid")) + + +_CREATED_CACHE: Final = { + "name": _STORAGE_TEST_CACHE_NAME, + "model": f"models/{_STORAGE_TEST_MODEL}", + "usageMetadata": {"totalTokenCount": 7200}, + "createTime": "2026-01-01T00:00:00Z", + "expireTime": "2026-01-01T00:30:00Z", +} + + +class _CreateCacheKwargs(TypedDict): + messages: ReadOnly[list[AllMessageValues]] + optional_params: ReadOnly[dict[str, object]] + api_key: ReadOnly[str] + api_base: ReadOnly[str | None] + model: ReadOnly[str] + client: ReadOnly[MagicMock] + timeout: ReadOnly[float] + logging_obj: ReadOnly[Logging] + custom_llm_provider: ReadOnly[Literal["gemini"]] + vertex_project: ReadOnly[str | None] + vertex_location: ReadOnly[str | None] + vertex_auth_header: ReadOnly[str | None] + + +def _create_kwargs(logging_obj: Logging, client: MagicMock) -> _CreateCacheKwargs: + return { + "messages": _storage_messages(), + "optional_params": {}, + "api_key": "test_key", + "api_base": None, + "model": _STORAGE_TEST_MODEL, + "client": client, + "timeout": 30.0, + "logging_obj": logging_obj, + "custom_llm_provider": "gemini", + "vertex_project": None, + "vertex_location": None, + "vertex_auth_header": None, + } + + +def test_creating_a_cache_records_its_storage_token_hours(local_model_cost_map: None) -> None: + logging_obj: Final = _storage_logging_obj() + client: Final = MagicMock(spec=HTTPHandler) + client.get.return_value = _json_response("GET", {}) + client.post.return_value = _json_response("POST", _CREATED_CACHE) + + _, _, cache_name = ContextCachingEndpoints().check_and_create_cache(**_create_kwargs(logging_obj, client)) + + assert cache_name == _STORAGE_TEST_CACHE_NAME + assert logging_obj.context_cache_storage_token_hours == pytest.approx(7200 * 0.5) + + +@pytest.mark.asyncio +async def test_async_creating_a_cache_records_its_storage_token_hours(local_model_cost_map: None) -> None: + logging_obj: Final = _storage_logging_obj() + client: Final = MagicMock(spec=AsyncHTTPHandler) + client.get = AsyncMock(return_value=_json_response("GET", {})) + client.post = AsyncMock(return_value=_json_response("POST", _CREATED_CACHE)) + + _, _, cache_name = await ContextCachingEndpoints().async_check_and_create_cache( + **_create_kwargs(logging_obj, client) + ) + + assert cache_name == _STORAGE_TEST_CACHE_NAME + assert logging_obj.context_cache_storage_token_hours == pytest.approx(7200 * 0.5) + + +def _listing_with(created_body: Mapping[str, object]) -> Mapping[str, object]: + return {"cachedContents": [{"name": _STORAGE_TEST_CACHE_NAME, "displayName": created_body["displayName"]}]} + + +def test_reusing_an_existing_cache_records_no_storage(local_model_cost_map: None) -> None: + client: Final = MagicMock(spec=HTTPHandler) + client.get.return_value = _json_response("GET", {}) + client.post.return_value = _json_response("POST", _CREATED_CACHE) + ContextCachingEndpoints().check_and_create_cache(**_create_kwargs(_storage_logging_obj(), client)) + client.get.return_value = _json_response("GET", _listing_with(client.post.call_args.kwargs["json"])) + reusing_logging_obj: Final = _storage_logging_obj() + + _, _, cache_name = ContextCachingEndpoints().check_and_create_cache(**_create_kwargs(reusing_logging_obj, client)) + + assert cache_name == _STORAGE_TEST_CACHE_NAME + assert client.post.call_count == 1 + assert reusing_logging_obj.context_cache_storage_token_hours == 0.0 + + +@pytest.mark.asyncio +async def test_async_reusing_an_existing_cache_records_no_storage(local_model_cost_map: None) -> None: + client: Final = MagicMock(spec=AsyncHTTPHandler) + client.get = AsyncMock(return_value=_json_response("GET", {})) + client.post = AsyncMock(return_value=_json_response("POST", _CREATED_CACHE)) + await ContextCachingEndpoints().async_check_and_create_cache(**_create_kwargs(_storage_logging_obj(), client)) + client.get.return_value = _json_response("GET", _listing_with(client.post.call_args.kwargs["json"])) + reusing_logging_obj: Final = _storage_logging_obj() + + _, _, cache_name = await ContextCachingEndpoints().async_check_and_create_cache( + **_create_kwargs(reusing_logging_obj, client) + ) + + assert cache_name == _STORAGE_TEST_CACHE_NAME + assert client.post.call_count == 1 + assert reusing_logging_obj.context_cache_storage_token_hours == 0.0 diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 7d01b1b2906b..2851ec8f766c 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -1,6 +1,8 @@ import datetime +import math import time -from collections.abc import Mapping +from collections.abc import Iterator, Mapping +from pathlib import Path from types import MappingProxyType, SimpleNamespace from typing import Final, Literal, cast @@ -5817,6 +5819,211 @@ def test_completion_cost_bills_base_when_gemini_serves_on_demand( assert cost == pytest.approx(100 * 0.001 + 50 * 0.002) +_STORAGE_RATE_KEY: Final = "cache_storage_cost_per_token_per_hour" + + +_FIXED_START_TIME: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc) + + +@pytest.fixture +def _storage_cost_map(_local_model_cost_map: None) -> Iterator[None]: + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +def _first_chat_model(provider_prefix: str, *, with_storage_rate: bool) -> str: + names: Final = [name for name in litellm.model_cost if isinstance(name, str)] + return next( + name + for name in names + if _is_priced_chat_entry(_cost_map_entry(name), provider_prefix, with_storage_rate=with_storage_rate) + ) + + +def _cost_map_entry(name: str) -> object: + entry: Final[object] = litellm.model_cost.get(name) + return entry + + +def _is_priced_chat_entry(entry: object, provider_prefix: str, *, with_storage_rate: bool) -> bool: + return ( + isinstance(entry, dict) + and entry.get("mode") == "chat" + and isinstance(provider := entry.get("litellm_provider"), str) + and provider.startswith(provider_prefix) + and entry.get("input_cost_per_token") is not None + and (_STORAGE_RATE_KEY in entry) == with_storage_rate + ) + + +def _cache_storage_logging_obj(model: str, token_hours: float) -> Logging: + logging_obj: Final = Logging( + model=model, + messages=[{"role": "user", "content": "Hello"}], + stream=False, + call_type="completion", + start_time=_FIXED_START_TIME, + litellm_call_id="context-cache-storage", + function_id="f", + ) + if token_hours: + logging_obj.record_context_cache_storage(token_hours) + return logging_obj + + +def _chat_cost(model: str, logging_obj: Logging, **kwargs: object) -> float: + usage: Final = Usage(prompt_tokens=1_000, completion_tokens=100, total_tokens=1_100) + return completion_cost( + completion_response=ModelResponse(model=model, usage=usage), + model=model, + litellm_logging_obj=logging_obj, + **kwargs, + ) + + +def test_context_cache_storage_is_billed_on_the_creating_request(_storage_cost_map: None) -> None: + model: Final = _first_chat_model("vertex_ai", with_storage_rate=True) + rate: Final = litellm.get_model_info(model)[_STORAGE_RATE_KEY] + creating_request: Final = _cache_storage_logging_obj(model, token_hours=3_600.0) + + with_storage: Final = _chat_cost(model, creating_request) + without_storage: Final = _chat_cost(model, _cache_storage_logging_obj(model, token_hours=0.0)) + + assert rate is not None and rate > 0 + assert with_storage - without_storage == pytest.approx(3_600.0 * rate) + assert creating_request.cost_breakdown["additional_costs"] == pytest.approx({"cache_storage_cost": 3_600.0 * rate}) + assert creating_request.cost_breakdown["total_cost"] == pytest.approx(with_storage) + + +def test_context_cache_storage_costs_nothing_for_models_without_a_storage_rate(_storage_cost_map: None) -> None: + model: Final = _first_chat_model("gemini", with_storage_rate=False) + creating_request: Final = _cache_storage_logging_obj(model, token_hours=3_600.0) + + with_storage: Final = _chat_cost(model, creating_request) + + assert with_storage == _chat_cost(model, _cache_storage_logging_obj(model, token_hours=0.0)) + assert creating_request.cost_breakdown.get("additional_costs") is None + + +def test_context_cache_storage_uses_the_deployment_storage_rate(_storage_cost_map: None) -> None: + model: Final = _first_chat_model("vertex_ai", with_storage_rate=True) + deployment_rate: Final = litellm.get_model_info(model)[_STORAGE_RATE_KEY] * 3 + litellm.register_model( + { + "storage-cost-deployment": { + "litellm_provider": "vertex_ai", + "mode": "chat", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + _STORAGE_RATE_KEY: deployment_rate, + } + } + ) + pricing: Final = {"custom_pricing": True, "router_model_id": "storage-cost-deployment"} + + with_storage: Final = _chat_cost(model, _cache_storage_logging_obj(model, token_hours=500.0), **pricing) + without_storage: Final = _chat_cost(model, _cache_storage_logging_obj(model, token_hours=0.0), **pricing) + + assert with_storage - without_storage == pytest.approx(500.0 * deployment_rate) + + +def test_context_cache_storage_joins_the_provider_additional_costs(_storage_cost_map: None) -> None: + litellm.register_model( + {"router-storage-deployment": {"litellm_provider": "azure_ai", "mode": "chat", _STORAGE_RATE_KEY: 1e-6}} + ) + logging_obj: Final = _cache_storage_logging_obj("azure_ai/model-router", token_hours=500.0) + usage: Final = Usage(prompt_tokens=1_000_000, completion_tokens=100, total_tokens=1_000_100) + + cost: Final = completion_cost( + completion_response=ModelResponse(model="azure_ai/gpt-4o-2024-08-06", usage=usage), + model="azure_ai/model-router", + custom_llm_provider="azure_ai", + litellm_logging_obj=logging_obj, + custom_pricing=True, + router_model_id="router-storage-deployment", + ) + + breakdown: Final = logging_obj.cost_breakdown + assert breakdown is not None + additional_costs: Final = breakdown.get("additional_costs") or {} + assert additional_costs["Azure Model Router Flat Cost"] > 0 + assert math.isclose(additional_costs["cache_storage_cost"], 500.0 * 1e-6) + assert math.isclose(cost, sum(additional_costs.values())) + + +_STORAGE_DEPLOYMENT_MODEL: Final = "gemini-2.5-flash" + + +def _routed_request_cost( + token_prices: Mapping[str, float], token_hours: float, *, storage_rate: float | None = None +) -> float | None: + from litellm import Router + + router: Final = Router( + model_list=[ + { + "model_name": "storage-cost-deployment", + "litellm_params": {"model": f"vertex_ai/{_STORAGE_DEPLOYMENT_MODEL}", **token_prices}, + "model_info": {} if storage_rate is None else {_STORAGE_RATE_KEY: storage_rate}, + } + ] + ) + deployment_id: Final = router.model_list[0]["model_info"]["id"] + assert isinstance(deployment_id, str) + return _vertex_request_cost({**token_prices, "metadata": {"model_info": {"id": deployment_id}}}, token_hours) + + +def _vertex_request_cost(litellm_params: Mapping[str, object], token_hours: float) -> float | None: + logging_obj: Final = _cache_storage_logging_obj(_STORAGE_DEPLOYMENT_MODEL, token_hours) + logging_obj.update_environment_variables( + model=_STORAGE_DEPLOYMENT_MODEL, + litellm_params=dict(litellm_params), + optional_params={}, + custom_llm_provider="vertex_ai", + ) + usage: Final = Usage(prompt_tokens=1_000, completion_tokens=100, total_tokens=1_100) + return logging_obj.response_cost_calculator(ModelResponse(model=_STORAGE_DEPLOYMENT_MODEL, usage=usage)) + + +def test_custom_token_prices_without_a_storage_rate_bill_storage_at_the_underlying_model_rate( + _storage_cost_map: None, +) -> None: + cost_map_rate: Final = litellm.get_model_info(f"vertex_ai/{_STORAGE_DEPLOYMENT_MODEL}")[_STORAGE_RATE_KEY] + token_prices: Final = {"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6} + + with_storage: Final = _routed_request_cost(token_prices, token_hours=500.0) + without_storage: Final = _routed_request_cost(token_prices, token_hours=0.0) + + assert cost_map_rate is not None and cost_map_rate > 0 + assert without_storage == pytest.approx(1_000 * 1e-6 + 100 * 2e-6) + assert with_storage == pytest.approx(without_storage + 500.0 * cost_map_rate) + + +def test_a_storage_only_override_keeps_the_underlying_token_prices(_storage_cost_map: None) -> None: + cost_map_rate: Final = litellm.get_model_info(f"vertex_ai/{_STORAGE_DEPLOYMENT_MODEL}")[_STORAGE_RATE_KEY] + assert isinstance(cost_map_rate, float) + deployment_rate: Final = cost_map_rate * 3 + + no_override: Final = _vertex_request_cost({}, token_hours=0.0) + without_storage: Final = _routed_request_cost({}, token_hours=0.0, storage_rate=deployment_rate) + with_storage: Final = _routed_request_cost({}, token_hours=500.0, storage_rate=deployment_rate) + + assert no_override is not None and no_override > 0 + assert without_storage == pytest.approx(no_override) + assert with_storage == pytest.approx(no_override + 500.0 * deployment_rate) + + +def test_a_deployment_storage_rate_stays_off_the_shared_backend_key(_storage_cost_map: None) -> None: + backend_key: Final = f"vertex_ai/{_STORAGE_DEPLOYMENT_MODEL}" + cost_map_rate: Final = litellm.get_model_info(backend_key)[_STORAGE_RATE_KEY] + assert isinstance(cost_map_rate, float) + + _routed_request_cost({}, token_hours=0.0, storage_rate=cost_map_rate * 3) + + assert litellm.get_model_info(backend_key)[_STORAGE_RATE_KEY] == cost_map_rate + + def test_custom_pricing_as_completion_cost_param(): from litellm import Choices, Message, ModelResponse from litellm.utils import Usage diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 4aed84d05c8c..d9404b3922e4 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -1471,13 +1471,17 @@ def test_custom_pricing_field_denylist_covers_all_builtin_pricing_fields(): """ import typing - from litellm.types.utils import CustomPricingLiteLLMParams, ModelInfoBase + from litellm.types.utils import ( + DEPLOYMENT_SCOPED_SEPARATE_CHARGE_FIELDS, + CustomPricingLiteLLMParams, + ModelInfoBase, + ) pricing_markers = ("cost", "price", "uplift", "vector_size", "tiered_pricing") builtin_pricing_fields = { name for name in typing.get_type_hints(ModelInfoBase) if any(marker in name for marker in pricing_markers) } - denylisted_fields = set(CustomPricingLiteLLMParams.model_fields.keys()) + denylisted_fields = set(CustomPricingLiteLLMParams.model_fields.keys()) | DEPLOYMENT_SCOPED_SEPARATE_CHARGE_FIELDS uncovered = sorted(builtin_pricing_fields - denylisted_fields) assert not uncovered, ( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 7551a3109778..375c34f0e844 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -808,6 +808,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "cache_creation_input_token_cost_flex": {"type": "number"}, "cache_creation_input_token_cost_priority": {"type": "number"}, "cache_creation_input_token_cost_ultrafast": {"type": "number"}, + "cache_storage_cost_per_token_per_hour": {"type": "number"}, "cache_read_input_token_cost": {"type": "number"}, "cache_read_input_token_cost_above_32k_tokens": {"type": "number"}, "cache_read_input_token_cost_above_100k_tokens": {"type": "number"}, @@ -7214,6 +7215,26 @@ async def yielding_truncate(messages): assert counting_logger.logged_results[0] is inner_result +def test_get_model_info_surfaces_context_cache_storage_rate(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + local_cost_map: Final[Mapping[str, object]] = litellm.get_model_cost_map(url="") + monkeypatch.setattr(litellm, "model_cost", local_cost_map) + litellm.get_model_info.cache_clear() + priced: Final = { + name: entry["cache_storage_cost_per_token_per_hour"] + for name, entry in litellm.model_cost.items() + if isinstance(entry, dict) and "cache_storage_cost_per_token_per_hour" in entry + } + + surfaced: Final = { + name: litellm.get_model_info(name).get("cache_storage_cost_per_token_per_hour") for name in priced + } + litellm.get_model_info.cache_clear() + + assert priced + assert surfaced == priced + + def test_basic_trimming(): litellm.turn_on_debug() messages = [ diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 1108d4a8743d..d91c0353323c 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -50823,6 +50823,8 @@ export interface components { cache_creation_input_token_cost?: number | null; /** Cache Read Input Token Cost */ cache_read_input_token_cost?: number | null; + /** Cache Storage Cost Per Token Per Hour */ + cache_storage_cost_per_token_per_hour?: number | null; /** Cost Per Ptu Per Hour */ cost_per_ptu_per_hour?: number | null; /** Created At */