feat(vertex_ai): bill explicit context cache storage per token-hour

basedpyright-code-budget.json

@@ -99,7 +99,7 @@
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 44802
+    "limit": 44899
   },
   "reportUnknownLambdaType": {
     "limit": 109

litellm/cost_calculator.py

@@ -299,6 +299,41 @@ def _get_additional_costs(
     return None
 
 
+def _context_cache_storage_cost(
+    litellm_logging_obj: LitellmLoggingObject | None,
+    *,
+    requested_model: str | None,
+    completion_response: object | None,
+    base_model: str | None,
+    custom_llm_provider: str | None,
+    router_model_id: str | None,
+    region_name: str | None,
+) -> float:
+    token_hours: Final = litellm_logging_obj.context_cache_storage_token_hours if litellm_logging_obj else 0.0
+    if not isinstance(token_hours, int | float) or token_hours <= 0:
+        return 0.0
+    underlying_model: Final = select_model_name_for_cost_calc(
+        model=requested_model,
+        completion_response=completion_response,
+        base_model=base_model,
+        custom_llm_provider=custom_llm_provider,
+        region_name=region_name,
+    )
+    rate: Final = next(
+        (
+            price
+            for source in (
+                _deployment_model_info(litellm_logging_obj, custom_pricing=True, router_model_id=router_model_id),
+                _cost_map_model_info(underlying_model, custom_llm_provider) if underlying_model else None,
+            )
+            if source is not None
+            and isinstance(price := source.get("cache_storage_cost_per_token_per_hour"), int | float)
+        ),
+        0.0,
+    )
+    return token_hours * rate
+
+
 def _transcription_usage_has_token_details(
     usage_block: Usage | None,
 ) -> bool:
@@ -1455,6 +1490,7 @@ def completion_cost(
             region_name=region_name,
         )
 
+        requested_model: Final = model
         potential_model_names: Final = [
             selected_model,
             _get_response_model(completion_response),
@@ -1847,6 +1883,20 @@ def completion_cost(
                     )
                 else:
                     additional_costs = None
+                _storage_response: ModelResponse | None = (
+                    completion_response if isinstance(completion_response, ModelResponse) else None
+                )
+                _storage_provider: str | None = custom_llm_provider if isinstance(custom_llm_provider, str) else None
+                _storage_region: str | None = region_name if isinstance(region_name, str) else None
+                cache_storage_cost: float = _context_cache_storage_cost(
+                    litellm_logging_obj,
+                    requested_model=requested_model,
+                    completion_response=_storage_response,
+                    base_model=base_model,
+                    custom_llm_provider=_storage_provider,
+                    router_model_id=router_model_id,
+                    region_name=_storage_region,
+                )
 
                 _final_cost = prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar
                 cost_for_built_in_tools = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
@@ -1859,6 +1909,9 @@ def completion_cost(
                 _final_cost += cost_for_built_in_tools
                 if additional_costs:
                     _final_cost += sum(additional_costs.values())
+                if cache_storage_cost > 0:
+                    _final_cost += cache_storage_cost
+                    additional_costs = {**(additional_costs or {}), "cache_storage_cost": cache_storage_cost}
 
                 original_cost = _final_cost
                 if litellm.cost_discount_config:

litellm/litellm_core_utils/litellm_logging.py

@@ -144,6 +144,7 @@
 from litellm.types.rerank import RerankResponse
 from litellm.types.utils import (
     DEPLOYMENT_SCOPED_PRICING_FIELDS,
+    SEPARATE_CHARGE_PRICING_FIELDS,
     CachingDetails,
     CallTypes,
     CostBreakdown,
@@ -323,7 +324,9 @@ def _get_provider_request_id(original_exception: Exception) -> str | None:
 ### GLOBAL VARIABLES ###
 
 # Cache custom pricing keys as frozenset for O(1) lookups instead of looping through 49 keys
-_CUSTOM_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields.keys())
+_CUSTOM_PRICING_KEYS: Final[frozenset[str]] = (
+    frozenset(CustomPricingLiteLLMParams.model_fields.keys()) - SEPARATE_CHARGE_PRICING_FIELDS
+)
 _MODEL_INFO_CUSTOM_PRICING_KEYS: Final[frozenset[str]] = _CUSTOM_PRICING_KEYS | DEPLOYMENT_SCOPED_PRICING_FIELDS
 _UNSERIALIZABLE_METADATA_KEYS: Final[frozenset[str]] = frozenset(
     ("user_api_key_auth", "user_api_key_budget_reservation")
@@ -575,6 +578,10 @@ class Logging(LiteLLMLoggingBaseClass):
     classifier_input: Mapping[str, JsonValue] | None = None
     baseline_cache_context: "BaselineCacheContext | None" = None
     baseline_observation: "CapturedBaselineObservation | None" = None
+    context_cache_storage_token_hours: float = 0.0
+
+    def record_context_cache_storage(self, token_hours: float) -> None:
+        self.context_cache_storage_token_hours = token_hours
 
     @property
     def _defer_async_logging(self) -> bool:

litellm/llms/vertex_ai/context_caching/storage_cost.py

@@ -0,0 +1,56 @@
+from datetime import datetime
+from typing import Final
+
+from pydantic import BaseModel, ConfigDict, ValidationError
+
+DEFAULT_CONTEXT_CACHE_TTL_SECONDS: Final = 3600.0
+_SECONDS_PER_HOUR: Final = 3600.0
+
+
+class _CachedContentUsage(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    totalTokenCount: int = 0
+
+
+class _CreatedCachedContent(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    usageMetadata: _CachedContentUsage = _CachedContentUsage()
+    createTime: datetime | None = None
+    expireTime: datetime | None = None
+
+
+def _reported_lifetime_seconds(created: _CreatedCachedContent) -> float | None:
+    if created.createTime is None or created.expireTime is None:
+        return None
+    seconds: Final = (created.expireTime - created.createTime).total_seconds()
+    return seconds if seconds > 0 else None
+
+
+def _requested_ttl_seconds(requested_ttl: str | None) -> float | None:
+    if requested_ttl is None or not requested_ttl.endswith("s"):
+        return None
+    try:
+        seconds: Final = float(requested_ttl[:-1])
+    except ValueError:
+        return None
+    return seconds if seconds > 0 else None
+
+
+def context_cache_storage_token_hours(created_cached_content: object, requested_ttl: str | None) -> float:
+    """Token-hours a newly created cachedContents resource is billed for over its whole TTL.
+
+    The lifetime comes from the create response (expireTime - createTime), then the TTL that was
+    requested, then the provider default of one hour.
+    """
+    try:
+        created: Final = _CreatedCachedContent.model_validate(created_cached_content)
+    except ValidationError:
+        return 0.0
+    lifetime_seconds: Final = (
+        _reported_lifetime_seconds(created)
+        or _requested_ttl_seconds(requested_ttl)
+        or DEFAULT_CONTEXT_CACHE_TTL_SECONDS
+    )
+    return created.usageMetadata.totalTokenCount * lifetime_seconds / _SECONDS_PER_HOUR

litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py

@@ -21,6 +21,7 @@
 
 from ..common_utils import VertexAIError, get_vertex_base_url
 from ..vertex_llm_base import VertexBase
+from .storage_cost import context_cache_storage_token_hours
 from .transformation import (
     cached_messages_end_on_supported_turn,
     separate_cached_messages,
@@ -425,6 +426,9 @@ def check_and_create_cache(
             raise VertexAIError(status_code=408, message="Timeout error occurred.")
 
         raw_response_cached: Final = response.json()
+        logging_obj.record_context_cache_storage(
+            context_cache_storage_token_hours(raw_response_cached, cached_content_request_body.get("ttl"))
+        )
         cached_content_response_obj: Final = VertexAICachedContentResponseObject(
             name=raw_response_cached.get("name"), model=raw_response_cached.get("model")
         )
@@ -581,6 +585,9 @@ async def async_check_and_create_cache(
             raise VertexAIError(status_code=408, message="Timeout error occurred.")
 
         raw_response_cached: Final = response.json()
+        logging_obj.record_context_cache_storage(
+            context_cache_storage_token_hours(raw_response_cached, cached_content_request_body.get("ttl"))
+        )
         cached_content_response_obj: Final = VertexAICachedContentResponseObject(
             name=raw_response_cached.get("name"), model=raw_response_cached.get("model")
         )