diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 8fc428b38ae4..7fe81879d99d 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -707,6 +707,24 @@ def calculate_cost_component(model_info: ModelInfo, cost_key: str, usage_value: return 0.0 +def _calculate_modality_input_cost( + model_info: ModelInfo, + tokens: int, + token_cost_key: str, + units: float, + unit_cost_key: str, + token_fallback_key: str | None = None, + cached_tokens: int = 0, +) -> float: + token_rate: Final = _get_cost_per_unit(model_info, token_cost_key, None) + if token_rate is not None and tokens + cached_tokens > 0: + return tokens * token_rate + unit_rate: Final = _get_cost_per_unit(model_info, unit_cost_key, None) + if unit_rate is not None and units > 0: + return units * unit_rate + return calculate_cost_component(model_info, token_fallback_key or token_cost_key, tokens) + + def _get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: float | None = 0.0) -> float | None: # Sometimes the cost per unit is a string (e.g.: If a value like "3e-7" was read from the config.yaml) cost_per_unit: Final = model_info.get(cost_key) @@ -955,26 +973,30 @@ def _calculate_input_cost( audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost ) - ### AUDIO COST - if prompt_tokens_details["audio_tokens"]: - audio_cost_key: Final = _get_service_tier_cost_key("input_cost_per_audio_token", service_tier) - prompt_cost += calculate_cost_component(model_info, audio_cost_key, prompt_tokens_details["audio_tokens"]) - - ### IMAGE TOKEN COST - if prompt_tokens_details["image_tokens"]: - # For image token costs: - # First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token. - image_token_cost_key = "input_cost_per_image_token" - if model_info.get(image_token_cost_key) is None: - image_token_cost_key = "input_cost_per_token" - prompt_cost += calculate_cost_component(model_info, image_token_cost_key, prompt_tokens_details["image_tokens"]) - - ### VIDEO TOKEN COST - if prompt_tokens_details["video_tokens"]: - video_token_cost_key = "input_cost_per_video_token" - if model_info.get(video_token_cost_key) is None: - video_token_cost_key = "input_cost_per_token" - prompt_cost += calculate_cost_component(model_info, video_token_cost_key, prompt_tokens_details["video_tokens"]) + prompt_cost += _calculate_modality_input_cost( + model_info, + prompt_tokens_details["audio_tokens"], + _get_service_tier_cost_key("input_cost_per_audio_token", service_tier), + prompt_tokens_details["audio_length_seconds"], + "input_cost_per_audio_per_second", + cached_tokens=cache_hit_audio_tokens, + ) + prompt_cost += _calculate_modality_input_cost( + model_info, + prompt_tokens_details["image_tokens"], + "input_cost_per_image_token", + prompt_tokens_details["image_count"], + "input_cost_per_image", + token_fallback_key="input_cost_per_token", + ) + prompt_cost += _calculate_modality_input_cost( + model_info, + prompt_tokens_details["video_tokens"], + "input_cost_per_video_token", + prompt_tokens_details["video_length_seconds"], + "input_cost_per_video_per_second", + token_fallback_key="input_cost_per_token", + ) ### CACHE WRITING COST - Now uses tiered pricing if ( @@ -996,28 +1018,6 @@ def _calculate_input_cost( prompt_tokens_details["character_count"], ) - ### IMAGE COUNT COST - if prompt_tokens_details["image_count"]: - prompt_cost += calculate_cost_component( - model_info, "input_cost_per_image", prompt_tokens_details["image_count"] - ) - - ### VIDEO LENGTH COST - if prompt_tokens_details["video_length_seconds"]: - prompt_cost += calculate_cost_component( - model_info, - "input_cost_per_video_per_second", - prompt_tokens_details["video_length_seconds"], - ) - - ### AUDIO LENGTH COST - if prompt_tokens_details["audio_length_seconds"]: - prompt_cost += calculate_cost_component( - model_info, - "input_cost_per_audio_per_second", - prompt_tokens_details["audio_length_seconds"], - ) - if prompt_tokens_details["query_count"]: prompt_cost += calculate_cost_component( model_info, "input_cost_per_query", prompt_tokens_details["query_count"] diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py index e7fd9a0d08b4..3c4c49250916 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py @@ -374,25 +374,21 @@ def _usage_from_embed_content_response( details: Final[Sequence[PromptTokensDetails]] = usage_metadata.get("promptTokensDetails") or () text_tokens: Final = _tokens_for_modality(details, "TEXT") audio_tokens: Final = _tokens_for_modality(details, "AUDIO") + image_tokens: Final = _tokens_for_modality(details, "IMAGE") video_tokens: Final = _tokens_for_modality(details, "VIDEO") image_count: Final = _count_input_images(input, resolved_files) video_length_seconds: Final = video_tokens / _VIDEO_TOKENS_PER_SECOND if video_tokens > 0 else 0.0 audio_length_seconds: Final = audio_tokens / _AUDIO_TOKENS_PER_SECOND if audio_tokens > 0 else 0.0 - # generic_cost_per_token rewrites text_tokens to the full prompt minus - # other modalities when both text_tokens and image_count are zero. For - # video, that misallocates video tokens to text; a 1-token floor sidesteps - # the rewrite and keeps billing on input_cost_per_video_per_second. - needs_video_text_floor: Final = video_length_seconds > 0 and text_tokens == 0 and image_count == 0 - resolved_text_tokens: Final = 1 if needs_video_text_floor else text_tokens - return Usage( prompt_tokens=prompt_tokens, total_tokens=total_tokens, prompt_tokens_details=PromptTokensDetailsWrapper( - text_tokens=resolved_text_tokens, + text_tokens=text_tokens, audio_tokens=audio_tokens, + image_tokens=image_tokens, + video_tokens=video_tokens, image_count=image_count, video_length_seconds=video_length_seconds, audio_length_seconds=audio_length_seconds, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0c60ff26635f..611b575a02a1 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -25602,9 +25602,12 @@ }, "gemini-embedding-2-preview": { "input_cost_per_audio_per_second": 0.00016, + "input_cost_per_audio_token": 6.5e-06, "input_cost_per_image": 0.00012, + "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, "input_cost_per_video_per_second": 0.00079, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25622,6 +25625,7 @@ "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_video_per_second": 0.00079, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25634,9 +25638,12 @@ }, "vertex_ai/gemini-embedding-2-preview": { "input_cost_per_audio_per_second": 0.00016, + "input_cost_per_audio_token": 6.5e-06, "input_cost_per_image": 0.00012, + "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, "input_cost_per_video_per_second": 0.00079, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25655,6 +25662,7 @@ "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_video_per_second": 0.00079, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25694,9 +25702,12 @@ "gemini/gemini-embedding-2-preview": { "deprecation_date": "2026-08-10", "input_cost_per_audio_per_second": 0.00016, + "input_cost_per_audio_token": 6.5e-06, "input_cost_per_image": 0.00012, + "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, "input_cost_per_video_per_second": 0.00079, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "gemini", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25710,9 +25721,12 @@ }, "gemini/gemini-embedding-2": { "input_cost_per_audio_per_second": 0.00016, + "input_cost_per_audio_token": 6.5e-06, "input_cost_per_image": 0.00012, + "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, "input_cost_per_video_per_second": 0.00079, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "gemini", "max_input_tokens": 8192, "max_tokens": 8192, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0c60ff26635f..611b575a02a1 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -25602,9 +25602,12 @@ }, "gemini-embedding-2-preview": { "input_cost_per_audio_per_second": 0.00016, + "input_cost_per_audio_token": 6.5e-06, "input_cost_per_image": 0.00012, + "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, "input_cost_per_video_per_second": 0.00079, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25622,6 +25625,7 @@ "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_video_per_second": 0.00079, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25634,9 +25638,12 @@ }, "vertex_ai/gemini-embedding-2-preview": { "input_cost_per_audio_per_second": 0.00016, + "input_cost_per_audio_token": 6.5e-06, "input_cost_per_image": 0.00012, + "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, "input_cost_per_video_per_second": 0.00079, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25655,6 +25662,7 @@ "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_video_per_second": 0.00079, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25694,9 +25702,12 @@ "gemini/gemini-embedding-2-preview": { "deprecation_date": "2026-08-10", "input_cost_per_audio_per_second": 0.00016, + "input_cost_per_audio_token": 6.5e-06, "input_cost_per_image": 0.00012, + "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, "input_cost_per_video_per_second": 0.00079, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "gemini", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25710,9 +25721,12 @@ }, "gemini/gemini-embedding-2": { "input_cost_per_audio_per_second": 0.00016, + "input_cost_per_audio_token": 6.5e-06, "input_cost_per_image": 0.00012, + "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, "input_cost_per_video_per_second": 0.00079, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "gemini", "max_input_tokens": 8192, "max_tokens": 8192, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index d1ac3e67b2bc..eff3f192b3ca 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -375,6 +375,10 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "input_cost_per_video_token": { + "type": "number", + "minimum": 0 + }, "input_dbu_cost_per_token": { "type": "number", "minimum": 0 diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 2289de9a9513..f4bb990628a3 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1,5 +1,6 @@ import json from datetime import datetime, timezone +from typing import Final import pytest from fastapi.testclient import TestClient @@ -5334,3 +5335,44 @@ def test_realtime_models_bill_cached_text_and_audio_at_their_cache_read_rates( prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider=custom_llm_provider) assert prompt_cost == pytest.approx(expected_prompt_cost) + + +@pytest.mark.parametrize("tokens,cached_audio,token_rate,unit_rate,expected", [ + (10, 0, 0.1, 7.0, 3.0), + (10, 0, 0.0, 7.0, 0.0), + (10, 0, None, 7.0, 42.0), + (0, 0, 0.1, 7.0, 42.0), + (0, 10, 0.1, 7.0, 28.0), + (10, 0, None, None, 0.2), + (10, 0, None, 0.0, 0.0), +]) +def test_modality_tokens_and_physical_units_are_alternative_prices( + tokens: int, cached_audio: int, token_rate: float | None, + unit_rate: float | None, expected: float, +) -> None: + info: Final[ModelInfo] = { + "input_cost_per_token": 0.01, + "input_cost_per_audio_token": token_rate, + "input_cost_per_image_token": token_rate, + "input_cost_per_video_token": token_rate, + "input_cost_per_audio_per_second": unit_rate, + "input_cost_per_image": unit_rate, + "input_cost_per_video_per_second": unit_rate, + "cache_read_input_token_cost": 0.0, + } + usage: Final = Usage( + prompt_tokens=3 * tokens + cached_audio, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=0, audio_tokens=tokens + cached_audio, + image_tokens=tokens, video_tokens=tokens, + cached_tokens=cached_audio, + cached_tokens_details={"audio_tokens": cached_audio}, + image_count=2, audio_length_seconds=2.0, + video_length_seconds=2.0, + ), + ) + prompt_cost, completion_cost = generic_cost_per_token( + "modality-unit-test", usage, "vertex_ai", model_info=info, + ) + assert prompt_cost == pytest.approx(expected) + assert completion_cost == 0 diff --git a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py index 86b3f0976ab5..e1d514660319 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py @@ -299,15 +299,25 @@ def test_nested_non_string_element_raises(self): ) +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("model,provider", [ + ("gemini-embedding-2", "vertex_ai"), + ("gemini-embedding-2-preview", "vertex_ai"), + ("vertex_ai/gemini-embedding-2", "vertex_ai"), + ("vertex_ai/gemini-embedding-2-preview", "vertex_ai"), + ("gemini-embedding-2", "gemini"), + ("gemini-embedding-2-preview", "gemini"), +]) class TestProcessEmbedContentResponseUsage: """Gemini Embedding 2 embedContent usageMetadata must drive spend. Regression for multimodal calls recording prompt_tokens=0 / spend=$0. """ - MODEL = "gemini-embedding-2" - - def test_multimodal_image_preserves_usage_metadata(self): + @pytest.mark.parametrize("inputs,image_count", [(IMAGE_DATA_URI, 1), ([], 0)]) + def test_multimodal_image_preserves_usage_metadata( + self, model: str, provider: str, inputs: str | list[str], image_count: int, + ) -> None: response_json = { "embedding": {"values": [0.1, 0.2, 0.3]}, "usageMetadata": { @@ -317,23 +327,24 @@ def test_multimodal_image_preserves_usage_metadata(self): }, } result = process_embed_content_response( - input=[IMAGE_DATA_URI], + input=inputs, model_response=EmbeddingResponse(), - model=self.MODEL, + model=model, response_json=response_json, ) assert result.usage.prompt_tokens == 258 assert result.usage.total_tokens == 258 - assert result.usage.prompt_tokens_details.image_count == 1 + assert result.usage.prompt_tokens_details.image_count == image_count + assert result.usage.prompt_tokens_details.image_tokens == 258 prompt_cost, _ = generic_cost_per_token( - model=self.MODEL, + model=model, usage=result.usage, - custom_llm_provider="vertex_ai", + custom_llm_provider=provider, ) - assert prompt_cost > 0 + assert prompt_cost == pytest.approx(258 * 4.5e-7) - def test_text_modality_detail_populated(self): + def test_text_modality_detail_populated(self, model: str, provider: str) -> None: response_json = { "embedding": {"values": [0.1, 0.2]}, "usageMetadata": { @@ -345,20 +356,20 @@ def test_text_modality_detail_populated(self): result = process_embed_content_response( input="a short caption", model_response=EmbeddingResponse(), - model=self.MODEL, + model=model, response_json=response_json, ) assert result.usage.prompt_tokens == 12 assert result.usage.prompt_tokens_details.text_tokens == 12 prompt_cost, _ = generic_cost_per_token( - model=self.MODEL, + model=model, usage=result.usage, - custom_llm_provider="vertex_ai", + custom_llm_provider=provider, ) - assert prompt_cost > 0 + assert prompt_cost == pytest.approx(12 * 2e-7) - def test_video_modality_derives_seconds_and_text_floor(self): + def test_video_modality_preserves_tokens_without_text(self, model: str, provider: str) -> None: response_json = { "embedding": {"values": [0.1]}, "usageMetadata": { @@ -370,38 +381,38 @@ def test_video_modality_derives_seconds_and_text_floor(self): result = process_embed_content_response( input=["gs://bucket/clip.mp4"], model_response=EmbeddingResponse(), - model=self.MODEL, + model=model, response_json=response_json, ) assert result.usage.prompt_tokens == 516 assert result.usage.prompt_tokens_details.video_length_seconds == pytest.approx( 2.0 ) - assert result.usage.prompt_tokens_details.text_tokens == 1 + assert result.usage.prompt_tokens_details.video_tokens == 516 + assert result.usage.prompt_tokens_details.text_tokens == 0 - def test_missing_usage_metadata_does_not_estimate_from_base64(self): + def test_missing_usage_metadata_does_not_estimate_from_base64(self, model: str, provider: str) -> None: response_json = {"embedding": {"values": [0.1, 0.2]}} result = process_embed_content_response( input=[IMAGE_DATA_URI], model_response=EmbeddingResponse(), - model=self.MODEL, + model=model, response_json=response_json, ) assert result.usage.prompt_tokens == 0 assert result.usage.total_tokens == 0 - def test_missing_usage_metadata_text_falls_back_to_token_counter(self): + def test_missing_usage_metadata_text_falls_back_to_token_counter(self, model: str, provider: str) -> None: response_json = {"embedding": {"values": [0.1, 0.2]}} result = process_embed_content_response( input="hello world this is plain text", model_response=EmbeddingResponse(), - model=self.MODEL, + model=model, response_json=response_json, ) assert result.usage.prompt_tokens > 0 - def test_file_reference_image_billed_per_image_not_text(self): - """files/... image refs must bill per-image, not at the text token rate.""" + def test_file_reference_image_billed_as_image_tokens(self, model: str, provider: str) -> None: response_json = { "embedding": {"values": [0.1, 0.2, 0.3]}, "usageMetadata": { @@ -413,7 +424,7 @@ def test_file_reference_image_billed_per_image_not_text(self): result = process_embed_content_response( input=["files/img123"], model_response=EmbeddingResponse(), - model=self.MODEL, + model=model, response_json=response_json, resolved_files={ "files/img123": { @@ -426,13 +437,14 @@ def test_file_reference_image_billed_per_image_not_text(self): assert result.usage.prompt_tokens_details.text_tokens == 0 prompt_cost, _ = generic_cost_per_token( - model=self.MODEL, + model=model, usage=result.usage, - custom_llm_provider="vertex_ai", + custom_llm_provider=provider, ) - assert prompt_cost == pytest.approx(0.00012) + assert result.usage.prompt_tokens_details.image_tokens == 258 + assert prompt_cost == pytest.approx(258 * 4.5e-7) - def test_file_reference_non_image_not_counted_as_image(self): + def test_file_reference_non_image_not_counted_as_image(self, model: str, provider: str) -> None: """A files/... ref resolving to a non-image mime must not be image-counted.""" response_json = { "embedding": {"values": [0.1, 0.2]}, @@ -445,7 +457,7 @@ def test_file_reference_non_image_not_counted_as_image(self): result = process_embed_content_response( input=["files/clip1"], model_response=EmbeddingResponse(), - model=self.MODEL, + model=model, response_json=response_json, resolved_files={ "files/clip1": { @@ -461,13 +473,13 @@ def test_file_reference_non_image_not_counted_as_image(self): ) prompt_cost, _ = generic_cost_per_token( - model=self.MODEL, + model=model, usage=result.usage, - custom_llm_provider="vertex_ai", + custom_llm_provider=provider, ) - assert prompt_cost == pytest.approx(2.0 * 0.00016) + assert prompt_cost == pytest.approx(64 * 6.5e-6) - def test_video_plus_audio_does_not_double_bill_text(self): + def test_video_plus_audio_does_not_double_bill_text(self, model: str, provider: str) -> None: """Video+audio responses must not get video tokens reassigned to text.""" response_json = { "embedding": {"values": [0.1]}, @@ -483,10 +495,11 @@ def test_video_plus_audio_does_not_double_bill_text(self): result = process_embed_content_response( input=["gs://bucket/clip.mp4"], model_response=EmbeddingResponse(), - model=self.MODEL, + model=model, response_json=response_json, ) - assert result.usage.prompt_tokens_details.text_tokens == 1 + assert result.usage.prompt_tokens_details.video_tokens == 516 + assert result.usage.prompt_tokens_details.text_tokens == 0 assert result.usage.prompt_tokens_details.video_length_seconds == pytest.approx( 2.0 ) @@ -495,9 +508,8 @@ def test_video_plus_audio_does_not_double_bill_text(self): ) prompt_cost, _ = generic_cost_per_token( - model=self.MODEL, + model=model, usage=result.usage, - custom_llm_provider="vertex_ai", + custom_llm_provider=provider, ) - # 1 floor text token at 2e-7 + 2s of video at 7.9e-4 + 2s of audio at 1.6e-4 - assert prompt_cost == pytest.approx(1 * 2e-7 + 2 * 0.00079 + 2 * 0.00016) + assert prompt_cost == pytest.approx(516 * 12e-6 + 64 * 6.5e-6)