diff --git a/litellm/llms/perplexity/cost_calculator.py b/litellm/llms/perplexity/cost_calculator.py index 0f9c3cad841..bf055f91aa0 100644 --- a/litellm/llms/perplexity/cost_calculator.py +++ b/litellm/llms/perplexity/cost_calculator.py @@ -58,11 +58,8 @@ def _safe_float_cast( ## CALCULATE OUTPUT COST output_cost_per_token = _safe_float_cast(model_info.get("output_cost_per_token")) - completion_cost: float = (usage.completion_tokens or 0) * output_cost_per_token - ## ADD REASONING TOKENS COST (if present) reasoning_tokens = getattr(usage, "reasoning_tokens", 0) or 0 - # Also check completion_tokens_details if reasoning_tokens is not directly available if ( reasoning_tokens == 0 and hasattr(usage, "completion_tokens_details") @@ -73,9 +70,19 @@ def _safe_float_cast( ) reasoning_cost_value = model_info.get("output_cost_per_reasoning_token") + + # `completion_tokens` includes `reasoning_tokens` per the OpenAI/Perplexity usage + # convention (codified for the central path in PR #18607). When a reasoning rate is + # configured we subtract before the output-rate multiplication so the reasoning + # tokens are not billed twice. if reasoning_tokens > 0 and reasoning_cost_value is not None: - reasoning_cost_per_token = _safe_float_cast(reasoning_cost_value) - completion_cost += reasoning_tokens * reasoning_cost_per_token + non_reasoning_completion_tokens = max( + 0, (usage.completion_tokens or 0) - reasoning_tokens + ) + completion_cost: float = non_reasoning_completion_tokens * output_cost_per_token + completion_cost += reasoning_tokens * _safe_float_cast(reasoning_cost_value) + else: + completion_cost = (usage.completion_tokens or 0) * output_cost_per_token ## ADD SEARCH QUERIES COST (if present) num_search_queries = 0 diff --git a/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py b/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py index d408f55c004..e2d1ab72c5e 100644 --- a/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py +++ b/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py @@ -1,7 +1,7 @@ """ Test file for Perplexity cost calculator functionality. -Tests the cost calculation for Perplexity models including citation tokens, +Tests the cost calculation for Perplexity models including citation tokens, search queries, and reasoning tokens. """ @@ -21,7 +21,11 @@ from litellm.llms.perplexity.cost_calculator import ( cost_per_token as perplexity_cost_per_token, ) -from litellm.types.utils import Usage, PromptTokensDetailsWrapper +from litellm.types.utils import ( + CompletionTokensDetailsWrapper, + Usage, + PromptTokensDetailsWrapper, +) from litellm.utils import get_model_info @@ -135,13 +139,14 @@ def test_reasoning_tokens_from_direct_attribute(self): model="sonar-deep-research", usage=usage ) - # Expected costs: - # Input: 100 tokens * $2e-6 = $0.0002 - # Output: 50 tokens * $8e-6 = $0.0004 - # Reasoning: 20 tokens * $3e-6 = $0.00006 - # Total completion cost: $0.00046 + # `completion_tokens` includes `reasoning_tokens` per the OpenAI/Perplexity + # convention codified in PR #18607. Non-reasoning portion = 50 - 20 = 30. + # Input: 100 tokens * $2e-6 = $0.0002 + # Output (text): 30 tokens * $8e-6 = $0.00024 + # Reasoning: 20 tokens * $3e-6 = $0.00006 + # Total completion cost = $0.0003 expected_prompt_cost = 100 * 2e-6 - expected_completion_cost = (50 * 8e-6) + (20 * 3e-6) + expected_completion_cost = ((50 - 20) * 8e-6) + (20 * 3e-6) assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6) assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6) @@ -159,13 +164,10 @@ def test_reasoning_tokens_from_completion_tokens_details(self): model="sonar-deep-research", usage=usage ) - # Expected costs: - # Input: 100 tokens * $2e-6 = $0.0002 - # Output: 50 tokens * $8e-6 = $0.0004 - # Reasoning: 20 tokens * $3e-6 = $0.00006 - # Total completion cost: $0.00046 + # Same convention as the direct-attribute case above; reasoning is a subset of + # completion_tokens, so non-reasoning portion = 50 - 20 = 30. expected_prompt_cost = 100 * 2e-6 - expected_completion_cost = (50 * 8e-6) + (20 * 3e-6) + expected_completion_cost = ((50 - 20) * 8e-6) + (20 * 3e-6) assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6) assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6) @@ -187,16 +189,16 @@ def test_comprehensive_cost_calculation(self): model="sonar-deep-research", usage=usage ) - # Expected costs: - # Input: 100 tokens * $2e-6 = $0.0002 - # Citation: 30 tokens * $2e-6 = $0.00006 - # Total prompt cost: $0.00026 - # Output: 50 tokens * $8e-6 = $0.0004 - # Reasoning: 15 tokens * $3e-6 = $0.000045 - # Search: 2 queries * ($0.005 / 1000) = $0.00001 - # Total completion cost: $0.000455 + # Expected costs (reasoning is a subset of completion_tokens): + # Input: 100 tokens * $2e-6 = $0.0002 + # Citation: 30 tokens * $2e-6 = $0.00006 + # Total prompt cost = $0.00026 + # Output (text): (50 - 15) tokens * $8e-6 = $0.00028 + # Reasoning: 15 tokens * $3e-6 = $0.000045 + # Search: 2 queries * ($0.005 / 1000) = $0.00001 + # Total completion cost = $0.000335 expected_prompt_cost = (100 * 2e-6) + (30 * 2e-6) - expected_completion_cost = (50 * 8e-6) + (15 * 3e-6) + (2 / 1000 * 0.005) + expected_completion_cost = ((50 - 15) * 8e-6) + (15 * 3e-6) + (2 / 1000 * 0.005) assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6) assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6) @@ -306,11 +308,11 @@ def test_integration_with_completion_cost_function(self): completion_response=response, custom_llm_provider="perplexity" ) - # Calculate expected total cost + # Calculate expected total cost (reasoning is a subset of completion_tokens) expected_prompt_cost = (100 * 2e-6) + (15 * 2e-6) # Input + citation expected_completion_cost = ( - (50 * 8e-6) + (10 * 3e-6) + (1 / 1000 * 0.005) - ) # Output + reasoning + search + ((50 - 10) * 8e-6) + (10 * 3e-6) + (1 / 1000 * 0.005) + ) # Output (text) + reasoning + search expected_total = expected_prompt_cost + expected_completion_cost assert math.isclose(total_cost, expected_total, rel_tol=1e-6) @@ -353,10 +355,13 @@ def test_cost_calculation_combinations( model="sonar-deep-research", usage=usage ) - # Calculate expected costs + # Calculate expected costs. `completion_tokens` includes `reasoning_tokens`, + # so non-reasoning portion = 50 - reasoning_tokens. expected_prompt_cost = (100 * 2e-6) + (citation_tokens * 2e-6) expected_completion_cost = ( - (50 * 8e-6) + (reasoning_tokens * 3e-6) + (search_queries / 1000 * 0.005) + ((50 - reasoning_tokens) * 8e-6) + + (reasoning_tokens * 3e-6) + + (search_queries / 1000 * 0.005) ) assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6) @@ -413,3 +418,36 @@ def test_falls_back_to_manual_calculation_when_no_cost_provided(self): assert math.isclose(prompt_cost, expected_prompt, rel_tol=1e-6) assert math.isclose(completion_cost, expected_completion, rel_tol=1e-6) + + def test_reasoning_tokens_not_double_billed(self): + """ + Regression: `completion_tokens` includes `reasoning_tokens` per the + OpenAI/Perplexity usage convention (codified for the central path in PR #18607). + When `output_cost_per_reasoning_token` is configured the manual fallback must + subtract reasoning from completion before applying the output rate so the + reasoning tokens are not billed at BOTH the output rate and the reasoning rate. + + Uses the exact usage shape produced by the live response fixture in + `tests/llm_translation/test_perplexity_reasoning.py`. + """ + usage = Usage( + prompt_tokens=9, + completion_tokens=20, + total_tokens=29, + completion_tokens_details=CompletionTokensDetailsWrapper( + reasoning_tokens=15 + ), + ) + + prompt_cost, completion_cost = perplexity_cost_per_token( + model="sonar-deep-research", usage=usage + ) + + # sonar-deep-research rates: input 2e-6, output 8e-6, reasoning 3e-6. + # Non-reasoning portion of the 20 completion tokens = 20 - 15 = 5. + # Pre-fix this asserted 20 * 8e-6 + 15 * 3e-6 = 2.05e-4 (a 2.16x overcharge). + expected_prompt = 9 * 2e-6 + expected_completion = (20 - 15) * 8e-6 + 15 * 3e-6 + + assert math.isclose(prompt_cost, expected_prompt, rel_tol=1e-9) + assert math.isclose(completion_cost, expected_completion, rel_tol=1e-9) diff --git a/tests/test_litellm/llms/perplexity/test_perplexity_integration.py b/tests/test_litellm/llms/perplexity/test_perplexity_integration.py index 1b03fd7df88..e59fbc9f272 100644 --- a/tests/test_litellm/llms/perplexity/test_perplexity_integration.py +++ b/tests/test_litellm/llms/perplexity/test_perplexity_integration.py @@ -104,12 +104,10 @@ def test_end_to_end_cost_calculation_with_transformation(self): ) citation_tokens = citation_chars // 4 - expected_prompt_cost = (100 * 2e-6) + ( - citation_tokens * 2e-6 - ) # Input + citation + expected_prompt_cost = (100 * 2e-6) + (citation_tokens * 2e-6) expected_completion_cost = ( - (50 * 8e-6) + (10 * 3e-6) + (2 / 1000 * 0.005) - ) # Output + reasoning + search + ((50 - 10) * 8e-6) + (10 * 3e-6) + (2 / 1000 * 0.005) + ) expected_total = expected_prompt_cost + expected_completion_cost assert math.isclose(total_cost, expected_total, rel_tol=1e-6) @@ -152,11 +150,10 @@ def test_main_cost_calculator_integration(self): usage_object=usage, ) - # Calculate expected costs - expected_prompt_cost = (200 * 2e-6) + (40 * 2e-6) # Input + citation + expected_prompt_cost = (200 * 2e-6) + (40 * 2e-6) expected_completion_cost = ( - (100 * 8e-6) + (25 * 3e-6) + (3 / 1000 * 0.005) - ) # Output + reasoning + search + ((100 - 25) * 8e-6) + (25 * 3e-6) + (3 / 1000 * 0.005) + ) assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6) assert math.isclose(completion_cost_val, expected_completion_cost, rel_tol=1e-6) @@ -263,15 +260,14 @@ def test_high_volume_cost_calculation(self): custom_llm_provider="perplexity", ) - # Calculate expected cost - expected_prompt_cost = (50000 * 2e-6) + (5000 * 2e-6) # $0.11 + expected_prompt_cost = (50000 * 2e-6) + (5000 * 2e-6) expected_completion_cost = ( - (25000 * 8e-6) + (10000 * 3e-6) + (100 / 1000 * 0.005) - ) # $0.23 - expected_total = expected_prompt_cost + expected_completion_cost # $0.34 + ((25000 - 10000) * 8e-6) + (10000 * 3e-6) + (100 / 1000 * 0.005) + ) + expected_total = expected_prompt_cost + expected_completion_cost assert math.isclose(total_cost, expected_total, rel_tol=1e-6) - assert total_cost > 0.3 # Sanity check for high-volume scenario + assert total_cost > 0.25 def test_transformation_preserves_existing_usage_fields(self): """Test that transformation doesn't overwrite existing standard usage fields."""