Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 60 additions & 28 deletions litellm/proxy/common_request_processing.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from datetime import datetime
from functools import lru_cache
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, overload
from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, Protocol, overload

import anyio
import httpx
Expand Down Expand Up @@ -928,28 +928,42 @@ def _override_openai_response_model(
)


class CostBreakdownHeaderValues(NamedTuple):
original_cost: float | None = None
discount_amount: float | None = None
margin_total_amount: float | None = None
margin_percent: float | None = None
input_cost: float | None = None
output_cost: float | None = None
cache_read_cost: float | None = None
cache_creation_cost: float | None = None
reasoning_cost: float | None = None
tool_usage_cost: float | None = None


def _get_cost_breakdown_from_logging_obj(
litellm_logging_obj: LiteLLMLoggingObj | None,
) -> tuple[float | None, float | None, float | None, float | None]:
"""
Extract discount and margin information from logging object's cost breakdown.

Returns:
Tuple of (original_cost, discount_amount, margin_total_amount, margin_percent)
"""
) -> CostBreakdownHeaderValues:
"""Extract discount, margin, and per-component cost information from logging object's cost breakdown."""
if not litellm_logging_obj or not hasattr(litellm_logging_obj, "cost_breakdown"):
return None, None, None, None
return CostBreakdownHeaderValues()

cost_breakdown: Final = litellm_logging_obj.cost_breakdown
if not cost_breakdown:
return None, None, None, None

original_cost: Final = cost_breakdown.get("original_cost")
discount_amount: Final = cost_breakdown.get("discount_amount")
margin_total_amount: Final = cost_breakdown.get("margin_total_amount")
margin_percent: Final = cost_breakdown.get("margin_percent")

return original_cost, discount_amount, margin_total_amount, margin_percent
return CostBreakdownHeaderValues()

return CostBreakdownHeaderValues(
original_cost=cost_breakdown.get("original_cost"),
discount_amount=cost_breakdown.get("discount_amount"),
margin_total_amount=cost_breakdown.get("margin_total_amount"),
margin_percent=cost_breakdown.get("margin_percent"),
input_cost=cost_breakdown.get("input_cost"),
output_cost=cost_breakdown.get("output_cost"),
cache_read_cost=cost_breakdown.get("cache_read_cost"),
cache_creation_cost=cost_breakdown.get("cache_creation_cost"),
reasoning_cost=cost_breakdown.get("reasoning_cost"),
tool_usage_cost=cost_breakdown.get("tool_usage_cost"),
)


def _classifier_cost_from_request_data(request_data: Mapping[str, object] | None) -> float | None:
Expand Down Expand Up @@ -1075,13 +1089,7 @@ def get_custom_headers(
exclude_values: Final = {"", None, "None"}
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
hidden_params = hidden_params or {}

# Extract discount and margin info from cost_breakdown if available
(
original_cost,
discount_amount,
margin_total_amount,
margin_percent,
) = _get_cost_breakdown_from_logging_obj(litellm_logging_obj=litellm_logging_obj)
cost_breakdown: Final = _get_cost_breakdown_from_logging_obj(litellm_logging_obj=litellm_logging_obj)

# Calculate updated spend for header (include current response_cost)
current_spend: Final = user_api_key_dict.spend or 0.0
Expand Down Expand Up @@ -1110,12 +1118,36 @@ def get_custom_headers(
"x-litellm-version": version,
"x-litellm-model-region": model_region,
"x-litellm-response-cost": str(response_cost),
"x-litellm-response-cost-original": (str(original_cost) if original_cost is not None else None),
"x-litellm-response-cost-discount-amount": (str(discount_amount) if discount_amount is not None else None),
"x-litellm-response-cost-original": (
str(cost_breakdown.original_cost) if cost_breakdown.original_cost is not None else None
),
"x-litellm-response-cost-discount-amount": (
str(cost_breakdown.discount_amount) if cost_breakdown.discount_amount is not None else None
),
"x-litellm-response-cost-margin-amount": (
str(margin_total_amount) if margin_total_amount is not None else None
str(cost_breakdown.margin_total_amount) if cost_breakdown.margin_total_amount is not None else None
),
"x-litellm-response-cost-margin-percent": (
str(cost_breakdown.margin_percent) if cost_breakdown.margin_percent is not None else None
),
"x-litellm-response-cost-input": (
str(cost_breakdown.input_cost) if cost_breakdown.input_cost is not None else None
),
"x-litellm-response-cost-output": (
str(cost_breakdown.output_cost) if cost_breakdown.output_cost is not None else None
),
"x-litellm-response-cost-cache-read": (
str(cost_breakdown.cache_read_cost) if cost_breakdown.cache_read_cost is not None else None
),
"x-litellm-response-cost-cache-creation": (
str(cost_breakdown.cache_creation_cost) if cost_breakdown.cache_creation_cost is not None else None
),
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved
"x-litellm-response-cost-reasoning": (
str(cost_breakdown.reasoning_cost) if cost_breakdown.reasoning_cost is not None else None
),
"x-litellm-response-cost-tool-usage": (
str(cost_breakdown.tool_usage_cost) if cost_breakdown.tool_usage_cost is not None else None
),
"x-litellm-response-cost-margin-percent": (str(margin_percent) if margin_percent is not None else None),
"x-litellm-classifier-cost": (str(classifier_cost) if classifier_cost is not None else None),
"x-litellm-key-tpm-limit": str(user_api_key_dict.tpm_limit),
"x-litellm-key-rpm-limit": str(user_api_key_dict.rpm_limit),
Expand Down
230 changes: 190 additions & 40 deletions tests/test_litellm/proxy/test_common_request_processing.py
Original file line number Diff line number Diff line change
Expand Up @@ -834,6 +834,174 @@ def test_get_custom_headers_without_margin_info(self):
assert "x-litellm-response-cost-margin-amount" not in headers
assert "x-litellm-response-cost-margin-percent" not in headers

def test_get_custom_headers_per_component_cost_breakdown(self):
"""Test that per-component cost headers are included when component breakdown is available."""
from litellm.litellm_core_utils.litellm_logging import (
Logging as LiteLLMLoggingObj,
)

mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth)
mock_user_api_key_dict.tpm_limit = None
mock_user_api_key_dict.rpm_limit = None
mock_user_api_key_dict.max_budget = None
mock_user_api_key_dict.spend = 0

logging_obj = LiteLLMLoggingObj(
model="gpt-5.4-nano",
messages=[{"role": "user", "content": "hello"}],
stream=False,
call_type="completion",
start_time=None,
litellm_call_id="test-call-id-components",
function_id="test-function",
)

input_cost: Final = 0.00002
output_cost: Final = 0.00004
cache_read_cost: Final = 0.000005
cache_creation_cost: Final = 0.00001
reasoning_cost: Final = 0.000015
tool_usage_cost: Final = 0.00003
total_cost: Final = (
input_cost + cache_read_cost + cache_creation_cost + output_cost + tool_usage_cost
)

logging_obj.set_cost_breakdown(
input_cost=input_cost,
output_cost=output_cost,
total_cost=total_cost,
cost_for_built_in_tools_cost_usd_dollar=tool_usage_cost,
cache_read_cost=cache_read_cost,
cache_creation_cost=cache_creation_cost,
reasoning_cost=reasoning_cost,
)

headers = ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=mock_user_api_key_dict,
call_id="test-call-id-components",
response_cost=total_cost,
litellm_logging_obj=logging_obj,
)

assert "x-litellm-response-cost" in headers
assert float(headers["x-litellm-response-cost"]) == pytest.approx(total_cost)

assert "x-litellm-response-cost-input" in headers
assert float(headers["x-litellm-response-cost-input"]) == pytest.approx(input_cost)

assert "x-litellm-response-cost-output" in headers
assert float(headers["x-litellm-response-cost-output"]) == pytest.approx(output_cost)

assert "x-litellm-response-cost-cache-read" in headers
assert float(headers["x-litellm-response-cost-cache-read"]) == pytest.approx(cache_read_cost)

assert "x-litellm-response-cost-cache-creation" in headers
assert float(headers["x-litellm-response-cost-cache-creation"]) == pytest.approx(cache_creation_cost)

assert "x-litellm-response-cost-reasoning" in headers
assert float(headers["x-litellm-response-cost-reasoning"]) == pytest.approx(reasoning_cost)

assert "x-litellm-response-cost-tool-usage" in headers
assert float(headers["x-litellm-response-cost-tool-usage"]) == pytest.approx(tool_usage_cost)

component_sum: Final = (
float(headers["x-litellm-response-cost-input"])
+ float(headers["x-litellm-response-cost-cache-read"])
+ float(headers["x-litellm-response-cost-cache-creation"])
+ float(headers["x-litellm-response-cost-output"])
+ float(headers["x-litellm-response-cost-tool-usage"])
)
assert component_sum == pytest.approx(float(headers["x-litellm-response-cost"]))
assert float(headers["x-litellm-response-cost-reasoning"]) <= float(headers["x-litellm-response-cost-output"])

def test_get_custom_headers_without_cost_breakdown_omits_component_headers(self):
"""Test that when litellm_logging_obj has no cost_breakdown, component headers are omitted."""
from litellm.litellm_core_utils.litellm_logging import (
Logging as LiteLLMLoggingObj,
)

mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth)
mock_user_api_key_dict.tpm_limit = None
mock_user_api_key_dict.rpm_limit = None
mock_user_api_key_dict.max_budget = None
mock_user_api_key_dict.spend = 0

logging_obj = LiteLLMLoggingObj(
model="gpt-4",
messages=[],
stream=False,
call_type="completion",
start_time=None,
litellm_call_id="test-call-id-no-breakdown",
function_id="test-function",
)

headers = ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=mock_user_api_key_dict,
response_cost=0.0001,
litellm_logging_obj=logging_obj,
)

assert "x-litellm-response-cost" in headers
assert "x-litellm-response-cost-input" not in headers
assert "x-litellm-response-cost-output" not in headers
assert "x-litellm-response-cost-cache-read" not in headers
assert "x-litellm-response-cost-cache-creation" not in headers
assert "x-litellm-response-cost-reasoning" not in headers
assert "x-litellm-response-cost-tool-usage" not in headers

def test_get_custom_headers_per_component_with_discount_and_margin(self):
"""Test that component headers co-exist accurately with discount and margin headers."""
from litellm.litellm_core_utils.litellm_logging import (
Logging as LiteLLMLoggingObj,
)

mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth)
mock_user_api_key_dict.tpm_limit = None
mock_user_api_key_dict.rpm_limit = None
mock_user_api_key_dict.max_budget = None
mock_user_api_key_dict.spend = 0

logging_obj = LiteLLMLoggingObj(
model="gpt-4",
messages=[],
stream=False,
call_type="completion",
start_time=None,
litellm_call_id="test-call-id-combined",
function_id="test-function",
)

logging_obj.set_cost_breakdown(
input_cost=0.00006,
output_cost=0.00004,
total_cost=0.000105,
cost_for_built_in_tools_cost_usd_dollar=0.0,
original_cost=0.0001,
discount_percent=0.05,
discount_amount=0.000005,
margin_percent=0.10,
margin_total_amount=0.00001,
)

headers = ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=mock_user_api_key_dict,
response_cost=0.000105,
litellm_logging_obj=logging_obj,
)

assert float(headers["x-litellm-response-cost"]) == pytest.approx(0.000105)
assert float(headers["x-litellm-response-cost-original"]) == pytest.approx(0.0001)
assert float(headers["x-litellm-response-cost-discount-amount"]) == pytest.approx(0.000005)
assert float(headers["x-litellm-response-cost-margin-amount"]) == pytest.approx(0.00001)
assert float(headers["x-litellm-response-cost-margin-percent"]) == pytest.approx(0.10)
assert float(headers["x-litellm-response-cost-input"]) == pytest.approx(0.00006)
assert float(headers["x-litellm-response-cost-output"]) == pytest.approx(0.00004)
assert "x-litellm-response-cost-cache-read" not in headers
assert "x-litellm-response-cost-cache-creation" not in headers
assert "x-litellm-response-cost-reasoning" not in headers
assert float(headers["x-litellm-response-cost-tool-usage"]) == pytest.approx(0.0)

@pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"])
def test_get_custom_headers_classifier_cost_from_routing_decision(self, metadata_key):
"""The auto-router's LLM classifier cost must surface as its own header.
Expand Down Expand Up @@ -919,16 +1087,14 @@ def test_get_cost_breakdown_from_logging_obj_helper(self):
discount_amount=0.000005,
)

(
original_cost,
discount_amount,
margin_total_amount,
margin_percent,
) = _get_cost_breakdown_from_logging_obj(logging_obj)
assert original_cost == 0.0001
assert discount_amount == 0.000005
assert margin_total_amount is None
assert margin_percent is None
breakdown = _get_cost_breakdown_from_logging_obj(logging_obj)
assert breakdown.original_cost == 0.0001
assert breakdown.discount_amount == 0.000005
assert breakdown.margin_total_amount is None
assert breakdown.margin_percent is None
assert breakdown.input_cost == 0.00005
assert breakdown.output_cost == 0.00005
assert breakdown.tool_usage_cost == 0.0

# Test with margin info
logging_obj_with_margin = LiteLLMLoggingObj(
Expand All @@ -950,16 +1116,11 @@ def test_get_cost_breakdown_from_logging_obj_helper(self):
margin_total_amount=0.00001,
)

(
original_cost,
discount_amount,
margin_total_amount,
margin_percent,
) = _get_cost_breakdown_from_logging_obj(logging_obj_with_margin)
assert original_cost == 0.0001
assert discount_amount is None
assert margin_total_amount == 0.00001
assert margin_percent == 0.10
breakdown_with_margin = _get_cost_breakdown_from_logging_obj(logging_obj_with_margin)
assert breakdown_with_margin.original_cost == 0.0001
assert breakdown_with_margin.discount_amount is None
assert breakdown_with_margin.margin_total_amount == 0.00001
assert breakdown_with_margin.margin_percent == 0.10

# Test with no discount or margin info
logging_obj_no_discount = LiteLLMLoggingObj(
Expand All @@ -978,28 +1139,17 @@ def test_get_cost_breakdown_from_logging_obj_helper(self):
cost_for_built_in_tools_cost_usd_dollar=0.0,
)

(
original_cost,
discount_amount,
margin_total_amount,
margin_percent,
) = _get_cost_breakdown_from_logging_obj(logging_obj_no_discount)
assert original_cost is None
assert discount_amount is None
assert margin_total_amount is None
assert margin_percent is None
breakdown_no_discount = _get_cost_breakdown_from_logging_obj(logging_obj_no_discount)
assert breakdown_no_discount.original_cost is None
assert breakdown_no_discount.discount_amount is None
assert breakdown_no_discount.margin_total_amount is None
assert breakdown_no_discount.margin_percent is None
assert breakdown_no_discount.input_cost == 0.00005
assert breakdown_no_discount.output_cost == 0.00005

# Test with None logging object
(
original_cost,
discount_amount,
margin_total_amount,
margin_percent,
) = _get_cost_breakdown_from_logging_obj(None)
assert original_cost is None
assert discount_amount is None
assert margin_total_amount is None
assert margin_percent is None
breakdown_none = _get_cost_breakdown_from_logging_obj(None)
assert all(value is None for value in breakdown_none)

def test_get_custom_headers_key_spend_includes_response_cost(self):
"""
Expand Down
Loading