diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index 2dfa7adddea..e4b7e755bd0 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -44,3 +44,10 @@ - {id: quota_management.spend_tracking.failure.writes_failure_row, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [writes_failure_row], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_log_error_logger.py", rationale: "A failed call writes a failure-status spend row"} - {id: quota_management.spend_tracking.spend_calculate.returns_cost, module: quota_management, tier: P2, behavior: spend_tracking, variant: spend_calculate, assertions: [returns_cost], exercised_on: [spend_calculate], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "/spend/calculate prices a hypothetical request at nonzero cost"} - {id: quota_management.spend_tracking.pagination.keeps_total, module: quota_management, tier: P2, behavior: spend_tracking, variant: pagination, assertions: [keeps_total], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "Spend-logs v2 pagination caps page size without losing the total"} +- {id: quota_management.spend_tracking.cache_write.bills_cache_creation_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_write, assertions: [bills_cache_creation_rate], exercised_on: [chat_completions], source: "litellm_core_utils/llm_cost_calc/utils.py", rationale: "OpenAI cache-write tokens land on the spend row as cache-creation tokens billed at the cache-creation rate, not silently at the input rate (#34046)"} +- {id: quota_management.spend_tracking.cost_breakdown.reports_component_costs, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_breakdown, assertions: [reports_component_costs], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "The spend row's metadata.cost_breakdown itemizes cache-read, cache-creation, output, and reasoning costs at the deployment's own rates and they sum to the row's spend (#31686)"} +- {id: quota_management.spend_tracking.stream_cache_read.bills_cache_read_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream_cache_read, assertions: [bills_cache_read_rate], exercised_on: [chat_completions], source: "litellm_core_utils/streaming_chunk_builder_utils.py", rationale: "A streamed call's reassembled usage keeps the cached-token detail so cache reads bill at the cache-read discount, not full input price (#34812)"} +- {id: quota_management.spend_tracking.messages_bridge.keeps_cache_tokens, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [keeps_cache_tokens], exercised_on: [messages], source: "llms/anthropic/experimental_pass_through/responses_adapters/handler.py", rationale: "A /v1/messages request served by a Responses-only OpenAI model keeps its cache-read tokens and their discounted billing across the bridge (#34957)"} +- {id: quota_management.spend_tracking.service_tier.bills_tier_rates, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier, assertions: [bills_tier_rates], exercised_on: [chat_completions], source: "cost_calculator.py", rationale: "A priority service_tier call bills input, output, and reasoning at the deployment's *_priority rates and records the tier on the row (#35923, #35925)"} +- {id: quota_management.spend_tracking.cost_headers.additive_components, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_headers, assertions: [additive_components], exercised_on: [chat_completions], source: "proxy/common_request_processing.py", rationale: "The x-litellm-response-cost-* component headers sum to the total, input covers only fresh tokens, and reasoning stays a subset of output (#36965)"} +- {id: quota_management.spend_tracking.passthrough_stream.injects_usage_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: passthrough_stream, assertions: [injects_usage_cost], exercised_on: [openai_passthrough], source: "proxy/pass_through_endpoints/streaming_handler.py", rationale: "With include_cost_in_streaming_usage on, the /openai passthrough's final streaming usage frame carries the proxy-computed cost (#36503). Uncovered: the flag is only settable in litellm_settings, and the shared e2e stack does not turn it on yet"} diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 619f4dcacfe..f1d5733253a 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -452,6 +452,7 @@ class AnthropicMessagesResponse(BaseModel): model: str | None = None content: list[AnthropicContentBlock] | None = None choices: list[ChatChoice] | None = None + usage: Usage | None = None class CountTokensResponse(BaseModel): @@ -716,8 +717,10 @@ class FineTuningJobsResponse(BaseModel): class LiteLLMParamsBody(BaseModel): """POST /model/new litellm_params: `model` is the only required field; `api_key` et al may be an `os.environ/FOO` reference the proxy resolves at call time. - `input_cost_per_token`/`output_cost_per_token` register a per-deployment custom - pricing override; left None (and dropped from the body) the deployment keeps the + The `*_cost_per_token` / `*_token_cost` fields register a per-deployment custom + pricing override (the cache and `_priority` rates only apply when both base + rates are set, which is what makes the proxy register the deployment's full + pricing entry); left None (and dropped from the body) the deployment keeps the backend's canonical rate.""" model: str @@ -744,6 +747,10 @@ class LiteLLMParamsBody(BaseModel): aws_external_id: str | None = None input_cost_per_token: float | None = None output_cost_per_token: float | None = None + cache_read_input_token_cost: float | None = None + cache_creation_input_token_cost: float | None = None + input_cost_per_token_priority: float | None = None + output_cost_per_token_priority: float | None = None extra_headers: dict[str, str] | None = None use_in_pass_through: bool | None = None complexity_router_config: dict[str, object] | None = None diff --git a/tests/e2e/quota_management/spend_tracking/cost_rows.py b/tests/e2e/quota_management/spend_tracking/cost_rows.py new file mode 100644 index 00000000000..87af54fe83f --- /dev/null +++ b/tests/e2e/quota_management/spend_tracking/cost_rows.py @@ -0,0 +1,204 @@ +"""Cost-accounting helpers for the spend-tracking suite: the /spend/logs row shape +that carries the per-component cost breakdown, a poll that waits for it, and the +builders the cache-pricing tests share. + +The shared SpendLogRow deliberately stays thin (most tests only read totals), so +the component-cost tests model the metadata they assert on here instead: +`metadata.cost_breakdown` (input/output/cache-read/cache-creation/reasoning costs +plus the service-tier pricing basis) and `metadata.additional_usage_values` (the +cache token counts the biller derived from the provider's usage). + +Determinism strategy: every test registers its own deployment with explicit custom +rates for each component it asserts on (`register_priced_model`), so expected cost +is exactly tokens-on-the-row times configured rate, immune to provider price +changes. The rates are chosen ~100x above canonical and distinct from one another, +so a component billed at the wrong rate can never accidentally match. + +OpenAI prompt caching is implicit and keyed on the exact token prefix, with a +1024-token minimum. `cacheable_prefix` builds a prefix whose first word is the +run's unique marker: unique marker = the whole prefix is novel (a fresh cache +write), same marker + different question = a cache read that still misses the +proxy's own response cache. How long the prefix has to be before the provider +actually reports a read varies by model, so callers pass `words` to suit theirs. + +Two facts about the recorded bill that the assertions here encode, because the +two surfaces disagree on purpose. On the spend row, `input_cost` is gross: it +already contains the cache-read and cache-creation costs, so the row's total is +input + output + tool-usage and the fresh-token cost is input minus the two cache +components. In the response headers, `x-litellm-response-cost-input` is net of +cache, which is what makes the component headers sum to the total. +""" + +import time +from collections.abc import Callable + +from pydantic import BaseModel, RootModel + +from e2e_config import unique_marker +from e2e_http import Success +from lifecycle import ResourceManager +from models import LiteLLMParamsBody, SpendLogsParams +from proxy_client import ProxyClient + + +class CostBreakdownRow(BaseModel): + input_cost: float | None = None + output_cost: float | None = None + cache_read_cost: float | None = None + cache_creation_cost: float | None = None + reasoning_cost: float | None = None + tool_usage_cost: float | None = None + total_cost: float | None = None + service_tier: str | None = None + + +class AdditionalUsageValues(BaseModel): + cache_read_input_tokens: int | None = None + cache_creation_input_tokens: int | None = None + + +class CostRowMetadata(BaseModel): + cost_breakdown: CostBreakdownRow | None = None + additional_usage_values: AdditionalUsageValues | None = None + + +class CostRow(BaseModel): + request_id: str | None = None + spend: float | None = None + prompt_tokens: int | None = None + completion_tokens: int | None = None + metadata: CostRowMetadata | None = None + + @property + def breakdown(self) -> CostBreakdownRow: + assert self.metadata and self.metadata.cost_breakdown, ( + f"spend row {self.request_id} landed without a cost breakdown" + ) + return self.metadata.cost_breakdown + + @property + def cache_read_tokens(self) -> int: + if self.metadata and self.metadata.additional_usage_values: + return self.metadata.additional_usage_values.cache_read_input_tokens or 0 + return 0 + + @property + def cache_creation_tokens(self) -> int: + if self.metadata and self.metadata.additional_usage_values: + return self.metadata.additional_usage_values.cache_creation_input_tokens or 0 + return 0 + + +class CostRows(RootModel[list[CostRow]]): + pass + + +def approx_equal(actual: float, expected: float) -> bool: + """Within 1% or 1e-9 absolute - spend math, not exact float identity.""" + return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2) + + +def assert_total_is_sum_of_components(row: CostRow) -> None: + """The row's total is input + output + tool usage. The cache components are + already inside the gross input cost, so adding them again would double-bill.""" + breakdown = row.breakdown + components = sum( + cost or 0.0 + for cost in (breakdown.input_cost, breakdown.output_cost, breakdown.tool_usage_cost) + ) + assert breakdown.total_cost is not None and approx_equal(breakdown.total_cost, components), ( + f"total_cost {breakdown.total_cost} != input + output + tool usage ({components}): {breakdown}" + ) + assert row.spend is not None and approx_equal(row.spend, breakdown.total_cost), ( + f"row spend {row.spend} != breakdown total {breakdown.total_cost}" + ) + + +def assert_fresh_tokens_billed_at(row: CostRow, input_rate: float) -> None: + """Strip the cache components out of the gross input cost and what is left must + be the freshly-read tokens at the deployment's input rate.""" + breakdown = row.breakdown + fresh_tokens = (row.prompt_tokens or 0) - row.cache_read_tokens - row.cache_creation_tokens + fresh_cost = ( + (breakdown.input_cost or 0.0) + - (breakdown.cache_read_cost or 0.0) + - (breakdown.cache_creation_cost or 0.0) + ) + assert breakdown.input_cost is not None and approx_equal(fresh_cost, fresh_tokens * input_rate), ( + f"input_cost {breakdown.input_cost} less cache read {breakdown.cache_read_cost} and " + f"cache creation {breakdown.cache_creation_cost} leaves {fresh_cost}, not " + f"{fresh_tokens} fresh tokens * {input_rate} (prompt {row.prompt_tokens}, " + f"cache read {row.cache_read_tokens}, cache creation {row.cache_creation_tokens}); " + "cached tokens are being billed at the input rate" + ) + + +def poll_cost_row(proxy: ProxyClient, request_id: str) -> CostRow | None: + """Poll /spend/logs for the call's row until it lands with a cost breakdown + (rows flush ~60s behind the call via proxy_batch_write_at); None on timeout.""" + deadline = time.monotonic() + proxy.poll_timeout + while time.monotonic() < deadline: + result = proxy.transport.get( + "/spend/logs", + headers=proxy.transport.master, + params=SpendLogsParams(request_id=request_id), + response_type=CostRows, + ) + match result: + case Success(data=data): + rows = data.root + case _: + rows = [] + for row in rows: + if row.metadata and row.metadata.cost_breakdown: + return row + time.sleep(proxy.poll_interval) + return None + + +def poll_cost_row_where( + proxy: ProxyClient, api_key: str, predicate: Callable[[CostRow], bool] +) -> CostRow | None: + """Poll the key's own /spend/logs until one of its rows carries a cost breakdown + the predicate accepts; None on timeout. For calls whose response id is not the + id the bill is filed under, which is how a user finds the row in the UI anyway.""" + deadline = time.monotonic() + proxy.poll_timeout + while time.monotonic() < deadline: + result = proxy.transport.get( + "/spend/logs", + headers=proxy.transport.master, + params=SpendLogsParams(api_key=api_key), + response_type=CostRows, + ) + match result: + case Success(data=data): + rows = data.root + case _: + rows = [] + for row in rows: + if row.metadata and row.metadata.cost_breakdown and predicate(row): + return row + time.sleep(proxy.poll_interval) + return None + + +def register_priced_model( + proxy: ProxyClient, + resources: ResourceManager, + name_prefix: str, + litellm_params: LiteLLMParamsBody, +) -> str: + """Register a deployment with explicit custom rates (deleted on teardown) and + return its unique model name.""" + model_name = f"{name_prefix}-{unique_marker()}" + model_id = proxy.create_model(model_name, litellm_params) + resources.defer(lambda: proxy.delete_model(model_id)) + return model_name + + +def cacheable_prefix(marker: str, *, words: int = 1200) -> str: + """A prompt prefix above OpenAI's 1024-token caching minimum whose identity is + fully determined by `marker` (it is the first word, and prefix caching matches + from token zero). Raise `words` for models that only report a cache read on a + substantially longer prefix.""" + return " ".join(marker if i == 0 else f"token{i:04d}" for i in range(words)) diff --git a/tests/e2e/quota_management/spend_tracking/test_cache_cost_accounting_e2e.py b/tests/e2e/quota_management/spend_tracking/test_cache_cost_accounting_e2e.py new file mode 100644 index 00000000000..c50ec3d902f --- /dev/null +++ b/tests/e2e/quota_management/spend_tracking/test_cache_cost_accounting_e2e.py @@ -0,0 +1,287 @@ +"""Live e2e: prompt-cache token accounting bills each cache component at its own rate. + +Four regressions the gateway has shipped fixes for, pinned against real OpenAI +prompt caching (implicit, keyed on the token prefix). Every test registers its own +deployment with distinct custom rates for input / output / cache-read / +cache-creation, so the expected bill is exactly the row's token counts times the +configured rates and a component billed at the wrong rate can never pass: + +- cache writes: gpt-5.6's cache-write tokens must land on the spend row as + cache-creation tokens billed at the cache-creation rate, not silently at the + input rate (#34046) +- breakdown components: the row's metadata.cost_breakdown must itemize cache-read, + cache-creation, and reasoning costs, with reasoning a subset of output (#31686) +- streaming: a streamed call's reassembled usage must keep the cached-token detail + so cache reads bill at the cache-read discount, not full input price (#34812) +- /v1/messages bridge: a request served by a Responses-only OpenAI model crosses + the anthropic-messages -> Responses adapter and must keep its cache-read tokens + and their discounted billing (#34957) + +Each test drives the model that actually reports the component it bills, which is +not the same model throughout. gpt-5.6-luna reports cache-write tokens on every +call over the caching minimum and never reports a cache read, so it is the one +model that can prove cache-write billing and the one model that can never prove +cache-read billing. gpt-5.5 is the reverse: it reports cached tokens on the second +call and no cache writes at all. gpt-5.3-codex is Responses-only, which is what +forces the /v1/messages bridge, and it starts reporting cache reads once the +prefix is a few thousand tokens rather than one. + +OpenAI caching is best-effort, so each test retries with a fresh prefix (new +marker = brand-new cache identity) up to three times before failing; the prime and +measured calls share the prefix but differ in the trailing question, which defeats +the proxy's own response cache without touching the provider's prefix cache. + +The test that asserts on reasoning cost requests reasoning explicitly with +`reasoning_effort`, so that assertion rests on a parameter the test sets rather +than on whatever the model happens to do by default. Its prime call carries the +same value: OpenAI's prefix cache keys on the reasoning setting as well as the +tokens, so a prime at a different effort never produces a read. +""" + +import pytest + +from cost_rows import ( + CostRow, + approx_equal, + assert_fresh_tokens_billed_at, + assert_total_is_sum_of_components, + cacheable_prefix, + poll_cost_row, + poll_cost_row_where, + register_priced_model, +) +from e2e_config import unique_marker +from e2e_http import unwrap +from lifecycle import ResourceManager +from models import AnthropicMessagesBody, ChatBody, ChatMessage, LiteLLMParamsBody +from pydantic import BaseModel +from spend_e2e_client import SpendClient + +pytestmark = pytest.mark.e2e + +CACHE_WRITE_BACKEND = "openai/gpt-5.6-luna" +CACHE_READ_BACKEND = "openai/gpt-5.5" +BRIDGE_BACKEND = "openai/gpt-5.3-codex" +BRIDGE_PREFIX_WORDS = 3000 +OPENAI_API_KEY = "os.environ/OPENAI_API_KEY" +CACHE_ATTEMPTS = 3 + +INPUT_RATE = 4e-05 +OUTPUT_RATE = 8e-05 +CACHE_READ_RATE = 1e-05 +CACHE_WRITE_RATE = 5e-05 + +PRIME_QUESTION = "Reply with the single word ready." +REASONING_QUESTION = "Compute 47*83 - 19*7 step by step, then reply with just the final number." +REASONING_EFFORT = "high" + + +class _StreamChunk(BaseModel): + id: str | None = None + + +def _cache_priced_params(backend: str) -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=backend, + api_key=OPENAI_API_KEY, + input_cost_per_token=INPUT_RATE, + output_cost_per_token=OUTPUT_RATE, + cache_read_input_token_cost=CACHE_READ_RATE, + cache_creation_input_token_cost=CACHE_WRITE_RATE, + ) + + +def _chat_body( + model: str, content: str, *, stream: bool = False, reasoning_effort: str | None = None +) -> ChatBody: + return ChatBody( + model=model, + messages=[ChatMessage(role="user", content=content)], + stream=stream, + max_completion_tokens=4000, + reasoning_effort=reasoning_effort, + ) + + +def _require_row(client: SpendClient, request_id: str) -> CostRow: + row = poll_cost_row(client.proxy, request_id) + assert row is not None, f"no spend row with a cost breakdown landed for {request_id}" + return row + + +def _assert_cache_read_billed(row: CostRow) -> None: + assert row.breakdown.cache_read_cost is not None and approx_equal( + row.breakdown.cache_read_cost, row.cache_read_tokens * CACHE_READ_RATE + ), ( + f"cache_read_cost {row.breakdown.cache_read_cost} != " + f"{row.cache_read_tokens} cached tokens * {CACHE_READ_RATE}" + ) + assert_fresh_tokens_billed_at(row, INPUT_RATE) + assert_total_is_sum_of_components(row) + + +class TestCacheCostAccounting: + @pytest.mark.covers("quota_management.spend_tracking.cache_write.bills_cache_creation_rate") + def test_cache_write_tokens_billed_at_cache_creation_rate( + self, client: SpendClient, resources: ResourceManager, scoped_key: str + ) -> None: + model = register_priced_model( + client.proxy, resources, "cache-write-priced", _cache_priced_params(CACHE_WRITE_BACKEND) + ) + + for _ in range(CACHE_ATTEMPTS): + prompt = f"{cacheable_prefix(unique_marker())}\n{PRIME_QUESTION}" + chat = unwrap(client.proxy.chat(scoped_key, _chat_body(model, prompt))) + assert chat.id, f"chat response carried no id: {chat}" + row = _require_row(client, chat.id) + if row.cache_creation_tokens > 0: + break + else: + pytest.fail( + f"OpenAI reported no cache-write tokens across {CACHE_ATTEMPTS} fresh " + "~2k-token prompts; the cache-write billing path was never exercised" + ) + + assert row.breakdown.cache_creation_cost is not None and approx_equal( + row.breakdown.cache_creation_cost, row.cache_creation_tokens * CACHE_WRITE_RATE + ), ( + f"cache_creation_cost {row.breakdown.cache_creation_cost} != " + f"{row.cache_creation_tokens} cache-write tokens * {CACHE_WRITE_RATE}" + ) + assert_fresh_tokens_billed_at(row, INPUT_RATE) + assert_total_is_sum_of_components(row) + + @pytest.mark.covers("quota_management.spend_tracking.cost_breakdown.reports_component_costs") + def test_cost_breakdown_reports_component_costs( + self, client: SpendClient, resources: ResourceManager, scoped_key: str + ) -> None: + model = register_priced_model( + client.proxy, resources, "breakdown-priced", _cache_priced_params(CACHE_READ_BACKEND) + ) + + for _ in range(CACHE_ATTEMPTS): + prefix = cacheable_prefix(unique_marker()) + unwrap( + client.proxy.chat( + scoped_key, + _chat_body( + model, f"{prefix}\n{PRIME_QUESTION}", reasoning_effort=REASONING_EFFORT + ), + ) + ) + chat = unwrap( + client.proxy.chat( + scoped_key, + _chat_body( + model, + f"{prefix}\n{REASONING_QUESTION}", + reasoning_effort=REASONING_EFFORT, + ), + ) + ) + assert chat.id, f"chat response carried no id: {chat}" + row = _require_row(client, chat.id) + if row.cache_read_tokens > 0: + break + else: + pytest.fail( + f"no cache read landed across {CACHE_ATTEMPTS} prime+read rounds; " + "the component-cost breakdown was never exercised with cached input" + ) + + usage = chat.usage + assert usage is not None and usage.completion_tokens_details is not None, ( + f"no completion token details on the measured call: {chat}" + ) + reasoning_tokens = usage.completion_tokens_details.reasoning_tokens or 0 + assert reasoning_tokens > 0, f"the reasoning question produced no reasoning tokens: {usage}" + + breakdown = row.breakdown + assert breakdown.output_cost is not None and approx_equal( + breakdown.output_cost, (row.completion_tokens or 0) * OUTPUT_RATE + ), ( + f"output_cost {breakdown.output_cost} != " + f"{row.completion_tokens} completion tokens * {OUTPUT_RATE}" + ) + assert breakdown.reasoning_cost is not None and approx_equal( + breakdown.reasoning_cost, reasoning_tokens * OUTPUT_RATE + ), ( + f"reasoning_cost {breakdown.reasoning_cost} != " + f"{reasoning_tokens} reasoning tokens * {OUTPUT_RATE}" + ) + assert breakdown.reasoning_cost <= (breakdown.output_cost or 0.0) * 1.01, ( + f"reasoning_cost {breakdown.reasoning_cost} exceeds output_cost " + f"{breakdown.output_cost}; reasoning must be a subset of output" + ) + _assert_cache_read_billed(row) + + @pytest.mark.covers("quota_management.spend_tracking.stream_cache_read.bills_cache_read_rate") + def test_streaming_cache_read_billed_at_cache_read_rate( + self, client: SpendClient, resources: ResourceManager, scoped_key: str + ) -> None: + model = register_priced_model( + client.proxy, resources, "stream-cache-priced", _cache_priced_params(CACHE_READ_BACKEND) + ) + + for _ in range(CACHE_ATTEMPTS): + prefix = cacheable_prefix(unique_marker()) + unwrap(client.proxy.chat(scoped_key, _chat_body(model, f"{prefix}\n{PRIME_QUESTION}"))) + result = client.proxy.chat_stream( + scoped_key, + _chat_body(model, f"{prefix}\nReply with the single word cached.", stream=True), + ) + assert result.ok and result.stream_events, ( + f"streamed chat failed (status {result.status_code}): {result.body[:300]}" + ) + stream_id = _StreamChunk.model_validate_json(result.stream_events[0]).id + assert stream_id, f"first stream chunk carried no id: {result.stream_events[0][:200]}" + row = _require_row(client, stream_id) + if row.cache_read_tokens > 0: + break + else: + pytest.fail( + f"no cache read landed across {CACHE_ATTEMPTS} prime+stream rounds; " + "streaming cache-read billing was never exercised" + ) + + _assert_cache_read_billed(row) + + @pytest.mark.covers("quota_management.spend_tracking.messages_bridge.keeps_cache_tokens") + def test_messages_bridge_keeps_cache_tokens( + self, client: SpendClient, resources: ResourceManager, scoped_key: str + ) -> None: + model = register_priced_model( + client.proxy, resources, "bridge-cache-priced", _cache_priced_params(BRIDGE_BACKEND) + ) + + def bridge_call(content: str) -> int: + response = unwrap( + client.proxy.messages( + scoped_key, + AnthropicMessagesBody( + model=model, + messages=[ChatMessage(role="user", content=content)], + max_tokens=4000, + ), + ) + ) + assert response.usage is not None, f"bridged response carried no usage: {response}" + return response.usage.cache_read_input_tokens or 0 + + for _ in range(CACHE_ATTEMPTS): + prefix = cacheable_prefix(unique_marker(), words=BRIDGE_PREFIX_WORDS) + bridge_call(f"{prefix}\n{PRIME_QUESTION}") + if bridge_call(f"{prefix}\nReply with the single word bridged.") > 0: + break + else: + pytest.fail( + f"no cache read survived {CACHE_ATTEMPTS} bridged prime+read rounds; " + "cache tokens are not surviving the anthropic-messages -> Responses bridge" + ) + + row = poll_cost_row_where(client.proxy, scoped_key, lambda r: r.cache_read_tokens > 0) + assert row is not None, ( + "the bridged call reported cached tokens but no spend row for the key " + "recorded any; the cache tokens were dropped on the way to the bill" + ) + _assert_cache_read_billed(row) diff --git a/tests/e2e/quota_management/spend_tracking/test_cost_headers_e2e.py b/tests/e2e/quota_management/spend_tracking/test_cost_headers_e2e.py new file mode 100644 index 00000000000..203be611905 --- /dev/null +++ b/tests/e2e/quota_management/spend_tracking/test_cost_headers_e2e.py @@ -0,0 +1,136 @@ +"""Live e2e: the per-component x-litellm-response-cost-* headers keep their contract. + +Pins the header contract shipped in #36965: alongside the x-litellm-response-cost +total, every response carries the component costs (input, output, cache-read, +cache-creation, reasoning, tool-usage), where input covers only fresh tokens (the +cache components are subtracted out) so the components sum to the total, and +reasoning stays a subset of output. + +The deployment carries distinct custom rates per component, a prime call fills the +provider's prefix cache, and the measured call re-reads it, so the cache-read +header is exercised with a real nonzero value instead of passing vacuously. The +backend is gpt-5.5 because it reports cached tokens on the second call; the +gpt-5.6 line reports cache writes and never a read, which would leave the +cache-read header at zero forever. The raw-transport send is used because the +typed chat client validates bodies and drops headers. OpenAI caching is +best-effort, so the prime+measure round retries with a fresh prefix before +failing. +""" + +import pytest + +from cost_rows import approx_equal, cacheable_prefix, register_priced_model +from e2e_config import unique_marker +from e2e_http import StreamingResponse +from lifecycle import ResourceManager +from models import ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody +from spend_e2e_client import SpendClient + +pytestmark = pytest.mark.e2e + +BACKEND = "openai/gpt-5.5" +OPENAI_API_KEY = "os.environ/OPENAI_API_KEY" +CACHE_ATTEMPTS = 3 + +INPUT_RATE = 4e-05 +OUTPUT_RATE = 8e-05 +CACHE_READ_RATE = 1e-05 +CACHE_WRITE_RATE = 5e-05 + +COMPONENT_HEADERS = ( + "x-litellm-response-cost-input", + "x-litellm-response-cost-cache-read", + "x-litellm-response-cost-cache-creation", + "x-litellm-response-cost-output", + "x-litellm-response-cost-tool-usage", +) + + +def _header_cost(response: StreamingResponse, name: str) -> float: + value = response.headers.get(name) + return float(value) if value not in (None, "", "None") else 0.0 + + +class TestCostHeaders: + @pytest.mark.covers("quota_management.spend_tracking.cost_headers.additive_components") + def test_component_cost_headers_sum_to_total( + self, client: SpendClient, resources: ResourceManager, scoped_key: str + ) -> None: + model = register_priced_model( + client.proxy, + resources, + "header-priced", + LiteLLMParamsBody( + model=BACKEND, + api_key=OPENAI_API_KEY, + input_cost_per_token=INPUT_RATE, + output_cost_per_token=OUTPUT_RATE, + cache_read_input_token_cost=CACHE_READ_RATE, + cache_creation_input_token_cost=CACHE_WRITE_RATE, + ), + ) + + def priced_call(content: str) -> StreamingResponse: + response = client.proxy.transport.send( + "/chat/completions", + headers=client.proxy.transport.bearer(scoped_key), + json=ChatBody( + model=model, + messages=[ChatMessage(role="user", content=content)], + max_completion_tokens=4000, + ), + ) + assert response.ok, f"chat failed (status {response.status_code}): {response.body[:300]}" + return response + + for _ in range(CACHE_ATTEMPTS): + prefix = cacheable_prefix(unique_marker()) + priced_call(f"{prefix}\nReply with the single word ready.") + measured = priced_call(f"{prefix}\nReply with the single word measured.") + if _header_cost(measured, "x-litellm-response-cost-cache-read") > 0: + break + else: + pytest.fail( + f"no cache read landed across {CACHE_ATTEMPTS} prime+measure rounds; " + "the cache-read cost header was never exercised with a nonzero value" + ) + + total = measured.response_cost + assert total is not None and total > 0, ( + f"x-litellm-response-cost missing or zero: {measured.headers}" + ) + component_sum = sum(_header_cost(measured, name) for name in COMPONENT_HEADERS) + assert approx_equal(component_sum, total), ( + f"component headers sum to {component_sum}, not the total {total}: " + f"{ {name: measured.headers.get(name) for name in COMPONENT_HEADERS} }" + ) + + reasoning = _header_cost(measured, "x-litellm-response-cost-reasoning") + output = _header_cost(measured, "x-litellm-response-cost-output") + assert reasoning <= output * 1.01, ( + f"reasoning header {reasoning} exceeds output header {output}; " + "reasoning must be a subset of output" + ) + + usage = ChatResponse.model_validate_json(measured.body).usage + assert usage is not None, f"measured response carried no usage: {measured.body[:300]}" + cached_tokens = ( + usage.prompt_tokens_details.cached_tokens or 0 if usage.prompt_tokens_details else 0 + ) + cache_creation_tokens = usage.cache_creation_input_tokens or 0 + assert cached_tokens > 0, f"cache-read header nonzero but usage shows no cached tokens: {usage}" + assert approx_equal( + _header_cost(measured, "x-litellm-response-cost-cache-read"), + cached_tokens * CACHE_READ_RATE, + ), ( + f"cache-read header {measured.headers.get('x-litellm-response-cost-cache-read')} != " + f"{cached_tokens} cached tokens * {CACHE_READ_RATE}" + ) + fresh_tokens = (usage.prompt_tokens or 0) - cached_tokens - cache_creation_tokens + assert approx_equal( + _header_cost(measured, "x-litellm-response-cost-input"), fresh_tokens * INPUT_RATE + ), ( + f"input header {measured.headers.get('x-litellm-response-cost-input')} != " + f"{fresh_tokens} fresh tokens * {INPUT_RATE}; the input component is not " + "subtracting the cache components" + ) diff --git a/tests/e2e/quota_management/spend_tracking/test_service_tier_pricing_e2e.py b/tests/e2e/quota_management/spend_tracking/test_service_tier_pricing_e2e.py new file mode 100644 index 00000000000..770c5699b4e --- /dev/null +++ b/tests/e2e/quota_management/spend_tracking/test_service_tier_pricing_e2e.py @@ -0,0 +1,121 @@ +"""Live e2e: a service_tier request bills every component at the tier's own rates. + +Pins the tier-billing fixes (#35923, #35925): a priority-tier call must price +input and output at the deployment's `*_priority` rates, including the reasoning +tokens inside output (the shipped bug billed reasoning at the default-tier rate), +and the spend row must record the tier the bill was computed on. + +The deployment carries custom base AND priority rates, each distinct, so a bill +computed from the wrong tier (or a mix) cannot match the expected numbers. The +prompt is a fresh unique marker per run, keeping cached tokens out of the math. +The response's own `service_tier` echo is asserted first: if OpenAI ever declined +priority processing and served the default tier, the test fails there instead of +producing a vacuous rate comparison. Reasoning is requested explicitly with +`reasoning_effort`, so the reasoning-rate assertion rests on a parameter the test +sets rather than on whatever the model happens to do by default. +""" + +import pytest + +from cost_rows import ( + approx_equal, + assert_fresh_tokens_billed_at, + assert_total_is_sum_of_components, + poll_cost_row, + register_priced_model, +) +from e2e_config import unique_marker +from e2e_http import unwrap +from lifecycle import ResourceManager +from models import ChatBody, ChatMessage, LiteLLMParamsBody +from spend_e2e_client import SpendClient + +pytestmark = pytest.mark.e2e + +BACKEND = "openai/gpt-5.6-luna" +OPENAI_API_KEY = "os.environ/OPENAI_API_KEY" + +INPUT_RATE = 4e-05 +OUTPUT_RATE = 8e-05 +PRIORITY_INPUT_RATE = 6e-05 +PRIORITY_OUTPUT_RATE = 1.6e-04 + +REASONING_EFFORT = "high" + + +class TestServiceTierPricing: + @pytest.mark.covers("quota_management.spend_tracking.service_tier.bills_tier_rates") + def test_priority_tier_bills_priority_rates( + self, client: SpendClient, resources: ResourceManager, scoped_key: str + ) -> None: + model = register_priced_model( + client.proxy, + resources, + "tier-priced", + LiteLLMParamsBody( + model=BACKEND, + api_key=OPENAI_API_KEY, + input_cost_per_token=INPUT_RATE, + output_cost_per_token=OUTPUT_RATE, + input_cost_per_token_priority=PRIORITY_INPUT_RATE, + output_cost_per_token_priority=PRIORITY_OUTPUT_RATE, + ), + ) + + chat = unwrap( + client.proxy.chat( + scoped_key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content=( + f"{unique_marker()} Compute 47*83 - 19*7 step by step, " + "then reply with just the final number." + ), + ) + ], + max_completion_tokens=4000, + service_tier="priority", + reasoning_effort=REASONING_EFFORT, + ), + ) + ) + assert chat.service_tier == "priority", ( + f"OpenAI served tier {chat.service_tier!r} instead of priority; " + "tier billing was never exercised" + ) + assert chat.id, f"chat response carried no id: {chat}" + + row = poll_cost_row(client.proxy, chat.id) + assert row is not None, f"no spend row with a cost breakdown landed for {chat.id}" + breakdown = row.breakdown + + assert breakdown.service_tier == "priority", ( + f"the bill records pricing basis {breakdown.service_tier!r}, not priority" + ) + + assert_fresh_tokens_billed_at(row, PRIORITY_INPUT_RATE) + assert breakdown.output_cost is not None and approx_equal( + breakdown.output_cost, (row.completion_tokens or 0) * PRIORITY_OUTPUT_RATE + ), ( + f"output_cost {breakdown.output_cost} != {row.completion_tokens} tokens * priority rate " + f"{PRIORITY_OUTPUT_RATE} (base rate would give {(row.completion_tokens or 0) * OUTPUT_RATE})" + ) + + usage = chat.usage + assert usage is not None and usage.completion_tokens_details is not None, ( + f"no completion token details on the priority call: {chat}" + ) + reasoning_tokens = usage.completion_tokens_details.reasoning_tokens or 0 + assert reasoning_tokens > 0, f"the reasoning question produced no reasoning tokens: {usage}" + assert breakdown.reasoning_cost is not None and approx_equal( + breakdown.reasoning_cost, reasoning_tokens * PRIORITY_OUTPUT_RATE + ), ( + f"reasoning_cost {breakdown.reasoning_cost} != {reasoning_tokens} reasoning tokens * " + f"priority rate {PRIORITY_OUTPUT_RATE} (the default-tier rate would give " + f"{reasoning_tokens * OUTPUT_RATE})" + ) + + assert_total_is_sum_of_components(row)