diff --git a/config/litellm/.ruff.toml b/config/litellm/.ruff.toml new file mode 100644 index 000000000..2a5458af7 --- /dev/null +++ b/config/litellm/.ruff.toml @@ -0,0 +1,19 @@ +# Ruff settings for the egg-litellm image sources ONLY. +# +# Every other Python file in this repo runs on the repo's own interpreter +# (`requires-python = ">=3.14"`). These do not: `cost_callback.py` and the four +# modules `patch_litellm_cache.py` installs are baked into +# `ghcr.io/berriai/litellm:v1.86.2`, which ships **Python 3.11**. Formatting +# them for 3.14 emits syntax that image cannot import. +# +# That is not hypothetical. `ruff format` under `target-version = "py314"` +# rewrites `except (TypeError, ValueError):` to the PEP 758 unparenthesized +# form, which is a hard SyntaxError on 3.11 — and the formatter has no `noqa` +# escape, so the only place to say "these files target 3.11" is here. Ruff +# resolves settings per file by walking up from it, so this file governs +# exactly this directory. +# +# `extend` inherits the root select/ignore set, so the lint rules stay +# identical and only the language level differs. +extend = "../../pyproject.toml" +target-version = "py311" diff --git a/config/litellm/Dockerfile b/config/litellm/Dockerfile index 01c8c10e1..57dc2629f 100644 --- a/config/litellm/Dockerfile +++ b/config/litellm/Dockerfile @@ -4,9 +4,11 @@ # through this proxy: Claude Code -> gateway -> LiteLLM -> OpenRouter. The # stock image's Anthropic->OpenAI translation drops prompt-cache hits on # Qwen/DeepSeek, mis-streams reasoning models, silently drops params it does -# not recognise, and manufactures a reasoning ceiling from the caller's -# thinking budget, and never sends prior-turn reasoning back. -# patch_litellm_cache.py closes the ten gaps at build time +# not recognise, manufactures a reasoning ceiling from the caller's thinking +# budget, never sends prior-turn reasoning back, and discards the provider's +# own bill during stream reassembly while its rate card cannot price the route +# either. +# patch_litellm_cache.py closes the twelve gaps at build time # (see that script for the diagnosis); # cost_callback.py surfaces the resulting cost/cache stats into the pod log # stream. Both mirror the host-side dotfiles setup that took Qwen cache hit @@ -28,11 +30,12 @@ COPY patch_litellm_cache.py /egg/patch_litellm_cache.py # script installs into every litellm tree (see NEW_MODULES). Keeping them real # files means they stay lintable and testable in the egg repo rather than # living as string literals — which is why none of them imports litellm at -# module scope (tests/config/ imports all three directly). +# module scope (tests/config/ imports all five directly). COPY openrouter_capabilities.py /egg/openrouter_capabilities.py COPY drop_params_visibility.py /egg/drop_params_visibility.py COPY anthropic_thinking_policy.py /egg/anthropic_thinking_policy.py COPY openrouter_reasoning_roundtrip.py /egg/openrouter_reasoning_roundtrip.py +COPY stream_cost_preservation.py /egg/stream_cost_preservation.py RUN python3 /egg/patch_litellm_cache.py # Custom cost/cache logger, registered as `cost_callback.cost_logger` under diff --git a/config/litellm/cost_callback.py b/config/litellm/cost_callback.py index a2b3ccddb..49c1c6637 100644 --- a/config/litellm/cost_callback.py +++ b/config/litellm/cost_callback.py @@ -20,28 +20,40 @@ snapshot, because the per-turn ratio is noisy on short turns (a single tool-result message can dominate the prompt budget). -Cost on the streaming path is intentionally reported as ``null``, not 0. -Claude Code streams its ``/v1/messages`` requests, and LiteLLM reassembles -the streamed chunks via ``stream_chunk_builder`` -> ``ChunkProcessor. -calculate_usage``, which rebuilds a fresh ``Usage`` carrying only the -token/cache counts and DROPS the upstream provider's ``cost`` / -``cost_details``. So on real agent traffic the upstream-billed cost is not -recoverable at this seam, and we emit ``cost: null`` (per call and in the -session totals) rather than coercing the missing value to ``0.0`` — a -``0.0`` would read in the logs as "this route is free", the exact opposite -of the cost-visibility signal this module exists to provide (#2799). The -cache-read/write and token counts DO survive reassembly, so the -cache-hit-rate metric (the primary cq-6 signal) is unaffected. Real cost is -still captured on the non-streaming path, where ``original_response`` -carries the raw provider JSON with ``usage.cost``. - -Because the billed cost is therefore unknown on essentially every agent -call, each line also carries ``cost_estimated``: LiteLLM's own -``response_cost``, computed at logging time from the assembled usage and -its pricing map, which survives streaming (issue #3175). It is kept -strictly separate from ``cost`` — an estimate from a possibly-stale rate -card must never be mistaken for a bill — and follows the same null-not-zero -rule when LiteLLM cannot price the model. +Cost arrives by one of two routes depending on how the call streamed. +Non-streaming, ``original_response`` carries the raw provider JSON and +``usage.cost`` with it. Streaming — which is essentially all agent traffic, +since Claude Code streams its ``/v1/messages`` requests — LiteLLM +reassembles the chunks via ``stream_chunk_builder`` -> ``ChunkProcessor. +calculate_usage``, a rebuild that enumerates the token/cache counts and +originally DROPPED the provider's ``cost`` / ``cost_details`` outright. +That is why this module recorded ``cost: null`` on 1252 of 1252 sampled +calls in run 6. The egg-litellm image's **patch 11** now carries those two +fields across the rebuild (``config/litellm/stream_cost_preservation.py``), +so the billed figure reaches ``_extract_cost`` on the streaming path too +and this module needs no change to read it — the value simply stops being +absent (#3691). + +``cost: null`` therefore no longer means "streaming"; it means the cost was +genuinely unavailable — a stock (unpatched) LiteLLM under this callback, or +a provider that does not report one. It is still emitted as null rather +than ``0.0``: a zero would read in the logs as "this route is free", the +exact opposite of the cost-visibility signal this module exists to provide +(#2799). + +Each line also carries ``cost_estimated``: LiteLLM's own ``response_cost``, +computed at logging time from the assembled usage and its pricing map +(issue #3175). It is kept strictly separate from ``cost`` — an estimate +from a possibly-stale rate card must never be mistaken for a bill — and +follows the same null-not-zero rule when LiteLLM cannot price the model. +That was the case for every route egg uses until the image's **patch 12** +taught the model-info lookup to read OpenRouter's published rate card; it +remains the case for a model whose prompt-length surcharge lands on a +boundary or component LiteLLM's map has no slot for, which is declined whole +rather than translated in part (see ``openrouter_capabilities``). With +both patches in place the two fields are independent measurements of the +same turn, and a persistent gap between them is a signal in its own right: +a stale rate card, an unexpected provider, or a surcharge tier. Each line also carries ``request_params``: the decoding configuration that actually went upstream on that call (issue #3599). Repetition and @@ -149,11 +161,13 @@ def _coerce_usage(usage): def _usage_from_response_obj(response_obj): """Read ``usage`` off the assembled response object LiteLLM hands the - success hook. On the streaming path this is the reliable source for the - token/cache counts (the final usage chunk's counts are folded into - ``response_obj.usage`` by ``stream_chunk_builder``), but NOT for cost: - that reassembly rebuilds a fresh ``Usage`` and drops ``cost`` / - ``cost_details``, so ``_extract_cost`` returns None here on streaming.""" + success hook. On the streaming path this is the source for the token/cache + counts (the final usage chunk's counts are folded into + ``response_obj.usage`` by ``stream_chunk_builder``) and, on the egg-litellm + image, for cost as well: that reassembly rebuilds a fresh ``Usage`` and + stock drops ``cost`` / ``cost_details`` with it, which patch 11 restores. + Under a stock LiteLLM the counts still arrive and ``_extract_cost`` returns + None — see the module docstring.""" if response_obj is None: return None usage = getattr(response_obj, "usage", None) @@ -181,8 +195,8 @@ def _extract_cost(usage): provider, so fall back to ``cost_details.upstream_inference_cost`` (what the upstream provider will bill for the same request). Either way, the number we record matches real spend on that turn. Returns None when no - positive cost is present — notably on the streaming path, where LiteLLM's - chunk reassembly drops the upstream cost (see ``_usage_from_response_obj``). + positive cost is present — a provider that reports none, or a stock LiteLLM + whose chunk reassembly drops it (see ``_usage_from_response_obj``). Callers must treat None as "unknown", not "$0". ``_positive`` rejects non-finite values as well as non-positive ones: a @@ -303,14 +317,16 @@ def _extract_attribution(mcd): def _extract_estimated_cost(mcd): """LiteLLM's own computed cost for the call, as an *estimate*. - Unlike the upstream-billed ``cost`` (dropped by stream-chunk reassembly — - see the module docstring), ``response_cost`` is computed by LiteLLM's - logging layer from the assembled usage and its model pricing map, so it - survives the streaming path that carries essentially all agent traffic. - It is an estimate, not a bill: the pricing map may lag the provider's - rates or lack cache-discount entries for a model. Returns None — never - 0.0 — when LiteLLM couldn't price the call (model absent from the map), - mirroring the billed-cost "unknown ≠ free" discipline. + ``response_cost`` is computed by LiteLLM's logging layer from the assembled + usage and its model pricing map, independently of whether the provider + reported a bill. It is an estimate, not a bill: the pricing map may lag the + provider's rates or lack cache-discount entries for a model. Returns None — + never 0.0 — when LiteLLM couldn't price the call, mirroring the billed-cost + "unknown ≠ free" discipline. On the egg-litellm image patch 12 supplies + OpenRouter's published rates for slugs the bundled map does not carry, so a + None here now means a genuinely unpriceable model (an inexpressible + prompt-length surcharge, or a provider with no live card to read) rather + than the routine case it was. Reads the top-level ``response_cost`` first, then falls back to ``standard_logging_object.response_cost`` — the latter is LiteLLM's @@ -358,10 +374,12 @@ def _extract_model(mcd): # line by orders of magnitude and spill task text into a stream that is a # cost/observability sink, not a transcript sink. # -# ``stream`` is included because it is the reason ``cost`` reads null on a -# line (see the module docstring) — worth having next to the null rather than -# inferred. ``max_tokens`` and ``n`` are not sampling knobs but shape the -# generation, and are cheap to carry. +# ``stream`` is included because it selects which of the two paths the cost on +# this line came through (see the module docstring), and it was the reason +# ``cost`` read null on every line before patch 11 — worth having next to the +# number rather than inferred, and worth keeping now that the null case is rare +# enough to need explaining when it happens. ``max_tokens`` and ``n`` are not +# sampling knobs but shape the generation, and are cheap to carry. _REQUEST_PARAM_KEYS = ( "temperature", "top_p", @@ -729,13 +747,15 @@ def _record(self, mcd, response_obj): prompt, cached, cache_write, reasoning = _extract_cache_stats(usage) if cost is None and prompt == 0 and cached == 0: return - # ``cost`` stays None when the upstream cost is unrecoverable - # (the streaming path — see module docstring). We accumulate only - # known costs and count how many calls contributed one, so a - # session that never saw a real cost reports ``cost: null`` rather - # than a misleading ``0.0``. ``cost_estimated`` (LiteLLM's own - # pricing-map figure, which DOES survive streaming) follows the - # same discipline under its own counters. + # ``cost`` stays None when the provider reported no cost, or when + # a stock LiteLLM discarded it in reassembly (see module + # docstring). We accumulate only known costs and count how many + # calls contributed one, so a session that never saw a real cost + # reports ``cost: null`` rather than a misleading ``0.0``. + # ``cost_estimated`` (LiteLLM's own pricing-map figure) follows the + # same discipline under its own counters — the two counters are + # what make a partially-known session readable, since either field + # can be the one that is missing. sid = _extract_session_id(mcd) or "_no_session" model = _extract_model(mcd) attribution = _extract_attribution(mcd) @@ -783,7 +803,10 @@ def _record(self, mcd, response_obj): 2, ) # Report session cost as null until at least one call carried a - # known cost, so all-streaming sessions don't read as "$0 spent". + # known cost, so a session that never learned one doesn't read as + # "$0 spent". Note the session total is a sum over the calls that + # DID report — read it against ``cost_known_calls``/``calls``, not + # as the session's whole bill, whenever those two differ. # Counts (calls and token tallies) are integer-valued — emit them # as ``int`` so the log line reads ``cost_known_calls: 1`` rather # than ``1.0`` (the aggregate is held as float for uniform +=). diff --git a/config/litellm/openrouter_capabilities.py b/config/litellm/openrouter_capabilities.py index cd910c2aa..ddb9bb8b2 100644 --- a/config/litellm/openrouter_capabilities.py +++ b/config/litellm/openrouter_capabilities.py @@ -1,18 +1,60 @@ -"""Live capability lookup for OpenRouter models. - -LiteLLM decides which optional params a provider accepts by consulting the -bundled model-cost map. For OpenRouter that map is wrong by construction: -OpenRouter publishes new slugs continuously, the bundled map lags behind, and -``litellm.supports_reasoning`` answers ``False`` for anything it has not caught -up to yet. Because ``OpenrouterConfig.get_supported_openai_params`` uses that -answer as a bare gate, the failure is closed and silent: a ``reasoning_effort`` -set on a current model is discarded before the request body is built, with no -exception and (before the ``drop_params`` warning) no log line. - -OpenRouter publishes the authoritative answer itself. ``GET /api/v1/models`` -returns every model with a ``supported_parameters`` list and requires no API -key. This module reads that, caches it for the life of the process, and hands -callers a set of parameter names for a given slug. +"""Live capability and pricing lookup for OpenRouter models. + +LiteLLM decides which optional params a provider accepts, and what a call cost, +by consulting the bundled model-cost map. For OpenRouter that map is wrong by +construction: OpenRouter publishes new slugs continuously, the bundled map lags +behind, and it answers for neither question on a slug it has not caught up to. +Two distinct silent failures follow from the one root cause: + +* **Parameters.** ``litellm.supports_reasoning`` answers ``False`` for an + unmapped slug, and ``OpenrouterConfig.get_supported_openai_params`` uses that + answer as a bare gate, so the failure is closed: a ``reasoning_effort`` set + on a current model is discarded before the request body is built, with no + exception and (before the ``drop_params`` warning) no log line. +* **Pricing.** ``_get_model_info_helper`` raises "This model isn't mapped yet" + for an unmapped slug, so LiteLLM's own ``response_cost`` is never computed + and egg's ``cost_estimated`` reads null on every routed call (#3691). Every + OpenRouter slug egg routes is absent from the pinned 1.86.2 map, so this is + 100% of routed traffic, not an edge case. + +OpenRouter publishes the authoritative answer to both itself. ``GET +/api/v1/models`` returns every model with a ``supported_parameters`` list and a +``pricing`` block, and requires no API key. This module reads that once, caches +it for the life of the process, and hands callers either the parameter-name set +(``get_supported_parameters``) or a LiteLLM-shaped model-cost entry +(``get_model_cost_entry``) for a given slug. + +Two deliberate limits on the pricing half, both about not trading a known +unknown for a confident wrong number: + +* **Cost fields only.** The entry carries the rate card, ``litellm_provider`` + and ``mode`` — not context lengths, not ``supports_*`` flags. Registering a + model's capabilities through this door would change behaviour well beyond + cost: ``supports_reasoning: true`` alone makes stock + ``get_supported_openai_params`` admit ``thinking``, which Patch 2's notes + explain would forward an Anthropic-shaped block verbatim to a provider that + expects ``reasoning``. Patch 7 remains the only path by which a parameter + becomes admissible, and it admits exactly ``reasoning_effort``. +* **Tiered rate cards are translated only where LiteLLM can hold them.** + OpenRouter expresses a long-context surcharge as ``pricing.overrides`` — a + list keyed by an arbitrary ``min_prompt_tokens`` (32000 and 128000 on + qwen3-max, for instance). LiteLLM's model-info schema has *named* slots at + three fixed boundaries — ``*_above_128k_tokens``, ``*_above_200k_tokens``, + ``*_above_272k_tokens`` — and ``_get_model_info_helper`` builds its return by + enumerating those field names explicitly, so a tier at any other boundary is + not merely unlikely to survive, it is inexpressible. Slot coverage is also + uneven *per component*: there is no ``cache_read_input_token_cost`` slot at + 128k, and no ``cache_creation_input_token_cost`` slot at 128k or 272k. So the + rule is all-or-nothing per model: every published boundary must have slots, + and every priced component published in each override must have a slot at + that boundary, or the whole card is declined. Translating the tiers we *can* + hold and dropping a component we cannot would under-report the dropped one on + exactly the long-prompt turns the tier exists to charge for — the same silent + understatement that registering a base-tier-only entry would produce, which is + why neither is done. A declined model is left unpriced, its ``cost_estimated`` + stays null, and the reason is logged once. The provider-billed ``cost`` (see + ``stream_cost_preservation``) is the number to read for those models, and it + is exact. Design constraints, because this sits behind a hot, synchronous code path: @@ -33,7 +75,11 @@ Operator knobs (all optional; see ``docs/guides/per-agent-models.md``): * ``LITELLM_OPENROUTER_CAPABILITY_FETCH=0`` disables the lookup entirely and - restores the previous model-map-only behaviour. + restores the previous model-map-only behaviour — parameters and pricing + both, since one fetch serves both. +* ``LITELLM_OPENROUTER_PRICING=0`` disables only the pricing half, leaving the + parameter lookup running. For an operator who wants LiteLLM's bundled map to + be the sole authority on cost while keeping Patch 7's parameter fix. * ``LITELLM_OPENROUTER_CAPABILITY_TTL`` seconds between refreshes (default 3600). ``0`` disables caching and re-fetches on every lookup — a debugging aid, not a production setting. @@ -49,6 +95,7 @@ """ import json +import math import os import threading import time @@ -62,11 +109,18 @@ DEFAULT_TTL_SECONDS = 3600.0 DEFAULT_TIMEOUT_SECONDS = 5.0 -# Cache of slug -> supported parameter names. ``None`` means "not populated". -# An empty dict is a real, meaningful state: it records a failed fetch, so the -# negative cache below can suppress retries without conflating "we asked and -# got nothing" with "we never asked". -_CACHE: dict[str, set[str]] | None = None +# Cache of slug -> record. ``None`` means "not populated". An empty dict is a +# real, meaningful state: it records a failed fetch, so the negative cache below +# can suppress retries without conflating "we asked and got nothing" with "we +# never asked". +# +# Each record is ``{"id": str, "parameters": set[str] | None, "cost_entry": dict +# | None, "declined_thresholds": tuple[int, ...] | None}``. Both payloads are optional and +# independent: OpenRouter publishes entries carrying one and not the other, and +# a slug that answers for parameters but not pricing (or the reverse) is a real +# answer for the half it has rather than a reason to drop the model. The +# thresholds are kept only so the declined-pricing warning can name them. +_CACHE: dict[str, dict] | None = None _CACHE_STAMP: float = 0.0 _LOCK = threading.Lock() @@ -76,6 +130,12 @@ # blip at startup does not permanently mute a real outage hours later. _WARNED_FETCH_FAILURE = False +# Slugs whose tiered rate card has already been reported as declined. Bounded by +# the roster size, and cleared by every successful refetch — not only by +# ``reset_cache`` — so a pricing change upstream is reported against the roster +# it was read from rather than muted for the life of the pod. +_WARNED_DECLINED_PRICING: set[str] = set() + # Env-var complaints already emitted, keyed by ``(name, raw value)``. # ``_env_float`` is reached from ``_ttl_seconds`` on *every* lookup, ahead of # the freshness check, so an unconditional warning there is one WARNING line per @@ -213,7 +273,218 @@ def _ttl_seconds() -> float: return _env_float("LITELLM_OPENROUTER_CAPABILITY_TTL", DEFAULT_TTL_SECONDS, allow_zero=True) -def _fetch() -> dict[str, set[str]]: +# OpenRouter pricing key -> LiteLLM model-cost key. Every value is USD per +# token, published as a decimal *string*, which is why ``_price`` parses rather +# than casts. +# +# ``input_cache_write`` maps to LiteLLM's ``cache_creation_*`` spelling: the two +# name the same thing (what you pay to put a prefix in the cache), and getting +# this pair backwards would silently price cache writes at the read rate, which +# on these routes is a ~5x understatement of the turn that costs the most. +# +# The last two pairs were added after review: both are per-token rates LiteLLM +# actually consumes on the chat path (``generic_cost_per_token`` reads +# ``output_cost_per_reasoning_token`` for reasoning tokens, and +# ``calculate_cache_writing_cost`` reads the ``_above_1hr`` write rate for a 1h +# TTL block), so omitting them under-reported those components rather than +# leaving them unknown. +# +# Components OpenRouter publishes that are deliberately NOT mapped, because +# LiteLLM 1.86.2 has no per-token slot that its chat-path cost calculation +# reads: ``request`` (per-request, not per-token), ``web_search`` (per-request; +# ``search_context_cost_per_query`` is a responses-API surface, not a chat one), +# ``image`` / ``audio`` / ``input_audio_cache`` (modality rates that egg's routed +# traffic does not exercise), and ``discount``. A model whose bill is dominated +# by one of those will read low under ``cost_estimated``; the provider-billed +# ``cost`` remains exact. +_PRICE_KEYS = ( + ("prompt", "input_cost_per_token"), + ("completion", "output_cost_per_token"), + ("input_cache_read", "cache_read_input_token_cost"), + ("input_cache_write", "cache_creation_input_token_cost"), + ("input_cache_write_1h", "cache_creation_input_token_cost_above_1hr"), + ("internal_reasoning", "output_cost_per_reasoning_token"), +) + +# Reverse index, used by the tier translation to tell "a component we price and +# cannot express at this boundary" (fatal — it would under-report) from "a +# component we do not price at the base tier either" (already out of scope, and +# no more wrong at 200k than at 0). +_MAPPED_PRICE_KEYS = frozenset(published for published, _ in _PRICE_KEYS) + +# The two rates without which an entry cannot price a chat turn at all. A +# missing cache rate degrades the estimate; a missing prompt or completion rate +# would make it meaningless, so the entry is declined instead. +_REQUIRED_PRICE_KEYS = ("prompt", "completion") + +# OpenRouter ``min_prompt_tokens`` boundary -> the LiteLLM model-cost keys that +# exist at that boundary, per published component. Transcribed from the explicit +# field enumeration in ``_get_model_info_helper`` (litellm/utils.py, 1.86.2): +# a key absent from that enumeration is dropped on the way out of the lookup, so +# the gaps below are the real shape of the schema and not a conservative guess. +_TIER_SLOTS: dict[int, dict[str, str]] = { + 128000: { + "prompt": "input_cost_per_token_above_128k_tokens", + "completion": "output_cost_per_token_above_128k_tokens", + }, + 200000: { + "prompt": "input_cost_per_token_above_200k_tokens", + "completion": "output_cost_per_token_above_200k_tokens", + "input_cache_read": "cache_read_input_token_cost_above_200k_tokens", + "input_cache_write": "cache_creation_input_token_cost_above_200k_tokens", + }, + 272000: { + "prompt": "input_cost_per_token_above_272k_tokens", + "completion": "output_cost_per_token_above_272k_tokens", + "input_cache_read": "cache_read_input_token_cost_above_272k_tokens", + }, +} + + +def _price(raw: object) -> float | None: + """Parse one published rate. ``None`` when it is not a usable number. + + Zero is usable and is kept: a ``:free`` variant really is priced at zero, + and the resulting zero estimate is filtered by ``cost_callback``'s own + ``_positive`` gate rather than being invented here. Negative and non-finite + values are refused — neither is a rate, and an ``inf`` would propagate into + a session total and into the emitted JSON as a token that makes the line + unparseable. + """ + if isinstance(raw, bool) or raw is None: + return None + try: + value = float(raw) + except (TypeError, ValueError): + return None + if value < 0 or not math.isfinite(value): + return None + return value + + +def _boundary(raw: object) -> int | None: + """Parse one ``min_prompt_tokens`` value. ``None`` when it is not a count. + + Accepts the decimal-*string* spelling as well as the integer one. Every + other number in this payload is published as a string + (``"prompt": "0.0000012"``), so a schema that switched ``min_prompt_tokens`` + to ``"128000"`` for consistency would be entirely in character — and an + ``isinstance(raw, int)`` test would have read that as "unparseable", turning + an expressible tier into a declined model with a warning blaming + OpenRouter's schema. A float that is not a whole number is refused rather + than truncated: a fractional token boundary is not a thing, so it means the + field is not what this thinks it is. + """ + if isinstance(raw, bool) or raw is None: + return None + if isinstance(raw, int): + return raw + try: + value = float(raw) + except (TypeError, ValueError): + return None + if not math.isfinite(value) or value != int(value): + return None + return int(value) + + +def _tier_rates(overrides: list) -> tuple[dict[str, float] | None, tuple[int, ...]]: + """Translate ``pricing.overrides`` into LiteLLM tiered-rate keys. + + Returns ``(rates, thresholds)``. ``rates`` is None when the card cannot be + held faithfully — see the module docstring — and ``thresholds`` always names + whatever boundaries parsed, so the decline warning can be specific even when + the reason for declining was one of them failing to parse. + + All-or-nothing on purpose. Emitting the tiers that fit and dropping the rest + is the failure mode this whole path exists to avoid: LiteLLM applies at most + one boundary per call and falls back to the *base* rate for any component + with no key at that boundary, so a partial translation reports a surcharged + turn at the un-surcharged rate for the dropped component — silently, and + only on long prompts. + """ + parsed: list[tuple[int, dict]] = [] + readable = True + for override in overrides: + raw = override.get("min_prompt_tokens") if isinstance(override, dict) else None + boundary = _boundary(raw) + if boundary is None: + readable = False + continue + parsed.append((boundary, override)) + + thresholds = tuple(sorted(boundary for boundary, _ in parsed)) + if not readable or not parsed: + return None, thresholds + + rates: dict[str, float] = {} + for boundary, override in parsed: + slots = _TIER_SLOTS.get(boundary) + if slots is None: + return None, thresholds + # A surcharge with no ``prompt`` rate is unreachable, not merely + # incomplete: LiteLLM finds the applicable boundary by scanning for + # ``input_cost_per_token_above_*`` keys, so a tier that publishes only a + # completion surcharge would contribute a key nothing ever reads and + # bill the whole turn at base. + if _price(override.get("prompt")) is None: + return None, thresholds + for published, raw in override.items(): + if published == "min_prompt_tokens" or published not in _MAPPED_PRICE_KEYS: + # An unmapped component is out of scope at the base tier too, so + # declining the model over it would withhold a rate card that is + # no less complete above the boundary than below it. + continue + price = _price(raw) + slot = slots.get(published) + if price is None or slot is None: + return None, thresholds + rates[slot] = price + return rates, thresholds + + +def _cost_entry(model_id: str, pricing: object) -> tuple[dict | None, tuple[int, ...] | None]: + """Translate an OpenRouter ``pricing`` block to a LiteLLM model-cost entry. + + Returns ``(entry, declined_thresholds)``. ``entry`` is None when no faithful + translation exists; ``declined_thresholds`` is None unless the reason was a + tiered rate card LiteLLM cannot hold, in which case it names the + ``min_prompt_tokens`` boundaries — the module docstring has the reasoning. + + The two are separate returns rather than one truthiness test because a + tiered card whose boundaries do not parse is still a tiered card: an empty + tuple must keep meaning "declined for tiering, boundaries unknown", so the + operator still gets told why the model is unpriced instead of only the + models whose overrides happened to be well-formed. + + ``key`` is set to the OpenRouter slug rather than left to the caller: it is + what ``get_model_info`` reports as the entry's identity, and an operator + reading ``/model/info`` should see the slug the rate actually came from. + """ + if not isinstance(pricing, dict): + return None, None + + tier_rates: dict[str, float] = {} + overrides = pricing.get("overrides") + if isinstance(overrides, list) and overrides: + translated, thresholds = _tier_rates(overrides) + if translated is None: + return None, thresholds + tier_rates = translated + + if any(_price(pricing.get(key)) is None for key in _REQUIRED_PRICE_KEYS): + return None, None + + entry: dict = {"key": model_id, "litellm_provider": "openrouter", "mode": "chat"} + for published, litellm_key in _PRICE_KEYS: + value = _price(pricing.get(published)) + if value is not None: + entry[litellm_key] = value + entry.update(tier_rates) + return entry, None + + +def _fetch() -> dict[str, dict]: """Fetch the model list. Returns ``{}`` on any failure.""" global _WARNED_FETCH_FAILURE @@ -253,15 +524,32 @@ def _fetch() -> dict[str, set[str]]: _log_fetch_failure("response had no `data` list; falling back to the model-cost map") return {} - capabilities: dict[str, set[str]] = {} + capabilities: dict[str, dict] = {} for entry in entries: if not isinstance(entry, dict): continue model_id = entry.get("id") + if not isinstance(model_id, str) or not model_id: + continue params = entry.get("supported_parameters") - if not isinstance(model_id, str) or not isinstance(params, list): + parameters = {p for p in params if isinstance(p, str)} if isinstance(params, list) else None + cost_entry, declined = _cost_entry(model_id, entry.get("pricing")) + # An entry that answers neither question carries no information, and + # keeping it would make a roster of such entries look like a successful + # fetch to the "no usable entries" check below. A declined rate card is + # an answer — "we saw this model and will not price it" — so it keeps + # the record alive for the warning. + if parameters is None and cost_entry is None and declined is None: continue - capabilities[model_id] = {p for p in params if isinstance(p, str)} + capabilities[model_id] = { + # The published slug, not the spelling the caller looked up: a + # record reached via an ``openrouter/``-prefixed or ``:free``-suffixed + # candidate must still name itself the way OpenRouter does. + "id": model_id, + "parameters": parameters, + "cost_entry": cost_entry, + "declined_thresholds": declined, + } if not capabilities: # A 200 whose `data` list is empty, or every entry of which is @@ -281,6 +569,12 @@ def _fetch() -> dict[str, set[str]]: # silencing exactly the case _log_fetch_failure exists to surface — but # re-arming on an empty parse would report a recovery that did not happen. _WARNED_FETCH_FAILURE = False + # Re-arm the per-slug decline warning against the roster it was derived + # from. Without this the latch outlives every refetch, so a model that + # dropped its surcharge tiers — or acquired them — reports the change once + # per pod lifetime at most, which for a long-lived proxy means never. The + # roster this replaces is the only thing the old keys described. + _WARNED_DECLINED_PRICING.clear() return capabilities @@ -304,7 +598,7 @@ def _log_fetch_failure(message: str, *args: object) -> None: _WARNED_FETCH_FAILURE = True -def _get_cache() -> dict[str, set[str]]: +def _get_cache() -> dict[str, dict]: global _CACHE, _CACHE_STAMP ttl = _ttl_seconds() @@ -335,7 +629,7 @@ def _get_cache() -> dict[str, set[str]]: _LOCK.release() -def _candidate_slugs(model: str) -> list[str]: +def _candidate_slugs(model: str, *, strip_variant: bool = True) -> list[str]: """Spellings of ``model`` that may appear as an OpenRouter model id. Callers reach this from several directions: a bare slug @@ -343,6 +637,19 @@ def _candidate_slugs(model: str) -> list[str]: from a ``litellm_params.model``), or a slug carrying an OpenRouter variant suffix (``qwen/qwen3-max:free``). The ids returned by the API are bare slugs, with ``:free`` published as its own id. + + ``strip_variant=False`` drops the base-slug fallback for a ``:``-bearing + model. The two halves want different answers here. For parameters the + fallback is safe because the lookup is union-only and never subtractive: a + variant accepts at least what its base does, so inheriting the base's list + can admit a parameter, never withdraw one. For pricing it is not, because + the variant suffix is frequently *what changes the rate* — a ``:free`` + variant is 0 against a paid base, and a ``:batch`` variant is half its base + on every model that publishes one — so inheriting would report a confident, + authoritative-looking number that is wrong by construction, which is the one + outcome this module refuses everywhere else. + Stripping the ``openrouter/`` prefix is kept for both: that names the same + model, not a different rate card. """ candidates: list[str] = [] @@ -355,17 +662,59 @@ def add(value: str) -> None: add(model[len("openrouter/") :]) # Variant suffixes (:free, :nitro, :floor, ...) are sometimes their own id # and sometimes only a routing hint on the base model, so try both. - for candidate in list(candidates): - if ":" in candidate: - add(candidate.split(":", 1)[0]) + if strip_variant: + for candidate in list(candidates): + if ":" in candidate: + add(candidate.split(":", 1)[0]) return candidates +def _lookup(model: str, field: str, *, strip_variant: bool = True) -> dict | None: + """The published record answering for ``field``, or None for no opinion. + + "No opinion" covers every way the answer can be unknown: the lookup is + disabled, the fetch failed, or no candidate spelling is in the published + list. + + ``field`` is taken rather than assumed because a record can answer one half + and not the other: OpenRouter publishes entries with a rate card and no + ``supported_parameters``, and stopping at the first record found would let + such an entry shadow a later candidate that does answer. Before this + module's pricing half existed, ``_fetch`` dropped those entries outright and + the candidate loop fell through them by accident; keeping them for their + pricing turned that accident into a silent regression of the parameter fix + — ``reasoning_effort`` dropped for a variant slug whose entry happens to + carry only rates. So candidates that cannot answer are skipped, and the + first record seen is still returned as a fallback so + ``get_model_cost_entry`` can report *why* a model it has definitely seen is + going unpriced. + """ + if not _env_flag("LITELLM_OPENROUTER_CAPABILITY_FETCH", True): + return None + if not model: + return None + + cache = _get_cache() + if not cache: + return None + + fallback: dict | None = None + for candidate in _candidate_slugs(model, strip_variant=strip_variant): + record = cache.get(candidate) + if record is None: + continue + if record.get(field) is not None: + return record + if fallback is None: + fallback = record + return fallback + + def get_supported_parameters(model: str) -> set[str] | None: """Parameter names OpenRouter advertises for ``model``. - Returns ``None`` when the answer is unknown for any reason: the lookup is - disabled, the fetch failed, or the slug is not in the published list. A + Returns ``None`` when the answer is unknown for any reason (see ``_lookup``) + or when the roster entry carried no ``supported_parameters`` list. A ``None`` return means "no opinion" and callers must fall back to whatever they did before. @@ -373,22 +722,107 @@ def get_supported_parameters(model: str) -> set[str] | None: TTL, so handing out the stored object would let one caller's ``add`` change what every later caller sees. """ - if not _env_flag("LITELLM_OPENROUTER_CAPABILITY_FETCH", True): + record = _lookup(model, "parameters") + if record is None: return None - if not model: + params = record.get("parameters") + return set(params) if params is not None else None + + +def get_model_cost_entry(model: str, custom_llm_provider: str | None = None) -> dict | None: + """A LiteLLM-shaped model-cost entry for ``model``, or None (#3691). + + None means "no opinion", exactly as in ``get_supported_parameters``, and the + caller must fall back to whatever it did before — which for + ``_get_model_info_helper`` is the stock "This model isn't mapped yet" + ValueError. There are four ways to get it: the lookup is off, the pricing + half specifically is off, the slug is unknown, or its rate card is tiered at + a boundary LiteLLM cannot hold (see the module docstring; that case is + warned about once per slug, because unlike the others it is a model egg + *can* see and still will not price). + + Unlike the parameter half, this does **not** fall back from a variant slug + to its base — ``_candidate_slugs``' docstring has the reasoning. So a + ``:free`` or ``:batch`` route that OpenRouter has retired from the roster + reads null here rather than inheriting the base model's paid rate. + ``entry["key"]`` is therefore always the slug the rates were published + under, and ``/model/info`` can be read as naming the real source. + + ``custom_llm_provider`` is accepted and checked rather than ignored: this + module speaks only for OpenRouter, and the call site sits on a generic + lookup that every provider reaches. Answering there for, say, a Bedrock slug + that happens to share a name would attach OpenRouter's rate card to someone + else's bill. None is permitted because the caller resolves the provider from + the model string, and a bare ``qwen/qwen3-max`` legitimately arrives + unattributed. + + The returned dict is a copy: the cache is process-wide, and LiteLLM's + model-info path is free to mutate what it is handed. + + Note on freshness: LiteLLM memoizes ``_get_model_info_helper`` behind an + ``lru_cache``, so the FIRST successful answer for a slug is what the + process uses until it restarts — this module's TTL governs how often the + roster is re-read, not how often a priced model is re-priced. That is the + same staleness every entry in the bundled map already has, and rates move + on a scale where it does not matter; the provider-billed ``cost`` is + unaffected either way. ``lru_cache`` does not memoize exceptions, so a + lookup that failed while the roster was unreachable is retried rather than + latched. + """ + if custom_llm_provider is not None and custom_llm_provider != "openrouter": + return None + if not _env_flag("LITELLM_OPENROUTER_PRICING", True): return None - cache = _get_cache() - if not cache: + record = _lookup(model, "cost_entry", strip_variant=False) + if record is None: return None - for candidate in _candidate_slugs(model): - params = cache.get(candidate) - if params is not None: - return set(params) + entry = record.get("cost_entry") + if entry is not None: + return dict(entry) + + declined = record.get("declined_thresholds") + if declined is not None: + _warn_declined_pricing(record, declined) return None +def _warn_declined_pricing(record: dict, thresholds: tuple[int, ...]) -> None: + """Report a tiered rate card we will not translate, once per slug. + + Warn-level and once: an operator looking at a null ``cost_estimated`` for a + model that plainly *has* a published price needs to find this, and this runs + on the per-call cost path, so repeating it would bury the very log stream + the cost figures land in. + """ + slug = record.get("id") or "" + if slug in _WARNED_DECLINED_PRICING: + return + where = ( + f"tiers at {', '.join(str(t) for t in thresholds)} tokens" + if thresholds + else "tier boundaries unparseable" + ) + # Deliberately says "this card", not "tiered cards": most of them are now + # translated (see ``_TIER_SLOTS``), so a message asserting that LiteLLM + # cannot express tier boundaries would be false in the general case and + # would send an operator looking for a limit that is not the one they hit. + # The boundaries are named so the specific reason is checkable against + # ``_TIER_SLOTS`` without reading the roster. + if _log( + "warning", + "openrouter capabilities: %s prices by prompt length (%s) in a shape LiteLLM " + "cannot hold — it has rate slots only at 128000/200000/272000 tokens, and not " + "for every component at each. cost_estimated stays null for this model rather " + "than under-reporting long prompts. Read the provider-billed `cost` field " + "instead; it is exact.", + slug, + where, + ): + _WARNED_DECLINED_PRICING.add(slug) + + def reset_cache() -> None: """Drop cached capability data. Intended for tests.""" global _CACHE, _CACHE_STAMP, _WARNED_FETCH_FAILURE @@ -397,3 +831,4 @@ def reset_cache() -> None: _CACHE_STAMP = 0.0 _WARNED_FETCH_FAILURE = False _WARNED_ENV.clear() + _WARNED_DECLINED_PRICING.clear() diff --git a/config/litellm/patch_litellm_cache.py b/config/litellm/patch_litellm_cache.py index 9043caff8..25603133f 100644 --- a/config/litellm/patch_litellm_cache.py +++ b/config/litellm/patch_litellm_cache.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Apply egg's LiteLLM prompt-cache and reasoning-stream patches at image-build time. +"""Apply egg's LiteLLM prompt-cache, reasoning-stream and cost patches at image-build time. LiteLLM's stock Anthropic->OpenAI translation (the path Claude Code's ``/v1/messages`` requests take when routed at a non-Claude OpenRouter @@ -7,10 +7,12 @@ reasoning models, and its OpenRouter param gate reads a model-cost map that does not carry current OpenRouter slugs, and it discards caller-specified params in total silence, while manufacturing a -reasoning ceiling nobody asked for and never sending prior-turn -reasoning back. Ten independent gaps cause it; this -script closes all ten by editing the installed ``litellm`` -package in place (and installing four new modules), then +reasoning ceiling nobody asked for, never sending prior-turn reasoning +back, and it destroys the provider's own bill during stream reassembly +while its rate card cannot price the route either. Twelve independent +gaps cause it; this script closes all twelve by editing the installed +``litellm`` +package in place (and installing five new modules), then ``config/litellm/Dockerfile`` bakes the result into the ``egg-litellm`` image. @@ -201,6 +203,37 @@ upstream's and predates every egg and fork change: jwbron/litellm#8 touches only ``get_supported_openai_params``, which acts on ``optional_params`` and cannot reach a message field. + 11. ``ChunkProcessor.calculate_usage`` + (litellm_core_utils/streaming_chunk_builder_utils.py) carry the + provider-billed ``cost`` / ``cost_details`` across stream + reassembly. OpenRouter reports what it charged on the final usage + chunk and stock already asks for it (``transform_request`` sets + ``usage: {"include": true}`` unconditionally), but this rebuild + enumerates token counts only and re-constructs ``Usage`` from its + own ``model_dump()``, so the bill is dropped. Claude Code streams + every ``/v1/messages`` request, so this is ~100% of routed traffic: + 1252 of 1252 sampled ``cost_callback`` lines on run 6 carried + ``cost: null`` (#3691). The companion module + ``litellm_core_utils/_egg_stream_cost.py`` transports the two + fields and interprets neither — a zero ``cost`` is the BYOK truth + and its fall-through partner sits under ``cost_details``. + 12. ``_get_model_info_helper`` (utils.py) price OpenRouter slugs the + bundled map has never heard of. Same root cause as 7, second + symptom: ``model_prices_and_context_window.json`` does not carry + current slugs, so the lookup raises "This model isn't mapped yet", + LiteLLM's ``response_cost`` is never computed, and egg's + ``cost_estimated`` reads null beside the null ``cost`` that 11 + fixes. The hook sits at the raise site, after every stock lookup + has failed, so a mapped slug keeps the bundled answer and the live + card can add a model but never reprice one. ``_egg_capabilities`` + grows a second entry point for this off the roster it already + caches; it answers for OpenRouter alone, carries cost fields only + (a ``supports_*`` flag through this door would change parameter + admission, which is 7's job), and translates a prompt-length + surcharge only into the rate slots LiteLLM actually has, declining + the whole card when one does not fit rather than registering a base + tier that would silently under-report the long prompts agent traffic + is made of. Idempotent: each patch detects whether it is already applied. Fails loudly (non-zero exit) if a needle is missing, so a LiteLLM version bump @@ -307,6 +340,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - F2 = "llms/anthropic/experimental_pass_through/adapters/transformation.py" F3 = "llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py" F4 = "utils.py" +F5 = "litellm_core_utils/streaming_chunk_builder_utils.py" # Every patch as a self-contained spec: (file, present marker, needle, # replacement, label). Module-level so tests can apply them to a checked-in @@ -329,7 +363,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - ' QWEN = "qwen"\n' ' DEEPSEEK = "deepseek"\n' ), - "label": "Patch 1/10 (CacheControlSupportedModels)", + "label": "Patch 1/12 (CacheControlSupportedModels)", }, # Patch 2 — broaden ONLY the cache_control gate (not the shared # is_anthropic_claude_model predicate, which also gates thinking @@ -359,7 +393,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " )\n" " ):\n" ), - "label": "Patch 2/10 (cache_control gate)", + "label": "Patch 2/12 (cache_control gate)", }, # Patch 3 — drop x-anthropic-billing-header during Anthropic->OpenAI translation. { @@ -393,7 +427,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - ' "text": text,\n' " }\n" ), - "label": "Patch 3/10 (x-anthropic-billing-header filter)", + "label": "Patch 3/12 (x-anthropic-billing-header filter)", }, # Patch 4 — OpenRouter-style reasoning_content must open a thinking # content block, not fall through to a text block. The bare @@ -431,7 +465,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - ' choice.delta, "thinking_blocks"\n' " ):\n" ), - "label": "Patch 4/10 (reasoning_content thinking block)", + "label": "Patch 4/12 (reasoning_content thinking block)", }, # Patch 5a — sync __next__: don't drop the first delta on text or # thinking block transitions. @@ -531,7 +565,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " ):\n" " self.chunk_queue.append(processed_chunk)\n" ), - "label": "Patch 5a/10 (sync first-delta requeue)", + "label": "Patch 5a/12 (sync first-delta requeue)", }, # Patch 5b — async __anext__: same first-delta preservation. { @@ -629,7 +663,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " ):\n" " self.chunk_queue.append(processed_chunk)\n" ), - "label": "Patch 5b/10 (async first-delta requeue)", + "label": "Patch 5b/12 (async first-delta requeue)", }, # Patch 6 — streamed usage must report provider-automatic cache hits. # The needle spans the whole usage-merge region so both edit points @@ -721,7 +755,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " elif cached_tokens > 0:\n" ' usage_dict["cache_read_input_tokens"] = cached_tokens\n' ), - "label": "Patch 6/10 (streaming cache_read fallback)", + "label": "Patch 6/12 (streaming cache_read fallback)", }, # Patch 7 — OpenrouterConfig.get_supported_openai_params: consult # OpenRouter's published capabilities instead of only the bundled @@ -792,7 +826,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " pass\n" " try:\n" ), - "label": "Patch 7/10 (openrouter live capabilities)", + "label": "Patch 7/12 (openrouter live capabilities)", }, # Patch 8 — get_optional_params: log what ``drop_params`` discards. # @@ -841,7 +875,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " for k in unsupported_params.keys():\n" " non_default_params.pop(k, None)\n" ), - "label": "Patch 8/10 (drop_params visibility)", + "label": "Patch 8/12 (drop_params visibility)", }, # Patch 9 — _translate_thinking_to_openai: stop synthesizing # ``reasoning_effort`` from the caller's ``thinking`` block for non-Claude @@ -934,7 +968,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - "\n" ' summary = thinking.get("summary") if isinstance(thinking, dict) else None\n' ), - "label": "Patch 9/10 (thinking->reasoning_effort synthesis gate)", + "label": "Patch 9/12 (thinking->reasoning_effort synthesis gate)", }, # Patch 10 — OpenrouterConfig.transform_request: carry prior-turn assistant # reasoning back to the provider. @@ -1015,7 +1049,167 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - "\n" ' extra_body = optional_params.pop("extra_body", {})\n' ), - "label": "Patch 10/10 (assistant reasoning round-trip)", + "label": "Patch 10/12 (assistant reasoning round-trip)", + }, + # Patch 11 — ChunkProcessor.calculate_usage: carry the provider-billed cost + # across stream reassembly. + # + # OpenRouter reports what it charged on the final streamed usage chunk, and + # stock ``OpenrouterConfig.transform_request`` already asks for it + # (``usage: {"include": true}`` on every request). The number reaches + # litellm intact: ``chunk_parser`` hands the raw block to + # ``ModelResponseStream``, whose ``Usage`` keeps ``cost`` as a declared + # field and ``cost_details`` as a pydantic extra. Then ``calculate_usage`` + # rebuilds a fresh ``Usage`` field-by-field over the counts it enumerates + # and re-constructs it from its own ``model_dump()`` — and the bill is gone. + # + # Claude Code streams every /v1/messages request, so this seam is on ~100% + # of egg's routed traffic: run 6 sampled 1252 cost_callback lines and 1252 + # carried ``cost: null`` (#3691). The non-streaming path was unaffected + # (``original_response`` there holds the raw provider JSON), which is why + # this read as a property of the route rather than as a transport bug. + # + # Placed AFTER the ``Usage(**model_dump())`` rebuild, not before: the + # constructor deletes a ``cost`` attribute it is handed as None, so setting + # it first would put the value somewhere the rebuild is entitled to discard. + # The companion module (``litellm_core_utils/_egg_stream_cost.py``, in + # NEW_MODULES) never overwrites a value litellm already carried, so a future + # release that fixes this upstream turns the patch into a no-op instead of a + # competing second opinion. + { + "file": F5, + "present": "# egg cost patch. Carry the provider-billed cost", + "needle": ( + " # Return a new usage object with the new values\n" + "\n" + " returned_usage = Usage(**returned_usage.model_dump())\n" + "\n" + " return returned_usage\n" + ), + "replacement": ( + " # Return a new usage object with the new values\n" + "\n" + " returned_usage = Usage(**returned_usage.model_dump())\n" + "\n" + " # egg cost patch. Carry the provider-billed cost across this\n" + " # rebuild — it enumerates token counts only, so `cost` /\n" + " # `cost_details` (what OpenRouter actually charged) are dropped\n" + " # here on every streamed call. See patch 11 notes in\n" + " # patch_litellm_cache.py.\n" + " try:\n" + " from litellm.litellm_core_utils._egg_stream_cost import (\n" + " carry_upstream_cost as _egg_carry_upstream_cost,\n" + " )\n" + "\n" + " returned_usage = _egg_carry_upstream_cost(chunks, returned_usage)\n" + " except Exception as _egg_exc:\n" + " # Swallowed, because a cost figure must never break a\n" + " # response — but not silently. A failed import here looks\n" + " # from the outside exactly like `the provider reported no\n" + " # cost`, which is the symptom this patch exists to remove.\n" + " # Warned once per process, and the latch is set only after\n" + " # the emit succeeded; verbose_logger is imported here rather\n" + " # than read from module scope because this file does not\n" + " # import it.\n" + " try:\n" + " if not globals().get('_egg_warned_stream_cost'):\n" + " from litellm._logging import verbose_logger\n" + "\n" + " verbose_logger.warning(\n" + " 'egg cost patch: streamed cost preservation is '\n" + " 'inactive (%s: %s); the provider-billed `cost` will '\n" + " 'read null on every streamed call.',\n" + " type(_egg_exc).__name__,\n" + " _egg_exc,\n" + " )\n" + " globals()['_egg_warned_stream_cost'] = True\n" + " except Exception:\n" + " pass\n" + "\n" + " return returned_usage\n" + ), + "label": "Patch 11/12 (streamed cost preservation)", + }, + # Patch 12 — _get_model_info_helper: price OpenRouter slugs the bundled map + # has never heard of. + # + # Same root cause as patch 7, second symptom. LiteLLM's own + # ``response_cost`` is computed from ``model_prices_and_context_window.json``, + # which does not carry current OpenRouter slugs, so the lookup raises "This + # model isn't mapped yet" and egg's ``cost_estimated`` reads null on every + # routed call (#3691). Patch 11 recovers the *billed* figure; this one + # restores the independent estimate beside it, which is what remains + # readable if a provider ever stops reporting cost. + # + # Placed at the raise site, so it is reached only once every stock lookup + # has failed: a slug the bundled map DOES carry keeps the bundled answer, + # and the live rate card can add a model but never reprice one. The + # companion module answers for OpenRouter alone (it checks + # ``custom_llm_provider``) and holds a prompt-length surcharge only in the + # rate slots LiteLLM has, declining the whole card when a published boundary + # or component does not fit rather than registering rates that would + # under-report long prompts — see ``openrouter_capabilities``. + # + # NEEDLE DISAMBIGUATION: utils.py carries the "isn't mapped yet" string + # twice. The other one (~line 5999) is the outer handler's re-raise, with a + # different message body and indentation; the needle pins the ValueError + # form together with the ``if _model_info is None or key is None:`` guard + # that precedes only this one. + { + "file": F4, + "present": "# egg pricing patch. Consult OpenRouter's published rate card", + "needle": ( + " if _model_info is None or key is None:\n" + " raise ValueError(\n" + " \"This model isn't mapped yet. Add it here - " + 'https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"\n' + " )\n" + ), + "replacement": ( + " if _model_info is None or key is None:\n" + " # egg pricing patch. Consult OpenRouter's published rate card\n" + " # before giving up — the bundled map lags its slugs by\n" + " # construction, so every route egg uses lands here and every\n" + " # routed call reports a null cost estimate. Cost fields only;\n" + " # see patch 12 notes in patch_litellm_cache.py.\n" + " try:\n" + " from litellm.llms.openrouter._egg_capabilities import (\n" + " get_model_cost_entry as _egg_openrouter_cost_entry,\n" + " )\n" + "\n" + " _egg_entry = _egg_openrouter_cost_entry(\n" + " model, custom_llm_provider\n" + " )\n" + " except Exception as _egg_exc:\n" + " _egg_entry = None\n" + " # Same reasoning as patch 11's handler: swallowed so a\n" + " # rate-card lookup can never break a request, warned once\n" + " # so its absence is not indistinguishable from a slug the\n" + " # roster simply does not carry. verbose_logger is already\n" + " # module-scope in utils.py; the warning is still wrapped\n" + " # because raising from an except block would propagate.\n" + " try:\n" + " if not globals().get('_egg_warned_pricing'):\n" + " verbose_logger.warning(\n" + " 'egg pricing patch: OpenRouter rate-card '\n" + " 'lookup is inactive (%s: %s); cost_estimated '\n" + " 'will read null for unmapped slugs.',\n" + " type(_egg_exc).__name__,\n" + " _egg_exc,\n" + " )\n" + " globals()['_egg_warned_pricing'] = True\n" + " except Exception:\n" + " pass\n" + " if _egg_entry is not None:\n" + " _model_info = _egg_entry\n" + ' key = _egg_entry.get("key") or model\n' + " if _model_info is None or key is None:\n" + " raise ValueError(\n" + " \"This model isn't mapped yet. Add it here - " + 'https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"\n' + " )\n" + ), + "label": "Patch 12/12 (openrouter live pricing)", }, ] @@ -1044,22 +1238,27 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - { "source": "openrouter_capabilities.py", "dest": "llms/openrouter/_egg_capabilities.py", - "label": "Module 1/4 (openrouter capabilities)", + "label": "Module 1/5 (openrouter capabilities + pricing)", }, { "source": "drop_params_visibility.py", "dest": "_egg_drop_params_visibility.py", - "label": "Module 2/4 (drop_params visibility)", + "label": "Module 2/5 (drop_params visibility)", }, { "source": "anthropic_thinking_policy.py", "dest": "_egg_anthropic_thinking_policy.py", - "label": "Module 3/4 (thinking synthesis policy)", + "label": "Module 3/5 (thinking synthesis policy)", }, { "source": "openrouter_reasoning_roundtrip.py", "dest": "llms/openrouter/_egg_reasoning_roundtrip.py", - "label": "Module 4/4 (openrouter reasoning round-trip)", + "label": "Module 4/5 (openrouter reasoning round-trip)", + }, + { + "source": "stream_cost_preservation.py", + "dest": "litellm_core_utils/_egg_stream_cost.py", + "label": "Module 5/5 (streamed cost preservation)", }, ] diff --git a/config/litellm/stream_cost_preservation.py b/config/litellm/stream_cost_preservation.py new file mode 100644 index 000000000..0d071fa5a --- /dev/null +++ b/config/litellm/stream_cost_preservation.py @@ -0,0 +1,150 @@ +"""Carry the provider's billed cost through LiteLLM's stream reassembly. + +LiteLLM rebuilds a single ``Usage`` from the streamed chunks in +``ChunkProcessor.calculate_usage``. That rebuild is field-by-field over the +counts it knows about — prompt/completion tokens, the Anthropic cache +read/write pair, the token-detail wrappers — and it ends by re-constructing +the object from its own ``model_dump()``. Anything the provider attached that +LiteLLM did not enumerate is gone at that seam. + +For OpenRouter that "anything" is the bill. ``usage.cost`` is what OpenRouter +charges for the turn, and ``cost_details.upstream_inference_cost`` is what the +upstream provider charges under BYOK (where the former is 0 because billing +routes past OpenRouter). Both arrive on the final streamed chunk: stock +``OpenrouterConfig.transform_request`` already sets ``usage: {"include": true}`` +on every request, and ``OpenRouterChatCompletionStreamingHandler.chunk_parser`` +hands the raw block to ``ModelResponseStream``, whose ``Usage`` constructor +keeps ``cost`` as a declared field and ``cost_details`` as a pydantic extra. So +the number is present, correct, and one function call from the logger that +wants it — and then dropped. + +The consequence, measured on egg's run 6 (#3691): 1252 of 1252 sampled +``cost_callback`` lines carried ``cost: null``. Claude Code streams every +``/v1/messages`` request, so this seam is on ~100% of agent traffic, and egg +had no dollar figure at all for its LLM spend. The non-streaming path was +unaffected — ``original_response`` there carries the raw provider JSON — which +is why the module read as "cost is unavailable on this route" rather than as a +transport bug. + +This module is the transport half of the fix and nothing more. It copies the +two fields verbatim onto the reassembled usage and leaves every judgement to +the reader: + +* **A zero ``cost`` is copied, not skipped.** Under BYOK that zero is the + literal truth about the OpenRouter bill, and the real number is next to it + under ``cost_details``. ``cost_callback._extract_cost`` is the component that + knows to fall through from one to the other; a "positive only" filter here + would delete the evidence that the fall-through is the right reading. +* **Non-finite values are refused.** ``NaN``/``Inf`` on a cost field is not a + measurement, and downstream it is worse than absent: it accumulates into + egg's per-session total and poisons it for the pod's lifetime, and + ``json.dumps`` renders it as a non-standard token that makes the whole log + line invalid JSON. Same guard, same reason, as ``_finite_number`` there. +* **An existing value is never overwritten.** If a future LiteLLM starts + carrying cost through reassembly itself, its answer wins and this becomes a + no-op rather than a silent second opinion. + +Fails soft in the strictest sense: every path is wrapped, and any error leaves +the reassembled usage exactly as LiteLLM built it. A cost figure is +observability; it must never be able to break a response. + +Installed into every litellm tree as +``litellm_core_utils/_egg_stream_cost.py`` by +``config/litellm/patch_litellm_cache.py`` (Patch 11 is the call site). Kept as +a real file rather than a string literal in that script so it stays lintable +and unit-testable in the egg repo — which is why it imports no litellm symbols +at all and works structurally, off ``getattr``/``dict`` access. +""" + +import math + +# The provider-billed fields we carry across reassembly. ``cost`` is a declared +# field on litellm's ``Usage``; ``cost_details`` survives only as a pydantic +# extra, which is precisely why neither is enumerated by ``calculate_usage``. +COST_FIELD = "cost" +COST_DETAILS_FIELD = "cost_details" + + +def _usage_of(chunk): + """The usage block on one streamed chunk, or None. + + Mirrors the two sources ``ChunkProcessor._calculate_usage_per_chunk`` + reads, in the same order: the chunk's own ``usage`` (``ModelResponseStream`` + defines ``__contains__``/``__getitem__``, so the mapping-style access works + on the pydantic object as well as on a plain dict) and, failing that, the + ``usage`` stashed in ``_hidden_params``. Reading a different set of chunks + from the function whose output we are amending would make the cost and the + token counts capable of describing different turns. + """ + try: + if "usage" in chunk: + usage = chunk["usage"] + if usage is not None: + return usage + except Exception: # noqa: BLE001 - a chunk shape we don't understand is not an error + pass + try: + hidden = getattr(chunk, "_hidden_params", None) + if isinstance(hidden, dict): + return hidden.get("usage") + except Exception: # noqa: BLE001 + pass + return None + + +def _field_of(usage, name): + """Read ``name`` off a usage block that may be a dict or a pydantic model.""" + if isinstance(usage, dict): + return usage.get(name) + return getattr(usage, name, None) + + +def _finite_number(value): + """True for a real, finite number. + + ``bool`` is excluded because ``isinstance(True, int)`` is True and a + boolean is not a measurement: ``float(True)`` would record one dollar that + was never billed. + """ + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) + + +def _extract(chunks): + """Return ``(cost, cost_details)`` from the streamed chunks. + + Each is None when no chunk carried a usable value. The LAST usable value + wins: a provider that revises its usage block mid-stream is stating a + correction, and OpenRouter's single usage chunk makes the choice moot in + the case that actually runs. + """ + cost = None + cost_details = None + for chunk in chunks or []: + usage = _usage_of(chunk) + if usage is None: + continue + candidate = _field_of(usage, COST_FIELD) + if _finite_number(candidate): + cost = float(candidate) + candidate = _field_of(usage, COST_DETAILS_FIELD) + if isinstance(candidate, dict) and candidate: + cost_details = candidate + return cost, cost_details + + +def carry_upstream_cost(chunks, usage): + """Copy the provider-billed cost from ``chunks`` onto the rebuilt ``usage``. + + Returns ``usage`` so the call site can stay a single expression. Never + raises: the caller is on the response path, and every field this touches is + observability rather than payload. + """ + try: + cost, cost_details = _extract(chunks) + if cost is not None and not _finite_number(_field_of(usage, COST_FIELD)): + setattr(usage, COST_FIELD, cost) + if cost_details is not None and not isinstance(_field_of(usage, COST_DETAILS_FIELD), dict): + setattr(usage, COST_DETAILS_FIELD, cost_details) + except Exception: # noqa: BLE001 - a missing cost must never break a response + pass + return usage diff --git a/docs/development/STRUCTURE.md b/docs/development/STRUCTURE.md index d2e43a097..f139ee748 100644 --- a/docs/development/STRUCTURE.md +++ b/docs/development/STRUCTURE.md @@ -547,11 +547,13 @@ config/ ├── routing-policy.template.yaml # Operator template for the gateway's hot-reloadable model routing policy: switchover remaps + fallback chains (copy to ~/.config/egg/routing-policy.yaml) ├── litellm/ # egg-litellm image sources │ ├── Dockerfile # Builds egg-litellm: stock LiteLLM + prompt-cache and reasoning-parameter patches -│ ├── patch_litellm_cache.py # Build-time patches: cache_control passthrough on Qwen/DeepSeek routes, live OpenRouter capability lookup, drop_params visibility, no synthesized reasoning ceiling, prior-turn reasoning round-trip -│ ├── openrouter_capabilities.py # Patch 7: live GET /api/v1/models capability lookup, unioned with LiteLLM's bundled model-cost map +│ ├── .ruff.toml # Pins this directory to target-version = py311 — everything here runs on the litellm base image's interpreter, not the repo's 3.14, and `ruff format` under py314 emits PEP 758 syntax the image cannot import +│ ├── patch_litellm_cache.py # Build-time patches: cache_control passthrough on Qwen/DeepSeek routes, live OpenRouter capability + pricing lookup, drop_params visibility, no synthesized reasoning ceiling, prior-turn reasoning round-trip, streamed cost preservation +│ ├── openrouter_capabilities.py # Patches 7 + 12: live GET /api/v1/models lookup — advertised parameters (unioned with LiteLLM's bundled model-cost map) and the published rate card for slugs that map has never heard of, including prompt-length surcharges that fit LiteLLM's 128k/200k/272k rate slots │ ├── drop_params_visibility.py # Patch 8: warn once per proxy process per (provider, model, param-set) when drop_params discards a parameter │ ├── anthropic_thinking_policy.py # Patch 9: stop synthesizing a reasoning_effort ceiling from the caller's thinking budget on non-Claude models │ ├── openrouter_reasoning_roundtrip.py # Patch 10: map prior-turn assistant thinking_blocks onto reasoning_content so OpenRouter models that re-render prior thinking stop seeing empty +│ ├── stream_cost_preservation.py # Patch 11: carry the provider-billed cost / cost_details across LiteLLM's stream reassembly, which enumerates token counts only and drops them │ └── cost_callback.py # LiteLLM custom logger: upstream + estimated cost, per-role attribution (x-egg-* headers), cache hit rate, per-call decoding config -> pod stdout ├── redis/ # egg-redis image sources │ └── Dockerfile # Builds egg-redis: pinned stock Redis, repackaged for the local build/publish supply chain; backs the orchestrator's Redis Streams message store diff --git a/docs/guides/per-agent-models.md b/docs/guides/per-agent-models.md index 234261d22..f70e04757 100644 --- a/docs/guides/per-agent-models.md +++ b/docs/guides/per-agent-models.md @@ -593,22 +593,90 @@ data: > Claude Code prepends (the block's `cch=` hash invalidates the cache key > every turn). egg ships a custom **`egg-litellm`** image > ([`config/litellm/Dockerfile`](../../config/litellm/Dockerfile)) that -> bakes in ten patches closing those gaps and the reasoning-parameter ones -> below +> bakes in twelve patches closing those gaps, the reasoning ones below, +> and the cost-visibility ones after that > ([`config/litellm/patch_litellm_cache.py`](../../config/litellm/patch_litellm_cache.py)); > the build fails loudly if a LiteLLM bump moves the patched code. Pinning > the OpenRouter provider (`extra_body.provider.order` + `allow_fallbacks: > false`) then gives a stable cache surface — without it OpenRouter routes > across a cheapest-available pool whose cache support varies per turn. The -> bundled `cost_callback` logs the real upstream cost, LiteLLM's own -> pricing-map estimate (`cost_estimated`, which survives the streaming path -> where billed cost is unavailable), per-session cache hit rate, and -> per-role attribution (`pipeline_id`/`agent_role`/`phase` from the +> bundled `cost_callback` logs the provider-billed cost (`cost`), LiteLLM's +> own pricing-map estimate (`cost_estimated`), per-session cache hit rate, +> and per-role attribution (`pipeline_id`/`agent_role`/`phase` from the > gateway's `x-egg-*` headers — so per-role spend is a log query, not a > hand cross-reference of agent completion logs). Each call emits one JSON > line to the LiteLLM pod stream, visible via `get_service_logs` / the > structured-logging stream. +> **Where the dollar figures come from, and when they are null.** Both cost +> fields on a `cost_callback` line read null under a stock LiteLLM, for two +> unrelated reasons — which is why `egg-litellm` carries a patch for each +> (issue #3691; before them, 1252 of 1252 sampled calls on run 6 reported +> `cost: null`, so egg had no dollar figure at all for its LLM spend). +> +> - **Patch 11 — the bill survives streaming.** `cost` is what OpenRouter +> charged, reported on the final streamed usage chunk. No config change is +> needed to get it: stock LiteLLM's `OpenrouterConfig.transform_request` +> already sets `usage: {"include": true}` on every request, so the number +> was always arriving. LiteLLM's +> `stream_chunk_builder` rebuilds a fresh `Usage` from the token counts it +> enumerates and dropped `cost` / `cost_details` at that seam. Claude Code +> streams every `/v1/messages` request, so that was ~100% of routed +> traffic; the non-streaming path was never affected, which is why it read +> as a property of the route rather than as a transport bug. The patch +> copies the two fields across the rebuild and interprets neither: a `0` +> under BYOK is the literal truth about the OpenRouter bill, with the real +> number beside it under `cost_details.upstream_inference_cost`. +> - **Patch 12 — the estimate has a rate card.** `cost_estimated` is +> LiteLLM's own `response_cost`, computed from its bundled pricing map — +> which carries none of the slugs egg routes, exactly as it carries none of +> their `supported_parameters` (patch 7, same root cause). The patch reads +> OpenRouter's published rate card off the same `GET /api/v1/models` fetch +> patch 7 already makes, and hands it to the model-info lookup only after +> every bundled lookup has failed — so a mapped slug keeps its bundled +> rate, and the live card can add a model but never reprice one. +> `LITELLM_OPENROUTER_PRICING=0` turns off just this half. +> +> `cost_estimated` can still read null for a model that **prices by prompt +> length**. OpenRouter publishes a long-context surcharge as +> `pricing.overrides` keyed by an arbitrary `min_prompt_tokens`. LiteLLM has +> named rate slots at exactly three boundaries — 128000, 200000, 272000 — and +> its coverage is uneven *per component*: there is no cache-read slot at +> 128000 and no cache-write slot at 128000 or 272000. Patch 12 translates a +> surcharge when every published boundary and every priced component in it has +> a slot (about half the tiered models on the current roster, including the +> gpt-5.5 and grok-4.x families), and declines the **whole** card otherwise. +> Declining the whole card is the point: translating the components that fit +> and letting the rest fall back to the base rate would under-report the +> dropped component by 2-2.5x on precisely the long-prompt turns agent traffic +> is made of — silently, under a field name an operator would reasonably use +> to choose a model. `anthropic/claude-sonnet-4.5` is a live example (its 200k +> tier surcharges the 1h cache-write rate, which LiteLLM has no tiered slot +> for). A declined model is logged once, at `warning`, naming the boundaries. +> Read the provider-billed `cost` for those; it is exact. +> +> The estimate also omits components LiteLLM's chat cost path has no per-token +> slot for: `web_search` and `request` (both per-request), and the `image` / +> `audio` / `input_audio_cache` modality rates. A model whose bill is dominated +> by one of those reads low under `cost_estimated` — another reason to compare +> the two fields rather than trusting either alone. +> +> The two fields are independent measurements of the same turn, so read them +> together: a persistent gap between them is itself a signal (a stale rate +> card, an unexpected provider, or a surcharge tier). Neither is ever +> coerced to `0.0` when unknown — a zero would read in the logs as "this +> route is free", the exact inversion of the signal. The session totals sum +> only the calls that reported, so compare `cost_known_calls` / +> `cost_estimated_known_calls` against `calls` before treating a session +> total as the whole bill: +> +> ```bash +> kubectl logs -n egg-system deploy/litellm \ +> | jq -Rc 'fromjson? | select(.component == "cost_callback") | .context +> | {role: .agent_role, model, cost: .call.cost, +> est: .call.cost_estimated}' +> ``` + > **Reasoning depth on OpenRouter routes, and the five env vars that > control it.** Three of the baked-in patches decide what reasoning a > provider sees — two whether a reasoning *parameter* reaches it, one @@ -629,7 +697,10 @@ data: > `LITELLM_OPENROUTER_CAPABILITY_TIMEOUT` (default `5` seconds, per HTTP > phase) tune it. A fetch that fails is cached for the TTL too, so an > offline cluster costs one attempt per hour rather than one per request, -> and the first failure is logged at `warning`. +> and the first failure is logged at `warning`. Note `_FETCH=0` also +> disables patch 12's pricing lookup — one fetch serves both, so the master +> switch governs both; `LITELLM_OPENROUTER_PRICING=0` turns off only the +> pricing half. > - **Patch 9 — no synthesized reasoning ceiling.** On `/v1/messages` (the > route Claude Code uses) LiteLLM's Anthropic adapter converts each > request's `thinking: {budget_tokens: N}` into a bucketed diff --git a/k8s/base/litellm-deployment.yaml b/k8s/base/litellm-deployment.yaml index ec8145b19..52facf42c 100644 --- a/k8s/base/litellm-deployment.yaml +++ b/k8s/base/litellm-deployment.yaml @@ -112,6 +112,15 @@ spec: # - name: LITELLM_OPENROUTER_CAPABILITY_TIMEOUT # value: "5" # + # Patch 12 — live OpenRouter pricing, read off the same roster + # fetch as patch 7. ``0`` disables only the pricing half, leaving + # the capability lookup running: LiteLLM's bundled map becomes the + # sole authority on cost, and since it carries none of the slugs + # egg routes, ``cost_estimated`` goes back to null on every call. + # The provider-billed ``cost`` (patch 11) is unaffected by this. + # - name: LITELLM_OPENROUTER_PRICING + # value: "0" + # # Patch 9 — thinking -> reasoning_effort synthesis, off by # default. ``1`` restores stock LiteLLM: the Anthropic adapter # derives a bucketed ``reasoning_effort`` from each request's diff --git a/tests/config/test_cost_callback.py b/tests/config/test_cost_callback.py index a498e1c61..5e1e28768 100644 --- a/tests/config/test_cost_callback.py +++ b/tests/config/test_cost_callback.py @@ -6,12 +6,16 @@ stub the single symbol the module imports — ``CustomLogger`` — before loading it from its on-disk path. -The regression these tests lock in: on the streaming path (all real Claude -Code agent traffic) LiteLLM's chunk reassembly drops the upstream -``cost`` / ``cost_details`` while preserving the token/cache counts. The -callback must therefore report ``cost`` as ``null`` — never a misleading -``0.0`` that reads as "this route is free" — while still emitting working -cache stats. See the ``cost_callback`` module docstring for the full trace. +The regression these tests lock in is the callback's reading of cost, on both +sides of the #3691 fix. Stock LiteLLM's chunk reassembly drops the provider's +``cost`` / ``cost_details`` while preserving the token/cache counts, which put +``cost: null`` on 1252 of 1252 sampled calls; the egg-litellm image's patch 10 +carries them through, so a streamed call now arrives with the bill attached +and the callback reads it with no change of its own. Both states must work: +a cost that arrives is recorded, and one that does not is reported as ``null`` +— never a misleading ``0.0`` that reads as "this route is free" — with the +cache stats emitted either way. See the ``cost_callback`` module docstring for +the full trace. """ import importlib.util @@ -49,18 +53,21 @@ class _CustomLogger: # minimal stand-in; the module only subclasses it cc = _load_cost_callback() -def _streaming_usage() -> dict: - """The shape ``ChunkProcessor.calculate_usage`` produces on the streaming - path: token + cache fields present, ``cost`` / ``cost_details`` absent. +def _usage_without_cost() -> dict: + """Usage carrying token + cache fields but no cost of any kind. - NOTE: this fixture is hand-built to mirror ``calculate_usage``'s output in - the pinned ``litellm==1.86.2`` (the version baked into the egg-litellm - image). litellm can't run on the repo's Python 3.14, so the test can't - assert this shape against the real reassembler — revisit this fixture on a - litellm bump in case a newer version starts carrying ``cost`` through chunk - reassembly (which would make the ``cost: null`` behavior under test stale). - The build-time patcher already fails loudly on needle drift; this fixture - has no equivalent tripwire.""" + This is what stock ``ChunkProcessor.calculate_usage`` produces on the + streaming path, where the rebuild enumerates counts and discards the + provider's ``cost`` / ``cost_details`` (#3691). On the egg-litellm image + patch 10 carries them through, so this shape now stands for the residual + cases — an unpatched LiteLLM under this callback, or a provider that + reports no cost at all — rather than for streaming as such. Either way the + callback's contract is the same and is what these tests pin: unknown is + reported as ``null``, never coerced to ``0.0``. + + Hand-built rather than captured: litellm can't run on the repo's Python + 3.14. The build-time patcher fails loudly on needle drift, but this fixture + has no equivalent tripwire, so revisit it on a litellm bump.""" return { "prompt_tokens": 1000, "completion_tokens": 200, @@ -70,9 +77,13 @@ def _streaming_usage() -> dict: } -def _nonstreaming_usage() -> dict: - """Raw OpenRouter ``usage`` (non-streaming path) carrying a real - upstream-billed cost.""" +def _usage_with_cost() -> dict: + """Raw OpenRouter ``usage`` carrying a real provider-billed cost. + + Reaches the callback two ways: as ``original_response`` on the + non-streaming path, and — since patch 10 — on the reassembled usage object + of a streamed call. ``_extract_cost`` cannot tell the two apart, which is + the point: the transport was the bug, not the reading.""" return { "prompt_tokens": 1000, "completion_tokens": 200, @@ -81,6 +92,17 @@ def _nonstreaming_usage() -> dict: } +def _reassembled_usage_with_cost() -> dict: + """What ``calculate_usage`` hands the success hook once patch 10 has run: + the enumerated counts it always kept, plus the two cost fields it used to + drop. ``cost_details`` rides along as a pydantic extra.""" + return { + **_usage_without_cost(), + "cost": 0.0123, + "cost_details": {"upstream_inference_cost": 0.0456}, + } + + def _streaming_optional_params() -> dict: """``model_call_details['optional_params']`` as litellm 1.86.2 populates it on a streamed ``anthropic_messages`` call to an OpenRouter backend — the @@ -132,13 +154,13 @@ def _mcd( class TestExtractCost: - def test_streaming_usage_has_no_recoverable_cost(self): - # Reassembled streaming usage has no cost field -> must be None, - # which the recorder treats as "unknown", not "$0". - assert cc._extract_cost(_streaming_usage()) is None + def test_usage_without_a_cost_field_reads_as_unknown(self): + # No cost of any kind -> must be None, which the recorder treats as + # "unknown", not "$0". + assert cc._extract_cost(_usage_without_cost()) is None - def test_nonstreaming_usage_cost_is_extracted(self): - assert cc._extract_cost(_nonstreaming_usage()) == 0.0123 + def test_usage_cost_is_extracted(self): + assert cc._extract_cost(_usage_with_cost()) == 0.0123 def test_byok_falls_back_to_upstream_inference_cost(self): # Under BYOK the top-level cost is 0; the real spend is in @@ -233,16 +255,18 @@ def _capture(self, monkeypatch) -> list[dict]: monkeypatch.setattr(cc, "_emit", lambda payload: emitted.append(payload)) return emitted - def test_streaming_call_reports_null_cost_not_zero(self, monkeypatch): + def test_call_without_a_reported_cost_reports_null_not_zero(self, monkeypatch): emitted = self._capture(monkeypatch) - cc.LiteLLMCostLogger()._record(_mcd("s1"), types.SimpleNamespace(usage=_streaming_usage())) + cc.LiteLLMCostLogger()._record( + _mcd("s1"), types.SimpleNamespace(usage=_usage_without_cost()) + ) assert len(emitted) == 1 payload = emitted[0] # The bug this guards: cost must be null, never coerced to 0.0. assert payload["call"]["cost"] is None assert payload["session"]["cost"] is None assert payload["session"]["cost_known_calls"] == 0 - # Cache stats survive reassembly, so they must still be emitted. + # An unknown cost must not take the cache stats down with it. assert payload["call"]["cached_tokens"] == 600 assert payload["call"]["cache_write_tokens"] == 100 assert payload["call"]["reasoning_tokens"] == 50 @@ -253,21 +277,55 @@ def test_streaming_call_reports_null_cost_not_zero(self, monkeypatch): assert isinstance(payload["session"]["calls"], int) assert isinstance(payload["call"]["cached_tokens"], int) - def test_nonstreaming_call_records_real_cost(self, monkeypatch): + def test_raw_upstream_usage_records_real_cost(self, monkeypatch): emitted = self._capture(monkeypatch) - cc.LiteLLMCostLogger()._record(_mcd("s2", raw_usage=_nonstreaming_usage()), None) + cc.LiteLLMCostLogger()._record(_mcd("s2", raw_usage=_usage_with_cost()), None) payload = emitted[0] assert payload["call"]["cost"] == 0.0123 assert payload["session"]["cost"] == 0.0123 assert payload["session"]["cost_known_calls"] == 1 + def test_reassembled_streaming_usage_records_real_cost(self, monkeypatch): + """The #3691 fix, read from this side of the seam. + + Patch 10 carries ``cost`` / ``cost_details`` across + ``calculate_usage``'s rebuild, so a streamed call — which is + essentially all agent traffic — now arrives with the bill attached and + the callback needs no change to read it. This test is the regression + that would catch the transport being lost again on a litellm bump. + """ + emitted = self._capture(monkeypatch) + cc.LiteLLMCostLogger()._record( + _mcd("s4"), types.SimpleNamespace(usage=_reassembled_usage_with_cost()) + ) + payload = emitted[0] + assert payload["call"]["cost"] == 0.0123 + assert payload["session"]["cost_known_calls"] == 1 + # The counts the rebuild always kept are still there beside it. + assert payload["call"]["cached_tokens"] == 600 + assert payload["call"]["cache_write_tokens"] == 100 + + def test_reassembled_byok_usage_falls_through_to_upstream_cost(self, monkeypatch): + """Under BYOK the top-level ``cost`` is a literal 0 and the real number + is in ``cost_details``. Patch 10 transports both without judging + either, so the fall-through ``_extract_cost`` already implements is + what turns them into a figure — and a 0 must not be recorded as spend. + """ + emitted = self._capture(monkeypatch) + usage = { + **_usage_without_cost(), + "cost": 0, + "cost_details": {"upstream_inference_cost": 0.05}, + } + cc.LiteLLMCostLogger()._record(_mcd("s5"), types.SimpleNamespace(usage=usage)) + assert emitted[0]["call"]["cost"] == 0.05 + def test_mixed_session_sums_only_known_costs(self, monkeypatch): emitted = self._capture(monkeypatch) logger = cc.LiteLLMCostLogger() - # A streaming turn (unknown cost) followed by a non-streaming turn - # (real cost) in the same session. - logger._record(_mcd("s3"), types.SimpleNamespace(usage=_streaming_usage())) - logger._record(_mcd("s3", raw_usage=_nonstreaming_usage()), None) + # A turn whose cost never arrived, followed by one carrying a real cost. + logger._record(_mcd("s3"), types.SimpleNamespace(usage=_usage_without_cost())) + logger._record(_mcd("s3", raw_usage=_usage_with_cost()), None) session = emitted[-1]["session"] assert session["calls"] == 2 assert session["cost_known_calls"] == 1 @@ -276,9 +334,10 @@ def test_mixed_session_sums_only_known_costs(self, monkeypatch): class TestEstimatedCost: - """LiteLLM's pricing-map ``response_cost`` survives streaming and is - surfaced as ``cost_estimated``, strictly separate from the billed - ``cost`` and under the same null-not-zero discipline (#3175).""" + """LiteLLM's pricing-map ``response_cost``, surfaced as ``cost_estimated`` + — strictly separate from the billed ``cost`` and under the same + null-not-zero discipline (#3175). The two are independent measurements of + the same turn, so each must be readable when the other is missing.""" def setup_method(self): cc._session_totals.clear() @@ -288,25 +347,41 @@ def _capture(self, monkeypatch) -> list[dict]: monkeypatch.setattr(cc, "_emit", lambda payload: emitted.append(payload)) return emitted - def test_streaming_call_carries_estimate_while_cost_stays_null(self, monkeypatch): + def test_estimate_is_carried_when_the_billed_cost_is_missing(self, monkeypatch): emitted = self._capture(monkeypatch) cc.LiteLLMCostLogger()._record( _mcd("e1", response_cost=0.021), - types.SimpleNamespace(usage=_streaming_usage()), + types.SimpleNamespace(usage=_usage_without_cost()), ) payload = emitted[0] - # The billed cost is still unrecoverable on streaming — must stay null. assert payload["call"]["cost"] is None assert payload["call"]["cost_estimated"] == 0.021 assert payload["session"]["cost"] is None assert payload["session"]["cost_estimated"] == 0.021 assert payload["session"]["cost_estimated_known_calls"] == 1 + def test_both_figures_are_reported_side_by_side(self, monkeypatch): + """With patches 10 and 11 both in place this is the ordinary line, and + a persistent gap between the two is a signal (stale rate card, + unexpected provider, surcharge tier) rather than an error.""" + emitted = self._capture(monkeypatch) + cc.LiteLLMCostLogger()._record( + _mcd("e4", response_cost=0.0119), + types.SimpleNamespace(usage=_reassembled_usage_with_cost()), + ) + payload = emitted[0] + assert payload["call"]["cost"] == 0.0123 + assert payload["call"]["cost_estimated"] == 0.0119 + assert payload["session"]["cost_known_calls"] == 1 + assert payload["session"]["cost_estimated_known_calls"] == 1 + def test_unpriceable_model_reports_null_estimate(self, monkeypatch): # No response_cost (model absent from LiteLLM's pricing map) must # read as "unknown", never "$0". emitted = self._capture(monkeypatch) - cc.LiteLLMCostLogger()._record(_mcd("e2"), types.SimpleNamespace(usage=_streaming_usage())) + cc.LiteLLMCostLogger()._record( + _mcd("e2"), types.SimpleNamespace(usage=_usage_without_cost()) + ) payload = emitted[0] assert payload["call"]["cost_estimated"] is None assert payload["session"]["cost_estimated"] is None @@ -316,11 +391,11 @@ def test_estimate_accumulates_only_known_calls(self, monkeypatch): emitted = self._capture(monkeypatch) logger = cc.LiteLLMCostLogger() logger._record( - _mcd("e3", response_cost=0.01), types.SimpleNamespace(usage=_streaming_usage()) + _mcd("e3", response_cost=0.01), types.SimpleNamespace(usage=_usage_without_cost()) ) - logger._record(_mcd("e3"), types.SimpleNamespace(usage=_streaming_usage())) + logger._record(_mcd("e3"), types.SimpleNamespace(usage=_usage_without_cost())) logger._record( - _mcd("e3", response_cost=0.02), types.SimpleNamespace(usage=_streaming_usage()) + _mcd("e3", response_cost=0.02), types.SimpleNamespace(usage=_usage_without_cost()) ) session = emitted[-1]["session"] assert session["calls"] == 3 @@ -396,7 +471,7 @@ def test_attribution_headers_are_emitted(self, monkeypatch): emitted = self._capture(monkeypatch) cc.LiteLLMCostLogger()._record( _mcd("a1", extra_headers=_ATTRIBUTION_HEADERS), - types.SimpleNamespace(usage=_streaming_usage()), + types.SimpleNamespace(usage=_usage_without_cost()), ) payload = emitted[0] assert payload["pipeline_id"] == "pipeline-20260612-abc" @@ -405,7 +480,9 @@ def test_attribution_headers_are_emitted(self, monkeypatch): def test_missing_attribution_reads_as_none(self, monkeypatch): emitted = self._capture(monkeypatch) - cc.LiteLLMCostLogger()._record(_mcd("a2"), types.SimpleNamespace(usage=_streaming_usage())) + cc.LiteLLMCostLogger()._record( + _mcd("a2"), types.SimpleNamespace(usage=_usage_without_cost()) + ) payload = emitted[0] assert payload["pipeline_id"] is None assert payload["agent_role"] is None @@ -416,7 +493,7 @@ def test_header_casing_is_normalized(self, monkeypatch): emitted = self._capture(monkeypatch) cc.LiteLLMCostLogger()._record( _mcd("a3", extra_headers={"X-Egg-Agent-Role": "coder"}), - types.SimpleNamespace(usage=_streaming_usage()), + types.SimpleNamespace(usage=_usage_without_cost()), ) assert emitted[0]["agent_role"] == "coder" @@ -850,7 +927,7 @@ def test_emitted_line_carries_request_params(self, monkeypatch): emitted = self._capture(monkeypatch) cc.LiteLLMCostLogger()._record( _mcd("p1", optional_params=_streaming_optional_params()), - types.SimpleNamespace(usage=_streaming_usage()), + types.SimpleNamespace(usage=_usage_without_cost()), ) payload = emitted[0] assert payload["request_params"]["max_tokens"] == 32000 @@ -865,7 +942,9 @@ def test_absent_optional_params_does_not_suppress_the_line(self, monkeypatch): # Cost/cache visibility must not regress on a LiteLLM shape that # carries no optional_params. emitted = self._capture(monkeypatch) - cc.LiteLLMCostLogger()._record(_mcd("p2"), types.SimpleNamespace(usage=_streaming_usage())) + cc.LiteLLMCostLogger()._record( + _mcd("p2"), types.SimpleNamespace(usage=_usage_without_cost()) + ) payload = emitted[0] assert payload["request_params"] is None assert payload["call"]["cached_tokens"] == 600 diff --git a/tests/config/test_litellm_runtime_modules.py b/tests/config/test_litellm_runtime_modules.py index 67a1888e5..3267a6cb5 100644 --- a/tests/config/test_litellm_runtime_modules.py +++ b/tests/config/test_litellm_runtime_modules.py @@ -1,8 +1,8 @@ -"""Unit tests for the four modules the patch script installs into litellm. +"""Unit tests for the five modules the patch script installs into litellm. ``config/litellm/{openrouter_capabilities,drop_params_visibility, -anthropic_thinking_policy,openrouter_reasoning_roundtrip}.py`` are staged by -the Dockerfile and copied into +anthropic_thinking_policy,openrouter_reasoning_roundtrip, +stream_cost_preservation}.py`` are staged by the Dockerfile and copied into every litellm tree by ``patch_litellm_cache.py``. They are kept as real files rather than string literals inside the patch script precisely so they can be linted and tested here — but that only works if they import without litellm @@ -15,6 +15,7 @@ ``verbose_logger`` and ``HTTPHandler`` — are stubbed into ``sys.modules``. """ +import ast import importlib.util import sys import types @@ -105,6 +106,7 @@ def caps(monkeypatch): "LITELLM_OPENROUTER_CAPABILITY_FETCH", "LITELLM_OPENROUTER_CAPABILITY_TTL", "LITELLM_OPENROUTER_CAPABILITY_TIMEOUT", + "LITELLM_OPENROUTER_PRICING", ): monkeypatch.delenv(var, raising=False) return module @@ -147,11 +149,44 @@ def get(self, url): return calls +# Shapes taken from live `GET /api/v1/models` responses: rates are decimal +# STRINGS in USD per token, `input_cache_write` is absent on plenty of models, +# and a long-context surcharge appears as `pricing.overrides` keyed by an +# arbitrary `min_prompt_tokens` (qwen3-max really does publish 32000 and +# 128000, which is what makes it untranslatable). _PAYLOAD = { "data": [ - {"id": "moonshotai/kimi-k3", "supported_parameters": ["reasoning", "reasoning_effort"]}, - {"id": "poolside/laguna-s-2.1", "supported_parameters": ["reasoning"]}, + { + "id": "moonshotai/kimi-k3", + "supported_parameters": ["reasoning", "reasoning_effort"], + "pricing": { + "prompt": "0.0000006", + "completion": "0.0000025", + "input_cache_read": "0.00000015", + }, + }, + { + "id": "poolside/laguna-s-2.1", + "supported_parameters": ["reasoning"], + "pricing": { + "prompt": "0.0000001", + "completion": "0.0000002", + "input_cache_read": "0.00000001", + "input_cache_write": "0.0000005", + }, + }, {"id": "qwen/qwen3-max:free", "supported_parameters": ["temperature"]}, + { + "id": "qwen/qwen3-max", + "pricing": { + "prompt": "0.00000078", + "completion": "0.0000039", + "overrides": [ + {"min_prompt_tokens": 32000, "prompt": "0.00000156"}, + {"min_prompt_tokens": 128000, "prompt": "0.00000195"}, + ], + }, + }, {"id": "malformed-no-params"}, "not-a-dict", ] @@ -457,6 +492,464 @@ def test_module_imports_without_litellm(monkeypatch): assert _load("drop_params_visibility") is not None assert _load("anthropic_thinking_policy") is not None assert _load("openrouter_reasoning_roundtrip") is not None + assert _load("stream_cost_preservation") is not None + + +# -------------------------------------------------------------------------- +# openrouter_capabilities — pricing half (#3691) +# -------------------------------------------------------------------------- + + +def test_pricing_translates_the_published_rate_card(caps, monkeypatch): + """The whole point: a slug LiteLLM's bundled map has never heard of gets a + usable model-cost entry, so ``cost_estimated`` stops reading null.""" + _install_http_stub(monkeypatch, payload=_PAYLOAD) + entry = caps.get_model_cost_entry("openrouter/poolside/laguna-s-2.1") + assert entry == { + "key": "poolside/laguna-s-2.1", + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_token": 1e-07, + "output_cost_per_token": 2e-07, + "cache_read_input_token_cost": 1e-08, + # OpenRouter's `input_cache_write` is LiteLLM's `cache_creation_*`. + # Swapping the pair would price cache writes at the read rate — a ~5x + # understatement of the most expensive turn in a session. + "cache_creation_input_token_cost": 5e-07, + } + + +def test_pricing_carries_cost_fields_only(caps, monkeypatch): + """Capabilities must not arrive through the pricing door. + + ``supports_reasoning: true`` alone makes stock + ``get_supported_openai_params`` admit ``thinking``, which Patch 2's notes + explain would forward an Anthropic-shaped block to a provider expecting + ``reasoning``. Patch 7 stays the only path by which a param is admitted. + """ + _install_http_stub(monkeypatch, payload=_PAYLOAD) + entry = caps.get_model_cost_entry("moonshotai/kimi-k3") + assert entry is not None + assert not [k for k in entry if k.startswith("supports_")] + assert not [k for k in entry if "tokens" in k], "no context lengths either" + + +def test_pricing_omits_a_rate_the_provider_did_not_publish(caps, monkeypatch): + """A missing cache-write rate degrades the estimate; inventing one (or + reusing the read rate for it) would misprice the turns that cost most.""" + _install_http_stub(monkeypatch, payload=_PAYLOAD) + entry = caps.get_model_cost_entry("moonshotai/kimi-k3") + assert "cache_read_input_token_cost" in entry + assert "cache_creation_input_token_cost" not in entry + + +def test_tiered_rate_card_is_declined_rather_than_under_reported(caps, logger, monkeypatch): + """qwen3-max prices by prompt length at boundaries LiteLLM cannot express. + + Registering the base tier anyway would under-report by 2-2.5x on exactly + the long-prompt turns agent traffic is made of, silently, under a field an + operator would use to choose a model. Null is the honest answer, and the + reason has to be findable — so it is a warning, and it names the tiers. + """ + _install_http_stub(monkeypatch, payload=_PAYLOAD) + assert caps.get_model_cost_entry("openrouter/qwen/qwen3-max") is None + + (message,) = logger.messages("warning") + assert "qwen/qwen3-max" in message + assert "32000, 128000" in message + + # Once per slug: this runs on the per-call cost path, so repeating it would + # bury the log stream the cost figures themselves land in. + for _ in range(50): + caps.get_model_cost_entry("openrouter/qwen/qwen3-max") + assert len(logger.messages("warning")) == 1 + + +def _tiered_payload(model_id, override, base=None): + pricing = dict(base or {"prompt": "0.000003", "completion": "0.000015"}) + pricing["overrides"] = [override] if isinstance(override, dict) else list(override) + return {"data": [{"id": model_id, "pricing": pricing}]} + + +def test_a_tier_landing_on_a_litellm_slot_is_translated_not_declined(caps, logger, monkeypatch): + """Declining every tiered card left most of them unpriced for no reason. + + LiteLLM has real rate slots at 128000/200000/272000, and ``x-ai/grok-4.5`` + and friends publish a single boundary that lands exactly on one. Declining + those cost ``cost_estimated`` on ~7% of the live roster — including several + likely routing targets — while the warning told the operator LiteLLM could + not express a boundary it can. + """ + _install_http_stub( + monkeypatch, + payload=_tiered_payload( + "x-ai/grok-4.5", + { + "min_prompt_tokens": 200000, + "prompt": "0.000004", + "completion": "0.000012", + "input_cache_read": "0.0000006", + }, + base={ + "prompt": "0.000002", + "completion": "0.000006", + "input_cache_read": "0.0000003", + }, + ), + ) + assert caps.get_model_cost_entry("x-ai/grok-4.5") == { + "key": "x-ai/grok-4.5", + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token_above_200k_tokens": 4e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, + } + assert logger.messages("warning") == [], "nothing was declined, so nothing to explain" + + +def test_a_tier_is_declined_when_one_published_component_has_no_slot(caps, logger, monkeypatch): + """Slot coverage is uneven *per component*, not just per boundary. + + ``openai/gpt-5.6-luna-pro`` really does surcharge ``input_cache_write`` at + 272000, and LiteLLM has no ``cache_creation_input_token_cost_above_272k_tokens`` + — the field is absent from the explicit enumeration ``_get_model_info_helper`` + builds its return from, so a key by that name is dropped on the way out. + Emitting the three components that *do* fit would leave cache writes billed + at the base rate above the boundary: a silent under-report on exactly the + long-prompt turns the surcharge exists for, which is the failure the + all-or-nothing rule exists to prevent. + """ + _install_http_stub( + monkeypatch, + payload=_tiered_payload( + "openai/gpt-5.6-luna-pro", + { + "min_prompt_tokens": 272000, + "prompt": "0.000001", + "completion": "0.0000045", + "input_cache_read": "0.0000001", + "input_cache_write": "0.00000125", + }, + ), + ) + assert caps.get_model_cost_entry("openai/gpt-5.6-luna-pro") is None + (message,) = logger.messages("warning") + assert "272000" in message + # The old wording asserted LiteLLM "cannot express those boundaries", which + # is now false for most cards and would send an operator after the wrong + # limit. 272000 *is* an expressible boundary; the component is not. + assert "cannot express those boundaries" not in message + + +def test_a_tier_publishing_only_a_component_we_never_price_is_still_translated(caps, monkeypatch): + """An unmapped component is out of scope at the base tier too. + + ``image`` has no per-token slot LiteLLM's chat-path calculation reads, so it + is already absent from the base entry. Declining the whole card over it + would withhold a rate card that is no less complete above the boundary than + below it — the all-or-nothing rule is about components we *do* price. + """ + _install_http_stub( + monkeypatch, + payload=_tiered_payload( + "some/multimodal", + {"min_prompt_tokens": 128000, "prompt": "0.000006", "image": "0.003"}, + ), + ) + entry = caps.get_model_cost_entry("some/multimodal") + assert entry["input_cost_per_token_above_128k_tokens"] == 6e-06 + assert not [k for k in entry if "image" in k] + + +def test_a_tier_with_no_prompt_surcharge_is_declined_as_unreachable(caps, monkeypatch): + """LiteLLM finds the applicable boundary by scanning for + ``input_cost_per_token_above_*`` keys, so a tier publishing only a + completion surcharge would contribute a key nothing ever reads — and bill + the whole turn at base while looking translated.""" + _install_http_stub( + monkeypatch, + payload=_tiered_payload( + "some/completion-only", + {"min_prompt_tokens": 200000, "completion": "0.00003"}, + ), + ) + assert caps.get_model_cost_entry("some/completion-only") is None + + +def test_a_string_tier_boundary_is_parsed_like_every_other_number(caps, monkeypatch): + """``pricing.prompt`` right beside it arrives as a decimal string, so + OpenRouter demonstrably does serialize numbers as strings in this block. An + ``isinstance(raw, int)`` gate would report a perfectly expressible boundary + as "unparseable" the day ``min_prompt_tokens`` follows suit.""" + _install_http_stub( + monkeypatch, + payload=_tiered_payload( + "some/stringy", + {"min_prompt_tokens": "272000", "prompt": "0.00001", "completion": "0.000045"}, + ), + ) + entry = caps.get_model_cost_entry("some/stringy") + assert entry["input_cost_per_token_above_272k_tokens"] == 1e-05 + assert entry["output_cost_per_token_above_272k_tokens"] == 4.5e-05 + + +@pytest.mark.parametrize("boundary", [32000, 256000, 128000.5, True, None]) +def test_an_inexpressible_boundary_is_still_declined(caps, monkeypatch, boundary): + """The narrowing is to the boundaries LiteLLM actually has slots for, not + to "anything that looks like a number". A fractional token count means the + field is not what this thinks it is; a bool is not a count at all.""" + _install_http_stub( + monkeypatch, + payload=_tiered_payload( + "some/arbitrary", + {"min_prompt_tokens": boundary, "prompt": "0.000006", "completion": "0.00003"}, + ), + ) + assert caps.get_model_cost_entry("some/arbitrary") is None + + +def test_the_1h_cache_write_and_reasoning_rates_are_carried(caps, monkeypatch): + """Two more components with an exact LiteLLM destination that is read on + the chat cost path: ``generic_cost_per_token`` prices reasoning tokens off + ``output_cost_per_reasoning_token``, and ``calculate_cache_writing_cost`` + prices a 1h TTL block off ``cache_creation_input_token_cost_above_1hr``. + + ``input_cache_write_1h`` is 2x the 5m rate on every Anthropic route. Claude + Code defaults to 5m so it does not fire today — but dropping it meant that + the moment anything sets ``ttl: "1h"``, cache writes get priced at half of + actual with no signal, which is the same under-report this module declines + a whole rate card to avoid. + """ + _install_http_stub( + monkeypatch, + payload={ + "data": [ + { + "id": "anthropic/claude-opus-4.8", + "pricing": { + "prompt": "0.000005", + "completion": "0.000025", + "input_cache_write": "0.00000625", + "input_cache_write_1h": "0.00001", + "internal_reasoning": "0.000025", + # Published, but per-request rather than per-token, and + # LiteLLM's chat cost path never reads a query rate. + "web_search": "0.01", + }, + } + ] + }, + ) + entry = caps.get_model_cost_entry("anthropic/claude-opus-4.8") + assert entry["cache_creation_input_token_cost"] == 6.25e-06 + assert entry["cache_creation_input_token_cost_above_1hr"] == 1e-05 + assert entry["output_cost_per_reasoning_token"] == 2.5e-05 + assert "input_cost_per_query" not in entry + + +def test_a_pricing_only_entry_does_not_shadow_a_parameter_answer(caps, monkeypatch): + """The candidate loop must skip a record that cannot answer the asked half. + + Before the pricing half existed, ``_fetch`` dropped entries with no + ``supported_parameters`` outright and the loop fell through them by + accident. Keeping them for their rates turned that accident into a silent + regression of Patch 7: ``reasoning_effort`` dropped for a variant slug + whose roster entry happens to carry only a rate card. + """ + _install_http_stub( + monkeypatch, + payload={ + "data": [ + { + "id": "some/model:beta", + "pricing": {"prompt": "0.000001", "completion": "0.000002"}, + }, + {"id": "some/model", "supported_parameters": ["reasoning_effort"]}, + ] + }, + ) + assert caps.get_supported_parameters("some/model:beta") == {"reasoning_effort"} + # ...and the reverse direction still resolves against the variant itself. + assert caps.get_model_cost_entry("some/model:beta")["key"] == "some/model:beta" + + +def test_a_variant_slug_never_inherits_its_base_models_rate_card(caps, monkeypatch): + """Union-only is safe for parameters and wrong for pricing. + + A variant accepts at least what its base does, so inheriting a parameter + list can admit a param and never withdraw one. A variant suffix is + frequently *what changes the rate* — ``:free`` is 0 against a paid base, + ``:batch`` is half — so inheriting would report a confident, authoritative + number that is wrong by construction. OpenRouter retires ``:free`` variants + routinely, which is exactly when the fallback would have fired. + """ + _install_http_stub( + monkeypatch, + payload={ + "data": [ + { + "id": "poolside/laguna-s-2.1", + "supported_parameters": ["reasoning"], + "pricing": {"prompt": "0.0000001", "completion": "0.0000002"}, + } + ] + }, + ) + assert caps.get_model_cost_entry("poolside/laguna-s-2.1:free") is None + # The parameter half keeps the fallback, and keeps it deliberately. + assert caps.get_supported_parameters("poolside/laguna-s-2.1:free") == {"reasoning"} + + +def test_the_decline_latch_is_rearmed_by_a_refetch_not_only_by_reset_cache(caps, monkeypatch): + """``reset_cache`` is test-only, so a latch cleared solely there is latched + for the life of the pod — the exact outcome the comment beside it claimed + to prevent. A model that acquires or drops a surcharge tier has to be + reported against the roster the change was read from.""" + _install_http_stub( + monkeypatch, + payload=_tiered_payload("some/tiered", {"min_prompt_tokens": 32000, "prompt": "0.000006"}), + ) + monkeypatch.setenv("LITELLM_OPENROUTER_CAPABILITY_TTL", "0") + recorder = _install_logger(monkeypatch, _RecordingLogger()) + + assert caps.get_model_cost_entry("some/tiered") is None + assert len(recorder.messages("warning")) == 1 + assert caps.get_model_cost_entry("some/tiered") is None + assert len(recorder.messages("warning")) == 2, "each refetch re-arms the per-slug latch" + + +def test_a_tiered_card_with_unparseable_boundaries_is_still_explained(caps, logger, monkeypatch): + """A tiered card whose boundaries do not parse is still a tiered card. + + Collapsing "declined for tiering" into the truthiness of the threshold + tuple would leave exactly these models unpriced AND unexplained — the one + combination an operator cannot debug. + """ + _install_http_stub( + monkeypatch, + payload={ + "data": [ + { + "id": "some/tiered", + "pricing": { + "prompt": "0.000001", + "completion": "0.000002", + "overrides": [{"min_prompt_tokens": "128k"}], + }, + } + ] + }, + ) + assert caps.get_model_cost_entry("some/tiered") is None + (message,) = logger.messages("warning") + assert "some/tiered" in message + assert "unparseable" in message + + +def test_declined_pricing_warning_is_not_lost_to_a_swallowed_emit_failure(caps, monkeypatch): + """Same latch discipline as the module's other three warn-once sites.""" + recorder = _install_logger(monkeypatch, _FlakyLogger()) + _install_http_stub(monkeypatch, payload=_PAYLOAD) + + assert caps.get_model_cost_entry("qwen/qwen3-max") is None + assert recorder.messages("warning") == [], "first emit raised, and was swallowed" + assert caps._WARNED_DECLINED_PRICING == set(), "nothing emitted, so nothing latched" + + assert caps.get_model_cost_entry("qwen/qwen3-max") is None + assert len(recorder.messages("warning")) == 1, "the signal must survive the failure" + + assert caps.get_model_cost_entry("qwen/qwen3-max") is None + assert len(recorder.messages("warning")) == 1, "and dedup still holds once it is out" + + +def test_pricing_answers_for_openrouter_only(caps, logger, monkeypatch): + """The call site is a generic lookup every provider reaches. Answering for + a same-named slug on another provider would attach OpenRouter's rate card + to somebody else's bill.""" + _install_http_stub(monkeypatch, payload=_PAYLOAD) + assert caps.get_model_cost_entry("poolside/laguna-s-2.1", "bedrock") is None + assert caps.get_model_cost_entry("poolside/laguna-s-2.1", "openrouter") is not None + # None is "the caller could not attribute it", which a bare slug legitimately is. + assert caps.get_model_cost_entry("poolside/laguna-s-2.1", None) is not None + + +def test_unknown_slug_has_no_price_opinion(caps, monkeypatch): + _install_http_stub(monkeypatch, payload=_PAYLOAD) + assert caps.get_model_cost_entry("some/model-the-api-never-heard-of") is None + + +def test_entry_without_pricing_is_still_a_parameter_answer(caps, monkeypatch): + """The two halves are independent: a roster entry carrying one and not the + other is a real answer for the half it has.""" + _install_http_stub(monkeypatch, payload=_PAYLOAD) + assert caps.get_supported_parameters("qwen/qwen3-max:free") == {"temperature"} + assert caps.get_model_cost_entry("qwen/qwen3-max:free") is None + # ...and the reverse: qwen3-max publishes pricing but no parameter list. + assert caps.get_supported_parameters("qwen/qwen3-max") is None + + +def test_returned_cost_entry_is_a_copy(caps, monkeypatch): + """The cache is process-wide and LiteLLM's model-info path mutates what it + is handed.""" + _install_http_stub(monkeypatch, payload=_PAYLOAD) + first = caps.get_model_cost_entry("poolside/laguna-s-2.1") + first["input_cost_per_token"] = 999.0 + assert caps.get_model_cost_entry("poolside/laguna-s-2.1")["input_cost_per_token"] == 1e-07 + + +def test_pricing_can_be_disabled_without_disabling_capabilities(caps, monkeypatch): + """For an operator who wants LiteLLM's bundled map to be the sole authority + on cost while keeping Patch 7's parameter fix.""" + _install_http_stub(monkeypatch, payload=_PAYLOAD) + monkeypatch.setenv("LITELLM_OPENROUTER_PRICING", "0") + assert caps.get_model_cost_entry("poolside/laguna-s-2.1") is None + assert caps.get_supported_parameters("poolside/laguna-s-2.1") == {"reasoning"} + + +def test_fetch_disabled_disables_pricing_too(caps, monkeypatch): + """One fetch serves both halves, so the master switch governs both.""" + calls = _install_http_stub(monkeypatch, payload=_PAYLOAD) + monkeypatch.setenv("LITELLM_OPENROUTER_CAPABILITY_FETCH", "0") + assert caps.get_model_cost_entry("poolside/laguna-s-2.1") is None + assert calls == [], "no network call may be made when the lookup is off" + + +@pytest.mark.parametrize( + "pricing", + [ + {"completion": "0.000002"}, # no prompt rate + {"prompt": "0.000001"}, # no completion rate + {"prompt": "not-a-number", "completion": "0.000002"}, + {"prompt": "-0.000001", "completion": "0.000002"}, + {"prompt": "Infinity", "completion": "0.000002"}, + "not-a-dict", + ], +) +def test_unusable_rate_card_is_no_opinion(caps, monkeypatch, pricing): + """Without both the prompt and completion rate the entry cannot price a + chat turn at all, and a negative or non-finite rate is not a rate — an + ``inf`` would propagate into egg's session total and into the emitted JSON + as a token that makes the log line unparseable.""" + _install_http_stub(monkeypatch, payload={"data": [{"id": "some/model", "pricing": pricing}]}) + assert caps.get_model_cost_entry("some/model") is None + + +def test_zero_is_a_real_rate(caps, monkeypatch): + """A ``:free`` variant really is priced at zero. Filtering the resulting + zero estimate is ``cost_callback``'s job, not this module's.""" + _install_http_stub( + monkeypatch, + payload={ + "data": [{"id": "some/model:free", "pricing": {"prompt": "0", "completion": "0"}}] + }, + ) + entry = caps.get_model_cost_entry("some/model:free") + assert entry["input_cost_per_token"] == 0.0 + assert entry["output_cost_per_token"] == 0.0 # -------------------------------------------------------------------------- @@ -1048,3 +1541,276 @@ def test_unrecognized_knob_value_warns_once_and_keeps_the_default(roundtrip, mon warnings = logger.messages("warning") assert len(warnings) == 1 assert roundtrip.ENV_VAR in warnings[0] + + +# -------------------------------------------------------------------------- +# stream_cost_preservation (#3691) +# -------------------------------------------------------------------------- + + +@pytest.fixture +def streamcost(): + return _load("stream_cost_preservation") + + +class _Usage: + """Stand-in for litellm's ``Usage``: attribute access, arbitrary extras. + + ``cost`` is a declared field there and ``cost_details`` survives only as a + pydantic extra, so both must work through plain ``setattr`` — which is what + this models. Deliberately NOT a dict: the object the patch amends is the + one ``calculate_usage`` just rebuilt. + """ + + def __init__(self, **fields): + for key, value in fields.items(): + setattr(self, key, value) + + +class _Chunk: + """Stand-in for ``ModelResponseStream``: mapping-style access over + attributes, which is how ``ChunkProcessor`` reads a chunk's usage.""" + + def __init__(self, usage=None, hidden_usage=None): + if usage is not None: + self.usage = usage + if hidden_usage is not None: + self._hidden_params = {"usage": hidden_usage} + + def __contains__(self, key): + return hasattr(self, key) + + def __getitem__(self, key): + return getattr(self, key) + + +def test_cost_is_carried_off_the_final_usage_chunk(streamcost): + """The fix itself: 1252 of 1252 sampled calls reported ``cost: null`` + because this value was dropped between the chunk and the rebuilt usage.""" + chunks = [ + _Chunk(), + _Chunk(usage=_Usage(prompt_tokens=100, cost=0.00123)), + ] + usage = _Usage(prompt_tokens=100, completion_tokens=10) + streamcost.carry_upstream_cost(chunks, usage) + assert usage.cost == 0.00123 + + +def test_cost_details_is_carried_too(streamcost): + """Under BYOK ``cost`` is 0 and the real number lives here, so carrying one + without the other would leave the BYOK bill unrecoverable.""" + chunks = [_Chunk(usage=_Usage(cost=0.0, cost_details={"upstream_inference_cost": 0.0045}))] + usage = _Usage(prompt_tokens=100) + streamcost.carry_upstream_cost(chunks, usage) + assert usage.cost == 0.0 + assert usage.cost_details == {"upstream_inference_cost": 0.0045} + + +def test_a_zero_cost_is_transported_not_filtered(streamcost): + """This module transports; ``cost_callback._extract_cost`` interprets. A + "positive only" filter here would delete the evidence that the + ``cost``->``cost_details`` fall-through is the right reading of a BYOK + turn, leaving the callback unable to tell it from a missing field.""" + chunks = [_Chunk(usage=_Usage(cost=0.0))] + usage = _Usage() + streamcost.carry_upstream_cost(chunks, usage) + assert usage.cost == 0.0 + + +def test_hidden_params_usage_is_read_as_well(streamcost): + """``ChunkProcessor`` reads both sources; reading a different set of chunks + than the counts came from would let cost and tokens describe different + turns.""" + chunks = [_Chunk(hidden_usage={"cost": 0.5})] + usage = _Usage() + streamcost.carry_upstream_cost(chunks, usage) + assert usage.cost == 0.5 + + +def test_the_last_reported_value_wins(streamcost): + """A provider that revises its usage block mid-stream is stating a + correction.""" + chunks = [_Chunk(usage=_Usage(cost=0.1)), _Chunk(usage=_Usage(cost=0.2))] + usage = _Usage() + streamcost.carry_upstream_cost(chunks, usage) + assert usage.cost == 0.2 + + +def test_an_existing_value_is_never_overwritten(streamcost): + """If a future LiteLLM carries cost through reassembly itself, its answer + wins and this becomes a no-op rather than a competing second opinion.""" + chunks = [_Chunk(usage=_Usage(cost=0.2, cost_details={"upstream_inference_cost": 9.0}))] + usage = _Usage(cost=0.1, cost_details={"upstream_inference_cost": 1.0}) + streamcost.carry_upstream_cost(chunks, usage) + assert usage.cost == 0.1 + assert usage.cost_details == {"upstream_inference_cost": 1.0} + + +@pytest.mark.parametrize("bad", [float("nan"), float("inf"), float("-inf"), True, "0.5", None]) +def test_a_non_measurement_is_refused(streamcost, bad): + """``NaN``/``Inf`` on a cost field is worse than absent: it accumulates into + egg's per-session total and poisons it for the pod's lifetime, and + ``json.dumps`` renders it as a non-standard token that makes the whole log + line invalid JSON. ``True`` is excluded for the same reason + ``_finite_number`` excludes it in ``cost_callback`` — one dollar that was + never billed.""" + usage = _Usage() + streamcost.carry_upstream_cost([_Chunk(usage=_Usage(cost=bad))], usage) + assert not hasattr(usage, "cost") + + +def test_an_empty_cost_details_is_not_carried(streamcost): + """``{}`` says nothing, and writing it would make the field look answered.""" + usage = _Usage() + streamcost.carry_upstream_cost([_Chunk(usage=_Usage(cost_details={}))], usage) + assert not hasattr(usage, "cost_details") + + +def test_chunks_without_usage_are_a_no_op(streamcost): + usage = _Usage(prompt_tokens=100) + streamcost.carry_upstream_cost([_Chunk(), _Chunk(usage=None)], usage) + assert not hasattr(usage, "cost") + + +@pytest.mark.parametrize("chunks", [None, [], [object()], ["not-a-chunk"], [{"usage": None}]]) +def test_a_shape_we_do_not_understand_never_raises(streamcost, chunks): + """A cost figure is observability; it must never break a response.""" + usage = _Usage(prompt_tokens=100) + assert streamcost.carry_upstream_cost(chunks, usage) is usage + + +def test_a_dict_usage_chunk_is_read(streamcost): + """Provider iterators hand back plain dicts on some paths.""" + usage = _Usage() + streamcost.carry_upstream_cost([{"usage": {"cost": 0.75}}], usage) + assert usage.cost == 0.75 + + +def test_a_hostile_usage_object_cannot_break_the_response(streamcost): + """Every read is guarded, including the ones on the usage being amended.""" + + class _Exploding: + def __getattr__(self, name): + raise RuntimeError("boom") + + usage = _Usage() + assert streamcost.carry_upstream_cost([_Chunk(usage=_Exploding())], usage) is usage + + +def test_an_undeclared_field_survives_on_a_real_pydantic_usage(streamcost): + """``cost_details`` is not a declared field on litellm's ``Usage``. + + It exists only because ``Usage`` inherits openai's ``BaseModel``, which sets + ``ConfigDict(extra="allow")``; under the default ``extra`` the same + ``setattr`` raises ``ValueError: "Usage" object has no field "cost_details"``. + ``carry_upstream_cost`` swallows that — correctly, a cost must never break a + response — so if a litellm bump ever tightens the config, the ``cost_details`` + half becomes a completely silent no-op. The plain-Python ``_Usage`` double + above cannot express that failure: ``setattr`` always works on it. + + This models the real shape (declared ``cost``, undeclared ``cost_details``) + with an actual pydantic model, and asserts the round-trip through + ``model_dump()`` as well — which is how ``cost_callback._coerce_usage`` + reads it, so an extra that survives ``setattr`` but is dropped by the dump + would still be a silent loss. + """ + pydantic = pytest.importorskip("pydantic") + + class _PydanticUsage(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="allow") + + prompt_tokens: int = 0 + cost: float | None = None + + usage = _PydanticUsage(prompt_tokens=100) + chunks = [_Chunk(usage=_Usage(cost=0.0, cost_details={"upstream_inference_cost": 0.0045}))] + streamcost.carry_upstream_cost(chunks, usage) + + assert usage.cost == 0.0 + assert usage.cost_details == {"upstream_inference_cost": 0.0045} + dumped = usage.model_dump() + assert dumped["cost"] == 0.0 + assert dumped["cost_details"] == {"upstream_inference_cost": 0.0045} + + +def test_the_carried_cost_is_readable_by_the_callback_that_consumes_it(streamcost, monkeypatch): + """Joins the producer to the consumer, which prose alone was doing. + + ``test_cost_callback.py`` asserts against a hand-built dict that *describes* + what patch 10 is supposed to leave behind; nothing ran the real + ``carry_upstream_cost`` output through the real ``_coerce_usage`` / + ``_extract_cost``. This does, on the BYOK shape — ``cost`` 0, the money in + ``cost_details.upstream_inference_cost`` — which is the one where the two + modules have to agree on a nested key name to produce a number at all. + """ + litellm = sys.modules.get("litellm") or types.ModuleType("litellm") + integrations = types.ModuleType("litellm.integrations") + custom_logger_mod = types.ModuleType("litellm.integrations.custom_logger") + + class _CustomLogger: # the module only subclasses it + pass + + custom_logger_mod.CustomLogger = _CustomLogger + monkeypatch.setitem(sys.modules, "litellm", litellm) + monkeypatch.setitem(sys.modules, "litellm.integrations", integrations) + monkeypatch.setitem(sys.modules, "litellm.integrations.custom_logger", custom_logger_mod) + cc = _load("cost_callback") + + pydantic = pytest.importorskip("pydantic") + + class _PydanticUsage(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="allow") + + prompt_tokens: int = 0 + completion_tokens: int = 0 + cost: float | None = None + + usage = _PydanticUsage(prompt_tokens=1000, completion_tokens=50) + chunks = [ + _Chunk(usage=_Usage(cost=0.0, cost_details={"upstream_inference_cost": 0.0045})), + ] + carried = streamcost.carry_upstream_cost(chunks, usage) + + assert cc._extract_cost(cc._coerce_usage(carried)) == 0.0045 + + +# -------------------------------------------------------------------------- +# Image-interpreter compatibility +# -------------------------------------------------------------------------- + + +# The Python the egg-litellm base image ships (ghcr.io/berriai/litellm:v1.86.2). +# Every file under config/litellm/ runs there, not on the repo's interpreter. +_IMAGE_PYTHON = (3, 11) + +_IMAGE_SOURCES = ( + "cost_callback.py", + "openrouter_capabilities.py", + "drop_params_visibility.py", + "anthropic_thinking_policy.py", + "openrouter_reasoning_roundtrip.py", + "stream_cost_preservation.py", +) + + +@pytest.mark.parametrize("name", _IMAGE_SOURCES) +def test_image_sources_parse_on_the_image_interpreter(name): + """These files must be valid Python 3.11, not just valid on the repo's 3.14. + + They are the only Python in this repo that runs on a different interpreter, + and nothing else notices: ruff formats for the repo's ``target-version``, + mypy checks against ``python_version = "3.14"``, and the tests import them + on 3.14 too. The formatter is the sharp edge — under ``py314`` it rewrites + ``except (TypeError, ValueError):`` into the PEP 758 unparenthesized form, + a hard SyntaxError on 3.11, and a formatter rewrite has no ``noqa`` + escape. ``config/litellm/.ruff.toml`` pins that directory to ``py311`` so + it cannot happen; this asserts the outcome rather than the mechanism, so a + future config reshuffle that loses the pin fails here. + + Without this the failure surfaces as a Docker build error at the patch + script's parse check (fail-loud, but only once someone builds the image) or + — for ``cost_callback.py``, which the patch script never parses — as a pod + CrashLoopBackOff at proxy startup. + """ + source = (CONFIG_DIR / name).read_text() + ast.parse(source, filename=str(CONFIG_DIR / name), feature_version=_IMAGE_PYTHON) diff --git a/tests/config/test_patch_litellm_cache.py b/tests/config/test_patch_litellm_cache.py index 4f82fa184..24122dead 100644 --- a/tests/config/test_patch_litellm_cache.py +++ b/tests/config/test_patch_litellm_cache.py @@ -15,6 +15,7 @@ flows into the fixture the test patches. """ +import ast import importlib.util import sys from pathlib import Path @@ -661,3 +662,167 @@ def test_patch10_maps_onto_the_request_not_the_response(tmp_path): assert "transform_response" not in patch10["needle"] assert "transform_response" not in patch10["replacement"] assert patch10["file"] == plc.F1 + + +def test_patch11_sets_cost_after_the_rebuild_not_before(tmp_path): + """Patch 11 must land on the far side of ``Usage(**model_dump())``. + + That constructor deletes a ``cost`` attribute it is handed as None, so a + carry inserted *before* the rebuild would be writing into an object the + rebuild is entitled to discard — the patch would apply cleanly, the build + would pass, and ``cost`` would still read null on every streamed call, + which is the bug it exists to fix.""" + patch11 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 11/")) + + target = tmp_path / patch11["file"] + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(patch11["needle"]) + plc._apply( + str(target), + present=patch11["present"], + needle=patch11["needle"], + replacement=patch11["replacement"], + label=patch11["label"], + ) + result = target.read_text() + + rebuild = " returned_usage = Usage(**returned_usage.model_dump())\n" + assert rebuild in result, "the stock rebuild must survive the patch" + assert result.index(rebuild) < result.index("_egg_carry_upstream_cost"), ( + "the carry must run after the rebuild, not before it" + ) + assert result.index("_egg_carry_upstream_cost") < result.index(" return returned_usage") + # An import failure must never propagate: this is on the response path. + assert "except Exception as _egg_exc:" in result + # And it must not be silent either — a swallowed import here looks exactly + # like "the provider reported no cost", the symptom the patch removes. + # verbose_logger is imported inside the handler because + # streaming_chunk_builder_utils.py, unlike utils.py, does not carry it at + # module scope; the latch is set only once the emit succeeded. + assert "from litellm._logging import verbose_logger" in result + assert result.index("verbose_logger.warning(") < result.index( + "globals()['_egg_warned_stream_cost'] = True" + ), "the latch must be set after the emit, not before it" + + +def test_patch12_hooks_the_unmapped_branch_and_leaves_the_stock_raise(tmp_path): + """Patch 12 must be a fallback, not a replacement. + + It sits at the "isn't mapped yet" raise, so it runs only once every stock + lookup has already failed: a slug the bundled map DOES carry keeps the + bundled rate, and the live card can add a model but never reprice one. + The stock ValueError must still be reachable, for the slug OpenRouter has + not heard of either.""" + patch12 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 12/")) + + target = tmp_path / patch12["file"] + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(patch12["needle"]) + plc._apply( + str(target), + present=patch12["present"], + needle=patch12["needle"], + replacement=patch12["replacement"], + label=patch12["label"], + ) + result = target.read_text() + + assert result.count("raise ValueError(") == 1, "the stock raise must survive, exactly once" + assert result.index("_egg_openrouter_cost_entry") < result.index("raise ValueError(") + # The lookup is handed the provider so it can decline to answer for + # anything that is not OpenRouter — this call site is generic. + assert "_egg_openrouter_cost_entry(\n model, custom_llm_provider\n" in ( + result + ) + # A missing or broken module leaves the stock behaviour exactly as it was. + assert "except Exception as _egg_exc:\n _egg_entry = None\n" in result + # ...but not silently. A failed import is otherwise indistinguishable from + # "OpenRouter has no rate for this slug", which is the null-cost_estimated + # symptom patch 12 exists to remove. Warned once, and the latch is set only + # after the emit — inside its own try, because raising from an except block + # would propagate into a live request. + assert "_egg_warned_pricing" in result + assert result.index("verbose_logger.warning(") < result.index( + "globals()['_egg_warned_pricing'] = True" + ), "the latch must be set after the emit, not before it" + + +# The indentation each replacement is spliced in at, and enough enclosing +# scope to make it a parseable module. Both bodies now carry a nested +# ``try``/``except`` for their warn-once latch, which is exactly the kind of +# hand-written indentation a string-literal patch payload gets wrong — and the +# build's own ``_check_parses`` would only catch it after a full image build. +_BODY_CONTEXTS = ( + ( + "Patch 11/", + "class ChunkProcessor:\n" + " def calculate_usage(self, chunks):\n" + " Usage = dict\n" + " returned_usage = Usage()\n", + ), + ( + "Patch 12/", + "def _get_model_info_helper(model, custom_llm_provider):\n" + " _model_info = None\n" + " key = None\n" + " if True:\n" + " if True:\n", + ), +) + + +@pytest.mark.parametrize(("prefix", "preamble"), _BODY_CONTEXTS, ids=["patch11", "patch12"]) +def test_patch_bodies_parse_at_their_insertion_indentation(prefix, preamble): + """The spliced payload must be valid Python 3.11 — the image's interpreter. + + These replacements are string literals assembled line by line, so a wrong + indent inside the nested warn-once handler is a plain typo that no other + test here would catch: the ``_apply`` tests assert substrings, not syntax, + and the build's ``_check_parses`` runs only during a real image build. + ``feature_version`` pins the check to 3.11 for the same reason + ``config/litellm/.ruff.toml`` does — this code runs on the litellm base + image, not on the repo's interpreter. + """ + patch = next(p for p in plc.PATCHES if p["label"].startswith(prefix)) + ast.parse(preamble + patch["replacement"], feature_version=(3, 11)) + + +def test_patch12_needle_disambiguates_the_two_unmapped_messages(tmp_path): + """utils.py carries the "isn't mapped yet" string twice. + + The other one is the outer handler's re-raise, with a different message + body and indentation. Matching loosely would insert a pricing fallback into + an exception handler, where ``_model_info`` and ``key`` are not even in + scope — same needle-uniqueness trap as Patches 4 and 8.""" + patch12 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 12/")) + + sibling = ( + "SENTINEL_OUTER_HANDLER_BEGIN\n" + " except Exception as e:\n" + ' verbose_logger.debug(f"Error getting model info: {e}")\n' + " raise Exception(\n" + " \"This model isn't mapped yet. model={}, custom_llm_provider={}. " + "Add it here - https://github.com/BerriAI/litellm/blob/main/" + 'model_prices_and_context_window.json.".format(\n' + " model, custom_llm_provider\n" + " )\n" + " )\n" + "SENTINEL_OUTER_HANDLER_END\n" + ) + target = tmp_path / patch12["file"] + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(patch12["needle"] + "\n" + sibling) + + plc._apply( + str(target), + present=patch12["present"], + needle=patch12["needle"], + replacement=patch12["replacement"], + label=patch12["label"], + ) + result = target.read_text() + + assert result.count(patch12["present"]) == 1 + start = result.index("SENTINEL_OUTER_HANDLER_BEGIN") + end = result.index("SENTINEL_OUTER_HANDLER_END") + len("SENTINEL_OUTER_HANDLER_END\n") + assert result[start:end] == sibling, "patch 12 rewrote the outer handler's re-raise"