diff --git a/config/litellm/Dockerfile b/config/litellm/Dockerfile index 57dc2629f..c196afbc2 100644 --- a/config/litellm/Dockerfile +++ b/config/litellm/Dockerfile @@ -5,20 +5,32 @@ # stock image's Anthropic->OpenAI translation drops prompt-cache hits on # Qwen/DeepSeek, mis-streams reasoning models, silently drops params it does # not recognise, manufactures a reasoning ceiling from the caller's thinking -# budget, never sends prior-turn reasoning back, and discards the provider's -# own bill during stream reassembly while its rate card cannot price the route -# either. -# patch_litellm_cache.py closes the twelve gaps at build time +# budget, never sends prior-turn reasoning back, loses the BYOK half of the +# provider's bill during stream reassembly while its rate card cannot price +# the route either, and seeds its own cost calculator with a BYOK zero that +# suppresses the estimate. +# patch_litellm_cache.py closes the ten gaps at build time # (see that script for the diagnosis); # cost_callback.py surfaces the resulting cost/cache stats into the pod log # stream. Both mirror the host-side dotfiles setup that took Qwen cache hit # rate from 0% to ~99.99%. # -# Pinned to v1.86.2 — the version the patches and cost_callback were -# validated against, and the same release the dotfiles host install pins. +# Pinned to v1.94.0 — the newest stable published release, and the version +# the patches and cost_callback are validated against (#3697). The bump from +# v1.86.2 retired four patches upstream absorbed, narrowed a fifth, and added +# a tenth for a regression the bump itself brings in (1.94.0 hands its cost +# calculator the provider's `cost` even when that is a BYOK zero, which +# suppresses the independent estimate and makes the rate-card patch dead code +# on the streaming path). +# # The patch script fails the build loudly if a needle stops matching, so a -# future bump can't silently ship an unpatched (full-input-rate) image. -FROM ghcr.io/berriai/litellm:v1.86.2 +# future bump can't silently ship an unpatched (full-input-rate) image. Read +# that failure as "look at this", not as "upstream fixed it": at this bump, +# two needles stopped matching purely because upstream reflowed the code +# around them, and deleting those patches would have shipped an image paying +# full input rate on every Qwen turn. Confirm each miss against the new +# source before retiring anything. +FROM ghcr.io/berriai/litellm:v1.94.0 # Patches edit the installed litellm package (root-owned site-packages); the # build runs as root. The runtime pod still drops to UID 1000 via the diff --git a/config/litellm/anthropic_thinking_policy.py b/config/litellm/anthropic_thinking_policy.py index de8d73e4a..ce0ff029c 100644 --- a/config/litellm/anthropic_thinking_policy.py +++ b/config/litellm/anthropic_thinking_policy.py @@ -28,15 +28,15 @@ Historically this never mattered: the model-cost map did not carry these slugs, ``OpenrouterConfig`` advertised no reasoning knobs, and ``drop_params`` silently discarded the synthesized param — which is precisely why these models have been -running at full depth. Patch 7 makes the OpenRouter param gate accurate, which +running at full depth. Patch 4 makes the OpenRouter param gate accurate, which is right for an *operator-configured* ``reasoning_effort`` and wrong for this adapter-manufactured one: it would turn a knob nobody set into the effective -setting, with no config file mentioning it and nothing in the logs (Patch 8 +setting, with no config file mentioning it and nothing in the logs (Patch 5 only fires on drops, and this param would no longer be dropped). -So Patch 9 gates the synthesis, defaulting it off. ``thinking`` stays out of +So Patch 6 gates the synthesis, defaulting it off. ``thinking`` stays out of the OpenAI body for non-Claude models and only an explicitly configured -``reasoning_effort`` reaches the wire — exactly the property Patch 7 exists to +``reasoning_effort`` reaches the wire — exactly the property Patch 4 exists to restore, without the adapter's bucket riding along. The gate covers the *derived* value only. On an adaptive request diff --git a/config/litellm/cost_callback.py b/config/litellm/cost_callback.py index 49c1c6637..6ec542268 100644 --- a/config/litellm/cost_callback.py +++ b/config/litellm/cost_callback.py @@ -28,32 +28,45 @@ calculate_usage``, a rebuild that enumerates the token/cache counts and originally DROPPED the provider's ``cost`` / ``cost_details`` outright. That is why this module recorded ``cost: null`` on 1252 of 1252 sampled -calls in run 6. The egg-litellm image's **patch 11** now carries those two -fields across the rebuild (``config/litellm/stream_cost_preservation.py``), -so the billed figure reaches ``_extract_cost`` on the streaming path too -and this module needs no change to read it — the value simply stops being -absent (#3691). +calls in run 6. litellm 1.94.0 carries ``cost`` across that rebuild, and the +egg-litellm image's **cost_details preservation patch** carries the +``cost_details`` it still drops (``config/litellm/stream_cost_preservation.py``) +— that is where the BYOK number lives. Between them the billed figure reaches +``_extract_cost`` on the streaming path too, and this module needs no change to +read it — the value simply stops being absent (#3691). ``cost: null`` therefore no longer means "streaming"; it means the cost was -genuinely unavailable — a stock (unpatched) LiteLLM under this callback, or -a provider that does not report one. It is still emitted as null rather -than ``0.0``: a zero would read in the logs as "this route is free", the -exact opposite of the cost-visibility signal this module exists to provide -(#2799). +genuinely unavailable — a LiteLLM older than 1.94.0 under this callback, a +BYOK route on an unpatched image, or a provider that does not report one. It +is still emitted as null rather than ``0.0``: a zero would read in the logs as +"this route is free", the exact opposite of the cost-visibility signal this +module exists to provide (#2799). Each line also carries ``cost_estimated``: LiteLLM's own ``response_cost``, computed at logging time from the assembled usage and its pricing map (issue #3175). It is kept strictly separate from ``cost`` — an estimate from a possibly-stale rate card must never be mistaken for a bill — and follows the same null-not-zero rule when LiteLLM cannot price the model. -That was the case for every route egg uses until the image's **patch 12** -taught the model-info lookup to read OpenRouter's published rate card; it -remains the case for a model whose prompt-length surcharge lands on a +That was the case for every route egg uses until the image's **live pricing +patch** taught the model-info lookup to read OpenRouter's published rate card; +it remains the case for a model whose prompt-length surcharge lands on a boundary or component LiteLLM's map has no slot for, which is declined whole -rather than translated in part (see ``openrouter_capabilities``). With -both patches in place the two fields are independent measurements of the -same turn, and a persistent gap between them is a signal in its own right: -a stale rate card, an unexpected provider, or a surcharge tier. +rather than translated in part (see ``openrouter_capabilities``). + +The two fields are independent measurements ONLY WHERE THE PROVIDER REPORTS NO +CHARGE, which on egg's BYOK routes is every call. From 1.94.0 litellm copies a +reported ``usage.cost`` into ``_hidden_params`` and ``response_cost_calculator`` +returns that value directly, so where a provider does bill a positive figure +``cost_estimated`` mirrors ``cost`` rather than pricing the turn independently. +The image's **billable-cost gate patch** narrows that propagation to a positive, +finite charge — without it a BYOK ``cost: 0.0`` would be handed to the +calculator, ``cost_estimated`` would read null on ~100% of egg's traffic, and +the live pricing patch would be dead code on the streaming path (it lives inside +``completion_cost()``, which the short circuit returns before). So on a BYOK +route the two fields remain independent and a persistent gap between them is a +signal in its own right — a stale rate card, an unexpected provider, or a +surcharge tier — while on a route that reports a real charge, agreement between +them is arithmetic, not corroboration. Each line also carries ``request_params``: the decoding configuration that actually went upstream on that call (issue #3599). Repetition and @@ -165,9 +178,10 @@ def _usage_from_response_obj(response_obj): counts (the final usage chunk's counts are folded into ``response_obj.usage`` by ``stream_chunk_builder``) and, on the egg-litellm image, for cost as well: that reassembly rebuilds a fresh ``Usage`` and - stock drops ``cost`` / ``cost_details`` with it, which patch 11 restores. - Under a stock LiteLLM the counts still arrive and ``_extract_cost`` returns - None — see the module docstring.""" + older releases dropped ``cost`` / ``cost_details`` with it; 1.94.0 carries + ``cost`` and the image's cost_details preservation patch supplies + ``cost_details``. Under a stock LiteLLM the counts still arrive and + ``_extract_cost`` returns None — see the module docstring.""" if response_obj is None: return None usage = getattr(response_obj, "usage", None) @@ -195,8 +209,8 @@ def _extract_cost(usage): provider, so fall back to ``cost_details.upstream_inference_cost`` (what the upstream provider will bill for the same request). Either way, the number we record matches real spend on that turn. Returns None when no - positive cost is present — a provider that reports none, or a stock LiteLLM - whose chunk reassembly drops it (see ``_usage_from_response_obj``). + positive cost is present — a provider that reports none, or a LiteLLM whose + chunk reassembly drops it (see ``_usage_from_response_obj``). Callers must treat None as "unknown", not "$0". ``_positive`` rejects non-finite values as well as non-positive ones: a @@ -322,11 +336,22 @@ def _extract_estimated_cost(mcd): reported a bill. It is an estimate, not a bill: the pricing map may lag the provider's rates or lack cache-discount entries for a model. Returns None — never 0.0 — when LiteLLM couldn't price the call, mirroring the billed-cost - "unknown ≠ free" discipline. On the egg-litellm image patch 12 supplies - OpenRouter's published rates for slugs the bundled map does not carry, so a - None here now means a genuinely unpriceable model (an inexpressible - prompt-length surcharge, or a provider with no live card to read) rather - than the routine case it was. + "unknown ≠ free" discipline. On the egg-litellm image the live pricing patch + supplies OpenRouter's published rates for slugs the bundled map does not + carry, so a None here now means a genuinely unpriceable model (an + inexpressible prompt-length surcharge, or a provider with no live card to + read) rather than the routine case it was. + + "Independently of whether the provider reported a bill" holds only where the + provider reported no charge — which on egg's BYOK routes is every call. From + 1.94.0, litellm copies a reported ``usage.cost`` into ``_hidden_params`` and + ``response_cost_calculator`` returns it before consulting the pricing map at + all, so on a route that does bill a positive figure this returns ``cost`` + itself and the two fields stop being independent. The image's billable-cost + gate patch narrows that propagation to a positive, finite charge — it is + also what keeps the live pricing patch reachable on the streaming path, + since that patch lives inside ``completion_cost()`` and the short circuit + returns before it. Reads the top-level ``response_cost`` first, then falls back to ``standard_logging_object.response_cost`` — the latter is LiteLLM's @@ -376,10 +401,11 @@ def _extract_model(mcd): # # ``stream`` is included because it selects which of the two paths the cost on # this line came through (see the module docstring), and it was the reason -# ``cost`` read null on every line before patch 11 — worth having next to the -# number rather than inferred, and worth keeping now that the null case is rare -# enough to need explaining when it happens. ``max_tokens`` and ``n`` are not -# sampling knobs but shape the generation, and are cheap to carry. +# ``cost`` read null on every line before 1.94.0 and the cost_details +# preservation patch — worth having next to the number rather than inferred, and +# worth keeping now that the null case is rare enough to need explaining when it +# happens. ``max_tokens`` and ``n`` are not sampling knobs but shape the +# generation, and are cheap to carry. _REQUEST_PARAM_KEYS = ( "temperature", "top_p", @@ -643,7 +669,7 @@ def _extract_request_params(mcd): Source is ``model_call_details['optional_params']``: LiteLLM's POST-mapping parameter set, i.e. the dict that becomes the upstream - request body. Verified against the pinned litellm 1.86.2 on the + request body. Verified against the pinned litellm 1.94.0 on the ``anthropic_messages`` route (the route every Claude Code agent request takes), streaming and non-streaming — the streaming case matters most, since that is essentially all real agent traffic. @@ -845,7 +871,7 @@ def _record(self, mcd, response_obj): # /v1/messages route (#3624): stock LiteLLM rewrites the # caller's ``thinking`` block into a bucketed # ``reasoning_effort``, but that bucket is a cap below the - # model default, so egg-litellm's patch 9 gates the + # model default, so egg-litellm's patch 6 gates the # synthesis off by default. A missing key here means the # request ran at the model's own reasoning depth, not that # the field failed to record. diff --git a/config/litellm/drop_params_visibility.py b/config/litellm/drop_params_visibility.py index 43e755179..ee5910fa2 100644 --- a/config/litellm/drop_params_visibility.py +++ b/config/litellm/drop_params_visibility.py @@ -2,7 +2,7 @@ ``drop_params`` exists so an unsupported parameter does not fail the whole request, and that tradeoff is right. But dropping a parameter *changes -generation behaviour*, and in stock LiteLLM 1.86.2 it happens with no signal +generation behaviour*, and in stock LiteLLM it happens with no signal at all: the branch that pops them is a bare loop with no logging. A ``reasoning_effort``, ``temperature`` or penalty set in a proxy config simply never reaches the provider, and nothing in the logs or the response says so. @@ -15,14 +15,14 @@ to notice (jwbron/egg#3620, #3624). One log line would have made it a five-minute question. -Patch 7 fixes the OpenRouter false-negative specifically; this covers the rest. +Patch 4 fixes the OpenRouter false-negative specifically; this covers the rest. A drop can still be *correct* and worth knowing about: ``poolside/laguna-s-2.1`` genuinely does not accept ``reasoning_effort``, so the knob is dropped on purpose, and without this the operator has no way to learn why their config line does nothing. Mirrors jwbron/litellm#7 (merged into the fork the host proxy runs). The -cluster image pins stock 1.86.2, which predates it, hence this patch. +cluster image pins a stock release, which does not carry it, hence this patch. """ # Warn-once bookkeeping, keyed by (provider, model, sorted dropped param diff --git a/config/litellm/openrouter_capabilities.py b/config/litellm/openrouter_capabilities.py index ddb9bb8b2..2370661b3 100644 --- a/config/litellm/openrouter_capabilities.py +++ b/config/litellm/openrouter_capabilities.py @@ -33,7 +33,7 @@ cost: ``supports_reasoning: true`` alone makes stock ``get_supported_openai_params`` admit ``thinking``, which Patch 2's notes explain would forward an Anthropic-shaped block verbatim to a provider that - expects ``reasoning``. Patch 7 remains the only path by which a parameter + expects ``reasoning``. Patch 4 remains the only path by which a parameter becomes admissible, and it admits exactly ``reasoning_effort``. * **Tiered rate cards are translated only where LiteLLM can hold them.** OpenRouter expresses a long-context surcharge as ``pricing.overrides`` — a @@ -79,7 +79,7 @@ both, since one fetch serves both. * ``LITELLM_OPENROUTER_PRICING=0`` disables only the pricing half, leaving the parameter lookup running. For an operator who wants LiteLLM's bundled map to - be the sole authority on cost while keeping Patch 7's parameter fix. + be the sole authority on cost while keeping Patch 4's parameter fix. * ``LITELLM_OPENROUTER_CAPABILITY_TTL`` seconds between refreshes (default 3600). ``0`` disables caching and re-fetches on every lookup — a debugging aid, not a production setting. @@ -584,7 +584,7 @@ def _log_fetch_failure(message: str, *args: object) -> None: A permanently unreachable endpoint is the exact case where behaviour silently reverts to the model-cost map, and litellm's default log level is INFO — so debug-only reporting reproduces, one file over, the silence - Patch 8 exists to remove. Warning once is enough to be findable without + Patch 5 exists to remove. Warning once is enough to be findable without turning an offline deployment into a log flood. """ global _WARNED_FETCH_FAILURE diff --git a/config/litellm/patch_litellm_cache.py b/config/litellm/patch_litellm_cache.py index 25603133f..15784e0fa 100644 --- a/config/litellm/patch_litellm_cache.py +++ b/config/litellm/patch_litellm_cache.py @@ -1,34 +1,44 @@ #!/usr/bin/env python3 -"""Apply egg's LiteLLM prompt-cache, reasoning-stream and cost patches at image-build time. +"""Apply egg's LiteLLM cache, reasoning and cost patches at image-build time. LiteLLM's stock Anthropic->OpenAI translation (the path Claude Code's ``/v1/messages`` requests take when routed at a non-Claude OpenRouter -backend) drops prompt-cache hits for Qwen/DeepSeek and mis-streams -reasoning models, and its OpenRouter param gate reads a model-cost map -that does not carry current OpenRouter slugs, and it discards -caller-specified params in total silence, while manufacturing a -reasoning ceiling nobody asked for, never sending prior-turn reasoning -back, and it destroys the provider's own bill during stream reassembly -while its rate card cannot price the route either. Twelve independent -gaps cause it; this script closes all twelve by editing the installed -``litellm`` -package in place (and installing five new modules), then -``config/litellm/Dockerfile`` bakes the result into the ``egg-litellm`` -image. +backend) drops prompt-cache hits for Qwen/DeepSeek, reads a model-cost map +that carries neither the parameters nor the prices of current OpenRouter +slugs, discards caller-specified params in total silence, manufactures a +reasoning ceiling nobody asked for, never sends prior-turn reasoning back, +loses the BYOK half of the provider's bill during stream reassembly, and — +new at 1.94.0 — hands its own cost calculator a BYOK zero that suppresses +the estimate entirely. Ten independent gaps cause it; this script closes +all ten by editing the installed ``litellm`` package in place (and +installing five new modules), then ``config/litellm/Dockerfile`` bakes the +result into the ``egg-litellm`` image. -The cache patches mirror the host-side ``relitellm`` script -(jwbron/dotfiles), which was validated empirically against -``litellm==1.86.2``: cache hit rate on Qwen via OpenRouter went 0% -> -~99.99%, ~10x input-cost cut on identical-prefix turns. The reasoning -patches mirror jwbron/litellm#4, validated against Kimi K2.7-Code via -OpenRouter. The image pins that same version (see the Dockerfile -``FROM``), so the needles below are known to match. +Pinned to ``litellm==1.94.0`` (see the Dockerfile ``FROM``), so the needles +below are known to match. The bump from 1.86.2 retired four patches whose +fixes upstream absorbed — the streaming ``reasoning_content`` thinking +block, the first-delta requeue on block transitions (both sync and async), +and the ``prompt_tokens_details.cached_tokens`` fallback — narrowed a +fifth (8) to the half upstream still does not carry, and ADDED one (10) for +a regression the bump itself brings in. Patch 7 arrived separately, from +#3698, and its needle was re-verified against the 1.94.0 wheel rather than +assumed to survive the bump. See issue #3697 for the audit, and read its +warning before assuming the next bump retires anything: TWO of the patches +below (2 and 5) had their needles stop matching at 1.94.0 purely because +upstream reflowed the surrounding code, and retiring them on the strength +of that miss would have silently shipped an image paying full input rate on +every Qwen turn. 1. ``CacheControlSupportedModels`` (openrouter/chat/transformation.py) add QWEN + DEEPSEEK so ``cache_control`` survives the OpenRouter handler's strip step. Without it every turn pays full input rate. - 2. ``_add_cache_control_to_target`` cache_control gate (anthropic adapter - transformation.py) broaden ONLY that gate to qwen + deepseek so + 1.94.0 still lists only claude/gemini/minimax/glm/z-ai. NOTE the + jwbron/litellm fork adds QWEN but deliberately omits DEEPSEEK, on the + grounds that DeepSeek caching is automatic/prefix-based and ignores + ``cache_control`` — if that holds, egg's DEEPSEEK arm is inert rather + than wrong, and is a candidate for removal once measured. + 2. ``_add_cache_control_if_applicable`` cache_control gate (anthropic + adapter transformation.py) broaden ONLY that gate to qwen + deepseek so ``cache_control`` survives the Anthropic->OpenAI translation the ``/v1/messages`` endpoint forces. Without it patch 1 never sees ``cache_control`` (stripped earlier). We deliberately do NOT widen the @@ -45,44 +55,10 @@ Claude Code injects this as the FIRST system text block, ahead of the ``cache_control`` marker, carrying a per-request ``cch=``; that hash invalidates the prefix-cache key every turn, so - ``cache_read_input_tokens`` stays 0 forever. The sibling - ``messages/transformation.py`` path already filters it via - ``_filter_billing_headers_from_system``; the adapter path missed it. - 4. ``_translate_streaming_openai_chunk_to_anthropic_content_block`` - (anthropic adapter transformation.py) add a ``reasoning_content`` - branch that opens a proper ``thinking`` content block. OpenRouter-style - reasoning models stream ``delta.reasoning_content`` rather than - Anthropic-native ``delta.thinking_blocks``; without this patch the - block start is typed ``text`` while the deltas are - ``thinking_delta`` — a malformed stream that clients (e.g. Claude - Code) render as visible assistant text and feed back as assistant - content on later turns. - 5. ``AnthropicStreamWrapper`` block-transition requeue (anthropic adapter - streaming_iterator.py) preserve the trigger chunk's first delta on - text/thinking block transitions, not just on ``input_json_delta`` - tool transitions. Without it the first reasoning token of every - thinking block and the first answer token after thinking are silently - dropped; a one-chunk answer can vanish entirely. Applied to both the - sync ``__next__`` and async ``__anext__`` paths. - 6. ``AnthropicStreamWrapper`` usage merge (same streaming_iterator.py) - report provider-automatic cache hits in the streamed usage. The stock - code subtracts ``prompt_tokens_details.cached_tokens`` from - ``input_tokens`` but only emits ``cache_read_input_tokens`` from the - internal ``_cache_read_input_tokens`` field, which OpenAI/OpenRouter- - style usage never populates (only Anthropic- and DeepSeek-native - fields do). On any auto-caching route (Moonshot kimi-k3 ~94% hit - rate, Fireworks deepseek-flash, DeepInfra glm) the cached tokens - simply vanish from the usage Anthropic-format clients see: Claude - Code's context tracking sees only the uncached tail (~0% context - forever), so auto-compaction never triggers and the orchestrator's - transcript-derived token accounting undercounts. Mirrors the fix - upstreamed via jwbron/litellm#6 (upstream now falls back to - ``prompt_tokens_details.cached_tokens`` natively). Unlike Patch 5, - this has no sync ``__next__`` twin: the cache-token usage merge - exists only in the async ``__anext__`` path upstream (the sync path - has no equivalent merge block), and that async path is the one the - litellm proxy drives for Claude Code streaming. - 7. ``OpenrouterConfig.get_supported_openai_params`` + ``cache_read_input_tokens`` stays 0 forever. 1.94.0 filters it in + ``anthropic/chat/transformation.py`` and + ``messages/transformation.py`` but still not in this adapter path. + 4. ``OpenrouterConfig.get_supported_openai_params`` (openrouter/chat/transformation.py) consult OpenRouter's published per-model ``supported_parameters`` instead of only the bundled model-cost map. The stock gate asks ``litellm.supports_reasoning``, @@ -90,10 +66,9 @@ ships new slugs faster than that map tracks them, so a current model answers False. The gate is a bare ``if``, so it fails CLOSED, and ``drop_params: true`` discards the parameter with no exception - and no log line. Every OpenRouter slug egg routes is absent from the - 1.86.2 map, so a reasoning knob set on any of them never reached the - wire. The companion module ``llms/openrouter/_egg_capabilities.py`` - (installed by ``NEW_MODULES``) reads OpenRouter's unauthenticated + and no log line. The companion module + ``llms/openrouter/_egg_capabilities.py`` (installed by + ``NEW_MODULES``) reads OpenRouter's unauthenticated ``/api/v1/models`` and is UNIONED with the map answer, never subtractive: ``supported_parameters`` under-reports ``reasoning_effort`` (deepseek-r1 advertises only ``reasoning``, @@ -102,30 +77,30 @@ admitted — OpenRouter's ``reasoning`` field is a different wire shape from Anthropic's ``thinking``, not a spelling of it. Fails soft: any fetch error yields no opinion and the stock path runs unchanged. - Mirrors jwbron/litellm#8 and jwbron/egg#3624. Patch 9 is the other + Mirrors jwbron/litellm#8 and jwbron/egg#3624. Patch 6 is the other half of this one: read them together. - 8. ``get_optional_params`` drop site (utils.py) log what - ``drop_params`` discards. Stock 1.86.2 pops unsupported params in a - bare loop with no logging, so a param set in a proxy config that - never reaches the provider is a real behavioural difference with no - signal attached — the condition that made patch 7's bug take a full - investigation to find. Patch 7 removes the OpenRouter + 5. ``get_optional_params`` drop site (utils.py) log what + ``drop_params`` discards. Stock pops unsupported params in a bare + loop with no logging, so a param set in a proxy config that never + reaches the provider is a real behavioural difference with no signal + attached — the condition that made patch 4's bug take a full + investigation to find. Patch 4 removes the OpenRouter false-negative; this covers the rest, including drops that are CORRECT: laguna-s-2.1 genuinely does not accept ``reasoning_effort``, so it is dropped on purpose and the operator otherwise has no way to learn why their config line does nothing. Deduped per - (provider, model, param-set) and bounded. NOTE the needle: 1.86.2 has - two ``drop_params`` branches in utils.py and the shared condition - alone matches the wrong one, so the needle includes the pop loop. + (provider, model, param-set) and bounded. NOTE the needle: 1.94.0 + still has two ``drop_params`` branches in utils.py sharing one + spelling, so the needle includes the pop loop. Mirrors jwbron/litellm#7, merged into the fork the HOST proxy runs; - the cluster image pins stock 1.86.2, which predates it. The message - offers the ``allowed_openai_params`` remedy GATED on the params - having come from this model's ``litellm_params``, and names the - synthesized case alongside it: the param most often dropped here is - one litellm manufactured from the request itself (see 9), so an - unconditional config edit would send that operator hunting for a - line that does not exist. - 9. ``_translate_thinking_to_openai`` (anthropic adapter + BerriAI has not taken it. The message offers the + ``allowed_openai_params`` remedy GATED on the params having come from + this model's ``litellm_params``, and names the synthesized case + alongside it: the param most often dropped here is one litellm + manufactured from the request itself (see 6), so an unconditional + config edit would send that operator hunting for a line that does not + exist. + 6. ``_translate_thinking_to_openai`` (anthropic adapter transformation.py) stop synthesizing ``reasoning_effort`` from the caller's ``thinking`` block for non-Claude models. On ``/v1/messages`` — egg's primary route — Claude Code sends @@ -135,15 +110,15 @@ non-Claude branch where the adapter REPLACES that block with a bucketed ``reasoning_effort``. Nothing in ``litellm-models.yaml`` is involved: the value is manufactured per request. That was harmless - only because patch 7's bug dropped it; the measurements in + only because patch 4's bug dropped it; the measurements in jwbron/egg#3624 show the bucket is a CAP BELOW the model default (kimi-k3: 3130 reasoning tokens with no param, 340 with - ``reasoning_effort: high``, non-overlapping), so shipping patch 7 + ``reasoning_effort: high``, non-overlapping), so shipping patch 4 without this would cut reasoning ~9x per agent turn with no config - file to point at and nothing logged — patch 8 fires only on drops, + file to point at and nothing logged — patch 5 fires only on drops, and this param would no longer be dropped. Gating the synthesis (off by default, ``LITELLM_ANTHROPIC_THINKING_TO_REASONING_EFFORT=1`` to - restore stock) keeps patch 7's actual goal: a knob an operator + restore stock) keeps patch 4's actual goal: a knob an operator configured reaches the wire, one nobody configured does not. The Claude branch is untouched, and so is an effort the CALLER stated outright: on an adaptive request (``thinking: {"type": "adaptive"}`` @@ -154,7 +129,7 @@ with the derived effort, because stock carries the summary only as a field of the ``reasoning_effort`` dict and there is no wire shape for "summary, no effort". - 10. ``OpenrouterConfig.transform_request`` + 7. ``OpenrouterConfig.transform_request`` (openrouter/chat/transformation.py) carry prior-turn assistant reasoning back to the provider. The Anthropic adapter converts incoming ``thinking`` content blocks into @@ -195,50 +170,71 @@ OpenRouter's ``reasoning_details`` is the right shape — a separate change), and ``LITELLM_OPENROUTER_REASONING_ROUNDTRIP=0`` backs the patch out of a live cluster without an image rebuild, as for - patches 7 and 9. Fails soft per message: a block it cannot parse + patches 4 and 6. Fails soft per message: a block it cannot parse leaves that message exactly as it arrived, and says so once. - NOTE the DIRECTION — this is request-path - (client -> provider); patches 4, 5a and 5b are response-path - (provider -> client). Adjacent, not the same thing. The gap is - upstream's and predates every egg and fork change: jwbron/litellm#8 - touches only ``get_supported_openai_params``, which acts on - ``optional_params`` and cannot reach a message field. - 11. ``ChunkProcessor.calculate_usage`` + NOTE the DIRECTION — this is request-path (client -> provider), + where the three patches the 1.94.0 bump retired (the streaming + ``reasoning_content`` thinking block and the two first-delta + requeues) were response-path (provider -> client). Adjacent, not + the same thing, which is why upstream absorbing those says nothing + about this one: its needle was checked against the 1.94.0 wheel + directly and still matches, once. The gap is upstream's and + predates every egg and fork change: jwbron/litellm#8 touches only + ``get_supported_openai_params``, which acts on ``optional_params`` + and cannot reach a message field. + 8. ``ChunkProcessor.calculate_usage`` (litellm_core_utils/streaming_chunk_builder_utils.py) carry the - provider-billed ``cost`` / ``cost_details`` across stream - reassembly. OpenRouter reports what it charged on the final usage - chunk and stock already asks for it (``transform_request`` sets - ``usage: {"include": true}`` unconditionally), but this rebuild - enumerates token counts only and re-constructs ``Usage`` from its - own ``model_dump()``, so the bill is dropped. Claude Code streams - every ``/v1/messages`` request, so this is ~100% of routed traffic: - 1252 of 1252 sampled ``cost_callback`` lines on run 6 carried - ``cost: null`` (#3691). The companion module - ``litellm_core_utils/_egg_stream_cost.py`` transports the two - fields and interprets neither — a zero ``cost`` is the BYOK truth - and its fall-through partner sits under ``cost_details``. - 12. ``_get_model_info_helper`` (utils.py) price OpenRouter slugs the - bundled map has never heard of. Same root cause as 7, second - symptom: ``model_prices_and_context_window.json`` does not carry - current slugs, so the lookup raises "This model isn't mapped yet", - LiteLLM's ``response_cost`` is never computed, and egg's - ``cost_estimated`` reads null beside the null ``cost`` that 11 - fixes. The hook sits at the raise site, after every stock lookup - has failed, so a mapped slug keeps the bundled answer and the live - card can add a model but never reprice one. ``_egg_capabilities`` - grows a second entry point for this off the roster it already - caches; it answers for OpenRouter alone, carries cost fields only - (a ``supports_*`` flag through this door would change parameter - admission, which is 7's job), and translates a prompt-length - surcharge only into the rate slots LiteLLM actually has, declining - the whole card when one does not fit rather than registering a base - tier that would silently under-report the long prompts agent traffic - is made of. + provider's ``cost_details`` across stream reassembly. 1.86.2 dropped + the whole bill here, which put ``cost: null`` on 1252 of 1252 sampled + calls (#3691); 1.94.0 carries ``cost`` natively and feeds it to the + cost calculator, but still not ``cost_details`` — which is where the + BYOK number lives, since OpenRouter's top-level ``cost`` is 0 when + billing routes past them. The companion module + ``litellm_core_utils/_egg_stream_cost.py`` never overwrites a value + litellm already set, so it no-ops on ``cost`` and supplies only the + missing half. + 9. ``_get_model_info_helper`` (utils.py) price OpenRouter slugs the + bundled map has never heard of. Same root cause as 4, second + symptom: the map does not carry current slugs, so the lookup raises + "This model isn't mapped yet", ``response_cost`` is never computed, + and ``cost_estimated`` reads null. The hook sits at the raise site, + after every stock lookup has failed, so a mapped slug keeps the + bundled answer and the live card can add a model but never reprice + one. ``_egg_capabilities`` grows a second entry point for this off + the roster it already caches; it answers for OpenRouter alone, + carries cost fields only (a ``supports_*`` flag through this door + would change parameter admission, which is 4's job), and translates + a prompt-length surcharge only into the rate slots LiteLLM actually + has, declining the whole card when one does not fit rather than + registering a base tier that would silently under-report the long + prompts agent traffic is made of. Proposed upstream as + jwbron/litellm#10. + 10. ``CustomStreamWrapper._propagate_usage_cost_to_hidden_params`` + (litellm_core_utils/streaming_handler.py) require the provider's + ``cost`` to be POSITIVE and finite before it is allowed to stand in + for the cost calculator. New at 1.94.0 and a regression the bump + itself brings in: the method did not exist in 1.86.2, and it copies + ``usage.cost`` into ``_hidden_params`` on both streaming completion + paths, where ``response_cost_calculator`` returns it verbatim and + ``completion_cost()`` never runs. Its guard is ``is not None``, and + under BYOK OpenRouter's ``cost`` is a literal ``0.0`` — so on egg's + route (Claude Code streams every request) the calculator short-circuits + on a zero, ``cost_estimated`` reads null on every streamed call, and + patch 9 is dead code, because its hook sits inside ``completion_cost``. + Requiring a positive figure restores both for the BYOK case and leaves + the real-charge case alone. NOTE what that leaves standing: where a + provider DOES report a positive charge, 1.94.0 seeds its own calculator + with it, so ``cost_estimated`` mirrors ``cost`` rather than being an + independent rate-card read — see ``cost_callback._extract_estimated_cost``. + Non-finite is excluded too: ``+inf`` clears a bare ``> 0`` and would be + written into a spend header as ``Infinity``. Idempotent: each patch detects whether it is already applied. Fails loudly (non-zero exit) if a needle is missing, so a LiteLLM version bump that moves the code surfaces at build time instead of silently shipping an unpatched image that bills full input rate or drops reasoning tokens. +A needle miss means "look at this", NOT "upstream fixed it" — see the +1.94.0 note above. """ import ast @@ -341,6 +337,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - F3 = "llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py" F4 = "utils.py" F5 = "litellm_core_utils/streaming_chunk_builder_utils.py" +F6 = "litellm_core_utils/streaming_handler.py" # Every patch as a self-contained spec: (file, present marker, needle, # replacement, label). Module-level so tests can apply them to a checked-in @@ -363,16 +360,29 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - ' QWEN = "qwen"\n' ' DEEPSEEK = "deepseek"\n' ), - "label": "Patch 1/12 (CacheControlSupportedModels)", + "label": "Patch 1/10 (CacheControlSupportedModels)", }, # Patch 2 — broaden ONLY the cache_control gate (not the shared # is_anthropic_claude_model predicate, which also gates thinking # translation — see module docstring). Touches the single call site in - # _add_cache_control_to_target. + # _add_cache_control_if_applicable. + # + # NOT retired by the 1.94.0 bump, despite looking like it was. Upstream + # rewrote this gate — collapsed it to one line and added + # ``is_bedrock_arn_model`` — so the old needle stopped matching, and a + # needle miss is indistinguishable from "upstream fixed it" if you only + # look at the miss. Upstream added no qwen/deepseek arm: 1.94.0's + # ``CacheControlSupportedModels`` still lists only claude/gemini/minimax/ + # glm/z-ai. Retiring this on the strength of the miss would have shipped an + # image that pays FULL INPUT RATE on every Qwen turn — the most expensive + # single regression available here, and a silent one. { "file": F2, "present": "# egg cache patch. Broaden ONLY the cache_control gate", - "needle": " if cache_control and model and self.is_anthropic_claude_model(model):\n", + "needle": ( + " if cache_control and model and " + "(self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model)):\n" + ), "replacement": ( " # egg cache patch. Broaden ONLY the cache_control gate to\n" " # cover the OpenRouter qwen/deepseek routes. Do NOT widen\n" @@ -388,12 +398,13 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " and model\n" " and (\n" " self.is_anthropic_claude_model(model)\n" + " or self.is_bedrock_arn_model(model)\n" ' or "qwen" in _model_lower\n' ' or "deepseek" in _model_lower\n' " )\n" " ):\n" ), - "label": "Patch 2/12 (cache_control gate)", + "label": "Patch 2/10 (cache_control gate)", }, # Patch 3 — drop x-anthropic-billing-header during Anthropic->OpenAI translation. { @@ -427,337 +438,9 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - ' "text": text,\n' " }\n" ), - "label": "Patch 3/12 (x-anthropic-billing-header filter)", - }, - # Patch 4 — OpenRouter-style reasoning_content must open a thinking - # content block, not fall through to a text block. The bare - # ``thinking_blocks`` elif appears in two sibling functions; we anchor the - # needle on the preceding text-block elif (``choice.delta.content is not - # None ...``), which is unique to - # _translate_streaming_openai_chunk_to_anthropic_content_block — the other - # function's preceding branch is the ``tool_calls`` block. Without this - # anchor, an upstream reorder could silently retarget the str.replace. - { - "file": F2, - "present": "# egg reasoning patch: OpenRouter-style reasoning_content", - "needle": ( - " elif choice.delta.content is not None and len(choice.delta.content) > 0:\n" - ' return "text", TextBlock(type="text", text="")\n' - " elif isinstance(choice, StreamingChoices) and hasattr(\n" - ' choice.delta, "thinking_blocks"\n' - " ):\n" - ), - "replacement": ( - " elif choice.delta.content is not None and len(choice.delta.content) > 0:\n" - ' return "text", TextBlock(type="text", text="")\n' - " elif isinstance(choice, StreamingChoices) and getattr(\n" - ' choice.delta, "reasoning_content", None\n' - " ):\n" - " # egg reasoning patch: OpenRouter-style reasoning_content\n" - " # streams must open a thinking content block. Without\n" - " # this, thinking_delta events are emitted inside a\n" - " # text-typed block, which clients render as visible\n" - " # assistant text and feed back as assistant content.\n" - ' return "thinking", ChatCompletionThinkingBlock(\n' - ' type="thinking", thinking="", signature=""\n' - " )\n" - " elif isinstance(choice, StreamingChoices) and hasattr(\n" - ' choice.delta, "thinking_blocks"\n' - " ):\n" - ), - "label": "Patch 4/12 (reasoning_content thinking block)", - }, - # Patch 5a — sync __next__: don't drop the first delta on text or - # thinking block transitions. - { - "file": F3, - "present": "# egg reasoning patch (sync first-delta)", - "needle": ( - " # Queue the sequence: content_block_stop -> content_block_start\n" - " # For text blocks the trigger chunk is not emitted as a separate\n" - " # delta because content_block_start carries the information.\n" - " # For tool_use blocks we must also emit the trigger chunk's delta\n" - " # when it carries input_json_delta data, because some providers\n" - " # (e.g. xAI, Gemini) include tool arguments in the same streaming\n" - " # chunk as the function name/id.\n" - "\n" - " # 1. Stop current content block\n" - " self.chunk_queue.append(\n" - " {\n" - ' "type": "content_block_stop",\n' - ' "index": max(self.current_content_block_index - 1, 0),\n' - " }\n" - " )\n" - "\n" - " # 2. Start new content block\n" - " self.chunk_queue.append(\n" - " {\n" - ' "type": "content_block_start",\n' - ' "index": self.current_content_block_index,\n' - ' "content_block": self.current_content_block_start,\n' - " }\n" - " )\n" - "\n" - " # 3. If the trigger chunk carries tool argument data, queue it\n" - " # so the input_json_delta is not silently dropped.\n" - " if (\n" - ' processed_chunk.get("type") == "content_block_delta"\n' - ' and isinstance(processed_chunk.get("delta"), dict)\n' - ' and processed_chunk["delta"].get("type") == "input_json_delta"\n' - ' and processed_chunk["delta"].get("partial_json")\n' - " ):\n" - " self.chunk_queue.append(processed_chunk)\n" - ), - "replacement": ( - " # Queue the sequence: content_block_stop -> content_block_start,\n" - " # then re-queue the trigger chunk's delta when it carries payload\n" - " # the new content_block_start doesn't already include (see step 3).\n" - " # egg reasoning patch (sync first-delta)\n" - "\n" - " # 1. Stop current content block\n" - " self.chunk_queue.append(\n" - " {\n" - ' "type": "content_block_stop",\n' - ' "index": max(self.current_content_block_index - 1, 0),\n' - " }\n" - " )\n" - "\n" - " # 2. Start new content block\n" - " self.chunk_queue.append(\n" - " {\n" - ' "type": "content_block_start",\n' - ' "index": self.current_content_block_index,\n' - ' "content_block": self.current_content_block_start,\n' - " }\n" - " )\n" - "\n" - " # 3. If the trigger chunk itself carries delta payload not\n" - " # already embedded in the new content_block_start, queue it\n" - " # so the first delta of the block isn't silently dropped:\n" - " # - input_json_delta: some providers (e.g. xAI, Gemini)\n" - " # include tool arguments in the same streaming chunk as\n" - " # the function name/id.\n" - " # - text_delta: the text block start is always empty, so\n" - " # the trigger chunk's text is the block's first token.\n" - " # - thinking_delta: same, but only when the block start\n" - " # doesn't already embed the thinking content (it does\n" - " # for providers that send Anthropic-native\n" - " # thinking_blocks).\n" - ' if processed_chunk.get("type") == "content_block_delta" and isinstance(\n' - ' processed_chunk.get("delta"), dict\n' - " ):\n" - ' trigger_delta = processed_chunk["delta"]\n' - ' trigger_delta_type = trigger_delta.get("type")\n' - " if (\n" - " (\n" - ' trigger_delta_type == "input_json_delta"\n' - ' and trigger_delta.get("partial_json")\n' - " )\n" - " or (\n" - ' trigger_delta_type == "text_delta"\n' - ' and trigger_delta.get("text")\n' - " )\n" - " or (\n" - ' trigger_delta_type == "thinking_delta"\n' - ' and trigger_delta.get("thinking")\n' - ' and not self.current_content_block_start.get("thinking")\n' - " )\n" - " ):\n" - " self.chunk_queue.append(processed_chunk)\n" - ), - "label": "Patch 5a/12 (sync first-delta requeue)", - }, - # Patch 5b — async __anext__: same first-delta preservation. - { - "file": F3, - "present": "# egg reasoning patch (async first-delta)", - "needle": ( - " # Queue the sequence: content_block_stop -> content_block_start\n" - " # For text blocks the trigger chunk is not emitted as a separate\n" - " # delta because content_block_start carries the information.\n" - " # For tool_use blocks we must also emit the trigger chunk's delta\n" - " # when it carries input_json_delta data, because some providers\n" - " # (e.g. xAI, Gemini) include tool arguments in the same streaming\n" - " # chunk as the function name/id.\n" - "\n" - " # 1. Stop current content block\n" - " self.chunk_queue.append(\n" - " {\n" - ' "type": "content_block_stop",\n' - ' "index": max(self.current_content_block_index - 1, 0),\n' - " }\n" - " )\n" - " self.chunk_queue.append(\n" - " {\n" - ' "type": "content_block_start",\n' - ' "index": self.current_content_block_index,\n' - ' "content_block": self.current_content_block_start,\n' - " }\n" - " )\n" - "\n" - " # 3. If the trigger chunk carries tool argument data, queue it\n" - " # so the input_json_delta is not silently dropped.\n" - " if (\n" - ' processed_chunk.get("type") == "content_block_delta"\n' - ' and isinstance(processed_chunk.get("delta"), dict)\n' - ' and processed_chunk["delta"].get("type")\n' - ' == "input_json_delta"\n' - ' and processed_chunk["delta"].get("partial_json")\n' - " ):\n" - " self.chunk_queue.append(processed_chunk)\n" - ), - "replacement": ( - " # Queue the sequence: content_block_stop -> content_block_start,\n" - " # then re-queue the trigger chunk's delta when it carries payload\n" - " # the new content_block_start doesn't already include (see step 3).\n" - " # egg reasoning patch (async first-delta)\n" - "\n" - " # 1. Stop current content block\n" - " self.chunk_queue.append(\n" - " {\n" - ' "type": "content_block_stop",\n' - ' "index": max(self.current_content_block_index - 1, 0),\n' - " }\n" - " )\n" - " self.chunk_queue.append(\n" - " {\n" - ' "type": "content_block_start",\n' - ' "index": self.current_content_block_index,\n' - ' "content_block": self.current_content_block_start,\n' - " }\n" - " )\n" - "\n" - " # 3. If the trigger chunk itself carries delta payload\n" - " # not already embedded in the new content_block_start,\n" - " # queue it so the first delta of the block isn't\n" - " # silently dropped:\n" - " # - input_json_delta: some providers (e.g. xAI, Gemini)\n" - " # include tool arguments in the same streaming chunk\n" - " # as the function name/id.\n" - " # - text_delta: the text block start is always empty,\n" - " # so the trigger chunk's text is the block's first\n" - " # token.\n" - " # - thinking_delta: same, but only when the block start\n" - " # doesn't already embed the thinking content (it does\n" - " # for providers that send Anthropic-native\n" - " # thinking_blocks).\n" - ' if processed_chunk.get("type") == "content_block_delta" and isinstance(\n' - ' processed_chunk.get("delta"), dict\n' - " ):\n" - ' trigger_delta = processed_chunk["delta"]\n' - ' trigger_delta_type = trigger_delta.get("type")\n' - " if (\n" - " (\n" - ' trigger_delta_type == "input_json_delta"\n' - ' and trigger_delta.get("partial_json")\n' - " )\n" - " or (\n" - ' trigger_delta_type == "text_delta"\n' - ' and trigger_delta.get("text")\n' - " )\n" - " or (\n" - ' trigger_delta_type == "thinking_delta"\n' - ' and trigger_delta.get("thinking")\n' - ' and not self.current_content_block_start.get("thinking")\n' - " )\n" - " ):\n" - " self.chunk_queue.append(processed_chunk)\n" - ), - "label": "Patch 5b/12 (async first-delta requeue)", - }, - # Patch 6 — streamed usage must report provider-automatic cache hits. - # The needle spans the whole usage-merge region so both edit points - # (the cached_tokens hoist and the cache_read fallback) land atomically. - { - "file": F3, - "present": "# egg cache patch (streaming cache_read fallback)", - "needle": ( - " # Add usage to the held chunk\n" - " uncached_input_tokens = chunk.usage.prompt_tokens or 0\n" - " if (\n" - ' hasattr(chunk.usage, "prompt_tokens_details")\n' - " and chunk.usage.prompt_tokens_details\n" - " ):\n" - " cached_tokens = (\n" - " getattr(\n" - ' chunk.usage.prompt_tokens_details, "cached_tokens", 0\n' - " )\n" - " or 0\n" - " )\n" - " uncached_input_tokens -= cached_tokens\n" - "\n" - " usage_dict: UsageDelta = {\n" - ' "input_tokens": uncached_input_tokens,\n' - ' "output_tokens": chunk.usage.completion_tokens or 0,\n' - " }\n" - " # Add cache tokens if available (for prompt caching support)\n" - " if (\n" - ' hasattr(chunk.usage, "_cache_creation_input_tokens")\n' - " and chunk.usage._cache_creation_input_tokens > 0\n" - " ):\n" - ' usage_dict["cache_creation_input_tokens"] = (\n' - " chunk.usage._cache_creation_input_tokens\n" - " )\n" - " if (\n" - ' hasattr(chunk.usage, "_cache_read_input_tokens")\n' - " and chunk.usage._cache_read_input_tokens > 0\n" - " ):\n" - ' usage_dict["cache_read_input_tokens"] = (\n' - " chunk.usage._cache_read_input_tokens\n" - " )\n" - ), - "replacement": ( - " # Add usage to the held chunk\n" - " uncached_input_tokens = chunk.usage.prompt_tokens or 0\n" - " cached_tokens = 0\n" - " if (\n" - ' hasattr(chunk.usage, "prompt_tokens_details")\n' - " and chunk.usage.prompt_tokens_details\n" - " ):\n" - " cached_tokens = (\n" - " getattr(\n" - ' chunk.usage.prompt_tokens_details, "cached_tokens", 0\n' - " )\n" - " or 0\n" - " )\n" - " uncached_input_tokens -= cached_tokens\n" - "\n" - " usage_dict: UsageDelta = {\n" - ' "input_tokens": uncached_input_tokens,\n' - ' "output_tokens": chunk.usage.completion_tokens or 0,\n' - " }\n" - " # Add cache tokens if available (for prompt caching support)\n" - " if (\n" - ' hasattr(chunk.usage, "_cache_creation_input_tokens")\n' - " and chunk.usage._cache_creation_input_tokens > 0\n" - " ):\n" - ' usage_dict["cache_creation_input_tokens"] = (\n' - " chunk.usage._cache_creation_input_tokens\n" - " )\n" - " if (\n" - ' hasattr(chunk.usage, "_cache_read_input_tokens")\n' - " and chunk.usage._cache_read_input_tokens > 0\n" - " ):\n" - ' usage_dict["cache_read_input_tokens"] = (\n' - " chunk.usage._cache_read_input_tokens\n" - " )\n" - " # egg cache patch (streaming cache_read fallback).\n" - " # OpenAI/OpenRouter-style usage reports the cache hit in\n" - " # prompt_tokens_details.cached_tokens; Usage.__init__ only\n" - " # populates _cache_read_input_tokens from Anthropic- or\n" - " # DeepSeek-native fields. Without this fallback the cached\n" - " # tokens are subtracted from input_tokens above but never\n" - " # reported, so Anthropic-format clients (Claude Code\n" - " # context tracking, the orchestrator token accounting)\n" - " # undercount the prompt by the cached amount on every\n" - " # cache hit. Mirrors jwbron/litellm#6 (fixed natively\n" - " # upstream after v1.86.2).\n" - " elif cached_tokens > 0:\n" - ' usage_dict["cache_read_input_tokens"] = cached_tokens\n' - ), - "label": "Patch 6/12 (streaming cache_read fallback)", + "label": "Patch 3/10 (x-anthropic-billing-header filter)", }, - # Patch 7 — OpenrouterConfig.get_supported_openai_params: consult + # Patch 4 — OpenrouterConfig.get_supported_openai_params: consult # OpenRouter's published capabilities instead of only the bundled # model-cost map. # @@ -767,7 +450,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - # bundled map lags, so a current model answers False. The gate is a bare # ``if``, so it fails CLOSED, and ``drop_params: true`` then discards the # parameter with no exception and no log line. Every OpenRouter slug egg - # routes is absent from the 1.86.2 map (kimi-k3, glm-5.2, laguna-s-2.1, + # routes is absent from the bundled map (kimi-k3, glm-5.2, laguna-s-2.1, # deepseek-v4-*), so any reasoning knob set on them never reached the wire. # # The companion module (installed by ``NEW_MODULES`` below) reads @@ -812,7 +495,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " # supported_parameters over an unauthenticated endpoint; the bundled\n" " # model-cost map does not carry current slugs, so the stock gate\n" " # below fails closed and the knob is dropped in silence. Unioned,\n" - " # never subtractive — see patch 7 notes in patch_litellm_cache.py.\n" + " # never subtractive — see patch 4 notes in patch_litellm_cache.py.\n" " try:\n" " from litellm.llms.openrouter._egg_capabilities import (\n" " get_supported_parameters as _egg_openrouter_capabilities,\n" @@ -826,38 +509,39 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " pass\n" " try:\n" ), - "label": "Patch 7/12 (openrouter live capabilities)", + "label": "Patch 4/10 (openrouter live capabilities)", }, - # Patch 8 — get_optional_params: log what ``drop_params`` discards. + # Patch 5 — get_optional_params: log what ``drop_params`` discards. # - # Patch 7 removes the OpenRouter false-negative, but a drop can still be + # Patch 4 removes the OpenRouter false-negative, but a drop can still be # correct and still worth knowing about: laguna-s-2.1 genuinely does not # accept ``reasoning_effort``, so the knob is dropped on purpose and the # operator has no way to learn why their config line does nothing. Stock - # 1.86.2 pops unsupported params in a bare loop with no logging at all. + # pops unsupported params in a bare loop with no logging at all. # - # NEEDLE DISAMBIGUATION: 1.86.2 has TWO ``if litellm.drop_params is True or - # (...)`` sites in utils.py. The other one (~line 3303, the embeddings - # path) is followed by a bare ``pass``; this one is followed by the pop - # loop. The needle therefore includes the loop line — the shared condition - # alone would match whichever comes first and patch the wrong function. + # NOT retired by the 1.94.0 bump. Upstream reflowed the condition onto one + # line, so the old needle stopped matching — but nothing upstream logs the + # drop; the warn-once bookkeeping exists only in jwbron/litellm (PR #7), + # which BerriAI has not taken. Repointed, not deleted. # - # Mirrors jwbron/litellm#7, merged into the fork the HOST proxy runs. The - # cluster image pins stock 1.86.2, which predates it. + # NEEDLE DISAMBIGUATION: 1.94.0 still has TWO ``if litellm.drop_params is + # True or (...)`` sites in utils.py with this exact one-line spelling — the + # other (the embeddings path) is followed by a bare ``pass``, this one by + # the pop loop. Verified: the bare condition matches twice, the condition + # plus the loop matches once. The needle therefore keeps the loop lines; + # matching on the condition alone would patch whichever came first. { "file": F4, "present": "# egg drop_params visibility patch", "needle": ( - " if litellm.drop_params is True or (\n" - " drop_params is not None and drop_params is True\n" - " ):\n" + " if litellm.drop_params is True or " + "(drop_params is not None and drop_params is True):\n" " for k in unsupported_params.keys():\n" " non_default_params.pop(k, None)\n" ), "replacement": ( - " if litellm.drop_params is True or (\n" - " drop_params is not None and drop_params is True\n" - " ):\n" + " if litellm.drop_params is True or " + "(drop_params is not None and drop_params is True):\n" " # egg drop_params visibility patch. Dropping a param changes\n" " # generation behaviour; stock does it with no signal at all.\n" " try:\n" @@ -875,13 +559,13 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " for k in unsupported_params.keys():\n" " non_default_params.pop(k, None)\n" ), - "label": "Patch 8/12 (drop_params visibility)", + "label": "Patch 5/10 (drop_params visibility)", }, - # Patch 9 — _translate_thinking_to_openai: stop synthesizing + # Patch 6 — _translate_thinking_to_openai: stop synthesizing # ``reasoning_effort`` from the caller's ``thinking`` block for non-Claude # models. # - # This is the other half of patch 7, and without it patch 7 is a + # This is the other half of patch 4, and without it patch 4 is a # regression on egg's primary route. On /v1/messages (Claude Code -> # gateway -> litellm -> OpenRouter) the request body carries # ``thinking: {"type": "enabled", "budget_tokens": N}``. @@ -894,16 +578,16 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - # # Until now that param was silently dropped (the model-cost map does not # carry these slugs), which is exactly why these models have been running - # at full reasoning depth. Patch 7 unblocks the param — correct for an + # at full reasoning depth. Patch 4 unblocks the param — correct for an # operator-configured value, wrong for this one, because the measurements # in jwbron/egg#3624 show the bucket is a CAP BELOW the model default: # kimi-k3 means 3130 reasoning tokens with no param vs 340 with # ``reasoning_effort: high``, distributions non-overlapping. Shipping - # patch 7 alone would cut reasoning ~9x on every agent turn with nothing - # logged (patch 8 only fires on drops, and this is no longer dropped) and + # patch 4 alone would cut reasoning ~9x on every agent turn with nothing + # logged (patch 5 only fires on drops, and this is no longer dropped) and # no config file to point at. # - # Gating the synthesis keeps patch 7's actual goal — a configured knob + # Gating the synthesis keeps patch 4's actual goal — a configured knob # reaches the wire — without letting the adapter's bucket become the # effective setting. ``LITELLM_ANTHROPIC_THINKING_TO_REASONING_EFFORT=1`` # restores stock behaviour. The Claude branch above is untouched. @@ -947,7 +631,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " # egg thinking-synthesis patch. Everything above DERIVES an effort\n" " # from the caller's thinking budget, and that bucket is a cap BELOW\n" " # the model default on every model egg routes, so sending it\n" - " # silently shallows reasoning. Off by default; see the patch 9 notes\n" + " # silently shallows reasoning. Off by default; see the patch 6 notes\n" " # in patch_litellm_cache.py. An effort the caller stated outright\n" " # (output_config.effort) is an instruction rather than a\n" " # manufactured ceiling, and is never suppressed.\n" @@ -968,9 +652,9 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - "\n" ' summary = thinking.get("summary") if isinstance(thinking, dict) else None\n' ), - "label": "Patch 9/12 (thinking->reasoning_effort synthesis gate)", + "label": "Patch 6/10 (thinking->reasoning_effort synthesis gate)", }, - # Patch 10 — OpenrouterConfig.transform_request: carry prior-turn assistant + # Patch 7 — OpenrouterConfig.transform_request: carry prior-turn assistant # reasoning back to the provider. # # The Anthropic adapter converts incoming ``thinking`` content blocks into @@ -989,15 +673,23 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - # empty ````, which Poolside's model card warns degrades # follow-up behaviour. # - # DIRECTION: this is request-path (client -> provider). Patches 4, 5a and 5b - # are response-path (provider -> client). Adjacent, not the same thing. + # DIRECTION: this is request-path (client -> provider); the three patches + # the 1.94.0 bump retired (the streaming ``reasoning_content`` thinking + # block and the two first-delta requeues) were response-path + # (provider -> client). Adjacent, not the same thing — which is why + # upstream absorbing those says nothing about this one. # - # NEEDLE ANCHORING: ``_supports_cache_control_in_content`` appears twice in - # 1.86.2 (its own ``def`` and the call in - # ``remove_cache_control_flag_from_messages_and_tools``), and - # ``_move_cache_control_to_content`` likewise. Neither call alone is unique. - # The needle therefore spans the cache_control pair AND the following - # ``extra_body`` pop, a sequence that occurs only in ``transform_request``. + # CARRIED ACROSS THE 1.94.0 BUMP: this patch arrived on the 1.86.2 base + # (#3698) and its needle was re-checked against the 1.94.0 wheel rather + # than assumed to still fit. It matches, once, unmodified. + # + # NEEDLE ANCHORING: ``_supports_cache_control_in_content`` appears THREE + # times in 1.94.0 (its own ``def``, the call in + # ``remove_cache_control_flag_from_messages_and_tools``, and the call in + # ``transform_request``), and ``_move_cache_control_to_content`` twice. + # No single call is a safe anchor on its own. The needle therefore spans + # the cache_control pair AND the following ``extra_body`` pop, a sequence + # that occurs only in ``transform_request``. { "file": F1, "present": "# egg reasoning round-trip patch", @@ -1018,7 +710,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " # template re-renders prior thinking stop seeing empty .\n" " # `model` is passed so the module can decline the routes whose\n" " # upstream re-verifies replayed reasoning (anthropic/*, google/*).\n" - " # See patch 10 notes in patch_litellm_cache.py.\n" + " # See patch 7 notes in patch_litellm_cache.py.\n" " try:\n" " from litellm.llms.openrouter._egg_reasoning_roundtrip import (\n" " map_thinking_blocks_to_reasoning_content as _egg_map_reasoning,\n" @@ -1030,7 +722,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " # so this is unreachable in a built image. Say so once rather\n" " # than reverting to stock in silence — an invisible no-op here\n" " # shows up only as a model quietly reasoning worse, which is\n" - " # the condition patch 8 exists to stop repeating.\n" + " # the condition patch 5 exists to stop repeating.\n" " try:\n" " from litellm._logging import verbose_logger\n" "\n" @@ -1049,36 +741,43 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - "\n" ' extra_body = optional_params.pop("extra_body", {})\n' ), - "label": "Patch 10/12 (assistant reasoning round-trip)", + "label": "Patch 7/10 (assistant reasoning round-trip)", }, - # Patch 11 — ChunkProcessor.calculate_usage: carry the provider-billed cost - # across stream reassembly. + # Patch 8 — ChunkProcessor.calculate_usage: carry the provider's + # ``cost_details`` across stream reassembly. + # + # SCOPE CHANGED at the 1.94.0 bump, and the half that remains is the + # subtler one. 1.86.2 dropped BOTH ``cost`` and ``cost_details`` here: the + # rebuild enumerates token counts and re-constructs ``Usage`` from its own + # ``model_dump()``, so anything the provider attached that litellm did not + # name was gone. Claude Code streams every /v1/messages request, so that + # seam was ~100% of egg's routed traffic — run 6 sampled 1252 + # ``cost_callback`` lines and 1252 carried ``cost: null`` (#3691). # - # OpenRouter reports what it charged on the final streamed usage chunk, and - # stock ``OpenrouterConfig.transform_request`` already asks for it - # (``usage: {"include": true}`` on every request). The number reaches - # litellm intact: ``chunk_parser`` hands the raw block to - # ``ModelResponseStream``, whose ``Usage`` keeps ``cost`` as a declared - # field and ``cost_details`` as a pydantic extra. Then ``calculate_usage`` - # rebuilds a fresh ``Usage`` field-by-field over the counts it enumerates - # and re-constructs it from its own ``model_dump()`` — and the bill is gone. + # 1.94.0 carries ``cost`` natively (upstream 8a49423, a port of + # BerriAI/litellm#16162) and goes further than egg did, feeding the figure + # into litellm's own cost calculator via ``_hidden_params``. Verified + # empirically against the stock 1.94.0 wheel: ``cost`` survives the rebuild, + # ``cost_details`` does not. # - # Claude Code streams every /v1/messages request, so this seam is on ~100% - # of egg's routed traffic: run 6 sampled 1252 cost_callback lines and 1252 - # carried ``cost: null`` (#3691). The non-streaming path was unaffected - # (``original_response`` there holds the raw provider JSON), which is why - # this read as a property of the route rather than as a transport bug. + # ``cost_details`` is where the BYOK bill lives. Under BYOK OpenRouter's + # top-level ``cost`` is a literal 0 because billing routes past them, and + # the real number is ``cost_details.upstream_inference_cost``. Upstream + # records the zero and stops, so without this the BYOK path still reports no + # spend — a null that looks identical to the bug #3691 fixed. Nothing + # upstream carries this field; see the note at the end of + # jwbron/litellm#10 for the shape a BerriAI PR would take. # # Placed AFTER the ``Usage(**model_dump())`` rebuild, not before: the # constructor deletes a ``cost`` attribute it is handed as None, so setting # it first would put the value somewhere the rebuild is entitled to discard. # The companion module (``litellm_core_utils/_egg_stream_cost.py``, in - # NEW_MODULES) never overwrites a value litellm already carried, so a future - # release that fixes this upstream turns the patch into a no-op instead of a - # competing second opinion. + # NEW_MODULES) never overwrites a value litellm already carried, which is + # exactly why the scope narrowing needed no code change: it now no-ops on + # ``cost`` and still supplies ``cost_details``. { "file": F5, - "present": "# egg cost patch. Carry the provider-billed cost", + "present": "# egg cost patch. Carry the provider's cost_details", "needle": ( " # Return a new usage object with the new values\n" "\n" @@ -1091,11 +790,11 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - "\n" " returned_usage = Usage(**returned_usage.model_dump())\n" "\n" - " # egg cost patch. Carry the provider-billed cost across this\n" - " # rebuild — it enumerates token counts only, so `cost` /\n" - " # `cost_details` (what OpenRouter actually charged) are dropped\n" - " # here on every streamed call. See patch 11 notes in\n" - " # patch_litellm_cache.py.\n" + " # egg cost patch. Carry the provider's cost_details across this\n" + " # rebuild. Upstream carries `cost` but not `cost_details`, which\n" + " # is where the BYOK bill lives (`cost` is 0 there). The helper\n" + " # never overwrites what litellm already set, so it no-ops on\n" + " # `cost`. See patch 7 notes in patch_litellm_cache.py.\n" " try:\n" " from litellm.litellm_core_utils._egg_stream_cost import (\n" " carry_upstream_cost as _egg_carry_upstream_cost,\n" @@ -1110,15 +809,20 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " # Warned once per process, and the latch is set only after\n" " # the emit succeeded; verbose_logger is imported here rather\n" " # than read from module scope because this file does not\n" - " # import it.\n" + " # import it. The message names `cost_details`, not `cost`:\n" + " # since 1.94.0 upstream carries `cost` itself, so what a\n" + " # failure here costs is the BYOK figure alone, and sending\n" + " # the operator to a field that reads correctly would be\n" + " # worse than not warning at all.\n" " try:\n" " if not globals().get('_egg_warned_stream_cost'):\n" " from litellm._logging import verbose_logger\n" "\n" " verbose_logger.warning(\n" - " 'egg cost patch: streamed cost preservation is '\n" - " 'inactive (%s: %s); the provider-billed `cost` will '\n" - " 'read null on every streamed call.',\n" + " 'egg cost patch: streamed cost_details preservation '\n" + " 'is inactive (%s: %s); `cost_details` will read null '\n" + " 'on every streamed call, so BYOK routes (where the '\n" + " 'provider-billed `cost` is 0) will report no spend.',\n" " type(_egg_exc).__name__,\n" " _egg_exc,\n" " )\n" @@ -1128,19 +832,27 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - "\n" " return returned_usage\n" ), - "label": "Patch 11/12 (streamed cost preservation)", + "label": "Patch 8/10 (streamed cost_details preservation)", }, - # Patch 12 — _get_model_info_helper: price OpenRouter slugs the bundled map + # Patch 9 — _get_model_info_helper: price OpenRouter slugs the bundled map # has never heard of. # - # Same root cause as patch 7, second symptom. LiteLLM's own + # Same root cause as patch 4, second symptom. LiteLLM's own # ``response_cost`` is computed from ``model_prices_and_context_window.json``, # which does not carry current OpenRouter slugs, so the lookup raises "This # model isn't mapped yet" and egg's ``cost_estimated`` reads null on every - # routed call (#3691). Patch 11 recovers the *billed* figure; this one + # routed call (#3691). Patch 7 recovers the BYOK *billed* figure; this one # restores the independent estimate beside it, which is what remains # readable if a provider ever stops reporting cost. # + # PATCH 9 IS LOAD-BEARING FOR THIS ONE on the streaming path. The hook below + # sits inside ``_get_model_info_helper``, reachable only via + # ``completion_cost()`` — and 1.94.0's ``_propagate_usage_cost_to_hidden_params`` + # makes ``response_cost_calculator`` return before it ever calls that, + # whenever the provider reported any ``cost`` at all including a BYOK zero. + # Without patch 9 this patch applies cleanly, passes every direct test of + # ``_get_model_info_helper``, and is dead code on ~100% of egg's traffic. + # # Placed at the raise site, so it is reached only once every stock lookup # has failed: a slug the bundled map DOES carry keeps the bundled answer, # and the live rate card can add a model but never reprice one. The @@ -1171,7 +883,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " # before giving up — the bundled map lags its slugs by\n" " # construction, so every route egg uses lands here and every\n" " # routed call reports a null cost estimate. Cost fields only;\n" - " # see patch 12 notes in patch_litellm_cache.py.\n" + " # see patch 8 notes in patch_litellm_cache.py.\n" " try:\n" " from litellm.llms.openrouter._egg_capabilities import (\n" " get_model_cost_entry as _egg_openrouter_cost_entry,\n" @@ -1182,7 +894,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " )\n" " except Exception as _egg_exc:\n" " _egg_entry = None\n" - " # Same reasoning as patch 11's handler: swallowed so a\n" + " # Same reasoning as patch 7's handler: swallowed so a\n" " # rate-card lookup can never break a request, warned once\n" " # so its absence is not indistinguishable from a slug the\n" " # roster simply does not carry. verbose_logger is already\n" @@ -1209,7 +921,81 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - 'https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"\n' " )\n" ), - "label": "Patch 12/12 (openrouter live pricing)", + "label": "Patch 9/10 (openrouter live pricing)", + }, + # Patch 10 — _propagate_usage_cost_to_hidden_params: require a POSITIVE, + # finite provider cost before it stands in for the cost calculator. + # + # This one is not a gap in 1.86.2 that survived the bump; it is a + # regression the bump BRINGS IN. The method does not exist in 1.86.2 + # (``grep -rn _propagate_usage_cost_to_hidden_params`` on that wheel returns + # nothing). 1.94.0 calls it on ``complete_streaming_response`` from both + # streaming completion paths (``__next__`` and ``__anext__``), immediately + # before ``logging_obj.success_handler`` sees the copy, and it copies + # ``usage.cost`` into ``_hidden_params["additional_headers"]``. Downstream, + # ``response_cost_calculator`` returns that header verbatim and + # ``completion_cost()`` never runs. + # + # Its guard is ``_usage.cost is not None``, and + # ``get_response_cost_from_hidden_params`` only declines on a literal None — + # ``0.0`` is returned as ``0.0``. Under BYOK OpenRouter's top-level ``cost`` + # IS ``0.0`` (billing routes past them; the real number lives in + # ``cost_details``, which is patch 7's job). Claude Code streams every + # /v1/messages request, so on egg's route the stock behaviour is: calculator + # short-circuits on a zero -> ``response_cost`` is 0.0 -> egg's + # ``cost_estimated`` reads null on EVERY streamed call (it is gated on + # ``_positive``), and patch 8 never fires at all, because its hook sits in + # ``_get_model_info_helper``'s unmapped branch, reachable only through + # ``completion_cost()``. Both of those are things this image exists to + # provide, and both would fail silently: a null cost_estimated is + # indistinguishable from an unpriceable model. + # + # Requiring a positive figure restores ``completion_cost()`` (and with it + # patch 8) for the BYOK case and leaves the real-charge case alone — the + # narrow fix, not a blanket suppression: where a provider reports a genuine + # charge, letting litellm bill from it rather than from a rate card is + # upstream's improvement and worth keeping. The cost of that choice is + # recorded in ``cost_callback._extract_estimated_cost``: on such a route + # ``cost_estimated`` mirrors ``cost`` instead of being an independent read. + # + # Finiteness is checked too, for the reason ``cost_callback._positive`` + # gives: ``+inf`` clears a bare ``> 0``, and this value is written into a + # spend header where it would serialize as the non-standard ``Infinity``. + # ``bool`` is excluded for the same reason it is there — ``isinstance(True, + # int)`` is True, and a True is not a dollar. + # + # The needle spans the assignment and the guard together: the guard line + # alone is what changes, but the assignment above it is what makes the + # match unambiguous, and ``_egg_cost`` has to be computed before it. + { + "file": F6, + "present": "# egg cost-propagation patch", + "needle": ( + ' _usage = getattr(response, "usage", None)\n' + ' if _usage is not None and hasattr(_usage, "cost") ' + "and _usage.cost is not None:\n" + ), + "replacement": ( + ' _usage = getattr(response, "usage", None)\n' + " # egg cost-propagation patch. Upstream's guard is `is not None`,\n" + " # and this value short-circuits litellm's cost calculator. Under\n" + " # BYOK OpenRouter reports `cost: 0.0` (the real figure is in\n" + " # cost_details), so the stock guard hands the calculator a zero,\n" + " # `response_cost` reads 0.0, egg's `cost_estimated` reads null,\n" + " # and the OpenRouter rate-card patch never runs — it lives inside\n" + " # completion_cost(), which this return skips. Require a positive,\n" + " # finite number; a real charge still propagates unchanged. See\n" + " # patch 9 notes in patch_litellm_cache.py.\n" + ' _egg_cost = getattr(_usage, "cost", None) if _usage is not None else None\n' + " _egg_cost_is_billable = (\n" + " isinstance(_egg_cost, (int, float))\n" + " and not isinstance(_egg_cost, bool)\n" + " and _egg_cost > 0\n" + ' and _egg_cost < float("inf")\n' + " )\n" + " if _egg_cost_is_billable:\n" + ), + "label": "Patch 10/10 (billable-cost gate on hidden params)", }, ] diff --git a/config/litellm/stream_cost_preservation.py b/config/litellm/stream_cost_preservation.py index 0d071fa5a..34d3aee63 100644 --- a/config/litellm/stream_cost_preservation.py +++ b/config/litellm/stream_cost_preservation.py @@ -50,7 +50,7 @@ Installed into every litellm tree as ``litellm_core_utils/_egg_stream_cost.py`` by -``config/litellm/patch_litellm_cache.py`` (Patch 11 is the call site). Kept as +``config/litellm/patch_litellm_cache.py`` (Patch 8 is the call site). Kept as a real file rather than a string literal in that script so it stays lintable and unit-testable in the egg repo — which is why it imports no litellm symbols at all and works structurally, off ``getattr``/``dict`` access. diff --git a/docs/development/STRUCTURE.md b/docs/development/STRUCTURE.md index f139ee748..ef632047f 100644 --- a/docs/development/STRUCTURE.md +++ b/docs/development/STRUCTURE.md @@ -548,12 +548,12 @@ config/ ├── litellm/ # egg-litellm image sources │ ├── Dockerfile # Builds egg-litellm: stock LiteLLM + prompt-cache and reasoning-parameter patches │ ├── .ruff.toml # Pins this directory to target-version = py311 — everything here runs on the litellm base image's interpreter, not the repo's 3.14, and `ruff format` under py314 emits PEP 758 syntax the image cannot import -│ ├── patch_litellm_cache.py # Build-time patches: cache_control passthrough on Qwen/DeepSeek routes, live OpenRouter capability + pricing lookup, drop_params visibility, no synthesized reasoning ceiling, prior-turn reasoning round-trip, streamed cost preservation -│ ├── openrouter_capabilities.py # Patches 7 + 12: live GET /api/v1/models lookup — advertised parameters (unioned with LiteLLM's bundled model-cost map) and the published rate card for slugs that map has never heard of, including prompt-length surcharges that fit LiteLLM's 128k/200k/272k rate slots -│ ├── drop_params_visibility.py # Patch 8: warn once per proxy process per (provider, model, param-set) when drop_params discards a parameter -│ ├── anthropic_thinking_policy.py # Patch 9: stop synthesizing a reasoning_effort ceiling from the caller's thinking budget on non-Claude models -│ ├── openrouter_reasoning_roundtrip.py # Patch 10: map prior-turn assistant thinking_blocks onto reasoning_content so OpenRouter models that re-render prior thinking stop seeing empty -│ ├── stream_cost_preservation.py # Patch 11: carry the provider-billed cost / cost_details across LiteLLM's stream reassembly, which enumerates token counts only and drops them +│ ├── patch_litellm_cache.py # Build-time patches: cache_control passthrough on Qwen/DeepSeek routes, live OpenRouter capability + pricing lookup, drop_params visibility, no synthesized reasoning ceiling, prior-turn reasoning round-trip, streamed cost preservation, billable-cost gate on hidden params (patch 10 — inline, no companion module) +│ ├── openrouter_capabilities.py # Patches 4 + 9: live GET /api/v1/models lookup — advertised parameters (unioned with LiteLLM's bundled model-cost map) and the published rate card for slugs that map has never heard of, including prompt-length surcharges that fit LiteLLM's 128k/200k/272k rate slots +│ ├── drop_params_visibility.py # Patch 5: warn once per proxy process per (provider, model, param-set) when drop_params discards a parameter +│ ├── anthropic_thinking_policy.py # Patch 6: stop synthesizing a reasoning_effort ceiling from the caller's thinking budget on non-Claude models +│ ├── openrouter_reasoning_roundtrip.py # Patch 7: map prior-turn assistant thinking_blocks onto reasoning_content so OpenRouter models that re-render prior thinking stop seeing empty +│ ├── stream_cost_preservation.py # Patch 8: carry the provider's cost_details across LiteLLM's stream reassembly (1.94.0 carries `cost` itself, but not this — it is where the BYOK bill lives) │ └── cost_callback.py # LiteLLM custom logger: upstream + estimated cost, per-role attribution (x-egg-* headers), cache hit rate, per-call decoding config -> pod stdout ├── redis/ # egg-redis image sources │ └── Dockerfile # Builds egg-redis: pinned stock Redis, repackaged for the local build/publish supply chain; backs the orchestrator's Redis Streams message store diff --git a/docs/guides/per-agent-models.md b/docs/guides/per-agent-models.md index f70e04757..084810ba2 100644 --- a/docs/guides/per-agent-models.md +++ b/docs/guides/per-agent-models.md @@ -593,8 +593,8 @@ data: > Claude Code prepends (the block's `cch=` hash invalidates the cache key > every turn). egg ships a custom **`egg-litellm`** image > ([`config/litellm/Dockerfile`](../../config/litellm/Dockerfile)) that -> bakes in twelve patches closing those gaps, the reasoning ones below, -> and the cost-visibility ones after that +> bakes in ten patches closing those gaps, the reasoning-parameter ones +> below, and the cost-visibility ones after that > ([`config/litellm/patch_litellm_cache.py`](../../config/litellm/patch_litellm_cache.py)); > the build fails loudly if a LiteLLM bump moves the patched code. Pinning > the OpenRouter provider (`extra_body.provider.order` + `allow_fallbacks: @@ -614,7 +614,7 @@ data: > (issue #3691; before them, 1252 of 1252 sampled calls on run 6 reported > `cost: null`, so egg had no dollar figure at all for its LLM spend). > -> - **Patch 11 — the bill survives streaming.** `cost` is what OpenRouter +> - **Patch 8 — the BYOK bill survives streaming.** `cost` is what OpenRouter > charged, reported on the final streamed usage chunk. No config change is > needed to get it: stock LiteLLM's `OpenrouterConfig.transform_request` > already sets `usage: {"include": true}` on every request, so the number @@ -623,26 +623,37 @@ data: > enumerates and dropped `cost` / `cost_details` at that seam. Claude Code > streams every `/v1/messages` request, so that was ~100% of routed > traffic; the non-streaming path was never affected, which is why it read -> as a property of the route rather than as a transport bug. The patch -> copies the two fields across the rebuild and interprets neither: a `0` -> under BYOK is the literal truth about the OpenRouter bill, with the real -> number beside it under `cost_details.upstream_inference_cost`. -> - **Patch 12 — the estimate has a rate card.** `cost_estimated` is +> as a property of the route rather than as a transport bug. litellm 1.94.0 +> now carries `cost` natively, so the patch is down to `cost_details` — +> which is the half that matters under BYOK, where the top-level `cost` is a +> literal `0` and the real number is `cost_details.upstream_inference_cost`. +> The patch interprets neither field; it only transports. +> - **Patch 9 — the estimate has a rate card.** `cost_estimated` is > LiteLLM's own `response_cost`, computed from its bundled pricing map — > which carries none of the slugs egg routes, exactly as it carries none of -> their `supported_parameters` (patch 7, same root cause). The patch reads +> their `supported_parameters` (patch 4, same root cause). The patch reads > OpenRouter's published rate card off the same `GET /api/v1/models` fetch -> patch 7 already makes, and hands it to the model-info lookup only after +> patch 4 already makes, and hands it to the model-info lookup only after > every bundled lookup has failed — so a mapped slug keeps its bundled > rate, and the live card can add a model but never reprice one. > `LITELLM_OPENROUTER_PRICING=0` turns off just this half. +> - **Patch 10 — the estimate stays independent under BYOK.** From 1.94.0 +> LiteLLM copies a provider-reported `usage.cost` into `_hidden_params` and +> `response_cost_calculator` returns that value *before* it reaches +> `completion_cost()`. Its guard is `is not None`, and under BYOK OpenRouter +> reports `cost: 0.0` — so unpatched, the calculator is handed a zero, +> `cost_estimated` reads null on ~100% of egg's traffic, and patch 9 (which +> lives inside `completion_cost()`) never runs at all. The patch narrows the +> guard to a positive, finite charge. A route that reports a real bill still +> propagates it unchanged, in which case `cost_estimated` mirrors `cost` +> rather than pricing the turn independently. > > `cost_estimated` can still read null for a model that **prices by prompt > length**. OpenRouter publishes a long-context surcharge as > `pricing.overrides` keyed by an arbitrary `min_prompt_tokens`. LiteLLM has > named rate slots at exactly three boundaries — 128000, 200000, 272000 — and > its coverage is uneven *per component*: there is no cache-read slot at -> 128000 and no cache-write slot at 128000 or 272000. Patch 12 translates a +> 128000 and no cache-write slot at 128000 or 272000. Patch 9 translates a > surcharge when every published boundary and every priced component in it has > a slot (about half the tiered models on the current roster, including the > gpt-5.5 and grok-4.x families), and declines the **whole** card otherwise. @@ -661,9 +672,13 @@ data: > by one of those reads low under `cost_estimated` — another reason to compare > the two fields rather than trusting either alone. > -> The two fields are independent measurements of the same turn, so read them -> together: a persistent gap between them is itself a signal (a stale rate -> card, an unexpected provider, or a surcharge tier). Neither is ever +> On a BYOK route — every route egg currently uses — the two fields are +> independent measurements of the same turn, so read them together: a +> persistent gap between them is itself a signal (a stale rate card, an +> unexpected provider, or a surcharge tier). On a route where the provider +> reports a positive charge, patch 10 lets that charge through to LiteLLM's +> calculator and `cost_estimated` mirrors `cost`; agreement there is +> arithmetic, not corroboration. Neither is ever > coerced to `0.0` when unknown — a zero would read in the logs as "this > route is free", the exact inversion of the signal. The session totals sum > only the calls that reported, so compare `cost_known_calls` / @@ -684,7 +699,7 @@ data: > [`k8s/base/litellm-deployment.yaml`](../../k8s/base/litellm-deployment.yaml) > (where they are present but commented out) without rebuilding the image: > -> - **Patch 7 — live capability lookup.** LiteLLM gates reasoning params on +> - **Patch 4 — live capability lookup.** LiteLLM gates reasoning params on > its bundled model-cost map, which does not carry current OpenRouter > slugs, so a `reasoning_effort` set in `litellm_params` was discarded > before the request body was built — no exception, no log line. The patch @@ -698,10 +713,10 @@ data: > phase) tune it. A fetch that fails is cached for the TTL too, so an > offline cluster costs one attempt per hour rather than one per request, > and the first failure is logged at `warning`. Note `_FETCH=0` also -> disables patch 12's pricing lookup — one fetch serves both, so the master +> disables patch 9's pricing lookup — one fetch serves both, so the master > switch governs both; `LITELLM_OPENROUTER_PRICING=0` turns off only the > pricing half. -> - **Patch 9 — no synthesized reasoning ceiling.** On `/v1/messages` (the +> - **Patch 6 — no synthesized reasoning ceiling.** On `/v1/messages` (the > route Claude Code uses) LiteLLM's Anthropic adapter converts each > request's `thinking: {budget_tokens: N}` into a bucketed > `reasoning_effort` for any non-Claude model. That value is not in any @@ -716,7 +731,7 @@ data: > adaptive request that names an effort outright > (`output_config: {effort: ...}`) is an instruction rather than a > manufactured ceiling, and still reaches the provider with this off. -> - **Patch 10 — prior-turn reasoning actually reaches the provider.** The +> - **Patch 7 — prior-turn reasoning actually reaches the provider.** The > same adapter parks each historical assistant turn's thinking on a > `thinking_blocks` field that no OpenRouter request-path code reads, so > every previous turn arrived with its reasoning missing. On a model whose @@ -755,7 +770,7 @@ data: > - **It is read after `drop_params` has acted**, so it reports what the wire > carried, not what your config asked for. A knob you set in > `litellm_params` that does not appear here was discarded or relocated -> into `extra_body` by LiteLLM's parameter mapper. Patch 8 +> into `extra_body` by LiteLLM's parameter mapper. Patch 5 > (`drop_params_visibility.py`) also logs a `litellm.drop_params: dropped > ...` warning once per (provider, model, param-set) combo, for as long as > the proxy's bookkeeping set holds that combo, naming the params and — when diff --git a/k8s/base/litellm-deployment.yaml b/k8s/base/litellm-deployment.yaml index 52facf42c..4b164ff19 100644 --- a/k8s/base/litellm-deployment.yaml +++ b/k8s/base/litellm-deployment.yaml @@ -99,7 +99,7 @@ spec: # are what the patches were measured with. Uncomment to revert an # individual patch at runtime without rebuilding the image. # - # Patch 7 — live OpenRouter capability lookup. ``0`` restores the + # Patch 4 — live OpenRouter capability lookup. ``0`` restores the # bundled model-cost map as the only source of truth, which # silently drops reasoning knobs on any slug the map has not caught # up to. ``_TTL`` seconds between refreshes (default 3600; 0 means @@ -112,16 +112,16 @@ spec: # - name: LITELLM_OPENROUTER_CAPABILITY_TIMEOUT # value: "5" # - # Patch 12 — live OpenRouter pricing, read off the same roster - # fetch as patch 7. ``0`` disables only the pricing half, leaving + # Patch 9 — live OpenRouter pricing, read off the same roster + # fetch as patch 4. ``0`` disables only the pricing half, leaving # the capability lookup running: LiteLLM's bundled map becomes the # sole authority on cost, and since it carries none of the slugs # egg routes, ``cost_estimated`` goes back to null on every call. - # The provider-billed ``cost`` (patch 11) is unaffected by this. + # The provider-billed ``cost`` (patch 8) is unaffected by this. # - name: LITELLM_OPENROUTER_PRICING # value: "0" # - # Patch 9 — thinking -> reasoning_effort synthesis, off by + # Patch 6 — thinking -> reasoning_effort synthesis, off by # default. ``1`` restores stock LiteLLM: the Anthropic adapter # derives a bucketed ``reasoning_effort`` from each request's # thinking budget on /v1/messages. Measured on kimi-k3 that bucket @@ -131,7 +131,7 @@ spec: # - name: LITELLM_ANTHROPIC_THINKING_TO_REASONING_EFFORT # value: "1" # - # Patch 10 — prior-turn assistant reasoning round-trip, on by + # Patch 7 — prior-turn assistant reasoning round-trip, on by # default. ``0`` restores stock LiteLLM: the reasoning of every # historical assistant turn rides to OpenRouter on a # ``thinking_blocks`` field no request-path code reads, so models diff --git a/tests/config/test_patch_litellm_cache.py b/tests/config/test_patch_litellm_cache.py index 24122dead..60042823b 100644 --- a/tests/config/test_patch_litellm_cache.py +++ b/tests/config/test_patch_litellm_cache.py @@ -2,17 +2,30 @@ ``config/litellm/patch_litellm_cache.py`` edits the installed ``litellm`` package in place at image-build time (see PR #3190 / #3199). The real -``litellm==1.86.2`` is not a project dependency — and cannot run on the +``litellm==1.94.0`` is not a project dependency — and cannot run on the repo's Python — so we exercise the patch *mechanics* against fixtures that embed the exact needles the script targets, then assert the resulting source. This is the patch-script regression the CI image build doesn't give us: the build catches version drift via the fail-loud needle check, but -nothing else exercises idempotency, the replacement payload, or the Patch 4 -needle-uniqueness anchor. +nothing else exercises idempotency, the replacement payload, or the +needle-uniqueness anchors on the two patches whose spelling appears twice in +``utils.py``. -The fixtures are derived from ``PATCHES`` itself (the module-level spec +Most fixtures are derived from ``PATCHES`` itself (the module-level spec list), so there is no fixture/needle drift: a needle change automatically flows into the fixture the test patches. + +THE EXCEPTIONS ARE THE HAND-WRITTEN STOCK SNAPSHOTS — ``_STOCK_CAPABILITY_GATE`` +and ``_STOCK_THINKING_TAIL_*`` below. They are transcribed from the pinned +litellm source so a test can assert what a patch leaves ALONE, which a +needle-derived fixture cannot express. Nothing checks them against the real +tree at test time (litellm is not installed here), so they are the one place +in this file that can silently become a snapshot of a version the image no +longer runs — exactly the false-pass shape ``_patch_by_description`` exists to +remove from the lookup key. ``_PINNED_LITELLM_VERSION`` is asserted against the +Dockerfile's ``FROM`` so a bump cannot land without this file being edited; +when you edit it, RE-TRANSCRIBE the snapshots from the new wheel rather than +bumping the constant alone. """ import ast @@ -24,6 +37,12 @@ CONFIG_DIR = Path(__file__).resolve().parent.parent.parent / "config" / "litellm" +# The litellm the hand-written stock snapshots in this file were transcribed +# from. Asserted against the Dockerfile's ``FROM`` below, so a version bump +# cannot land without someone editing this line — and the docstring above says +# what has to happen when they do. +_PINNED_LITELLM_VERSION = "1.94.0" + def _load_patch_module(): """Load ``patch_litellm_cache`` from disk (it isn't an importable package).""" @@ -39,6 +58,24 @@ def _load_patch_module(): plc = _load_patch_module() +def _patch_by_description(description: str) -> dict: + """Find a patch spec by the descriptive tail of its label. + + Deliberately NOT by number. Numbers are positional and get reused: the + 1.94.0 bump (#3697) retired four patches and renumbered the rest, and a + lookup keyed on ``"Patch 4/"`` silently re-bound to a completely different + patch and kept passing — testing nothing, while reading green. The + description is the stable identity, and an exact-count assertion turns a + retired patch into a loud failure instead of a false pass. + """ + matches = [p for p in plc.PATCHES if p["label"].endswith(f"({description})")] + assert len(matches) == 1, ( + f"expected exactly one patch described {description!r}, found " + f"{[p['label'] for p in matches]}" + ) + return matches[0] + + def _build_fixture_root(root: Path) -> None: """Write one fixture file per patched ``litellm`` path, each containing every needle that targets that path concatenated verbatim.""" @@ -51,6 +88,30 @@ def _build_fixture_root(root: Path) -> None: fpath.write_text("# fixture head\n\n" + "\n\n".join(needles) + "\n\n# fixture tail\n") +def test_stock_snapshots_are_pinned_to_the_image_litellm(): + """The hand-written snapshots below are only as good as their version. + + They cannot be checked against the real tree here — litellm is not a + project dependency and does not run on the repo's Python — so the closest + thing to a drift detector is making a version bump impossible to land + without editing this file. At the 1.86.2 -> 1.94.0 bump BOTH snapshots + changed (a reflowed gate, a new ``is_bedrock_arn_model`` arm) while every + test kept passing, because a snapshot of an old version is still a valid + fixture for the patch mechanics — it just stops describing the image.""" + dockerfile = (CONFIG_DIR / "Dockerfile").read_text() + pins = [ + line.split(":v", 1)[1].strip() + for line in dockerfile.splitlines() + if line.startswith("FROM ghcr.io/berriai/litellm:v") + ] + assert pins == [_PINNED_LITELLM_VERSION], ( + f"Dockerfile pins litellm {pins} but the stock snapshots in this file were " + f"transcribed from {_PINNED_LITELLM_VERSION}. Re-transcribe " + "_STOCK_CAPABILITY_GATE and _STOCK_THINKING_TAIL_* from the new wheel, " + "then update _PINNED_LITELLM_VERSION." + ) + + def test_each_needle_occurs_once_in_its_fixture(tmp_path): """Sanity: the needles are distinct, so the fixture contains each exactly once. If this fails the other assertions can't be trusted.""" @@ -213,6 +274,43 @@ def test_installed_module_parse_error_points_at_the_real_line(tmp_path, monkeypa assert "staged module source does not parse" in message +def test_new_modules_are_installed_into_each_root(tmp_path): + """``NEW_MODULES`` drops whole files that have no stock counterpart. + + The capability patch's gate imports + ``litellm.llms.openrouter._egg_capabilities``, so if the module install + silently no-ops the patched gate raises ImportError on every request — + caught by its ``except Exception``, which would put us right back at the + silent-drop behaviour the patch exists to remove.""" + _build_fixture_root(tmp_path) + plc._patch_root(str(tmp_path)) + + for spec in plc.NEW_MODULES: + dest = tmp_path / spec["dest"] + assert dest.is_file(), f"{spec['label']}: not installed" + source = Path(plc._module_source(spec["source"], spec["label"])) + installed = dest.read_text() + assert installed.startswith(plc.EGG_MODULE_HEADER), f"{spec['label']}: no provenance" + assert installed == plc.EGG_MODULE_HEADER + source.read_text(), ( + f"{spec['label']}: content drift" + ) + + +def test_new_module_install_is_idempotent(tmp_path): + _build_fixture_root(tmp_path) + plc._patch_root(str(tmp_path)) + first = {spec["dest"]: (tmp_path / spec["dest"]).read_text() for spec in plc.NEW_MODULES} + plc._patch_root(str(tmp_path)) + for dest, content in first.items(): + assert (tmp_path / dest).read_text() == content + + +def test_missing_staged_module_fails_loud(): + """A missing staged file must abort the build, not skip the install.""" + with pytest.raises(SystemExit): + plc._module_source("definitely-not-a-real-module.py", "test label") + + def test_new_module_refuses_to_clobber_a_foreign_file(tmp_path): """Every other operation in this script is fail-loud on drift; so is this. @@ -262,127 +360,47 @@ def test_new_module_destinations_carry_the_egg_prefix(tmp_path): assert "must be prefixed" in str(excinfo.value) -def test_patch4_needle_anchors_on_content_block_function(tmp_path): - """Regression for the Patch 4 needle-uniqueness fix (#3199 review). - - The bare ``thinking_blocks`` elif appears in two sibling functions. The - Patch 4 needle anchors on the preceding text-block elif - (``choice.delta.content is not None ...``), which is unique to - ``_translate_streaming_openai_chunk_to_anthropic_content_block``. A - fixture containing a *second* (sibling-function-style) bare - ``thinking_blocks`` elif — preceded by a ``tool_calls`` block, not the - text elif — must be left untouched.""" - # Matched on "Patch 4/" rather than the full label so adding a patch (and - # renumbering the denominators) does not silently turn this into a - # StopIteration instead of a real assertion. - patch4 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 4/")) - - # Sibling function: bare thinking_blocks elif preceded by a tool_calls - # branch (mirrors _translate_streaming_openai_chunk_to_anthropic). It must - # NOT match the Patch 4 needle. - sibling = ( - "SENTINEL_SIBLING_BEGIN\n" - " if choice.delta.tool_calls is not None:\n" - " partial_json = ''\n" - " elif isinstance(choice, StreamingChoices) and hasattr(\n" - ' choice.delta, "thinking_blocks"\n' - " ):\n" - " pass\n" - "SENTINEL_SIBLING_END\n" - ) - fixture = sibling + "\n" + patch4["needle"] - - f2 = tmp_path / patch4["file"] - f2.parent.mkdir(parents=True, exist_ok=True) - f2.write_text(fixture) - - plc._apply( - str(f2), - present=patch4["present"], - needle=patch4["needle"], - replacement=patch4["replacement"], - label=patch4["label"], - ) - result = f2.read_text() - - # The intended branch was rewritten exactly once. - assert result.count(patch4["present"]) == 1 - # The sibling block is byte-for-byte unchanged. - start = result.index("SENTINEL_SIBLING_BEGIN") - end = result.index("SENTINEL_SIBLING_END") + len("SENTINEL_SIBLING_END\n") - assert result[start:end] == sibling - - -def test_new_modules_are_installed_into_each_root(tmp_path): - """``NEW_MODULES`` drops whole files that have no stock counterpart. - - Patch 7's gate imports ``litellm.llms.openrouter._egg_capabilities``, so if - the module install silently no-ops the patched gate raises ImportError on - every request — caught by its ``except Exception``, which would put us right - back at the silent-drop behaviour the patch exists to remove.""" - _build_fixture_root(tmp_path) - plc._patch_root(str(tmp_path)) - - for spec in plc.NEW_MODULES: - dest = tmp_path / spec["dest"] - assert dest.is_file(), f"{spec['label']}: not installed" - source = Path(plc._module_source(spec["source"], spec["label"])) - installed = dest.read_text() - assert installed.startswith(plc.EGG_MODULE_HEADER), f"{spec['label']}: no provenance" - assert installed == plc.EGG_MODULE_HEADER + source.read_text(), ( - f"{spec['label']}: content drift" - ) - - -def test_new_module_install_is_idempotent(tmp_path): - _build_fixture_root(tmp_path) - plc._patch_root(str(tmp_path)) - first = {spec["dest"]: (tmp_path / spec["dest"]).read_text() for spec in plc.NEW_MODULES} - plc._patch_root(str(tmp_path)) - for dest, content in first.items(): - assert (tmp_path / dest).read_text() == content - - -def test_missing_staged_module_fails_loud(): - """A missing staged file must abort the build, not skip the install.""" - with pytest.raises(SystemExit): - plc._module_source("definitely-not-a-real-module.py", "test label") +# The stock ``get_supported_openai_params`` gate, transcribed verbatim from +# litellm 1.94.0 (``llms/openrouter/chat/transformation.py``), picking up where +# the capability patch's needle ends. A PINNED SNAPSHOT — see the module +# docstring: 1.86.2 spelled this across three lines and 1.94.0 reflowed it, and +# nothing here would have noticed. Re-transcribe on every bump. +_STOCK_CAPABILITY_GATE = ( + " if litellm.supports_reasoning(" + 'model=model, custom_llm_provider="openrouter") or litellm.supports_reasoning(\n' + " model=model\n" + " ):\n" + ' supported_params.append("reasoning_effort")\n' + ' supported_params.append("thinking")\n' + " except Exception:\n" + " pass\n" + " return list(dict.fromkeys(supported_params))\n" +) -def test_patch7_gate_is_additive_not_substitutive(tmp_path): - """Patch 7 must UNION the live answer with the stock model-map answer. +def test_capability_gate_is_additive_not_substitutive(tmp_path): + """The capability patch must UNION live data with the stock model map. OpenRouter's ``supported_parameters`` under-reports ``reasoning_effort`` (deepseek-r1 advertises only ``reasoning``), so letting live data win outright would drop a knob the map correctly allows — trading one silent drop for another. The stock ``supports_reasoning`` branch must therefore survive the patch.""" - patch7 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 7/")) - - # The stock gate, verbatim from 1.86.2, following the needle. Applying the - # patch to this and asserting on the RESULT tests the invariant the - # docstring claims; substring checks against the replacement string alone - # would pass on a patch that deleted the branch entirely. - stock_gate = ( - " if litellm.supports_reasoning(\n" - ' model=model, custom_llm_provider="openrouter"\n' - " ) or litellm.supports_reasoning(model=model):\n" - ' supported_params.append("reasoning_effort")\n' - ' supported_params.append("thinking")\n' - " except Exception:\n" - " pass\n" - " return list(dict.fromkeys(supported_params))\n" - ) - target = tmp_path / patch7["file"] + capability_patch = _patch_by_description("openrouter live capabilities") + + # Applying the patch to the stock snapshot and asserting on the RESULT + # tests the invariant the docstring claims; substring checks against the + # replacement string alone would pass on a patch that deleted the branch. + target = tmp_path / capability_patch["file"] target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(patch7["needle"] + stock_gate) + target.write_text(capability_patch["needle"] + _STOCK_CAPABILITY_GATE) plc._apply( str(target), - present=patch7["present"], - needle=patch7["needle"], - replacement=patch7["replacement"], - label=patch7["label"], + present=capability_patch["present"], + needle=capability_patch["needle"], + replacement=capability_patch["replacement"], + label=capability_patch["label"], ) result = target.read_text() @@ -391,11 +409,15 @@ def test_patch7_gate_is_additive_not_substitutive(tmp_path): assert '"reasoning_effort" in _advertised' in result # ...and the stock model-map branch is still there, untouched, downstream # of it: the live answer can ADD a knob, never withhold one. - assert stock_gate in result, "patch 7 must not replace the stock model-map branch" + assert _STOCK_CAPABILITY_GATE in result, ( + "the capability patch must not replace the stock model-map branch" + ) # The inserted block precedes it and does not return out of the function. - inserted = result[: result.index(stock_gate)] + inserted = result[: result.index(_STOCK_CAPABILITY_GATE)] assert "supported_params.append" in inserted - assert "\n return" not in inserted, "patch 7 must not short-circuit the stock branch" + assert "\n return" not in inserted, ( + "the capability patch must not short-circuit the stock branch" + ) # Failures in the lookup must never propagate into a request. assert "except Exception:" in inserted # Only reasoning_effort is admitted. OpenRouter's `reasoning` field is a @@ -403,19 +425,22 @@ def test_patch7_gate_is_additive_not_substitutive(tmp_path): assert '"thinking"' not in inserted -# The tail of ``_translate_thinking_to_openai`` as it stands in 1.86.2, -# verbatim, from the Claude branch through the assignments. Patch 9's needle is -# a slice of this; keeping the surrounding lines lets the test assert what the -# gate sits between, which is the whole invariant. +# The tail of ``_translate_thinking_to_openai`` as it stands in 1.94.0, +# verbatim, from the Claude branch through the assignments. The synthesis +# gate's needle is a slice of this; keeping the surrounding lines lets the test +# assert what the gate sits between, which is the whole invariant. A PINNED +# SNAPSHOT — see the module docstring. Both halves moved at the 1.86.2 -> 1.94.0 +# bump (the Claude branch gained a ``is_bedrock_arn_model`` arm and the +# derivation call collapsed onto one line), which is exactly the drift nothing +# in this file can detect for you. Re-transcribe on every bump. _STOCK_THINKING_TAIL_HEAD = ( ' model = new_kwargs.get("model", "")\n' - " if self.is_anthropic_claude_model(model):\n" + " if self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model):\n" ' new_kwargs["thinking"] = thinking # type: ignore\n' " return\n" "\n" - " reasoning_effort = self.translate_anthropic_thinking_to_reasoning_effort(\n" - " cast(Dict[str, Any], thinking)\n" - " )\n" + " reasoning_effort = self.translate_anthropic_thinking_to_reasoning_effort(" + "cast(Dict[str, Any], thinking))\n" " if not reasoning_effort:\n" " return\n" "\n" @@ -443,39 +468,39 @@ def test_patch7_gate_is_additive_not_substitutive(tmp_path): ) -def test_patch9_gates_synthesis_without_touching_the_claude_branch(tmp_path): - """Patch 9 must stop the adapter manufacturing a ``reasoning_effort``. +def test_thinking_gate_leaves_the_claude_branch_alone(tmp_path): + """The synthesis gate must stop the adapter manufacturing an effort. On ``/v1/messages`` litellm derives ``reasoning_effort`` from the caller's ``thinking`` budget for every non-Claude model. That derived value is a cap BELOW the model default (#3624: kimi-k3 means 3130 reasoning tokens with no - param vs 340 with ``high``), so Patch 7 alone would silently shallow every - agent turn. The Claude branch, which forwards ``thinking`` unchanged, must - be unaffected — and so must the assignments, which the gate returns before - rather than rewriting.""" - patch9 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 9/")) + param vs 340 with ``high``), so the capability patch alone would silently + shallow every agent turn. The Claude branch, which forwards ``thinking`` + unchanged, must be unaffected — and so must the assignments, which the gate + returns before rather than rewriting.""" + gate_patch = _patch_by_description("thinking->reasoning_effort synthesis gate") - target = tmp_path / patch9["file"] + target = tmp_path / gate_patch["file"] target.parent.mkdir(parents=True, exist_ok=True) target.write_text( - _STOCK_THINKING_TAIL_HEAD + patch9["needle"] + _STOCK_THINKING_TAIL_FOOT, + _STOCK_THINKING_TAIL_HEAD + gate_patch["needle"] + _STOCK_THINKING_TAIL_FOOT, ) plc._apply( str(target), - present=patch9["present"], - needle=patch9["needle"], - replacement=patch9["replacement"], - label=patch9["label"], + present=gate_patch["present"], + needle=gate_patch["needle"], + replacement=gate_patch["replacement"], + label=gate_patch["label"], ) result = target.read_text() - assert _STOCK_THINKING_TAIL_HEAD in result, "patch 9 must leave the Claude path alone" - assert _STOCK_THINKING_TAIL_FOOT in result, "patch 9 must not rewrite the assignments" + assert _STOCK_THINKING_TAIL_HEAD in result, "the gate must leave the Claude path alone" + assert _STOCK_THINKING_TAIL_FOOT in result, "the gate must not rewrite the assignments" # The gate sits after the derivation and returns (rather than falling # through) when synthesis is off, so nothing is assigned. - gate = result[result.index(patch9["present"]) : result.index(_STOCK_THINKING_TAIL_FOOT)] + gate = result[result.index(gate_patch["present"]) : result.index(_STOCK_THINKING_TAIL_FOOT)] assert "_egg_anthropic_thinking_policy" in gate assert "if not _egg_synthesize:\n return\n" in gate # A missing policy module must fall back to the policy's OWN default (off), @@ -483,7 +508,7 @@ def test_patch9_gates_synthesis_without_touching_the_claude_branch(tmp_path): assert "_egg_synthesize = False" in gate -def test_patch9_does_not_suppress_an_explicitly_requested_effort(tmp_path): +def test_thinking_gate_does_not_suppress_an_explicit_effort(tmp_path): """The gate's scope is the *derived* bucket, not the whole function. Stock reaches the assignment two ways: from ``budget_tokens`` (a ceiling @@ -491,42 +516,42 @@ def test_patch9_does_not_suppress_an_explicitly_requested_effort(tmp_path): which is the caller saying outright what they want. Suppressing the second would be discarding an instruction, not declining to invent one — a different change from the one the patch documents.""" - patch9 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 9/")) + gate_patch = _patch_by_description("thinking->reasoning_effort synthesis gate") - target = tmp_path / patch9["file"] + target = tmp_path / gate_patch["file"] target.parent.mkdir(parents=True, exist_ok=True) target.write_text( - _STOCK_THINKING_TAIL_HEAD + patch9["needle"] + _STOCK_THINKING_TAIL_FOOT, + _STOCK_THINKING_TAIL_HEAD + gate_patch["needle"] + _STOCK_THINKING_TAIL_FOOT, ) plc._apply( str(target), - present=patch9["present"], - needle=patch9["needle"], - replacement=patch9["replacement"], - label=patch9["label"], + present=gate_patch["present"], + needle=gate_patch["needle"], + replacement=gate_patch["replacement"], + label=gate_patch["label"], ) result = target.read_text() # The adaptive override still runs, and it is what exempts the request. override = ' reasoning_effort = output_config["effort"]\n' assert override in result, "the adaptive override must survive the patch" - assert result.index(override) < result.index(patch9["present"]), ( + assert result.index(override) < result.index(gate_patch["present"]), ( "the gate must sit after the override, not before it" ) assert f"{override} _egg_effort_is_explicit = True\n" in result assert "if not _egg_effort_is_explicit:\n" in result -def test_patch8_needle_disambiguates_the_two_drop_sites(tmp_path): - """litellm 1.86.2 has TWO ``drop_params`` branches in utils.py. +def test_drop_params_needle_disambiguates_the_two_drop_sites(tmp_path): + """litellm 1.94.0 has TWO ``drop_params`` branches in utils.py. They share the identical ``if litellm.drop_params is True or (...)`` condition; only what follows differs (a bare ``pass`` in the embeddings path, the pop loop in ``get_optional_params``). Matching on the shared condition would patch whichever came first — the same needle-uniqueness - trap as Patch 4. A fixture carrying the sibling ``pass`` form first must be - left untouched.""" - patch8 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 8/")) + trap as the capability gate. A fixture carrying the sibling ``pass`` form + first must be left untouched.""" + drop_patch = _patch_by_description("drop_params visibility") sibling = ( "SENTINEL_PASS_SITE_BEGIN\n" @@ -536,37 +561,82 @@ def test_patch8_needle_disambiguates_the_two_drop_sites(tmp_path): " pass\n" "SENTINEL_PASS_SITE_END\n" ) - fixture = sibling + "\n" + patch8["needle"] + fixture = sibling + "\n" + drop_patch["needle"] - target = tmp_path / patch8["file"] + target = tmp_path / drop_patch["file"] target.parent.mkdir(parents=True, exist_ok=True) target.write_text(fixture) plc._apply( str(target), - present=patch8["present"], - needle=patch8["needle"], - replacement=patch8["replacement"], - label=patch8["label"], + present=drop_patch["present"], + needle=drop_patch["needle"], + replacement=drop_patch["replacement"], + label=drop_patch["label"], ) result = target.read_text() - assert result.count(patch8["present"]) == 1 + assert result.count(drop_patch["present"]) == 1 start = result.index("SENTINEL_PASS_SITE_BEGIN") end = result.index("SENTINEL_PASS_SITE_END") + len("SENTINEL_PASS_SITE_END\n") - assert result[start:end] == sibling, "patch 8 rewrote the embeddings-path drop site" + assert result[start:end] == sibling, ( + "the drop_params patch rewrote the embeddings-path drop site" + ) + + +def test_cost_details_carry_runs_after_the_rebuild_not_before(tmp_path): + """The carry must land on the far side of ``Usage(**model_dump())``. + That constructor drops the fields it is not declared to carry, so a carry + inserted *before* the rebuild would be writing into an object the rebuild is + entitled to discard — the patch would apply cleanly, the build would pass, + and ``cost_details`` would still read null on every streamed call. On a BYOK + route that is the whole bill: the provider-billed figure lives in + ``cost_details.upstream_inference_cost`` and top-level ``cost`` is 0.""" + carry_patch = _patch_by_description("streamed cost_details preservation") + + target = tmp_path / carry_patch["file"] + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(carry_patch["needle"]) + plc._apply( + str(target), + present=carry_patch["present"], + needle=carry_patch["needle"], + replacement=carry_patch["replacement"], + label=carry_patch["label"], + ) + result = target.read_text() -def test_patch10_needle_anchors_on_transform_request(tmp_path): - """Patch 10 must land in ``transform_request`` and nowhere else. + rebuild = " returned_usage = Usage(**returned_usage.model_dump())\n" + assert rebuild in result, "the stock rebuild must survive the patch" + assert result.index(rebuild) < result.index("_egg_carry_upstream_cost"), ( + "the carry must run after the rebuild, not before it" + ) + assert result.index("_egg_carry_upstream_cost") < result.index(" return returned_usage") + # An import failure must never propagate: this is on the response path. + assert "except Exception as _egg_exc:" in result + # And it must not be silent either — a swallowed import here looks exactly + # like "the provider reported no cost", the symptom the patch removes. + # verbose_logger is imported inside the handler because + # streaming_chunk_builder_utils.py, unlike utils.py, does not carry it at + # module scope; the latch is set only once the emit succeeded. + assert "from litellm._logging import verbose_logger" in result + assert result.index("verbose_logger.warning(") < result.index( + "globals()['_egg_warned_stream_cost'] = True" + ), "the latch must be set after the emit, not before it" - Both cache_control helpers are *named twice* in 1.86.2: each has its own - ``def``, and ``remove_cache_control_flag_from_messages_and_tools`` calls - ``_supports_cache_control_in_content`` too. A needle built from either call - alone could retarget after an upstream reorder — the same trap Patches 4 - and 8 document. The needle therefore runs through the ``extra_body`` pop, - which happens only here.""" - patch10 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 10/")) + +def test_roundtrip_needle_anchors_on_transform_request(tmp_path): + """The round-trip patch must land in ``transform_request`` and nowhere else. + + ``_supports_cache_control_in_content`` is named THREE times in 1.94.0 (its + own ``def``, the call in ``remove_cache_control_flag_from_messages_and_tools`` + and the call in ``transform_request``), and ``_move_cache_control_to_content`` + twice. A needle built from either call alone could retarget after an upstream + reorder — the same trap the drop_params and live-pricing needles document. + The needle therefore runs through the ``extra_body`` pop, which happens only + here.""" + roundtrip_patch = _patch_by_description("assistant reasoning round-trip") # A sibling that mentions the same helper, ahead of the real site. sibling = ( @@ -576,41 +646,41 @@ def test_patch10_needle_anchors_on_transform_request(tmp_path): " if self._supports_cache_control_in_content(model):\n" " return messages, tools\n" ) - target = tmp_path / patch10["file"] + target = tmp_path / roundtrip_patch["file"] target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(sibling + "\n" + patch10["needle"]) + target.write_text(sibling + "\n" + roundtrip_patch["needle"]) plc._apply( str(target), - present=patch10["present"], - needle=patch10["needle"], - replacement=patch10["replacement"], - label=patch10["label"], + present=roundtrip_patch["present"], + needle=roundtrip_patch["needle"], + replacement=roundtrip_patch["replacement"], + label=roundtrip_patch["label"], ) result = target.read_text() assert sibling in result, "the sibling call site must be left alone" - inserted_at = result.index(patch10["present"]) - assert inserted_at > result.index(sibling), "patch 10 must land after the sibling" + inserted_at = result.index(roundtrip_patch["present"]) + assert inserted_at > result.index(sibling), "the round-trip patch must land after the sibling" -def test_patch10_preserves_the_cache_control_step(tmp_path): - """Patch 10 shares its needle with Patch 1's file and sits directly on top - of the cache_control rewrite. Asserting on the RESULT (rather than on the - replacement literal) is what catches a patch that dropped that step while - inserting its own.""" - patch10 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 10/")) +def test_roundtrip_preserves_the_cache_control_step(tmp_path): + """The round-trip patch shares its needle's file with Patch 1 and sits + directly on top of the cache_control rewrite. Asserting on the RESULT + (rather than on the replacement literal) is what catches a patch that + dropped that step while inserting its own.""" + roundtrip_patch = _patch_by_description("assistant reasoning round-trip") - target = tmp_path / patch10["file"] + target = tmp_path / roundtrip_patch["file"] target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(patch10["needle"]) + target.write_text(roundtrip_patch["needle"]) plc._apply( str(target), - present=patch10["present"], - needle=patch10["needle"], - replacement=patch10["replacement"], - label=patch10["label"], + present=roundtrip_patch["present"], + needle=roundtrip_patch["needle"], + replacement=roundtrip_patch["replacement"], + label=roundtrip_patch["label"], ) result = target.read_text() @@ -625,26 +695,26 @@ def test_patch10_preserves_the_cache_control_step(tmp_path): assert "_egg_reasoning_roundtrip" in result -def test_patch10_passes_the_model_and_reports_an_unavailable_module(tmp_path): +def test_roundtrip_passes_the_model_and_reports_an_unavailable_module(tmp_path): """Two properties of the injected call site. The module declines routes whose upstream re-verifies replayed reasoning (anthropic/*, google/*), which it can only do if the call site hands it ``model``. And an import failure means a broken image, not a bad request: reverting to stock in silence there is invisible until a model quietly - reasons worse — the condition Patch 8 exists to stop repeating.""" - patch10 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 10/")) + reasons worse — the condition Patch 5 exists to stop repeating.""" + roundtrip_patch = _patch_by_description("assistant reasoning round-trip") - target = tmp_path / patch10["file"] + target = tmp_path / roundtrip_patch["file"] target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(patch10["needle"]) + target.write_text(roundtrip_patch["needle"]) plc._apply( str(target), - present=patch10["present"], - needle=patch10["needle"], - replacement=patch10["replacement"], - label=patch10["label"], + present=roundtrip_patch["present"], + needle=roundtrip_patch["needle"], + replacement=roundtrip_patch["replacement"], + label=roundtrip_patch["label"], ) result = target.read_text() @@ -653,77 +723,125 @@ def test_patch10_passes_the_model_and_reports_an_unavailable_module(tmp_path): assert "_egg_reasoning_roundtrip_warned" in result, "warned once, not once per request" -def test_patch10_maps_onto_the_request_not_the_response(tmp_path): - """Guard against the adjacent-patch confusion the docstring calls out: - patches 4/5a/5b are provider -> client, this one is client -> provider. - A patch that touched ``transform_response`` would be a different bug.""" - patch10 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 10/")) +def test_roundtrip_maps_onto_the_request_not_the_response(tmp_path): + """Guard against the adjacent-patch confusion the docstring calls out: the + three patches the 1.94.0 bump retired were provider -> client, this one is + client -> provider. A patch that touched ``transform_response`` would be a + different bug.""" + roundtrip_patch = _patch_by_description("assistant reasoning round-trip") + + assert "transform_response" not in roundtrip_patch["needle"] + assert "transform_response" not in roundtrip_patch["replacement"] + assert roundtrip_patch["file"] == plc.F1 + + +# The body ``_propagate_usage_cost_to_hidden_params`` runs once its guard is +# true, transcribed from 1.94.0 and picking up where the billable-cost patch's +# needle ends. A PINNED SNAPSHOT — see the module docstring. It is here rather +# than beside the other two because this test EXECUTES the patched result: an +# assertion that the gate rejects a BYOK zero is only meaningful if the thing +# behind the gate is the thing that actually sets the header. +_STOCK_HIDDEN_PARAMS_BODY = ( + ' if "additional_headers" not in response._hidden_params:\n' + ' response._hidden_params["additional_headers"] = {}\n' + ' response._hidden_params["additional_headers"]' + '["llm_provider-x-litellm-response-cost"] = float(_usage.cost)\n' +) - assert "transform_response" not in patch10["needle"] - assert "transform_response" not in patch10["replacement"] - assert patch10["file"] == plc.F1 +_HIDDEN_PARAMS_SCAFFOLD = "class _Wrapper:\n @staticmethod\n def propagate(response):\n" -def test_patch11_sets_cost_after_the_rebuild_not_before(tmp_path): - """Patch 11 must land on the far side of ``Usage(**model_dump())``. +def _run_billable_cost_gate(tmp_path, cost): + """Apply the billable-cost patch, then run the result against a fake usage. - That constructor deletes a ``cost`` attribute it is handed as None, so a - carry inserted *before* the rebuild would be writing into an object the - rebuild is entitled to discard — the patch would apply cleanly, the build - would pass, and ``cost`` would still read null on every streamed call, - which is the bug it exists to fix.""" - patch11 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 11/")) + Returns the ``additional_headers`` dict the patched body produced, so the + caller can assert on behaviour rather than on substrings.""" + gate_patch = _patch_by_description("billable-cost gate on hidden params") - target = tmp_path / patch11["file"] + target = tmp_path / gate_patch["file"] target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(patch11["needle"]) + target.write_text( + _HIDDEN_PARAMS_SCAFFOLD + gate_patch["needle"] + _STOCK_HIDDEN_PARAMS_BODY, + ) plc._apply( str(target), - present=patch11["present"], - needle=patch11["needle"], - replacement=patch11["replacement"], - label=patch11["label"], + present=gate_patch["present"], + needle=gate_patch["needle"], + replacement=gate_patch["replacement"], + label=gate_patch["label"], ) - result = target.read_text() - rebuild = " returned_usage = Usage(**returned_usage.model_dump())\n" - assert rebuild in result, "the stock rebuild must survive the patch" - assert result.index(rebuild) < result.index("_egg_carry_upstream_cost"), ( - "the carry must run after the rebuild, not before it" - ) - assert result.index("_egg_carry_upstream_cost") < result.index(" return returned_usage") - # An import failure must never propagate: this is on the response path. - assert "except Exception as _egg_exc:" in result - # And it must not be silent either — a swallowed import here looks exactly - # like "the provider reported no cost", the symptom the patch removes. - # verbose_logger is imported inside the handler because - # streaming_chunk_builder_utils.py, unlike utils.py, does not carry it at - # module scope; the latch is set only once the emit succeeded. - assert "from litellm._logging import verbose_logger" in result - assert result.index("verbose_logger.warning(") < result.index( - "globals()['_egg_warned_stream_cost'] = True" - ), "the latch must be set after the emit, not before it" + namespace: dict = {} + exec(compile(target.read_text(), str(target), "exec"), namespace) # noqa: S102 + + class _Usage: + pass + + class _Response: + pass + + usage = _Usage() + if cost is not _MISSING: + usage.cost = cost + response = _Response() + response.usage = usage + response._hidden_params = {} + + namespace["_Wrapper"].propagate(response) + return response._hidden_params.get("additional_headers", {}) + +_MISSING = object() +_COST_HEADER = "llm_provider-x-litellm-response-cost" -def test_patch12_hooks_the_unmapped_branch_and_leaves_the_stock_raise(tmp_path): - """Patch 12 must be a fallback, not a replacement. + +@pytest.mark.parametrize( + "cost", + [0.0, 0, -1.0, None, float("inf"), float("nan"), True, _MISSING], +) +def test_billable_cost_gate_declines_a_non_charge(tmp_path, cost): + """The gate exists because 1.94.0 short-circuits its own cost calculator. + + ``response_cost_calculator`` returns this header's value before it ever + reaches ``completion_cost()``, and stock propagates on ``is not None``. + Under BYOK OpenRouter reports ``cost: 0.0`` — the real figure lives in + ``cost_details`` — so stock hands the calculator a zero, ``response_cost`` + reads 0.0, egg's ``cost_estimated`` reads null, and the live rate-card patch + (which lives inside ``completion_cost()``) becomes dead code on ~100% of + egg's traffic. ``True`` is in the list because it is an ``int`` that is + ``> 0``: a bool reaching a currency field is a bug upstream, not a bill.""" + assert _COST_HEADER not in _run_billable_cost_gate(tmp_path, cost) + + +def test_billable_cost_gate_propagates_a_real_charge(tmp_path): + """The narrow fix, not blanket suppression. + + Where a provider does report a positive charge, upstream's propagation is + an improvement over a token-based estimate and must survive untouched — + ``cost_estimated`` then mirrors ``cost`` by design.""" + headers = _run_billable_cost_gate(tmp_path, 0.0031) + assert headers[_COST_HEADER] == 0.0031 + + +def test_pricing_hooks_the_unmapped_branch_and_leaves_the_stock_raise(tmp_path): + """The live rate card must be a fallback, not a replacement. It sits at the "isn't mapped yet" raise, so it runs only once every stock lookup has already failed: a slug the bundled map DOES carry keeps the bundled rate, and the live card can add a model but never reprice one. The stock ValueError must still be reachable, for the slug OpenRouter has not heard of either.""" - patch12 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 12/")) + pricing_patch = _patch_by_description("openrouter live pricing") - target = tmp_path / patch12["file"] + target = tmp_path / pricing_patch["file"] target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(patch12["needle"]) + target.write_text(pricing_patch["needle"]) plc._apply( str(target), - present=patch12["present"], - needle=patch12["needle"], - replacement=patch12["replacement"], - label=patch12["label"], + present=pricing_patch["present"], + needle=pricing_patch["needle"], + replacement=pricing_patch["replacement"], + label=pricing_patch["label"], ) result = target.read_text() @@ -738,7 +856,7 @@ def test_patch12_hooks_the_unmapped_branch_and_leaves_the_stock_raise(tmp_path): assert "except Exception as _egg_exc:\n _egg_entry = None\n" in result # ...but not silently. A failed import is otherwise indistinguishable from # "OpenRouter has no rate for this slug", which is the null-cost_estimated - # symptom patch 12 exists to remove. Warned once, and the latch is set only + # symptom the pricing patch exists to remove. Warned once, and the latch is set only # after the emit — inside its own try, because raising from an except block # would propagate into a live request. assert "_egg_warned_pricing" in result @@ -754,14 +872,14 @@ def test_patch12_hooks_the_unmapped_branch_and_leaves_the_stock_raise(tmp_path): # build's own ``_check_parses`` would only catch it after a full image build. _BODY_CONTEXTS = ( ( - "Patch 11/", + "streamed cost_details preservation", "class ChunkProcessor:\n" " def calculate_usage(self, chunks):\n" " Usage = dict\n" " returned_usage = Usage()\n", ), ( - "Patch 12/", + "openrouter live pricing", "def _get_model_info_helper(model, custom_llm_provider):\n" " _model_info = None\n" " key = None\n" @@ -771,8 +889,12 @@ def test_patch12_hooks_the_unmapped_branch_and_leaves_the_stock_raise(tmp_path): ) -@pytest.mark.parametrize(("prefix", "preamble"), _BODY_CONTEXTS, ids=["patch11", "patch12"]) -def test_patch_bodies_parse_at_their_insertion_indentation(prefix, preamble): +@pytest.mark.parametrize( + ("description", "preamble"), + _BODY_CONTEXTS, + ids=["stream-cost", "openrouter-pricing"], +) +def test_patch_bodies_parse_at_their_insertion_indentation(description, preamble): """The spliced payload must be valid Python 3.11 — the image's interpreter. These replacements are string literals assembled line by line, so a wrong @@ -783,18 +905,18 @@ def test_patch_bodies_parse_at_their_insertion_indentation(prefix, preamble): ``config/litellm/.ruff.toml`` does — this code runs on the litellm base image, not on the repo's interpreter. """ - patch = next(p for p in plc.PATCHES if p["label"].startswith(prefix)) + patch = _patch_by_description(description) ast.parse(preamble + patch["replacement"], feature_version=(3, 11)) -def test_patch12_needle_disambiguates_the_two_unmapped_messages(tmp_path): +def test_pricing_needle_disambiguates_the_two_unmapped_messages(tmp_path): """utils.py carries the "isn't mapped yet" string twice. The other one is the outer handler's re-raise, with a different message body and indentation. Matching loosely would insert a pricing fallback into an exception handler, where ``_model_info`` and ``key`` are not even in scope — same needle-uniqueness trap as Patches 4 and 8.""" - patch12 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 12/")) + pricing_patch = _patch_by_description("openrouter live pricing") sibling = ( "SENTINEL_OUTER_HANDLER_BEGIN\n" @@ -809,20 +931,20 @@ def test_patch12_needle_disambiguates_the_two_unmapped_messages(tmp_path): " )\n" "SENTINEL_OUTER_HANDLER_END\n" ) - target = tmp_path / patch12["file"] + target = tmp_path / pricing_patch["file"] target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(patch12["needle"] + "\n" + sibling) + target.write_text(pricing_patch["needle"] + "\n" + sibling) plc._apply( str(target), - present=patch12["present"], - needle=patch12["needle"], - replacement=patch12["replacement"], - label=patch12["label"], + present=pricing_patch["present"], + needle=pricing_patch["needle"], + replacement=pricing_patch["replacement"], + label=pricing_patch["label"], ) result = target.read_text() - assert result.count(patch12["present"]) == 1 + assert result.count(pricing_patch["present"]) == 1 start = result.index("SENTINEL_OUTER_HANDLER_BEGIN") end = result.index("SENTINEL_OUTER_HANDLER_END") + len("SENTINEL_OUTER_HANDLER_END\n") - assert result[start:end] == sibling, "patch 12 rewrote the outer handler's re-raise" + assert result[start:end] == sibling, "the pricing patch rewrote the outer handler's re-raise"