From f80b1bef113fb829256eba0758cbd00b7afe4b5a Mon Sep 17 00:00:00 2001 From: James Wiesebron Date: Tue, 28 Jul 2026 22:04:46 -0700 Subject: [PATCH 1/2] Bump litellm 1.86.2 -> 1.94.0 and retire four absorbed patches (#3697) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four of the twelve patch entries are now dead weight: upstream absorbed the fixes. Retired, with the upstream equivalent verified in the 1.94.0 source rather than inferred from a needle miss: - the streaming `reasoning_content` -> thinking-block branch - the first-delta requeue on block transitions, both the sync and async entries, now generalized upstream behind `_delta_has_content` so it covers text and thinking rather than only `input_json_delta` - the `prompt_tokens_details.cached_tokens` fallback A fifth narrows rather than dies. 1.94.0 carries streamed `cost` natively (upstream 8a49423, a port of BerriAI/litellm#16162) and feeds it to the cost calculator, which is more than egg's patch did. It still drops `cost_details`, where the BYOK number lives — OpenRouter's top-level `cost` is 0 when billing routes past them. So patch 7 is now `cost_details` only. That needed no code change: the helper never overwrote a value litellm already set, so it no-ops on `cost` and supplies the missing half. Twelve entries -> eight. TWO patches had to be REPOINTED, and this is the part worth reading before the next bump. Patch 2 (the cache_control gate) and patch 5 (the drop_params warning) both stopped matching at 1.94.0 purely because upstream reflowed the surrounding code — the gate collapsed onto one line and gained `is_bedrock_arn_model`, the drop site lost a line break. Neither fix was absorbed. Retiring them on the strength of the miss would have shipped an image paying full input rate on every Qwen turn, silently. Verified against the stock 1.94.0 wheel, which is what makes this a finding rather than a worry: on stock, qwen and deepseek lose `cache_control` and the `x-anthropic-billing-header` block leaks into every request. On the patched tree both hold, and gpt-4o-mini still correctly does NOT get `cache_control` — the gate is broadened, not widened. A needle miss means "look at this", not "upstream fixed it". Both the script docstring and the Dockerfile now say so. Also fixes a live false pass in the patch tests. They looked patches up by number prefix (`"Patch 4/"`), and numbers are positional: retiring patch 4 silently re-bound that test to the openrouter capability patch, where it applied an unrelated needle to a fixture built from it and kept reading green. Lookups are now by the descriptive tail of the label, which is stable across renumbering, and assert exactly one match so a retired patch fails loudly instead. The test for the retired patch 4 is deleted rather than repointed. Verification, all against the real 1.94.0 wheel with the patch script applied: - all 8 patches apply cleanly and idempotently - `cost` and `cost_details` both survive stream reassembly - laguna-s-2.1 prices from the live card; cost math matches hand arithmetic - cache_control survives for qwen/deepseek and not for gpt-4o-mini; the billing header is filtered on every route - the drop_params warning fires; the synthesized reasoning_effort stays suppressed by default Still outstanding, and not doable from here: the live re-measurement #3697 lists — cache hit rate on real traffic, and the #3624 reasoning-depth numbers. Both need a cluster and a model budget. The static behaviour they depend on is verified above, but the numbers themselves are unconfirmed on 1.94.0. --- config/litellm/Dockerfile | 21 +- config/litellm/anthropic_thinking_policy.py | 6 +- config/litellm/cost_callback.py | 29 +- config/litellm/drop_params_visibility.py | 6 +- config/litellm/openrouter_capabilities.py | 4 +- config/litellm/patch_litellm_cache.py | 652 +++++--------------- config/litellm/stream_cost_preservation.py | 2 +- docs/development/STRUCTURE.md | 8 +- docs/guides/per-agent-models.md | 27 +- k8s/base/litellm-deployment.yaml | 10 +- tests/config/test_patch_litellm_cache.py | 133 +--- 11 files changed, 252 insertions(+), 646 deletions(-) diff --git a/config/litellm/Dockerfile b/config/litellm/Dockerfile index 844b77121..bb8ac98d5 100644 --- a/config/litellm/Dockerfile +++ b/config/litellm/Dockerfile @@ -5,19 +5,26 @@ # stock image's Anthropic->OpenAI translation drops prompt-cache hits on # Qwen/DeepSeek, mis-streams reasoning models, silently drops params it does # not recognise, manufactures a reasoning ceiling from the caller's thinking -# budget, and discards the provider's own bill during stream reassembly while -# its rate card cannot price the route either. -# patch_litellm_cache.py closes the eleven gaps at build time +# budget, and loses the BYOK half of the provider's bill during stream +# reassembly while its rate card cannot price the route either. +# patch_litellm_cache.py closes the eight gaps at build time # (see that script for the diagnosis); # cost_callback.py surfaces the resulting cost/cache stats into the pod log # stream. Both mirror the host-side dotfiles setup that took Qwen cache hit # rate from 0% to ~99.99%. # -# Pinned to v1.86.2 — the version the patches and cost_callback were -# validated against, and the same release the dotfiles host install pins. +# Pinned to v1.94.0 — the newest stable published release, and the version +# the patches and cost_callback are validated against (#3697). The bump from +# v1.86.2 retired four patches upstream absorbed and narrowed a fifth. +# # The patch script fails the build loudly if a needle stops matching, so a -# future bump can't silently ship an unpatched (full-input-rate) image. -FROM ghcr.io/berriai/litellm:v1.86.2 +# future bump can't silently ship an unpatched (full-input-rate) image. Read +# that failure as "look at this", not as "upstream fixed it": at this bump, +# two needles stopped matching purely because upstream reflowed the code +# around them, and deleting those patches would have shipped an image paying +# full input rate on every Qwen turn. Confirm each miss against the new +# source before retiring anything. +FROM ghcr.io/berriai/litellm:v1.94.0 # Patches edit the installed litellm package (root-owned site-packages); the # build runs as root. The runtime pod still drops to UID 1000 via the diff --git a/config/litellm/anthropic_thinking_policy.py b/config/litellm/anthropic_thinking_policy.py index de8d73e4a..2d7840fcd 100644 --- a/config/litellm/anthropic_thinking_policy.py +++ b/config/litellm/anthropic_thinking_policy.py @@ -28,15 +28,15 @@ Historically this never mattered: the model-cost map did not carry these slugs, ``OpenrouterConfig`` advertised no reasoning knobs, and ``drop_params`` silently discarded the synthesized param — which is precisely why these models have been -running at full depth. Patch 7 makes the OpenRouter param gate accurate, which +running at full depth. Patch 4 makes the OpenRouter param gate accurate, which is right for an *operator-configured* ``reasoning_effort`` and wrong for this adapter-manufactured one: it would turn a knob nobody set into the effective setting, with no config file mentioning it and nothing in the logs (Patch 8 only fires on drops, and this param would no longer be dropped). -So Patch 9 gates the synthesis, defaulting it off. ``thinking`` stays out of +So Patch 6 gates the synthesis, defaulting it off. ``thinking`` stays out of the OpenAI body for non-Claude models and only an explicitly configured -``reasoning_effort`` reaches the wire — exactly the property Patch 7 exists to +``reasoning_effort`` reaches the wire — exactly the property Patch 4 exists to restore, without the adapter's bucket riding along. The gate covers the *derived* value only. On an adaptive request diff --git a/config/litellm/cost_callback.py b/config/litellm/cost_callback.py index 4e32f1005..a12407000 100644 --- a/config/litellm/cost_callback.py +++ b/config/litellm/cost_callback.py @@ -28,15 +28,17 @@ calculate_usage``, a rebuild that enumerates the token/cache counts and originally DROPPED the provider's ``cost`` / ``cost_details`` outright. That is why this module recorded ``cost: null`` on 1252 of 1252 sampled -calls in run 6. The egg-litellm image's **patch 10** now carries those two -fields across the rebuild (``config/litellm/stream_cost_preservation.py``), -so the billed figure reaches ``_extract_cost`` on the streaming path too +calls in run 6. litellm 1.94.0 carries ``cost`` across that rebuild, and the +egg-litellm image's **patch 7** carries the ``cost_details`` it still drops +(``config/litellm/stream_cost_preservation.py``) — that is where the BYOK +number lives. Between them the billed figure reaches ``_extract_cost`` on +the streaming path too and this module needs no change to read it — the value simply stops being absent (#3691). ``cost: null`` therefore no longer means "streaming"; it means the cost was -genuinely unavailable — a stock (unpatched) LiteLLM under this callback, or -a provider that does not report one. It is still emitted as null rather +genuinely unavailable — a LiteLLM older than 1.94.0 under this callback, a +BYOK route on an unpatched image, or a provider that does not report one. It is still emitted as null rather than ``0.0``: a zero would read in the logs as "this route is free", the exact opposite of the cost-visibility signal this module exists to provide (#2799). @@ -46,7 +48,7 @@ (issue #3175). It is kept strictly separate from ``cost`` — an estimate from a possibly-stale rate card must never be mistaken for a bill — and follows the same null-not-zero rule when LiteLLM cannot price the model. -That was the case for every route egg uses until the image's **patch 11** +That was the case for every route egg uses until the image's **patch 8** taught the model-info lookup to read OpenRouter's published rate card; it remains the case for a model whose rate card is tiered by prompt length, which LiteLLM's map cannot express (see ``openrouter_capabilities``). With @@ -164,7 +166,8 @@ def _usage_from_response_obj(response_obj): counts (the final usage chunk's counts are folded into ``response_obj.usage`` by ``stream_chunk_builder``) and, on the egg-litellm image, for cost as well: that reassembly rebuilds a fresh ``Usage`` and - stock drops ``cost`` / ``cost_details`` with it, which patch 10 restores. + older releases dropped ``cost`` / ``cost_details`` with it; 1.94.0 carries + ``cost`` and patch 7 supplies ``cost_details``. Under a stock LiteLLM the counts still arrive and ``_extract_cost`` returns None — see the module docstring.""" if response_obj is None: @@ -194,8 +197,8 @@ def _extract_cost(usage): provider, so fall back to ``cost_details.upstream_inference_cost`` (what the upstream provider will bill for the same request). Either way, the number we record matches real spend on that turn. Returns None when no - positive cost is present — a provider that reports none, or a stock LiteLLM - whose chunk reassembly drops it (see ``_usage_from_response_obj``). + positive cost is present — a provider that reports none, or a LiteLLM whose + chunk reassembly drops it (see ``_usage_from_response_obj``). Callers must treat None as "unknown", not "$0". ``_positive`` rejects non-finite values as well as non-positive ones: a @@ -321,7 +324,7 @@ def _extract_estimated_cost(mcd): reported a bill. It is an estimate, not a bill: the pricing map may lag the provider's rates or lack cache-discount entries for a model. Returns None — never 0.0 — when LiteLLM couldn't price the call, mirroring the billed-cost - "unknown ≠ free" discipline. On the egg-litellm image patch 11 supplies + "unknown ≠ free" discipline. On the egg-litellm image patch 8 supplies OpenRouter's published rates for slugs the bundled map does not carry, so a None here now means a genuinely unpriceable model (a tiered rate card, or a provider with no live card to read) rather than the routine case it was. @@ -374,7 +377,7 @@ def _extract_model(mcd): # # ``stream`` is included because it selects which of the two paths the cost on # this line came through (see the module docstring), and it was the reason -# ``cost`` read null on every line before patch 10 — worth having next to the +# ``cost`` read null on every line before 1.94.0 and patch 7 — worth having next to the # number rather than inferred, and worth keeping now that the null case is rare # enough to need explaining when it happens. ``max_tokens`` and ``n`` are not # sampling knobs but shape the generation, and are cheap to carry. @@ -641,7 +644,7 @@ def _extract_request_params(mcd): Source is ``model_call_details['optional_params']``: LiteLLM's POST-mapping parameter set, i.e. the dict that becomes the upstream - request body. Verified against the pinned litellm 1.86.2 on the + request body. Verified against the pinned litellm 1.94.0 on the ``anthropic_messages`` route (the route every Claude Code agent request takes), streaming and non-streaming — the streaming case matters most, since that is essentially all real agent traffic. @@ -843,7 +846,7 @@ def _record(self, mcd, response_obj): # /v1/messages route (#3624): stock LiteLLM rewrites the # caller's ``thinking`` block into a bucketed # ``reasoning_effort``, but that bucket is a cap below the - # model default, so egg-litellm's patch 9 gates the + # model default, so egg-litellm's patch 6 gates the # synthesis off by default. A missing key here means the # request ran at the model's own reasoning depth, not that # the field failed to record. diff --git a/config/litellm/drop_params_visibility.py b/config/litellm/drop_params_visibility.py index 43e755179..ee5910fa2 100644 --- a/config/litellm/drop_params_visibility.py +++ b/config/litellm/drop_params_visibility.py @@ -2,7 +2,7 @@ ``drop_params`` exists so an unsupported parameter does not fail the whole request, and that tradeoff is right. But dropping a parameter *changes -generation behaviour*, and in stock LiteLLM 1.86.2 it happens with no signal +generation behaviour*, and in stock LiteLLM it happens with no signal at all: the branch that pops them is a bare loop with no logging. A ``reasoning_effort``, ``temperature`` or penalty set in a proxy config simply never reaches the provider, and nothing in the logs or the response says so. @@ -15,14 +15,14 @@ to notice (jwbron/egg#3620, #3624). One log line would have made it a five-minute question. -Patch 7 fixes the OpenRouter false-negative specifically; this covers the rest. +Patch 4 fixes the OpenRouter false-negative specifically; this covers the rest. A drop can still be *correct* and worth knowing about: ``poolside/laguna-s-2.1`` genuinely does not accept ``reasoning_effort``, so the knob is dropped on purpose, and without this the operator has no way to learn why their config line does nothing. Mirrors jwbron/litellm#7 (merged into the fork the host proxy runs). The -cluster image pins stock 1.86.2, which predates it, hence this patch. +cluster image pins a stock release, which does not carry it, hence this patch. """ # Warn-once bookkeeping, keyed by (provider, model, sorted dropped param diff --git a/config/litellm/openrouter_capabilities.py b/config/litellm/openrouter_capabilities.py index 31b508c0b..13ffeef72 100644 --- a/config/litellm/openrouter_capabilities.py +++ b/config/litellm/openrouter_capabilities.py @@ -33,7 +33,7 @@ cost: ``supports_reasoning: true`` alone makes stock ``get_supported_openai_params`` admit ``thinking``, which Patch 2's notes explain would forward an Anthropic-shaped block verbatim to a provider that - expects ``reasoning``. Patch 7 remains the only path by which a parameter + expects ``reasoning``. Patch 4 remains the only path by which a parameter becomes admissible, and it admits exactly ``reasoning_effort``. * **Tiered rate cards are declined outright.** OpenRouter expresses a long-context surcharge as ``pricing.overrides`` — a list keyed by arbitrary @@ -70,7 +70,7 @@ both, since one fetch serves both. * ``LITELLM_OPENROUTER_PRICING=0`` disables only the pricing half, leaving the parameter lookup running. For an operator who wants LiteLLM's bundled map to - be the sole authority on cost while keeping Patch 7's parameter fix. + be the sole authority on cost while keeping Patch 4's parameter fix. * ``LITELLM_OPENROUTER_CAPABILITY_TTL`` seconds between refreshes (default 3600). ``0`` disables caching and re-fetches on every lookup — a debugging aid, not a production setting. diff --git a/config/litellm/patch_litellm_cache.py b/config/litellm/patch_litellm_cache.py index bea565844..caa83836e 100644 --- a/config/litellm/patch_litellm_cache.py +++ b/config/litellm/patch_litellm_cache.py @@ -1,33 +1,40 @@ #!/usr/bin/env python3 -"""Apply egg's LiteLLM prompt-cache, reasoning-stream and cost patches at image-build time. +"""Apply egg's LiteLLM cache, reasoning and cost patches at image-build time. LiteLLM's stock Anthropic->OpenAI translation (the path Claude Code's ``/v1/messages`` requests take when routed at a non-Claude OpenRouter -backend) drops prompt-cache hits for Qwen/DeepSeek and mis-streams -reasoning models, and its OpenRouter param gate reads a model-cost map -that does not carry current OpenRouter slugs, and it discards -caller-specified params in total silence, while manufacturing a -reasoning ceiling nobody asked for, and it destroys the provider's own -bill during stream reassembly while its rate card cannot price the -route either. Eleven independent gaps cause it; this -script closes all eleven by editing the installed ``litellm`` -package in place (and installing four new modules), then +backend) drops prompt-cache hits for Qwen/DeepSeek, reads a model-cost map +that carries neither the parameters nor the prices of current OpenRouter +slugs, discards caller-specified params in total silence, manufactures a +reasoning ceiling nobody asked for, and loses the BYOK half of the +provider's bill during stream reassembly. Eight independent gaps cause it; +this script closes all eight by editing the installed ``litellm`` package +in place (and installing four new modules), then ``config/litellm/Dockerfile`` bakes the result into the ``egg-litellm`` image. -The cache patches mirror the host-side ``relitellm`` script -(jwbron/dotfiles), which was validated empirically against -``litellm==1.86.2``: cache hit rate on Qwen via OpenRouter went 0% -> -~99.99%, ~10x input-cost cut on identical-prefix turns. The reasoning -patches mirror jwbron/litellm#4, validated against Kimi K2.7-Code via -OpenRouter. The image pins that same version (see the Dockerfile -``FROM``), so the needles below are known to match. +Pinned to ``litellm==1.94.0`` (see the Dockerfile ``FROM``), so the needles +below are known to match. The bump from 1.86.2 retired four patches whose +fixes upstream absorbed — the streaming ``reasoning_content`` thinking +block, the first-delta requeue on block transitions (both sync and async), +and the ``prompt_tokens_details.cached_tokens`` fallback — and narrowed a +fifth (7) to the half upstream still does not carry. See issue #3697 for +the audit, and read its warning before assuming the next bump retires +anything: TWO of the patches below (2 and 5) had their needles stop +matching at 1.94.0 purely because upstream reflowed the surrounding code, +and retiring them on the strength of that miss would have silently shipped +an image paying full input rate on every Qwen turn. 1. ``CacheControlSupportedModels`` (openrouter/chat/transformation.py) add QWEN + DEEPSEEK so ``cache_control`` survives the OpenRouter handler's strip step. Without it every turn pays full input rate. - 2. ``_add_cache_control_to_target`` cache_control gate (anthropic adapter - transformation.py) broaden ONLY that gate to qwen + deepseek so + 1.94.0 still lists only claude/gemini/minimax/glm/z-ai. NOTE the + jwbron/litellm fork adds QWEN but deliberately omits DEEPSEEK, on the + grounds that DeepSeek caching is automatic/prefix-based and ignores + ``cache_control`` — if that holds, egg's DEEPSEEK arm is inert rather + than wrong, and is a candidate for removal once measured. + 2. ``_add_cache_control_if_applicable`` cache_control gate (anthropic + adapter transformation.py) broaden ONLY that gate to qwen + deepseek so ``cache_control`` survives the Anthropic->OpenAI translation the ``/v1/messages`` endpoint forces. Without it patch 1 never sees ``cache_control`` (stripped earlier). We deliberately do NOT widen the @@ -44,44 +51,10 @@ Claude Code injects this as the FIRST system text block, ahead of the ``cache_control`` marker, carrying a per-request ``cch=``; that hash invalidates the prefix-cache key every turn, so - ``cache_read_input_tokens`` stays 0 forever. The sibling - ``messages/transformation.py`` path already filters it via - ``_filter_billing_headers_from_system``; the adapter path missed it. - 4. ``_translate_streaming_openai_chunk_to_anthropic_content_block`` - (anthropic adapter transformation.py) add a ``reasoning_content`` - branch that opens a proper ``thinking`` content block. OpenRouter-style - reasoning models stream ``delta.reasoning_content`` rather than - Anthropic-native ``delta.thinking_blocks``; without this patch the - block start is typed ``text`` while the deltas are - ``thinking_delta`` — a malformed stream that clients (e.g. Claude - Code) render as visible assistant text and feed back as assistant - content on later turns. - 5. ``AnthropicStreamWrapper`` block-transition requeue (anthropic adapter - streaming_iterator.py) preserve the trigger chunk's first delta on - text/thinking block transitions, not just on ``input_json_delta`` - tool transitions. Without it the first reasoning token of every - thinking block and the first answer token after thinking are silently - dropped; a one-chunk answer can vanish entirely. Applied to both the - sync ``__next__`` and async ``__anext__`` paths. - 6. ``AnthropicStreamWrapper`` usage merge (same streaming_iterator.py) - report provider-automatic cache hits in the streamed usage. The stock - code subtracts ``prompt_tokens_details.cached_tokens`` from - ``input_tokens`` but only emits ``cache_read_input_tokens`` from the - internal ``_cache_read_input_tokens`` field, which OpenAI/OpenRouter- - style usage never populates (only Anthropic- and DeepSeek-native - fields do). On any auto-caching route (Moonshot kimi-k3 ~94% hit - rate, Fireworks deepseek-flash, DeepInfra glm) the cached tokens - simply vanish from the usage Anthropic-format clients see: Claude - Code's context tracking sees only the uncached tail (~0% context - forever), so auto-compaction never triggers and the orchestrator's - transcript-derived token accounting undercounts. Mirrors the fix - upstreamed via jwbron/litellm#6 (upstream now falls back to - ``prompt_tokens_details.cached_tokens`` natively). Unlike Patch 5, - this has no sync ``__next__`` twin: the cache-token usage merge - exists only in the async ``__anext__`` path upstream (the sync path - has no equivalent merge block), and that async path is the one the - litellm proxy drives for Claude Code streaming. - 7. ``OpenrouterConfig.get_supported_openai_params`` + ``cache_read_input_tokens`` stays 0 forever. 1.94.0 filters it in + ``anthropic/chat/transformation.py`` and + ``messages/transformation.py`` but still not in this adapter path. + 4. ``OpenrouterConfig.get_supported_openai_params`` (openrouter/chat/transformation.py) consult OpenRouter's published per-model ``supported_parameters`` instead of only the bundled model-cost map. The stock gate asks ``litellm.supports_reasoning``, @@ -89,10 +62,9 @@ ships new slugs faster than that map tracks them, so a current model answers False. The gate is a bare ``if``, so it fails CLOSED, and ``drop_params: true`` discards the parameter with no exception - and no log line. Every OpenRouter slug egg routes is absent from the - 1.86.2 map, so a reasoning knob set on any of them never reached the - wire. The companion module ``llms/openrouter/_egg_capabilities.py`` - (installed by ``NEW_MODULES``) reads OpenRouter's unauthenticated + and no log line. The companion module + ``llms/openrouter/_egg_capabilities.py`` (installed by + ``NEW_MODULES``) reads OpenRouter's unauthenticated ``/api/v1/models`` and is UNIONED with the map answer, never subtractive: ``supported_parameters`` under-reports ``reasoning_effort`` (deepseek-r1 advertises only ``reasoning``, @@ -101,30 +73,30 @@ admitted — OpenRouter's ``reasoning`` field is a different wire shape from Anthropic's ``thinking``, not a spelling of it. Fails soft: any fetch error yields no opinion and the stock path runs unchanged. - Mirrors jwbron/litellm#8 and jwbron/egg#3624. Patch 9 is the other + Mirrors jwbron/litellm#8 and jwbron/egg#3624. Patch 6 is the other half of this one: read them together. - 8. ``get_optional_params`` drop site (utils.py) log what - ``drop_params`` discards. Stock 1.86.2 pops unsupported params in a - bare loop with no logging, so a param set in a proxy config that - never reaches the provider is a real behavioural difference with no - signal attached — the condition that made patch 7's bug take a full - investigation to find. Patch 7 removes the OpenRouter + 5. ``get_optional_params`` drop site (utils.py) log what + ``drop_params`` discards. Stock pops unsupported params in a bare + loop with no logging, so a param set in a proxy config that never + reaches the provider is a real behavioural difference with no signal + attached — the condition that made patch 4's bug take a full + investigation to find. Patch 4 removes the OpenRouter false-negative; this covers the rest, including drops that are CORRECT: laguna-s-2.1 genuinely does not accept ``reasoning_effort``, so it is dropped on purpose and the operator otherwise has no way to learn why their config line does nothing. Deduped per - (provider, model, param-set) and bounded. NOTE the needle: 1.86.2 has - two ``drop_params`` branches in utils.py and the shared condition - alone matches the wrong one, so the needle includes the pop loop. + (provider, model, param-set) and bounded. NOTE the needle: 1.94.0 + still has two ``drop_params`` branches in utils.py sharing one + spelling, so the needle includes the pop loop. Mirrors jwbron/litellm#7, merged into the fork the HOST proxy runs; - the cluster image pins stock 1.86.2, which predates it. The message - offers the ``allowed_openai_params`` remedy GATED on the params - having come from this model's ``litellm_params``, and names the - synthesized case alongside it: the param most often dropped here is - one litellm manufactured from the request itself (see 9), so an - unconditional config edit would send that operator hunting for a - line that does not exist. - 9. ``_translate_thinking_to_openai`` (anthropic adapter + BerriAI has not taken it. The message offers the + ``allowed_openai_params`` remedy GATED on the params having come from + this model's ``litellm_params``, and names the synthesized case + alongside it: the param most often dropped here is one litellm + manufactured from the request itself (see 6), so an unconditional + config edit would send that operator hunting for a line that does not + exist. + 6. ``_translate_thinking_to_openai`` (anthropic adapter transformation.py) stop synthesizing ``reasoning_effort`` from the caller's ``thinking`` block for non-Claude models. On ``/v1/messages`` — egg's primary route — Claude Code sends @@ -134,15 +106,15 @@ non-Claude branch where the adapter REPLACES that block with a bucketed ``reasoning_effort``. Nothing in ``litellm-models.yaml`` is involved: the value is manufactured per request. That was harmless - only because patch 7's bug dropped it; the measurements in + only because patch 4's bug dropped it; the measurements in jwbron/egg#3624 show the bucket is a CAP BELOW the model default (kimi-k3: 3130 reasoning tokens with no param, 340 with - ``reasoning_effort: high``, non-overlapping), so shipping patch 7 + ``reasoning_effort: high``, non-overlapping), so shipping patch 4 without this would cut reasoning ~9x per agent turn with no config - file to point at and nothing logged — patch 8 fires only on drops, + file to point at and nothing logged — patch 5 fires only on drops, and this param would no longer be dropped. Gating the synthesis (off by default, ``LITELLM_ANTHROPIC_THINKING_TO_REASONING_EFFORT=1`` to - restore stock) keeps patch 7's actual goal: a knob an operator + restore stock) keeps patch 4's actual goal: a knob an operator configured reaches the wire, one nobody configured does not. The Claude branch is untouched, and so is an effort the CALLER stated outright: on an adaptive request (``thinking: {"type": "adaptive"}`` @@ -153,40 +125,38 @@ with the derived effort, because stock carries the summary only as a field of the ``reasoning_effort`` dict and there is no wire shape for "summary, no effort". - 10. ``ChunkProcessor.calculate_usage`` + 7. ``ChunkProcessor.calculate_usage`` (litellm_core_utils/streaming_chunk_builder_utils.py) carry the - provider-billed ``cost`` / ``cost_details`` across stream - reassembly. OpenRouter reports what it charged on the final usage - chunk and stock already asks for it (``transform_request`` sets - ``usage: {"include": true}`` unconditionally), but this rebuild - enumerates token counts only and re-constructs ``Usage`` from its - own ``model_dump()``, so the bill is dropped. Claude Code streams - every ``/v1/messages`` request, so this is ~100% of routed traffic: - 1252 of 1252 sampled ``cost_callback`` lines on run 6 carried - ``cost: null`` (#3691). The companion module - ``litellm_core_utils/_egg_stream_cost.py`` transports the two - fields and interprets neither — a zero ``cost`` is the BYOK truth - and its fall-through partner sits under ``cost_details``. - 11. ``_get_model_info_helper`` (utils.py) price OpenRouter slugs the - bundled map has never heard of. Same root cause as 7, second - symptom: ``model_prices_and_context_window.json`` does not carry - current slugs, so the lookup raises "This model isn't mapped yet", - LiteLLM's ``response_cost`` is never computed, and egg's - ``cost_estimated`` reads null beside the null ``cost`` that 10 - fixes. The hook sits at the raise site, after every stock lookup - has failed, so a mapped slug keeps the bundled answer and the live - card can add a model but never reprice one. ``_egg_capabilities`` - grows a second entry point for this off the roster it already - caches; it answers for OpenRouter alone, carries cost fields only - (a ``supports_*`` flag through this door would change parameter - admission, which is 7's job), and declines a tiered rate card - outright rather than registering a base tier that would silently - under-report the long prompts agent traffic is made of. + provider's ``cost_details`` across stream reassembly. 1.86.2 dropped + the whole bill here, which put ``cost: null`` on 1252 of 1252 sampled + calls (#3691); 1.94.0 carries ``cost`` natively and feeds it to the + cost calculator, but still not ``cost_details`` — which is where the + BYOK number lives, since OpenRouter's top-level ``cost`` is 0 when + billing routes past them. The companion module + ``litellm_core_utils/_egg_stream_cost.py`` never overwrites a value + litellm already set, so it no-ops on ``cost`` and supplies only the + missing half. + 8. ``_get_model_info_helper`` (utils.py) price OpenRouter slugs the + bundled map has never heard of. Same root cause as 4, second + symptom: the map does not carry current slugs, so the lookup raises + "This model isn't mapped yet", ``response_cost`` is never computed, + and ``cost_estimated`` reads null. The hook sits at the raise site, + after every stock lookup has failed, so a mapped slug keeps the + bundled answer and the live card can add a model but never reprice + one. ``_egg_capabilities`` grows a second entry point for this off + the roster it already caches; it answers for OpenRouter alone, + carries cost fields only (a ``supports_*`` flag through this door + would change parameter admission, which is 4's job), and declines a + tiered rate card outright rather than registering a base tier that + would silently under-report the long prompts agent traffic is made + of. Proposed upstream as jwbron/litellm#10. Idempotent: each patch detects whether it is already applied. Fails loudly (non-zero exit) if a needle is missing, so a LiteLLM version bump that moves the code surfaces at build time instead of silently shipping an unpatched image that bills full input rate or drops reasoning tokens. +A needle miss means "look at this", NOT "upstream fixed it" — see the +1.94.0 note above. """ import ast @@ -311,16 +281,29 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - ' QWEN = "qwen"\n' ' DEEPSEEK = "deepseek"\n' ), - "label": "Patch 1/11 (CacheControlSupportedModels)", + "label": "Patch 1/8 (CacheControlSupportedModels)", }, # Patch 2 — broaden ONLY the cache_control gate (not the shared # is_anthropic_claude_model predicate, which also gates thinking # translation — see module docstring). Touches the single call site in - # _add_cache_control_to_target. + # _add_cache_control_if_applicable. + # + # NOT retired by the 1.94.0 bump, despite looking like it was. Upstream + # rewrote this gate — collapsed it to one line and added + # ``is_bedrock_arn_model`` — so the old needle stopped matching, and a + # needle miss is indistinguishable from "upstream fixed it" if you only + # look at the miss. Upstream added no qwen/deepseek arm: 1.94.0's + # ``CacheControlSupportedModels`` still lists only claude/gemini/minimax/ + # glm/z-ai. Retiring this on the strength of the miss would have shipped an + # image that pays FULL INPUT RATE on every Qwen turn — the most expensive + # single regression available here, and a silent one. { "file": F2, "present": "# egg cache patch. Broaden ONLY the cache_control gate", - "needle": " if cache_control and model and self.is_anthropic_claude_model(model):\n", + "needle": ( + " if cache_control and model and " + "(self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model)):\n" + ), "replacement": ( " # egg cache patch. Broaden ONLY the cache_control gate to\n" " # cover the OpenRouter qwen/deepseek routes. Do NOT widen\n" @@ -336,12 +319,13 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " and model\n" " and (\n" " self.is_anthropic_claude_model(model)\n" + " or self.is_bedrock_arn_model(model)\n" ' or "qwen" in _model_lower\n' ' or "deepseek" in _model_lower\n' " )\n" " ):\n" ), - "label": "Patch 2/11 (cache_control gate)", + "label": "Patch 2/8 (cache_control gate)", }, # Patch 3 — drop x-anthropic-billing-header during Anthropic->OpenAI translation. { @@ -375,337 +359,9 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - ' "text": text,\n' " }\n" ), - "label": "Patch 3/11 (x-anthropic-billing-header filter)", - }, - # Patch 4 — OpenRouter-style reasoning_content must open a thinking - # content block, not fall through to a text block. The bare - # ``thinking_blocks`` elif appears in two sibling functions; we anchor the - # needle on the preceding text-block elif (``choice.delta.content is not - # None ...``), which is unique to - # _translate_streaming_openai_chunk_to_anthropic_content_block — the other - # function's preceding branch is the ``tool_calls`` block. Without this - # anchor, an upstream reorder could silently retarget the str.replace. - { - "file": F2, - "present": "# egg reasoning patch: OpenRouter-style reasoning_content", - "needle": ( - " elif choice.delta.content is not None and len(choice.delta.content) > 0:\n" - ' return "text", TextBlock(type="text", text="")\n' - " elif isinstance(choice, StreamingChoices) and hasattr(\n" - ' choice.delta, "thinking_blocks"\n' - " ):\n" - ), - "replacement": ( - " elif choice.delta.content is not None and len(choice.delta.content) > 0:\n" - ' return "text", TextBlock(type="text", text="")\n' - " elif isinstance(choice, StreamingChoices) and getattr(\n" - ' choice.delta, "reasoning_content", None\n' - " ):\n" - " # egg reasoning patch: OpenRouter-style reasoning_content\n" - " # streams must open a thinking content block. Without\n" - " # this, thinking_delta events are emitted inside a\n" - " # text-typed block, which clients render as visible\n" - " # assistant text and feed back as assistant content.\n" - ' return "thinking", ChatCompletionThinkingBlock(\n' - ' type="thinking", thinking="", signature=""\n' - " )\n" - " elif isinstance(choice, StreamingChoices) and hasattr(\n" - ' choice.delta, "thinking_blocks"\n' - " ):\n" - ), - "label": "Patch 4/11 (reasoning_content thinking block)", - }, - # Patch 5a — sync __next__: don't drop the first delta on text or - # thinking block transitions. - { - "file": F3, - "present": "# egg reasoning patch (sync first-delta)", - "needle": ( - " # Queue the sequence: content_block_stop -> content_block_start\n" - " # For text blocks the trigger chunk is not emitted as a separate\n" - " # delta because content_block_start carries the information.\n" - " # For tool_use blocks we must also emit the trigger chunk's delta\n" - " # when it carries input_json_delta data, because some providers\n" - " # (e.g. xAI, Gemini) include tool arguments in the same streaming\n" - " # chunk as the function name/id.\n" - "\n" - " # 1. Stop current content block\n" - " self.chunk_queue.append(\n" - " {\n" - ' "type": "content_block_stop",\n' - ' "index": max(self.current_content_block_index - 1, 0),\n' - " }\n" - " )\n" - "\n" - " # 2. Start new content block\n" - " self.chunk_queue.append(\n" - " {\n" - ' "type": "content_block_start",\n' - ' "index": self.current_content_block_index,\n' - ' "content_block": self.current_content_block_start,\n' - " }\n" - " )\n" - "\n" - " # 3. If the trigger chunk carries tool argument data, queue it\n" - " # so the input_json_delta is not silently dropped.\n" - " if (\n" - ' processed_chunk.get("type") == "content_block_delta"\n' - ' and isinstance(processed_chunk.get("delta"), dict)\n' - ' and processed_chunk["delta"].get("type") == "input_json_delta"\n' - ' and processed_chunk["delta"].get("partial_json")\n' - " ):\n" - " self.chunk_queue.append(processed_chunk)\n" - ), - "replacement": ( - " # Queue the sequence: content_block_stop -> content_block_start,\n" - " # then re-queue the trigger chunk's delta when it carries payload\n" - " # the new content_block_start doesn't already include (see step 3).\n" - " # egg reasoning patch (sync first-delta)\n" - "\n" - " # 1. Stop current content block\n" - " self.chunk_queue.append(\n" - " {\n" - ' "type": "content_block_stop",\n' - ' "index": max(self.current_content_block_index - 1, 0),\n' - " }\n" - " )\n" - "\n" - " # 2. Start new content block\n" - " self.chunk_queue.append(\n" - " {\n" - ' "type": "content_block_start",\n' - ' "index": self.current_content_block_index,\n' - ' "content_block": self.current_content_block_start,\n' - " }\n" - " )\n" - "\n" - " # 3. If the trigger chunk itself carries delta payload not\n" - " # already embedded in the new content_block_start, queue it\n" - " # so the first delta of the block isn't silently dropped:\n" - " # - input_json_delta: some providers (e.g. xAI, Gemini)\n" - " # include tool arguments in the same streaming chunk as\n" - " # the function name/id.\n" - " # - text_delta: the text block start is always empty, so\n" - " # the trigger chunk's text is the block's first token.\n" - " # - thinking_delta: same, but only when the block start\n" - " # doesn't already embed the thinking content (it does\n" - " # for providers that send Anthropic-native\n" - " # thinking_blocks).\n" - ' if processed_chunk.get("type") == "content_block_delta" and isinstance(\n' - ' processed_chunk.get("delta"), dict\n' - " ):\n" - ' trigger_delta = processed_chunk["delta"]\n' - ' trigger_delta_type = trigger_delta.get("type")\n' - " if (\n" - " (\n" - ' trigger_delta_type == "input_json_delta"\n' - ' and trigger_delta.get("partial_json")\n' - " )\n" - " or (\n" - ' trigger_delta_type == "text_delta"\n' - ' and trigger_delta.get("text")\n' - " )\n" - " or (\n" - ' trigger_delta_type == "thinking_delta"\n' - ' and trigger_delta.get("thinking")\n' - ' and not self.current_content_block_start.get("thinking")\n' - " )\n" - " ):\n" - " self.chunk_queue.append(processed_chunk)\n" - ), - "label": "Patch 5a/11 (sync first-delta requeue)", - }, - # Patch 5b — async __anext__: same first-delta preservation. - { - "file": F3, - "present": "# egg reasoning patch (async first-delta)", - "needle": ( - " # Queue the sequence: content_block_stop -> content_block_start\n" - " # For text blocks the trigger chunk is not emitted as a separate\n" - " # delta because content_block_start carries the information.\n" - " # For tool_use blocks we must also emit the trigger chunk's delta\n" - " # when it carries input_json_delta data, because some providers\n" - " # (e.g. xAI, Gemini) include tool arguments in the same streaming\n" - " # chunk as the function name/id.\n" - "\n" - " # 1. Stop current content block\n" - " self.chunk_queue.append(\n" - " {\n" - ' "type": "content_block_stop",\n' - ' "index": max(self.current_content_block_index - 1, 0),\n' - " }\n" - " )\n" - " self.chunk_queue.append(\n" - " {\n" - ' "type": "content_block_start",\n' - ' "index": self.current_content_block_index,\n' - ' "content_block": self.current_content_block_start,\n' - " }\n" - " )\n" - "\n" - " # 3. If the trigger chunk carries tool argument data, queue it\n" - " # so the input_json_delta is not silently dropped.\n" - " if (\n" - ' processed_chunk.get("type") == "content_block_delta"\n' - ' and isinstance(processed_chunk.get("delta"), dict)\n' - ' and processed_chunk["delta"].get("type")\n' - ' == "input_json_delta"\n' - ' and processed_chunk["delta"].get("partial_json")\n' - " ):\n" - " self.chunk_queue.append(processed_chunk)\n" - ), - "replacement": ( - " # Queue the sequence: content_block_stop -> content_block_start,\n" - " # then re-queue the trigger chunk's delta when it carries payload\n" - " # the new content_block_start doesn't already include (see step 3).\n" - " # egg reasoning patch (async first-delta)\n" - "\n" - " # 1. Stop current content block\n" - " self.chunk_queue.append(\n" - " {\n" - ' "type": "content_block_stop",\n' - ' "index": max(self.current_content_block_index - 1, 0),\n' - " }\n" - " )\n" - " self.chunk_queue.append(\n" - " {\n" - ' "type": "content_block_start",\n' - ' "index": self.current_content_block_index,\n' - ' "content_block": self.current_content_block_start,\n' - " }\n" - " )\n" - "\n" - " # 3. If the trigger chunk itself carries delta payload\n" - " # not already embedded in the new content_block_start,\n" - " # queue it so the first delta of the block isn't\n" - " # silently dropped:\n" - " # - input_json_delta: some providers (e.g. xAI, Gemini)\n" - " # include tool arguments in the same streaming chunk\n" - " # as the function name/id.\n" - " # - text_delta: the text block start is always empty,\n" - " # so the trigger chunk's text is the block's first\n" - " # token.\n" - " # - thinking_delta: same, but only when the block start\n" - " # doesn't already embed the thinking content (it does\n" - " # for providers that send Anthropic-native\n" - " # thinking_blocks).\n" - ' if processed_chunk.get("type") == "content_block_delta" and isinstance(\n' - ' processed_chunk.get("delta"), dict\n' - " ):\n" - ' trigger_delta = processed_chunk["delta"]\n' - ' trigger_delta_type = trigger_delta.get("type")\n' - " if (\n" - " (\n" - ' trigger_delta_type == "input_json_delta"\n' - ' and trigger_delta.get("partial_json")\n' - " )\n" - " or (\n" - ' trigger_delta_type == "text_delta"\n' - ' and trigger_delta.get("text")\n' - " )\n" - " or (\n" - ' trigger_delta_type == "thinking_delta"\n' - ' and trigger_delta.get("thinking")\n' - ' and not self.current_content_block_start.get("thinking")\n' - " )\n" - " ):\n" - " self.chunk_queue.append(processed_chunk)\n" - ), - "label": "Patch 5b/11 (async first-delta requeue)", - }, - # Patch 6 — streamed usage must report provider-automatic cache hits. - # The needle spans the whole usage-merge region so both edit points - # (the cached_tokens hoist and the cache_read fallback) land atomically. - { - "file": F3, - "present": "# egg cache patch (streaming cache_read fallback)", - "needle": ( - " # Add usage to the held chunk\n" - " uncached_input_tokens = chunk.usage.prompt_tokens or 0\n" - " if (\n" - ' hasattr(chunk.usage, "prompt_tokens_details")\n' - " and chunk.usage.prompt_tokens_details\n" - " ):\n" - " cached_tokens = (\n" - " getattr(\n" - ' chunk.usage.prompt_tokens_details, "cached_tokens", 0\n' - " )\n" - " or 0\n" - " )\n" - " uncached_input_tokens -= cached_tokens\n" - "\n" - " usage_dict: UsageDelta = {\n" - ' "input_tokens": uncached_input_tokens,\n' - ' "output_tokens": chunk.usage.completion_tokens or 0,\n' - " }\n" - " # Add cache tokens if available (for prompt caching support)\n" - " if (\n" - ' hasattr(chunk.usage, "_cache_creation_input_tokens")\n' - " and chunk.usage._cache_creation_input_tokens > 0\n" - " ):\n" - ' usage_dict["cache_creation_input_tokens"] = (\n' - " chunk.usage._cache_creation_input_tokens\n" - " )\n" - " if (\n" - ' hasattr(chunk.usage, "_cache_read_input_tokens")\n' - " and chunk.usage._cache_read_input_tokens > 0\n" - " ):\n" - ' usage_dict["cache_read_input_tokens"] = (\n' - " chunk.usage._cache_read_input_tokens\n" - " )\n" - ), - "replacement": ( - " # Add usage to the held chunk\n" - " uncached_input_tokens = chunk.usage.prompt_tokens or 0\n" - " cached_tokens = 0\n" - " if (\n" - ' hasattr(chunk.usage, "prompt_tokens_details")\n' - " and chunk.usage.prompt_tokens_details\n" - " ):\n" - " cached_tokens = (\n" - " getattr(\n" - ' chunk.usage.prompt_tokens_details, "cached_tokens", 0\n' - " )\n" - " or 0\n" - " )\n" - " uncached_input_tokens -= cached_tokens\n" - "\n" - " usage_dict: UsageDelta = {\n" - ' "input_tokens": uncached_input_tokens,\n' - ' "output_tokens": chunk.usage.completion_tokens or 0,\n' - " }\n" - " # Add cache tokens if available (for prompt caching support)\n" - " if (\n" - ' hasattr(chunk.usage, "_cache_creation_input_tokens")\n' - " and chunk.usage._cache_creation_input_tokens > 0\n" - " ):\n" - ' usage_dict["cache_creation_input_tokens"] = (\n' - " chunk.usage._cache_creation_input_tokens\n" - " )\n" - " if (\n" - ' hasattr(chunk.usage, "_cache_read_input_tokens")\n' - " and chunk.usage._cache_read_input_tokens > 0\n" - " ):\n" - ' usage_dict["cache_read_input_tokens"] = (\n' - " chunk.usage._cache_read_input_tokens\n" - " )\n" - " # egg cache patch (streaming cache_read fallback).\n" - " # OpenAI/OpenRouter-style usage reports the cache hit in\n" - " # prompt_tokens_details.cached_tokens; Usage.__init__ only\n" - " # populates _cache_read_input_tokens from Anthropic- or\n" - " # DeepSeek-native fields. Without this fallback the cached\n" - " # tokens are subtracted from input_tokens above but never\n" - " # reported, so Anthropic-format clients (Claude Code\n" - " # context tracking, the orchestrator token accounting)\n" - " # undercount the prompt by the cached amount on every\n" - " # cache hit. Mirrors jwbron/litellm#6 (fixed natively\n" - " # upstream after v1.86.2).\n" - " elif cached_tokens > 0:\n" - ' usage_dict["cache_read_input_tokens"] = cached_tokens\n' - ), - "label": "Patch 6/11 (streaming cache_read fallback)", + "label": "Patch 3/8 (x-anthropic-billing-header filter)", }, - # Patch 7 — OpenrouterConfig.get_supported_openai_params: consult + # Patch 4 — OpenrouterConfig.get_supported_openai_params: consult # OpenRouter's published capabilities instead of only the bundled # model-cost map. # @@ -715,7 +371,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - # bundled map lags, so a current model answers False. The gate is a bare # ``if``, so it fails CLOSED, and ``drop_params: true`` then discards the # parameter with no exception and no log line. Every OpenRouter slug egg - # routes is absent from the 1.86.2 map (kimi-k3, glm-5.2, laguna-s-2.1, + # routes is absent from the bundled map (kimi-k3, glm-5.2, laguna-s-2.1, # deepseek-v4-*), so any reasoning knob set on them never reached the wire. # # The companion module (installed by ``NEW_MODULES`` below) reads @@ -774,7 +430,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " pass\n" " try:\n" ), - "label": "Patch 7/11 (openrouter live capabilities)", + "label": "Patch 4/8 (openrouter live capabilities)", }, # Patch 8 — get_optional_params: log what ``drop_params`` discards. # @@ -782,30 +438,31 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - # correct and still worth knowing about: laguna-s-2.1 genuinely does not # accept ``reasoning_effort``, so the knob is dropped on purpose and the # operator has no way to learn why their config line does nothing. Stock - # 1.86.2 pops unsupported params in a bare loop with no logging at all. + # Stock pops unsupported params in a bare loop with no logging at all. # - # NEEDLE DISAMBIGUATION: 1.86.2 has TWO ``if litellm.drop_params is True or - # (...)`` sites in utils.py. The other one (~line 3303, the embeddings - # path) is followed by a bare ``pass``; this one is followed by the pop - # loop. The needle therefore includes the loop line — the shared condition - # alone would match whichever comes first and patch the wrong function. + # NOT retired by the 1.94.0 bump. Upstream reflowed the condition onto one + # line, so the old needle stopped matching — but nothing upstream logs the + # drop; the warn-once bookkeeping exists only in jwbron/litellm (PR #7), + # which BerriAI has not taken. Repointed, not deleted. # - # Mirrors jwbron/litellm#7, merged into the fork the HOST proxy runs. The - # cluster image pins stock 1.86.2, which predates it. + # NEEDLE DISAMBIGUATION: 1.94.0 still has TWO ``if litellm.drop_params is + # True or (...)`` sites in utils.py with this exact one-line spelling — the + # other (the embeddings path) is followed by a bare ``pass``, this one by + # the pop loop. Verified: the bare condition matches twice, the condition + # plus the loop matches once. The needle therefore keeps the loop lines; + # matching on the condition alone would patch whichever came first. { "file": F4, "present": "# egg drop_params visibility patch", "needle": ( - " if litellm.drop_params is True or (\n" - " drop_params is not None and drop_params is True\n" - " ):\n" + " if litellm.drop_params is True or " + "(drop_params is not None and drop_params is True):\n" " for k in unsupported_params.keys():\n" " non_default_params.pop(k, None)\n" ), "replacement": ( - " if litellm.drop_params is True or (\n" - " drop_params is not None and drop_params is True\n" - " ):\n" + " if litellm.drop_params is True or " + "(drop_params is not None and drop_params is True):\n" " # egg drop_params visibility patch. Dropping a param changes\n" " # generation behaviour; stock does it with no signal at all.\n" " try:\n" @@ -823,9 +480,9 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " for k in unsupported_params.keys():\n" " non_default_params.pop(k, None)\n" ), - "label": "Patch 8/11 (drop_params visibility)", + "label": "Patch 5/8 (drop_params visibility)", }, - # Patch 9 — _translate_thinking_to_openai: stop synthesizing + # Patch 6 — _translate_thinking_to_openai: stop synthesizing # ``reasoning_effort`` from the caller's ``thinking`` block for non-Claude # models. # @@ -895,7 +552,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " # egg thinking-synthesis patch. Everything above DERIVES an effort\n" " # from the caller's thinking budget, and that bucket is a cap BELOW\n" " # the model default on every model egg routes, so sending it\n" - " # silently shallows reasoning. Off by default; see the patch 9 notes\n" + " # silently shallows reasoning. Off by default; see the patch 6 notes\n" " # in patch_litellm_cache.py. An effort the caller stated outright\n" " # (output_config.effort) is an instruction rather than a\n" " # manufactured ceiling, and is never suppressed.\n" @@ -916,36 +573,43 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - "\n" ' summary = thinking.get("summary") if isinstance(thinking, dict) else None\n' ), - "label": "Patch 9/11 (thinking->reasoning_effort synthesis gate)", + "label": "Patch 6/8 (thinking->reasoning_effort synthesis gate)", }, - # Patch 10 — ChunkProcessor.calculate_usage: carry the provider-billed cost - # across stream reassembly. + # Patch 7 — ChunkProcessor.calculate_usage: carry the provider's + # ``cost_details`` across stream reassembly. + # + # SCOPE CHANGED at the 1.94.0 bump, and the half that remains is the + # subtler one. 1.86.2 dropped BOTH ``cost`` and ``cost_details`` here: the + # rebuild enumerates token counts and re-constructs ``Usage`` from its own + # ``model_dump()``, so anything the provider attached that litellm did not + # name was gone. Claude Code streams every /v1/messages request, so that + # seam was ~100% of egg's routed traffic — run 6 sampled 1252 + # ``cost_callback`` lines and 1252 carried ``cost: null`` (#3691). # - # OpenRouter reports what it charged on the final streamed usage chunk, and - # stock ``OpenrouterConfig.transform_request`` already asks for it - # (``usage: {"include": true}`` on every request). The number reaches - # litellm intact: ``chunk_parser`` hands the raw block to - # ``ModelResponseStream``, whose ``Usage`` keeps ``cost`` as a declared - # field and ``cost_details`` as a pydantic extra. Then ``calculate_usage`` - # rebuilds a fresh ``Usage`` field-by-field over the counts it enumerates - # and re-constructs it from its own ``model_dump()`` — and the bill is gone. + # 1.94.0 carries ``cost`` natively (upstream 8a49423, a port of + # BerriAI/litellm#16162) and goes further than egg did, feeding the figure + # into litellm's own cost calculator via ``_hidden_params``. Verified + # empirically against the stock 1.94.0 wheel: ``cost`` survives the rebuild, + # ``cost_details`` does not. # - # Claude Code streams every /v1/messages request, so this seam is on ~100% - # of egg's routed traffic: run 6 sampled 1252 cost_callback lines and 1252 - # carried ``cost: null`` (#3691). The non-streaming path was unaffected - # (``original_response`` there holds the raw provider JSON), which is why - # this read as a property of the route rather than as a transport bug. + # ``cost_details`` is where the BYOK bill lives. Under BYOK OpenRouter's + # top-level ``cost`` is a literal 0 because billing routes past them, and + # the real number is ``cost_details.upstream_inference_cost``. Upstream + # records the zero and stops, so without this the BYOK path still reports no + # spend — a null that looks identical to the bug #3691 fixed. Nothing + # upstream carries this field; see the note at the end of + # jwbron/litellm#10 for the shape a BerriAI PR would take. # # Placed AFTER the ``Usage(**model_dump())`` rebuild, not before: the # constructor deletes a ``cost`` attribute it is handed as None, so setting # it first would put the value somewhere the rebuild is entitled to discard. # The companion module (``litellm_core_utils/_egg_stream_cost.py``, in - # NEW_MODULES) never overwrites a value litellm already carried, so a future - # release that fixes this upstream turns the patch into a no-op instead of a - # competing second opinion. + # NEW_MODULES) never overwrites a value litellm already carried, which is + # exactly why the scope narrowing needed no code change: it now no-ops on + # ``cost`` and still supplies ``cost_details``. { "file": F5, - "present": "# egg cost patch. Carry the provider-billed cost", + "present": "# egg cost patch. Carry the provider's cost_details", "needle": ( " # Return a new usage object with the new values\n" "\n" @@ -958,11 +622,11 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - "\n" " returned_usage = Usage(**returned_usage.model_dump())\n" "\n" - " # egg cost patch. Carry the provider-billed cost across this\n" - " # rebuild — it enumerates token counts only, so `cost` /\n" - " # `cost_details` (what OpenRouter actually charged) are dropped\n" - " # here on every streamed call. See patch 10 notes in\n" - " # patch_litellm_cache.py.\n" + " # egg cost patch. Carry the provider's cost_details across this\n" + " # rebuild. Upstream carries `cost` but not `cost_details`, which\n" + " # is where the BYOK bill lives (`cost` is 0 there). The helper\n" + " # never overwrites what litellm already set, so it no-ops on\n" + " # `cost`. See patch 7 notes in patch_litellm_cache.py.\n" " try:\n" " from litellm.litellm_core_utils._egg_stream_cost import (\n" " carry_upstream_cost as _egg_carry_upstream_cost,\n" @@ -974,16 +638,16 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - "\n" " return returned_usage\n" ), - "label": "Patch 10/11 (streamed cost preservation)", + "label": "Patch 7/8 (streamed cost_details preservation)", }, - # Patch 11 — _get_model_info_helper: price OpenRouter slugs the bundled map + # Patch 8 — _get_model_info_helper: price OpenRouter slugs the bundled map # has never heard of. # # Same root cause as patch 7, second symptom. LiteLLM's own # ``response_cost`` is computed from ``model_prices_and_context_window.json``, # which does not carry current OpenRouter slugs, so the lookup raises "This # model isn't mapped yet" and egg's ``cost_estimated`` reads null on every - # routed call (#3691). Patch 10 recovers the *billed* figure; this one + # routed call (#3691). Patch 7 recovers the BYOK *billed* figure; this one # restores the independent estimate beside it, which is what remains # readable if a provider ever stops reporting cost. # @@ -1016,7 +680,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " # before giving up — the bundled map lags its slugs by\n" " # construction, so every route egg uses lands here and every\n" " # routed call reports a null cost estimate. Cost fields only;\n" - " # see patch 11 notes in patch_litellm_cache.py.\n" + " # see patch 8 notes in patch_litellm_cache.py.\n" " try:\n" " from litellm.llms.openrouter._egg_capabilities import (\n" " get_model_cost_entry as _egg_openrouter_cost_entry,\n" @@ -1036,7 +700,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - 'https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"\n' " )\n" ), - "label": "Patch 11/11 (openrouter live pricing)", + "label": "Patch 8/8 (openrouter live pricing)", }, ] diff --git a/config/litellm/stream_cost_preservation.py b/config/litellm/stream_cost_preservation.py index e814ccee7..bc8287c4a 100644 --- a/config/litellm/stream_cost_preservation.py +++ b/config/litellm/stream_cost_preservation.py @@ -50,7 +50,7 @@ Installed into every litellm tree as ``litellm_core_utils/_egg_stream_cost.py`` by -``config/litellm/patch_litellm_cache.py`` (Patch 10 is the call site). Kept as +``config/litellm/patch_litellm_cache.py`` (Patch 7 is the call site). Kept as a real file rather than a string literal in that script so it stays lintable and unit-testable in the egg repo — which is why it imports no litellm symbols at all and works structurally, off ``getattr``/``dict`` access. diff --git a/docs/development/STRUCTURE.md b/docs/development/STRUCTURE.md index b24fedcd5..2c78260f4 100644 --- a/docs/development/STRUCTURE.md +++ b/docs/development/STRUCTURE.md @@ -543,10 +543,10 @@ config/ ├── litellm/ # egg-litellm image sources │ ├── Dockerfile # Builds egg-litellm: stock LiteLLM + prompt-cache and reasoning-parameter patches │ ├── patch_litellm_cache.py # Build-time patches: cache_control passthrough on Qwen/DeepSeek routes, live OpenRouter capability + pricing lookup, drop_params visibility, no synthesized reasoning ceiling, streamed cost preservation -│ ├── openrouter_capabilities.py # Patches 7 + 11: live GET /api/v1/models lookup — advertised parameters (unioned with LiteLLM's bundled model-cost map) and the published rate card for slugs that map has never heard of -│ ├── drop_params_visibility.py # Patch 8: warn once per proxy process per (provider, model, param-set) when drop_params discards a parameter -│ ├── anthropic_thinking_policy.py # Patch 9: stop synthesizing a reasoning_effort ceiling from the caller's thinking budget on non-Claude models -│ ├── stream_cost_preservation.py # Patch 10: carry the provider-billed cost / cost_details across LiteLLM's stream reassembly, which enumerates token counts only and drops them +│ ├── openrouter_capabilities.py # Patches 4 + 8: live GET /api/v1/models lookup — advertised parameters (unioned with LiteLLM's bundled model-cost map) and the published rate card for slugs that map has never heard of +│ ├── drop_params_visibility.py # Patch 5: warn once per proxy process per (provider, model, param-set) when drop_params discards a parameter +│ ├── anthropic_thinking_policy.py # Patch 6: stop synthesizing a reasoning_effort ceiling from the caller's thinking budget on non-Claude models +│ ├── stream_cost_preservation.py # Patch 7: carry the provider's cost_details across LiteLLM's stream reassembly (1.94.0 carries `cost` itself, but not this — it is where the BYOK bill lives) │ └── cost_callback.py # LiteLLM custom logger: upstream + estimated cost, per-role attribution (x-egg-* headers), cache hit rate, per-call decoding config -> pod stdout ├── redis/ # egg-redis image sources │ └── Dockerfile # Builds egg-redis: pinned stock Redis, repackaged for the local build/publish supply chain; backs the orchestrator's Redis Streams message store diff --git a/docs/guides/per-agent-models.md b/docs/guides/per-agent-models.md index b31e26f0b..d36611913 100644 --- a/docs/guides/per-agent-models.md +++ b/docs/guides/per-agent-models.md @@ -593,7 +593,7 @@ data: > Claude Code prepends (the block's `cch=` hash invalidates the cache key > every turn). egg ships a custom **`egg-litellm`** image > ([`config/litellm/Dockerfile`](../../config/litellm/Dockerfile)) that -> bakes in eleven patches closing those gaps, the reasoning-parameter ones +> bakes in eight patches closing those gaps, the reasoning-parameter ones > below, and the cost-visibility ones after that > ([`config/litellm/patch_litellm_cache.py`](../../config/litellm/patch_litellm_cache.py)); > the build fails loudly if a LiteLLM bump moves the patched code. Pinning @@ -614,7 +614,7 @@ data: > (issue #3691; before them, 1252 of 1252 sampled calls on run 6 reported > `cost: null`, so egg had no dollar figure at all for its LLM spend). > -> - **Patch 10 — the bill survives streaming.** `cost` is what OpenRouter +> - **Patch 7 — the BYOK bill survives streaming.** `cost` is what OpenRouter > charged, reported on the final streamed usage chunk. No config change is > needed to get it: stock LiteLLM's `OpenrouterConfig.transform_request` > already sets `usage: {"include": true}` on every request, so the number @@ -623,16 +623,17 @@ data: > enumerates and dropped `cost` / `cost_details` at that seam. Claude Code > streams every `/v1/messages` request, so that was ~100% of routed > traffic; the non-streaming path was never affected, which is why it read -> as a property of the route rather than as a transport bug. The patch -> copies the two fields across the rebuild and interprets neither: a `0` -> under BYOK is the literal truth about the OpenRouter bill, with the real -> number beside it under `cost_details.upstream_inference_cost`. -> - **Patch 11 — the estimate has a rate card.** `cost_estimated` is +> as a property of the route rather than as a transport bug. litellm 1.94.0 +> now carries `cost` natively, so the patch is down to `cost_details` — +> which is the half that matters under BYOK, where the top-level `cost` is a +> literal `0` and the real number is `cost_details.upstream_inference_cost`. +> The patch interprets neither field; it only transports. +> - **Patch 8 — the estimate has a rate card.** `cost_estimated` is > LiteLLM's own `response_cost`, computed from its bundled pricing map — > which carries none of the slugs egg routes, exactly as it carries none of -> their `supported_parameters` (patch 7, same root cause). The patch reads +> their `supported_parameters` (patch 4, same root cause). The patch reads > OpenRouter's published rate card off the same `GET /api/v1/models` fetch -> patch 7 already makes, and hands it to the model-info lookup only after +> patch 4 already makes, and hands it to the model-info lookup only after > every bundled lookup has failed — so a mapped slug keeps its bundled > rate, and the live card can add a model but never reprice one. > `LITELLM_OPENROUTER_PRICING=0` turns off just this half. @@ -669,7 +670,7 @@ data: > [`k8s/base/litellm-deployment.yaml`](../../k8s/base/litellm-deployment.yaml) > (where they are present but commented out) without rebuilding the image: > -> - **Patch 7 — live capability lookup.** LiteLLM gates reasoning params on +> - **Patch 4 — live capability lookup.** LiteLLM gates reasoning params on > its bundled model-cost map, which does not carry current OpenRouter > slugs, so a `reasoning_effort` set in `litellm_params` was discarded > before the request body was built — no exception, no log line. The patch @@ -683,10 +684,10 @@ data: > phase) tune it. A fetch that fails is cached for the TTL too, so an > offline cluster costs one attempt per hour rather than one per request, > and the first failure is logged at `warning`. Note `_FETCH=0` also -> disables patch 11's pricing lookup — one fetch serves both, so the master +> disables patch 8's pricing lookup — one fetch serves both, so the master > switch governs both; `LITELLM_OPENROUTER_PRICING=0` turns off only the > pricing half. -> - **Patch 9 — no synthesized reasoning ceiling.** On `/v1/messages` (the +> - **Patch 6 — no synthesized reasoning ceiling.** On `/v1/messages` (the > route Claude Code uses) LiteLLM's Anthropic adapter converts each > request's `thinking: {budget_tokens: N}` into a bucketed > `reasoning_effort` for any non-Claude model. That value is not in any @@ -726,7 +727,7 @@ data: > - **It is read after `drop_params` has acted**, so it reports what the wire > carried, not what your config asked for. A knob you set in > `litellm_params` that does not appear here was discarded or relocated -> into `extra_body` by LiteLLM's parameter mapper. Patch 8 +> into `extra_body` by LiteLLM's parameter mapper. Patch 5 > (`drop_params_visibility.py`) also logs a `litellm.drop_params: dropped > ...` warning once per (provider, model, param-set) combo, for as long as > the proxy's bookkeeping set holds that combo, naming the params and — when diff --git a/k8s/base/litellm-deployment.yaml b/k8s/base/litellm-deployment.yaml index 7353b07ff..09cd0fddc 100644 --- a/k8s/base/litellm-deployment.yaml +++ b/k8s/base/litellm-deployment.yaml @@ -99,7 +99,7 @@ spec: # are what the patches were measured with. Uncomment to revert an # individual patch at runtime without rebuilding the image. # - # Patch 7 — live OpenRouter capability lookup. ``0`` restores the + # Patch 4 — live OpenRouter capability lookup. ``0`` restores the # bundled model-cost map as the only source of truth, which # silently drops reasoning knobs on any slug the map has not caught # up to. ``_TTL`` seconds between refreshes (default 3600; 0 means @@ -112,16 +112,16 @@ spec: # - name: LITELLM_OPENROUTER_CAPABILITY_TIMEOUT # value: "5" # - # Patch 11 — live OpenRouter pricing, read off the same roster - # fetch as patch 7. ``0`` disables only the pricing half, leaving + # Patch 8 — live OpenRouter pricing, read off the same roster + # fetch as patch 4. ``0`` disables only the pricing half, leaving # the capability lookup running: LiteLLM's bundled map becomes the # sole authority on cost, and since it carries none of the slugs # egg routes, ``cost_estimated`` goes back to null on every call. - # The provider-billed ``cost`` (patch 10) is unaffected by this. + # The provider-billed ``cost`` is unaffected by this. # - name: LITELLM_OPENROUTER_PRICING # value: "0" # - # Patch 9 — thinking -> reasoning_effort synthesis, off by + # Patch 6 — thinking -> reasoning_effort synthesis, off by # default. ``1`` restores stock LiteLLM: the Anthropic adapter # derives a bucketed ``reasoning_effort`` from each request's # thinking budget on /v1/messages. Measured on kimi-k3 that bucket diff --git a/tests/config/test_patch_litellm_cache.py b/tests/config/test_patch_litellm_cache.py index 61013f7c2..416472bf6 100644 --- a/tests/config/test_patch_litellm_cache.py +++ b/tests/config/test_patch_litellm_cache.py @@ -38,6 +38,24 @@ def _load_patch_module(): plc = _load_patch_module() +def _patch_by_description(description: str) -> dict: + """Find a patch spec by the descriptive tail of its label. + + Deliberately NOT by number. Numbers are positional and get reused: the + 1.94.0 bump (#3697) retired four patches and renumbered the rest, and a + lookup keyed on ``"Patch 4/"`` silently re-bound to a completely different + patch and kept passing — testing nothing, while reading green. The + description is the stable identity, and an exact-count assertion turns a + retired patch into a loud failure instead of a false pass. + """ + matches = [p for p in plc.PATCHES if p["label"].endswith(f"({description})")] + assert len(matches) == 1, ( + f"expected exactly one patch described {description!r}, found " + f"{[p['label'] for p in matches]}" + ) + return matches[0] + + def _build_fixture_root(root: Path) -> None: """Write one fixture file per patched ``litellm`` path, each containing every needle that targets that path concatenated verbatim.""" @@ -261,94 +279,7 @@ def test_new_module_destinations_carry_the_egg_prefix(tmp_path): assert "must be prefixed" in str(excinfo.value) -def test_patch4_needle_anchors_on_content_block_function(tmp_path): - """Regression for the Patch 4 needle-uniqueness fix (#3199 review). - - The bare ``thinking_blocks`` elif appears in two sibling functions. The - Patch 4 needle anchors on the preceding text-block elif - (``choice.delta.content is not None ...``), which is unique to - ``_translate_streaming_openai_chunk_to_anthropic_content_block``. A - fixture containing a *second* (sibling-function-style) bare - ``thinking_blocks`` elif — preceded by a ``tool_calls`` block, not the - text elif — must be left untouched.""" - # Matched on "Patch 4/" rather than the full label so adding a patch (and - # renumbering the denominators) does not silently turn this into a - # StopIteration instead of a real assertion. - patch4 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 4/")) - - # Sibling function: bare thinking_blocks elif preceded by a tool_calls - # branch (mirrors _translate_streaming_openai_chunk_to_anthropic). It must - # NOT match the Patch 4 needle. - sibling = ( - "SENTINEL_SIBLING_BEGIN\n" - " if choice.delta.tool_calls is not None:\n" - " partial_json = ''\n" - " elif isinstance(choice, StreamingChoices) and hasattr(\n" - ' choice.delta, "thinking_blocks"\n' - " ):\n" - " pass\n" - "SENTINEL_SIBLING_END\n" - ) - fixture = sibling + "\n" + patch4["needle"] - - f2 = tmp_path / patch4["file"] - f2.parent.mkdir(parents=True, exist_ok=True) - f2.write_text(fixture) - - plc._apply( - str(f2), - present=patch4["present"], - needle=patch4["needle"], - replacement=patch4["replacement"], - label=patch4["label"], - ) - result = f2.read_text() - - # The intended branch was rewritten exactly once. - assert result.count(patch4["present"]) == 1 - # The sibling block is byte-for-byte unchanged. - start = result.index("SENTINEL_SIBLING_BEGIN") - end = result.index("SENTINEL_SIBLING_END") + len("SENTINEL_SIBLING_END\n") - assert result[start:end] == sibling - - -def test_new_modules_are_installed_into_each_root(tmp_path): - """``NEW_MODULES`` drops whole files that have no stock counterpart. - - Patch 7's gate imports ``litellm.llms.openrouter._egg_capabilities``, so if - the module install silently no-ops the patched gate raises ImportError on - every request — caught by its ``except Exception``, which would put us right - back at the silent-drop behaviour the patch exists to remove.""" - _build_fixture_root(tmp_path) - plc._patch_root(str(tmp_path)) - - for spec in plc.NEW_MODULES: - dest = tmp_path / spec["dest"] - assert dest.is_file(), f"{spec['label']}: not installed" - source = Path(plc._module_source(spec["source"], spec["label"])) - installed = dest.read_text() - assert installed.startswith(plc.EGG_MODULE_HEADER), f"{spec['label']}: no provenance" - assert installed == plc.EGG_MODULE_HEADER + source.read_text(), ( - f"{spec['label']}: content drift" - ) - - -def test_new_module_install_is_idempotent(tmp_path): - _build_fixture_root(tmp_path) - plc._patch_root(str(tmp_path)) - first = {spec["dest"]: (tmp_path / spec["dest"]).read_text() for spec in plc.NEW_MODULES} - plc._patch_root(str(tmp_path)) - for dest, content in first.items(): - assert (tmp_path / dest).read_text() == content - - -def test_missing_staged_module_fails_loud(): - """A missing staged file must abort the build, not skip the install.""" - with pytest.raises(SystemExit): - plc._module_source("definitely-not-a-real-module.py", "test label") - - -def test_patch7_gate_is_additive_not_substitutive(tmp_path): +def test_capability_gate_is_additive_not_substitutive(tmp_path): """Patch 7 must UNION the live answer with the stock model-map answer. OpenRouter's ``supported_parameters`` under-reports ``reasoning_effort`` @@ -356,7 +287,7 @@ def test_patch7_gate_is_additive_not_substitutive(tmp_path): outright would drop a knob the map correctly allows — trading one silent drop for another. The stock ``supports_reasoning`` branch must therefore survive the patch.""" - patch7 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 7/")) + patch7 = _patch_by_description("openrouter live capabilities") # The stock gate, verbatim from 1.86.2, following the needle. Applying the # patch to this and asserting on the RESULT tests the invariant the @@ -442,7 +373,7 @@ def test_patch7_gate_is_additive_not_substitutive(tmp_path): ) -def test_patch9_gates_synthesis_without_touching_the_claude_branch(tmp_path): +def test_thinking_gate_leaves_the_claude_branch_alone(tmp_path): """Patch 9 must stop the adapter manufacturing a ``reasoning_effort``. On ``/v1/messages`` litellm derives ``reasoning_effort`` from the caller's @@ -452,7 +383,7 @@ def test_patch9_gates_synthesis_without_touching_the_claude_branch(tmp_path): agent turn. The Claude branch, which forwards ``thinking`` unchanged, must be unaffected — and so must the assignments, which the gate returns before rather than rewriting.""" - patch9 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 9/")) + patch9 = _patch_by_description("thinking->reasoning_effort synthesis gate") target = tmp_path / patch9["file"] target.parent.mkdir(parents=True, exist_ok=True) @@ -482,7 +413,7 @@ def test_patch9_gates_synthesis_without_touching_the_claude_branch(tmp_path): assert "_egg_synthesize = False" in gate -def test_patch9_does_not_suppress_an_explicitly_requested_effort(tmp_path): +def test_thinking_gate_does_not_suppress_an_explicit_effort(tmp_path): """The gate's scope is the *derived* bucket, not the whole function. Stock reaches the assignment two ways: from ``budget_tokens`` (a ceiling @@ -490,7 +421,7 @@ def test_patch9_does_not_suppress_an_explicitly_requested_effort(tmp_path): which is the caller saying outright what they want. Suppressing the second would be discarding an instruction, not declining to invent one — a different change from the one the patch documents.""" - patch9 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 9/")) + patch9 = _patch_by_description("thinking->reasoning_effort synthesis gate") target = tmp_path / patch9["file"] target.parent.mkdir(parents=True, exist_ok=True) @@ -516,7 +447,7 @@ def test_patch9_does_not_suppress_an_explicitly_requested_effort(tmp_path): assert "if not _egg_effort_is_explicit:\n" in result -def test_patch8_needle_disambiguates_the_two_drop_sites(tmp_path): +def test_drop_params_needle_disambiguates_the_two_drop_sites(tmp_path): """litellm 1.86.2 has TWO ``drop_params`` branches in utils.py. They share the identical ``if litellm.drop_params is True or (...)`` @@ -525,7 +456,7 @@ def test_patch8_needle_disambiguates_the_two_drop_sites(tmp_path): condition would patch whichever came first — the same needle-uniqueness trap as Patch 4. A fixture carrying the sibling ``pass`` form first must be left untouched.""" - patch8 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 8/")) + patch8 = _patch_by_description("drop_params visibility") sibling = ( "SENTINEL_PASS_SITE_BEGIN\n" @@ -556,7 +487,7 @@ def test_patch8_needle_disambiguates_the_two_drop_sites(tmp_path): assert result[start:end] == sibling, "patch 8 rewrote the embeddings-path drop site" -def test_patch10_sets_cost_after_the_rebuild_not_before(tmp_path): +def test_cost_details_carry_runs_after_the_rebuild_not_before(tmp_path): """Patch 10 must land on the far side of ``Usage(**model_dump())``. That constructor deletes a ``cost`` attribute it is handed as None, so a @@ -564,7 +495,7 @@ def test_patch10_sets_cost_after_the_rebuild_not_before(tmp_path): rebuild is entitled to discard — the patch would apply cleanly, the build would pass, and ``cost`` would still read null on every streamed call, which is the bug it exists to fix.""" - patch10 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 10/")) + patch10 = _patch_by_description("streamed cost_details preservation") target = tmp_path / patch10["file"] target.parent.mkdir(parents=True, exist_ok=True) @@ -588,7 +519,7 @@ def test_patch10_sets_cost_after_the_rebuild_not_before(tmp_path): assert "except Exception:" in result -def test_patch11_hooks_the_unmapped_branch_and_leaves_the_stock_raise(tmp_path): +def test_pricing_hooks_the_unmapped_branch_and_leaves_the_stock_raise(tmp_path): """Patch 11 must be a fallback, not a replacement. It sits at the "isn't mapped yet" raise, so it runs only once every stock @@ -596,7 +527,7 @@ def test_patch11_hooks_the_unmapped_branch_and_leaves_the_stock_raise(tmp_path): bundled rate, and the live card can add a model but never reprice one. The stock ValueError must still be reachable, for the slug OpenRouter has not heard of either.""" - patch11 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 11/")) + patch11 = _patch_by_description("openrouter live pricing") target = tmp_path / patch11["file"] target.parent.mkdir(parents=True, exist_ok=True) @@ -621,14 +552,14 @@ def test_patch11_hooks_the_unmapped_branch_and_leaves_the_stock_raise(tmp_path): assert "except Exception:\n _egg_entry = None\n" in result -def test_patch11_needle_disambiguates_the_two_unmapped_messages(tmp_path): +def test_pricing_needle_disambiguates_the_two_unmapped_messages(tmp_path): """utils.py carries the "isn't mapped yet" string twice. The other one is the outer handler's re-raise, with a different message body and indentation. Matching loosely would insert a pricing fallback into an exception handler, where ``_model_info`` and ``key`` are not even in scope — same needle-uniqueness trap as Patches 4 and 8.""" - patch11 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 11/")) + patch11 = _patch_by_description("openrouter live pricing") sibling = ( "SENTINEL_OUTER_HANDLER_BEGIN\n" From b2fb977d33b6d1ae9dba0e302f7159018717c766 Mon Sep 17 00:00:00 2001 From: "egg-reviewer[bot]" <261018737+egg-reviewer[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 07:09:43 +0000 Subject: [PATCH 2/2] Address review: patch 9 for 1.94.0 cost short-circuit, restore module tests, refresh stock snapshots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add patch 9 (billable-cost gate on hidden params): 1.94.0's _propagate_usage_cost_to_hidden_params short-circuits response_cost_calculator on any non-None usage.cost, including a BYOK 0.0 — which suppressed cost_estimated and made the live pricing patch dead code on the streaming path. Narrowed to a positive, finite charge; a real bill propagates unchanged. - Restore the three NEW_MODULES tests dropped in the rewrite, plus a functional test for the new gate. - Re-transcribe the hand-written stock snapshots from the real 1.94.0 wheel (both had drifted) and pin them with a Dockerfile FROM assertion. - Fix stale patch cross-references, the patch 7 warn message naming the wrong field, test prose/locals still keyed on old numbers, and cost_callback prose. --- config/litellm/Dockerfile | 13 +- config/litellm/anthropic_thinking_policy.py | 2 +- config/litellm/cost_callback.py | 81 ++-- config/litellm/openrouter_capabilities.py | 2 +- config/litellm/patch_litellm_cache.py | 177 +++++++-- docs/development/STRUCTURE.md | 2 +- docs/guides/per-agent-models.md | 22 +- tests/config/test_patch_litellm_cache.py | 393 ++++++++++++++------ 8 files changed, 514 insertions(+), 178 deletions(-) diff --git a/config/litellm/Dockerfile b/config/litellm/Dockerfile index bb8ac98d5..11c40dc29 100644 --- a/config/litellm/Dockerfile +++ b/config/litellm/Dockerfile @@ -5,9 +5,10 @@ # stock image's Anthropic->OpenAI translation drops prompt-cache hits on # Qwen/DeepSeek, mis-streams reasoning models, silently drops params it does # not recognise, manufactures a reasoning ceiling from the caller's thinking -# budget, and loses the BYOK half of the provider's bill during stream -# reassembly while its rate card cannot price the route either. -# patch_litellm_cache.py closes the eight gaps at build time +# budget, loses the BYOK half of the provider's bill during stream +# reassembly while its rate card cannot price the route either, and seeds its +# own cost calculator with a BYOK zero that suppresses the estimate. +# patch_litellm_cache.py closes the nine gaps at build time # (see that script for the diagnosis); # cost_callback.py surfaces the resulting cost/cache stats into the pod log # stream. Both mirror the host-side dotfiles setup that took Qwen cache hit @@ -15,7 +16,11 @@ # # Pinned to v1.94.0 — the newest stable published release, and the version # the patches and cost_callback are validated against (#3697). The bump from -# v1.86.2 retired four patches upstream absorbed and narrowed a fifth. +# v1.86.2 retired four patches upstream absorbed, narrowed a fifth, and added +# a ninth for a regression the bump itself brings in (1.94.0 hands its cost +# calculator the provider's `cost` even when that is a BYOK zero, which +# suppresses the independent estimate and makes the rate-card patch dead code +# on the streaming path). # # The patch script fails the build loudly if a needle stops matching, so a # future bump can't silently ship an unpatched (full-input-rate) image. Read diff --git a/config/litellm/anthropic_thinking_policy.py b/config/litellm/anthropic_thinking_policy.py index 2d7840fcd..ce0ff029c 100644 --- a/config/litellm/anthropic_thinking_policy.py +++ b/config/litellm/anthropic_thinking_policy.py @@ -31,7 +31,7 @@ running at full depth. Patch 4 makes the OpenRouter param gate accurate, which is right for an *operator-configured* ``reasoning_effort`` and wrong for this adapter-manufactured one: it would turn a knob nobody set into the effective -setting, with no config file mentioning it and nothing in the logs (Patch 8 +setting, with no config file mentioning it and nothing in the logs (Patch 5 only fires on drops, and this param would no longer be dropped). So Patch 6 gates the synthesis, defaulting it off. ``thinking`` stays out of diff --git a/config/litellm/cost_callback.py b/config/litellm/cost_callback.py index a4c7eac1c..6ec542268 100644 --- a/config/litellm/cost_callback.py +++ b/config/litellm/cost_callback.py @@ -29,33 +29,44 @@ originally DROPPED the provider's ``cost`` / ``cost_details`` outright. That is why this module recorded ``cost: null`` on 1252 of 1252 sampled calls in run 6. litellm 1.94.0 carries ``cost`` across that rebuild, and the -egg-litellm image's **patch 7** carries the ``cost_details`` it still drops -(``config/litellm/stream_cost_preservation.py``) — that is where the BYOK -number lives. Between them the billed figure reaches ``_extract_cost`` on -the streaming path too -and this module needs no change to read it — the value simply stops being -absent (#3691). +egg-litellm image's **cost_details preservation patch** carries the +``cost_details`` it still drops (``config/litellm/stream_cost_preservation.py``) +— that is where the BYOK number lives. Between them the billed figure reaches +``_extract_cost`` on the streaming path too, and this module needs no change to +read it — the value simply stops being absent (#3691). ``cost: null`` therefore no longer means "streaming"; it means the cost was genuinely unavailable — a LiteLLM older than 1.94.0 under this callback, a -BYOK route on an unpatched image, or a provider that does not report one. It is still emitted as null rather -than ``0.0``: a zero would read in the logs as "this route is free", the -exact opposite of the cost-visibility signal this module exists to provide -(#2799). +BYOK route on an unpatched image, or a provider that does not report one. It +is still emitted as null rather than ``0.0``: a zero would read in the logs as +"this route is free", the exact opposite of the cost-visibility signal this +module exists to provide (#2799). Each line also carries ``cost_estimated``: LiteLLM's own ``response_cost``, computed at logging time from the assembled usage and its pricing map (issue #3175). It is kept strictly separate from ``cost`` — an estimate from a possibly-stale rate card must never be mistaken for a bill — and follows the same null-not-zero rule when LiteLLM cannot price the model. -That was the case for every route egg uses until the image's **patch 8** -taught the model-info lookup to read OpenRouter's published rate card; it -remains the case for a model whose prompt-length surcharge lands on a +That was the case for every route egg uses until the image's **live pricing +patch** taught the model-info lookup to read OpenRouter's published rate card; +it remains the case for a model whose prompt-length surcharge lands on a boundary or component LiteLLM's map has no slot for, which is declined whole -rather than translated in part (see ``openrouter_capabilities``). With -both patches in place the two fields are independent measurements of the -same turn, and a persistent gap between them is a signal in its own right: -a stale rate card, an unexpected provider, or a surcharge tier. +rather than translated in part (see ``openrouter_capabilities``). + +The two fields are independent measurements ONLY WHERE THE PROVIDER REPORTS NO +CHARGE, which on egg's BYOK routes is every call. From 1.94.0 litellm copies a +reported ``usage.cost`` into ``_hidden_params`` and ``response_cost_calculator`` +returns that value directly, so where a provider does bill a positive figure +``cost_estimated`` mirrors ``cost`` rather than pricing the turn independently. +The image's **billable-cost gate patch** narrows that propagation to a positive, +finite charge — without it a BYOK ``cost: 0.0`` would be handed to the +calculator, ``cost_estimated`` would read null on ~100% of egg's traffic, and +the live pricing patch would be dead code on the streaming path (it lives inside +``completion_cost()``, which the short circuit returns before). So on a BYOK +route the two fields remain independent and a persistent gap between them is a +signal in its own right — a stale rate card, an unexpected provider, or a +surcharge tier — while on a route that reports a real charge, agreement between +them is arithmetic, not corroboration. Each line also carries ``request_params``: the decoding configuration that actually went upstream on that call (issue #3599). Repetition and @@ -168,9 +179,9 @@ def _usage_from_response_obj(response_obj): ``response_obj.usage`` by ``stream_chunk_builder``) and, on the egg-litellm image, for cost as well: that reassembly rebuilds a fresh ``Usage`` and older releases dropped ``cost`` / ``cost_details`` with it; 1.94.0 carries - ``cost`` and patch 7 supplies ``cost_details``. - Under a stock LiteLLM the counts still arrive and ``_extract_cost`` returns - None — see the module docstring.""" + ``cost`` and the image's cost_details preservation patch supplies + ``cost_details``. Under a stock LiteLLM the counts still arrive and + ``_extract_cost`` returns None — see the module docstring.""" if response_obj is None: return None usage = getattr(response_obj, "usage", None) @@ -325,11 +336,22 @@ def _extract_estimated_cost(mcd): reported a bill. It is an estimate, not a bill: the pricing map may lag the provider's rates or lack cache-discount entries for a model. Returns None — never 0.0 — when LiteLLM couldn't price the call, mirroring the billed-cost - "unknown ≠ free" discipline. On the egg-litellm image patch 8 supplies - OpenRouter's published rates for slugs the bundled map does not carry, so a - None here now means a genuinely unpriceable model (an inexpressible - prompt-length surcharge, or a provider with no live card to read) rather - than the routine case it was. + "unknown ≠ free" discipline. On the egg-litellm image the live pricing patch + supplies OpenRouter's published rates for slugs the bundled map does not + carry, so a None here now means a genuinely unpriceable model (an + inexpressible prompt-length surcharge, or a provider with no live card to + read) rather than the routine case it was. + + "Independently of whether the provider reported a bill" holds only where the + provider reported no charge — which on egg's BYOK routes is every call. From + 1.94.0, litellm copies a reported ``usage.cost`` into ``_hidden_params`` and + ``response_cost_calculator`` returns it before consulting the pricing map at + all, so on a route that does bill a positive figure this returns ``cost`` + itself and the two fields stop being independent. The image's billable-cost + gate patch narrows that propagation to a positive, finite charge — it is + also what keeps the live pricing patch reachable on the streaming path, + since that patch lives inside ``completion_cost()`` and the short circuit + returns before it. Reads the top-level ``response_cost`` first, then falls back to ``standard_logging_object.response_cost`` — the latter is LiteLLM's @@ -379,10 +401,11 @@ def _extract_model(mcd): # # ``stream`` is included because it selects which of the two paths the cost on # this line came through (see the module docstring), and it was the reason -# ``cost`` read null on every line before 1.94.0 and patch 7 — worth having next to the -# number rather than inferred, and worth keeping now that the null case is rare -# enough to need explaining when it happens. ``max_tokens`` and ``n`` are not -# sampling knobs but shape the generation, and are cheap to carry. +# ``cost`` read null on every line before 1.94.0 and the cost_details +# preservation patch — worth having next to the number rather than inferred, and +# worth keeping now that the null case is rare enough to need explaining when it +# happens. ``max_tokens`` and ``n`` are not sampling knobs but shape the +# generation, and are cheap to carry. _REQUEST_PARAM_KEYS = ( "temperature", "top_p", diff --git a/config/litellm/openrouter_capabilities.py b/config/litellm/openrouter_capabilities.py index 9163f5067..2370661b3 100644 --- a/config/litellm/openrouter_capabilities.py +++ b/config/litellm/openrouter_capabilities.py @@ -584,7 +584,7 @@ def _log_fetch_failure(message: str, *args: object) -> None: A permanently unreachable endpoint is the exact case where behaviour silently reverts to the model-cost map, and litellm's default log level is INFO — so debug-only reporting reproduces, one file over, the silence - Patch 8 exists to remove. Warning once is enough to be findable without + Patch 5 exists to remove. Warning once is enough to be findable without turning an offline deployment into a log flood. """ global _WARNED_FETCH_FAILURE diff --git a/config/litellm/patch_litellm_cache.py b/config/litellm/patch_litellm_cache.py index 7423b7e3f..0f110257b 100644 --- a/config/litellm/patch_litellm_cache.py +++ b/config/litellm/patch_litellm_cache.py @@ -6,24 +6,26 @@ backend) drops prompt-cache hits for Qwen/DeepSeek, reads a model-cost map that carries neither the parameters nor the prices of current OpenRouter slugs, discards caller-specified params in total silence, manufactures a -reasoning ceiling nobody asked for, and loses the BYOK half of the -provider's bill during stream reassembly. Eight independent gaps cause it; -this script closes all eight by editing the installed ``litellm`` package -in place (and installing four new modules), then -``config/litellm/Dockerfile`` bakes the result into the ``egg-litellm`` +reasoning ceiling nobody asked for, loses the BYOK half of the provider's +bill during stream reassembly, and — new at 1.94.0 — hands its own cost +calculator a BYOK zero that suppresses the estimate entirely. Nine +independent gaps cause it; this script closes all nine by editing the +installed ``litellm`` package in place (and installing four new modules), +then ``config/litellm/Dockerfile`` bakes the result into the ``egg-litellm`` image. Pinned to ``litellm==1.94.0`` (see the Dockerfile ``FROM``), so the needles below are known to match. The bump from 1.86.2 retired four patches whose fixes upstream absorbed — the streaming ``reasoning_content`` thinking block, the first-delta requeue on block transitions (both sync and async), -and the ``prompt_tokens_details.cached_tokens`` fallback — and narrowed a -fifth (7) to the half upstream still does not carry. See issue #3697 for -the audit, and read its warning before assuming the next bump retires -anything: TWO of the patches below (2 and 5) had their needles stop -matching at 1.94.0 purely because upstream reflowed the surrounding code, -and retiring them on the strength of that miss would have silently shipped -an image paying full input rate on every Qwen turn. +and the ``prompt_tokens_details.cached_tokens`` fallback — narrowed a +fifth (7) to the half upstream still does not carry, and ADDED one (9) for +a regression the bump itself brings in. See issue #3697 for the audit, and +read its warning before assuming the next bump retires anything: TWO of the +patches below (2 and 5) had their needles stop matching at 1.94.0 purely +because upstream reflowed the surrounding code, and retiring them on the +strength of that miss would have silently shipped an image paying full +input rate on every Qwen turn. 1. ``CacheControlSupportedModels`` (openrouter/chat/transformation.py) add QWEN + DEEPSEEK so ``cache_control`` survives the OpenRouter @@ -152,6 +154,25 @@ registering a base tier that would silently under-report the long prompts agent traffic is made of. Proposed upstream as jwbron/litellm#10. + 9. ``CustomStreamWrapper._propagate_usage_cost_to_hidden_params`` + (litellm_core_utils/streaming_handler.py) require the provider's + ``cost`` to be POSITIVE and finite before it is allowed to stand in + for the cost calculator. New at 1.94.0 and a regression the bump + itself brings in: the method did not exist in 1.86.2, and it copies + ``usage.cost`` into ``_hidden_params`` on both streaming completion + paths, where ``response_cost_calculator`` returns it verbatim and + ``completion_cost()`` never runs. Its guard is ``is not None``, and + under BYOK OpenRouter's ``cost`` is a literal ``0.0`` — so on egg's + route (Claude Code streams every request) the calculator short-circuits + on a zero, ``cost_estimated`` reads null on every streamed call, and + patch 8 is dead code, because its hook sits inside ``completion_cost``. + Requiring a positive figure restores both for the BYOK case and leaves + the real-charge case alone. NOTE what that leaves standing: where a + provider DOES report a positive charge, 1.94.0 seeds its own calculator + with it, so ``cost_estimated`` mirrors ``cost`` rather than being an + independent rate-card read — see ``cost_callback._extract_estimated_cost``. + Non-finite is excluded too: ``+inf`` clears a bare ``> 0`` and would be + written into a spend header as ``Infinity``. Idempotent: each patch detects whether it is already applied. Fails loudly (non-zero exit) if a needle is missing, so a LiteLLM version bump @@ -261,6 +282,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - F3 = "llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py" F4 = "utils.py" F5 = "litellm_core_utils/streaming_chunk_builder_utils.py" +F6 = "litellm_core_utils/streaming_handler.py" # Every patch as a self-contained spec: (file, present marker, needle, # replacement, label). Module-level so tests can apply them to a checked-in @@ -283,7 +305,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - ' QWEN = "qwen"\n' ' DEEPSEEK = "deepseek"\n' ), - "label": "Patch 1/8 (CacheControlSupportedModels)", + "label": "Patch 1/9 (CacheControlSupportedModels)", }, # Patch 2 — broaden ONLY the cache_control gate (not the shared # is_anthropic_claude_model predicate, which also gates thinking @@ -327,7 +349,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " )\n" " ):\n" ), - "label": "Patch 2/8 (cache_control gate)", + "label": "Patch 2/9 (cache_control gate)", }, # Patch 3 — drop x-anthropic-billing-header during Anthropic->OpenAI translation. { @@ -361,7 +383,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - ' "text": text,\n' " }\n" ), - "label": "Patch 3/8 (x-anthropic-billing-header filter)", + "label": "Patch 3/9 (x-anthropic-billing-header filter)", }, # Patch 4 — OpenrouterConfig.get_supported_openai_params: consult # OpenRouter's published capabilities instead of only the bundled @@ -418,7 +440,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " # supported_parameters over an unauthenticated endpoint; the bundled\n" " # model-cost map does not carry current slugs, so the stock gate\n" " # below fails closed and the knob is dropped in silence. Unioned,\n" - " # never subtractive — see patch 7 notes in patch_litellm_cache.py.\n" + " # never subtractive — see patch 4 notes in patch_litellm_cache.py.\n" " try:\n" " from litellm.llms.openrouter._egg_capabilities import (\n" " get_supported_parameters as _egg_openrouter_capabilities,\n" @@ -432,15 +454,15 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " pass\n" " try:\n" ), - "label": "Patch 4/8 (openrouter live capabilities)", + "label": "Patch 4/9 (openrouter live capabilities)", }, - # Patch 8 — get_optional_params: log what ``drop_params`` discards. + # Patch 5 — get_optional_params: log what ``drop_params`` discards. # - # Patch 7 removes the OpenRouter false-negative, but a drop can still be + # Patch 4 removes the OpenRouter false-negative, but a drop can still be # correct and still worth knowing about: laguna-s-2.1 genuinely does not # accept ``reasoning_effort``, so the knob is dropped on purpose and the # operator has no way to learn why their config line does nothing. Stock - # Stock pops unsupported params in a bare loop with no logging at all. + # pops unsupported params in a bare loop with no logging at all. # # NOT retired by the 1.94.0 bump. Upstream reflowed the condition onto one # line, so the old needle stopped matching — but nothing upstream logs the @@ -482,13 +504,13 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " for k in unsupported_params.keys():\n" " non_default_params.pop(k, None)\n" ), - "label": "Patch 5/8 (drop_params visibility)", + "label": "Patch 5/9 (drop_params visibility)", }, # Patch 6 — _translate_thinking_to_openai: stop synthesizing # ``reasoning_effort`` from the caller's ``thinking`` block for non-Claude # models. # - # This is the other half of patch 7, and without it patch 7 is a + # This is the other half of patch 4, and without it patch 4 is a # regression on egg's primary route. On /v1/messages (Claude Code -> # gateway -> litellm -> OpenRouter) the request body carries # ``thinking: {"type": "enabled", "budget_tokens": N}``. @@ -501,16 +523,16 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - # # Until now that param was silently dropped (the model-cost map does not # carry these slugs), which is exactly why these models have been running - # at full reasoning depth. Patch 7 unblocks the param — correct for an + # at full reasoning depth. Patch 4 unblocks the param — correct for an # operator-configured value, wrong for this one, because the measurements # in jwbron/egg#3624 show the bucket is a CAP BELOW the model default: # kimi-k3 means 3130 reasoning tokens with no param vs 340 with # ``reasoning_effort: high``, distributions non-overlapping. Shipping - # patch 7 alone would cut reasoning ~9x on every agent turn with nothing - # logged (patch 8 only fires on drops, and this is no longer dropped) and + # patch 4 alone would cut reasoning ~9x on every agent turn with nothing + # logged (patch 5 only fires on drops, and this is no longer dropped) and # no config file to point at. # - # Gating the synthesis keeps patch 7's actual goal — a configured knob + # Gating the synthesis keeps patch 4's actual goal — a configured knob # reaches the wire — without letting the adapter's bucket become the # effective setting. ``LITELLM_ANTHROPIC_THINKING_TO_REASONING_EFFORT=1`` # restores stock behaviour. The Claude branch above is untouched. @@ -575,7 +597,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - "\n" ' summary = thinking.get("summary") if isinstance(thinking, dict) else None\n' ), - "label": "Patch 6/8 (thinking->reasoning_effort synthesis gate)", + "label": "Patch 6/9 (thinking->reasoning_effort synthesis gate)", }, # Patch 7 — ChunkProcessor.calculate_usage: carry the provider's # ``cost_details`` across stream reassembly. @@ -643,15 +665,20 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " # Warned once per process, and the latch is set only after\n" " # the emit succeeded; verbose_logger is imported here rather\n" " # than read from module scope because this file does not\n" - " # import it.\n" + " # import it. The message names `cost_details`, not `cost`:\n" + " # since 1.94.0 upstream carries `cost` itself, so what a\n" + " # failure here costs is the BYOK figure alone, and sending\n" + " # the operator to a field that reads correctly would be\n" + " # worse than not warning at all.\n" " try:\n" " if not globals().get('_egg_warned_stream_cost'):\n" " from litellm._logging import verbose_logger\n" "\n" " verbose_logger.warning(\n" - " 'egg cost patch: streamed cost preservation is '\n" - " 'inactive (%s: %s); the provider-billed `cost` will '\n" - " 'read null on every streamed call.',\n" + " 'egg cost patch: streamed cost_details preservation '\n" + " 'is inactive (%s: %s); `cost_details` will read null '\n" + " 'on every streamed call, so BYOK routes (where the '\n" + " 'provider-billed `cost` is 0) will report no spend.',\n" " type(_egg_exc).__name__,\n" " _egg_exc,\n" " )\n" @@ -661,12 +688,12 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - "\n" " return returned_usage\n" ), - "label": "Patch 7/8 (streamed cost_details preservation)", + "label": "Patch 7/9 (streamed cost_details preservation)", }, # Patch 8 — _get_model_info_helper: price OpenRouter slugs the bundled map # has never heard of. # - # Same root cause as patch 7, second symptom. LiteLLM's own + # Same root cause as patch 4, second symptom. LiteLLM's own # ``response_cost`` is computed from ``model_prices_and_context_window.json``, # which does not carry current OpenRouter slugs, so the lookup raises "This # model isn't mapped yet" and egg's ``cost_estimated`` reads null on every @@ -674,6 +701,14 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - # restores the independent estimate beside it, which is what remains # readable if a provider ever stops reporting cost. # + # PATCH 9 IS LOAD-BEARING FOR THIS ONE on the streaming path. The hook below + # sits inside ``_get_model_info_helper``, reachable only via + # ``completion_cost()`` — and 1.94.0's ``_propagate_usage_cost_to_hidden_params`` + # makes ``response_cost_calculator`` return before it ever calls that, + # whenever the provider reported any ``cost`` at all including a BYOK zero. + # Without patch 9 this patch applies cleanly, passes every direct test of + # ``_get_model_info_helper``, and is dead code on ~100% of egg's traffic. + # # Placed at the raise site, so it is reached only once every stock lookup # has failed: a slug the bundled map DOES carry keeps the bundled answer, # and the live rate card can add a model but never reprice one. The @@ -742,7 +777,81 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - 'https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"\n' " )\n" ), - "label": "Patch 8/8 (openrouter live pricing)", + "label": "Patch 8/9 (openrouter live pricing)", + }, + # Patch 9 — _propagate_usage_cost_to_hidden_params: require a POSITIVE, + # finite provider cost before it stands in for the cost calculator. + # + # This one is not a gap in 1.86.2 that survived the bump; it is a + # regression the bump BRINGS IN. The method does not exist in 1.86.2 + # (``grep -rn _propagate_usage_cost_to_hidden_params`` on that wheel returns + # nothing). 1.94.0 calls it on ``complete_streaming_response`` from both + # streaming completion paths (``__next__`` and ``__anext__``), immediately + # before ``logging_obj.success_handler`` sees the copy, and it copies + # ``usage.cost`` into ``_hidden_params["additional_headers"]``. Downstream, + # ``response_cost_calculator`` returns that header verbatim and + # ``completion_cost()`` never runs. + # + # Its guard is ``_usage.cost is not None``, and + # ``get_response_cost_from_hidden_params`` only declines on a literal None — + # ``0.0`` is returned as ``0.0``. Under BYOK OpenRouter's top-level ``cost`` + # IS ``0.0`` (billing routes past them; the real number lives in + # ``cost_details``, which is patch 7's job). Claude Code streams every + # /v1/messages request, so on egg's route the stock behaviour is: calculator + # short-circuits on a zero -> ``response_cost`` is 0.0 -> egg's + # ``cost_estimated`` reads null on EVERY streamed call (it is gated on + # ``_positive``), and patch 8 never fires at all, because its hook sits in + # ``_get_model_info_helper``'s unmapped branch, reachable only through + # ``completion_cost()``. Both of those are things this image exists to + # provide, and both would fail silently: a null cost_estimated is + # indistinguishable from an unpriceable model. + # + # Requiring a positive figure restores ``completion_cost()`` (and with it + # patch 8) for the BYOK case and leaves the real-charge case alone — the + # narrow fix, not a blanket suppression: where a provider reports a genuine + # charge, letting litellm bill from it rather than from a rate card is + # upstream's improvement and worth keeping. The cost of that choice is + # recorded in ``cost_callback._extract_estimated_cost``: on such a route + # ``cost_estimated`` mirrors ``cost`` instead of being an independent read. + # + # Finiteness is checked too, for the reason ``cost_callback._positive`` + # gives: ``+inf`` clears a bare ``> 0``, and this value is written into a + # spend header where it would serialize as the non-standard ``Infinity``. + # ``bool`` is excluded for the same reason it is there — ``isinstance(True, + # int)`` is True, and a True is not a dollar. + # + # The needle spans the assignment and the guard together: the guard line + # alone is what changes, but the assignment above it is what makes the + # match unambiguous, and ``_egg_cost`` has to be computed before it. + { + "file": F6, + "present": "# egg cost-propagation patch", + "needle": ( + ' _usage = getattr(response, "usage", None)\n' + ' if _usage is not None and hasattr(_usage, "cost") ' + "and _usage.cost is not None:\n" + ), + "replacement": ( + ' _usage = getattr(response, "usage", None)\n' + " # egg cost-propagation patch. Upstream's guard is `is not None`,\n" + " # and this value short-circuits litellm's cost calculator. Under\n" + " # BYOK OpenRouter reports `cost: 0.0` (the real figure is in\n" + " # cost_details), so the stock guard hands the calculator a zero,\n" + " # `response_cost` reads 0.0, egg's `cost_estimated` reads null,\n" + " # and the OpenRouter rate-card patch never runs — it lives inside\n" + " # completion_cost(), which this return skips. Require a positive,\n" + " # finite number; a real charge still propagates unchanged. See\n" + " # patch 9 notes in patch_litellm_cache.py.\n" + ' _egg_cost = getattr(_usage, "cost", None) if _usage is not None else None\n' + " _egg_cost_is_billable = (\n" + " isinstance(_egg_cost, (int, float))\n" + " and not isinstance(_egg_cost, bool)\n" + " and _egg_cost > 0\n" + ' and _egg_cost < float("inf")\n' + " )\n" + " if _egg_cost_is_billable:\n" + ), + "label": "Patch 9/9 (billable-cost gate on hidden params)", }, ] diff --git a/docs/development/STRUCTURE.md b/docs/development/STRUCTURE.md index 0db1b644c..21e868687 100644 --- a/docs/development/STRUCTURE.md +++ b/docs/development/STRUCTURE.md @@ -543,7 +543,7 @@ config/ ├── litellm/ # egg-litellm image sources │ ├── Dockerfile # Builds egg-litellm: stock LiteLLM + prompt-cache and reasoning-parameter patches │ ├── .ruff.toml # Pins this directory to target-version = py311 — everything here runs on the litellm base image's interpreter, not the repo's 3.14, and `ruff format` under py314 emits PEP 758 syntax the image cannot import -│ ├── patch_litellm_cache.py # Build-time patches: cache_control passthrough on Qwen/DeepSeek routes, live OpenRouter capability + pricing lookup, drop_params visibility, no synthesized reasoning ceiling, streamed cost preservation +│ ├── patch_litellm_cache.py # Build-time patches: cache_control passthrough on Qwen/DeepSeek routes, live OpenRouter capability + pricing lookup, drop_params visibility, no synthesized reasoning ceiling, streamed cost preservation, billable-cost gate on hidden params (patch 9 — inline, no companion module) │ ├── openrouter_capabilities.py # Patches 4 + 8: live GET /api/v1/models lookup — advertised parameters (unioned with LiteLLM's bundled model-cost map) and the published rate card for slugs that map has never heard of, including prompt-length surcharges that fit LiteLLM's 128k/200k/272k rate slots │ ├── drop_params_visibility.py # Patch 5: warn once per proxy process per (provider, model, param-set) when drop_params discards a parameter │ ├── anthropic_thinking_policy.py # Patch 6: stop synthesizing a reasoning_effort ceiling from the caller's thinking budget on non-Claude models diff --git a/docs/guides/per-agent-models.md b/docs/guides/per-agent-models.md index 369384fd4..b4bfbf309 100644 --- a/docs/guides/per-agent-models.md +++ b/docs/guides/per-agent-models.md @@ -593,7 +593,7 @@ data: > Claude Code prepends (the block's `cch=` hash invalidates the cache key > every turn). egg ships a custom **`egg-litellm`** image > ([`config/litellm/Dockerfile`](../../config/litellm/Dockerfile)) that -> bakes in eight patches closing those gaps, the reasoning-parameter ones +> bakes in nine patches closing those gaps, the reasoning-parameter ones > below, and the cost-visibility ones after that > ([`config/litellm/patch_litellm_cache.py`](../../config/litellm/patch_litellm_cache.py)); > the build fails loudly if a LiteLLM bump moves the patched code. Pinning @@ -637,6 +637,16 @@ data: > every bundled lookup has failed — so a mapped slug keeps its bundled > rate, and the live card can add a model but never reprice one. > `LITELLM_OPENROUTER_PRICING=0` turns off just this half. +> - **Patch 9 — the estimate stays independent under BYOK.** From 1.94.0 +> LiteLLM copies a provider-reported `usage.cost` into `_hidden_params` and +> `response_cost_calculator` returns that value *before* it reaches +> `completion_cost()`. Its guard is `is not None`, and under BYOK OpenRouter +> reports `cost: 0.0` — so unpatched, the calculator is handed a zero, +> `cost_estimated` reads null on ~100% of egg's traffic, and patch 8 (which +> lives inside `completion_cost()`) never runs at all. The patch narrows the +> guard to a positive, finite charge. A route that reports a real bill still +> propagates it unchanged, in which case `cost_estimated` mirrors `cost` +> rather than pricing the turn independently. > > `cost_estimated` can still read null for a model that **prices by prompt > length**. OpenRouter publishes a long-context surcharge as @@ -662,9 +672,13 @@ data: > by one of those reads low under `cost_estimated` — another reason to compare > the two fields rather than trusting either alone. > -> The two fields are independent measurements of the same turn, so read them -> together: a persistent gap between them is itself a signal (a stale rate -> card, an unexpected provider, or a surcharge tier). Neither is ever +> On a BYOK route — every route egg currently uses — the two fields are +> independent measurements of the same turn, so read them together: a +> persistent gap between them is itself a signal (a stale rate card, an +> unexpected provider, or a surcharge tier). On a route where the provider +> reports a positive charge, patch 9 lets that charge through to LiteLLM's +> calculator and `cost_estimated` mirrors `cost`; agreement there is +> arithmetic, not corroboration. Neither is ever > coerced to `0.0` when unknown — a zero would read in the logs as "this > route is free", the exact inversion of the signal. The session totals sum > only the calls that reported, so compare `cost_known_calls` / diff --git a/tests/config/test_patch_litellm_cache.py b/tests/config/test_patch_litellm_cache.py index 217319817..08b708ed5 100644 --- a/tests/config/test_patch_litellm_cache.py +++ b/tests/config/test_patch_litellm_cache.py @@ -2,17 +2,30 @@ ``config/litellm/patch_litellm_cache.py`` edits the installed ``litellm`` package in place at image-build time (see PR #3190 / #3199). The real -``litellm==1.86.2`` is not a project dependency — and cannot run on the +``litellm==1.94.0`` is not a project dependency — and cannot run on the repo's Python — so we exercise the patch *mechanics* against fixtures that embed the exact needles the script targets, then assert the resulting source. This is the patch-script regression the CI image build doesn't give us: the build catches version drift via the fail-loud needle check, but -nothing else exercises idempotency, the replacement payload, or the Patch 4 -needle-uniqueness anchor. +nothing else exercises idempotency, the replacement payload, or the +needle-uniqueness anchors on the two patches whose spelling appears twice in +``utils.py``. -The fixtures are derived from ``PATCHES`` itself (the module-level spec +Most fixtures are derived from ``PATCHES`` itself (the module-level spec list), so there is no fixture/needle drift: a needle change automatically flows into the fixture the test patches. + +THE EXCEPTIONS ARE THE HAND-WRITTEN STOCK SNAPSHOTS — ``_STOCK_CAPABILITY_GATE`` +and ``_STOCK_THINKING_TAIL_*`` below. They are transcribed from the pinned +litellm source so a test can assert what a patch leaves ALONE, which a +needle-derived fixture cannot express. Nothing checks them against the real +tree at test time (litellm is not installed here), so they are the one place +in this file that can silently become a snapshot of a version the image no +longer runs — exactly the false-pass shape ``_patch_by_description`` exists to +remove from the lookup key. ``_PINNED_LITELLM_VERSION`` is asserted against the +Dockerfile's ``FROM`` so a bump cannot land without this file being edited; +when you edit it, RE-TRANSCRIBE the snapshots from the new wheel rather than +bumping the constant alone. """ import ast @@ -24,6 +37,12 @@ CONFIG_DIR = Path(__file__).resolve().parent.parent.parent / "config" / "litellm" +# The litellm the hand-written stock snapshots in this file were transcribed +# from. Asserted against the Dockerfile's ``FROM`` below, so a version bump +# cannot land without someone editing this line — and the docstring above says +# what has to happen when they do. +_PINNED_LITELLM_VERSION = "1.94.0" + def _load_patch_module(): """Load ``patch_litellm_cache`` from disk (it isn't an importable package).""" @@ -69,6 +88,30 @@ def _build_fixture_root(root: Path) -> None: fpath.write_text("# fixture head\n\n" + "\n\n".join(needles) + "\n\n# fixture tail\n") +def test_stock_snapshots_are_pinned_to_the_image_litellm(): + """The hand-written snapshots below are only as good as their version. + + They cannot be checked against the real tree here — litellm is not a + project dependency and does not run on the repo's Python — so the closest + thing to a drift detector is making a version bump impossible to land + without editing this file. At the 1.86.2 -> 1.94.0 bump BOTH snapshots + changed (a reflowed gate, a new ``is_bedrock_arn_model`` arm) while every + test kept passing, because a snapshot of an old version is still a valid + fixture for the patch mechanics — it just stops describing the image.""" + dockerfile = (CONFIG_DIR / "Dockerfile").read_text() + pins = [ + line.split(":v", 1)[1].strip() + for line in dockerfile.splitlines() + if line.startswith("FROM ghcr.io/berriai/litellm:v") + ] + assert pins == [_PINNED_LITELLM_VERSION], ( + f"Dockerfile pins litellm {pins} but the stock snapshots in this file were " + f"transcribed from {_PINNED_LITELLM_VERSION}. Re-transcribe " + "_STOCK_CAPABILITY_GATE and _STOCK_THINKING_TAIL_* from the new wheel, " + "then update _PINNED_LITELLM_VERSION." + ) + + def test_each_needle_occurs_once_in_its_fixture(tmp_path): """Sanity: the needles are distinct, so the fixture contains each exactly once. If this fails the other assertions can't be trusted.""" @@ -231,6 +274,43 @@ def test_installed_module_parse_error_points_at_the_real_line(tmp_path, monkeypa assert "staged module source does not parse" in message +def test_new_modules_are_installed_into_each_root(tmp_path): + """``NEW_MODULES`` drops whole files that have no stock counterpart. + + The capability patch's gate imports + ``litellm.llms.openrouter._egg_capabilities``, so if the module install + silently no-ops the patched gate raises ImportError on every request — + caught by its ``except Exception``, which would put us right back at the + silent-drop behaviour the patch exists to remove.""" + _build_fixture_root(tmp_path) + plc._patch_root(str(tmp_path)) + + for spec in plc.NEW_MODULES: + dest = tmp_path / spec["dest"] + assert dest.is_file(), f"{spec['label']}: not installed" + source = Path(plc._module_source(spec["source"], spec["label"])) + installed = dest.read_text() + assert installed.startswith(plc.EGG_MODULE_HEADER), f"{spec['label']}: no provenance" + assert installed == plc.EGG_MODULE_HEADER + source.read_text(), ( + f"{spec['label']}: content drift" + ) + + +def test_new_module_install_is_idempotent(tmp_path): + _build_fixture_root(tmp_path) + plc._patch_root(str(tmp_path)) + first = {spec["dest"]: (tmp_path / spec["dest"]).read_text() for spec in plc.NEW_MODULES} + plc._patch_root(str(tmp_path)) + for dest, content in first.items(): + assert (tmp_path / dest).read_text() == content + + +def test_missing_staged_module_fails_loud(): + """A missing staged file must abort the build, not skip the install.""" + with pytest.raises(SystemExit): + plc._module_source("definitely-not-a-real-module.py", "test label") + + def test_new_module_refuses_to_clobber_a_foreign_file(tmp_path): """Every other operation in this script is fail-loud on drift; so is this. @@ -280,40 +360,47 @@ def test_new_module_destinations_carry_the_egg_prefix(tmp_path): assert "must be prefixed" in str(excinfo.value) +# The stock ``get_supported_openai_params`` gate, transcribed verbatim from +# litellm 1.94.0 (``llms/openrouter/chat/transformation.py``), picking up where +# the capability patch's needle ends. A PINNED SNAPSHOT — see the module +# docstring: 1.86.2 spelled this across three lines and 1.94.0 reflowed it, and +# nothing here would have noticed. Re-transcribe on every bump. +_STOCK_CAPABILITY_GATE = ( + " if litellm.supports_reasoning(" + 'model=model, custom_llm_provider="openrouter") or litellm.supports_reasoning(\n' + " model=model\n" + " ):\n" + ' supported_params.append("reasoning_effort")\n' + ' supported_params.append("thinking")\n' + " except Exception:\n" + " pass\n" + " return list(dict.fromkeys(supported_params))\n" +) + + def test_capability_gate_is_additive_not_substitutive(tmp_path): - """Patch 7 must UNION the live answer with the stock model-map answer. + """The capability patch must UNION live data with the stock model map. OpenRouter's ``supported_parameters`` under-reports ``reasoning_effort`` (deepseek-r1 advertises only ``reasoning``), so letting live data win outright would drop a knob the map correctly allows — trading one silent drop for another. The stock ``supports_reasoning`` branch must therefore survive the patch.""" - patch7 = _patch_by_description("openrouter live capabilities") - - # The stock gate, verbatim from 1.86.2, following the needle. Applying the - # patch to this and asserting on the RESULT tests the invariant the - # docstring claims; substring checks against the replacement string alone - # would pass on a patch that deleted the branch entirely. - stock_gate = ( - " if litellm.supports_reasoning(\n" - ' model=model, custom_llm_provider="openrouter"\n' - " ) or litellm.supports_reasoning(model=model):\n" - ' supported_params.append("reasoning_effort")\n' - ' supported_params.append("thinking")\n' - " except Exception:\n" - " pass\n" - " return list(dict.fromkeys(supported_params))\n" - ) - target = tmp_path / patch7["file"] + capability_patch = _patch_by_description("openrouter live capabilities") + + # Applying the patch to the stock snapshot and asserting on the RESULT + # tests the invariant the docstring claims; substring checks against the + # replacement string alone would pass on a patch that deleted the branch. + target = tmp_path / capability_patch["file"] target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(patch7["needle"] + stock_gate) + target.write_text(capability_patch["needle"] + _STOCK_CAPABILITY_GATE) plc._apply( str(target), - present=patch7["present"], - needle=patch7["needle"], - replacement=patch7["replacement"], - label=patch7["label"], + present=capability_patch["present"], + needle=capability_patch["needle"], + replacement=capability_patch["replacement"], + label=capability_patch["label"], ) result = target.read_text() @@ -322,11 +409,15 @@ def test_capability_gate_is_additive_not_substitutive(tmp_path): assert '"reasoning_effort" in _advertised' in result # ...and the stock model-map branch is still there, untouched, downstream # of it: the live answer can ADD a knob, never withhold one. - assert stock_gate in result, "patch 7 must not replace the stock model-map branch" + assert _STOCK_CAPABILITY_GATE in result, ( + "the capability patch must not replace the stock model-map branch" + ) # The inserted block precedes it and does not return out of the function. - inserted = result[: result.index(stock_gate)] + inserted = result[: result.index(_STOCK_CAPABILITY_GATE)] assert "supported_params.append" in inserted - assert "\n return" not in inserted, "patch 7 must not short-circuit the stock branch" + assert "\n return" not in inserted, ( + "the capability patch must not short-circuit the stock branch" + ) # Failures in the lookup must never propagate into a request. assert "except Exception:" in inserted # Only reasoning_effort is admitted. OpenRouter's `reasoning` field is a @@ -334,19 +425,22 @@ def test_capability_gate_is_additive_not_substitutive(tmp_path): assert '"thinking"' not in inserted -# The tail of ``_translate_thinking_to_openai`` as it stands in 1.86.2, -# verbatim, from the Claude branch through the assignments. Patch 9's needle is -# a slice of this; keeping the surrounding lines lets the test assert what the -# gate sits between, which is the whole invariant. +# The tail of ``_translate_thinking_to_openai`` as it stands in 1.94.0, +# verbatim, from the Claude branch through the assignments. The synthesis +# gate's needle is a slice of this; keeping the surrounding lines lets the test +# assert what the gate sits between, which is the whole invariant. A PINNED +# SNAPSHOT — see the module docstring. Both halves moved at the 1.86.2 -> 1.94.0 +# bump (the Claude branch gained a ``is_bedrock_arn_model`` arm and the +# derivation call collapsed onto one line), which is exactly the drift nothing +# in this file can detect for you. Re-transcribe on every bump. _STOCK_THINKING_TAIL_HEAD = ( ' model = new_kwargs.get("model", "")\n' - " if self.is_anthropic_claude_model(model):\n" + " if self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model):\n" ' new_kwargs["thinking"] = thinking # type: ignore\n' " return\n" "\n" - " reasoning_effort = self.translate_anthropic_thinking_to_reasoning_effort(\n" - " cast(Dict[str, Any], thinking)\n" - " )\n" + " reasoning_effort = self.translate_anthropic_thinking_to_reasoning_effort(" + "cast(Dict[str, Any], thinking))\n" " if not reasoning_effort:\n" " return\n" "\n" @@ -375,38 +469,38 @@ def test_capability_gate_is_additive_not_substitutive(tmp_path): def test_thinking_gate_leaves_the_claude_branch_alone(tmp_path): - """Patch 9 must stop the adapter manufacturing a ``reasoning_effort``. + """The synthesis gate must stop the adapter manufacturing an effort. On ``/v1/messages`` litellm derives ``reasoning_effort`` from the caller's ``thinking`` budget for every non-Claude model. That derived value is a cap BELOW the model default (#3624: kimi-k3 means 3130 reasoning tokens with no - param vs 340 with ``high``), so Patch 7 alone would silently shallow every - agent turn. The Claude branch, which forwards ``thinking`` unchanged, must - be unaffected — and so must the assignments, which the gate returns before - rather than rewriting.""" - patch9 = _patch_by_description("thinking->reasoning_effort synthesis gate") + param vs 340 with ``high``), so the capability patch alone would silently + shallow every agent turn. The Claude branch, which forwards ``thinking`` + unchanged, must be unaffected — and so must the assignments, which the gate + returns before rather than rewriting.""" + gate_patch = _patch_by_description("thinking->reasoning_effort synthesis gate") - target = tmp_path / patch9["file"] + target = tmp_path / gate_patch["file"] target.parent.mkdir(parents=True, exist_ok=True) target.write_text( - _STOCK_THINKING_TAIL_HEAD + patch9["needle"] + _STOCK_THINKING_TAIL_FOOT, + _STOCK_THINKING_TAIL_HEAD + gate_patch["needle"] + _STOCK_THINKING_TAIL_FOOT, ) plc._apply( str(target), - present=patch9["present"], - needle=patch9["needle"], - replacement=patch9["replacement"], - label=patch9["label"], + present=gate_patch["present"], + needle=gate_patch["needle"], + replacement=gate_patch["replacement"], + label=gate_patch["label"], ) result = target.read_text() - assert _STOCK_THINKING_TAIL_HEAD in result, "patch 9 must leave the Claude path alone" - assert _STOCK_THINKING_TAIL_FOOT in result, "patch 9 must not rewrite the assignments" + assert _STOCK_THINKING_TAIL_HEAD in result, "the gate must leave the Claude path alone" + assert _STOCK_THINKING_TAIL_FOOT in result, "the gate must not rewrite the assignments" # The gate sits after the derivation and returns (rather than falling # through) when synthesis is off, so nothing is assigned. - gate = result[result.index(patch9["present"]) : result.index(_STOCK_THINKING_TAIL_FOOT)] + gate = result[result.index(gate_patch["present"]) : result.index(_STOCK_THINKING_TAIL_FOOT)] assert "_egg_anthropic_thinking_policy" in gate assert "if not _egg_synthesize:\n return\n" in gate # A missing policy module must fall back to the policy's OWN default (off), @@ -422,26 +516,26 @@ def test_thinking_gate_does_not_suppress_an_explicit_effort(tmp_path): which is the caller saying outright what they want. Suppressing the second would be discarding an instruction, not declining to invent one — a different change from the one the patch documents.""" - patch9 = _patch_by_description("thinking->reasoning_effort synthesis gate") + gate_patch = _patch_by_description("thinking->reasoning_effort synthesis gate") - target = tmp_path / patch9["file"] + target = tmp_path / gate_patch["file"] target.parent.mkdir(parents=True, exist_ok=True) target.write_text( - _STOCK_THINKING_TAIL_HEAD + patch9["needle"] + _STOCK_THINKING_TAIL_FOOT, + _STOCK_THINKING_TAIL_HEAD + gate_patch["needle"] + _STOCK_THINKING_TAIL_FOOT, ) plc._apply( str(target), - present=patch9["present"], - needle=patch9["needle"], - replacement=patch9["replacement"], - label=patch9["label"], + present=gate_patch["present"], + needle=gate_patch["needle"], + replacement=gate_patch["replacement"], + label=gate_patch["label"], ) result = target.read_text() # The adaptive override still runs, and it is what exempts the request. override = ' reasoning_effort = output_config["effort"]\n' assert override in result, "the adaptive override must survive the patch" - assert result.index(override) < result.index(patch9["present"]), ( + assert result.index(override) < result.index(gate_patch["present"]), ( "the gate must sit after the override, not before it" ) assert f"{override} _egg_effort_is_explicit = True\n" in result @@ -449,15 +543,15 @@ def test_thinking_gate_does_not_suppress_an_explicit_effort(tmp_path): def test_drop_params_needle_disambiguates_the_two_drop_sites(tmp_path): - """litellm 1.86.2 has TWO ``drop_params`` branches in utils.py. + """litellm 1.94.0 has TWO ``drop_params`` branches in utils.py. They share the identical ``if litellm.drop_params is True or (...)`` condition; only what follows differs (a bare ``pass`` in the embeddings path, the pop loop in ``get_optional_params``). Matching on the shared condition would patch whichever came first — the same needle-uniqueness - trap as Patch 4. A fixture carrying the sibling ``pass`` form first must be - left untouched.""" - patch8 = _patch_by_description("drop_params visibility") + trap as the capability gate. A fixture carrying the sibling ``pass`` form + first must be left untouched.""" + drop_patch = _patch_by_description("drop_params visibility") sibling = ( "SENTINEL_PASS_SITE_BEGIN\n" @@ -467,46 +561,49 @@ def test_drop_params_needle_disambiguates_the_two_drop_sites(tmp_path): " pass\n" "SENTINEL_PASS_SITE_END\n" ) - fixture = sibling + "\n" + patch8["needle"] + fixture = sibling + "\n" + drop_patch["needle"] - target = tmp_path / patch8["file"] + target = tmp_path / drop_patch["file"] target.parent.mkdir(parents=True, exist_ok=True) target.write_text(fixture) plc._apply( str(target), - present=patch8["present"], - needle=patch8["needle"], - replacement=patch8["replacement"], - label=patch8["label"], + present=drop_patch["present"], + needle=drop_patch["needle"], + replacement=drop_patch["replacement"], + label=drop_patch["label"], ) result = target.read_text() - assert result.count(patch8["present"]) == 1 + assert result.count(drop_patch["present"]) == 1 start = result.index("SENTINEL_PASS_SITE_BEGIN") end = result.index("SENTINEL_PASS_SITE_END") + len("SENTINEL_PASS_SITE_END\n") - assert result[start:end] == sibling, "patch 8 rewrote the embeddings-path drop site" + assert result[start:end] == sibling, ( + "the drop_params patch rewrote the embeddings-path drop site" + ) def test_cost_details_carry_runs_after_the_rebuild_not_before(tmp_path): - """Patch 10 must land on the far side of ``Usage(**model_dump())``. + """The carry must land on the far side of ``Usage(**model_dump())``. - That constructor deletes a ``cost`` attribute it is handed as None, so a - carry inserted *before* the rebuild would be writing into an object the - rebuild is entitled to discard — the patch would apply cleanly, the build - would pass, and ``cost`` would still read null on every streamed call, - which is the bug it exists to fix.""" - patch10 = _patch_by_description("streamed cost_details preservation") + That constructor drops the fields it is not declared to carry, so a carry + inserted *before* the rebuild would be writing into an object the rebuild is + entitled to discard — the patch would apply cleanly, the build would pass, + and ``cost_details`` would still read null on every streamed call. On a BYOK + route that is the whole bill: the provider-billed figure lives in + ``cost_details.upstream_inference_cost`` and top-level ``cost`` is 0.""" + carry_patch = _patch_by_description("streamed cost_details preservation") - target = tmp_path / patch10["file"] + target = tmp_path / carry_patch["file"] target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(patch10["needle"]) + target.write_text(carry_patch["needle"]) plc._apply( str(target), - present=patch10["present"], - needle=patch10["needle"], - replacement=patch10["replacement"], - label=patch10["label"], + present=carry_patch["present"], + needle=carry_patch["needle"], + replacement=carry_patch["replacement"], + label=carry_patch["label"], ) result = target.read_text() @@ -529,25 +626,113 @@ def test_cost_details_carry_runs_after_the_rebuild_not_before(tmp_path): ), "the latch must be set after the emit, not before it" +# The body ``_propagate_usage_cost_to_hidden_params`` runs once its guard is +# true, transcribed from 1.94.0 and picking up where the billable-cost patch's +# needle ends. A PINNED SNAPSHOT — see the module docstring. It is here rather +# than beside the other two because this test EXECUTES the patched result: an +# assertion that the gate rejects a BYOK zero is only meaningful if the thing +# behind the gate is the thing that actually sets the header. +_STOCK_HIDDEN_PARAMS_BODY = ( + ' if "additional_headers" not in response._hidden_params:\n' + ' response._hidden_params["additional_headers"] = {}\n' + ' response._hidden_params["additional_headers"]' + '["llm_provider-x-litellm-response-cost"] = float(_usage.cost)\n' +) + +_HIDDEN_PARAMS_SCAFFOLD = "class _Wrapper:\n @staticmethod\n def propagate(response):\n" + + +def _run_billable_cost_gate(tmp_path, cost): + """Apply the billable-cost patch, then run the result against a fake usage. + + Returns the ``additional_headers`` dict the patched body produced, so the + caller can assert on behaviour rather than on substrings.""" + gate_patch = _patch_by_description("billable-cost gate on hidden params") + + target = tmp_path / gate_patch["file"] + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text( + _HIDDEN_PARAMS_SCAFFOLD + gate_patch["needle"] + _STOCK_HIDDEN_PARAMS_BODY, + ) + plc._apply( + str(target), + present=gate_patch["present"], + needle=gate_patch["needle"], + replacement=gate_patch["replacement"], + label=gate_patch["label"], + ) + + namespace: dict = {} + exec(compile(target.read_text(), str(target), "exec"), namespace) # noqa: S102 + + class _Usage: + pass + + class _Response: + pass + + usage = _Usage() + if cost is not _MISSING: + usage.cost = cost + response = _Response() + response.usage = usage + response._hidden_params = {} + + namespace["_Wrapper"].propagate(response) + return response._hidden_params.get("additional_headers", {}) + + +_MISSING = object() +_COST_HEADER = "llm_provider-x-litellm-response-cost" + + +@pytest.mark.parametrize( + "cost", + [0.0, 0, -1.0, None, float("inf"), float("nan"), True, _MISSING], +) +def test_billable_cost_gate_declines_a_non_charge(tmp_path, cost): + """The gate exists because 1.94.0 short-circuits its own cost calculator. + + ``response_cost_calculator`` returns this header's value before it ever + reaches ``completion_cost()``, and stock propagates on ``is not None``. + Under BYOK OpenRouter reports ``cost: 0.0`` — the real figure lives in + ``cost_details`` — so stock hands the calculator a zero, ``response_cost`` + reads 0.0, egg's ``cost_estimated`` reads null, and the live rate-card patch + (which lives inside ``completion_cost()``) becomes dead code on ~100% of + egg's traffic. ``True`` is in the list because it is an ``int`` that is + ``> 0``: a bool reaching a currency field is a bug upstream, not a bill.""" + assert _COST_HEADER not in _run_billable_cost_gate(tmp_path, cost) + + +def test_billable_cost_gate_propagates_a_real_charge(tmp_path): + """The narrow fix, not blanket suppression. + + Where a provider does report a positive charge, upstream's propagation is + an improvement over a token-based estimate and must survive untouched — + ``cost_estimated`` then mirrors ``cost`` by design.""" + headers = _run_billable_cost_gate(tmp_path, 0.0031) + assert headers[_COST_HEADER] == 0.0031 + + def test_pricing_hooks_the_unmapped_branch_and_leaves_the_stock_raise(tmp_path): - """Patch 11 must be a fallback, not a replacement. + """The live rate card must be a fallback, not a replacement. It sits at the "isn't mapped yet" raise, so it runs only once every stock lookup has already failed: a slug the bundled map DOES carry keeps the bundled rate, and the live card can add a model but never reprice one. The stock ValueError must still be reachable, for the slug OpenRouter has not heard of either.""" - patch11 = _patch_by_description("openrouter live pricing") + pricing_patch = _patch_by_description("openrouter live pricing") - target = tmp_path / patch11["file"] + target = tmp_path / pricing_patch["file"] target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(patch11["needle"]) + target.write_text(pricing_patch["needle"]) plc._apply( str(target), - present=patch11["present"], - needle=patch11["needle"], - replacement=patch11["replacement"], - label=patch11["label"], + present=pricing_patch["present"], + needle=pricing_patch["needle"], + replacement=pricing_patch["replacement"], + label=pricing_patch["label"], ) result = target.read_text() @@ -562,7 +747,7 @@ def test_pricing_hooks_the_unmapped_branch_and_leaves_the_stock_raise(tmp_path): assert "except Exception as _egg_exc:\n _egg_entry = None\n" in result # ...but not silently. A failed import is otherwise indistinguishable from # "OpenRouter has no rate for this slug", which is the null-cost_estimated - # symptom patch 11 exists to remove. Warned once, and the latch is set only + # symptom the pricing patch exists to remove. Warned once, and the latch is set only # after the emit — inside its own try, because raising from an except block # would propagate into a live request. assert "_egg_warned_pricing" in result @@ -622,7 +807,7 @@ def test_pricing_needle_disambiguates_the_two_unmapped_messages(tmp_path): body and indentation. Matching loosely would insert a pricing fallback into an exception handler, where ``_model_info`` and ``key`` are not even in scope — same needle-uniqueness trap as Patches 4 and 8.""" - patch11 = _patch_by_description("openrouter live pricing") + pricing_patch = _patch_by_description("openrouter live pricing") sibling = ( "SENTINEL_OUTER_HANDLER_BEGIN\n" @@ -637,20 +822,20 @@ def test_pricing_needle_disambiguates_the_two_unmapped_messages(tmp_path): " )\n" "SENTINEL_OUTER_HANDLER_END\n" ) - target = tmp_path / patch11["file"] + target = tmp_path / pricing_patch["file"] target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(patch11["needle"] + "\n" + sibling) + target.write_text(pricing_patch["needle"] + "\n" + sibling) plc._apply( str(target), - present=patch11["present"], - needle=patch11["needle"], - replacement=patch11["replacement"], - label=patch11["label"], + present=pricing_patch["present"], + needle=pricing_patch["needle"], + replacement=pricing_patch["replacement"], + label=pricing_patch["label"], ) result = target.read_text() - assert result.count(patch11["present"]) == 1 + assert result.count(pricing_patch["present"]) == 1 start = result.index("SENTINEL_OUTER_HANDLER_BEGIN") end = result.index("SENTINEL_OUTER_HANDLER_END") + len("SENTINEL_OUTER_HANDLER_END\n") - assert result[start:end] == sibling, "patch 11 rewrote the outer handler's re-raise" + assert result[start:end] == sibling, "the pricing patch rewrote the outer handler's re-raise"