diff --git a/config/litellm-models.template.yaml b/config/litellm-models.template.yaml index 752d9dc05..95b67799d 100644 --- a/config/litellm-models.template.yaml +++ b/config/litellm-models.template.yaml @@ -98,8 +98,14 @@ data: provider: order: [Alibaba] allow_fallbacks: false - # Closest knob to Opus xhigh — uncomment to opt into the - # deepest reasoning OpenRouter exposes for this model: + # Reasoning depth. MEASURE BEFORE UNCOMMENTING — on the models + # measured in #3624 this knob is a CAP BELOW the model's own + # default, not a ceiling above it. Mean reasoning tokens at + # max_tokens 16000, n=4: kimi-k3 3130 with no parameter vs 86 via + # `extra_body.reasoning.effort: high` (distributions do not + # overlap); glm-5.2 1689 vs 1516. Sending nothing is what gives + # these models full depth. Set it only for a model you have + # measured to reason MORE when explicitly asked: # reasoning: # effort: "high" # Paired `[1m]` alias — absorbs Claude Code startup-probe @@ -124,6 +130,8 @@ data: provider: order: [Alibaba] allow_fallbacks: false + # See the depth-vs-default measurement on the bare row above + # before uncommenting this. # reasoning: # effort: "high" diff --git a/config/litellm/Dockerfile b/config/litellm/Dockerfile index 24ee17042..734b92d4b 100644 --- a/config/litellm/Dockerfile +++ b/config/litellm/Dockerfile @@ -3,8 +3,10 @@ # egg routes non-Claude agents (the cq-6 OpenRouter/Qwen pilot, #2799) # through this proxy: Claude Code -> gateway -> LiteLLM -> OpenRouter. The # stock image's Anthropic->OpenAI translation drops prompt-cache hits on -# Qwen/DeepSeek and mis-streams reasoning models. patch_litellm_cache.py -# closes the five gaps at build time (see that script for the diagnosis); +# Qwen/DeepSeek, mis-streams reasoning models, silently drops params it does +# not recognise, and manufactures a reasoning ceiling from the caller's +# thinking budget. patch_litellm_cache.py closes the nine gaps at build time +# (see that script for the diagnosis); # cost_callback.py surfaces the resulting cost/cache stats into the pod log # stream. Both mirror the host-side dotfiles setup that took Qwen cache hit # rate from 0% to ~99.99%. @@ -21,6 +23,14 @@ FROM ghcr.io/berriai/litellm:v1.86.2 USER root COPY patch_litellm_cache.py /egg/patch_litellm_cache.py +# Staged, not baked directly into the patch script: these are whole modules the +# script installs into every litellm tree (see NEW_MODULES). Keeping them real +# files means they stay lintable and testable in the egg repo rather than +# living as string literals — which is why none of them imports litellm at +# module scope (tests/config/ imports all three directly). +COPY openrouter_capabilities.py /egg/openrouter_capabilities.py +COPY drop_params_visibility.py /egg/drop_params_visibility.py +COPY anthropic_thinking_policy.py /egg/anthropic_thinking_policy.py RUN python3 /egg/patch_litellm_cache.py # Custom cost/cache logger, registered as `cost_callback.cost_logger` under diff --git a/config/litellm/anthropic_thinking_policy.py b/config/litellm/anthropic_thinking_policy.py new file mode 100644 index 000000000..de8d73e4a --- /dev/null +++ b/config/litellm/anthropic_thinking_policy.py @@ -0,0 +1,125 @@ +"""Whether LiteLLM may synthesize ``reasoning_effort`` from ``thinking``. + +egg's primary route is ``/v1/messages``: Claude Code -> egg-gateway -> LiteLLM +-> OpenRouter, carrying an Anthropic-shaped body with +``thinking: {"type": "enabled", "budget_tokens": N}``. LiteLLM's Anthropic +adapter (``_translate_thinking_to_openai``) turns that into an OpenAI body. For +a Claude model it forwards ``thinking`` unchanged. For anything else — +``is_anthropic_claude_model`` is a substring test for ``anthropic``/``claude``, +so every OpenRouter slug egg routes falls here — it *replaces* the block with a +bucketed ``reasoning_effort``: ``>=10000 -> "high"``, ``>=5000 -> "medium"``, +``>=2000 -> "low"``. Nothing in ``litellm-models.yaml`` is involved; the value +is manufactured per request from the caller's thinking budget. + +That synthesized value is not a floor, it is a **cap below the model default**. +Measured directly against OpenRouter (``max_tokens: 16000``, n=4, mean +reasoning tokens): + +=================================== ============ =================== +Model no parameter ``effort: "high"`` +=================================== ============ =================== +``moonshotai/kimi-k3`` 3130 340 +``z-ai/glm-5.2`` 1689 1090 +=================================== ============ =================== + +On kimi-k3 the distributions do not overlap. So sending the adapter's bucket +costs roughly 9x the reasoning depth the model would have produced on its own. + +Historically this never mattered: the model-cost map did not carry these slugs, +``OpenrouterConfig`` advertised no reasoning knobs, and ``drop_params`` silently +discarded the synthesized param — which is precisely why these models have been +running at full depth. Patch 7 makes the OpenRouter param gate accurate, which +is right for an *operator-configured* ``reasoning_effort`` and wrong for this +adapter-manufactured one: it would turn a knob nobody set into the effective +setting, with no config file mentioning it and nothing in the logs (Patch 8 +only fires on drops, and this param would no longer be dropped). + +So Patch 9 gates the synthesis, defaulting it off. ``thinking`` stays out of +the OpenAI body for non-Claude models and only an explicitly configured +``reasoning_effort`` reaches the wire — exactly the property Patch 7 exists to +restore, without the adapter's bucket riding along. + +The gate covers the *derived* value only. On an adaptive request +(``thinking: {"type": "adaptive"}`` plus ``output_config: {"effort": ...}``) +the caller states an effort outright; that is an instruction rather than a +manufactured ceiling, and it still reaches the provider with this policy off. +A ``thinking.summary`` request is suppressed along with the derived effort, +because stock carries the summary only as a field of the ``reasoning_effort`` +dict — honouring it would mean sending the ceiling. There is no wire shape for +"summary, no effort". + +Set ``LITELLM_ANTHROPIC_THINKING_TO_REASONING_EFFORT=1`` to restore stock +behaviour (e.g. for a provider whose models do not reason unless asked, or +after measuring the ``/v1/messages`` path for a specific model). A value that is +neither a recognised on nor off spelling warns once and leaves the policy at its +default, rather than being read as off — off is also the default, so an operator +who typed ``=enabled`` would otherwise have no way to tell "ignored" apart from +"working as configured", on the highest-impact knob in this changeset. +""" + +import os + +ENV_VAR = "LITELLM_ANTHROPIC_THINKING_TO_REASONING_EFFORT" + +_TRUTHY = ("1", "true", "yes", "on") +_FALSY = ("0", "false", "no", "off", "") + +# Values already complained about. Bounded by construction: the environment does +# not change mid-process, so this holds at most one entry. Needed because this +# is read once per translated request, and an unconditional warning would be one +# WARNING line per request forever. +_WARNED_VALUES: set[str] = set() + + +def _log(level: str, message: str, *args: object) -> bool: + """Log via litellm's ``verbose_logger``, deferring the import. + + Kept out of module scope so this file stays importable — and therefore unit + testable — where litellm is not installed. Never raises: a diagnostic must + not be able to break a request. + + Returns whether the call completed without raising, not whether a record + reached a handler; see the same note in ``openrouter_capabilities._log``. + """ + try: + from litellm._logging import verbose_logger + + getattr(verbose_logger, level)(message, *args) + return True + except Exception: # noqa: BLE001 - diagnostics must never break a request + return False + + +def should_synthesize_reasoning_effort() -> bool: + """True when the adapter may derive ``reasoning_effort`` from ``thinking``. + + Defaults to False: on every model egg routes, the derived value measurably + reduces reasoning depth relative to sending nothing at all. + """ + raw = os.getenv(ENV_VAR) + if raw is None: + return False + value = raw.strip().lower() + if value in _TRUTHY: + return True + if value not in _FALSY and value not in _WARNED_VALUES: + # Recorded only once the emit did not raise, for the reason given in + # ``openrouter_capabilities._warn_env_once``: ``_log`` swallows its own + # failure, so recording first would let a logger that is not yet in + # place on the first request suppress the warning permanently. + if _log( + "warning", + "%s=%r is neither an on (%s) nor an off (%s) spelling; leaving " + "thinking -> reasoning_effort synthesis disabled, which is also the " + "default — set %s=1 if you meant to enable it.", + ENV_VAR, + raw, + ", ".join(_TRUTHY), + ", ".join(v for v in _FALSY if v), + ENV_VAR, + ): + _WARNED_VALUES.add(value) + return False + + +__all__ = ["ENV_VAR", "should_synthesize_reasoning_effort"] diff --git a/config/litellm/cost_callback.py b/config/litellm/cost_callback.py index c4b5b6487..a2b3ccddb 100644 --- a/config/litellm/cost_callback.py +++ b/config/litellm/cost_callback.py @@ -814,9 +814,18 @@ def _record(self, mcd, response_obj): # (#3599). Top-level, not nested under ``call``, so an # incident query can filter on it the same way it filters # on model/role. Per line rather than once per session - # because it is NOT session-stable: LiteLLM rewrites - # ``thinking`` into a ``reasoning_effort`` bucket, so the - # effective effort tracks the per-turn thinking budget. + # because it is NOT session-stable: these are per-request + # values, and a config change or an overlay edit takes + # effect mid-session. + # + # Note ``reasoning_effort`` is normally ABSENT on egg's + # /v1/messages route (#3624): stock LiteLLM rewrites the + # caller's ``thinking`` block into a bucketed + # ``reasoning_effort``, but that bucket is a cap below the + # model default, so egg-litellm's patch 9 gates the + # synthesis off by default. A missing key here means the + # request ran at the model's own reasoning depth, not that + # the field failed to record. "request_params": request_params, "call": { "cost": cost, diff --git a/config/litellm/drop_params_visibility.py b/config/litellm/drop_params_visibility.py new file mode 100644 index 000000000..43e755179 --- /dev/null +++ b/config/litellm/drop_params_visibility.py @@ -0,0 +1,124 @@ +"""Make ``drop_params`` say what it dropped. + +``drop_params`` exists so an unsupported parameter does not fail the whole +request, and that tradeoff is right. But dropping a parameter *changes +generation behaviour*, and in stock LiteLLM 1.86.2 it happens with no signal +at all: the branch that pops them is a bare loop with no logging. A +``reasoning_effort``, ``temperature`` or penalty set in a proxy config simply +never reaches the provider, and nothing in the logs or the response says so. +The config and the wire disagree, silently and indefinitely. + +That is not hypothetical here. Every OpenRouter slug egg routes is absent from +LiteLLM's bundled model-cost map, so ``OpenrouterConfig`` advertised no +reasoning knobs and every ``reasoning_effort: high`` in the operator overlay +was discarded before the request body was built. It took a full investigation +to notice (jwbron/egg#3620, #3624). One log line would have made it a +five-minute question. + +Patch 7 fixes the OpenRouter false-negative specifically; this covers the rest. +A drop can still be *correct* and worth knowing about: ``poolside/laguna-s-2.1`` +genuinely does not accept ``reasoning_effort``, so the knob is dropped on +purpose, and without this the operator has no way to learn why their config +line does nothing. + +Mirrors jwbron/litellm#7 (merged into the fork the host proxy runs). The +cluster image pins stock 1.86.2, which predates it, hence this patch. +""" + +# Warn-once bookkeeping, keyed by (provider, model, sorted dropped param +# names) so a route that drops the same params on every request warns once +# rather than once per call. Bounded so a long-lived proxy serving many models +# cannot grow it without limit; on overflow the set is CLEARED rather than +# frozen, because a frozen full set stops deduplicating and every subsequent +# request warns again forever. Clearing costs one extra warning per key per +# cycle and keeps both memory and log volume bounded. +_MAX_WARNINGS = 1000 +_SEEN: set[tuple[str, str, tuple[str, ...]]] = set() + + +def _log_warning(message: str, *args: object) -> bool: + """Log via litellm's ``verbose_logger``, deferring the import. + + Kept out of module scope so this file stays importable where litellm is + not installed — which is what makes it unit testable in the egg repo, the + stated reason (``config/litellm/Dockerfile``) for keeping it a real file + rather than a string literal in the patch script. + + Returns whether the call completed without raising — not whether a line + reached a handler, since a logger filtering the level away also returns + normally. That is enough for the warn-once bookkeeping below, whose failure + mode is an import or emit that *raised*: this module exists precisely so a + drop is not silent, and recording the dedup key ahead of a failed emit + would make that route silent for the life of the process. + """ + try: + from litellm._logging import verbose_logger + + verbose_logger.warning(message, *args) + except Exception: # noqa: BLE001 - diagnostics must never break a request + return False + return True + + +def warn_dropped_params( + unsupported_params: dict, + model: str | None, + custom_llm_provider: str | None, +) -> None: + """Log once per (provider, model, param-set) when params are discarded. + + Warns rather than debugs because the caller asked for something and did not + get it; at debug level it would be invisible in exactly the situation it + exists for. Never raises: a diagnostic must not be able to break a request. + """ + try: + if not unsupported_params: + return + dropped = tuple(sorted(unsupported_params.keys())) + key = (custom_llm_provider or "", model or "", dropped) + if key in _SEEN: + return + # States what is known, and prescribes only under a condition the + # operator can check. The param most likely to be dropped on this + # deployment is ``reasoning_effort``, and it is frequently NOT in any + # config file: litellm's own Anthropic adapter synthesises it from the + # caller's `thinking` block on the /v1/messages route. An unconditional + # "edit config.yaml" would send that operator looking for a line that + # does not exist, and an unconditional "force it through + # `allowed_openai_params`" would turn a correct drop on a genuinely + # non-reasoning model into a provider-side error. So the remedy is + # offered gated on "if they came from this model's litellm_params", + # with the synthesized case named alongside it — the operator who has + # such a line gets the fix, and the one who does not is told why the + # drop is expected instead of being sent editing. + emitted = _log_warning( + "litellm.drop_params: dropped %s for model=%s provider=%s — the " + "provider does not advertise support for them, so they did not " + "reach it and whatever behaviour they were meant to control is " + "unchanged. If they came from this model's litellm_params in " + "config.yaml, remove them or override with `allowed_openai_params: " + "%s`. If they were synthesized from the request (e.g. " + "reasoning_effort derived from an Anthropic `thinking` block), the " + "drop is expected and the model ran at its own default.", + list(dropped), + model, + custom_llm_provider, + list(dropped), + ) + # Recorded only once the emit did not raise. ``_log_warning`` swallows + # its own failure so a diagnostic cannot fail a request, and recording + # first would mean one failure on the *first* call — litellm's logger + # not yet in place, say — suppresses this route's warning forever, + # because every later call would find the key already there. The cost of + # the other ordering is one warning attempt per request until an emit + # succeeds, which is the right way round for a module whose whole job is + # to make a silent drop audible. + if emitted: + if len(_SEEN) >= _MAX_WARNINGS: + _SEEN.clear() + _SEEN.add(key) + except Exception: # noqa: BLE001 - diagnostics must never break a request + pass + + +__all__ = ["warn_dropped_params"] diff --git a/config/litellm/openrouter_capabilities.py b/config/litellm/openrouter_capabilities.py new file mode 100644 index 000000000..cd910c2aa --- /dev/null +++ b/config/litellm/openrouter_capabilities.py @@ -0,0 +1,399 @@ +"""Live capability lookup for OpenRouter models. + +LiteLLM decides which optional params a provider accepts by consulting the +bundled model-cost map. For OpenRouter that map is wrong by construction: +OpenRouter publishes new slugs continuously, the bundled map lags behind, and +``litellm.supports_reasoning`` answers ``False`` for anything it has not caught +up to yet. Because ``OpenrouterConfig.get_supported_openai_params`` uses that +answer as a bare gate, the failure is closed and silent: a ``reasoning_effort`` +set on a current model is discarded before the request body is built, with no +exception and (before the ``drop_params`` warning) no log line. + +OpenRouter publishes the authoritative answer itself. ``GET /api/v1/models`` +returns every model with a ``supported_parameters`` list and requires no API +key. This module reads that, caches it for the life of the process, and hands +callers a set of parameter names for a given slug. + +Design constraints, because this sits behind a hot, synchronous code path: + +* **Fail soft.** Any error, timeout, non-200, or malformed payload returns + ``None``, and every caller is expected to fall back to the existing + ``supports_reasoning`` behaviour. This module can make param handling more + accurate; it must never make a request fail. +* **Fetch at most once per TTL**, including after a failure. A negative cache + entry keeps an offline or firewalled deployment from attempting a network + call on every single request. +* **One fetch, not N**, and never a queue. A lock serialises refreshes so + concurrent requests do not stampede the endpoint; a thread that finds the + lock held serves the stale cache rather than waiting behind a network call. +* **Importable without litellm.** ``verbose_logger`` is imported inside + ``_log`` rather than at module scope so this file can be imported — and + therefore unit tested — in the egg repo, where litellm is not a dependency. + +Operator knobs (all optional; see ``docs/guides/per-agent-models.md``): + +* ``LITELLM_OPENROUTER_CAPABILITY_FETCH=0`` disables the lookup entirely and + restores the previous model-map-only behaviour. +* ``LITELLM_OPENROUTER_CAPABILITY_TTL`` seconds between refreshes + (default 3600). ``0`` disables caching and re-fetches on every lookup — + a debugging aid, not a production setting. +* ``LITELLM_OPENROUTER_CAPABILITY_TIMEOUT`` per-phase HTTP timeout in seconds + (default 5). + +An unparseable, unrecognized or out-of-range value for any of these is logged +and ignored rather than silently swallowed: an operator reaching for these vars +is very likely already debugging something. That covers a near miss on the +boolean too — ``FETCH=disabled`` is neither a recognised on nor off spelling, so +it warns and takes the default instead of being read as the opposite of what was +meant. +""" + +import json +import os +import threading +import time + +OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models" + +# The endpoint is served without authentication, so no key is read here on +# purpose: capability data must be available to a proxy that has not yet been +# handed credentials, and sending a key would make this lookup fail differently +# depending on which key happened to be in scope. +DEFAULT_TTL_SECONDS = 3600.0 +DEFAULT_TIMEOUT_SECONDS = 5.0 + +# Cache of slug -> supported parameter names. ``None`` means "not populated". +# An empty dict is a real, meaningful state: it records a failed fetch, so the +# negative cache below can suppress retries without conflating "we asked and +# got nothing" with "we never asked". +_CACHE: dict[str, set[str]] | None = None +_CACHE_STAMP: float = 0.0 +_LOCK = threading.Lock() + +# Whether a fetch failure has already been reported at warning level. The first +# failure is the one worth seeing (behaviour has silently reverted to the +# model-cost map); repeats are noise. Cleared again by a successful fetch, so a +# blip at startup does not permanently mute a real outage hours later. +_WARNED_FETCH_FAILURE = False + +# Env-var complaints already emitted, keyed by ``(name, raw value)``. +# ``_env_float`` is reached from ``_ttl_seconds`` on *every* lookup, ahead of +# the freshness check, so an unconditional warning there is one WARNING line per +# proxied request forever — a misconfigured TTL would bury the log stream egg's +# per-call cost observability reads. Bounded by construction: the environment +# does not change mid-process, so this holds at most one entry per knob. +_WARNED_ENV: set[tuple[str, str]] = set() + + +def _log(level: str, message: str, *args: object) -> bool: + """Log via litellm's ``verbose_logger``, deferring the import. + + Kept out of module scope so this file stays importable where litellm is + not installed. Never raises: a diagnostic must not be able to break a + request, and must not stop the capability lookup either. + + Returns whether the call completed without raising — *not* whether a record + reached a handler, since ``verbose_logger.debug(...)`` on a logger set to + INFO also returns normally. The distinction does not matter to either + caller: what the warn-once bookkeeping below must not do is mark a line + "already warned" when the attempt *raised* (litellm's logger not yet in + place, say), because swallowing the exception would then also swallow the + signal. Level filtering is the operator's own choice and is not a lost + signal. + """ + try: + from litellm._logging import verbose_logger + + getattr(verbose_logger, level)(message, *args) + return True + except Exception: # noqa: BLE001 - diagnostics must never break a request + return False + + +def _warn_env_once(name: str, raw: str, message: str, *args: object) -> None: + """Warn about a bad env value once per distinct ``(name, value)``. + + Same discipline as ``_log_fetch_failure`` here and ``_SEEN`` in + ``drop_params_visibility``: the first occurrence is the one that carries + information, and this one sits on the request path. + """ + key = (name, raw) + if key in _WARNED_ENV: + return + # Recorded only once the emit did not raise. ``_log`` deliberately never + # propagates, so recording first would mean a failure on the *first* call — + # litellm's logger not yet in place, say — permanently suppresses the + # warning: every later call would find the key already there. + if _log("warning", message, *args): + _WARNED_ENV.add(key) + + +_TRUTHY = ("1", "true", "yes", "on") +_FALSY = ("0", "false", "no", "off") + + +def _env_flag(name: str, default: bool) -> bool: + """Read a boolean env var, warning rather than guessing at a near miss. + + A bare ``raw not in _FALSY`` reads every unrecognized value as *enable*, so + a near-miss disable spelling (``=disabled``, ``=n``) does not fall back to + the default — it inverts the operator's instruction, silently. Anything + matching neither list warns once and takes the default, which is the same + discipline ``_env_float`` applies and the behaviour this module's docstring + promises for all three knobs. + """ + raw = os.getenv(name) + if raw is None: + return default + value = raw.strip().lower() + if value in _TRUTHY: + return True + if value in _FALSY: + return False + _warn_env_once( + name, + raw, + "openrouter capabilities: %s=%r is not a boolean (expected one of %s " + "or %s); using the default %s", + name, + raw, + ", ".join(_TRUTHY), + ", ".join(_FALSY), + default, + ) + return default + + +def _env_float(name: str, default: float, *, allow_zero: bool = False) -> float: + """Read a float env var, warning rather than silently falling back. + + A deliberately-set-but-unusable value (``TTL=0`` when zero is not allowed, + ``TIMEOUT=5s`` pasted with a unit suffix) previously became the default + with no signal at all, so an operator could set a knob, restart, observe + nothing change, and have no way to learn the proxy ignored them. + + The warning is deduplicated per ``(name, value)``: this is called from + ``_ttl_seconds`` on the per-request path, so warning unconditionally would + trade a silent fallback for an unbounded log flood. + """ + raw = os.getenv(name) + if raw is None: + return default + try: + value = float(raw) + except ValueError: + _warn_env_once( + name, + raw, + "openrouter capabilities: %s=%r is not a number; using the default %s", + name, + raw, + default, + ) + return default + if value < 0 or (value == 0 and not allow_zero): + _warn_env_once( + name, + raw, + "openrouter capabilities: %s=%r must be %s; using the default %s", + name, + raw, + ">= 0" if allow_zero else "> 0", + default, + ) + return default + return value + + +def _ttl_seconds() -> float: + # Zero is allowed and meaningful: it is the natural spelling of "never + # cache, always re-fetch", and rejecting it was the least discoverable of + # this module's silent fallbacks. It makes every lookup a network call, so + # it is a debugging setting rather than a production one. + return _env_float("LITELLM_OPENROUTER_CAPABILITY_TTL", DEFAULT_TTL_SECONDS, allow_zero=True) + + +def _fetch() -> dict[str, set[str]]: + """Fetch the model list. Returns ``{}`` on any failure.""" + global _WARNED_FETCH_FAILURE + + # Imported here rather than at module scope: http_handler pulls in a large + # slice of litellm, and this module is imported from a transformation that + # is itself imported during litellm's own startup. + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + timeout = _env_float("LITELLM_OPENROUTER_CAPABILITY_TIMEOUT", DEFAULT_TIMEOUT_SECONDS) + try: + response = HTTPHandler(timeout=timeout).get(OPENROUTER_MODELS_URL) + if response.status_code != 200: + _log_fetch_failure( + "%s returned HTTP %s; falling back to the bundled model-cost map", + OPENROUTER_MODELS_URL, + response.status_code, + ) + return {} + payload = response.json() + except Exception as exc: # noqa: BLE001 - never propagate into a request + _log_fetch_failure( + "fetch failed (%s: %s); falling back to the bundled model-cost map", + type(exc).__name__, + exc, + ) + return {} + + if isinstance(payload, (str, bytes)): + try: + payload = json.loads(payload) + except Exception: # noqa: BLE001 + _log_fetch_failure("response body was not JSON; falling back to the model-cost map") + return {} + + entries = payload.get("data") if isinstance(payload, dict) else None + if not isinstance(entries, list): + _log_fetch_failure("response had no `data` list; falling back to the model-cost map") + return {} + + capabilities: dict[str, set[str]] = {} + for entry in entries: + if not isinstance(entry, dict): + continue + model_id = entry.get("id") + params = entry.get("supported_parameters") + if not isinstance(model_id, str) or not isinstance(params, list): + continue + capabilities[model_id] = {p for p in params if isinstance(p, str)} + + if not capabilities: + # A 200 whose `data` list is empty, or every entry of which is + # unparseable, is an OpenRouter schema change or an empty roster — not + # the ordinary "we asked and got nothing usable" fallback it otherwise + # looks like from the outside. Reporting it keeps `{}` meaning exactly + # one thing (see the _CACHE comment) and makes a schema drift visible + # instead of indistinguishable from a fetch that simply had no opinion. + _log_fetch_failure( + "`data` list contained no usable entries; falling back to the bundled model-cost map" + ) + return {} + + # Re-arm the failure warning, and only on a fetch that actually produced + # capabilities. Latching it for the life of the process would mean a single + # blip during pod startup permanently demotes every later outage to debug — + # silencing exactly the case _log_fetch_failure exists to surface — but + # re-arming on an empty parse would report a recovery that did not happen. + _WARNED_FETCH_FAILURE = False + return capabilities + + +def _log_fetch_failure(message: str, *args: object) -> None: + """First failure at warning, the rest at debug. + + A permanently unreachable endpoint is the exact case where behaviour + silently reverts to the model-cost map, and litellm's default log level is + INFO — so debug-only reporting reproduces, one file over, the silence + Patch 8 exists to remove. Warning once is enough to be findable without + turning an offline deployment into a log flood. + """ + global _WARNED_FETCH_FAILURE + + level = "debug" if _WARNED_FETCH_FAILURE else "warning" + # Latched only once the emit did not raise, for the reason in ``_warn_env_once``: + # ``_log`` swallows its own failures, and marking "already warned" for a + # line that was never emitted would demote every later outage to debug + # without anyone having seen the first one. + if _log(level, "openrouter capabilities: " + message, *args): + _WARNED_FETCH_FAILURE = True + + +def _get_cache() -> dict[str, set[str]]: + global _CACHE, _CACHE_STAMP + + ttl = _ttl_seconds() + cache = _CACHE + if cache is not None and (time.monotonic() - _CACHE_STAMP) < ttl: + return cache + + # Exactly one thread refreshes. A thread that finds the lock held serves + # the cache it already has rather than queueing behind someone else's HTTP + # call: httpx timeouts are per-phase, not total, so a pathological + # connection can exceed the configured seconds and that latency would land + # on a live request once per TTL. Stale capability data is cheap; added + # request latency is not. Only the very first fetch — nothing cached to + # serve — actually blocks. + if not _LOCK.acquire(blocking=cache is None): + return cache # type: ignore[return-value] + try: + # Re-check under the lock: another thread may have refreshed while this + # one waited, and a second fetch would be pure waste. + if _CACHE is not None and (time.monotonic() - _CACHE_STAMP) < ttl: + return _CACHE + # Stamped even on failure, so an unreachable endpoint costs one attempt + # per TTL rather than one per request. + _CACHE = _fetch() + _CACHE_STAMP = time.monotonic() + return _CACHE + finally: + _LOCK.release() + + +def _candidate_slugs(model: str) -> list[str]: + """Spellings of ``model`` that may appear as an OpenRouter model id. + + Callers reach this from several directions: a bare slug + (``qwen/qwen3-max``), a provider-prefixed one (``openrouter/qwen/qwen3-max`` + from a ``litellm_params.model``), or a slug carrying an OpenRouter variant + suffix (``qwen/qwen3-max:free``). The ids returned by the API are bare + slugs, with ``:free`` published as its own id. + """ + candidates: list[str] = [] + + def add(value: str) -> None: + if value and value not in candidates: + candidates.append(value) + + add(model) + if model.startswith("openrouter/"): + add(model[len("openrouter/") :]) + # Variant suffixes (:free, :nitro, :floor, ...) are sometimes their own id + # and sometimes only a routing hint on the base model, so try both. + for candidate in list(candidates): + if ":" in candidate: + add(candidate.split(":", 1)[0]) + return candidates + + +def get_supported_parameters(model: str) -> set[str] | None: + """Parameter names OpenRouter advertises for ``model``. + + Returns ``None`` when the answer is unknown for any reason: the lookup is + disabled, the fetch failed, or the slug is not in the published list. A + ``None`` return means "no opinion" and callers must fall back to whatever + they did before. + + The returned set is a copy — the cache is process-wide and lives for the + TTL, so handing out the stored object would let one caller's ``add`` change + what every later caller sees. + """ + if not _env_flag("LITELLM_OPENROUTER_CAPABILITY_FETCH", True): + return None + if not model: + return None + + cache = _get_cache() + if not cache: + return None + + for candidate in _candidate_slugs(model): + params = cache.get(candidate) + if params is not None: + return set(params) + return None + + +def reset_cache() -> None: + """Drop cached capability data. Intended for tests.""" + global _CACHE, _CACHE_STAMP, _WARNED_FETCH_FAILURE + with _LOCK: + _CACHE = None + _CACHE_STAMP = 0.0 + _WARNED_FETCH_FAILURE = False + _WARNED_ENV.clear() diff --git a/config/litellm/patch_litellm_cache.py b/config/litellm/patch_litellm_cache.py index 244646651..b4e1b64a4 100644 --- a/config/litellm/patch_litellm_cache.py +++ b/config/litellm/patch_litellm_cache.py @@ -4,8 +4,12 @@ LiteLLM's stock Anthropic->OpenAI translation (the path Claude Code's ``/v1/messages`` requests take when routed at a non-Claude OpenRouter backend) drops prompt-cache hits for Qwen/DeepSeek and mis-streams -reasoning models. Six independent gaps cause it; this script closes all -six by editing the installed ``litellm`` package in place, then +reasoning models, and its OpenRouter param gate reads a model-cost map +that does not carry current OpenRouter slugs, and it discards +caller-specified params in total silence, while manufacturing a +reasoning ceiling nobody asked for. Nine independent gaps cause it; this +script closes all nine by editing the installed ``litellm`` +package in place (and installing three new modules), then ``config/litellm/Dockerfile`` bakes the result into the ``egg-litellm`` image. @@ -75,6 +79,78 @@ exists only in the async ``__anext__`` path upstream (the sync path has no equivalent merge block), and that async path is the one the litellm proxy drives for Claude Code streaming. + 7. ``OpenrouterConfig.get_supported_openai_params`` + (openrouter/chat/transformation.py) consult OpenRouter's published + per-model ``supported_parameters`` instead of only the bundled + model-cost map. The stock gate asks ``litellm.supports_reasoning``, + which reads ``model_prices_and_context_window.json``; OpenRouter + ships new slugs faster than that map tracks them, so a current + model answers False. The gate is a bare ``if``, so it fails CLOSED, + and ``drop_params: true`` discards the parameter with no exception + and no log line. Every OpenRouter slug egg routes is absent from the + 1.86.2 map, so a reasoning knob set on any of them never reached the + wire. The companion module ``llms/openrouter/_egg_capabilities.py`` + (installed by ``NEW_MODULES``) reads OpenRouter's unauthenticated + ``/api/v1/models`` and is UNIONED with the map answer, never + subtractive: ``supported_parameters`` under-reports + ``reasoning_effort`` (deepseek-r1 advertises only ``reasoning``, + treating the OpenAI spelling as an alias), so reading its absence as + a denial would drop a working param. Only ``reasoning_effort`` is + admitted — OpenRouter's ``reasoning`` field is a different wire shape + from Anthropic's ``thinking``, not a spelling of it. Fails soft: any + fetch error yields no opinion and the stock path runs unchanged. + Mirrors jwbron/litellm#8 and jwbron/egg#3624. Patch 9 is the other + half of this one: read them together. + 8. ``get_optional_params`` drop site (utils.py) log what + ``drop_params`` discards. Stock 1.86.2 pops unsupported params in a + bare loop with no logging, so a param set in a proxy config that + never reaches the provider is a real behavioural difference with no + signal attached — the condition that made patch 7's bug take a full + investigation to find. Patch 7 removes the OpenRouter + false-negative; this covers the rest, including drops that are + CORRECT: laguna-s-2.1 genuinely does not accept ``reasoning_effort``, + so it is dropped on purpose and the operator otherwise has no way to + learn why their config line does nothing. Deduped per + (provider, model, param-set) and bounded. NOTE the needle: 1.86.2 has + two ``drop_params`` branches in utils.py and the shared condition + alone matches the wrong one, so the needle includes the pop loop. + Mirrors jwbron/litellm#7, merged into the fork the HOST proxy runs; + the cluster image pins stock 1.86.2, which predates it. The message + offers the ``allowed_openai_params`` remedy GATED on the params + having come from this model's ``litellm_params``, and names the + synthesized case alongside it: the param most often dropped here is + one litellm manufactured from the request itself (see 9), so an + unconditional config edit would send that operator hunting for a + line that does not exist. + 9. ``_translate_thinking_to_openai`` (anthropic adapter + transformation.py) stop synthesizing ``reasoning_effort`` from the + caller's ``thinking`` block for non-Claude models. On ``/v1/messages`` + — egg's primary route — Claude Code sends + ``thinking: {"type": "enabled", "budget_tokens": N}``, and because + ``is_anthropic_claude_model`` is a substring test for + ``anthropic``/``claude``, every OpenRouter slug egg routes takes the + non-Claude branch where the adapter REPLACES that block with a + bucketed ``reasoning_effort``. Nothing in ``litellm-models.yaml`` is + involved: the value is manufactured per request. That was harmless + only because patch 7's bug dropped it; the measurements in + jwbron/egg#3624 show the bucket is a CAP BELOW the model default + (kimi-k3: 3130 reasoning tokens with no param, 340 with + ``reasoning_effort: high``, non-overlapping), so shipping patch 7 + without this would cut reasoning ~9x per agent turn with no config + file to point at and nothing logged — patch 8 fires only on drops, + and this param would no longer be dropped. Gating the synthesis (off + by default, ``LITELLM_ANTHROPIC_THINKING_TO_REASONING_EFFORT=1`` to + restore stock) keeps patch 7's actual goal: a knob an operator + configured reaches the wire, one nobody configured does not. The + Claude branch is untouched, and so is an effort the CALLER stated + outright: on an adaptive request (``thinking: {"type": "adaptive"}`` + plus ``output_config: {"effort": ...}``) the gate sits after stock's + override, so that value still reaches the provider with the policy + off. Only the DERIVED bucket is suppressed — the distinction is + structural, not a special case. A ``thinking.summary`` request goes + with the derived effort, because stock carries the summary only as a + field of the ``reasoning_effort`` dict and there is no wire shape for + "summary, no effort". Idempotent: each patch detects whether it is already applied. Fails loudly (non-zero exit) if a needle is missing, so a LiteLLM version bump @@ -82,6 +158,7 @@ an unpatched image that bills full input rate or drops reasoning tokens. """ +import ast import importlib.util import os import sys @@ -118,6 +195,36 @@ def _litellm_roots() -> list[str]: return out +def _parses(source: str) -> bool: + try: + ast.parse(source) + except SyntaxError: + return False + return True + + +def _check_parses(source: str, path: str, label: str, detail: str) -> None: + """Fail the build if we are about to write source Python cannot import. + + A needle miss already exits non-zero, but a replacement with wrong + indentation applies *cleanly* — the result would pass the build, ship in + the image, and surface as a pod CrashLoopBackOff at litellm import time, + long after the only thing that could have caught it. Same fail-loud + discipline as the needle check, one step later. + + ``detail`` names what is actually broken, because the two callers arrive + here from different directions: ``_apply`` has substituted a replacement + into an upstream file, while ``_install_module`` has read a staged module of + ours with no replacement involved at all. ``source`` must be the text whose + line numbering matches ``path``, so the reported line sends the operator to + the right one. + """ + try: + ast.parse(source, filename=path) + except SyntaxError as exc: + raise SystemExit(f"{label}: {detail} ({path}:{exc.lineno}: {exc.msg})") from exc + + def _apply(path: str, present: str, needle: str, replacement: str, label: str) -> None: if not os.path.isfile(path): raise SystemExit(f"{label}: file not found: {path}") @@ -128,14 +235,28 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - return if needle not in src: raise SystemExit(f"{label}: marker not found in {path} — LiteLLM version drift?") + patched = src.replace(needle, replacement, 1) + # Checked only when the input was valid Python to begin with: a patch can + # break a file, it cannot be blamed for one that never parsed. Every real + # litellm source does; the concatenated-needle fixtures in tests/config + # deliberately do not, and holding them to it would test the fixture rather + # than the patch. + if _parses(src): + _check_parses( + patched, + path, + label, + "patched source does not parse — the replacement is malformed", + ) with open(path, "w") as fh: - fh.write(src.replace(needle, replacement, 1)) + fh.write(patched) print(f"{label}: applied") F1 = "llms/openrouter/chat/transformation.py" F2 = "llms/anthropic/experimental_pass_through/adapters/transformation.py" F3 = "llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py" +F4 = "utils.py" # Every patch as a self-contained spec: (file, present marker, needle, # replacement, label). Module-level so tests can apply them to a checked-in @@ -158,7 +279,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - ' QWEN = "qwen"\n' ' DEEPSEEK = "deepseek"\n' ), - "label": "Patch 1/6 (CacheControlSupportedModels)", + "label": "Patch 1/9 (CacheControlSupportedModels)", }, # Patch 2 — broaden ONLY the cache_control gate (not the shared # is_anthropic_claude_model predicate, which also gates thinking @@ -188,7 +309,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " )\n" " ):\n" ), - "label": "Patch 2/6 (cache_control gate)", + "label": "Patch 2/9 (cache_control gate)", }, # Patch 3 — drop x-anthropic-billing-header during Anthropic->OpenAI translation. { @@ -222,7 +343,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - ' "text": text,\n' " }\n" ), - "label": "Patch 3/6 (x-anthropic-billing-header filter)", + "label": "Patch 3/9 (x-anthropic-billing-header filter)", }, # Patch 4 — OpenRouter-style reasoning_content must open a thinking # content block, not fall through to a text block. The bare @@ -260,7 +381,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - ' choice.delta, "thinking_blocks"\n' " ):\n" ), - "label": "Patch 4/6 (reasoning_content thinking block)", + "label": "Patch 4/9 (reasoning_content thinking block)", }, # Patch 5a — sync __next__: don't drop the first delta on text or # thinking block transitions. @@ -360,7 +481,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " ):\n" " self.chunk_queue.append(processed_chunk)\n" ), - "label": "Patch 5a/6 (sync first-delta requeue)", + "label": "Patch 5a/9 (sync first-delta requeue)", }, # Patch 5b — async __anext__: same first-delta preservation. { @@ -458,7 +579,7 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " ):\n" " self.chunk_queue.append(processed_chunk)\n" ), - "label": "Patch 5b/6 (async first-delta requeue)", + "label": "Patch 5b/9 (async first-delta requeue)", }, # Patch 6 — streamed usage must report provider-automatic cache hits. # The needle spans the whole usage-merge region so both edit points @@ -550,13 +671,332 @@ def _apply(path: str, present: str, needle: str, replacement: str, label: str) - " elif cached_tokens > 0:\n" ' usage_dict["cache_read_input_tokens"] = cached_tokens\n' ), - "label": "Patch 6/6 (streaming cache_read fallback)", + "label": "Patch 6/9 (streaming cache_read fallback)", + }, + # Patch 7 — OpenrouterConfig.get_supported_openai_params: consult + # OpenRouter's published capabilities instead of only the bundled + # model-cost map. + # + # The stock gate asks ``litellm.supports_reasoning``, which reads + # ``model_prices_and_context_window.json``. For OpenRouter that map is + # wrong by construction: OpenRouter ships new slugs continuously and the + # bundled map lags, so a current model answers False. The gate is a bare + # ``if``, so it fails CLOSED, and ``drop_params: true`` then discards the + # parameter with no exception and no log line. Every OpenRouter slug egg + # routes is absent from the 1.86.2 map (kimi-k3, glm-5.2, laguna-s-2.1, + # deepseek-v4-*), so any reasoning knob set on them never reached the wire. + # + # The companion module (installed by ``NEW_MODULES`` below) reads + # OpenRouter's unauthenticated /api/v1/models and is UNIONED with the + # existing map answer rather than replacing it. Live data can admit a knob + # the map does not know about but never withholds one the map allows, + # because ``supported_parameters`` under-reports ``reasoning_effort``: + # deepseek/deepseek-r1 is flagged supports_reasoning in the map and is + # plainly a reasoning model, yet OpenRouter advertises only ``reasoning`` + # for it, treating the OpenAI spelling as an alias. Reading that absence as + # a denial would drop a working param — the very failure this fixes. + # + # Only ``reasoning_effort`` is admitted. OpenRouter also advertises a + # ``reasoning`` param, but that is its own request field + # (``{"effort": ...}`` / ``{"max_tokens": ...}``) and is NOT a spelling of + # Anthropic's ``thinking`` (``{"type", "budget_tokens"}``). Mapping one to + # the other by name similarity would let a raw Anthropic-shaped ``thinking`` + # dict through to a non-Anthropic provider on the /chat/completions route — + # the same spelling conflation patch 2's notes are careful to avoid. + # + # Mirrors jwbron/litellm#8 and jwbron/egg#3624. Fails soft throughout: any + # fetch error yields no opinion and the stock path runs, so the worst case + # is exactly the unpatched behaviour. + { + "file": F1, + "present": "# egg openrouter capability patch", + "needle": ( + " def get_supported_openai_params(self, model: str) -> list:\n" + ' """\n' + " Allow reasoning parameters for models flagged as reasoning-capable.\n" + ' """\n' + " supported_params = super().get_supported_openai_params(model=model)\n" + " try:\n" + ), + "replacement": ( + " def get_supported_openai_params(self, model: str) -> list:\n" + ' """\n' + " Allow reasoning parameters for models flagged as reasoning-capable.\n" + ' """\n' + " supported_params = super().get_supported_openai_params(model=model)\n" + " # egg openrouter capability patch. OpenRouter publishes per-model\n" + " # supported_parameters over an unauthenticated endpoint; the bundled\n" + " # model-cost map does not carry current slugs, so the stock gate\n" + " # below fails closed and the knob is dropped in silence. Unioned,\n" + " # never subtractive — see patch 7 notes in patch_litellm_cache.py.\n" + " try:\n" + " from litellm.llms.openrouter._egg_capabilities import (\n" + " get_supported_parameters as _egg_openrouter_capabilities,\n" + " )\n" + "\n" + " _advertised = _egg_openrouter_capabilities(model)\n" + " if _advertised is not None:\n" + ' if "reasoning_effort" in _advertised:\n' + ' supported_params.append("reasoning_effort")\n' + " except Exception:\n" + " pass\n" + " try:\n" + ), + "label": "Patch 7/9 (openrouter live capabilities)", + }, + # Patch 8 — get_optional_params: log what ``drop_params`` discards. + # + # Patch 7 removes the OpenRouter false-negative, but a drop can still be + # correct and still worth knowing about: laguna-s-2.1 genuinely does not + # accept ``reasoning_effort``, so the knob is dropped on purpose and the + # operator has no way to learn why their config line does nothing. Stock + # 1.86.2 pops unsupported params in a bare loop with no logging at all. + # + # NEEDLE DISAMBIGUATION: 1.86.2 has TWO ``if litellm.drop_params is True or + # (...)`` sites in utils.py. The other one (~line 3303, the embeddings + # path) is followed by a bare ``pass``; this one is followed by the pop + # loop. The needle therefore includes the loop line — the shared condition + # alone would match whichever comes first and patch the wrong function. + # + # Mirrors jwbron/litellm#7, merged into the fork the HOST proxy runs. The + # cluster image pins stock 1.86.2, which predates it. + { + "file": F4, + "present": "# egg drop_params visibility patch", + "needle": ( + " if litellm.drop_params is True or (\n" + " drop_params is not None and drop_params is True\n" + " ):\n" + " for k in unsupported_params.keys():\n" + " non_default_params.pop(k, None)\n" + ), + "replacement": ( + " if litellm.drop_params is True or (\n" + " drop_params is not None and drop_params is True\n" + " ):\n" + " # egg drop_params visibility patch. Dropping a param changes\n" + " # generation behaviour; stock does it with no signal at all.\n" + " try:\n" + " from litellm._egg_drop_params_visibility import (\n" + " warn_dropped_params as _egg_warn_dropped_params,\n" + " )\n" + "\n" + " _egg_warn_dropped_params(\n" + " unsupported_params=unsupported_params,\n" + " model=model,\n" + " custom_llm_provider=custom_llm_provider,\n" + " )\n" + " except Exception:\n" + " pass\n" + " for k in unsupported_params.keys():\n" + " non_default_params.pop(k, None)\n" + ), + "label": "Patch 8/9 (drop_params visibility)", + }, + # Patch 9 — _translate_thinking_to_openai: stop synthesizing + # ``reasoning_effort`` from the caller's ``thinking`` block for non-Claude + # models. + # + # This is the other half of patch 7, and without it patch 7 is a + # regression on egg's primary route. On /v1/messages (Claude Code -> + # gateway -> litellm -> OpenRouter) the request body carries + # ``thinking: {"type": "enabled", "budget_tokens": N}``. + # ``is_anthropic_claude_model`` is a substring test for + # ``anthropic``/``claude``, so every OpenRouter slug egg routes takes the + # non-Claude branch, where the adapter REPLACES the block with a bucketed + # ``reasoning_effort`` (>=10000 -> "high", >=5000 -> "medium", + # >=2000 -> "low"). Nothing in litellm-models.yaml is involved: the value + # is manufactured per request. + # + # Until now that param was silently dropped (the model-cost map does not + # carry these slugs), which is exactly why these models have been running + # at full reasoning depth. Patch 7 unblocks the param — correct for an + # operator-configured value, wrong for this one, because the measurements + # in jwbron/egg#3624 show the bucket is a CAP BELOW the model default: + # kimi-k3 means 3130 reasoning tokens with no param vs 340 with + # ``reasoning_effort: high``, distributions non-overlapping. Shipping + # patch 7 alone would cut reasoning ~9x on every agent turn with nothing + # logged (patch 8 only fires on drops, and this is no longer dropped) and + # no config file to point at. + # + # Gating the synthesis keeps patch 7's actual goal — a configured knob + # reaches the wire — without letting the adapter's bucket become the + # effective setting. ``LITELLM_ANTHROPIC_THINKING_TO_REASONING_EFFORT=1`` + # restores stock behaviour. The Claude branch above is untouched. + # + # The gate sits AFTER the adaptive-thinking override, not before the whole + # block, and that placement is the contract. Stock reaches the + # ``reasoning_effort`` assignment two ways: derived from ``budget_tokens`` + # (the manufactured ceiling this patch exists to stop) or stated outright by + # the caller as ``output_config.effort`` on an adaptive request. Gating the + # whole function would discard the second — an explicit instruction, not an + # invented cap — so only the derived value is suppressed. egg's own route + # never sends the adaptive shape (Claude Code sends + # ``thinking.type == "enabled"``), so this is about the patch matching its + # own stated scope rather than a live behaviour today. + # + # A ``thinking.summary`` request is still suppressed along with the derived + # effort, and that is deliberate: stock carries the summary only as a field + # of the ``reasoning_effort`` dict, so honouring it would require sending + # the manufactured ceiling. There is no wire shape for "summary, no effort". + { + "file": F2, + "present": "# egg thinking-synthesis patch", + "needle": ( + " # For adaptive thinking, override with output_config.effort if available\n" + ' if isinstance(thinking, dict) and thinking.get("type") == "adaptive":\n' + ' output_config = anthropic_message_request.get("output_config")\n' + ' if isinstance(output_config, dict) and output_config.get("effort"):\n' + ' reasoning_effort = output_config["effort"]\n' + "\n" + ' summary = thinking.get("summary") if isinstance(thinking, dict) else None\n' + ), + "replacement": ( + " # For adaptive thinking, override with output_config.effort if available\n" + " _egg_effort_is_explicit = False\n" + ' if isinstance(thinking, dict) and thinking.get("type") == "adaptive":\n' + ' output_config = anthropic_message_request.get("output_config")\n' + ' if isinstance(output_config, dict) and output_config.get("effort"):\n' + ' reasoning_effort = output_config["effort"]\n' + " _egg_effort_is_explicit = True\n" + "\n" + " # egg thinking-synthesis patch. Everything above DERIVES an effort\n" + " # from the caller's thinking budget, and that bucket is a cap BELOW\n" + " # the model default on every model egg routes, so sending it\n" + " # silently shallows reasoning. Off by default; see the patch 9 notes\n" + " # in patch_litellm_cache.py. An effort the caller stated outright\n" + " # (output_config.effort) is an instruction rather than a\n" + " # manufactured ceiling, and is never suppressed.\n" + " if not _egg_effort_is_explicit:\n" + " try:\n" + " from litellm._egg_anthropic_thinking_policy import (\n" + " should_synthesize_reasoning_effort as _egg_should_synthesize,\n" + " )\n" + "\n" + " _egg_synthesize = _egg_should_synthesize()\n" + " except Exception:\n" + " # The module is installed by the same build step as this\n" + " # patch, so this is unreachable in a built image; fall back\n" + " # to the policy's own default, not to stock behaviour.\n" + " _egg_synthesize = False\n" + " if not _egg_synthesize:\n" + " return\n" + "\n" + ' summary = thinking.get("summary") if isinstance(thinking, dict) else None\n' + ), + "label": "Patch 9/9 (thinking->reasoning_effort synthesis gate)", + }, +] + +# Whole modules to drop into each litellm tree, sourced from files the +# Dockerfile stages under /egg. Unlike PATCHES these are additive: there is no +# stock file to collide with, so installation is a copy guarded by a content +# check rather than a needle match. +# +# Every destination carries the ``_egg_`` prefix. It is not decoration: it +# keeps a future upstream module from colliding with ours, since an unprefixed +# name (``capabilities.py``) is one upstream could plausibly take. +EGG_MODULE_PREFIX = "_egg_" + +# Provenance header written ahead of every installed module. The prefix above +# makes a collision unlikely; this is what makes the clobber guard *real*. +# Checking the prefix told us only that our own ``NEW_MODULES`` literals were +# spelled the way we spelled them — it could never fire on upstream drift, +# because it never looked at the file on disk. This does: a file at one of our +# destinations that does not carry this header is not ours, whoever put it +# there, and overwriting it would break litellm in a way nothing else in this +# script would report. +EGG_MODULE_MARKER = "egg-managed module (config/litellm/patch_litellm_cache.py)" +EGG_MODULE_HEADER = f"# {EGG_MODULE_MARKER} — do not edit in place.\n" + +NEW_MODULES: list[dict[str, str]] = [ + { + "source": "openrouter_capabilities.py", + "dest": "llms/openrouter/_egg_capabilities.py", + "label": "Module 1/3 (openrouter capabilities)", + }, + { + "source": "drop_params_visibility.py", + "dest": "_egg_drop_params_visibility.py", + "label": "Module 2/3 (drop_params visibility)", + }, + { + "source": "anthropic_thinking_policy.py", + "dest": "_egg_anthropic_thinking_policy.py", + "label": "Module 3/3 (thinking synthesis policy)", }, ] +def _module_source(name: str, label: str) -> str: + """Locate a staged module by basename. + + In the image the Dockerfile drops it beside this script under /egg; in the + repo (and in tests) it sits beside this script in config/litellm. Checking + both means the same script runs in either place without a path flag, and it + still fails loudly rather than silently skipping the install.""" + here = os.path.dirname(os.path.abspath(__file__)) + for candidate in (os.path.join("/egg", name), os.path.join(here, name)): + if os.path.isfile(candidate): + return candidate + raise SystemExit(f"{label}: staged source not found: {name} (looked in /egg and {here})") + + +def _install_module(root: str, spec: dict[str, str]) -> None: + label = spec["label"] + # A lint of this file's own constants rather than drift detection (the + # provenance check below is that): every destination must carry the prefix, + # so an upstream module can never occupy one of our paths to begin with. + if not os.path.basename(spec["dest"]).startswith(EGG_MODULE_PREFIX): + raise SystemExit( + f"{label}: destination {spec['dest']} must be prefixed {EGG_MODULE_PREFIX!r}" + ) + source = _module_source(spec["source"], label) + with open(source) as fh: + body = fh.read() + # The header goes on disk, not in the repo copy: it is the provenance the + # guard below reads. A leading comment leaves the module docstring as the + # first statement, so nothing about the module changes. + payload = EGG_MODULE_HEADER + body + # Same reason as in ``_apply``: a truncated COPY or a half-written staged + # file would install without complaint and only fail at litellm import. + # Checked against the un-headered text, whose line numbers match the file + # the operator will open — the header is a comment, so it cannot change + # whether the rest parses, but it does shift every reported line by one. + _check_parses(body, source, label, "staged module source does not parse") + dest = os.path.join(root, spec["dest"]) + dest_dir = os.path.dirname(dest) + if not os.path.isdir(dest_dir): + raise SystemExit( + f"{label}: destination package missing: {dest_dir} — LiteLLM version drift?" + ) + if os.path.isfile(dest): + with open(dest) as fh: + existing = fh.read() + if existing == payload: + print(f"{label}: already installed") + return + # Differing content that still carries our header is a stale install + # from an earlier image layer — overwrite it. Differing content WITHOUT + # the header means somebody else owns this path (upstream took the + # name, an operator dropped a file in), and clobbering it would break + # litellm in a way nothing else in this script would report. Every + # other operation here is fail-loud on drift; so is this. + if EGG_MODULE_MARKER not in existing: + raise SystemExit( + f"{label}: refusing to overwrite {dest} — it exists with different " + "content and no egg provenance header, so it is not ours. " + "LiteLLM version drift?" + ) + with open(dest, "w") as fh: + fh.write(payload) + print(f"{label}: installed") + + def _patch_root(root: str) -> None: print(f"== patching {root}") + for spec in NEW_MODULES: + _install_module(root, spec) for spec in PATCHES: _apply( os.path.join(root, spec["file"]), diff --git a/docs/guides/per-agent-models.md b/docs/guides/per-agent-models.md index cbb7c4c77..7ea40dd29 100644 --- a/docs/guides/per-agent-models.md +++ b/docs/guides/per-agent-models.md @@ -593,7 +593,8 @@ data: > Claude Code prepends (the block's `cch=` hash invalidates the cache key > every turn). egg ships a custom **`egg-litellm`** image > ([`config/litellm/Dockerfile`](../../config/litellm/Dockerfile)) that -> bakes in three patches closing those gaps +> bakes in nine patches closing those gaps and the reasoning-parameter ones +> below > ([`config/litellm/patch_litellm_cache.py`](../../config/litellm/patch_litellm_cache.py)); > the build fails loudly if a LiteLLM bump moves the patched code. Pinning > the OpenRouter provider (`extra_body.provider.order` + `allow_fallbacks: @@ -608,6 +609,48 @@ data: > line to the LiteLLM pod stream, visible via `get_service_logs` / the > structured-logging stream. +> **Reasoning depth on OpenRouter routes, and the four env vars that +> control it.** Two of the baked-in patches decide whether a reasoning +> parameter reaches the provider, and both are runtime-overridable on +> [`k8s/base/litellm-deployment.yaml`](../../k8s/base/litellm-deployment.yaml) +> (where they are present but commented out) without rebuilding the image: +> +> - **Patch 7 — live capability lookup.** LiteLLM gates reasoning params on +> its bundled model-cost map, which does not carry current OpenRouter +> slugs, so a `reasoning_effort` set in `litellm_params` was discarded +> before the request body was built — no exception, no log line. The patch +> also reads OpenRouter's unauthenticated `GET /api/v1/models`, **unioned** +> with the map answer so it can admit a knob the map does not know but +> never withhold one it allows. `LITELLM_OPENROUTER_CAPABILITY_FETCH=0` +> restores map-only behaviour exactly; +> `LITELLM_OPENROUTER_CAPABILITY_TTL` (default `3600` seconds; `0` means +> re-fetch on every lookup, a debugging setting) and +> `LITELLM_OPENROUTER_CAPABILITY_TIMEOUT` (default `5` seconds, per HTTP +> phase) tune it. A fetch that fails is cached for the TTL too, so an +> offline cluster costs one attempt per hour rather than one per request, +> and the first failure is logged at `warning`. +> - **Patch 9 — no synthesized reasoning ceiling.** On `/v1/messages` (the +> route Claude Code uses) LiteLLM's Anthropic adapter converts each +> request's `thinking: {budget_tokens: N}` into a bucketed +> `reasoning_effort` for any non-Claude model. That value is not in any +> config file — it is manufactured per request — and measurement (#3624) +> shows it acts as a **cap below the model's own default**: kimi-k3 means +> 3130 reasoning tokens with no parameter vs 340 with `high`, +> distributions non-overlapping. So the synthesis is off by default and +> only an explicitly configured `reasoning_effort` reaches the wire. Set +> `LITELLM_ANTHROPIC_THINKING_TO_REASONING_EFFORT=1` to restore stock +> behaviour — worth doing only for a model you have measured to reason +> *more* when asked explicitly. Only the *derived* value is suppressed: an +> adaptive request that names an effort outright +> (`output_config: {effort: ...}`) is an instruction rather than a +> manufactured ceiling, and still reaches the provider with this off. +> +> The same measurement is why the commented-out +> `extra_body.reasoning.effort: "high"` in +> [`config/litellm-models.template.yaml`](../../config/litellm-models.template.yaml) +> carries a "measure before uncommenting" warning: on these models, +> sending nothing is what buys full depth. + > **Which decoding config did this run under?** Every `cost_callback` line > also carries `request_params` — the sampling configuration that actually > went upstream on that call (`temperature`, `top_p`, `top_k`, the penalty diff --git a/k8s/base/litellm-deployment.yaml b/k8s/base/litellm-deployment.yaml index 82ca3d937..b555d8052 100644 --- a/k8s/base/litellm-deployment.yaml +++ b/k8s/base/litellm-deployment.yaml @@ -93,6 +93,34 @@ spec: name: gateway-secrets key: openrouter-api-key optional: true + # Escape hatches for the egg-litellm build-time patches (see + # ``config/litellm/patch_litellm_cache.py`` and + # ``docs/guides/per-agent-models.md``). All optional; the defaults + # are what the patches were measured with. Uncomment to revert an + # individual patch at runtime without rebuilding the image. + # + # Patch 7 — live OpenRouter capability lookup. ``0`` restores the + # bundled model-cost map as the only source of truth, which + # silently drops reasoning knobs on any slug the map has not caught + # up to. ``_TTL`` seconds between refreshes (default 3600; 0 means + # re-fetch every lookup — a debugging setting). ``_TIMEOUT`` + # per-phase HTTP timeout in seconds (default 5). + # - name: LITELLM_OPENROUTER_CAPABILITY_FETCH + # value: "0" + # - name: LITELLM_OPENROUTER_CAPABILITY_TTL + # value: "3600" + # - name: LITELLM_OPENROUTER_CAPABILITY_TIMEOUT + # value: "5" + # + # Patch 9 — thinking -> reasoning_effort synthesis, off by + # default. ``1`` restores stock LiteLLM: the Anthropic adapter + # derives a bucketed ``reasoning_effort`` from each request's + # thinking budget on /v1/messages. Measured on kimi-k3 that bucket + # is a CAP BELOW the model default (3130 reasoning tokens with no + # param vs 340 with "high"), so only turn this on for a model + # measured to need an explicit ask. + # - name: LITELLM_ANTHROPIC_THINKING_TO_REASONING_EFFORT + # value: "1" livenessProbe: httpGet: path: /health/liveliness diff --git a/tests/config/test_litellm_runtime_modules.py b/tests/config/test_litellm_runtime_modules.py new file mode 100644 index 000000000..6d80d644b --- /dev/null +++ b/tests/config/test_litellm_runtime_modules.py @@ -0,0 +1,627 @@ +"""Unit tests for the three modules the patch script installs into litellm. + +``config/litellm/{openrouter_capabilities,drop_params_visibility, +anthropic_thinking_policy}.py`` are staged by the Dockerfile and copied into +every litellm tree by ``patch_litellm_cache.py``. They are kept as real files +rather than string literals inside the patch script precisely so they can be +linted and tested here — but that only works if they import without litellm +installed, which is why each of them defers its ``litellm`` imports into the +function that needs them. These tests lock in both halves: the behaviour, and +the importability that makes the behaviour testable. + +litellm itself is not a project dependency (and 1.86.2 cannot run on the +repo's Python), so the handful of symbols the modules reach for at call time — +``verbose_logger`` and ``HTTPHandler`` — are stubbed into ``sys.modules``. +""" + +import importlib.util +import sys +import types +from pathlib import Path + +import pytest + +CONFIG_DIR = Path(__file__).resolve().parents[2] / "config" / "litellm" + + +def _load(name: str): + """Load a staged module from disk under a test-local module name.""" + path = CONFIG_DIR / f"{name}.py" + spec = importlib.util.spec_from_file_location(f"egg_staged_{name}", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class _RecordingLogger: + """Stand-in for litellm's ``verbose_logger``, capturing formatted calls.""" + + def __init__(self): + self.records: list[tuple[str, str]] = [] + + def _record(self, level, message, *args): + self.records.append((level, message % args if args else message)) + + def warning(self, message, *args): + self._record("warning", message, *args) + + def debug(self, message, *args): + self._record("debug", message, *args) + + def info(self, message, *args): + self._record("info", message, *args) + + def messages(self, level=None): + return [text for lvl, text in self.records if level is None or lvl == level] + + +class _FlakyLogger(_RecordingLogger): + """Raises on the first ``n`` emits, then records normally. + + Stands in for a logger that is not yet wired up when the first diagnostic + fires. The modules swallow that failure by design; the question these + tests ask is whether they also swallow the *signal*. + """ + + def __init__(self, failures=1): + super().__init__() + self.remaining = failures + + def warning(self, message, *args): + if self.remaining > 0: + self.remaining -= 1 + raise RuntimeError("logging subsystem not ready") + super().warning(message, *args) + + +def _install_logger(monkeypatch, recorder): + litellm = sys.modules.get("litellm") or types.ModuleType("litellm") + logging_mod = types.ModuleType("litellm._logging") + logging_mod.verbose_logger = recorder + monkeypatch.setitem(sys.modules, "litellm", litellm) + monkeypatch.setitem(sys.modules, "litellm._logging", logging_mod) + return recorder + + +@pytest.fixture +def logger(monkeypatch): + """Install a stub ``litellm._logging.verbose_logger``.""" + return _install_logger(monkeypatch, _RecordingLogger()) + + +# -------------------------------------------------------------------------- +# openrouter_capabilities +# -------------------------------------------------------------------------- + + +@pytest.fixture +def caps(monkeypatch): + module = _load("openrouter_capabilities") + module.reset_cache() + # Never let a test inherit a stray value from the ambient environment. + for var in ( + "LITELLM_OPENROUTER_CAPABILITY_FETCH", + "LITELLM_OPENROUTER_CAPABILITY_TTL", + "LITELLM_OPENROUTER_CAPABILITY_TIMEOUT", + ): + monkeypatch.delenv(var, raising=False) + return module + + +def _install_http_stub(monkeypatch, *, payload=None, status_code=200, boom=None): + """Stub ``litellm.llms.custom_httpx.http_handler.HTTPHandler``. + + Returns a list that records one entry per constructed handler, so a test + can assert how many fetches actually happened. + """ + calls: list[float] = [] + + class _Response: + status_code = None + + def json(self): + return payload + + class _Handler: + def __init__(self, timeout=None): + calls.append(timeout) + + def get(self, url): + if boom is not None: + raise boom + response = _Response() + response.status_code = status_code + return response + + handler_mod = types.ModuleType("litellm.llms.custom_httpx.http_handler") + handler_mod.HTTPHandler = _Handler + litellm = sys.modules.get("litellm") or types.ModuleType("litellm") + monkeypatch.setitem(sys.modules, "litellm", litellm) + monkeypatch.setitem(sys.modules, "litellm.llms", types.ModuleType("litellm.llms")) + monkeypatch.setitem( + sys.modules, "litellm.llms.custom_httpx", types.ModuleType("litellm.llms.custom_httpx") + ) + monkeypatch.setitem(sys.modules, "litellm.llms.custom_httpx.http_handler", handler_mod) + return calls + + +_PAYLOAD = { + "data": [ + {"id": "moonshotai/kimi-k3", "supported_parameters": ["reasoning", "reasoning_effort"]}, + {"id": "poolside/laguna-s-2.1", "supported_parameters": ["reasoning"]}, + {"id": "qwen/qwen3-max:free", "supported_parameters": ["temperature"]}, + {"id": "malformed-no-params"}, + "not-a-dict", + ] +} + + +def test_env_float_warns_on_unparseable_value(caps, logger, monkeypatch): + """A value the operator deliberately set must not vanish in silence.""" + monkeypatch.setenv("LITELLM_OPENROUTER_CAPABILITY_TIMEOUT", "5s") + assert caps._env_float("LITELLM_OPENROUTER_CAPABILITY_TIMEOUT", 5.0) == 5.0 + assert any("is not a number" in m for m in logger.messages("warning")) + + +def test_env_float_warns_on_out_of_range_value(caps, logger, monkeypatch): + monkeypatch.setenv("LITELLM_OPENROUTER_CAPABILITY_TIMEOUT", "0") + assert caps._env_float("LITELLM_OPENROUTER_CAPABILITY_TIMEOUT", 5.0) == 5.0 + assert any("must be > 0" in m for m in logger.messages("warning")) + + +def test_env_warning_is_deduplicated_not_emitted_per_request(caps, logger, monkeypatch): + """``_ttl_seconds`` runs on every lookup, ahead of the freshness check. + + Warning unconditionally there trades a silent fallback for one WARNING per + proxied request forever, burying the log stream egg's per-call cost + observability reads — the same unbounded-noise failure the fetch-failure + and drop_params dedups exist to avoid. + """ + monkeypatch.setenv("LITELLM_OPENROUTER_CAPABILITY_TTL", "1h") + _install_http_stub(monkeypatch, payload=_PAYLOAD) + for _ in range(200): + caps.get_supported_parameters("moonshotai/kimi-k3") + assert len(logger.messages("warning")) == 1 + + # A *different* bad value is a different mistake and is still reported. + monkeypatch.setenv("LITELLM_OPENROUTER_CAPABILITY_TTL", "-3") + caps.get_supported_parameters("moonshotai/kimi-k3") + assert len(logger.messages("warning")) == 2 + + +def test_env_warning_is_not_lost_to_a_swallowed_emit_failure(caps, monkeypatch): + """``_log`` never propagates, so recording the dedup key before the emit + would let one failure on the *first* call suppress the warning forever: + every later call finds the key already there.""" + recorder = _install_logger(monkeypatch, _FlakyLogger()) + monkeypatch.setenv("LITELLM_OPENROUTER_CAPABILITY_TTL", "1h") + + assert caps._ttl_seconds() == caps.DEFAULT_TTL_SECONDS + assert recorder.messages("warning") == [], "first emit raised, and was swallowed" + + assert caps._ttl_seconds() == caps.DEFAULT_TTL_SECONDS + assert len(recorder.messages("warning")) == 1, "the signal must survive the failure" + + assert caps._ttl_seconds() == caps.DEFAULT_TTL_SECONDS + assert len(recorder.messages("warning")) == 1, "and dedup still holds once it is out" + + +def test_fetch_failure_warning_is_not_lost_to_a_swallowed_emit_failure(caps, monkeypatch): + """Same reasoning as above for the other warn-once latch.""" + recorder = _install_logger(monkeypatch, _FlakyLogger()) + _install_http_stub(monkeypatch, boom=OSError("no route to host")) + + caps.get_supported_parameters("moonshotai/kimi-k3") + assert recorder.messages("warning") == [] + assert caps._WARNED_FETCH_FAILURE is False, "nothing was emitted, so nothing is latched" + + caps._CACHE = None + caps.get_supported_parameters("moonshotai/kimi-k3") + assert len(recorder.messages("warning")) == 1 + assert caps._WARNED_FETCH_FAILURE is True + + +def test_ttl_zero_is_accepted_and_disables_caching(caps, logger, monkeypatch): + """``TTL=0`` is the natural spelling of "always re-fetch", not an error.""" + monkeypatch.setenv("LITELLM_OPENROUTER_CAPABILITY_TTL", "0") + assert caps._ttl_seconds() == 0.0 + assert logger.messages("warning") == [] + + calls = _install_http_stub(monkeypatch, payload=_PAYLOAD) + caps.get_supported_parameters("moonshotai/kimi-k3") + caps.get_supported_parameters("moonshotai/kimi-k3") + assert len(calls) == 2, "TTL=0 must re-fetch on every lookup" + + +def test_ttl_negative_warns_and_falls_back(caps, logger, monkeypatch): + monkeypatch.setenv("LITELLM_OPENROUTER_CAPABILITY_TTL", "-1") + assert caps._ttl_seconds() == caps.DEFAULT_TTL_SECONDS + assert any("must be >= 0" in m for m in logger.messages("warning")) + + +@pytest.mark.parametrize( + "raw,expected", + [ + ("0", False), + ("off", False), + ("no", False), + ("FALSE", False), + ("1", True), + ("true", True), + ("On", True), + ("yes", True), + ], +) +def test_env_flag(caps, logger, monkeypatch, raw, expected): + monkeypatch.setenv("LITELLM_OPENROUTER_CAPABILITY_FETCH", raw) + assert caps._env_flag("LITELLM_OPENROUTER_CAPABILITY_FETCH", True) is expected + assert logger.messages("warning") == [], "a recognised spelling is not a complaint" + + +@pytest.mark.parametrize("raw", ["disabled", "n", "off ish", "2"]) +def test_env_flag_warns_rather_than_inverting_a_near_miss(caps, logger, monkeypatch, raw): + """``not in _FALSY`` read every unrecognized value as *enable*, so a + near-miss disable spelling did not fall back to the default — it inverted + the operator's instruction, silently. The default here is True, so the + observable behaviour is unchanged; what must not be silent is the typo.""" + monkeypatch.setenv("LITELLM_OPENROUTER_CAPABILITY_FETCH", raw) + assert caps._env_flag("LITELLM_OPENROUTER_CAPABILITY_FETCH", True) is True + assert caps._env_flag("LITELLM_OPENROUTER_CAPABILITY_FETCH", False) is False, ( + "an unrecognized value takes the caller's default, not a guess" + ) + (message,) = logger.messages("warning") + assert "is not a boolean" in message + assert raw in message + + # Dedup: this is read on every lookup, so an unconditional warning would be + # one WARNING line per proxied request forever. + caps._env_flag("LITELLM_OPENROUTER_CAPABILITY_FETCH", True) + assert len(logger.messages("warning")) == 1 + + +def test_candidate_slugs_covers_prefix_and_variant_spellings(caps): + assert caps._candidate_slugs("openrouter/qwen/qwen3-max:free") == [ + "openrouter/qwen/qwen3-max:free", + "qwen/qwen3-max:free", + "openrouter/qwen/qwen3-max", + "qwen/qwen3-max", + ] + + +def test_lookup_reads_advertised_parameters(caps, logger, monkeypatch): + _install_http_stub(monkeypatch, payload=_PAYLOAD) + assert caps.get_supported_parameters("openrouter/moonshotai/kimi-k3") == { + "reasoning", + "reasoning_effort", + } + # Advertising `reasoning` but not `reasoning_effort` is a real answer, and + # the caller (Patch 7) must be able to tell the two apart. + assert caps.get_supported_parameters("poolside/laguna-s-2.1") == {"reasoning"} + + +def test_unknown_slug_is_no_opinion_not_a_denial(caps, monkeypatch): + _install_http_stub(monkeypatch, payload=_PAYLOAD) + assert caps.get_supported_parameters("some/model-the-api-never-heard-of") is None + + +def test_malformed_entries_are_skipped_not_fatal(caps, monkeypatch): + _install_http_stub(monkeypatch, payload=_PAYLOAD) + assert caps.get_supported_parameters("malformed-no-params") is None + assert caps.get_supported_parameters("qwen/qwen3-max:free") == {"temperature"} + + +def test_returned_set_is_a_copy(caps, monkeypatch): + """The cache is process-wide; a caller's ``add`` must not poison it.""" + _install_http_stub(monkeypatch, payload=_PAYLOAD) + first = caps.get_supported_parameters("moonshotai/kimi-k3") + first.add("invented_param") + assert "invented_param" not in caps.get_supported_parameters("moonshotai/kimi-k3") + + +def test_result_is_cached_within_the_ttl(caps, monkeypatch): + calls = _install_http_stub(monkeypatch, payload=_PAYLOAD) + for _ in range(5): + caps.get_supported_parameters("moonshotai/kimi-k3") + assert len(calls) == 1 + + +def test_failed_fetch_is_negatively_cached(caps, logger, monkeypatch): + """An offline deployment costs one attempt per TTL, not one per request.""" + calls = _install_http_stub(monkeypatch, boom=OSError("no route to host")) + for _ in range(5): + assert caps.get_supported_parameters("moonshotai/kimi-k3") is None + assert len(calls) == 1 + + +def test_first_fetch_failure_warns_and_repeats_go_to_debug(caps, logger, monkeypatch): + """INFO is litellm's default level, so a debug-only line is invisible in + exactly the case where behaviour silently reverts to the model-cost map.""" + _install_http_stub(monkeypatch, boom=OSError("no route to host")) + caps.get_supported_parameters("moonshotai/kimi-k3") + assert len(logger.messages("warning")) == 1 + assert len(logger.messages("debug")) == 0 + + caps.reset_cache() + # reset_cache clears the warned flag, so re-arm it explicitly to exercise + # the repeat path. + caps._WARNED_FETCH_FAILURE = True + caps.get_supported_parameters("moonshotai/kimi-k3") + assert len(logger.messages("warning")) == 1 + assert len(logger.messages("debug")) == 1 + + +def test_successful_fetch_rearms_the_failure_warning(caps, logger, monkeypatch): + """Latching the flag for the process lifetime would let one blip at pod + startup permanently demote every later outage to debug — the exact silence + the warning exists to break.""" + _install_http_stub(monkeypatch, boom=OSError("no route to host")) + caps.get_supported_parameters("moonshotai/kimi-k3") + assert len(logger.messages("warning")) == 1 + + # Endpoint recovers... + caps._CACHE = None + _install_http_stub(monkeypatch, payload=_PAYLOAD) + assert caps.get_supported_parameters("moonshotai/kimi-k3") is not None + assert caps._WARNED_FETCH_FAILURE is False + + # ...and a genuinely new outage later is visible again. + caps._CACHE = None + _install_http_stub(monkeypatch, boom=OSError("no route to host")) + caps.get_supported_parameters("moonshotai/kimi-k3") + assert len(logger.messages("warning")) == 2 + + +def test_non_200_is_no_opinion(caps, logger, monkeypatch): + _install_http_stub(monkeypatch, payload=_PAYLOAD, status_code=503) + assert caps.get_supported_parameters("moonshotai/kimi-k3") is None + assert any("HTTP 503" in m for m in logger.messages("warning")) + + +def test_payload_without_data_list_is_no_opinion(caps, logger, monkeypatch): + _install_http_stub(monkeypatch, payload={"error": "nope"}) + assert caps.get_supported_parameters("moonshotai/kimi-k3") is None + + +@pytest.mark.parametrize( + "payload", + [ + {"data": []}, + {"data": ["not-a-dict", {"id": 5}, {"id": "x", "supported_parameters": "nope"}]}, + ], + ids=["empty-list", "every-entry-malformed"], +) +def test_a_200_that_parses_to_nothing_is_reported(caps, logger, monkeypatch, payload): + """Otherwise an OpenRouter schema change is indistinguishable from a fetch + that simply had no opinion, and the module goes back to the model-cost map + with nothing said.""" + _install_http_stub(monkeypatch, payload=payload) + assert caps.get_supported_parameters("moonshotai/kimi-k3") is None + assert any("no usable entries" in m for m in logger.messages("warning")) + + +def test_a_200_that_parses_to_nothing_does_not_rearm_the_failure_warning(caps, logger, monkeypatch): + """Re-arming there would report a recovery that did not happen: the point + of the flag is that the *first* line of a real outage is visible.""" + _install_http_stub(monkeypatch, boom=OSError("no route to host")) + caps.get_supported_parameters("moonshotai/kimi-k3") + assert caps._WARNED_FETCH_FAILURE is True + assert len(logger.messages("warning")) == 1 + + caps._CACHE = None + _install_http_stub(monkeypatch, payload={"data": []}) + assert caps.get_supported_parameters("moonshotai/kimi-k3") is None + assert caps._WARNED_FETCH_FAILURE is True, "an empty parse is not a recovery" + assert len(logger.messages("warning")) == 1, "and it is a repeat, so it goes to debug" + assert any("no usable entries" in m for m in logger.messages("debug")) + + +def test_fetch_disabled_skips_the_network_entirely(caps, monkeypatch): + """The kill switch must not merely discard the answer — it must not ask.""" + monkeypatch.setenv("LITELLM_OPENROUTER_CAPABILITY_FETCH", "0") + calls = _install_http_stub(monkeypatch, payload=_PAYLOAD) + assert caps.get_supported_parameters("moonshotai/kimi-k3") is None + assert calls == [] + + +def test_concurrent_refresh_serves_stale_rather_than_queueing(caps, monkeypatch): + """A thread that finds the refresh lock held must not block on HTTP. + + httpx timeouts are per-phase, not total, so queueing behind someone else's + fetch puts unbounded latency on a live request once per TTL. + """ + calls = _install_http_stub(monkeypatch, payload=_PAYLOAD) + caps.get_supported_parameters("moonshotai/kimi-k3") + assert len(calls) == 1 + + # Force the cache stale, then hold the lock as a competing refresher would. + caps._CACHE_STAMP -= caps.DEFAULT_TTL_SECONDS * 2 + caps._LOCK.acquire() + try: + assert caps.get_supported_parameters("moonshotai/kimi-k3") == { + "reasoning", + "reasoning_effort", + } + finally: + caps._LOCK.release() + assert len(calls) == 1, "stale read must not have triggered a second fetch" + + +def test_module_imports_without_litellm(monkeypatch): + """Regression: a module-scope ``from litellm...`` import made this file + unimportable in the repo, which is what left it untested.""" + for name in [n for n in sys.modules if n == "litellm" or n.startswith("litellm.")]: + monkeypatch.delitem(sys.modules, name, raising=False) + assert _load("openrouter_capabilities") is not None + assert _load("drop_params_visibility") is not None + assert _load("anthropic_thinking_policy") is not None + + +# -------------------------------------------------------------------------- +# drop_params_visibility +# -------------------------------------------------------------------------- + + +@pytest.fixture +def dropwarn(): + module = _load("drop_params_visibility") + module._SEEN.clear() + return module + + +def test_warns_once_per_provider_model_paramset(dropwarn, logger): + for _ in range(4): + dropwarn.warn_dropped_params( + {"reasoning_effort": "high"}, "kimi-k3", custom_llm_provider="openrouter" + ) + assert len(logger.messages("warning")) == 1 + dropwarn.warn_dropped_params({"temperature": 0.2}, "kimi-k3", custom_llm_provider="openrouter") + assert len(logger.messages("warning")) == 2 + + +def test_no_dropped_params_is_silent(dropwarn, logger): + dropwarn.warn_dropped_params({}, "kimi-k3", custom_llm_provider="openrouter") + assert logger.messages() == [] + + +def test_message_names_the_params_and_does_not_only_prescribe_a_config_edit(dropwarn, logger): + """The param most often dropped here is one litellm synthesized from the + request, so an unconditional "edit config.yaml" remedy sends the operator + looking for a line that does not exist.""" + dropwarn.warn_dropped_params( + {"reasoning_effort": "high"}, "laguna-s-2.1", custom_llm_provider="openrouter" + ) + (message,) = logger.messages("warning") + assert "reasoning_effort" in message + assert "laguna-s-2.1" in message + assert "openrouter" in message + assert "If they came from" in message + assert "synthesized from the request" in message + + +def test_seen_set_is_bounded_and_keeps_deduplicating(dropwarn, logger, monkeypatch): + """Freezing a full set would stop dedup and re-warn on every request; the + cap must clear instead, keeping both memory and log volume bounded.""" + monkeypatch.setattr(dropwarn, "_MAX_WARNINGS", 3) + for i in range(10): + dropwarn.warn_dropped_params({f"param_{i}": 1}, "m", custom_llm_provider="openrouter") + assert len(dropwarn._SEEN) <= 3 + assert len(logger.messages("warning")) == 10 + + # The key just recorded is still deduplicated — we did not go chatty. + before = len(logger.messages("warning")) + dropwarn.warn_dropped_params({"param_9": 1}, "m", custom_llm_provider="openrouter") + assert len(logger.messages("warning")) == before + + +def test_drop_warning_is_not_lost_to_a_swallowed_emit_failure(dropwarn, monkeypatch): + """The third of this changeset's three warn-once latches, held to the same + rule as the other two. + + ``warn_dropped_params`` swallows a logging failure by design, so recording + the dedup key before the emit would let one failure on the *first* call mute + that route for the life of the process — in the one module whose entire + purpose is to stop a drop being silent.""" + recorder = _install_logger(monkeypatch, _FlakyLogger()) + + dropwarn.warn_dropped_params( + {"reasoning_effort": "high"}, "kimi-k3", custom_llm_provider="openrouter" + ) + assert recorder.messages("warning") == [], "first emit raised, and was swallowed" + assert dropwarn._SEEN == set(), "nothing was emitted, so nothing is deduplicated" + + dropwarn.warn_dropped_params( + {"reasoning_effort": "high"}, "kimi-k3", custom_llm_provider="openrouter" + ) + assert len(recorder.messages("warning")) == 1, "the signal must survive the failure" + + dropwarn.warn_dropped_params( + {"reasoning_effort": "high"}, "kimi-k3", custom_llm_provider="openrouter" + ) + assert len(recorder.messages("warning")) == 1, "and dedup still holds once it is out" + + +def test_diagnostic_never_raises(dropwarn, logger): + """A logging failure must not be able to fail a request.""" + + class _Explode: + def keys(self): + raise RuntimeError("boom") + + def __bool__(self): + return True + + dropwarn.warn_dropped_params(_Explode(), "m", custom_llm_provider="openrouter") + + +# -------------------------------------------------------------------------- +# anthropic_thinking_policy +# -------------------------------------------------------------------------- + + +@pytest.fixture +def policy(monkeypatch): + module = _load("anthropic_thinking_policy") + monkeypatch.delenv(module.ENV_VAR, raising=False) + return module + + +def test_synthesis_is_off_by_default(policy): + """Patch 9's whole point: the adapter's bucketed reasoning_effort is a cap + below the model default on every model egg routes (kimi-k3: 3130 reasoning + tokens with no param vs 340 with ``high``).""" + assert policy.should_synthesize_reasoning_effort() is False + + +@pytest.mark.parametrize("raw", ["1", "true", "TRUE", "yes", "on"]) +def test_synthesis_opt_in(policy, monkeypatch, raw): + monkeypatch.setenv(policy.ENV_VAR, raw) + assert policy.should_synthesize_reasoning_effort() is True + + +@pytest.mark.parametrize("raw", ["0", "false", "off", "no", "", "maybe"]) +def test_synthesis_stays_off_for_anything_else(policy, monkeypatch, raw): + monkeypatch.setenv(policy.ENV_VAR, raw) + assert policy.should_synthesize_reasoning_effort() is False + + +@pytest.mark.parametrize("raw", ["0", "false", "OFF", "no", ""]) +def test_recognised_off_spellings_do_not_complain(policy, logger, monkeypatch, raw): + monkeypatch.setenv(policy.ENV_VAR, raw) + assert policy.should_synthesize_reasoning_effort() is False + assert logger.messages("warning") == [] + + +@pytest.mark.parametrize("raw", ["enabled", "y", "maybe", "2"]) +def test_unrecognised_value_warns_once(policy, logger, monkeypatch, raw): + """False is also the default, so without a warning an operator who typed + ``=enabled`` cannot tell "ignored" from "working as configured" — on the + knob with the ~9x measured effect on reasoning depth.""" + monkeypatch.setenv(policy.ENV_VAR, raw) + assert policy.should_synthesize_reasoning_effort() is False + (message,) = logger.messages("warning") + assert policy.ENV_VAR in message + assert raw in message + + # Read once per translated request, so the complaint must not repeat. + for _ in range(4): + policy.should_synthesize_reasoning_effort() + assert len(logger.messages("warning")) == 1 + + +def test_unrecognised_value_warning_survives_a_swallowed_emit_failure(policy, monkeypatch): + """Same latch discipline as the other three warn-once sites: the key is + recorded only once the emit did not raise, so a logger that is not yet in + place on the first request cannot mute the complaint permanently.""" + recorder = _install_logger(monkeypatch, _FlakyLogger()) + monkeypatch.setenv(policy.ENV_VAR, "enabled") + + assert policy.should_synthesize_reasoning_effort() is False + assert recorder.messages("warning") == [], "first emit raised, and was swallowed" + assert policy._WARNED_VALUES == set(), "nothing was emitted, so nothing is deduplicated" + + assert policy.should_synthesize_reasoning_effort() is False + assert len(recorder.messages("warning")) == 1, "the signal must survive the failure" + + assert policy.should_synthesize_reasoning_effort() is False + assert len(recorder.messages("warning")) == 1, "and dedup still holds once it is out" diff --git a/tests/config/test_patch_litellm_cache.py b/tests/config/test_patch_litellm_cache.py index a469c099a..7079440aa 100644 --- a/tests/config/test_patch_litellm_cache.py +++ b/tests/config/test_patch_litellm_cache.py @@ -103,10 +103,162 @@ def test_missing_needle_fails_loud(tmp_path): def test_missing_file_fails_loud(tmp_path): - """A missing target file must raise SystemExit, not pass silently.""" - # Empty root — none of the litellm paths exist. - with pytest.raises(SystemExit): + """A missing target file must raise SystemExit from ``_apply``. + + ``_patch_root`` installs ``NEW_MODULES`` before it runs ``PATCHES``, so an + empty root aborts in ``_install_module``'s "destination package missing" + branch and never reaches ``_apply`` at all — the assertion would pass while + testing something else entirely. Create the module destinations (but none + of the patch targets) so the failure under test is the one named.""" + for spec in plc.NEW_MODULES: + (tmp_path / spec["dest"]).parent.mkdir(parents=True, exist_ok=True) + + with pytest.raises(SystemExit) as excinfo: plc._patch_root(str(tmp_path)) + assert "file not found" in str(excinfo.value), "aborted before reaching _apply" + + +def test_malformed_replacement_fails_at_build_time(tmp_path): + """A replacement with wrong indentation applies *cleanly*. + + The needle check only proves we found the right spot, not that what we put + there is valid Python. Without this guard the broken result passes the + build, ships in the image, and surfaces as a pod CrashLoopBackOff when + litellm imports the file — the one place nothing is watching.""" + path = tmp_path / "victim.py" + stock = "def f():\n return 1\n" + path.write_text(stock) + + with pytest.raises(SystemExit) as excinfo: + plc._apply( + str(path), + present="# egg marker", + needle=" return 1\n", + replacement="# egg marker\nreturn 1\n", + label="Patch X", + ) + assert "does not parse" in str(excinfo.value) + assert path.read_text() == stock, "a rejected patch must not leave a broken file behind" + + +def test_wellformed_replacement_still_applies(tmp_path): + """The parse check must not reject a correct patch.""" + path = tmp_path / "victim.py" + path.write_text("def f():\n return 1\n") + plc._apply( + str(path), + present="# egg marker", + needle=" return 1\n", + replacement=" # egg marker\n return 2\n", + label="Patch X", + ) + assert "return 2" in path.read_text() + + +def test_unparseable_input_is_not_blamed_on_the_patch(tmp_path): + """The check asserts the patch did not break the file, not that the file + was ever valid — the concatenated-needle fixtures here never are, and + holding them to it would test the fixture rather than the patch.""" + path = tmp_path / "victim.py" + path.write_text("this is (not python\n") + plc._apply( + str(path), + present="# egg marker", + needle="not python", + replacement="# egg marker\nstill not python", + label="Patch X", + ) + assert "# egg marker" in path.read_text() + + +def test_installed_module_payload_must_parse(tmp_path, monkeypatch): + """Same guard on the other write path: a truncated COPY or a half-written + staged file would install without complaint and only fail at import.""" + spec = dict(plc.NEW_MODULES[0]) + (tmp_path / spec["dest"]).parent.mkdir(parents=True, exist_ok=True) + + broken = tmp_path / "staged" + broken.mkdir() + (broken / spec["source"]).write_text("def truncated(\n") + monkeypatch.setattr(plc, "_module_source", lambda name, label: str(broken / name)) + + with pytest.raises(SystemExit) as excinfo: + plc._install_module(str(tmp_path), spec) + assert "does not parse" in str(excinfo.value) + assert not (tmp_path / spec["dest"]).exists() + + +def test_installed_module_parse_error_points_at_the_real_line(tmp_path, monkeypatch): + """The reported line must be the one the operator will open. + + The provenance header is prepended before the file is written, so parsing + the *payload* shifts every line by one and sends whoever reads the build log + to the wrong place. This path also has no replacement in it — the staged + source itself is broken — so ``_apply``'s vocabulary would misdescribe it.""" + spec = dict(plc.NEW_MODULES[0]) + (tmp_path / spec["dest"]).parent.mkdir(parents=True, exist_ok=True) + + broken = tmp_path / "staged" + broken.mkdir() + # Syntax error deliberately on line 5, well clear of the off-by-one. + (broken / spec["source"]).write_text('"""Doc."""\n\nimport os\n\ndef truncated(\n') + monkeypatch.setattr(plc, "_module_source", lambda name, label: str(broken / name)) + + with pytest.raises(SystemExit) as excinfo: + plc._install_module(str(tmp_path), spec) + message = str(excinfo.value) + assert f"{spec['source']}:5:" in message, f"expected the real line, got: {message}" + assert "replacement" not in message, "no replacement is involved on the install path" + assert "staged module source does not parse" in message + + +def test_new_module_refuses_to_clobber_a_foreign_file(tmp_path): + """Every other operation in this script is fail-loud on drift; so is this. + + The guard must read the file *on disk*, not our own ``NEW_MODULES`` + literal: keying it on the ``_egg_`` prefix of a hardcoded destination could + only ever fire if someone edited this script, which is a lint of its own + constants and not upstream-drift detection. A real file at a real + destination, lacking egg's provenance header, must abort the build.""" + _build_fixture_root(tmp_path) + spec = plc.NEW_MODULES[0] + dest = tmp_path / spec["dest"] + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text("# upstream took this name\n") + + # An unrecognised file at our own destination aborts — no synthetic spec + # needed, which is the point: this path is reachable in production. + with pytest.raises(SystemExit) as excinfo: + plc._install_module(str(tmp_path), spec) + assert "refusing to overwrite" in str(excinfo.value) + assert dest.read_text() == "# upstream took this name\n", "clobbered anyway" + + +def test_new_module_overwrites_a_stale_egg_install(tmp_path): + """A previous image layer's copy carries the header, so it is ours to + replace — the guard must distinguish stale-ours from foreign.""" + _build_fixture_root(tmp_path) + spec = plc.NEW_MODULES[0] + dest = tmp_path / spec["dest"] + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(plc.EGG_MODULE_HEADER + "# an older revision of this module\n") + + plc._install_module(str(tmp_path), spec) + assert "an older revision" not in dest.read_text() + assert dest.read_text().startswith(plc.EGG_MODULE_HEADER) + + +def test_new_module_destinations_carry_the_egg_prefix(tmp_path): + """The prefix keeps upstream from ever taking one of our paths. Enforced at + install time so a future ``NEW_MODULES`` entry cannot quietly drop it.""" + for spec in plc.NEW_MODULES: + assert Path(spec["dest"]).name.startswith(plc.EGG_MODULE_PREFIX), spec["label"] + + _build_fixture_root(tmp_path) + unprefixed = dict(plc.NEW_MODULES[0], dest="llms/openrouter/capabilities.py") + with pytest.raises(SystemExit) as excinfo: + plc._install_module(str(tmp_path), unprefixed) + assert "must be prefixed" in str(excinfo.value) def test_patch4_needle_anchors_on_content_block_function(tmp_path): @@ -119,7 +271,10 @@ def test_patch4_needle_anchors_on_content_block_function(tmp_path): fixture containing a *second* (sibling-function-style) bare ``thinking_blocks`` elif — preceded by a ``tool_calls`` block, not the text elif — must be left untouched.""" - patch4 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 4/6")) + # Matched on "Patch 4/" rather than the full label so adding a patch (and + # renumbering the denominators) does not silently turn this into a + # StopIteration instead of a real assertion. + patch4 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 4/")) # Sibling function: bare thinking_blocks elif preceded by a tool_calls # branch (mirrors _translate_streaming_openai_chunk_to_anthropic). It must @@ -155,3 +310,247 @@ def test_patch4_needle_anchors_on_content_block_function(tmp_path): start = result.index("SENTINEL_SIBLING_BEGIN") end = result.index("SENTINEL_SIBLING_END") + len("SENTINEL_SIBLING_END\n") assert result[start:end] == sibling + + +def test_new_modules_are_installed_into_each_root(tmp_path): + """``NEW_MODULES`` drops whole files that have no stock counterpart. + + Patch 7's gate imports ``litellm.llms.openrouter._egg_capabilities``, so if + the module install silently no-ops the patched gate raises ImportError on + every request — caught by its ``except Exception``, which would put us right + back at the silent-drop behaviour the patch exists to remove.""" + _build_fixture_root(tmp_path) + plc._patch_root(str(tmp_path)) + + for spec in plc.NEW_MODULES: + dest = tmp_path / spec["dest"] + assert dest.is_file(), f"{spec['label']}: not installed" + source = Path(plc._module_source(spec["source"], spec["label"])) + installed = dest.read_text() + assert installed.startswith(plc.EGG_MODULE_HEADER), f"{spec['label']}: no provenance" + assert installed == plc.EGG_MODULE_HEADER + source.read_text(), ( + f"{spec['label']}: content drift" + ) + + +def test_new_module_install_is_idempotent(tmp_path): + _build_fixture_root(tmp_path) + plc._patch_root(str(tmp_path)) + first = {spec["dest"]: (tmp_path / spec["dest"]).read_text() for spec in plc.NEW_MODULES} + plc._patch_root(str(tmp_path)) + for dest, content in first.items(): + assert (tmp_path / dest).read_text() == content + + +def test_missing_staged_module_fails_loud(): + """A missing staged file must abort the build, not skip the install.""" + with pytest.raises(SystemExit): + plc._module_source("definitely-not-a-real-module.py", "test label") + + +def test_patch7_gate_is_additive_not_substitutive(tmp_path): + """Patch 7 must UNION the live answer with the stock model-map answer. + + OpenRouter's ``supported_parameters`` under-reports ``reasoning_effort`` + (deepseek-r1 advertises only ``reasoning``), so letting live data win + outright would drop a knob the map correctly allows — trading one silent + drop for another. The stock ``supports_reasoning`` branch must therefore + survive the patch.""" + patch7 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 7/")) + + # The stock gate, verbatim from 1.86.2, following the needle. Applying the + # patch to this and asserting on the RESULT tests the invariant the + # docstring claims; substring checks against the replacement string alone + # would pass on a patch that deleted the branch entirely. + stock_gate = ( + " if litellm.supports_reasoning(\n" + ' model=model, custom_llm_provider="openrouter"\n' + " ) or litellm.supports_reasoning(model=model):\n" + ' supported_params.append("reasoning_effort")\n' + ' supported_params.append("thinking")\n' + " except Exception:\n" + " pass\n" + " return list(dict.fromkeys(supported_params))\n" + ) + target = tmp_path / patch7["file"] + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(patch7["needle"] + stock_gate) + + plc._apply( + str(target), + present=patch7["present"], + needle=patch7["needle"], + replacement=patch7["replacement"], + label=patch7["label"], + ) + result = target.read_text() + + # The live lookup is consulted... + assert "_egg_capabilities" in result + assert '"reasoning_effort" in _advertised' in result + # ...and the stock model-map branch is still there, untouched, downstream + # of it: the live answer can ADD a knob, never withhold one. + assert stock_gate in result, "patch 7 must not replace the stock model-map branch" + # The inserted block precedes it and does not return out of the function. + inserted = result[: result.index(stock_gate)] + assert "supported_params.append" in inserted + assert "\n return" not in inserted, "patch 7 must not short-circuit the stock branch" + # Failures in the lookup must never propagate into a request. + assert "except Exception:" in inserted + # Only reasoning_effort is admitted. OpenRouter's `reasoning` field is a + # different wire shape from Anthropic's `thinking`, not a spelling of it. + assert '"thinking"' not in inserted + + +# The tail of ``_translate_thinking_to_openai`` as it stands in 1.86.2, +# verbatim, from the Claude branch through the assignments. Patch 9's needle is +# a slice of this; keeping the surrounding lines lets the test assert what the +# gate sits between, which is the whole invariant. +_STOCK_THINKING_TAIL_HEAD = ( + ' model = new_kwargs.get("model", "")\n' + " if self.is_anthropic_claude_model(model):\n" + ' new_kwargs["thinking"] = thinking # type: ignore\n' + " return\n" + "\n" + " reasoning_effort = self.translate_anthropic_thinking_to_reasoning_effort(\n" + " cast(Dict[str, Any], thinking)\n" + " )\n" + " if not reasoning_effort:\n" + " return\n" + "\n" +) +_STOCK_THINKING_TAIL_FOOT = ( + " auto_summary = is_reasoning_auto_summary_enabled()\n" + " if summary:\n" + ' new_kwargs["reasoning_effort"] = cast(\n' + " Any,\n" + " {\n" + ' "effort": reasoning_effort,\n' + ' "summary": summary,\n' + " },\n" + " )\n" + " elif auto_summary:\n" + ' new_kwargs["reasoning_effort"] = cast(\n' + " Any,\n" + " {\n" + ' "effort": reasoning_effort,\n' + ' "summary": "detailed",\n' + " },\n" + " )\n" + " else:\n" + ' new_kwargs["reasoning_effort"] = reasoning_effort\n' +) + + +def test_patch9_gates_synthesis_without_touching_the_claude_branch(tmp_path): + """Patch 9 must stop the adapter manufacturing a ``reasoning_effort``. + + On ``/v1/messages`` litellm derives ``reasoning_effort`` from the caller's + ``thinking`` budget for every non-Claude model. That derived value is a cap + BELOW the model default (#3624: kimi-k3 means 3130 reasoning tokens with no + param vs 340 with ``high``), so Patch 7 alone would silently shallow every + agent turn. The Claude branch, which forwards ``thinking`` unchanged, must + be unaffected — and so must the assignments, which the gate returns before + rather than rewriting.""" + patch9 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 9/")) + + target = tmp_path / patch9["file"] + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text( + _STOCK_THINKING_TAIL_HEAD + patch9["needle"] + _STOCK_THINKING_TAIL_FOOT, + ) + + plc._apply( + str(target), + present=patch9["present"], + needle=patch9["needle"], + replacement=patch9["replacement"], + label=patch9["label"], + ) + result = target.read_text() + + assert _STOCK_THINKING_TAIL_HEAD in result, "patch 9 must leave the Claude path alone" + assert _STOCK_THINKING_TAIL_FOOT in result, "patch 9 must not rewrite the assignments" + + # The gate sits after the derivation and returns (rather than falling + # through) when synthesis is off, so nothing is assigned. + gate = result[result.index(patch9["present"]) : result.index(_STOCK_THINKING_TAIL_FOOT)] + assert "_egg_anthropic_thinking_policy" in gate + assert "if not _egg_synthesize:\n return\n" in gate + # A missing policy module must fall back to the policy's OWN default (off), + # not to stock behaviour — otherwise the failure mode is the regression. + assert "_egg_synthesize = False" in gate + + +def test_patch9_does_not_suppress_an_explicitly_requested_effort(tmp_path): + """The gate's scope is the *derived* bucket, not the whole function. + + Stock reaches the assignment two ways: from ``budget_tokens`` (a ceiling + nobody asked for) or from ``output_config.effort`` on an adaptive request, + which is the caller saying outright what they want. Suppressing the second + would be discarding an instruction, not declining to invent one — a + different change from the one the patch documents.""" + patch9 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 9/")) + + target = tmp_path / patch9["file"] + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text( + _STOCK_THINKING_TAIL_HEAD + patch9["needle"] + _STOCK_THINKING_TAIL_FOOT, + ) + plc._apply( + str(target), + present=patch9["present"], + needle=patch9["needle"], + replacement=patch9["replacement"], + label=patch9["label"], + ) + result = target.read_text() + + # The adaptive override still runs, and it is what exempts the request. + override = ' reasoning_effort = output_config["effort"]\n' + assert override in result, "the adaptive override must survive the patch" + assert result.index(override) < result.index(patch9["present"]), ( + "the gate must sit after the override, not before it" + ) + assert f"{override} _egg_effort_is_explicit = True\n" in result + assert "if not _egg_effort_is_explicit:\n" in result + + +def test_patch8_needle_disambiguates_the_two_drop_sites(tmp_path): + """litellm 1.86.2 has TWO ``drop_params`` branches in utils.py. + + They share the identical ``if litellm.drop_params is True or (...)`` + condition; only what follows differs (a bare ``pass`` in the embeddings + path, the pop loop in ``get_optional_params``). Matching on the shared + condition would patch whichever came first — the same needle-uniqueness + trap as Patch 4. A fixture carrying the sibling ``pass`` form first must be + left untouched.""" + patch8 = next(p for p in plc.PATCHES if p["label"].startswith("Patch 8/")) + + sibling = ( + "SENTINEL_PASS_SITE_BEGIN\n" + " if litellm.drop_params is True or (\n" + " drop_params is not None and drop_params is True\n" + " ):\n" + " pass\n" + "SENTINEL_PASS_SITE_END\n" + ) + fixture = sibling + "\n" + patch8["needle"] + + target = tmp_path / patch8["file"] + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(fixture) + + plc._apply( + str(target), + present=patch8["present"], + needle=patch8["needle"], + replacement=patch8["replacement"], + label=patch8["label"], + ) + result = target.read_text() + + assert result.count(patch8["present"]) == 1 + start = result.index("SENTINEL_PASS_SITE_BEGIN") + end = result.index("SENTINEL_PASS_SITE_END") + len("SENTINEL_PASS_SITE_END\n") + assert result[start:end] == sibling, "patch 8 rewrote the embeddings-path drop site"