diff --git a/litellm/llms/openrouter/capabilities.py b/litellm/llms/openrouter/capabilities.py index 9d26b2538026..ff82dadfcf34 100644 --- a/litellm/llms/openrouter/capabilities.py +++ b/litellm/llms/openrouter/capabilities.py @@ -1,18 +1,51 @@ -"""Live capability lookup for OpenRouter models. - -LiteLLM decides which optional params a provider accepts by consulting the -bundled model-cost map. For OpenRouter that map is wrong by construction: -OpenRouter publishes new slugs continuously, the bundled map lags behind, and -``litellm.supports_reasoning`` answers ``False`` for anything it has not caught -up to yet. Because ``OpenrouterConfig.get_supported_openai_params`` uses that -answer as a bare gate, the failure is closed and silent: a ``reasoning_effort`` -set on a current model is discarded before the request body is built, with no -exception and (before the ``drop_params`` warning) no log line. - -OpenRouter publishes the authoritative answer itself. ``GET /api/v1/models`` -returns every model with a ``supported_parameters`` list and requires no API -key. This module reads that, caches it for the life of the process, and hands -callers a set of parameter names for a given slug. +"""Live capability and pricing lookup for OpenRouter models. + +LiteLLM decides which optional params a provider accepts, and what a call cost, +by consulting the bundled model-cost map. For OpenRouter that map is wrong by +construction: OpenRouter publishes new slugs continuously, the bundled map lags +behind, and it answers for neither question on a slug it has not caught up to. +Two distinct silent failures follow from the one root cause: + +* **Parameters.** ``litellm.supports_reasoning`` answers ``False`` for an + unmapped slug, and ``OpenrouterConfig.get_supported_openai_params`` uses that + answer as a bare gate, so the failure is closed: a ``reasoning_effort`` set + on a current model is discarded before the request body is built, with no + exception and (before the ``drop_params`` warning) no log line. +* **Pricing.** ``_get_model_info_helper`` raises "This model isn't mapped yet" + for an unmapped slug, so ``response_cost`` is never computed and every cost + consumer downstream of it reports nothing. For a proxy routing current + OpenRouter slugs this is not an edge case: it is every call. + +OpenRouter publishes the authoritative answer to both itself. ``GET +/api/v1/models`` returns every model with a ``supported_parameters`` list and a +``pricing`` block, and requires no API key. This module reads that once, caches +it for the life of the process, and hands callers either the parameter-name set +(``get_supported_parameters``) or a LiteLLM-shaped model-cost entry +(``get_model_cost_entry``) for a given slug. + +Two deliberate limits on the pricing half, both about not trading a known +unknown for a confident wrong number: + +* **Cost fields only.** The entry carries the rate card, ``litellm_provider`` + and ``mode`` — not context lengths, not ``supports_*`` flags. Registering a + model's capabilities through this door would change behaviour well beyond + cost: ``supports_reasoning: true`` alone would make + ``get_supported_openai_params`` admit ``thinking``, which is a different wire + shape from OpenRouter's ``reasoning`` and would be forwarded verbatim to a + provider that does not accept it. ``get_supported_parameters`` remains the + only path by which a parameter becomes admissible here, and the caller in + ``OpenrouterConfig`` admits exactly ``reasoning_effort``. +* **Tiered rate cards are declined outright.** OpenRouter expresses a + long-context surcharge as ``pricing.overrides`` — a list keyed by arbitrary + ``min_prompt_tokens`` (32000 and 128000 on qwen3-max, for instance). LiteLLM + has slots for three fixed thresholds and drops the rest, so a faithful + translation does not exist in general. Registering the base tier anyway would + under-report by 2-2.5x on exactly the long-prompt turns that make up agent + traffic, and it would do so silently, under a field name an operator would + reasonably use to pick a model. So a model with any override is left + unpriced, its ``cost_estimated`` stays null, and the reason is logged once. + The provider-reported ``usage.cost`` that ``ChunkProcessor.calculate_usage`` + carries through is the number to read for those models, and it is exact. Design constraints, because this sits behind a hot, synchronous code path: @@ -23,19 +56,46 @@ * **Fetch at most once per TTL**, including after a failure. A negative cache entry keeps an offline or firewalled deployment from attempting a network call on every single request. -* **One fetch, not N.** A lock serialises refreshes so concurrent first - requests do not stampede the endpoint. -* **Opt-out.** ``LITELLM_OPENROUTER_CAPABILITY_FETCH=0`` disables the lookup - entirely and restores the previous model-map-only behaviour. +* **One fetch, not N**, and never a queue. A lock serialises refreshes so + concurrent requests do not stampede the endpoint; a thread that finds the + lock held serves the stale cache rather than waiting behind a network call. +* **Importable without litellm.** ``verbose_logger`` is imported inside + ``_log`` rather than at module scope. This file is maintained as a twin of + ``config/litellm/openrouter_capabilities.py`` in jwbron/egg, which is unit + tested in a tree where litellm is not installed. The two are kept + line-for-line comparable — they differ only in formatting width (this repo + wraps at 120, that one at 100) and in the couple of docstring references + that name the other project — so a sync is a copy and a reformat rather than + a merge. Divergence beyond that is drift, and has bitten before. + +Operator knobs (all optional): + +* ``LITELLM_OPENROUTER_CAPABILITY_FETCH=0`` disables the lookup entirely and + restores the previous model-map-only behaviour — parameters and pricing + both, since one fetch serves both. +* ``LITELLM_OPENROUTER_PRICING=0`` disables only the pricing half, leaving the + parameter lookup running. For an operator who wants LiteLLM's bundled map to + be the sole authority on cost while keeping the parameter lookup. +* ``LITELLM_OPENROUTER_CAPABILITY_TTL`` seconds between refreshes + (default 3600). ``0`` disables caching and re-fetches on every lookup — + a debugging aid, not a production setting. +* ``LITELLM_OPENROUTER_CAPABILITY_TIMEOUT`` per-phase HTTP timeout in seconds + (default 5). + +An unparseable, unrecognized or out-of-range value for any of these is logged +and ignored rather than silently swallowed: an operator reaching for these vars +is very likely already debugging something. That covers a near miss on the +boolean too — ``FETCH=disabled`` is neither a recognised on nor off spelling, so +it warns and takes the default instead of being read as the opposite of what was +meant. """ import json +import math import os import threading import time -from litellm._logging import verbose_logger - OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models" # The endpoint is served without authentication, so no key is read here on @@ -45,39 +105,256 @@ DEFAULT_TTL_SECONDS = 3600.0 DEFAULT_TIMEOUT_SECONDS = 5.0 -# Cache of slug -> supported parameter names. ``None`` means "not populated". -# An empty dict is a real, meaningful state: it records a failed fetch, so the -# negative cache below can suppress retries without conflating "we asked and -# got nothing" with "we never asked". -_CACHE: dict[str, set[str]] | None = None +# Cache of slug -> record. ``None`` means "not populated". An empty dict is a +# real, meaningful state: it records a failed fetch, so the negative cache below +# can suppress retries without conflating "we asked and got nothing" with "we +# never asked". +# +# Each record is ``{"id": str, "parameters": set[str] | None, "cost_entry": dict +# | None, "declined_thresholds": tuple[int, ...]}``. Both payloads are optional and +# independent: OpenRouter publishes entries carrying one and not the other, and +# a slug that answers for parameters but not pricing (or the reverse) is a real +# answer for the half it has rather than a reason to drop the model. The +# thresholds are kept only so the declined-pricing warning can name them. +_CACHE: dict[str, dict] | None = None _CACHE_STAMP: float = 0.0 _LOCK = threading.Lock() +# Whether a fetch failure has already been reported at warning level. The first +# failure is the one worth seeing (behaviour has silently reverted to the +# model-cost map); repeats are noise. Cleared again by a successful fetch, so a +# blip at startup does not permanently mute a real outage hours later. +_WARNED_FETCH_FAILURE = False + +# Slugs whose tiered rate card has already been reported as declined. Bounded by +# the roster size, and reset with the cache so a pricing change upstream is +# reported again rather than muted for the pod's lifetime. +_WARNED_DECLINED_PRICING: set[str] = set() + +# Env-var complaints already emitted, keyed by ``(name, raw value)``. +# ``_env_float`` is reached from ``_ttl_seconds`` on *every* lookup, ahead of +# the freshness check, so an unconditional warning there is one WARNING line per +# proxied request forever — a misconfigured TTL would bury the same log stream +# the proxy's own per-call diagnostics land in. Bounded by construction: the environment +# does not change mid-process, so this holds at most one entry per knob. +_WARNED_ENV: set[tuple[str, str]] = set() + + +def _log(level: str, message: str, *args: object) -> bool: + """Log via litellm's ``verbose_logger``, deferring the import. + + Kept out of module scope so this file stays importable where litellm is + not installed. Never raises: a diagnostic must not be able to break a + request, and must not stop the capability lookup either. + + Returns whether the call completed without raising — *not* whether a record + reached a handler, since ``verbose_logger.debug(...)`` on a logger set to + INFO also returns normally. The distinction does not matter to either + caller: what the warn-once bookkeeping below must not do is mark a line + "already warned" when the attempt *raised* (litellm's logger not yet in + place, say), because swallowing the exception would then also swallow the + signal. Level filtering is the operator's own choice and is not a lost + signal. + """ + try: + from litellm._logging import verbose_logger + + getattr(verbose_logger, level)(message, *args) + return True + except Exception: # noqa: BLE001 - diagnostics must never break a request + return False + + +def _warn_env_once(name: str, raw: str, message: str, *args: object) -> None: + """Warn about a bad env value once per distinct ``(name, value)``. + + Same discipline as ``_log_fetch_failure`` here and ``_SEEN`` in + ``drop_params_visibility``: the first occurrence is the one that carries + information, and this one sits on the request path. + """ + key = (name, raw) + if key in _WARNED_ENV: + return + # Recorded only once the emit did not raise. ``_log`` deliberately never + # propagates, so recording first would mean a failure on the *first* call — + # litellm's logger not yet in place, say — permanently suppresses the + # warning: every later call would find the key already there. + if _log("warning", message, *args): + _WARNED_ENV.add(key) + + +_TRUTHY = ("1", "true", "yes", "on") +_FALSY = ("0", "false", "no", "off") + def _env_flag(name: str, default: bool) -> bool: + """Read a boolean env var, warning rather than guessing at a near miss. + + A bare ``raw not in _FALSY`` reads every unrecognized value as *enable*, so + a near-miss disable spelling (``=disabled``, ``=n``) does not fall back to + the default — it inverts the operator's instruction, silently. Anything + matching neither list warns once and takes the default, which is the same + discipline ``_env_float`` applies and the behaviour this module's docstring + promises for all three knobs. + """ raw = os.getenv(name) if raw is None: return default - return raw.strip().lower() not in ("0", "false", "no", "off") - - -def _env_float(name: str, default: float) -> float: + value = raw.strip().lower() + if value in _TRUTHY: + return True + if value in _FALSY: + return False + _warn_env_once( + name, + raw, + "openrouter capabilities: %s=%r is not a boolean (expected one of %s or %s); using the default %s", + name, + raw, + ", ".join(_TRUTHY), + ", ".join(_FALSY), + default, + ) + return default + + +def _env_float(name: str, default: float, *, allow_zero: bool = False) -> float: + """Read a float env var, warning rather than silently falling back. + + A deliberately-set-but-unusable value (``TTL=0`` when zero is not allowed, + ``TIMEOUT=5s`` pasted with a unit suffix) previously became the default + with no signal at all, so an operator could set a knob, restart, observe + nothing change, and have no way to learn the proxy ignored them. + + The warning is deduplicated per ``(name, value)``: this is called from + ``_ttl_seconds`` on the per-request path, so warning unconditionally would + trade a silent fallback for an unbounded log flood. + """ raw = os.getenv(name) if raw is None: return default try: value = float(raw) except ValueError: + _warn_env_once( + name, + raw, + "openrouter capabilities: %s=%r is not a number; using the default %s", + name, + raw, + default, + ) + return default + if value < 0 or (value == 0 and not allow_zero): + _warn_env_once( + name, + raw, + "openrouter capabilities: %s=%r must be %s; using the default %s", + name, + raw, + ">= 0" if allow_zero else "> 0", + default, + ) return default - return value if value > 0 else default + return value def _ttl_seconds() -> float: - return _env_float("LITELLM_OPENROUTER_CAPABILITY_TTL", DEFAULT_TTL_SECONDS) + # Zero is allowed and meaningful: it is the natural spelling of "never + # cache, always re-fetch", and rejecting it was the least discoverable of + # this module's silent fallbacks. It makes every lookup a network call, so + # it is a debugging setting rather than a production one. + return _env_float("LITELLM_OPENROUTER_CAPABILITY_TTL", DEFAULT_TTL_SECONDS, allow_zero=True) + + +# OpenRouter pricing key -> LiteLLM model-cost key. Every value is USD per +# token, published as a decimal *string*, which is why ``_price`` parses rather +# than casts. +# +# ``input_cache_write`` maps to LiteLLM's ``cache_creation_*`` spelling: the two +# name the same thing (what you pay to put a prefix in the cache), and getting +# this pair backwards would silently price cache writes at the read rate, which +# on these routes is a ~5x understatement of the turn that costs the most. +_PRICE_KEYS = ( + ("prompt", "input_cost_per_token"), + ("completion", "output_cost_per_token"), + ("input_cache_read", "cache_read_input_token_cost"), + ("input_cache_write", "cache_creation_input_token_cost"), +) + +# The two rates without which an entry cannot price a chat turn at all. A +# missing cache rate degrades the estimate; a missing prompt or completion rate +# would make it meaningless, so the entry is declined instead. +_REQUIRED_PRICE_KEYS = ("prompt", "completion") + + +def _price(raw: object) -> float | None: + """Parse one published rate. ``None`` when it is not a usable number. + + Zero is usable and is kept: a ``:free`` variant really is priced at zero, + and filtering the resulting zero estimate belongs to whoever reads it, not + here. Negative and non-finite values are refused — neither is a rate, and an + ``inf`` would propagate into every downstream total and serialize as a + non-standard JSON token that makes the emitting log line unparseable. + """ + if isinstance(raw, bool) or raw is None: + return None + try: + value = float(raw) + except (TypeError, ValueError): + return None + if value < 0 or not math.isfinite(value): + return None + return value + + +def _cost_entry(model_id: str, pricing: object) -> tuple[dict | None, tuple[int, ...] | None]: + """Translate an OpenRouter ``pricing`` block to a LiteLLM model-cost entry. + + Returns ``(entry, declined_thresholds)``. ``entry`` is None when no faithful + translation exists; ``declined_thresholds`` is None unless the reason was a + tiered rate card, in which case it names the ``min_prompt_tokens`` + boundaries — the module docstring has the reasoning; in short, LiteLLM + cannot express an arbitrary tier boundary and a base-tier-only entry would + under-report long-prompt turns silently. + The two are separate returns rather than one truthiness test because a + tiered card whose boundaries do not parse is still a tiered card: an empty + tuple must keep meaning "declined for tiering, boundaries unknown", so the + operator still gets told why the model is unpriced instead of only the + models whose overrides happened to be well-formed. + + ``key`` is set to the OpenRouter slug rather than left to the caller: it is + what ``get_model_info`` reports as the entry's identity, and an operator + reading ``/model/info`` should see the slug the rate actually came from. + """ + if not isinstance(pricing, dict): + return None, None + + overrides = pricing.get("overrides") + if isinstance(overrides, list) and overrides: + thresholds = [] + for override in overrides: + boundary = override.get("min_prompt_tokens") if isinstance(override, dict) else None + if isinstance(boundary, int) and not isinstance(boundary, bool): + thresholds.append(boundary) + return None, tuple(sorted(thresholds)) + + if any(_price(pricing.get(key)) is None for key in _REQUIRED_PRICE_KEYS): + return None, None + + entry: dict = {"key": model_id, "litellm_provider": "openrouter", "mode": "chat"} + for published, litellm_key in _PRICE_KEYS: + value = _price(pricing.get(published)) + if value is not None: + entry[litellm_key] = value + return entry, None + + +def _fetch() -> dict[str, dict]: + """Fetch the model list. Returns ``{}`` on any failure.""" + global _WARNED_FETCH_FAILURE -def _fetch() -> dict[str, set[str]]: - """Fetch the model list. Returns ``{}` on any failure.""" # Imported here rather than at module scope: http_handler pulls in a large # slice of litellm, and this module is imported from a transformation that # is itself imported during litellm's own startup. @@ -87,16 +364,16 @@ def _fetch() -> dict[str, set[str]]: try: response = HTTPHandler(timeout=timeout).get(OPENROUTER_MODELS_URL) if response.status_code != 200: - verbose_logger.debug( - "openrouter capabilities: %s returned HTTP %s; falling back to the bundled model-cost map", + _log_fetch_failure( + "%s returned HTTP %s; falling back to the bundled model-cost map", OPENROUTER_MODELS_URL, response.status_code, ) return {} payload = response.json() except Exception as exc: # noqa: BLE001 - never propagate into a request - verbose_logger.debug( - "openrouter capabilities: fetch failed (%s: %s); falling back to the bundled model-cost map", + _log_fetch_failure( + "fetch failed (%s: %s); falling back to the bundled model-cost map", type(exc).__name__, exc, ) @@ -106,46 +383,112 @@ def _fetch() -> dict[str, set[str]]: try: payload = json.loads(payload) except Exception: # noqa: BLE001 + _log_fetch_failure("response body was not JSON; falling back to the model-cost map") return {} entries = payload.get("data") if isinstance(payload, dict) else None if not isinstance(entries, list): + _log_fetch_failure("response had no `data` list; falling back to the model-cost map") return {} - capabilities: dict[str, set[str]] = {} + capabilities: dict[str, dict] = {} for entry in entries: if not isinstance(entry, dict): continue model_id = entry.get("id") + if not isinstance(model_id, str) or not model_id: + continue params = entry.get("supported_parameters") - if not isinstance(model_id, str) or not isinstance(params, list): + parameters = {p for p in params if isinstance(p, str)} if isinstance(params, list) else None + cost_entry, declined = _cost_entry(model_id, entry.get("pricing")) + # An entry that answers neither question carries no information, and + # keeping it would make a roster of such entries look like a successful + # fetch to the "no usable entries" check below. A declined rate card is + # an answer — "we saw this model and will not price it" — so it keeps + # the record alive for the warning. + if parameters is None and cost_entry is None and declined is None: continue - capabilities[model_id] = {p for p in params if isinstance(p, str)} + capabilities[model_id] = { + # The published slug, not the spelling the caller looked up: a + # record reached via an ``openrouter/``-prefixed or ``:free``-suffixed + # candidate must still name itself the way OpenRouter does. + "id": model_id, + "parameters": parameters, + "cost_entry": cost_entry, + "declined_thresholds": declined, + } + + if not capabilities: + # A 200 whose `data` list is empty, or every entry of which is + # unparseable, is an OpenRouter schema change or an empty roster — not + # the ordinary "we asked and got nothing usable" fallback it otherwise + # looks like from the outside. Reporting it keeps `{}` meaning exactly + # one thing (see the _CACHE comment) and makes a schema drift visible + # instead of indistinguishable from a fetch that simply had no opinion. + _log_fetch_failure("`data` list contained no usable entries; falling back to the bundled model-cost map") + return {} + + # Re-arm the failure warning, and only on a fetch that actually produced + # capabilities. Latching it for the life of the process would mean a single + # blip during pod startup permanently demotes every later outage to debug — + # silencing exactly the case _log_fetch_failure exists to surface — but + # re-arming on an empty parse would report a recovery that did not happen. + _WARNED_FETCH_FAILURE = False return capabilities -def _get_cache() -> dict[str, set[str]]: +def _log_fetch_failure(message: str, *args: object) -> None: + """First failure at warning, the rest at debug. + + A permanently unreachable endpoint is the exact case where behaviour + silently reverts to the model-cost map, and litellm's default log level is + INFO — so debug-only reporting reproduces the same silent-fallback problem + the ``drop_params`` warning exists to remove. Warning once is enough to be findable without + turning an offline deployment into a log flood. + """ + global _WARNED_FETCH_FAILURE + + level = "debug" if _WARNED_FETCH_FAILURE else "warning" + # Latched only once the emit did not raise, for the reason in ``_warn_env_once``: + # ``_log`` swallows its own failures, and marking "already warned" for a + # line that was never emitted would demote every later outage to debug + # without anyone having seen the first one. + if _log(level, "openrouter capabilities: " + message, *args): + _WARNED_FETCH_FAILURE = True + + +def _get_cache() -> dict[str, dict]: global _CACHE, _CACHE_STAMP - now = time.monotonic() + ttl = _ttl_seconds() cache = _CACHE - if cache is not None and (now - _CACHE_STAMP) < _ttl_seconds(): + if cache is not None and (time.monotonic() - _CACHE_STAMP) < ttl: return cache - with _LOCK: + # Exactly one thread refreshes. A thread that finds the lock held serves + # the cache it already has rather than queueing behind someone else's HTTP + # call: httpx timeouts are per-phase, not total, so a pathological + # connection can exceed the configured seconds and that latency would land + # on a live request once per TTL. Stale capability data is cheap; added + # request latency is not. Only the very first fetch — nothing cached to + # serve — actually blocks. + if not _LOCK.acquire(blocking=cache is None): + return cache # type: ignore[return-value] + try: # Re-check under the lock: another thread may have refreshed while this # one waited, and a second fetch would be pure waste. - now = time.monotonic() - if _CACHE is not None and (now - _CACHE_STAMP) < _ttl_seconds(): + if _CACHE is not None and (time.monotonic() - _CACHE_STAMP) < ttl: return _CACHE # Stamped even on failure, so an unreachable endpoint costs one attempt # per TTL rather than one per request. _CACHE = _fetch() _CACHE_STAMP = time.monotonic() return _CACHE + finally: + _LOCK.release() -def _candidate_slugs(model: str) -> list: +def _candidate_slugs(model: str) -> list[str]: """Spellings of ``model`` that may appear as an OpenRouter model id. Callers reach this from several directions: a bare slug @@ -154,7 +497,7 @@ def _candidate_slugs(model: str) -> list: suffix (``qwen/qwen3-max:free``). The ids returned by the API are bare slugs, with ``:free`` published as its own id. """ - candidates = [] + candidates: list[str] = [] def add(value: str) -> None: if value and value not in candidates: @@ -171,13 +514,11 @@ def add(value: str) -> None: return candidates -def get_supported_parameters(model: str) -> set[str] | None: - """Parameter names OpenRouter advertises for ``model``. +def _lookup(model: str) -> dict | None: + """The published record for ``model``, or None when there is no opinion. - Returns ``None`` when the answer is unknown for any reason: the lookup is - disabled, the fetch failed, or the slug is not in the published list. A - ``None`` return means "no opinion" and callers must fall back to whatever - they did before. + "No opinion" covers every way the answer can be unknown: the lookup is + disabled, the fetch failed, or the slug is not in the published list. """ if not _env_flag("LITELLM_OPENROUTER_CAPABILITY_FETCH", True): return None @@ -189,15 +530,113 @@ def get_supported_parameters(model: str) -> set[str] | None: return None for candidate in _candidate_slugs(model): - params = cache.get(candidate) - if params is not None: - return params + record = cache.get(candidate) + if record is not None: + return record return None +def get_supported_parameters(model: str) -> set[str] | None: + """Parameter names OpenRouter advertises for ``model``. + + Returns ``None`` when the answer is unknown for any reason (see ``_lookup``) + or when the roster entry carried no ``supported_parameters`` list. A + ``None`` return means "no opinion" and callers must fall back to whatever + they did before. + + The returned set is a copy — the cache is process-wide and lives for the + TTL, so handing out the stored object would let one caller's ``add`` change + what every later caller sees. + """ + record = _lookup(model) + if record is None: + return None + params = record.get("parameters") + return set(params) if params is not None else None + + +def get_model_cost_entry(model: str, custom_llm_provider: str | None = None) -> dict | None: + """A LiteLLM-shaped model-cost entry for ``model``, or None. + + None means "no opinion", exactly as in ``get_supported_parameters``, and the + caller must fall back to whatever it did before — which for + ``_get_model_info_helper`` is the stock "This model isn't mapped yet" + ValueError. There are four ways to get it: the lookup is off, the pricing + half specifically is off, the slug is unknown, or its rate card is tiered + and therefore untranslatable (see the module docstring; that case is warned + about once per slug, because unlike the others it is a model we *can* see + and still will not price). + + ``custom_llm_provider`` is accepted and checked rather than ignored: this + module speaks only for OpenRouter, and the call site sits on a generic + lookup that every provider reaches. Answering there for, say, a Bedrock slug + that happens to share a name would attach OpenRouter's rate card to someone + else's bill. None is permitted because the caller resolves the provider from + the model string, and a bare ``qwen/qwen3-max`` legitimately arrives + unattributed. + + The returned dict is a copy: the cache is process-wide, and LiteLLM's + model-info path is free to mutate what it is handed. + + Note on freshness: LiteLLM memoizes ``_get_model_info_helper`` behind an + ``lru_cache``, so the FIRST successful answer for a slug is what the + process uses until it restarts — this module's TTL governs how often the + roster is re-read, not how often a priced model is re-priced. That is the + same staleness every entry in the bundled map already has, and rates move + on a scale where it does not matter; the provider-billed ``cost`` is + unaffected either way. ``lru_cache`` does not memoize exceptions, so a + lookup that failed while the roster was unreachable is retried rather than + latched. + """ + if custom_llm_provider is not None and custom_llm_provider != "openrouter": + return None + if not _env_flag("LITELLM_OPENROUTER_PRICING", True): + return None + + record = _lookup(model) + if record is None: + return None + + entry = record.get("cost_entry") + if entry is not None: + return dict(entry) + + declined = record.get("declined_thresholds") + if declined is not None: + _warn_declined_pricing(record, declined) + return None + + +def _warn_declined_pricing(record: dict, thresholds: tuple[int, ...]) -> None: + """Report a tiered rate card we will not translate, once per slug. + + Warn-level and once: an operator looking at a null ``cost_estimated`` for a + model that plainly *has* a published price needs to find this, and this runs + on the per-call cost path, so repeating it would bury the very log stream + the cost figures land in. + """ + slug = record.get("id") or "" + if slug in _WARNED_DECLINED_PRICING: + return + where = f"tiers at {', '.join(str(t) for t in thresholds)} tokens" if thresholds else "tier boundaries unparseable" + if _log( + "warning", + "openrouter capabilities: %s prices by prompt length (%s); " + "LiteLLM cannot express those boundaries, so cost_estimated stays null for it " + "rather than under-reporting long prompts. Read the provider-billed `cost` " + "field instead.", + slug, + where, + ): + _WARNED_DECLINED_PRICING.add(slug) + + def reset_cache() -> None: """Drop cached capability data. Intended for tests.""" - global _CACHE, _CACHE_STAMP + global _CACHE, _CACHE_STAMP, _WARNED_FETCH_FAILURE with _LOCK: _CACHE = None _CACHE_STAMP = 0.0 + _WARNED_FETCH_FAILURE = False + _WARNED_ENV.clear() + _WARNED_DECLINED_PRICING.clear() diff --git a/litellm/utils.py b/litellm/utils.py index 3c19b3c33655..6343c1b8e712 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5434,6 +5434,30 @@ def _get_model_info_helper( if generalization is not None: key, _model_info = generalization + if _model_info is None or key is None: + # OpenRouter publishes its own rate card at GET /api/v1/models, + # and the bundled map lags its slugs by construction — the same + # root cause that makes `supports_reasoning` answer False for a + # current model. Consult the live card before giving up. + # + # Placed here, after every map lookup has already failed, so a + # slug the bundled map DOES carry keeps the bundled answer: the + # live card can add a model but never reprice one. Cost fields + # only; see llms/openrouter/capabilities.py for why capability + # flags deliberately do not arrive through this door, and why a + # rate card tiered by prompt length is declined rather than + # approximated. + try: + from litellm.llms.openrouter.capabilities import ( + get_model_cost_entry as _openrouter_cost_entry, + ) + + _live_entry = _openrouter_cost_entry(model, custom_llm_provider) + except Exception: + _live_entry = None + if _live_entry is not None: + _model_info = _live_entry + key = _live_entry.get("key") or model if _model_info is None or key is None: raise ValueError( "This model isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json" diff --git a/tests/test_litellm/llms/openrouter/test_openrouter_capabilities.py b/tests/test_litellm/llms/openrouter/test_openrouter_capabilities.py index 9f28abead2a9..aafbb9887a4f 100644 --- a/tests/test_litellm/llms/openrouter/test_openrouter_capabilities.py +++ b/tests/test_litellm/llms/openrouter/test_openrouter_capabilities.py @@ -37,9 +37,40 @@ "id": "openai/gpt-4o-mini", "supported_parameters": ["max_tokens", "temperature"], }, + { + # Prices by prompt length. OpenRouter really does publish qwen3-max + # this way; the boundaries are arbitrary and the model-cost map + # cannot express them, which is why this entry stays unpriced. + "id": "qwen/qwen3-max", + "supported_parameters": ["max_tokens"], + "pricing": { + "prompt": "0.00000078", + "completion": "0.0000039", + "overrides": [ + {"min_prompt_tokens": 32000, "prompt": "0.00000156"}, + {"min_prompt_tokens": 128000, "prompt": "0.00000195"}, + ], + }, + }, ] } +# Rates are published as decimal STRINGS in USD per token, and a model may omit +# any of the cache rates. Attached to the two slugs above that the pricing tests +# read; the rest stay price-free on purpose, so "advertises params but not +# pricing" is covered by the same payload. +CAPABILITY_PAYLOAD["data"][0]["pricing"] = { + "prompt": "0.0000006", + "completion": "0.0000025", + "input_cache_read": "0.00000015", +} +CAPABILITY_PAYLOAD["data"][1]["pricing"] = { + "prompt": "0.0000001", + "completion": "0.0000002", + "input_cache_read": "0.00000001", + "input_cache_write": "0.0000005", +} + def _mock_response(payload, status_code=200): response = MagicMock() @@ -61,15 +92,20 @@ def _enable_lookup(monkeypatch): @pytest.fixture def live_payload(): - with patch.object( - capabilities, - "_fetch", - return_value={ - entry["id"]: set(entry["supported_parameters"]) - for entry in CAPABILITY_PAYLOAD["data"] - }, - ) as mock_fetch: - yield mock_fetch + """Serve CAPABILITY_PAYLOAD over a stubbed HTTP handler. + + Patched at the transport rather than at ``_fetch`` so the parsing — which + is where the pricing translation lives — is under test rather than mocked + past. The returned mock records one call per constructed handler, so a test + can still assert how many fetches actually happened. + """ + handler = MagicMock() + handler.return_value.get.return_value = _mock_response(CAPABILITY_PAYLOAD) + with patch( + "litellm.llms.custom_httpx.http_handler.HTTPHandler", + handler, + ): + yield handler class TestCapabilityLookup: @@ -187,7 +223,14 @@ def test_live_data_never_withholds_what_the_cost_map_allows(self, live_payload): with patch.object( capabilities, "_fetch", - return_value={"deepseek/deepseek-r1": {"reasoning", "include_reasoning"}}, + return_value={ + "deepseek/deepseek-r1": { + "id": "deepseek/deepseek-r1", + "parameters": {"reasoning", "include_reasoning"}, + "cost_entry": None, + "declined_thresholds": None, + } + }, ): params = OpenrouterConfig().get_supported_openai_params( "deepseek/deepseek-r1" @@ -220,3 +263,184 @@ def test_fallback_matches_previous_behaviour_for_an_unknown_slug(self): "vendor/never-published" ) assert "reasoning_effort" not in params + + +class TestPricingLookup: + """The rate-card half of the same fetch. + + The bundled model-cost map does not carry current OpenRouter slugs, so + ``get_model_info`` raises for them and no cost is ever computed. These + assert the live card fills that gap without ever becoming a source of + anything other than cost. + """ + + def test_translates_the_published_rate_card(self, live_payload): + entry = capabilities.get_model_cost_entry("openrouter/poolside/laguna-s-2.1") + assert entry == { + "key": "poolside/laguna-s-2.1", + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_token": 1e-07, + "output_cost_per_token": 2e-07, + "cache_read_input_token_cost": 1e-08, + # OpenRouter's `input_cache_write` is this map's `cache_creation_*`. + # Swapping the pair prices cache writes at the read rate — a ~5x + # understatement of the most expensive turn in a session. + "cache_creation_input_token_cost": 5e-07, + } + + def test_carries_cost_fields_only(self, live_payload): + # A supports_* flag arriving through this door would change parameter + # admission, which is get_supported_parameters' job and is gated far + # more carefully (see the `thinking` note in the module docstring). + entry = capabilities.get_model_cost_entry("moonshotai/kimi-k3") + assert not [k for k in entry if k.startswith("supports_")] + assert not [k for k in entry if "tokens" in k], "no context lengths either" + + def test_omits_a_rate_the_provider_did_not_publish(self, live_payload): + entry = capabilities.get_model_cost_entry("moonshotai/kimi-k3") + assert "cache_read_input_token_cost" in entry + assert "cache_creation_input_token_cost" not in entry + + def test_a_tiered_rate_card_is_declined_rather_than_under_reported( + self, live_payload + ): + # qwen3-max charges 2x above 32000 prompt tokens and 2.5x above 128000. + # The map has slots for three fixed thresholds and drops the rest, so + # registering the base tier would under-report long prompts silently, + # under a field an operator would use to compare models. + assert capabilities.get_model_cost_entry("qwen/qwen3-max") is None + # ...and the parameter half of the same entry is unaffected. + assert capabilities.get_supported_parameters("qwen/qwen3-max") == {"max_tokens"} + + def test_answers_for_openrouter_only(self, live_payload): + # The call site is a generic model-info lookup every provider reaches. + assert capabilities.get_model_cost_entry("poolside/laguna-s-2.1", "bedrock") is None + assert ( + capabilities.get_model_cost_entry("poolside/laguna-s-2.1", "openrouter") + is not None + ) + # None means the caller could not attribute it, which a bare slug is. + assert capabilities.get_model_cost_entry("poolside/laguna-s-2.1", None) is not None + + def test_unknown_slug_has_no_price_opinion(self, live_payload): + assert capabilities.get_model_cost_entry("vendor/never-published") is None + + def test_an_entry_without_pricing_is_still_a_parameter_answer(self, live_payload): + # The two halves are independent; one missing is not the other's problem. + assert capabilities.get_supported_parameters("openai/gpt-4o-mini") is not None + assert capabilities.get_model_cost_entry("openai/gpt-4o-mini") is None + + def test_returned_entry_is_a_copy(self, live_payload): + # The cache is process-wide and get_model_info mutates what it is handed. + first = capabilities.get_model_cost_entry("poolside/laguna-s-2.1") + first["input_cost_per_token"] = 999.0 + assert ( + capabilities.get_model_cost_entry("poolside/laguna-s-2.1")[ + "input_cost_per_token" + ] + == 1e-07 + ) + + def test_pricing_can_be_disabled_without_disabling_capabilities( + self, live_payload, monkeypatch + ): + monkeypatch.setenv("LITELLM_OPENROUTER_PRICING", "0") + assert capabilities.get_model_cost_entry("poolside/laguna-s-2.1") is None + assert capabilities.get_supported_parameters("poolside/laguna-s-2.1") is not None + + def test_the_master_switch_disables_pricing_too(self, live_payload, monkeypatch): + monkeypatch.setenv("LITELLM_OPENROUTER_CAPABILITY_FETCH", "0") + assert capabilities.get_model_cost_entry("poolside/laguna-s-2.1") is None + assert live_payload.call_count == 0, "no network call when the lookup is off" + + @pytest.mark.parametrize( + "pricing", + [ + {"completion": "0.000002"}, + {"prompt": "0.000001"}, + {"prompt": "not-a-number", "completion": "0.000002"}, + {"prompt": "-0.000001", "completion": "0.000002"}, + {"prompt": "Infinity", "completion": "0.000002"}, + "not-a-dict", + ], + ) + def test_an_unusable_rate_card_is_no_opinion(self, pricing): + # Without both rates the entry cannot price a chat turn; a negative or + # non-finite rate is not a rate, and an `inf` would serialize as a + # non-standard JSON token downstream. + handler = MagicMock() + handler.return_value.get.return_value = _mock_response( + {"data": [{"id": "some/model", "pricing": pricing}]} + ) + with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler", handler): + assert capabilities.get_model_cost_entry("some/model") is None + + def test_zero_is_a_real_rate(self): + # A `:free` variant really is priced at zero. Deciding what to do with + # a zero estimate belongs to whoever reads it. + handler = MagicMock() + handler.return_value.get.return_value = _mock_response( + {"data": [{"id": "a/b:free", "pricing": {"prompt": "0", "completion": "0"}}]} + ) + with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler", handler): + entry = capabilities.get_model_cost_entry("a/b:free") + assert entry["input_cost_per_token"] == 0.0 + assert entry["output_cost_per_token"] == 0.0 + + +class TestModelInfoIntegration: + """The hook in ``_get_model_info_helper``, which is the point of all this.""" + + @pytest.fixture(autouse=True) + def _clear_model_info_cache(self): + # get_model_info memoizes, so without this each test here would be + # asserting against whatever a previous one happened to warm — and the + # order would decide which of them actually exercised the hook. + from litellm.utils import _cached_get_model_info, _cached_get_model_info_helper + + for cache in (_cached_get_model_info, _cached_get_model_info_helper): + cache.cache_clear() + yield + for cache in (_cached_get_model_info, _cached_get_model_info_helper): + cache.cache_clear() + + def test_an_unmapped_slug_is_priced_from_the_live_card(self, live_payload): + info = litellm.get_model_info(model="openrouter/poolside/laguna-s-2.1") + assert info["input_cost_per_token"] == 1e-07 + assert info["output_cost_per_token"] == 2e-07 + assert info["cache_read_input_token_cost"] == 1e-08 + assert info["litellm_provider"] == "openrouter" + + def test_cost_is_computed_end_to_end(self, live_payload): + # 10k uncached input + 90k cached + 2k output, against the rates above. + prompt, cached, completion = 100_000, 90_000, 2_000 + expected = (prompt - cached) * 1e-07 + cached * 1e-08 + completion * 2e-07 + usage = litellm.Usage( + prompt_tokens=prompt, + completion_tokens=completion, + total_tokens=prompt + completion, + prompt_tokens_details={"cached_tokens": cached}, + ) + prompt_cost, completion_cost = litellm.cost_per_token( + model="openrouter/poolside/laguna-s-2.1", + custom_llm_provider="openrouter", + usage_object=usage, + ) + assert prompt_cost + completion_cost == pytest.approx(expected) + + def test_a_declined_tiered_model_still_raises(self, live_payload): + # Better an explicit "not mapped" than a number that is 2.5x low on the + # prompt lengths this model is actually used at. + with pytest.raises(Exception, match="isn't mapped yet"): + litellm.get_model_info(model="openrouter/qwen/qwen3-max") + + def test_an_unknown_slug_still_raises(self, live_payload): + with pytest.raises(Exception, match="isn't mapped yet"): + litellm.get_model_info(model="openrouter/vendor/never-published") + + def test_a_mapped_slug_keeps_the_bundled_rate(self, live_payload): + # The hook sits after every map lookup, so the live card can add a + # model but never reprice one. + info = litellm.get_model_info(model="gpt-4o-mini") + assert info["litellm_provider"] == "openai" diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 38516bdb3d02..11ff13af86a9 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -4878,6 +4878,25 @@ class TestDropParamsVisibility: def setup_method(self): litellm.utils._DROPPED_PARAM_WARNINGS.clear() + @pytest.fixture(autouse=True) + def _deterministic_openrouter_lookups(self, monkeypatch): + """Keep these tests on the bundled model-cost map. + + `get_optional_params` reaches `get_model_info`, which consults + OpenRouter's live model list for both supported params and pricing. + Left on, these tests do real HTTP and their outcome depends on what + OpenRouter published that morning — and a model with a prompt-length + rate card emits a pricing warning here, which is correct behaviour but + not what this class is counting. Same discipline as the autouse + fixture in tests/test_litellm/llms/openrouter/conftest.py. + """ + monkeypatch.setenv("LITELLM_OPENROUTER_CAPABILITY_FETCH", "0") + from litellm.llms.openrouter import capabilities + + capabilities.reset_cache() + yield + capabilities.reset_cache() + def test_warns_when_an_unsupported_param_is_dropped(self): # openrouter advertises reasoning_effort only for models flagged # supports_reasoning in the model-cost map; an absent slug is not.