From 409932de804bd195ada3221d7c63362c631811e2 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 23:50:24 +0000 Subject: [PATCH 01/22] fix(proxy): expand partial wildcard prefixes against provider-stripped model ids Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../health_check_helpers.py | 23 ++- litellm/proxy/auth/model_checks.py | 98 ++++++++---- .../test_wildcard_partial_prefix_expansion.py | 149 ++++++++++++++++++ .../test_health_check_helpers.py | 1 + tests/unit/proxy/auth/test_model_checks.py | 35 ++++ 5 files changed, 267 insertions(+), 39 deletions(-) create mode 100644 tests/e2e/test_wildcard_partial_prefix_expansion.py diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 03af9076c793..2da21bf65a51 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -3,7 +3,7 @@ """ import base64 -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping, Sequence from typing import TYPE_CHECKING, Final, Literal from litellm.llms.base_llm.ocr.transformation import DocumentType @@ -80,6 +80,19 @@ def _ocr_health_check_document(model: str, custom_llm_provider: str) -> Document return native(model, custom_llm_provider) +def _wildcard_health_check_models(wildcard_model: str, cheapest_models: Sequence[str]) -> tuple[str, ...]: + """Substitute each candidate id into the wildcard's dynamic segment so the health + check probes a name the wildcard route actually expands, instead of the bare + cost-map id (e.g. `databricks/system.ai.*` + `databricks/databricks-x` -> + `databricks/system.ai.databricks-x`).""" + provider_prefix, sep, wildcard_suffix = wildcard_model.partition("/") + if not sep or wildcard_suffix == "*": + return tuple(cheapest_models) + return tuple( + f"{provider_prefix}/{wildcard_suffix.replace('*', model.partition('/')[-1], 1)}" for model in cheapest_models + ) + + class HealthCheckHelpers: @staticmethod async def ahealth_check_wildcard_models( @@ -99,11 +112,9 @@ async def ahealth_check_wildcard_models( raise Exception( f"Unable to health check wildcard model for provider {custom_llm_provider}. Add a model on your config.yaml or contribute here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json" ) - if len(cheapest_models) > 1: - fallback_models = cheapest_models[1:] # Pick the last 2 models from the shuffled list - else: - fallback_models = None - model_params["model"] = cheapest_models[0] + candidates: Final = _wildcard_health_check_models(wildcard_model=model, cheapest_models=cheapest_models) + fallback_models: Final = list(candidates[1:]) or None + model_params["model"] = candidates[0] model_params["litellm_logging_obj"] = litellm_logging_obj model_params["fallbacks"] = fallback_models model_params["max_tokens"] = model_params.get("max_tokens", 16) # GPT-5 models require max_output_tokens >= 16 diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index b70cdb08b96b..340586d3628a 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -269,6 +269,49 @@ def _hydrate_litellm_credential_name( return litellm_params +def _strip_known_provider_prefix(model: str, known_providers: frozenset[str]) -> str: + leading, sep, model_suffix = model.partition("/") + return model_suffix if sep and leading in known_providers else model + + +def _apply_partial_wildcard_prefix( + provider_models: Sequence[str], + model_prefix: str, + known_providers: frozenset[str], + live_model_listing: bool, +) -> tuple[str, ...]: + matching_provider_models: Final = tuple( + model + for model in provider_models + if _strip_known_provider_prefix(model, known_providers).startswith(model_prefix) + ) + if matching_provider_models: + return matching_provider_models + if live_model_listing: + return () + return tuple(f"{model_prefix}{_strip_known_provider_prefix(model, known_providers)}" for model in provider_models) + + +def _provider_has_live_model_listing(provider: str) -> bool: + if not litellm.check_provider_endpoint: + return False + try: + llm_provider: Final = LlmProviders(provider) + except ValueError: + return False + return ProviderConfigManager.get_provider_model_info(model=None, provider=llm_provider) is not None + + +def _normalize_known_wildcard_model( + model: str, + wildcard_provider_prefix: str, + known_providers: frozenset[str], +) -> str: + if model.startswith(f"{wildcard_provider_prefix}/"): + return model + return f"{wildcard_provider_prefix}/{_strip_known_provider_prefix(model, known_providers)}" + + def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_Params | None = None) -> list[str]: wildcard_model_to_expand: Final = ( litellm_params.model @@ -295,44 +338,33 @@ def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_ litellm_params = _hydrate_litellm_credential_name(litellm_params) - wildcard_models = get_provider_models(provider=provider, litellm_params=litellm_params) + provider_models: Final = get_provider_models(provider=provider, litellm_params=litellm_params) - if wildcard_models is None: + if provider_models is None: return [] - if wildcard_suffix != "*": - ## CHECK IF PARTIAL FILTER e.g. `gemini-*` - model_prefix: Final = wildcard_suffix.replace("*", "") - deployment_repeats_prefix: Final = litellm_params is not None and litellm_params.model.endswith( - f"/{wildcard_suffix}" + known_providers: Final = frozenset(provider.value for provider in LlmProviders) + + wildcard_models: Final = ( + provider_models + if wildcard_suffix == "*" + else _apply_partial_wildcard_prefix( + provider_models=provider_models, + model_prefix=wildcard_suffix.replace("*", ""), + known_providers=known_providers, + live_model_listing=_provider_has_live_model_listing(provider), ) - is_partial_filter: Final = deployment_repeats_prefix or any( - wc_model.startswith(model_prefix) for wc_model in wildcard_models + ) + + normalized_wildcard_models: Final = tuple( + _normalize_known_wildcard_model( + model=model, + wildcard_provider_prefix=wildcard_provider_prefix, + known_providers=known_providers, ) - if is_partial_filter: - filtered_wildcard_models = [wc_model for wc_model in wildcard_models if wc_model.startswith(model_prefix)] - wildcard_models = filtered_wildcard_models - else: - # add model prefix to wildcard models - wildcard_models = [f"{model_prefix}{model}" for model in wildcard_models] - - known_providers: Final = {provider.value for provider in LlmProviders} - suffix_appended_wildcard_models: Final = [] - for model in wildcard_models: - if not model.startswith(f"{wildcard_provider_prefix}/"): - # `get_provider_models` returns provider-prefixed ids (e.g. "ollama/gemma3:1b"). - # When the wildcard uses a custom prefix (e.g. "ollama_server1/*" to distinguish - # multiple instances), replace that existing provider prefix instead of stacking - # both, which would otherwise yield an uncallable "ollama_server1/ollama/gemma3:1b". - # Only strip the leading segment when it is a known provider, so ids whose first - # segment is an org rather than a provider (e.g. "meta-llama/Llama-3-8B") keep it. - leading, sep, model_suffix = model.partition("/") - if sep and leading in known_providers: - model = f"{wildcard_provider_prefix}/{model_suffix}" - else: - model = f"{wildcard_provider_prefix}/{model}" - suffix_appended_wildcard_models.append(model) - return suffix_appended_wildcard_models or [] + for model in wildcard_models + ) + return list(dict.fromkeys(normalized_wildcard_models)) def expand_wildcard_deployments_for_model_info( diff --git a/tests/e2e/test_wildcard_partial_prefix_expansion.py b/tests/e2e/test_wildcard_partial_prefix_expansion.py new file mode 100644 index 000000000000..48b265c2c62e --- /dev/null +++ b/tests/e2e/test_wildcard_partial_prefix_expansion.py @@ -0,0 +1,149 @@ +"""Regression: wildcard expansion of a partial provider prefix like +`databricks/system.ai.*` must produce `databricks/system.ai.` ids, +not `databricks/system.ai.databricks/databricks-`. + +On main, get_known_models_from_wildcard prepends the literal prefix before the +`*` ("system.ai.") to cost-map keys that already carry the `databricks/` +provider prefix, so /v1/models and /model/info list names that Databricks +rejects with "Invalid Unity Catalog name". See LIT-8910. + +The test boots its own proxy (the shared stack's model_list is fixed), with two +wildcard deployments: `databricks/system.ai.*` under test and `databricks/*` as +a control whose expansion must stay `databricks/databricks-*`. +""" + +import socket +import subprocess +import sys +import time +from collections.abc import Generator +from pathlib import Path +from typing import Final, cast + +import pytest +from e2e_http import URL, AuthHeaders, NoBody, get, probe, unwrap +from models import ModelsListParams, ModelsListResponse +from pydantic import BaseModel, ConfigDict + +REPO_ROOT: Final = Path(__file__).resolve().parents[2] + +MASTER_KEY: Final = "sk-1234" +AUTH: Final = AuthHeaders(authorization=f"Bearer {MASTER_KEY}") + +CONFIG_YAML: Final = """\ +model_list: + - model_name: databricks/system.ai.* + litellm_params: + model: databricks/system.ai.* + api_key: fake-databricks-token + api_base: https://example.invalid + - model_name: databricks/* + litellm_params: + model: databricks/* + api_key: fake-databricks-token + api_base: https://example.invalid +general_settings: + master_key: sk-1234 +""" + + +class _ModelInfoParams(BaseModel): + model_config = ConfigDict(protected_namespaces=(), extra="ignore") + model: str = "" + + +class _ModelInfoRow(BaseModel): + model_name: str + litellm_params: _ModelInfoParams = _ModelInfoParams() + + +class _ModelInfoResponse(BaseModel): + data: list[_ModelInfoRow] = [] + + +def _free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return cast("tuple[str, int]", sock.getsockname())[1] + + +@pytest.fixture(scope="module") +def wildcard_proxy(tmp_path_factory: pytest.TempPathFactory) -> Generator[str]: + config_path = tmp_path_factory.mktemp("wildcard") / "config.yaml" + config_path.write_text(CONFIG_YAML) + port = _free_port() + proc = subprocess.Popen( + [ + sys.executable, + str(REPO_ROOT / "litellm/proxy/proxy_cli.py"), + "--config", + str(config_path), + "--port", + str(port), + ], + cwd=REPO_ROOT, + env={ + "PATH": "/usr/bin:/bin:/usr/local/bin", + "HOME": str(Path.home()), + "PYTHONPATH": str(REPO_ROOT), + "LITELLM_LOCAL_MODEL_COST_MAP": "true", + "LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY": "true", + "LITELLM_MASTER_KEY": MASTER_KEY, + }, + ) + base = f"http://127.0.0.1:{port}" + try: + deadline = time.time() + 180 + while True: + alive = probe(URL(f"{base}/health/liveliness"), headers=NoBody(), params=NoBody(), timeout=5.0) + if alive.healthy: + break + if proc.poll() is not None: + pytest.fail(f"proxy exited during boot with {proc.returncode}") + assert time.time() < deadline, "proxy did not answer /health/liveliness within 180s" + time.sleep(2) + yield base + finally: + proc.terminate() + proc.wait(timeout=30) + + +def _list_model_ids(base: str) -> list[str]: + listed = get( + URL(f"{base}/v1/models"), + headers=AUTH, + params=ModelsListParams(return_wildcard_routes=False), + response_type=ModelsListResponse, + ) + return [entry.id for entry in unwrap(listed).data] + + +def test_partial_prefix_wildcard_expands_to_unity_catalog_names(wildcard_proxy: str) -> None: + ids = _list_model_ids(wildcard_proxy) + + system_ai_ids = [model_id for model_id in ids if model_id.startswith("databricks/system.ai.")] + assert system_ai_ids, f"no databricks/system.ai.* expansion in {ids}" + + malformed = [model_id for model_id in system_ai_ids if "/" in model_id.removeprefix("databricks/system.ai.")] + assert not malformed, f"expanded ids embed the provider-prefixed cost-map key after 'system.ai.': {malformed}" + + +def test_provider_wildcard_still_expands_to_cost_map_names(wildcard_proxy: str) -> None: + ids = _list_model_ids(wildcard_proxy) + + databricks_ids = [model_id for model_id in ids if model_id.startswith("databricks/databricks-")] + assert databricks_ids, f"databricks/* did not expand to cost-map names: {ids}" + + +def test_model_info_keeps_provider_model_for_expanded_deployments(wildcard_proxy: str) -> None: + info = unwrap( + get( + URL(f"{wildcard_proxy}/model/info"), + headers=AUTH, + params=NoBody(), + response_type=_ModelInfoResponse, + ) + ) + + bad = [row.model_name for row in info.data if "system.ai.databricks/" in row.litellm_params.model] + assert not bad, f"litellm_params.model carries the corrupted expanded name: {bad}" diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index 4f62c6fb0965..064673f3a984 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -18,6 +18,7 @@ HealthCheckHelpers, default_health_check_mode, native_health_check_mode, + _wildcard_health_check_models, ) from litellm.main import ahealth_check from litellm.proxy._types import UserAPIKeyAuth diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 9f9a66872103..e93a18949f09 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -478,6 +478,41 @@ def test_wildcard_custom_prefix_keeps_org_segment_for_non_provider_first_segment assert result == ["my_hf/meta-llama/Llama-3-8B"] +def test_partial_prefix_wildcard_expands_against_stripped_provider_ids(monkeypatch): + """A partial wildcard like ``databricks/system.ai.*`` must splice the prefix before + ``*`` onto the provider-stripped model id, not onto the whole cost-map key. Prepending + to the provider-prefixed key yielded ``databricks/system.ai.databricks/databricks-x``, + an id no upstream accepts. + """ + import litellm + from litellm.proxy.auth import model_checks + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + provider_ids: Final = list(litellm.models_by_provider["databricks"]) + monkeypatch.setattr( + model_checks, + "get_provider_models", + lambda provider, litellm_params=None: provider_ids, + ) + + result: Final = get_known_models_from_wildcard( + wildcard_model="databricks/system.ai.*", + litellm_params=LiteLLM_Params( + model="databricks/system.ai.*", + api_key="x", + api_base="https://example.invalid", + ), + ) + + expected: Final = [f"databricks/system.ai.{model_id.partition('/')[-1]}" for model_id in provider_ids] + assert result == expected + assert all( + model_id.startswith("databricks/system.ai.") and "/" not in model_id.removeprefix("databricks/system.ai.") + for model_id in result + ) + + def test_wildcard_credential_hydration_preserves_missing_credential_name( monkeypatch, ): From 7dec2cb876de6dc9f10298a6a30fb935d70fdb9e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 23:54:32 +0000 Subject: [PATCH 02/22] fix(health): splice wildcard prefix onto provider-stripped health check candidates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../health_check_helpers.py | 19 +++++++++----- .../test_health_check_helpers.py | 14 +++++++++++ tests/unit/proxy/auth/test_model_checks.py | 25 +++++++++++-------- 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 2da21bf65a51..683d840b92b5 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -9,7 +9,7 @@ from litellm.llms.base_llm.ocr.transformation import DocumentType from litellm.rust_bridge import runtime from litellm.rust_bridge.ocr.entrypoints import NATIVE_OCR_HEALTH_CHECK_DOCUMENT -from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS +from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS, LlmProviders if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging @@ -80,16 +80,23 @@ def _ocr_health_check_document(model: str, custom_llm_provider: str) -> Document return native(model, custom_llm_provider) +def _strip_known_provider_prefix(model: str, known_providers: frozenset[str]) -> str: + leading, sep, model_suffix = model.partition("/") + return model_suffix if sep and leading in known_providers else model + + def _wildcard_health_check_models(wildcard_model: str, cheapest_models: Sequence[str]) -> tuple[str, ...]: - """Substitute each candidate id into the wildcard's dynamic segment so the health - check probes a name the wildcard route actually expands, instead of the bare - cost-map id (e.g. `databricks/system.ai.*` + `databricks/databricks-x` -> - `databricks/system.ai.databricks-x`).""" provider_prefix, sep, wildcard_suffix = wildcard_model.partition("/") if not sep or wildcard_suffix == "*": return tuple(cheapest_models) + known_providers: Final = frozenset(provider.value for provider in LlmProviders) + literal_prefix: Final = wildcard_suffix.replace("*", "") + stripped_ids: Final = tuple(_strip_known_provider_prefix(model, known_providers) for model in cheapest_models) return tuple( - f"{provider_prefix}/{wildcard_suffix.replace('*', model.partition('/')[-1], 1)}" for model in cheapest_models + f"{provider_prefix}/{stripped}" + if stripped.startswith(literal_prefix) + else f"{provider_prefix}/{wildcard_suffix.replace('*', stripped, 1)}" + for stripped in stripped_ids ) diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index 064673f3a984..8cd3247b714e 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -1504,3 +1504,17 @@ async def test_audio_transcription_health_check_returns_transcribed_text( assert "error" not in response assert upstream.called assert b'name="file"' in upstream.calls.last.request.content + + +def test_wildcard_health_check_models_bare_provider_and_star_wildcard_unchanged(): + candidates: Final = ("databricks/databricks-gpt-5",) + assert _wildcard_health_check_models("databricks/*", candidates) == candidates + assert _wildcard_health_check_models("*", candidates) == candidates + + +def test_wildcard_health_check_models_partial_prefix_matching_literal_keeps_stripped_id(): + assert _wildcard_health_check_models("openai/gpt-4*", ["gpt-4o-mini"]) == ("openai/gpt-4o-mini",) + + +def test_wildcard_health_check_models_partial_prefix_splices_suffix_around_star(): + assert _wildcard_health_check_models("openai/ft:*", ["gpt-4o-mini"]) == ("openai/ft:gpt-4o-mini",) diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index e93a18949f09..3f2059fa1eac 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -247,7 +247,9 @@ def test_get_key_models_does_not_mutate_input(): ), ], ) -def test_get_complete_model_list_order(key_models, team_models, proxy_model_list, model_list, expected): +def test_get_complete_model_list_order( + key_models, team_models, proxy_model_list, model_list, expected +): """ Test that get_complete_model_list preserves order """ @@ -400,7 +402,9 @@ def fake_get_provider_models(provider, litellm_params=None): captured_params["api_key"] = litellm_params.api_key captured_params["api_version"] = litellm_params.api_version captured_params["credential_name"] = litellm_params.litellm_credential_name - captured_params["has_unexpected_field"] = hasattr(litellm_params, "unexpected_field") + captured_params["has_unexpected_field"] = hasattr( + litellm_params, "unexpected_field" + ) return ["gpt-4o"] monkeypatch.setattr(model_checks, "get_provider_models", fake_get_provider_models) @@ -445,7 +449,9 @@ def test_wildcard_custom_prefix_does_not_stack_provider_prefix(monkeypatch): result = get_known_models_from_wildcard( wildcard_model="ollama_server1/*", - litellm_params=LiteLLM_Params(model="ollama_chat/*", custom_llm_provider="ollama_chat"), + litellm_params=LiteLLM_Params( + model="ollama_chat/*", custom_llm_provider="ollama_chat" + ), ) assert result == ["ollama_server1/gemma3:1b", "ollama_server1/llama3:8b"] @@ -472,18 +478,15 @@ def test_wildcard_custom_prefix_keeps_org_segment_for_non_provider_first_segment result = get_known_models_from_wildcard( wildcard_model="my_hf/*", - litellm_params=LiteLLM_Params(model="huggingface/*", custom_llm_provider="huggingface"), + litellm_params=LiteLLM_Params( + model="huggingface/*", custom_llm_provider="huggingface" + ), ) assert result == ["my_hf/meta-llama/Llama-3-8B"] def test_partial_prefix_wildcard_expands_against_stripped_provider_ids(monkeypatch): - """A partial wildcard like ``databricks/system.ai.*`` must splice the prefix before - ``*`` onto the provider-stripped model id, not onto the whole cost-map key. Prepending - to the provider-prefixed key yielded ``databricks/system.ai.databricks/databricks-x``, - an id no upstream accepts. - """ import litellm from litellm.proxy.auth import model_checks from litellm.proxy.auth.model_checks import get_known_models_from_wildcard @@ -955,7 +958,9 @@ def test_add_known_models_refreshes_models_by_provider_for_wildcard_expansion(): assert fake_model not in litellm.models_by_provider["vertex_ai"] try: litellm.add_known_models( - model_cost_map={fake_model: {"litellm_provider": "vertex_ai-language-models", "mode": "chat"}} + model_cost_map={ + fake_model: {"litellm_provider": "vertex_ai-language-models", "mode": "chat"} + } ) assert fake_model in litellm.models_by_provider["vertex_ai"] assert litellm.models_by_provider is captured_reference From 409eb78eabcf5aee097ef9de8bab3ef6c2538505 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 00:02:22 +0000 Subject: [PATCH 03/22] fix(health): use provider-stripped wildcard suffix at the ahealth_check call site Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../health_check_helpers.py | 15 +++++---- .../test_health_check_helpers.py | 31 ++++++++++++++++--- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 683d840b92b5..0f0c17ce7e83 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -85,17 +85,18 @@ def _strip_known_provider_prefix(model: str, known_providers: frozenset[str]) -> return model_suffix if sep and leading in known_providers else model -def _wildcard_health_check_models(wildcard_model: str, cheapest_models: Sequence[str]) -> tuple[str, ...]: - provider_prefix, sep, wildcard_suffix = wildcard_model.partition("/") - if not sep or wildcard_suffix == "*": +def _wildcard_health_check_models( + wildcard_suffix: str, custom_llm_provider: str, cheapest_models: Sequence[str] +) -> tuple[str, ...]: + if wildcard_suffix == "*": return tuple(cheapest_models) known_providers: Final = frozenset(provider.value for provider in LlmProviders) literal_prefix: Final = wildcard_suffix.replace("*", "") stripped_ids: Final = tuple(_strip_known_provider_prefix(model, known_providers) for model in cheapest_models) return tuple( - f"{provider_prefix}/{stripped}" + f"{custom_llm_provider}/{stripped}" if stripped.startswith(literal_prefix) - else f"{provider_prefix}/{wildcard_suffix.replace('*', stripped, 1)}" + else f"{custom_llm_provider}/{wildcard_suffix.replace('*', stripped, 1)}" for stripped in stripped_ids ) @@ -119,7 +120,9 @@ async def ahealth_check_wildcard_models( raise Exception( f"Unable to health check wildcard model for provider {custom_llm_provider}. Add a model on your config.yaml or contribute here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json" ) - candidates: Final = _wildcard_health_check_models(wildcard_model=model, cheapest_models=cheapest_models) + candidates: Final = _wildcard_health_check_models( + wildcard_suffix=model, custom_llm_provider=custom_llm_provider, cheapest_models=cheapest_models + ) fallback_models: Final = list(candidates[1:]) or None model_params["model"] = candidates[0] model_params["litellm_logging_obj"] = litellm_logging_obj diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index 8cd3247b714e..a7778433ffc0 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -1506,15 +1506,36 @@ async def test_audio_transcription_health_check_returns_transcribed_text( assert b'name="file"' in upstream.calls.last.request.content -def test_wildcard_health_check_models_bare_provider_and_star_wildcard_unchanged(): +def test_wildcard_health_check_models_star_wildcard_unchanged(): candidates: Final = ("databricks/databricks-gpt-5",) - assert _wildcard_health_check_models("databricks/*", candidates) == candidates - assert _wildcard_health_check_models("*", candidates) == candidates + assert _wildcard_health_check_models("*", "databricks", candidates) == candidates def test_wildcard_health_check_models_partial_prefix_matching_literal_keeps_stripped_id(): - assert _wildcard_health_check_models("openai/gpt-4*", ["gpt-4o-mini"]) == ("openai/gpt-4o-mini",) + assert _wildcard_health_check_models("gpt-4*", "openai", ["gpt-4o-mini"]) == ("openai/gpt-4o-mini",) def test_wildcard_health_check_models_partial_prefix_splices_suffix_around_star(): - assert _wildcard_health_check_models("openai/ft:*", ["gpt-4o-mini"]) == ("openai/ft:gpt-4o-mini",) + assert _wildcard_health_check_models("ft:*", "openai", ["gpt-4o-mini"]) == ("openai/ft:gpt-4o-mini",) + + +@pytest.mark.asyncio +async def test_ahealth_check_partial_prefix_wildcard_probes_spliced_model_id(): + from litellm.litellm_core_utils.llm_request_utils import pick_cheapest_chat_models_from_llm_provider + + cheapest: Final = pick_cheapest_chat_models_from_llm_provider(custom_llm_provider="databricks", n=3) + stripped: Final = tuple(model_id.partition("/")[-1] for model_id in cheapest) + + captured: dict[str, object] = {} + + async def _capture(**kwargs): + captured.update(kwargs) + + with patch("litellm.acompletion", side_effect=_capture): + await ahealth_check( + model_params={"model": "databricks/system.ai.*", "api_key": "x", "api_base": "https://example.invalid"}, + mode="chat", + ) + + assert captured["model"] == f"databricks/system.ai.{stripped[0]}" + assert captured["fallbacks"] == [f"databricks/system.ai.{model_id}" for model_id in stripped[1:]] From 1ecff12955d925b6fcd140f972d5335413d98168 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 00:09:26 +0000 Subject: [PATCH 04/22] test(e2e): spawn wildcard repro proxy with isolated interpreter flag Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/test_wildcard_partial_prefix_expansion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/test_wildcard_partial_prefix_expansion.py b/tests/e2e/test_wildcard_partial_prefix_expansion.py index 48b265c2c62e..dbe85028e10d 100644 --- a/tests/e2e/test_wildcard_partial_prefix_expansion.py +++ b/tests/e2e/test_wildcard_partial_prefix_expansion.py @@ -75,6 +75,7 @@ def wildcard_proxy(tmp_path_factory: pytest.TempPathFactory) -> Generator[str]: proc = subprocess.Popen( [ sys.executable, + "-P", str(REPO_ROOT / "litellm/proxy/proxy_cli.py"), "--config", str(config_path), From d945e2e6e505adcdb9c9bbf5ac9eaa59454840d4 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 03:11:00 +0000 Subject: [PATCH 05/22] fix(health): filter wildcard health probes to ids matching the literal prefix Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../health_check_helpers.py | 20 +++++++++---------- .../test_wildcard_partial_prefix_expansion.py | 4 +++- .../test_health_check_helpers.py | 14 +++++++++++-- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 0f0c17ce7e83..9255b27ea42a 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -86,19 +86,17 @@ def _strip_known_provider_prefix(model: str, known_providers: frozenset[str]) -> def _wildcard_health_check_models( - wildcard_suffix: str, custom_llm_provider: str, cheapest_models: Sequence[str] + wildcard_suffix: str, custom_llm_provider: str, candidate_models: Sequence[str] ) -> tuple[str, ...]: if wildcard_suffix == "*": - return tuple(cheapest_models) + return tuple(candidate_models[:3]) known_providers: Final = frozenset(provider.value for provider in LlmProviders) literal_prefix: Final = wildcard_suffix.replace("*", "") - stripped_ids: Final = tuple(_strip_known_provider_prefix(model, known_providers) for model in cheapest_models) - return tuple( - f"{custom_llm_provider}/{stripped}" - if stripped.startswith(literal_prefix) - else f"{custom_llm_provider}/{wildcard_suffix.replace('*', stripped, 1)}" - for stripped in stripped_ids - ) + stripped_ids: Final = tuple(_strip_known_provider_prefix(model, known_providers) for model in candidate_models) + matching: Final = [stripped for stripped in stripped_ids if stripped.startswith(literal_prefix)] + if matching: + return tuple(f"{custom_llm_provider}/{stripped}" for stripped in matching[:3]) + return tuple(f"{custom_llm_provider}/{wildcard_suffix.replace('*', stripped, 1)}" for stripped in stripped_ids[:3]) class HealthCheckHelpers: @@ -115,13 +113,13 @@ async def ahealth_check_wildcard_models( ) # this is a wildcard model, we need to pick a random model from the provider - cheapest_models = pick_cheapest_chat_models_from_llm_provider(custom_llm_provider=custom_llm_provider, n=3) + cheapest_models = pick_cheapest_chat_models_from_llm_provider(custom_llm_provider=custom_llm_provider, n=10_000) if len(cheapest_models) == 0: raise Exception( f"Unable to health check wildcard model for provider {custom_llm_provider}. Add a model on your config.yaml or contribute here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json" ) candidates: Final = _wildcard_health_check_models( - wildcard_suffix=model, custom_llm_provider=custom_llm_provider, cheapest_models=cheapest_models + wildcard_suffix=model, custom_llm_provider=custom_llm_provider, candidate_models=cheapest_models ) fallback_models: Final = list(candidates[1:]) or None model_params["model"] = candidates[0] diff --git a/tests/e2e/test_wildcard_partial_prefix_expansion.py b/tests/e2e/test_wildcard_partial_prefix_expansion.py index dbe85028e10d..1281e9d921ba 100644 --- a/tests/e2e/test_wildcard_partial_prefix_expansion.py +++ b/tests/e2e/test_wildcard_partial_prefix_expansion.py @@ -146,5 +146,7 @@ def test_model_info_keeps_provider_model_for_expanded_deployments(wildcard_proxy ) ) - bad = [row.model_name for row in info.data if "system.ai.databricks/" in row.litellm_params.model] + expanded = [row for row in info.data if row.model_name.startswith("databricks/system.ai.")] + assert expanded, "model/info returned no expanded rows for databricks/system.ai.*" + bad = [row.model_name for row in expanded if "system.ai.databricks/" in row.litellm_params.model] assert not bad, f"litellm_params.model carries the corrupted expanded name: {bad}" diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index a7778433ffc0..1bd7056e6e5a 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -1507,14 +1507,24 @@ async def test_audio_transcription_health_check_returns_transcribed_text( def test_wildcard_health_check_models_star_wildcard_unchanged(): - candidates: Final = ("databricks/databricks-gpt-5",) - assert _wildcard_health_check_models("*", "databricks", candidates) == candidates + candidates: Final = ("databricks/a", "databricks/b", "databricks/c", "databricks/d") + assert _wildcard_health_check_models("*", "databricks", candidates) == ( + "databricks/a", + "databricks/b", + "databricks/c", + ) def test_wildcard_health_check_models_partial_prefix_matching_literal_keeps_stripped_id(): assert _wildcard_health_check_models("gpt-4*", "openai", ["gpt-4o-mini"]) == ("openai/gpt-4o-mini",) +def test_wildcard_health_check_models_partial_prefix_filters_candidates_to_literal_prefix(): + assert _wildcard_health_check_models( + "gpt-4*", "openai", ["gpt-5-nano", "gpt-4o-mini", "gpt-4.1-mini", "gpt-4o"] + ) == ("openai/gpt-4o-mini", "openai/gpt-4.1-mini", "openai/gpt-4o") + + def test_wildcard_health_check_models_partial_prefix_splices_suffix_around_star(): assert _wildcard_health_check_models("ft:*", "openai", ["gpt-4o-mini"]) == ("openai/ft:gpt-4o-mini",) From d15980d7f85a21d1f8e143072fad3545dc555286 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 03:18:48 +0000 Subject: [PATCH 06/22] test(integration): move partial wildcard expansion repro onto the shared proxy Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_wildcard_partial_prefix_expansion.py | 152 ------------------ .../test_wildcard_partial_prefix_expansion.py | 71 ++++++++ 2 files changed, 71 insertions(+), 152 deletions(-) delete mode 100644 tests/e2e/test_wildcard_partial_prefix_expansion.py create mode 100644 tests/integration/routing/test_wildcard_partial_prefix_expansion.py diff --git a/tests/e2e/test_wildcard_partial_prefix_expansion.py b/tests/e2e/test_wildcard_partial_prefix_expansion.py deleted file mode 100644 index 1281e9d921ba..000000000000 --- a/tests/e2e/test_wildcard_partial_prefix_expansion.py +++ /dev/null @@ -1,152 +0,0 @@ -"""Regression: wildcard expansion of a partial provider prefix like -`databricks/system.ai.*` must produce `databricks/system.ai.` ids, -not `databricks/system.ai.databricks/databricks-`. - -On main, get_known_models_from_wildcard prepends the literal prefix before the -`*` ("system.ai.") to cost-map keys that already carry the `databricks/` -provider prefix, so /v1/models and /model/info list names that Databricks -rejects with "Invalid Unity Catalog name". See LIT-8910. - -The test boots its own proxy (the shared stack's model_list is fixed), with two -wildcard deployments: `databricks/system.ai.*` under test and `databricks/*` as -a control whose expansion must stay `databricks/databricks-*`. -""" - -import socket -import subprocess -import sys -import time -from collections.abc import Generator -from pathlib import Path -from typing import Final, cast - -import pytest -from e2e_http import URL, AuthHeaders, NoBody, get, probe, unwrap -from models import ModelsListParams, ModelsListResponse -from pydantic import BaseModel, ConfigDict - -REPO_ROOT: Final = Path(__file__).resolve().parents[2] - -MASTER_KEY: Final = "sk-1234" -AUTH: Final = AuthHeaders(authorization=f"Bearer {MASTER_KEY}") - -CONFIG_YAML: Final = """\ -model_list: - - model_name: databricks/system.ai.* - litellm_params: - model: databricks/system.ai.* - api_key: fake-databricks-token - api_base: https://example.invalid - - model_name: databricks/* - litellm_params: - model: databricks/* - api_key: fake-databricks-token - api_base: https://example.invalid -general_settings: - master_key: sk-1234 -""" - - -class _ModelInfoParams(BaseModel): - model_config = ConfigDict(protected_namespaces=(), extra="ignore") - model: str = "" - - -class _ModelInfoRow(BaseModel): - model_name: str - litellm_params: _ModelInfoParams = _ModelInfoParams() - - -class _ModelInfoResponse(BaseModel): - data: list[_ModelInfoRow] = [] - - -def _free_port() -> int: - with socket.socket() as sock: - sock.bind(("127.0.0.1", 0)) - return cast("tuple[str, int]", sock.getsockname())[1] - - -@pytest.fixture(scope="module") -def wildcard_proxy(tmp_path_factory: pytest.TempPathFactory) -> Generator[str]: - config_path = tmp_path_factory.mktemp("wildcard") / "config.yaml" - config_path.write_text(CONFIG_YAML) - port = _free_port() - proc = subprocess.Popen( - [ - sys.executable, - "-P", - str(REPO_ROOT / "litellm/proxy/proxy_cli.py"), - "--config", - str(config_path), - "--port", - str(port), - ], - cwd=REPO_ROOT, - env={ - "PATH": "/usr/bin:/bin:/usr/local/bin", - "HOME": str(Path.home()), - "PYTHONPATH": str(REPO_ROOT), - "LITELLM_LOCAL_MODEL_COST_MAP": "true", - "LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY": "true", - "LITELLM_MASTER_KEY": MASTER_KEY, - }, - ) - base = f"http://127.0.0.1:{port}" - try: - deadline = time.time() + 180 - while True: - alive = probe(URL(f"{base}/health/liveliness"), headers=NoBody(), params=NoBody(), timeout=5.0) - if alive.healthy: - break - if proc.poll() is not None: - pytest.fail(f"proxy exited during boot with {proc.returncode}") - assert time.time() < deadline, "proxy did not answer /health/liveliness within 180s" - time.sleep(2) - yield base - finally: - proc.terminate() - proc.wait(timeout=30) - - -def _list_model_ids(base: str) -> list[str]: - listed = get( - URL(f"{base}/v1/models"), - headers=AUTH, - params=ModelsListParams(return_wildcard_routes=False), - response_type=ModelsListResponse, - ) - return [entry.id for entry in unwrap(listed).data] - - -def test_partial_prefix_wildcard_expands_to_unity_catalog_names(wildcard_proxy: str) -> None: - ids = _list_model_ids(wildcard_proxy) - - system_ai_ids = [model_id for model_id in ids if model_id.startswith("databricks/system.ai.")] - assert system_ai_ids, f"no databricks/system.ai.* expansion in {ids}" - - malformed = [model_id for model_id in system_ai_ids if "/" in model_id.removeprefix("databricks/system.ai.")] - assert not malformed, f"expanded ids embed the provider-prefixed cost-map key after 'system.ai.': {malformed}" - - -def test_provider_wildcard_still_expands_to_cost_map_names(wildcard_proxy: str) -> None: - ids = _list_model_ids(wildcard_proxy) - - databricks_ids = [model_id for model_id in ids if model_id.startswith("databricks/databricks-")] - assert databricks_ids, f"databricks/* did not expand to cost-map names: {ids}" - - -def test_model_info_keeps_provider_model_for_expanded_deployments(wildcard_proxy: str) -> None: - info = unwrap( - get( - URL(f"{wildcard_proxy}/model/info"), - headers=AUTH, - params=NoBody(), - response_type=_ModelInfoResponse, - ) - ) - - expanded = [row for row in info.data if row.model_name.startswith("databricks/system.ai.")] - assert expanded, "model/info returned no expanded rows for databricks/system.ai.*" - bad = [row.model_name for row in expanded if "system.ai.databricks/" in row.litellm_params.model] - assert not bad, f"litellm_params.model carries the corrupted expanded name: {bad}" diff --git a/tests/integration/routing/test_wildcard_partial_prefix_expansion.py b/tests/integration/routing/test_wildcard_partial_prefix_expansion.py new file mode 100644 index 000000000000..01206a2163a8 --- /dev/null +++ b/tests/integration/routing/test_wildcard_partial_prefix_expansion.py @@ -0,0 +1,71 @@ +"""Partial-prefix wildcards like ``databricks/system.ai.*`` must splice the literal +prefix onto provider-stripped cost-map ids, never onto the provider-prefixed key.""" + +from collections.abc import Iterator +from typing import Final + +import pytest + +from integration._support.client import Gateway, object_value, string_value + +PATTERNS: Final = ("databricks/system.ai.*", "databricks/*") + + +def _registered_model_id(gateway: Gateway, pattern: str) -> str: + created: Final = object_value( + gateway.post( + "/model/new", + { + "model_name": pattern, + "litellm_params": { + "model": pattern, + "api_key": "integration-wildcard-key", + "api_base": "https://example.invalid", + }, + }, + ) + ) + return string_value(object_value(created["model_info"])["id"]) + + +@pytest.fixture +def wildcard_deployments(gateway: Gateway) -> Iterator[None]: + with gateway.scenario() as scenario: + for pattern in PATTERNS: + scenario.cleanups.callback(scenario.delete_model, _registered_model_id(gateway, pattern)) + yield + + +def _listed_model_ids(gateway: Gateway) -> tuple[str, ...]: + data: Final = gateway.get("/v1/models")["data"] + assert isinstance(data, list) + return tuple(string_value(object_value(entry)["id"]) for entry in data) + + +def test_partial_prefix_wildcard_expands_to_unity_catalog_names(gateway: Gateway, wildcard_deployments: None) -> None: + prefixed: Final = tuple( + model_id for model_id in _listed_model_ids(gateway) if model_id.startswith("databricks/system.ai.") + ) + assert prefixed + assert all("/" not in model_id.removeprefix("databricks/system.ai.") for model_id in prefixed) + + +def test_provider_wildcard_still_expands_to_cost_map_names(gateway: Gateway, wildcard_deployments: None) -> None: + assert any(model_id.startswith("databricks/databricks-") for model_id in _listed_model_ids(gateway)) + + +def test_model_info_keeps_provider_model_for_expanded_deployments(gateway: Gateway, wildcard_deployments: None) -> None: + data: Final = gateway.get("/model/info")["data"] + assert isinstance(data, list) + expanded: Final = [ + row + for row in (object_value(entry) for entry in data) + if string_value(row["model_name"]).startswith("databricks/system.ai.") + ] + assert expanded, "model/info returned no expanded rows for databricks/system.ai.*" + bad: Final = [ + row["model_name"] + for row in expanded + if "system.ai.databricks/" in string_value(object_value(row["litellm_params"])["model"]) + ] + assert not bad, f"litellm_params.model carries the corrupted expanded name: {bad}" From b258196787627e987212e84f41565fe23f1915ad Mon Sep 17 00:00:00 2001 From: gabriele Date: Thu, 15 Oct 2026 10:20:55 +0000 Subject: [PATCH 07/22] test(proxy): cover full-map wildcard prefix expansion Co-authored-by: yuneng-jiang Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/unit/proxy/auth/test_model_checks.py | 112 +++++++++++++++------ 1 file changed, 79 insertions(+), 33 deletions(-) diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 3f2059fa1eac..b181f0fc1190 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -247,9 +247,7 @@ def test_get_key_models_does_not_mutate_input(): ), ], ) -def test_get_complete_model_list_order( - key_models, team_models, proxy_model_list, model_list, expected -): +def test_get_complete_model_list_order(key_models, team_models, proxy_model_list, model_list, expected): """ Test that get_complete_model_list preserves order """ @@ -402,9 +400,7 @@ def fake_get_provider_models(provider, litellm_params=None): captured_params["api_key"] = litellm_params.api_key captured_params["api_version"] = litellm_params.api_version captured_params["credential_name"] = litellm_params.litellm_credential_name - captured_params["has_unexpected_field"] = hasattr( - litellm_params, "unexpected_field" - ) + captured_params["has_unexpected_field"] = hasattr(litellm_params, "unexpected_field") return ["gpt-4o"] monkeypatch.setattr(model_checks, "get_provider_models", fake_get_provider_models) @@ -449,9 +445,7 @@ def test_wildcard_custom_prefix_does_not_stack_provider_prefix(monkeypatch): result = get_known_models_from_wildcard( wildcard_model="ollama_server1/*", - litellm_params=LiteLLM_Params( - model="ollama_chat/*", custom_llm_provider="ollama_chat" - ), + litellm_params=LiteLLM_Params(model="ollama_chat/*", custom_llm_provider="ollama_chat"), ) assert result == ["ollama_server1/gemma3:1b", "ollama_server1/llama3:8b"] @@ -478,43 +472,97 @@ def test_wildcard_custom_prefix_keeps_org_segment_for_non_provider_first_segment result = get_known_models_from_wildcard( wildcard_model="my_hf/*", - litellm_params=LiteLLM_Params( - model="huggingface/*", custom_llm_provider="huggingface" - ), + litellm_params=LiteLLM_Params(model="huggingface/*", custom_llm_provider="huggingface"), ) assert result == ["my_hf/meta-llama/Llama-3-8B"] -def test_partial_prefix_wildcard_expands_against_stripped_provider_ids(monkeypatch): +def test_cloudflare_partial_wildcard_matches_provider_stripped_cost_map_ids( + monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: import litellm - from litellm.proxy.auth import model_checks from litellm.proxy.auth.model_checks import get_known_models_from_wildcard from litellm.types.router import LiteLLM_Params - provider_ids: Final = list(litellm.models_by_provider["databricks"]) - monkeypatch.setattr( - model_checks, - "get_provider_models", - lambda provider, litellm_params=None: provider_ids, + wildcard: Final = "cloudflare/@cf/*" + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + + result: Final = tuple( + get_known_models_from_wildcard( + wildcard_model=wildcard, + litellm_params=LiteLLM_Params(model=wildcard), + ) ) + assert result + assert all(model_id.startswith("cloudflare/@cf/") for model_id in result) + assert all(model_id in litellm.model_cost for model_id in result) + assert all(model_id.count("@cf/") == 1 for model_id in result) + + +def test_gemini_partial_wildcard_deduplicates_cost_map_ids( + monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + wildcard: Final = "gemini/gemini-2.5*" + result: Final = get_known_models_from_wildcard( - wildcard_model="databricks/system.ai.*", - litellm_params=LiteLLM_Params( - model="databricks/system.ai.*", - api_key="x", - api_base="https://example.invalid", - ), + wildcard_model=wildcard, + litellm_params=LiteLLM_Params(model=wildcard), + ) + + assert len(result) == len(set(result)) + assert result.count("gemini/gemini-2.5-flash") == 1 + + +def test_gemini_model_info_partial_wildcard_has_unique_model_names( + monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + import litellm + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info + + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + wildcard: Final = "gemini/gemini-2.5*" + + expanded: Final = tuple( + expand_wildcard_deployments_for_model_info( + [{"model_name": wildcard, "litellm_params": {"model": wildcard}}] + ) + ) + model_names: Final = tuple(str(deployment["model_name"]) for deployment in expanded) + + assert model_names + assert len(model_names) == len(set(model_names)) + + +def test_static_databricks_partial_wildcard_splices_cost_map_ids( + monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + wildcard: Final = "databricks/system.ai.*" + provider_ids: Final = tuple(litellm.models_by_provider["databricks"]) + expected_ids: Final = frozenset( + f"databricks/system.ai.{model_id.partition('/')[-1]}" for model_id in provider_ids ) - expected: Final = [f"databricks/system.ai.{model_id.partition('/')[-1]}" for model_id in provider_ids] - assert result == expected - assert all( - model_id.startswith("databricks/system.ai.") and "/" not in model_id.removeprefix("databricks/system.ai.") - for model_id in result + result: Final = get_known_models_from_wildcard( + wildcard_model=wildcard, + litellm_params=LiteLLM_Params(model=wildcard, api_key="x", api_base="https://example.invalid"), ) + assert result + assert frozenset(result) == expected_ids + assert all(model_id.startswith("databricks/system.ai.") for model_id in result) + def test_wildcard_credential_hydration_preserves_missing_credential_name( monkeypatch, @@ -958,9 +1006,7 @@ def test_add_known_models_refreshes_models_by_provider_for_wildcard_expansion(): assert fake_model not in litellm.models_by_provider["vertex_ai"] try: litellm.add_known_models( - model_cost_map={ - fake_model: {"litellm_provider": "vertex_ai-language-models", "mode": "chat"} - } + model_cost_map={fake_model: {"litellm_provider": "vertex_ai-language-models", "mode": "chat"}} ) assert fake_model in litellm.models_by_provider["vertex_ai"] assert litellm.models_by_provider is captured_reference From e24584df8130eac5850abd71695ccf918dd1481c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 16 Oct 2026 00:45:07 -0700 Subject: [PATCH 08/22] fix(proxy): list partial wildcard ids through the deployment's own wildcard, drop guessed splices A partial wildcard now lists model_name's literal plus the suffix the router would substitute into litellm_params.model, so every listed id routes to a known provider model. A partial wildcard matching no known model lists nothing, and the health check falls back to main's cheapest candidates. --- .../health_check_helpers.py | 4 +- litellm/proxy/auth/model_checks.py | 64 +++++++++---------- .../test_wildcard_partial_prefix_expansion.py | 52 +++++++-------- .../test_health_check_helpers.py | 14 ++-- tests/unit/proxy/auth/test_model_checks.py | 63 ++++++++++++++---- 5 files changed, 117 insertions(+), 80 deletions(-) diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 9255b27ea42a..41d339c063e4 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -93,10 +93,10 @@ def _wildcard_health_check_models( known_providers: Final = frozenset(provider.value for provider in LlmProviders) literal_prefix: Final = wildcard_suffix.replace("*", "") stripped_ids: Final = tuple(_strip_known_provider_prefix(model, known_providers) for model in candidate_models) - matching: Final = [stripped for stripped in stripped_ids if stripped.startswith(literal_prefix)] + matching: Final = tuple(stripped for stripped in stripped_ids if stripped.startswith(literal_prefix)) if matching: return tuple(f"{custom_llm_provider}/{stripped}" for stripped in matching[:3]) - return tuple(f"{custom_llm_provider}/{wildcard_suffix.replace('*', stripped, 1)}" for stripped in stripped_ids[:3]) + return tuple(candidate_models[:3]) class HealthCheckHelpers: diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 340586d3628a..50f7a2d07248 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -274,32 +274,26 @@ def _strip_known_provider_prefix(model: str, known_providers: frozenset[str]) -> return model_suffix if sep and leading in known_providers else model -def _apply_partial_wildcard_prefix( +def _deployment_wildcard_literal(wildcard_literal: str, litellm_params: LiteLLM_Params | None) -> str: + if litellm_params is None or not _check_wildcard_routing(litellm_params.model): + return wildcard_literal + _, sep, deployment_suffix = litellm_params.model.partition("/") + return deployment_suffix.replace("*", "") if sep else wildcard_literal + + +def _expand_partial_wildcard( provider_models: Sequence[str], - model_prefix: str, + wildcard_provider_prefix: str, + wildcard_literal: str, + deployment_literal: str, known_providers: frozenset[str], - live_model_listing: bool, ) -> tuple[str, ...]: - matching_provider_models: Final = tuple( - model - for model in provider_models - if _strip_known_provider_prefix(model, known_providers).startswith(model_prefix) + stripped_models: Final = (_strip_known_provider_prefix(model, known_providers) for model in provider_models) + return tuple( + f"{wildcard_provider_prefix}/{wildcard_literal}{stripped.removeprefix(deployment_literal)}" + for stripped in stripped_models + if stripped.startswith(deployment_literal) ) - if matching_provider_models: - return matching_provider_models - if live_model_listing: - return () - return tuple(f"{model_prefix}{_strip_known_provider_prefix(model, known_providers)}" for model in provider_models) - - -def _provider_has_live_model_listing(provider: str) -> bool: - if not litellm.check_provider_endpoint: - return False - try: - llm_provider: Final = LlmProviders(provider) - except ValueError: - return False - return ProviderConfigManager.get_provider_model_info(model=None, provider=llm_provider) is not None def _normalize_known_wildcard_model( @@ -344,27 +338,27 @@ def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_ return [] known_providers: Final = frozenset(provider.value for provider in LlmProviders) + wildcard_literal: Final = wildcard_suffix.replace("*", "") wildcard_models: Final = ( - provider_models + tuple( + _normalize_known_wildcard_model( + model=model, + wildcard_provider_prefix=wildcard_provider_prefix, + known_providers=known_providers, + ) + for model in provider_models + ) if wildcard_suffix == "*" - else _apply_partial_wildcard_prefix( + else _expand_partial_wildcard( provider_models=provider_models, - model_prefix=wildcard_suffix.replace("*", ""), - known_providers=known_providers, - live_model_listing=_provider_has_live_model_listing(provider), - ) - ) - - normalized_wildcard_models: Final = tuple( - _normalize_known_wildcard_model( - model=model, wildcard_provider_prefix=wildcard_provider_prefix, + wildcard_literal=wildcard_literal, + deployment_literal=_deployment_wildcard_literal(wildcard_literal, litellm_params), known_providers=known_providers, ) - for model in wildcard_models ) - return list(dict.fromkeys(normalized_wildcard_models)) + return list(dict.fromkeys(wildcard_models)) def expand_wildcard_deployments_for_model_info( diff --git a/tests/integration/routing/test_wildcard_partial_prefix_expansion.py b/tests/integration/routing/test_wildcard_partial_prefix_expansion.py index 01206a2163a8..b79924ec47fa 100644 --- a/tests/integration/routing/test_wildcard_partial_prefix_expansion.py +++ b/tests/integration/routing/test_wildcard_partial_prefix_expansion.py @@ -1,5 +1,5 @@ -"""Partial-prefix wildcards like ``databricks/system.ai.*`` must splice the literal -prefix onto provider-stripped cost-map ids, never onto the provider-prefixed key.""" +"""Partial wildcards like ``cloudflare/@cf/*`` list each callable id once on /v1/models and /model/info, +and a partial wildcard that matches no known id, like ``databricks/system.ai.*``, lists no guessed ids.""" from collections.abc import Iterator from typing import Final @@ -8,7 +8,7 @@ from integration._support.client import Gateway, object_value, string_value -PATTERNS: Final = ("databricks/system.ai.*", "databricks/*") +PATTERNS: Final = ("cloudflare/@cf/*", "databricks/system.ai.*", "databricks/*") def _registered_model_id(gateway: Gateway, pattern: str) -> str: @@ -42,30 +42,32 @@ def _listed_model_ids(gateway: Gateway) -> tuple[str, ...]: return tuple(string_value(object_value(entry)["id"]) for entry in data) -def test_partial_prefix_wildcard_expands_to_unity_catalog_names(gateway: Gateway, wildcard_deployments: None) -> None: - prefixed: Final = tuple( - model_id for model_id in _listed_model_ids(gateway) if model_id.startswith("databricks/system.ai.") - ) - assert prefixed - assert all("/" not in model_id.removeprefix("databricks/system.ai.") for model_id in prefixed) +def _model_info_names(gateway: Gateway) -> tuple[str, ...]: + data: Final = gateway.get("/model/info")["data"] + assert isinstance(data, list) + return tuple(string_value(object_value(entry)["model_name"]) for entry in data) -def test_provider_wildcard_still_expands_to_cost_map_names(gateway: Gateway, wildcard_deployments: None) -> None: - assert any(model_id.startswith("databricks/databricks-") for model_id in _listed_model_ids(gateway)) +def test_partial_wildcard_lists_provider_ids_once_on_both_endpoints( + gateway: Gateway, wildcard_deployments: None +) -> None: + listed: Final = tuple(model_id for model_id in _listed_model_ids(gateway) if model_id.startswith("cloudflare/")) + model_info: Final = tuple(name for name in _model_info_names(gateway) if name.startswith("cloudflare/")) + assert listed + assert all(model_id.startswith("cloudflare/@cf/") and model_id.count("@cf/") == 1 for model_id in listed) + assert len(listed) == len(set(listed)) + assert sorted(model_info) == sorted(listed) -def test_model_info_keeps_provider_model_for_expanded_deployments(gateway: Gateway, wildcard_deployments: None) -> None: - data: Final = gateway.get("/model/info")["data"] - assert isinstance(data, list) - expanded: Final = [ - row - for row in (object_value(entry) for entry in data) - if string_value(row["model_name"]).startswith("databricks/system.ai.") - ] - assert expanded, "model/info returned no expanded rows for databricks/system.ai.*" - bad: Final = [ - row["model_name"] - for row in expanded - if "system.ai.databricks/" in string_value(object_value(row["litellm_params"])["model"]) + +def test_partial_wildcard_matching_no_known_id_lists_no_guessed_ids( + gateway: Gateway, wildcard_deployments: None +) -> None: + assert [model_id for model_id in _listed_model_ids(gateway) if model_id.startswith("databricks/system.ai.")] == [] + assert [name for name in _model_info_names(gateway) if name.startswith("databricks/system.ai.")] == [ + "databricks/system.ai.*" ] - assert not bad, f"litellm_params.model carries the corrupted expanded name: {bad}" + + +def test_provider_wildcard_still_expands_to_cost_map_names(gateway: Gateway, wildcard_deployments: None) -> None: + assert any(model_id.startswith("databricks/databricks-") for model_id in _listed_model_ids(gateway)) diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index 1bd7056e6e5a..12a90fe36199 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -1525,16 +1525,18 @@ def test_wildcard_health_check_models_partial_prefix_filters_candidates_to_liter ) == ("openai/gpt-4o-mini", "openai/gpt-4.1-mini", "openai/gpt-4o") -def test_wildcard_health_check_models_partial_prefix_splices_suffix_around_star(): - assert _wildcard_health_check_models("ft:*", "openai", ["gpt-4o-mini"]) == ("openai/ft:gpt-4o-mini",) +def test_wildcard_health_check_models_partial_prefix_matching_nothing_keeps_cheapest_candidates(): + assert _wildcard_health_check_models("ft:*", "openai", ["gpt-4o-mini", "gpt-4.1-nano"]) == ( + "gpt-4o-mini", + "gpt-4.1-nano", + ) @pytest.mark.asyncio -async def test_ahealth_check_partial_prefix_wildcard_probes_spliced_model_id(): +async def test_ahealth_check_partial_prefix_wildcard_never_probes_a_spliced_model_id(): from litellm.litellm_core_utils.llm_request_utils import pick_cheapest_chat_models_from_llm_provider cheapest: Final = pick_cheapest_chat_models_from_llm_provider(custom_llm_provider="databricks", n=3) - stripped: Final = tuple(model_id.partition("/")[-1] for model_id in cheapest) captured: dict[str, object] = {} @@ -1547,5 +1549,5 @@ async def _capture(**kwargs): mode="chat", ) - assert captured["model"] == f"databricks/system.ai.{stripped[0]}" - assert captured["fallbacks"] == [f"databricks/system.ai.{model_id}" for model_id in stripped[1:]] + assert captured["model"] == cheapest[0] + assert captured["fallbacks"] == list(cheapest[1:]) diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index b181f0fc1190..496d57e1cd08 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -540,28 +540,67 @@ def test_gemini_model_info_partial_wildcard_has_unique_model_names( assert len(model_names) == len(set(model_names)) -def test_static_databricks_partial_wildcard_splices_cost_map_ids( - monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +@pytest.mark.parametrize( + ("model_name", "deployment_model"), + [ + ("cloudflare/@cf/*", "cloudflare/@cf/*"), + ("gemini/gemini-2.5*", "gemini/gemini-2.5*"), + ("vertex_ai/gemini-*", "vertex_ai/gemini-*"), + ("fireworks_ai/accounts/fireworks/models/llama*", "fireworks_ai/accounts/fireworks/models/llama*"), + ("genai/test/*", "openai/*"), + ("custom/gemini-*", "gemini/*"), + ("team/claude-*", "anthropic/claude-*"), + ], +) +def test_every_listed_partial_wildcard_id_routes_to_a_known_provider_model( + monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, model_name: str, deployment_model: str ) -> None: import litellm from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.router_utils.pattern_match_deployments import PatternMatchRouter from litellm.types.router import LiteLLM_Params monkeypatch.setattr(litellm, "check_provider_endpoint", False) - wildcard: Final = "databricks/system.ai.*" - provider_ids: Final = tuple(litellm.models_by_provider["databricks"]) - expected_ids: Final = frozenset( - f"databricks/system.ai.{model_id.partition('/')[-1]}" for model_id in provider_ids + provider: Final = deployment_model.partition("/")[0] + known_ids: Final = frozenset( + model_id.removeprefix(f"{provider}/") for model_id in litellm.models_by_provider[provider] ) + router: Final = PatternMatchRouter() + router.add_pattern(model_name, {"model_name": model_name, "litellm_params": {"model": deployment_model}}) - result: Final = get_known_models_from_wildcard( - wildcard_model=wildcard, - litellm_params=LiteLLM_Params(model=wildcard, api_key="x", api_base="https://example.invalid"), + listed: Final = get_known_models_from_wildcard( + wildcard_model=model_name, + litellm_params=LiteLLM_Params(model=deployment_model), + ) + routed_ids: Final = tuple( + str((router.route(model_id) or [{}])[0].get("litellm_params", {}).get("model", "")) for model_id in listed ) - assert result - assert frozenset(result) == expected_ids - assert all(model_id.startswith("databricks/system.ai.") for model_id in result) + assert listed + assert len(listed) == len(set(listed)) + assert [routed for routed in routed_ids if routed.removeprefix(f"{provider}/") not in known_ids] == [] + + +@pytest.mark.parametrize( + "wildcard", + ["databricks/system.ai.*", "openai/ft:gpt-4o*", "anthropic/claude-nonexistent-*"], +) +def test_partial_wildcard_matching_no_known_model_lists_nothing( + monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, wildcard: str +) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + + assert ( + get_known_models_from_wildcard( + wildcard_model=wildcard, + litellm_params=LiteLLM_Params(model=wildcard, api_key="x", api_base="https://example.invalid"), + ) + == [] + ) def test_wildcard_credential_hydration_preserves_missing_credential_name( From 5041e7eb65334457c74bcab844beb0af39c06847 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 16 Oct 2026 02:10:34 -0700 Subject: [PATCH 09/22] fix(proxy): list wildcard ids only for routes that rename the namespace A wildcard route now lists ids only when it renames the namespace in front of its deployment's model pattern, so every listed id routes to the model it names. Routes that rewrite part of the model name, or a bare * over a partial deployment, list nothing instead of ids that 404. Strip only a provider-family prefix from cost-map ids so vendor namespaces like openai/ under wandb survive, share that helper with the wildcard health check, and de-dupe ids across deployments of one wildcard. --- .../get_llm_provider_logic.py | 13 ++ .../health_check_helpers.py | 28 ++-- litellm/proxy/auth/model_checks.py | 156 ++++++------------ .../test_health_check_helpers.py | 58 +++++-- tests/unit/proxy/auth/test_model_checks.py | 133 +++++++++++++-- 5 files changed, 245 insertions(+), 143 deletions(-) diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index c9e5167e7a9a..149491360091 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -14,6 +14,9 @@ from litellm.secret_managers.main import get_secret, get_secret_str from ..types.router import GenericLiteLLMParams, LiteLLM_Params +from ..types.utils import LlmProviders + +_KNOWN_PROVIDERS: Final = frozenset(provider.value for provider in LlmProviders) def _endpoint_matches_api_base(endpoint: str, api_base: str) -> bool: @@ -146,6 +149,16 @@ def declared_authenticating_provider(model: str | None, custom_llm_provider: str return declared if declared in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO else None +def strip_provider_family_prefix(model: str, custom_llm_provider: str) -> str: + leading, sep, model_id = model.partition("/") + if not sep: + return model + is_provider_family: Final = leading == custom_llm_provider or ( + leading in _KNOWN_PROVIDERS and custom_llm_provider.startswith(f"{leading}_") + ) + return model_id if is_provider_family else model + + def inferred_provider(model: str | None) -> str | None: if not model: return None diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 41d339c063e4..68d4935b1517 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -3,13 +3,16 @@ """ import base64 +import re +import sys from collections.abc import Awaitable, Callable, Mapping, Sequence from typing import TYPE_CHECKING, Final, Literal +from litellm.litellm_core_utils.get_llm_provider_logic import strip_provider_family_prefix from litellm.llms.base_llm.ocr.transformation import DocumentType from litellm.rust_bridge import runtime from litellm.rust_bridge.ocr.entrypoints import NATIVE_OCR_HEALTH_CHECK_DOCUMENT -from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS, LlmProviders +from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging @@ -80,23 +83,17 @@ def _ocr_health_check_document(model: str, custom_llm_provider: str) -> Document return native(model, custom_llm_provider) -def _strip_known_provider_prefix(model: str, known_providers: frozenset[str]) -> str: - leading, sep, model_suffix = model.partition("/") - return model_suffix if sep and leading in known_providers else model - - def _wildcard_health_check_models( wildcard_suffix: str, custom_llm_provider: str, candidate_models: Sequence[str] ) -> tuple[str, ...]: if wildcard_suffix == "*": return tuple(candidate_models[:3]) - known_providers: Final = frozenset(provider.value for provider in LlmProviders) - literal_prefix: Final = wildcard_suffix.replace("*", "") - stripped_ids: Final = tuple(_strip_known_provider_prefix(model, known_providers) for model in candidate_models) - matching: Final = tuple(stripped for stripped in stripped_ids if stripped.startswith(literal_prefix)) - if matching: - return tuple(f"{custom_llm_provider}/{stripped}" for stripped in matching[:3]) - return tuple(candidate_models[:3]) + suffix_regex: Final = re.compile(re.escape(wildcard_suffix).replace(r"\*", ".*")) + model_ids: Final = (strip_provider_family_prefix(model, custom_llm_provider) for model in candidate_models) + matching: Final = tuple( + f"{custom_llm_provider}/{model_id}" for model_id in model_ids if suffix_regex.fullmatch(model_id) + ) + return matching[:3] or tuple(candidate_models[:3]) class HealthCheckHelpers: @@ -112,8 +109,9 @@ async def ahealth_check_wildcard_models( pick_cheapest_chat_models_from_llm_provider, ) - # this is a wildcard model, we need to pick a random model from the provider - cheapest_models = pick_cheapest_chat_models_from_llm_provider(custom_llm_provider=custom_llm_provider, n=10_000) + cheapest_models = pick_cheapest_chat_models_from_llm_provider( + custom_llm_provider=custom_llm_provider, n=sys.maxsize + ) if len(cheapest_models) == 0: raise Exception( f"Unable to health check wildcard model for provider {custom_llm_provider}. Add a model on your config.yaml or contribute here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json" diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 50f7a2d07248..98c923338061 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -1,12 +1,15 @@ # What is this? ## Common checks for /v1/models and `/model/info` import copy +import re from collections.abc import Sequence +from itertools import chain from typing import Final import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.litellm_core_utils.get_llm_provider_logic import strip_provider_family_prefix from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth from litellm.repositories.object_permission_repository import ObjectPermissionRepository from litellm.router import Router @@ -269,96 +272,37 @@ def _hydrate_litellm_credential_name( return litellm_params -def _strip_known_provider_prefix(model: str, known_providers: frozenset[str]) -> str: - leading, sep, model_suffix = model.partition("/") - return model_suffix if sep and leading in known_providers else model - - -def _deployment_wildcard_literal(wildcard_literal: str, litellm_params: LiteLLM_Params | None) -> str: - if litellm_params is None or not _check_wildcard_routing(litellm_params.model): - return wildcard_literal - _, sep, deployment_suffix = litellm_params.model.partition("/") - return deployment_suffix.replace("*", "") if sep else wildcard_literal - - -def _expand_partial_wildcard( - provider_models: Sequence[str], - wildcard_provider_prefix: str, - wildcard_literal: str, - deployment_literal: str, - known_providers: frozenset[str], -) -> tuple[str, ...]: - stripped_models: Final = (_strip_known_provider_prefix(model, known_providers) for model in provider_models) - return tuple( - f"{wildcard_provider_prefix}/{wildcard_literal}{stripped.removeprefix(deployment_literal)}" - for stripped in stripped_models - if stripped.startswith(deployment_literal) - ) - - -def _normalize_known_wildcard_model( - model: str, - wildcard_provider_prefix: str, - known_providers: frozenset[str], -) -> str: - if model.startswith(f"{wildcard_provider_prefix}/"): - return model - return f"{wildcard_provider_prefix}/{_strip_known_provider_prefix(model, known_providers)}" - - def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_Params | None = None) -> list[str]: - wildcard_model_to_expand: Final = ( - litellm_params.model - if wildcard_model == "*" - and litellm_params is not None - and _check_wildcard_routing(litellm_params.model) - and "/" in litellm_params.model + deployment_model: Final = ( + litellm_params.model if litellm_params is not None and _check_wildcard_routing(litellm_params.model) else None + ) + _, deployment_sep, deployment_suffix = (deployment_model or "").partition("/") + route: Final = ( + deployment_model + if wildcard_model == "*" and deployment_model is not None and deployment_suffix == "*" else wildcard_model ) - try: - wildcard_provider_prefix, wildcard_suffix = wildcard_model_to_expand.split("/", 1) - except ValueError: # safely fail + route_provider, route_sep, route_suffix = route.partition("/") + if not route_sep: return [] - # Use provider from litellm_params when available, otherwise from wildcard prefix - # (e.g., "openai" from "openai/*" - needed for BYOK where wildcard isn't in router) - if litellm_params is not None: - try: - provider = litellm_params.model.split("/", 1)[0] - except ValueError: - provider = wildcard_provider_prefix - else: - provider = wildcard_provider_prefix - - litellm_params = _hydrate_litellm_credential_name(litellm_params) - - provider_models: Final = get_provider_models(provider=provider, litellm_params=litellm_params) + model_pattern: Final = deployment_suffix if deployment_sep else route_suffix + route_namespace: Final = route.removesuffix(model_pattern) + if not route.endswith(model_pattern) or not route_namespace.endswith("/"): + return [] + provider: Final = litellm_params.model.split("/", 1)[0] if litellm_params is not None else route_provider + provider_models: Final = get_provider_models( + provider=provider, litellm_params=_hydrate_litellm_credential_name(litellm_params) + ) if provider_models is None: return [] - known_providers: Final = frozenset(provider.value for provider in LlmProviders) - wildcard_literal: Final = wildcard_suffix.replace("*", "") - - wildcard_models: Final = ( - tuple( - _normalize_known_wildcard_model( - model=model, - wildcard_provider_prefix=wildcard_provider_prefix, - known_providers=known_providers, - ) - for model in provider_models - ) - if wildcard_suffix == "*" - else _expand_partial_wildcard( - provider_models=provider_models, - wildcard_provider_prefix=wildcard_provider_prefix, - wildcard_literal=wildcard_literal, - deployment_literal=_deployment_wildcard_literal(wildcard_literal, litellm_params), - known_providers=known_providers, - ) + model_regex: Final = re.compile(re.escape(model_pattern).replace(r"\*", ".*")) + model_ids: Final = (strip_provider_family_prefix(model, provider) for model in provider_models) + return list( + dict.fromkeys(f"{route_namespace}{model_id}" for model_id in model_ids if model_regex.fullmatch(model_id)) ) - return list(dict.fromkeys(wildcard_models)) def expand_wildcard_deployments_for_model_info( @@ -415,37 +359,41 @@ def expand_wildcard_deployments_for_model_info( return expanded +def _models_listed_for_wildcard_route( + model: str, + return_wildcard_route: bool, + llm_router: Router | None, + team_id: str | None, +) -> tuple[str, ...]: + model_list: Final = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router else None + expanded: Final = ( + chain.from_iterable( + get_known_models_from_wildcard( + wildcard_model=model, + litellm_params=LiteLLM_Params(**router_model["litellm_params"]), + ) + for router_model in model_list + ) + if model_list + else get_known_models_from_wildcard(wildcard_model=model, litellm_params=None) + ) + return (model, *expanded) if return_wildcard_route else tuple(expanded) + + def _get_wildcard_models( unique_models: list[str], return_wildcard_routes: bool | None = False, llm_router: Router | None = None, team_id: str | None = None, ) -> list[str]: - models_to_remove: Final = set() - all_wildcard_models: Final = [] - for model in unique_models: - if _check_wildcard_routing(model=model): - if return_wildcard_routes: - all_wildcard_models.append(model) - - models_to_remove.add(model) - - model_list = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router else None - if model_list: - for router_model in model_list: - all_wildcard_models.extend( - get_known_models_from_wildcard( - wildcard_model=model, - litellm_params=LiteLLM_Params(**router_model["litellm_params"]), - ) - ) - else: - all_wildcard_models.extend(get_known_models_from_wildcard(wildcard_model=model, litellm_params=None)) - - for model in models_to_remove: + wildcard_routes: Final = tuple(model for model in unique_models if _check_wildcard_routing(model=model)) + for model in wildcard_routes: unique_models.remove(model) - - return all_wildcard_models + listed: Final = chain.from_iterable( + _models_listed_for_wildcard_route(model, bool(return_wildcard_routes), llm_router, team_id) + for model in wildcard_routes + ) + return list(dict.fromkeys(listed)) def get_all_fallbacks( diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index 12a90fe36199..f7893797d2a3 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -1525,6 +1525,22 @@ def test_wildcard_health_check_models_partial_prefix_filters_candidates_to_liter ) == ("openai/gpt-4o-mini", "openai/gpt-4.1-mini", "openai/gpt-4o") +def test_wildcard_health_check_models_keeps_a_vendor_namespace_that_matches_another_provider_name(): + candidates: Final = ("openai/gpt-oss-20b", "wandb/openai/gpt-oss-120b") + + assert _wildcard_health_check_models("openai/gpt*", "wandb", candidates) == ( + "wandb/openai/gpt-oss-20b", + "wandb/openai/gpt-oss-120b", + ) + assert "wandb/gpt-oss-20b" not in _wildcard_health_check_models("gpt*", "wandb", candidates) + + +def test_wildcard_health_check_models_strips_the_provider_family_prefix(): + assert _wildcard_health_check_models("gemma*", "ollama_chat", ["ollama/llama3:8b", "ollama/gemma3:1b"]) == ( + "ollama_chat/gemma3:1b", + ) + + def test_wildcard_health_check_models_partial_prefix_matching_nothing_keeps_cheapest_candidates(): assert _wildcard_health_check_models("ft:*", "openai", ["gpt-4o-mini", "gpt-4.1-nano"]) == ( "gpt-4o-mini", @@ -1533,21 +1549,35 @@ def test_wildcard_health_check_models_partial_prefix_matching_nothing_keeps_chea @pytest.mark.asyncio -async def test_ahealth_check_partial_prefix_wildcard_never_probes_a_spliced_model_id(): +async def test_ahealth_check_partial_prefix_wildcard_never_probes_a_spliced_model_id( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: from litellm.litellm_core_utils.llm_request_utils import pick_cheapest_chat_models_from_llm_provider - cheapest: Final = pick_cheapest_chat_models_from_llm_provider(custom_llm_provider="databricks", n=3) - - captured: dict[str, object] = {} - - async def _capture(**kwargs): - captured.update(kwargs) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + cheapest: Final = pick_cheapest_chat_models_from_llm_provider(custom_llm_provider="databricks", n=1) + upstream: Final = respx_mock.post("https://dbx.test/serving-endpoints/chat/completions").respond( + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 0, + "model": "probe", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + ) - with patch("litellm.acompletion", side_effect=_capture): - await ahealth_check( - model_params={"model": "databricks/system.ai.*", "api_key": "x", "api_base": "https://example.invalid"}, - mode="chat", - ) + result: Final = await ahealth_check( + model_params={ + "model": "databricks/system.ai.*", + "api_key": "x", + "api_base": "https://dbx.test/serving-endpoints", + }, + mode="chat", + ) - assert captured["model"] == cheapest[0] - assert captured["fallbacks"] == list(cheapest[1:]) + assert "error" not in result, result + assert upstream.call_count == 1 + assert json.loads(upstream.calls[0].request.content)["model"] == cheapest[0].removeprefix("databricks/") diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 496d57e1cd08..32c42e82128b 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -541,19 +541,26 @@ def test_gemini_model_info_partial_wildcard_has_unique_model_names( @pytest.mark.parametrize( - ("model_name", "deployment_model"), + ("model_name", "deployment_model", "route_namespace"), [ - ("cloudflare/@cf/*", "cloudflare/@cf/*"), - ("gemini/gemini-2.5*", "gemini/gemini-2.5*"), - ("vertex_ai/gemini-*", "vertex_ai/gemini-*"), - ("fireworks_ai/accounts/fireworks/models/llama*", "fireworks_ai/accounts/fireworks/models/llama*"), - ("genai/test/*", "openai/*"), - ("custom/gemini-*", "gemini/*"), - ("team/claude-*", "anthropic/claude-*"), + ("cloudflare/@cf/*", "cloudflare/@cf/*", "cloudflare/"), + ("gemini/gemini-2.5*", "gemini/gemini-2.5*", "gemini/"), + ("vertex_ai/gemini-*", "vertex_ai/gemini-*", "vertex_ai/"), + ( + "fireworks_ai/accounts/fireworks/models/llama*", + "fireworks_ai/accounts/fireworks/models/llama*", + "fireworks_ai/", + ), + ("genai/test/*", "openai/*", "genai/test/"), + ("team/claude-*", "anthropic/claude-*", "team/"), ], ) -def test_every_listed_partial_wildcard_id_routes_to_a_known_provider_model( - monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, model_name: str, deployment_model: str +def test_every_listed_wildcard_id_routes_to_the_known_model_it_names( + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, + model_name: str, + deployment_model: str, + route_namespace: str, ) -> None: import litellm from litellm.proxy.auth.model_checks import get_known_models_from_wildcard @@ -579,6 +586,112 @@ def test_every_listed_partial_wildcard_id_routes_to_a_known_provider_model( assert listed assert len(listed) == len(set(listed)) assert [routed for routed in routed_ids if routed.removeprefix(f"{provider}/") not in known_ids] == [] + assert [ + (model_id, routed) + for model_id, routed in zip(listed, routed_ids) + if routed != f"{provider}/{model_id.removeprefix(route_namespace)}" + ] == [] + + +@pytest.mark.parametrize( + ("model_name", "deployment_model"), + [ + ("custom/gemini-*", "gemini/*"), + ("openai/gpt-4*", "openai/*"), + ("team/*", "anthropic/claude-*"), + ("*", "cloudflare/@cf/*"), + ("*", "gemini/gemini-2.5*"), + ], +) +def test_route_that_rewrites_part_of_the_model_name_lists_no_ids( + monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, model_name: str, deployment_model: str +) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + + assert get_known_models_from_wildcard( + wildcard_model=deployment_model, litellm_params=LiteLLM_Params(model=deployment_model) + ) + assert ( + get_known_models_from_wildcard(wildcard_model=model_name, litellm_params=LiteLLM_Params(model=deployment_model)) + == [] + ) + + +def test_partial_wildcard_without_a_deployment_lists_the_same_ids_as_with_one( + monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + wildcard: Final = "cloudflare/@cf/*" + + without_deployment: Final = get_known_models_from_wildcard(wildcard_model=wildcard, litellm_params=None) + + assert without_deployment + assert without_deployment == get_known_models_from_wildcard( + wildcard_model=wildcard, litellm_params=LiteLLM_Params(model=wildcard) + ) + + +@pytest.mark.parametrize( + ("wildcard", "expected"), + [ + ("wandb/*", ["wandb/openai/gpt-oss-20b", "wandb/openai/gpt-oss-120b"]), + ("wandb/openai/*", ["wandb/openai/gpt-oss-20b", "wandb/openai/gpt-oss-120b"]), + ("wandb/gpt*", []), + ], +) +def test_wildcard_keeps_a_vendor_namespace_that_matches_another_provider_name( + monkeypatch: pytest.MonkeyPatch, wildcard: str, expected: list[str] +) -> None: + from litellm.proxy.auth import model_checks + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr( + model_checks, + "get_provider_models", + lambda provider, litellm_params=None: ["openai/gpt-oss-20b", "wandb/openai/gpt-oss-120b"], + ) + + assert get_known_models_from_wildcard(wildcard_model=wildcard, litellm_params=LiteLLM_Params(model=wildcard)) == ( + expected + ) + + +def test_complete_model_list_lists_each_id_once_across_deployments_of_one_wildcard( + monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + import litellm + from litellm import Router + from litellm.proxy.auth.model_checks import get_complete_model_list + + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + wildcard: Final = "gemini/gemini-2.5*" + router: Final = Router( + model_list=[ + {"model_name": wildcard, "litellm_params": {"model": wildcard, "api_key": "key-a"}}, + {"model_name": wildcard, "litellm_params": {"model": wildcard, "api_key": "key-b"}}, + ] + ) + + listed: Final = get_complete_model_list( + key_models=[], + team_models=[], + proxy_model_list=[wildcard], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + + assert listed + assert len(listed) == len(set(listed)) @pytest.mark.parametrize( From a32b99b372c3b8bd4253d73f0736465e62beefea Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 16 Oct 2026 03:26:49 -0700 Subject: [PATCH 10/22] fix(proxy): bound wildcard matching and list /model/info ids by the same rule Wildcard ids are matched with fnmatchcase instead of a hand-built regex, so a key whose models hold many stars no longer backtracks for minutes. /model/info now passes the route name to the listing rule, so a * route over a partial wildcard lists the same ids as /v1/models, and /v1/models de-dupes an explicit model that a wildcard also covers. --- .../get_llm_provider_logic.py | 7 +++++ .../health_check_helpers.py | 8 ++--- litellm/proxy/auth/model_checks.py | 28 +++++------------ .../test_health_check_helpers.py | 14 +++++++++ tests/unit/proxy/auth/test_model_checks.py | 31 ++++++++++++++++++- 5 files changed, 63 insertions(+), 25 deletions(-) diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 149491360091..95b4dd143c37 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -1,3 +1,5 @@ +import re +from fnmatch import fnmatchcase from typing import Final, cast from urllib.parse import urlparse @@ -17,6 +19,7 @@ from ..types.utils import LlmProviders _KNOWN_PROVIDERS: Final = frozenset(provider.value for provider in LlmProviders) +_GLOB_CHARS_OTHER_THAN_STAR: Final = re.compile(r"[\[?]") def _endpoint_matches_api_base(endpoint: str, api_base: str) -> bool: @@ -159,6 +162,10 @@ def strip_provider_family_prefix(model: str, custom_llm_provider: str) -> str: return model_id if is_provider_family else model +def model_matches_wildcard(model_id: str, wildcard: str) -> bool: + return fnmatchcase(model_id, _GLOB_CHARS_OTHER_THAN_STAR.sub(r"[\g<0>]", wildcard)) + + def inferred_provider(model: str | None) -> str | None: if not model: return None diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 68d4935b1517..07b3169b92e7 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -3,12 +3,11 @@ """ import base64 -import re import sys from collections.abc import Awaitable, Callable, Mapping, Sequence from typing import TYPE_CHECKING, Final, Literal -from litellm.litellm_core_utils.get_llm_provider_logic import strip_provider_family_prefix +from litellm.litellm_core_utils.get_llm_provider_logic import model_matches_wildcard, strip_provider_family_prefix from litellm.llms.base_llm.ocr.transformation import DocumentType from litellm.rust_bridge import runtime from litellm.rust_bridge.ocr.entrypoints import NATIVE_OCR_HEALTH_CHECK_DOCUMENT @@ -88,10 +87,11 @@ def _wildcard_health_check_models( ) -> tuple[str, ...]: if wildcard_suffix == "*": return tuple(candidate_models[:3]) - suffix_regex: Final = re.compile(re.escape(wildcard_suffix).replace(r"\*", ".*")) model_ids: Final = (strip_provider_family_prefix(model, custom_llm_provider) for model in candidate_models) matching: Final = tuple( - f"{custom_llm_provider}/{model_id}" for model_id in model_ids if suffix_regex.fullmatch(model_id) + f"{custom_llm_provider}/{model_id}" + for model_id in model_ids + if model_matches_wildcard(model_id, wildcard_suffix) ) return matching[:3] or tuple(candidate_models[:3]) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 98c923338061..6cd4dc745b55 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -1,7 +1,6 @@ # What is this? ## Common checks for /v1/models and `/model/info` import copy -import re from collections.abc import Sequence from itertools import chain from typing import Final @@ -9,7 +8,7 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.credential_accessor import CredentialAccessor -from litellm.litellm_core_utils.get_llm_provider_logic import strip_provider_family_prefix +from litellm.litellm_core_utils.get_llm_provider_logic import model_matches_wildcard, strip_provider_family_prefix from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth from litellm.repositories.object_permission_repository import ObjectPermissionRepository from litellm.router import Router @@ -249,9 +248,7 @@ def append_unique(models): team_id=team_id, ) - complete_model_list: Final = unique_models + all_wildcard_models - - return complete_model_list + return list(dict.fromkeys(chain(unique_models, all_wildcard_models))) def _hydrate_litellm_credential_name( @@ -298,10 +295,11 @@ def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_ if provider_models is None: return [] - model_regex: Final = re.compile(re.escape(model_pattern).replace(r"\*", ".*")) model_ids: Final = (strip_provider_family_prefix(model, provider) for model in provider_models) return list( - dict.fromkeys(f"{route_namespace}{model_id}" for model_id in model_ids if model_regex.fullmatch(model_id)) + dict.fromkeys( + f"{route_namespace}{model_id}" for model_id in model_ids if model_matches_wildcard(model_id, model_pattern) + ) ) @@ -320,18 +318,8 @@ def expand_wildcard_deployments_for_model_info( model_name = str(deployment.get("model_name") or "") raw_params = deployment.get("litellm_params") litellm_params_dict: dict[str, object] = raw_params if isinstance(raw_params, dict) else {} - litellm_model = str(litellm_params_dict.get("model") or "") - - # Determine the wildcard pattern to expand. - # Branch order matters: only fall to litellm_model when model_name is - # also a wildcard, so a concrete model_name is never overwritten. - if _check_wildcard_routing(model_name) and "/" in model_name: - wildcard_pattern = model_name - elif _check_wildcard_routing(model_name) and _check_wildcard_routing(litellm_model): - wildcard_pattern = litellm_model - elif _check_wildcard_routing(model_name): - wildcard_pattern = model_name - else: + + if not _check_wildcard_routing(model_name): expanded.append(deployment) continue @@ -341,7 +329,7 @@ def expand_wildcard_deployments_for_model_info( expanded.append(deployment) continue expanded_names = get_known_models_from_wildcard( - wildcard_model=wildcard_pattern, + wildcard_model=model_name, litellm_params=litellm_params, ) if not expanded_names: diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index f7893797d2a3..be3be40bc6fb 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -1541,6 +1541,20 @@ def test_wildcard_health_check_models_strips_the_provider_family_prefix(): ) +def test_wildcard_health_check_models_treats_glob_characters_other_than_star_literally(): + assert _wildcard_health_check_models("gpt-4?*", "openai", ["gpt-4o", "gpt-4?-preview"]) == ( + "openai/gpt-4?-preview", + ) + assert _wildcard_health_check_models("gpt-[4]*", "openai", ["gpt-4o", "gpt-[4]o"]) == ("openai/gpt-[4]o",) + + +@pytest.mark.timeout(5) +def test_wildcard_health_check_models_with_many_stars_returns_promptly(): + candidates: Final = ("gpt-4o-mini", "gpt-4.1-nano") + + assert _wildcard_health_check_models("*" * 30 + "!", "openai", candidates) == candidates + + def test_wildcard_health_check_models_partial_prefix_matching_nothing_keeps_cheapest_candidates(): assert _wildcard_health_check_models("ft:*", "openai", ["gpt-4o-mini", "gpt-4.1-nano"]) == ( "gpt-4o-mini", diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 32c42e82128b..93ef90b1fa53 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -607,10 +607,13 @@ def test_route_that_rewrites_part_of_the_model_name_lists_no_ids( monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, model_name: str, deployment_model: str ) -> None: import litellm - from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_known_models_from_wildcard from litellm.types.router import LiteLLM_Params monkeypatch.setattr(litellm, "check_provider_endpoint", False) + model_info_rows: Final = expand_wildcard_deployments_for_model_info( + [{"model_name": model_name, "litellm_params": {"model": deployment_model}}] + ) assert get_known_models_from_wildcard( wildcard_model=deployment_model, litellm_params=LiteLLM_Params(model=deployment_model) @@ -619,6 +622,7 @@ def test_route_that_rewrites_part_of_the_model_name_lists_no_ids( get_known_models_from_wildcard(wildcard_model=model_name, litellm_params=LiteLLM_Params(model=deployment_model)) == [] ) + assert [row["model_name"] for row in model_info_rows] == [model_name] def test_partial_wildcard_without_a_deployment_lists_the_same_ids_as_with_one( @@ -639,6 +643,31 @@ def test_partial_wildcard_without_a_deployment_lists_the_same_ids_as_with_one( ) +def test_explicit_model_that_a_wildcard_also_covers_is_listed_once(local_model_cost_map: None) -> None: + from litellm.proxy.auth.model_checks import get_complete_model_list + + listed: Final = get_complete_model_list( + key_models=[], + team_models=[], + proxy_model_list=["openai/gpt-4o-mini", "openai/*"], + user_model=None, + infer_model_from_keys=False, + ) + + assert "openai/gpt-4o-mini" in listed + assert len(listed) == len(set(listed)) + + +@pytest.mark.timeout(5) +def test_key_wildcard_with_many_stars_lists_promptly(local_model_cost_map: None) -> None: + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + + many_stars: Final = "openai/" + "*" * 30 + + assert get_known_models_from_wildcard(wildcard_model=f"{many_stars}!", litellm_params=None) == [] + assert "openai/gpt-4o" in get_known_models_from_wildcard(wildcard_model=f"{many_stars}o", litellm_params=None) + + @pytest.mark.parametrize( ("wildcard", "expected"), [ From 07c16946b7c3ac519d68e54b8d6f404fa6de8d4c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 16 Oct 2026 05:13:26 -0700 Subject: [PATCH 11/22] fix(proxy): list each wildcard id that routes to a known model, through infix and alias routes Listed ids are the route with the capture that reaches a known model, so prefix aliases and responses/, converse/ and invoke/ deployments list again, a fixed target lists nothing, and /model/info rows carry the routed model. --- litellm/proxy/auth/model_checks.py | 101 +++++++++++++----- tests/unit/proxy/auth/test_model_checks.py | 101 +++++++++--------- .../proxy_server/test_routes_model_info.py | 4 +- 3 files changed, 127 insertions(+), 79 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 6cd4dc745b55..991a2922bc85 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -2,7 +2,9 @@ ## Common checks for /v1/models and `/model/info` import copy from collections.abc import Sequence +from dataclasses import dataclass from itertools import chain +from types import MappingProxyType from typing import Final import litellm @@ -269,38 +271,82 @@ def _hydrate_litellm_credential_name( return litellm_params -def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_Params | None = None) -> list[str]: - deployment_model: Final = ( - litellm_params.model if litellm_params is not None and _check_wildcard_routing(litellm_params.model) else None +_RESPONSES_ROUTE_PREFIX: Final = "responses/" +_ROUTE_TYPE_PREFIXES_BY_PROVIDER: Final = MappingProxyType({"bedrock": ("converse/", "invoke/")}) + + +@dataclass(frozen=True, slots=True) +class _WildcardListing: + listed_id: str + routed_model: str + + +def _without_route_type_prefix(deployment_path: str, provider: str) -> str: + prefixes: Final = (_RESPONSES_ROUTE_PREFIX, *_ROUTE_TYPE_PREFIXES_BY_PROVIDER.get(provider, ())) + return next( + (deployment_path.removeprefix(prefix) for prefix in prefixes if deployment_path.startswith(prefix)), + deployment_path, ) - _, deployment_sep, deployment_suffix = (deployment_model or "").partition("/") - route: Final = ( - deployment_model - if wildcard_model == "*" and deployment_model is not None and deployment_suffix == "*" - else wildcard_model + + +def _single_star_listing( + route: str, provider: str, deployment_path: str, model_pattern: str, model_id: str +) -> _WildcardListing | None: + model_prefix, _, model_suffix = model_pattern.partition("*") + capture_end: Final = len(model_id) - len(model_suffix) + if capture_end < len(model_prefix) or not (model_id.startswith(model_prefix) and model_id.endswith(model_suffix)): + return None + capture: Final = model_id[len(model_prefix) : capture_end] + return _WildcardListing( + listed_id=route.replace("*", capture, 1), + routed_model=f"{provider}/{deployment_path.replace('*', capture, 1)}", ) - route_provider, route_sep, route_suffix = route.partition("/") - if not route_sep: - return [] - model_pattern: Final = deployment_suffix if deployment_sep else route_suffix - route_namespace: Final = route.removesuffix(model_pattern) - if not route.endswith(model_pattern) or not route_namespace.endswith("/"): - return [] - provider: Final = litellm_params.model.split("/", 1)[0] if litellm_params is not None else route_provider +def _same_pattern_listing( + route_namespace: str, provider: str, route_type_prefix: str, model_pattern: str, model_id: str +) -> _WildcardListing | None: + if not model_matches_wildcard(model_id, model_pattern): + return None + return _WildcardListing( + listed_id=f"{route_namespace}{model_id}", routed_model=f"{provider}/{route_type_prefix}{model_id}" + ) + + +def _wildcard_listings(wildcard_model: str, litellm_params: LiteLLM_Params | None) -> tuple[_WildcardListing, ...]: + deployment_model: Final = litellm_params.model if litellm_params is not None else wildcard_model + provider, provider_sep, deployment_path = deployment_model.partition("/") + if not provider_sep: + return () + + model_pattern: Final = _without_route_type_prefix(deployment_path, provider) + route_type_prefix: Final = deployment_path.removesuffix(model_pattern) + route_namespace: Final = wildcard_model.removesuffix(model_pattern) + is_single_star: Final = wildcard_model.count("*") == 1 and model_pattern.count("*") == 1 + is_same_pattern: Final = ( + wildcard_model.endswith(model_pattern) and route_namespace.endswith("/") and "*" not in route_namespace + ) + if not (is_single_star or is_same_pattern): + return () + provider_models: Final = get_provider_models( provider=provider, litellm_params=_hydrate_litellm_credential_name(litellm_params) ) if provider_models is None: - return [] + return () model_ids: Final = (strip_provider_family_prefix(model, provider) for model in provider_models) - return list( - dict.fromkeys( - f"{route_namespace}{model_id}" for model_id in model_ids if model_matches_wildcard(model_id, model_pattern) - ) + listings: Final = ( + _single_star_listing(wildcard_model, provider, deployment_path, model_pattern, model_id) + if is_single_star + else _same_pattern_listing(route_namespace, provider, route_type_prefix, model_pattern, model_id) + for model_id in model_ids ) + return tuple(dict.fromkeys(filter(None, listings))) + + +def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_Params | None = None) -> list[str]: + return [listing.listed_id for listing in _wildcard_listings(wildcard_model, litellm_params)] def expand_wildcard_deployments_for_model_info( @@ -328,20 +374,17 @@ def expand_wildcard_deployments_for_model_info( except Exception: expanded.append(deployment) continue - expanded_names = get_known_models_from_wildcard( - wildcard_model=model_name, - litellm_params=litellm_params, - ) - if not expanded_names: + listings = _wildcard_listings(wildcard_model=model_name, litellm_params=litellm_params) + if not listings: expanded.append(deployment) continue - for name in expanded_names: + for listing in listings: row = copy.deepcopy(deployment) - row["model_name"] = name + row["model_name"] = listing.listed_id params = row.get("litellm_params") if isinstance(params, dict): - params["model"] = name + params["model"] = listing.routed_model expanded.append(row) return expanded diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 93ef90b1fa53..f2abe9a3b5ac 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -541,34 +541,45 @@ def test_gemini_model_info_partial_wildcard_has_unique_model_names( @pytest.mark.parametrize( - ("model_name", "deployment_model", "route_namespace"), + ("model_name", "deployment_model", "route_type_prefix"), [ - ("cloudflare/@cf/*", "cloudflare/@cf/*", "cloudflare/"), - ("gemini/gemini-2.5*", "gemini/gemini-2.5*", "gemini/"), - ("vertex_ai/gemini-*", "vertex_ai/gemini-*", "vertex_ai/"), - ( - "fireworks_ai/accounts/fireworks/models/llama*", - "fireworks_ai/accounts/fireworks/models/llama*", - "fireworks_ai/", - ), - ("genai/test/*", "openai/*", "genai/test/"), - ("team/claude-*", "anthropic/claude-*", "team/"), + ("cloudflare/@cf/*", "cloudflare/@cf/*", ""), + ("gemini/gemini-2.5*", "gemini/gemini-2.5*", ""), + ("vertex_ai/gemini-*", "vertex_ai/gemini-*", ""), + ("fireworks_ai/accounts/fireworks/models/llama*", "fireworks_ai/accounts/fireworks/models/llama*", ""), + ("genai/test/*", "openai/*", ""), + ("team/claude-*", "anthropic/claude-*", ""), + ("team/oai-*", "openai/*", ""), + ("team/*", "anthropic/claude-*", ""), + ("custom/gemini-*", "gemini/*", ""), + ("*", "openai/*", ""), + ("*", "cloudflare/@cf/*", ""), + ("*", "gemini/gemini-2.5*", ""), + ("openai/*", "openai/responses/*", "responses/"), + ("bedrock/*", "bedrock/converse/*", "converse/"), + ("bedrock/*", "bedrock/invoke/*", "invoke/"), ], ) -def test_every_listed_wildcard_id_routes_to_the_known_model_it_names( +def test_every_known_model_a_wildcard_reaches_is_listed_once_under_the_id_that_routes_to_it( monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, model_name: str, deployment_model: str, - route_namespace: str, + route_type_prefix: str, ) -> None: + from fnmatch import fnmatchcase + import litellm - from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.proxy.auth.model_checks import ( + expand_wildcard_deployments_for_model_info, + get_known_models_from_wildcard, + ) from litellm.router_utils.pattern_match_deployments import PatternMatchRouter from litellm.types.router import LiteLLM_Params monkeypatch.setattr(litellm, "check_provider_endpoint", False) - provider: Final = deployment_model.partition("/")[0] + provider, _, deployment_path = deployment_model.partition("/") + model_pattern: Final = deployment_path.removeprefix(route_type_prefix) known_ids: Final = frozenset( model_id.removeprefix(f"{provider}/") for model_id in litellm.models_by_provider[provider] ) @@ -579,50 +590,47 @@ def test_every_listed_wildcard_id_routes_to_the_known_model_it_names( wildcard_model=model_name, litellm_params=LiteLLM_Params(model=deployment_model), ) - routed_ids: Final = tuple( + routed: Final = tuple( str((router.route(model_id) or [{}])[0].get("litellm_params", {}).get("model", "")) for model_id in listed ) + model_info_rows: Final = expand_wildcard_deployments_for_model_info( + [{"model_name": model_name, "litellm_params": {"model": deployment_model}}] + ) + routed_ids: Final = frozenset(model.removeprefix(f"{provider}/{route_type_prefix}") for model in routed) assert listed assert len(listed) == len(set(listed)) - assert [routed for routed in routed_ids if routed.removeprefix(f"{provider}/") not in known_ids] == [] - assert [ - (model_id, routed) - for model_id, routed in zip(listed, routed_ids) - if routed != f"{provider}/{model_id.removeprefix(route_namespace)}" - ] == [] + assert [model for model in routed if not model.startswith(f"{provider}/{route_type_prefix}")] == [] + assert routed_ids == {model_id for model_id in known_ids if fnmatchcase(model_id, model_pattern)} + assert [(row["model_name"], row["litellm_params"]["model"]) for row in model_info_rows] == list(zip(listed, routed)) @pytest.mark.parametrize( ("model_name", "deployment_model"), [ - ("custom/gemini-*", "gemini/*"), - ("openai/gpt-4*", "openai/*"), - ("team/*", "anthropic/claude-*"), - ("*", "cloudflare/@cf/*"), - ("*", "gemini/gemini-2.5*"), + ("team/*", "openai/gpt-4o-mini"), + ("*", "*"), + ("team/*", "openai/fo::*::static::*"), ], ) -def test_route_that_rewrites_part_of_the_model_name_lists_no_ids( +def test_wildcard_route_that_cannot_name_a_known_model_lists_no_ids( monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, model_name: str, deployment_model: str ) -> None: import litellm - from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_known_models_from_wildcard + from litellm.proxy.auth.model_checks import ( + expand_wildcard_deployments_for_model_info, + get_known_models_from_wildcard, + ) from litellm.types.router import LiteLLM_Params monkeypatch.setattr(litellm, "check_provider_endpoint", False) - model_info_rows: Final = expand_wildcard_deployments_for_model_info( - [{"model_name": model_name, "litellm_params": {"model": deployment_model}}] - ) + deployment: Final = {"model_name": model_name, "litellm_params": {"model": deployment_model}} - assert get_known_models_from_wildcard( - wildcard_model=deployment_model, litellm_params=LiteLLM_Params(model=deployment_model) - ) assert ( get_known_models_from_wildcard(wildcard_model=model_name, litellm_params=LiteLLM_Params(model=deployment_model)) == [] ) - assert [row["model_name"] for row in model_info_rows] == [model_name] + assert expand_wildcard_deployments_for_model_info([deployment]) == [deployment] def test_partial_wildcard_without_a_deployment_lists_the_same_ids_as_with_one( @@ -679,15 +687,12 @@ def test_key_wildcard_with_many_stars_lists_promptly(local_model_cost_map: None) def test_wildcard_keeps_a_vendor_namespace_that_matches_another_provider_name( monkeypatch: pytest.MonkeyPatch, wildcard: str, expected: list[str] ) -> None: - from litellm.proxy.auth import model_checks + import litellm from litellm.proxy.auth.model_checks import get_known_models_from_wildcard from litellm.types.router import LiteLLM_Params - monkeypatch.setattr( - model_checks, - "get_provider_models", - lambda provider, litellm_params=None: ["openai/gpt-oss-20b", "wandb/openai/gpt-oss-120b"], - ) + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + monkeypatch.setitem(litellm.models_by_provider, "wandb", ["openai/gpt-oss-20b", "wandb/openai/gpt-oss-120b"]) assert get_known_models_from_wildcard(wildcard_model=wildcard, litellm_params=LiteLLM_Params(model=wildcard)) == ( expected @@ -1041,23 +1046,21 @@ def test_expand_wildcard_deployments_non_wildcard_passthrough(): assert result == [deployment] -def test_expand_wildcard_deployments_openai_wildcard(): +def test_expand_wildcard_deployments_openai_wildcard(monkeypatch: pytest.MonkeyPatch): """openai/* should expand into ≥1 known openai model entries.""" - + import litellm from litellm.proxy.auth.model_checks import ( expand_wildcard_deployments_for_model_info, ) fake_models = ["openai/gpt-4o", "openai/gpt-4o-mini"] + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + monkeypatch.setitem(litellm.models_by_provider, "openai", ["gpt-4o", "gpt-4o-mini"]) deployment = { "model_name": "openai/*", "litellm_params": {"model": "openai/*"}, } - with patch( - "litellm.proxy.auth.model_checks.get_known_models_from_wildcard", - return_value=fake_models, - ): - result = expand_wildcard_deployments_for_model_info([deployment]) + result = expand_wildcard_deployments_for_model_info([deployment]) assert len(result) == 2 assert all(r["model_name"] in fake_models for r in result) diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 37e020591793..ecf10b2c53aa 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -496,7 +496,9 @@ def fake_get_provider_models(provider, litellm_params=None): allowed_model_names=allowed_model_names, ) - assert [model["model_name"] for model in result] == ["openai/gpt-4o"] + listed: Final = [(model["model_name"], model["litellm_params"]["model"]) for model in result] + + assert listed == [("gpt-4o", "openai/gpt-4o")] # --------------------------------------------------------------------------- From 1354c513f28f5e6f9dd463f65b8e862ae5e8da43 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 16 Oct 2026 07:20:13 -0700 Subject: [PATCH 12/22] fix(proxy): list ids through every route-type infix, fixed targets, and shadowed routes --- .../get_llm_provider_logic.py | 24 ++++ .../health_check_helpers.py | 19 ++-- litellm/proxy/auth/model_checks.py | 51 +++++---- litellm/proxy/proxy_server.py | 2 +- .../router_utils/pattern_match_deployments.py | 5 + .../test_health_check_helpers.py | 25 +++++ tests/unit/proxy/auth/test_model_checks.py | 105 +++++++++++++++++- .../proxy_server/test_routes_model_info.py | 4 + .../test_pattern_match_deployments.py | 19 ++++ 9 files changed, 218 insertions(+), 36 deletions(-) diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 95b4dd143c37..479fac7ec679 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -1,5 +1,6 @@ import re from fnmatch import fnmatchcase +from types import MappingProxyType from typing import Final, cast from urllib.parse import urlparse @@ -20,6 +21,23 @@ _KNOWN_PROVIDERS: Final = frozenset(provider.value for provider in LlmProviders) _GLOB_CHARS_OTHER_THAN_STAR: Final = re.compile(r"[\[?]") +_ROUTE_TYPE_PREFIXES_BY_PROVIDER: Final = MappingProxyType( + { + "azure": ("o_series/", "gpt5_series/"), + "bedrock": ( + "converse/", + "invoke/", + "claude_platform/", + "converse_like/", + "async_invoke/", + "openai/", + "mantle/", + "chat_completions/", + "nova/", + "nova-2/", + ), + } +) def _endpoint_matches_api_base(endpoint: str, api_base: str) -> bool: @@ -166,6 +184,12 @@ def model_matches_wildcard(model_id: str, wildcard: str) -> bool: return fnmatchcase(model_id, _GLOB_CHARS_OTHER_THAN_STAR.sub(r"[\g<0>]", wildcard)) +def split_route_type_prefix(model_path: str, custom_llm_provider: str) -> tuple[str, str]: + prefixes: Final = ("responses/", *_ROUTE_TYPE_PREFIXES_BY_PROVIDER.get(custom_llm_provider, ())) + prefix: Final = next((prefix for prefix in prefixes if model_path.startswith(prefix)), "") + return prefix, model_path.removeprefix(prefix) + + def inferred_provider(model: str | None) -> str | None: if not model: return None diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 07b3169b92e7..a98fd76f18f8 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -7,7 +7,11 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence from typing import TYPE_CHECKING, Final, Literal -from litellm.litellm_core_utils.get_llm_provider_logic import model_matches_wildcard, strip_provider_family_prefix +from litellm.litellm_core_utils.get_llm_provider_logic import ( + model_matches_wildcard, + split_route_type_prefix, + strip_provider_family_prefix, +) from litellm.llms.base_llm.ocr.transformation import DocumentType from litellm.rust_bridge import runtime from litellm.rust_bridge.ocr.entrypoints import NATIVE_OCR_HEALTH_CHECK_DOCUMENT @@ -85,15 +89,16 @@ def _ocr_health_check_document(model: str, custom_llm_provider: str) -> Document def _wildcard_health_check_models( wildcard_suffix: str, custom_llm_provider: str, candidate_models: Sequence[str] ) -> tuple[str, ...]: - if wildcard_suffix == "*": - return tuple(candidate_models[:3]) - model_ids: Final = (strip_provider_family_prefix(model, custom_llm_provider) for model in candidate_models) + route_type_prefix, model_pattern = split_route_type_prefix(wildcard_suffix, custom_llm_provider) + model_ids: Final = dict.fromkeys( + strip_provider_family_prefix(model, custom_llm_provider) for model in candidate_models + ) matching: Final = tuple( - f"{custom_llm_provider}/{model_id}" + f"{custom_llm_provider}/{route_type_prefix}{model_id}" for model_id in model_ids - if model_matches_wildcard(model_id, wildcard_suffix) + if model_matches_wildcard(model_id, model_pattern) ) - return matching[:3] or tuple(candidate_models[:3]) + return matching[:3] or tuple(dict.fromkeys(candidate_models))[:3] class HealthCheckHelpers: diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 991a2922bc85..507fc4de5a40 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -4,17 +4,21 @@ from collections.abc import Sequence from dataclasses import dataclass from itertools import chain -from types import MappingProxyType from typing import Final import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.credential_accessor import CredentialAccessor -from litellm.litellm_core_utils.get_llm_provider_logic import model_matches_wildcard, strip_provider_family_prefix +from litellm.litellm_core_utils.get_llm_provider_logic import ( + model_matches_wildcard, + split_route_type_prefix, + strip_provider_family_prefix, +) from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth from litellm.repositories.object_permission_repository import ObjectPermissionRepository from litellm.router import Router from litellm.router_utils.fallback_event_handlers import get_fallback_model_group +from litellm.router_utils.pattern_match_deployments import PatternMatchRouter from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager, get_valid_models @@ -271,24 +275,12 @@ def _hydrate_litellm_credential_name( return litellm_params -_RESPONSES_ROUTE_PREFIX: Final = "responses/" -_ROUTE_TYPE_PREFIXES_BY_PROVIDER: Final = MappingProxyType({"bedrock": ("converse/", "invoke/")}) - - @dataclass(frozen=True, slots=True) class _WildcardListing: listed_id: str routed_model: str -def _without_route_type_prefix(deployment_path: str, provider: str) -> str: - prefixes: Final = (_RESPONSES_ROUTE_PREFIX, *_ROUTE_TYPE_PREFIXES_BY_PROVIDER.get(provider, ())) - return next( - (deployment_path.removeprefix(prefix) for prefix in prefixes if deployment_path.startswith(prefix)), - deployment_path, - ) - - def _single_star_listing( route: str, provider: str, deployment_path: str, model_pattern: str, model_id: str ) -> _WildcardListing | None: @@ -319,8 +311,10 @@ def _wildcard_listings(wildcard_model: str, litellm_params: LiteLLM_Params | Non if not provider_sep: return () - model_pattern: Final = _without_route_type_prefix(deployment_path, provider) - route_type_prefix: Final = deployment_path.removesuffix(model_pattern) + route_type_prefix, model_pattern = split_route_type_prefix(deployment_path, provider) + if wildcard_model.count("*") == 1 and "*" not in model_pattern: + return (_WildcardListing(listed_id=wildcard_model.replace("*", model_pattern), routed_model=deployment_model),) + route_namespace: Final = wildcard_model.removesuffix(model_pattern) is_single_star: Final = wildcard_model.count("*") == 1 and model_pattern.count("*") == 1 is_same_pattern: Final = ( @@ -345,12 +339,22 @@ def _wildcard_listings(wildcard_model: str, litellm_params: LiteLLM_Params | Non return tuple(dict.fromkeys(filter(None, listings))) +def _listings_served_by_route( + wildcard_model: str, litellm_params: LiteLLM_Params | None, pattern_router: PatternMatchRouter | None +) -> tuple[_WildcardListing, ...]: + listings: Final = _wildcard_listings(wildcard_model, litellm_params) + if pattern_router is None: + return listings + return tuple(listing for listing in listings if not pattern_router.is_shadowed(listing.listed_id, wildcard_model)) + + def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_Params | None = None) -> list[str]: return [listing.listed_id for listing in _wildcard_listings(wildcard_model, litellm_params)] def expand_wildcard_deployments_for_model_info( deployments: list[dict[str, object]], + pattern_router: PatternMatchRouter | None = None, ) -> list[dict[str, object]]: """Expand wildcard deployments into one row per known provider model. @@ -374,7 +378,7 @@ def expand_wildcard_deployments_for_model_info( except Exception: expanded.append(deployment) continue - listings = _wildcard_listings(wildcard_model=model_name, litellm_params=litellm_params) + listings = _listings_served_by_route(model_name, litellm_params, pattern_router) if not listings: expanded.append(deployment) continue @@ -397,18 +401,17 @@ def _models_listed_for_wildcard_route( team_id: str | None, ) -> tuple[str, ...]: model_list: Final = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router else None - expanded: Final = ( + pattern_router: Final = llm_router.pattern_router if llm_router else None + listings: Final = ( chain.from_iterable( - get_known_models_from_wildcard( - wildcard_model=model, - litellm_params=LiteLLM_Params(**router_model["litellm_params"]), - ) + _listings_served_by_route(model, LiteLLM_Params(**router_model["litellm_params"]), pattern_router) for router_model in model_list ) if model_list - else get_known_models_from_wildcard(wildcard_model=model, litellm_params=None) + else _listings_served_by_route(model, None, pattern_router) ) - return (model, *expanded) if return_wildcard_route else tuple(expanded) + expanded: Final = tuple(listing.listed_id for listing in listings) + return (model, *expanded) if return_wildcard_route else expanded def _get_wildcard_models( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 03ab1752a920..ac69c7aaa561 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -16728,7 +16728,7 @@ async def model_info_v1( alias_models: Final = copy.deepcopy(llm_router.get_model_list_from_model_alias()) all_models.extend(alias_models) - all_models = expand_wildcard_deployments_for_model_info(all_models) + all_models = expand_wildcard_deployments_for_model_info(all_models, pattern_router=llm_router.pattern_router) allowed_model_names: Final = _get_v1_model_info_allowed_model_names( user_api_key_dict=user_api_key_dict, diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index 62ba5592f90b..e293aab16c99 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -172,6 +172,11 @@ def route(self, request: str | None, filtered_model_names: list[str] | None = No return None # No matching pattern found + def is_shadowed(self, request: str, wildcard: str) -> bool: + own_regex: Final = self.pattern_to_regex(wildcard) + first_match: Final = next((regex for regex in self.patterns if re.match(regex, request)), own_regex) + return own_regex in self.patterns and first_match != own_regex + def routes_to_provider(self, request: str, custom_llm_provider: str) -> bool: deployments: Final = _ROUTED_DEPLOYMENTS.validate_python(self.route(request) or ()) return any( diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index be3be40bc6fb..bcd99ab63167 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -1555,6 +1555,31 @@ def test_wildcard_health_check_models_with_many_stars_returns_promptly(): assert _wildcard_health_check_models("*" * 30 + "!", "openai", candidates) == candidates +def test_wildcard_health_check_models_probes_each_model_once(): + candidates: Final = ("gemini/gemini-exp-1206", "gemini-exp-1206", "gemini/gemini-exp-1114", "gemini-2.0-flash") + + assert _wildcard_health_check_models("gemini-exp*", "gemini", candidates) == ( + "gemini/gemini-exp-1206", + "gemini/gemini-exp-1114", + ) + + +@pytest.mark.parametrize( + ("wildcard_suffix", "custom_llm_provider", "expected"), + [ + ("converse/*", "bedrock", ("bedrock/converse/amazon.nova-micro-v1:0", "bedrock/converse/anthropic.claude-v2")), + ("nova/amazon.*", "bedrock", ("bedrock/nova/amazon.nova-micro-v1:0",)), + ("o_series/*", "azure", ("azure/o_series/amazon.nova-micro-v1:0", "azure/o_series/anthropic.claude-v2")), + ], +) +def test_wildcard_health_check_models_probes_through_the_route_type_prefix( + wildcard_suffix: str, custom_llm_provider: str, expected: tuple[str, ...] +) -> None: + candidates: Final = (f"{custom_llm_provider}/amazon.nova-micro-v1:0", "anthropic.claude-v2") + + assert _wildcard_health_check_models(wildcard_suffix, custom_llm_provider, candidates) == expected + + def test_wildcard_health_check_models_partial_prefix_matching_nothing_keeps_cheapest_candidates(): assert _wildcard_health_check_models("ft:*", "openai", ["gpt-4o-mini", "gpt-4.1-nano"]) == ( "gpt-4o-mini", diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index f2abe9a3b5ac..1145065c1064 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -1,10 +1,18 @@ -from typing import Final +from typing import Final, get_args from unittest.mock import patch import httpx import pytest import respx +from litellm.llms.bedrock.common_utils import BedrockRoute + +_BEDROCK_MODEL_ROUTES: Final = ( + *(route for route in get_args(BedrockRoute) if route not in ("agent", "agentcore")), + "nova", + "nova-2", +) + def test_get_team_models_for_all_models_and_team_only_models(): from litellm.proxy.auth.model_checks import get_team_models @@ -556,8 +564,7 @@ def test_gemini_model_info_partial_wildcard_has_unique_model_names( ("*", "cloudflare/@cf/*", ""), ("*", "gemini/gemini-2.5*", ""), ("openai/*", "openai/responses/*", "responses/"), - ("bedrock/*", "bedrock/converse/*", "converse/"), - ("bedrock/*", "bedrock/invoke/*", "invoke/"), + *(("bedrock/*", f"bedrock/{route}/*", f"{route}/") for route in _BEDROCK_MODEL_ROUTES), ], ) def test_every_known_model_a_wildcard_reaches_is_listed_once_under_the_id_that_routes_to_it( @@ -605,10 +612,27 @@ def test_every_known_model_a_wildcard_reaches_is_listed_once_under_the_id_that_r assert [(row["model_name"], row["litellm_params"]["model"]) for row in model_info_rows] == list(zip(listed, routed)) +@pytest.mark.parametrize("route_type_prefix", ["o_series/", "gpt5_series/", "responses/"]) +def test_azure_route_type_infix_lists_the_same_ids_as_the_route_without_it( + monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, route_type_prefix: str +) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + + without_infix: Final = get_known_models_from_wildcard("azure/*", LiteLLM_Params(model="azure/*")) + + assert without_infix + assert get_known_models_from_wildcard("azure/*", LiteLLM_Params(model=f"azure/{route_type_prefix}*")) == ( + without_infix + ) + + @pytest.mark.parametrize( ("model_name", "deployment_model"), [ - ("team/*", "openai/gpt-4o-mini"), ("*", "*"), ("team/*", "openai/fo::*::static::*"), ], @@ -633,6 +657,79 @@ def test_wildcard_route_that_cannot_name_a_known_model_lists_no_ids( assert expand_wildcard_deployments_for_model_info([deployment]) == [deployment] +@pytest.mark.parametrize( + ("model_name", "deployment_model", "listed_id"), + [ + ("team/*", "openai/gpt-4o-mini", "team/gpt-4o-mini"), + ("*", "openai/gpt-4o-mini", "gpt-4o-mini"), + ("team/*", "bedrock/converse/amazon.nova-micro-v1:0", "team/amazon.nova-micro-v1:0"), + ], +) +def test_wildcard_route_over_one_fixed_model_lists_that_model_once( + model_name: str, deployment_model: str, listed_id: str +) -> None: + from litellm.proxy.auth.model_checks import ( + expand_wildcard_deployments_for_model_info, + get_known_models_from_wildcard, + ) + from litellm.router_utils.pattern_match_deployments import PatternMatchRouter + from litellm.types.router import LiteLLM_Params + + deployment: Final = {"model_name": model_name, "litellm_params": {"model": deployment_model}} + router: Final = PatternMatchRouter() + router.add_pattern(model_name, deployment) + + assert ( + get_known_models_from_wildcard(wildcard_model=model_name, litellm_params=LiteLLM_Params(model=deployment_model)) + == [listed_id] + ) + assert (router.route(listed_id) or [{}])[0]["litellm_params"]["model"] == deployment_model + assert [ + (row["model_name"], row["litellm_params"]["model"]) + for row in expand_wildcard_deployments_for_model_info([deployment]) + ] == [(listed_id, deployment_model)] + + +def test_wildcard_lists_no_id_that_a_more_specific_route_serves( + monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + import copy + + import litellm + from litellm import Router + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list + + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + monkeypatch.setitem(litellm.models_by_provider, "openrouter", ["openrouter/openai/gpt-4o", "meta-llama/llama-3-8b"]) + router: Final = Router( + model_list=[ + {"model_name": "*", "litellm_params": {"model": "openrouter/*", "api_key": "key"}}, + {"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": "key"}}, + ] + ) + + def served_by(model_id: str) -> str: + return str((router.pattern_router.route(model_id) or [{}])[0]["litellm_params"]["model"]) + + listed_by_star: Final = get_complete_model_list( + key_models=["*"], + team_models=[], + proxy_model_list=[], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + model_info_rows: Final = expand_wildcard_deployments_for_model_info( + copy.deepcopy(router.model_list), pattern_router=router.pattern_router + ) + + assert listed_by_star == ["meta-llama/llama-3-8b"] + assert "openai/gpt-4o" in [row["model_name"] for row in model_info_rows] + assert [ + row["model_name"] for row in model_info_rows if served_by(row["model_name"]) != row["litellm_params"]["model"] + ] == [] + + def test_partial_wildcard_without_a_deployment_lists_the_same_ids_as_with_one( monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None ) -> None: diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index ecf10b2c53aa..637348f8fbe4 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -24,6 +24,7 @@ from litellm.caching.llm_caching_handler import LLMClientCache from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy import proxy_server +from litellm.router_utils.pattern_match_deployments import PatternMatchRouter from litellm.utils import _invalidate_model_cost_lowercase_map from .conftest import normalize # type: ignore[import-not-found] @@ -480,6 +481,8 @@ def fake_get_provider_models(provider, litellm_params=None): router.get_model_access_groups = MagicMock(return_value={}) router.get_model_names = MagicMock(return_value=["*"]) router.get_model_list = MagicMock(return_value=[deployment]) + router.pattern_router = PatternMatchRouter() + router.pattern_router.add_pattern("*", deployment) monkeypatch.setattr(model_checks, "get_provider_models", fake_get_provider_models) expanded_deployments = proxy_server.expand_wildcard_deployments_for_model_info([deployment]) @@ -638,6 +641,7 @@ def model_group_info_router(monkeypatch): router.get_model_names.return_value = model_names router.get_model_access_groups.return_value = {} router.get_model_list.return_value = [] + router.pattern_router = PatternMatchRouter() def model_group_info(*, llm_router, all_models_str, model_group): return [ModelGroupInfoProxy(model_group=name, providers=[]) for name in all_models_str] diff --git a/tests/unit/router_utils/test_pattern_match_deployments.py b/tests/unit/router_utils/test_pattern_match_deployments.py index 01af021112be..280b1f1198e0 100644 --- a/tests/unit/router_utils/test_pattern_match_deployments.py +++ b/tests/unit/router_utils/test_pattern_match_deployments.py @@ -378,6 +378,25 @@ def test_pattern_matching_router_uses_default_wildcard_for_unmatched_model() -> assert tuple(deployment["model_name"] for deployment in deployments) == ("*",) +@pytest.mark.parametrize( + ("request_model", "wildcard", "shadowed"), + [ + ("openai/gpt-4o", "*", True), + ("openai/gpt-4o", "openai/*", False), + ("meta-llama/llama-3-8b", "*", False), + ("openai/gpt-4o", "team/*", False), + ], +) +def test_is_shadowed_only_when_a_more_specific_registered_pattern_serves_the_request( + request_model: str, wildcard: str, shadowed: bool +) -> None: + pattern_router: Final = PatternMatchRouter() + pattern_router.add_pattern("*", _wildcard_deployment("*")) + pattern_router.add_pattern("openai/*", _wildcard_deployment("openai/*")) + + assert pattern_router.is_shadowed(request_model, wildcard) is shadowed + + def test_sorted_patterns(): """ Tests that the pattern specificity is calculated correctly From 3c833266f5f975cc7fe8e93946daa98cacdfa58e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 16 Oct 2026 08:57:10 -0700 Subject: [PATCH 13/22] test(proxy): cover wildcard listing routes, row precedence, health probes and many-star listing time Adds integration cells that call each listed wildcard id through the proxy and check the upstream model, the /model/info row and its price, which deployment each row names when a narrower route or an exact deployment wins, key-scoped listing beside a narrower route, duplicate-free listing across overlapping deployments, the wildcard health probe, and GET /v1/models time beside a route with many stars --- .../test_wildcard_partial_prefix_expansion.py | 268 ++++++++++++++++-- 1 file changed, 248 insertions(+), 20 deletions(-) diff --git a/tests/integration/routing/test_wildcard_partial_prefix_expansion.py b/tests/integration/routing/test_wildcard_partial_prefix_expansion.py index b79924ec47fa..ed3db354cb07 100644 --- a/tests/integration/routing/test_wildcard_partial_prefix_expansion.py +++ b/tests/integration/routing/test_wildcard_partial_prefix_expansion.py @@ -1,51 +1,134 @@ """Partial wildcards like ``cloudflare/@cf/*`` list each callable id once on /v1/models and /model/info, -and a partial wildcard that matches no known id, like ``databricks/system.ai.*``, lists no guessed ids.""" +and a partial wildcard that matches no known id, like ``databricks/system.ai.*``, lists no guessed ids. +Every id a wildcard lists reaches the upstream as the model its /model/info row names, priced from that model.""" -from collections.abc import Iterator +import uuid +from collections.abc import Iterator, Sequence +from pathlib import Path +from types import MappingProxyType from typing import Final +import httpx import pytest - -from integration._support.client import Gateway, object_value, string_value +from integration._support.client import JSON_OBJECT, Gateway, Scenario, list_value, object_value, string_value +from integration._support.openai_wire import answering_model_discovery, chat_reply, responses_reply +from integration._support.timing import assert_within_factor_of_control, timed +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue PATTERNS: Final = ("cloudflare/@cf/*", "databricks/system.ai.*", "databricks/*") +UNREACHABLE_API_BASE: Final = "https://example.invalid" +COST_MAP: Final = JSON_OBJECT.validate_json( + (Path(__file__).resolve().parents[3] / "litellm" / "model_prices_and_context_window_backup.json").read_bytes() +) +ROUTES: Final = MappingProxyType( + { + "alias_over_partial_model": ("{namespace}/*", "openai/gpt-4*"), + "alias_over_one_fixed_model": ("{namespace}/*", "openai/gpt-4o-mini"), + "catch_all_over_provider": ("*", "openai/*"), + } +) +SAMPLED_IDS_PER_ROUTE: Final = 6 +MANY_STARS: Final = "*" * 7 + "!" +CONTROL_LISTINGS: Final = 3 +LISTING_SLOWDOWN_FACTOR: Final = 25 +LISTING_TIMEOUT_SECONDS: Final = 120 -def _registered_model_id(gateway: Gateway, pattern: str) -> str: +def _register(scenario: Scenario, model_name: str, model: str, api_base: str) -> str: created: Final = object_value( - gateway.post( + scenario.gateway.post( "/model/new", { - "model_name": pattern, - "litellm_params": { - "model": pattern, - "api_key": "integration-wildcard-key", - "api_base": "https://example.invalid", - }, + "model_name": model_name, + "litellm_params": {"model": model, "api_key": "integration-wildcard-key", "api_base": api_base}, }, ) ) - return string_value(object_value(created["model_info"])["id"]) + identity: Final = string_value(object_value(created["model_info"])["id"]) + scenario.cleanups.callback(scenario.delete_model, identity) + return identity @pytest.fixture def wildcard_deployments(gateway: Gateway) -> Iterator[None]: with gateway.scenario() as scenario: for pattern in PATTERNS: - scenario.cleanups.callback(scenario.delete_model, _registered_model_id(gateway, pattern)) + _register(scenario, pattern, pattern, UNREACHABLE_API_BASE) yield -def _listed_model_ids(gateway: Gateway) -> tuple[str, ...]: - data: Final = gateway.get("/v1/models")["data"] +def _rows(gateway: Gateway, path: str) -> tuple[dict[str, JsonValue], ...]: + data: Final = gateway.get(path)["data"] assert isinstance(data, list) - return tuple(string_value(object_value(entry)["id"]) for entry in data) + return tuple(object_value(entry) for entry in data) + + +def _listed_model_ids(gateway: Gateway) -> tuple[str, ...]: + return tuple(string_value(entry["id"]) for entry in _rows(gateway, "/v1/models")) def _model_info_names(gateway: Gateway) -> tuple[str, ...]: - data: Final = gateway.get("/model/info")["data"] - assert isinstance(data, list) - return tuple(string_value(object_value(entry)["model_name"]) for entry in data) + return tuple(string_value(entry["model_name"]) for entry in _rows(gateway, "/model/info")) + + +def _model_group_names(gateway: Gateway) -> tuple[str, ...]: + return tuple(string_value(entry["model_group"]) for entry in _rows(gateway, "/model_group/info")) + + +def _openai_reply(request: Request) -> Reply: + model: Final = string_value(JSON_OBJECT.validate_json(request.body)["model"]) + identity: Final = f"wildcard-{uuid.uuid4().hex}" + if request.target.endswith("/responses"): + return responses_reply(identity, model, "listed id answered", stream=False) + return chat_reply(identity, model, "listed id answered", stream=False) + + +OPENAI_UPSTREAM: Final = answering_model_discovery(_openai_reply) + + +def _call(gateway: Gateway, model: str) -> httpx.Response: + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "which model am I"}], "cache": {"no-cache": True}}, + ) + assert response.status_code == 200, f"{model}: {response.status_code} {response.text}" + return response + + +def _upstream_model(wire: Wire) -> str: + (request,) = tuple(request for request in wire.drain() if request.method == "POST") + return string_value(JSON_OBJECT.validate_json(request.body)["model"]) + + +def _upstream_model_of_call(gateway: Gateway, wire: Wire, model: str) -> str: + _call(gateway, model) + return _upstream_model(wire) + + +def _spread(ids: Sequence[str], count: int) -> tuple[str, ...]: + ordered: Final = sorted(ids) + return tuple(ordered[:: max(1, len(ordered) // count)][:count]) + + +def _openai_cost_entry(model: str) -> dict[str, JsonValue] | None: + entry: Final = COST_MAP.get(model) + if not isinstance(entry, dict) or entry.get("litellm_provider") != "openai": + return None + return entry + + +def _assert_row_names_priced_model( + rows: Sequence[dict[str, JsonValue]], name: str, deployment_id: str, upstream_model: str +) -> None: + (row,) = tuple( + row for row in rows if row["model_name"] == name and object_value(row["model_info"])["id"] == deployment_id + ) + cost_entry: Final = _openai_cost_entry(upstream_model) + assert cost_entry is not None, f"{name} reached {upstream_model}, which no openai price names" + assert object_value(row["litellm_params"])["model"] == f"openai/{upstream_model}", name + assert object_value(row["model_info"])["input_cost_per_token"] == cost_entry.get("input_cost_per_token"), name def test_partial_wildcard_lists_provider_ids_once_on_both_endpoints( @@ -71,3 +154,148 @@ def test_partial_wildcard_matching_no_known_id_lists_no_guessed_ids( def test_provider_wildcard_still_expands_to_cost_map_names(gateway: Gateway, wildcard_deployments: None) -> None: assert any(model_id.startswith("databricks/databricks-") for model_id in _listed_model_ids(gateway)) + + +@pytest.mark.parametrize("route", tuple(ROUTES)) +def test_listed_wildcard_ids_reach_the_upstream_as_the_priced_model_their_row_names( + gateway: Gateway, route: str +) -> None: + model_name_template, deployment_model = ROUTES[route] + model_name: Final = model_name_template.format(namespace=f"listing-{uuid.uuid4().hex[:12]}") + already_listed: Final = frozenset(_listed_model_ids(gateway)) + + with wire_server(OPENAI_UPSTREAM) as wire, gateway.scenario() as scenario: + deployment_id: Final = _register(scenario, model_name, deployment_model, f"{wire.url}/v1") + listed: Final = tuple(model_id for model_id in _listed_model_ids(gateway) if model_id not in already_listed) + rows: Final = _rows(gateway, "/model/info") + groups: Final = _model_group_names(gateway) + sampled: Final = _spread(listed, SAMPLED_IDS_PER_ROUTE) + wire.drain() + + assert sampled, f"{model_name} over {deployment_model} listed no ids" + assert all(groups.count(model_id) == 1 for model_id in sampled), sampled + upstream_models: Final = tuple(_upstream_model_of_call(gateway, wire, model_id) for model_id in sampled) + + for model_id, upstream_model in zip(sampled, upstream_models, strict=True): + _assert_row_names_priced_model(rows, model_id, deployment_id, upstream_model) + assert len(set(upstream_models)) == len(sampled), dict(zip(sampled, upstream_models, strict=True)) + + +@pytest.mark.parametrize( + ("narrow_route", "narrow_model"), + (("gpt-4o*", "openai/gpt-4o*"), ("gpt-4o-mini", "openai/gpt-4o-mini")), + ids=("narrower_wildcard", "exact_name"), +) +def test_each_wildcard_row_names_the_deployment_its_id_routes_to( + gateway: Gateway, narrow_route: str, narrow_model: str +) -> None: + namespace: Final = f"shadow-{uuid.uuid4().hex[:12]}" + + with wire_server(OPENAI_UPSTREAM) as wire, gateway.scenario() as scenario: + narrow_id: Final = _register(scenario, f"{namespace}/{narrow_route}", narrow_model, f"{wire.url}/v1") + broad_id: Final = _register(scenario, f"{namespace}/*", "openai/*", f"{wire.url}/v1") + rows: Final = tuple( + (string_value(row["model_name"]), string_value(object_value(row["model_info"])["id"])) + for row in _rows(gateway, "/model/info") + if string_value(row["model_name"]).startswith(f"{namespace}/") + ) + contested: Final = tuple(row for row in rows if row[0].startswith(f"{namespace}/gpt-4o")) + broad_only: Final = tuple(row for row in rows if not row[0].startswith(f"{namespace}/gpt-4o")) + sampled_broad: Final = frozenset(_spread([name for name, _ in broad_only], 3)) + checked: Final = contested + tuple(row for row in broad_only if row[0] in sampled_broad) + served: Final = tuple( + (name, row_deployment, _call(gateway, name).headers.get("x-litellm-model-id")) + for name, row_deployment in checked + ) + + assert any(row_deployment == narrow_id for _, row_deployment in contested) + assert any(row_deployment == broad_id for _, row_deployment in broad_only) + assert [entry for entry in served if entry[1] != entry[2]] == [] + + +def test_key_scoped_to_a_wildcard_lists_an_id_a_narrower_route_sends_to_the_same_model(gateway: Gateway) -> None: + namespace: Final = f"keyscope-{uuid.uuid4().hex[:12]}" + narrowly_served: Final = f"{namespace}/gpt-4.1" + + with wire_server(OPENAI_UPSTREAM) as wire, gateway.scenario() as scenario: + _register(scenario, f"{namespace}/*", "openai/*", f"{wire.url}/v1") + _register(scenario, f"{namespace}/gpt-4.1*", "openai/gpt-4.1*", f"{wire.url}/v1") + key: Final = scenario.key(models=[f"{namespace}/*"]) + listing: Final = gateway.request("GET", "/v1/models", key=key) + wire.drain() + call: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": narrowly_served, "messages": [{"role": "user", "content": "hi"}], "cache": {"no-cache": True}}, + key=key, + ) + upstream_model: Final = _upstream_model(wire) + + assert call.status_code == 200, call.text + assert upstream_model == "gpt-4.1" + assert listing.status_code == 200, listing.text + listed: Final = tuple( + string_value(object_value(entry)["id"]) + for entry in list_value(JSON_OBJECT.validate_json(listing.content)["data"]) + ) + assert narrowly_served in listed, sorted(model_id for model_id in listed if model_id.startswith(namespace)) + + +def test_overlapping_deployments_of_one_wildcard_list_each_id_once(gateway: Gateway) -> None: + namespace: Final = f"overlap-{uuid.uuid4().hex[:12]}" + + with gateway.scenario() as scenario: + for _ in range(2): + _register(scenario, f"{namespace}/gpt-4o*", "openai/gpt-4o*", UNREACHABLE_API_BASE) + _register(scenario, f"{namespace}/gpt-4o-mini", "openai/gpt-4o-mini", UNREACHABLE_API_BASE) + listed: Final = tuple( + model_id for model_id in _listed_model_ids(gateway) if model_id.startswith(f"{namespace}/") + ) + groups: Final = tuple(name for name in _model_group_names(gateway) if name.startswith(f"{namespace}/")) + + assert f"{namespace}/gpt-4o-mini" in listed + assert len(listed) == len(set(listed)), sorted(listed) + assert len(groups) == len(set(groups)), sorted(groups) + assert set(groups) == set(listed) + + +@pytest.mark.parametrize(("pattern", "probes_match"), (("gpt-4o*", True), ("no-such-model-*", False))) +def test_wildcard_health_check_probes_a_model_its_pattern_names( + gateway: Gateway, pattern: str, probes_match: bool +) -> None: + model_name: Final = f"health-{uuid.uuid4().hex[:12]}/{pattern}" + + with wire_server(OPENAI_UPSTREAM) as wire, gateway.scenario() as scenario: + _register(scenario, model_name, f"openai/{pattern}", f"{wire.url}/v1") + wire.drain() + health: Final = gateway.get("/health", {"model": model_name}) + probe: Final = _upstream_model(wire) + + assert (health["healthy_count"], health["unhealthy_count"]) == (1, 0), health + assert probe.startswith(pattern.removesuffix("*")) is probes_match, probe + + +def _namespace_listing_seconds(gateway: Gateway, namespace: str) -> float: + listing: Final = timed( + lambda: gateway.client.get( + "/v1/models", headers={"Authorization": f"Bearer {gateway.key}"}, timeout=LISTING_TIMEOUT_SECONDS + ) + ) + assert listing.value.status_code == 200, listing.value.text + assert any( + string_value(object_value(entry)["id"]).startswith(f"{namespace}/") + for entry in list_value(JSON_OBJECT.validate_json(listing.value.content)["data"]) + ) + return listing.seconds + + +def test_wildcard_listing_answers_promptly_beside_a_route_with_many_stars(gateway: Gateway) -> None: + namespace: Final = f"stars-{uuid.uuid4().hex[:12]}" + + with gateway.scenario() as scenario: + _register(scenario, f"{namespace}/*", "openai/*", UNREACHABLE_API_BASE) + control: Final = max(_namespace_listing_seconds(gateway, namespace) for _ in range(CONTROL_LISTINGS)) + _register(scenario, f"{namespace}/{MANY_STARS}", f"openai/{MANY_STARS}", UNREACHABLE_API_BASE) + observed: Final = _namespace_listing_seconds(gateway, namespace) + + assert_within_factor_of_control(observed, control=control, factor=LISTING_SLOWDOWN_FACTOR) From 668fa295631550b89b16f572ad42265498517916 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:46:45 -0700 Subject: [PATCH 14/22] fix(proxy): keep wildcard ids a narrower route sends to the same model, list each id under its serving deployment, and match shadow checks without backtracking --- litellm/proxy/auth/model_checks.py | 24 +++++++- .../router_utils/pattern_match_deployments.py | 29 ++++++++- tests/unit/proxy/auth/test_model_checks.py | 61 +++++++++++++++++++ .../test_pattern_match_deployments.py | 57 ++++++++++++++++- 4 files changed, 163 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 507fc4de5a40..f78b092ee04d 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -345,7 +345,25 @@ def _listings_served_by_route( listings: Final = _wildcard_listings(wildcard_model, litellm_params) if pattern_router is None: return listings - return tuple(listing for listing in listings if not pattern_router.is_shadowed(listing.listed_id, wildcard_model)) + return tuple( + listing + for listing in listings + if not pattern_router.routes_elsewhere(listing.listed_id, wildcard_model, listing.routed_model) + ) + + +def _rows_served_by_route( + wildcard_model: str, + litellm_params: LiteLLM_Params | None, + pattern_router: PatternMatchRouter | None, + exact_model_names: frozenset[str], +) -> tuple[_WildcardListing, ...]: + return tuple( + listing + for listing in _wildcard_listings(wildcard_model, litellm_params) + if listing.listed_id not in exact_model_names + and not (pattern_router is not None and pattern_router.shadowing_regex(listing.listed_id, wildcard_model)) + ) def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_Params | None = None) -> list[str]: @@ -363,6 +381,8 @@ def expand_wildcard_deployments_for_model_info( on top of that: a wildcard deployment like model_name="*" / litellm_params.model="openai/*" becomes one entry per known openai model, matching /v1/models behaviour. """ + model_names: Final = tuple(str(deployment.get("model_name") or "") for deployment in deployments) + exact_model_names: Final = frozenset(name for name in model_names if not _check_wildcard_routing(name)) expanded: Final[list[dict[str, object]]] = [] for deployment in deployments: model_name = str(deployment.get("model_name") or "") @@ -378,7 +398,7 @@ def expand_wildcard_deployments_for_model_info( except Exception: expanded.append(deployment) continue - listings = _listings_served_by_route(model_name, litellm_params, pattern_router) + listings = _rows_served_by_route(model_name, litellm_params, pattern_router, exact_model_names) if not listings: expanded.append(deployment) continue diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index e293aab16c99..e5249d1506e8 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -4,6 +4,7 @@ import copy import re +from fnmatch import fnmatchcase from re import Match from typing import Final @@ -21,6 +22,15 @@ class _RoutedDeployment(BaseModel): _ROUTED_DEPLOYMENTS: Final = TypeAdapter(tuple[_RoutedDeployment, ...]) +_ESCAPED_CHARACTER: Final = re.compile(r"\\(.)", re.DOTALL) +_GLOB_LITERALS: Final = str.maketrans({"?": "[?]", "[": "[[]"}) + + +def matches_from_start(regex: str, request: str) -> bool: + literals: Final = ( + _ESCAPED_CHARACTER.sub(r"\1", segment).translate(_GLOB_LITERALS) for segment in regex.split("(.*)") + ) + return fnmatchcase(request, "*".join(literals) + "*") class PatternUtils: @@ -172,10 +182,23 @@ def route(self, request: str | None, filtered_model_names: list[str] | None = No return None # No matching pattern found - def is_shadowed(self, request: str, wildcard: str) -> bool: + def shadowing_regex(self, request: str, wildcard: str) -> str | None: own_regex: Final = self.pattern_to_regex(wildcard) - first_match: Final = next((regex for regex in self.patterns if re.match(regex, request)), own_regex) - return own_regex in self.patterns and first_match != own_regex + if own_regex not in self.patterns: + return None + serving_regex: Final = next((regex for regex in self.patterns if matches_from_start(regex, request)), own_regex) + return None if serving_regex == own_regex else serving_regex + + def routes_elsewhere(self, request: str, wildcard: str, routed_model: str) -> bool: + serving_regex: Final = self.shadowing_regex(request, wildcard) + matched: Final = re.match(serving_regex, request) if serving_regex is not None else None + if serving_regex is None or matched is None: + return False + deployments: Final = _ROUTED_DEPLOYMENTS.validate_python(self.patterns[serving_regex]) + return all( + self.set_deployment_model_name(matched, deployment.litellm_params.model) != routed_model + for deployment in deployments + ) def routes_to_provider(self, request: str, custom_llm_provider: str) -> bool: deployments: Final = _ROUTED_DEPLOYMENTS.validate_python(self.route(request) or ()) diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 1145065c1064..96d8ad6309ec 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -1164,6 +1164,67 @@ def test_expand_wildcard_deployments_openai_wildcard(monkeypatch: pytest.MonkeyP assert all(r["litellm_params"]["model"] in fake_models for r in result) +def test_expand_wildcard_lists_each_id_once_under_the_deployment_that_serves_it( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info + from litellm.router_utils.pattern_match_deployments import PatternMatchRouter + + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + monkeypatch.setitem(litellm.models_by_provider, "openai", ["gpt-4o", "gpt-4o-mini", "gpt-4.1-mini"]) + wide: Final = {"model_name": "openai/*", "litellm_params": {"model": "openai/*"}, "model_info": {"id": "wide"}} + narrow: Final = { + "model_name": "openai/gpt-4.1*", + "litellm_params": {"model": "openai/gpt-4.1*"}, + "model_info": {"id": "narrow"}, + } + exact: Final = { + "model_name": "openai/gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini"}, + "model_info": {"id": "exact"}, + } + pattern_router: Final = PatternMatchRouter() + pattern_router.add_pattern("openai/*", wide) + pattern_router.add_pattern("openai/gpt-4.1*", narrow) + + rows: Final = expand_wildcard_deployments_for_model_info([wide, narrow, exact], pattern_router=pattern_router) + + assert sorted((row["model_name"], row["model_info"]["id"]) for row in rows) == [ + ("openai/gpt-4.1-mini", "narrow"), + ("openai/gpt-4o", "wide"), + ("openai/gpt-4o-mini", "exact"), + ] + + +def test_key_scoped_to_a_wide_route_lists_ids_a_narrower_route_sends_to_the_same_model( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm import Router + from litellm.proxy.auth.model_checks import get_complete_model_list + + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + monkeypatch.setitem(litellm.models_by_provider, "openai", ["gpt-4o", "gpt-4.1-mini", "gpt-4.1-nano"]) + router: Final = Router( + model_list=[ + {"model_name": "openai/*", "litellm_params": {"model": "openai/*"}}, + {"model_name": "openai/gpt-4.1*", "litellm_params": {"model": "openai/gpt-4.1*"}}, + ] + ) + + listed: Final = get_complete_model_list( + key_models=["openai/*"], + team_models=[], + proxy_model_list=["openai/*", "openai/gpt-4.1*"], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + + assert sorted(listed) == ["openai/gpt-4.1-mini", "openai/gpt-4.1-nano", "openai/gpt-4o"] + + def test_expand_wildcard_concrete_model_name_with_wildcard_litellm_params(): """Concrete model_name must not be overwritten when only litellm_params.model is wildcard.""" from litellm.proxy.auth.model_checks import ( diff --git a/tests/unit/router_utils/test_pattern_match_deployments.py b/tests/unit/router_utils/test_pattern_match_deployments.py index 280b1f1198e0..4a063d0baa04 100644 --- a/tests/unit/router_utils/test_pattern_match_deployments.py +++ b/tests/unit/router_utils/test_pattern_match_deployments.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re from typing import Final from unittest.mock import Mock, patch @@ -13,7 +14,7 @@ from litellm import Router from litellm.litellm_core_utils import get_llm_provider_logic from litellm.router import Deployment, LiteLLM_Params -from litellm.router_utils.pattern_match_deployments import PatternMatchRouter, PatternUtils +from litellm.router_utils.pattern_match_deployments import PatternMatchRouter, PatternUtils, matches_from_start from litellm.types.router import ModelInfo import json from unittest.mock import MagicMock @@ -387,14 +388,64 @@ def test_pattern_matching_router_uses_default_wildcard_for_unmatched_model() -> ("openai/gpt-4o", "team/*", False), ], ) -def test_is_shadowed_only_when_a_more_specific_registered_pattern_serves_the_request( +def test_shadowing_regex_only_when_a_more_specific_registered_pattern_serves_the_request( request_model: str, wildcard: str, shadowed: bool ) -> None: pattern_router: Final = PatternMatchRouter() pattern_router.add_pattern("*", _wildcard_deployment("*")) pattern_router.add_pattern("openai/*", _wildcard_deployment("openai/*")) - assert pattern_router.is_shadowed(request_model, wildcard) is shadowed + assert (pattern_router.shadowing_regex(request_model, wildcard) is not None) is shadowed + + +@pytest.mark.parametrize( + ("pattern", "request_model"), + [ + ("openai/*", "openai/gpt-4o"), + ("openai/*", "azure/gpt-4o"), + ("a*b", "axxbyy"), + ("a*b", "ayy"), + ("stars/*******!", "stars/abc!"), + ("stars/*******!", "stars/abc"), + ("odd?[name]/*", "odd?[name]/x"), + ("odd?[name]/*", "oddX[name]/x"), + ], +) +def test_matches_from_start_agrees_with_the_routing_regex(pattern: str, request_model: str) -> None: + regex: Final = PatternMatchRouter().pattern_to_regex(pattern) + + assert matches_from_start(regex, request_model) is (re.match(regex, request_model) is not None) + + +@pytest.mark.timeout(10) +def test_shadowing_regex_answers_promptly_beside_a_route_with_many_stars() -> None: + pattern_router: Final = PatternMatchRouter() + pattern_router.add_pattern("stars/*", _wildcard_deployment("stars/*")) + pattern_router.add_pattern("stars/*******!", _wildcard_deployment("stars/*******!")) + long_id: Final = "stars/" + "x" * 200 + + assert pattern_router.shadowing_regex(long_id, "stars/*") is None + assert pattern_router.shadowing_regex(f"{long_id}!", "stars/*") == pattern_router.pattern_to_regex("stars/*******!") + + +@pytest.mark.parametrize( + ("request_model", "wildcard", "routed_model", "routes_elsewhere"), + [ + ("openai/gpt-4.1-mini", "openai/*", "openai/gpt-4.1-mini", False), + ("openai/gpt-oss-120b", "*", "groq/openai/gpt-oss-120b", True), + ("llama-3-8b", "*", "groq/llama-3-8b", False), + ("openai/gpt-4.1-mini", "openai/gpt-4.1*", "openai/gpt-4.1-mini", False), + ], +) +def test_routes_elsewhere_only_when_the_serving_route_sends_another_model( + request_model: str, wildcard: str, routed_model: str, routes_elsewhere: bool +) -> None: + pattern_router: Final = PatternMatchRouter() + pattern_router.add_pattern("*", {"model_name": "*", "litellm_params": {"model": "groq/*"}}) + pattern_router.add_pattern("openai/*", _wildcard_deployment("openai/*")) + pattern_router.add_pattern("openai/gpt-4.1*", _wildcard_deployment("openai/gpt-4.1*")) + + assert pattern_router.routes_elsewhere(request_model, wildcard, routed_model) is routes_elsewhere def test_sorted_patterns(): From 557f368f0a4e8d66db49320263d405b2b9225339 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:32:20 -0700 Subject: [PATCH 15/22] test(proxy): cover different-model narrower routes in key listing and many-star routes that match listed ids A key scoped to a wildcard drops an id a narrower route sends to another model, and keeps it when the route reaches the same model. The listing-time cell now runs beside an 8-star route whose tail sits in no listed id and one whose tail sits early in most listed ids, on both /v1/models and /model/info. --- .../test_wildcard_partial_prefix_expansion.py | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/tests/integration/routing/test_wildcard_partial_prefix_expansion.py b/tests/integration/routing/test_wildcard_partial_prefix_expansion.py index ed3db354cb07..8122ccccf539 100644 --- a/tests/integration/routing/test_wildcard_partial_prefix_expansion.py +++ b/tests/integration/routing/test_wildcard_partial_prefix_expansion.py @@ -29,7 +29,8 @@ } ) SAMPLED_IDS_PER_ROUTE: Final = 6 -MANY_STARS: Final = "*" * 7 + "!" +MANY_STARS: Final = "*" * 8 +LISTED_NAME_FIELDS: Final = MappingProxyType({"/v1/models": "id", "/model/info": "model_name"}) CONTROL_LISTINGS: Final = 3 LISTING_SLOWDOWN_FACTOR: Final = 25 LISTING_TIMEOUT_SECONDS: Final = 120 @@ -213,13 +214,19 @@ def test_each_wildcard_row_names_the_deployment_its_id_routes_to( assert [entry for entry in served if entry[1] != entry[2]] == [] -def test_key_scoped_to_a_wildcard_lists_an_id_a_narrower_route_sends_to_the_same_model(gateway: Gateway) -> None: +@pytest.mark.parametrize( + "narrow_model", ("openai/gpt-4.1*", "openai/gpt-4o-mini"), ids=("same_model", "different_model") +) +def test_key_scoped_to_a_wildcard_lists_an_id_a_narrower_route_serves_only_if_it_reaches_the_named_model( + gateway: Gateway, narrow_model: str +) -> None: namespace: Final = f"keyscope-{uuid.uuid4().hex[:12]}" - narrowly_served: Final = f"{namespace}/gpt-4.1" + named_model: Final = "gpt-4.1" + narrowly_served: Final = f"{namespace}/{named_model}" with wire_server(OPENAI_UPSTREAM) as wire, gateway.scenario() as scenario: _register(scenario, f"{namespace}/*", "openai/*", f"{wire.url}/v1") - _register(scenario, f"{namespace}/gpt-4.1*", "openai/gpt-4.1*", f"{wire.url}/v1") + _register(scenario, f"{namespace}/{named_model}*", narrow_model, f"{wire.url}/v1") key: Final = scenario.key(models=[f"{namespace}/*"]) listing: Final = gateway.request("GET", "/v1/models", key=key) wire.drain() @@ -232,13 +239,15 @@ def test_key_scoped_to_a_wildcard_lists_an_id_a_narrower_route_sends_to_the_same upstream_model: Final = _upstream_model(wire) assert call.status_code == 200, call.text - assert upstream_model == "gpt-4.1" + assert upstream_model == narrow_model.removeprefix("openai/").removesuffix("*") assert listing.status_code == 200, listing.text listed: Final = tuple( string_value(object_value(entry)["id"]) for entry in list_value(JSON_OBJECT.validate_json(listing.content)["data"]) ) - assert narrowly_served in listed, sorted(model_id for model_id in listed if model_id.startswith(namespace)) + in_namespace: Final = sorted(model_id for model_id in listed if model_id.startswith(namespace)) + assert f"{namespace}/gpt-4o-mini" in listed, in_namespace + assert (narrowly_served in listed) is (upstream_model == named_model), in_namespace def test_overlapping_deployments_of_one_wildcard_list_each_id_once(gateway: Gateway) -> None: @@ -275,27 +284,32 @@ def test_wildcard_health_check_probes_a_model_its_pattern_names( assert probe.startswith(pattern.removesuffix("*")) is probes_match, probe -def _namespace_listing_seconds(gateway: Gateway, namespace: str) -> float: +def _namespace_listing_seconds(gateway: Gateway, path: str, namespace: str) -> float: listing: Final = timed( lambda: gateway.client.get( - "/v1/models", headers={"Authorization": f"Bearer {gateway.key}"}, timeout=LISTING_TIMEOUT_SECONDS + path, headers={"Authorization": f"Bearer {gateway.key}"}, timeout=LISTING_TIMEOUT_SECONDS ) ) assert listing.value.status_code == 200, listing.value.text assert any( - string_value(object_value(entry)["id"]).startswith(f"{namespace}/") + string_value(object_value(entry)[LISTED_NAME_FIELDS[path]]).startswith(f"{namespace}/") for entry in list_value(JSON_OBJECT.validate_json(listing.value.content)["data"]) ) return listing.seconds -def test_wildcard_listing_answers_promptly_beside_a_route_with_many_stars(gateway: Gateway) -> None: +@pytest.mark.parametrize("path", tuple(LISTED_NAME_FIELDS), ids=("v1_models", "model_info")) +@pytest.mark.parametrize("route_tail", ("!", "g"), ids=("tail_in_no_listed_id", "tail_early_in_listed_ids")) +def test_wildcard_listing_answers_promptly_beside_a_route_with_many_stars( + gateway: Gateway, path: str, route_tail: str +) -> None: namespace: Final = f"stars-{uuid.uuid4().hex[:12]}" + many_stars_route: Final = f"{namespace}/{MANY_STARS}{route_tail}" with gateway.scenario() as scenario: _register(scenario, f"{namespace}/*", "openai/*", UNREACHABLE_API_BASE) - control: Final = max(_namespace_listing_seconds(gateway, namespace) for _ in range(CONTROL_LISTINGS)) - _register(scenario, f"{namespace}/{MANY_STARS}", f"openai/{MANY_STARS}", UNREACHABLE_API_BASE) - observed: Final = _namespace_listing_seconds(gateway, namespace) + control: Final = max(_namespace_listing_seconds(gateway, path, namespace) for _ in range(CONTROL_LISTINGS)) + _register(scenario, many_stars_route, f"openai/{MANY_STARS}{route_tail}", UNREACHABLE_API_BASE) + observed: Final = _namespace_listing_seconds(gateway, path, namespace) assert_within_factor_of_control(observed, control=control, factor=LISTING_SLOWDOWN_FACTOR) From fa613b11bcc93d0fd5e2a9d78284cdea45ca92ca Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:07:42 -0700 Subject: [PATCH 16/22] fix(proxy): keep wildcard ids a narrower route sends to the same model by another path, and read serving-route captures without backtracking A wider route now drops a listed id only when the route that serves it reaches a different model after the provider, route-type infix, and provider-family prefix are dropped, so responses/converse infixes, a bare model with custom_llm_provider, and a narrower route on another provider keep their ids. /model/info keeps such a row under the wider route when the serving route cannot list it. The serving route's captures come from a right-to-left literal placement that matches re.match's greedy groups in linear time. Health probes for a partial wildcard fall back to the provider's cheapest models after the pattern's own. --- .../get_llm_provider_logic.py | 7 ++ .../health_check_helpers.py | 16 +++- litellm/proxy/auth/model_checks.py | 61 ++++++++------ .../router_utils/pattern_match_deployments.py | 57 ++++++++----- .../test_health_check_helpers.py | 49 ++++++++++++ tests/unit/proxy/auth/test_model_checks.py | 50 ++++++++++++ .../test_pattern_match_deployments.py | 80 ++++++++++++++++++- 7 files changed, 272 insertions(+), 48 deletions(-) diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 479fac7ec679..5ab61909c0d8 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -190,6 +190,13 @@ def split_route_type_prefix(model_path: str, custom_llm_provider: str) -> tuple[ return prefix, model_path.removeprefix(prefix) +def model_name_at_provider(model: str, custom_llm_provider: str | None) -> str: + leading, sep, path = model.partition("/") + provider: Final = custom_llm_provider or (leading if sep else "") + model_path: Final = path if sep and leading == provider else model + return strip_provider_family_prefix(split_route_type_prefix(model_path, provider)[1], provider) + + def inferred_provider(model: str | None) -> str | None: if not model: return None diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index a98fd76f18f8..d1bb632002e1 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -9,6 +9,7 @@ from litellm.litellm_core_utils.get_llm_provider_logic import ( model_matches_wildcard, + model_name_at_provider, split_route_type_prefix, strip_provider_family_prefix, ) @@ -101,6 +102,17 @@ def _wildcard_health_check_models( return matching[:3] or tuple(dict.fromkeys(candidate_models))[:3] +def _wildcard_probe_models( + wildcard_suffix: str, custom_llm_provider: str, cheapest_models: Sequence[str] +) -> tuple[str, ...]: + route_models: Final = _wildcard_health_check_models(wildcard_suffix, custom_llm_provider, cheapest_models) + probed: Final = frozenset(model_name_at_provider(model, custom_llm_provider) for model in route_models) + provider_models: Final = ( + model for model in cheapest_models[:3] if model_name_at_provider(model, custom_llm_provider) not in probed + ) + return (*route_models, *provider_models) + + class HealthCheckHelpers: @staticmethod async def ahealth_check_wildcard_models( @@ -121,9 +133,7 @@ async def ahealth_check_wildcard_models( raise Exception( f"Unable to health check wildcard model for provider {custom_llm_provider}. Add a model on your config.yaml or contribute here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json" ) - candidates: Final = _wildcard_health_check_models( - wildcard_suffix=model, custom_llm_provider=custom_llm_provider, candidate_models=cheapest_models - ) + candidates: Final = _wildcard_probe_models(model, custom_llm_provider, cheapest_models) fallback_models: Final = list(candidates[1:]) or None model_params["model"] = candidates[0] model_params["litellm_logging_obj"] = litellm_logging_obj diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index f78b092ee04d..39c05dc60661 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -352,17 +352,33 @@ def _listings_served_by_route( ) -def _rows_served_by_route( - wildcard_model: str, - litellm_params: LiteLLM_Params | None, - pattern_router: PatternMatchRouter | None, - exact_model_names: frozenset[str], -) -> tuple[_WildcardListing, ...]: +@dataclass(frozen=True, slots=True) +class _ModelInfoRow: + listing: _WildcardListing + shadowed: bool + + +def _model_info_rows( + deployment: dict[str, object], pattern_router: PatternMatchRouter | None, exact_model_names: frozenset[str] +) -> tuple[_ModelInfoRow, ...]: + model_name: Final = str(deployment.get("model_name") or "") + raw_params: Final = deployment.get("litellm_params") + if not _check_wildcard_routing(model_name): + return () + try: + litellm_params: Final = ( + LiteLLM_Params.model_validate(raw_params) if isinstance(raw_params, dict) and raw_params else None + ) + except Exception: + return () return tuple( - listing - for listing in _wildcard_listings(wildcard_model, litellm_params) + _ModelInfoRow( + listing=listing, + shadowed=pattern_router is not None + and pattern_router.shadowing_regex(listing.listed_id, model_name) is not None, + ) + for listing in _listings_served_by_route(model_name, litellm_params, pattern_router) if listing.listed_id not in exact_model_names - and not (pattern_router is not None and pattern_router.shadowing_regex(listing.listed_id, wildcard_model)) ) @@ -383,22 +399,19 @@ def expand_wildcard_deployments_for_model_info( """ model_names: Final = tuple(str(deployment.get("model_name") or "") for deployment in deployments) exact_model_names: Final = frozenset(name for name in model_names if not _check_wildcard_routing(name)) + rows_by_deployment: Final = tuple( + _model_info_rows(deployment, pattern_router, exact_model_names) for deployment in deployments + ) + served_ids: Final = frozenset( + candidate.listing.listed_id for candidate in chain.from_iterable(rows_by_deployment) if not candidate.shadowed + ) expanded: Final[list[dict[str, object]]] = [] - for deployment in deployments: - model_name = str(deployment.get("model_name") or "") - raw_params = deployment.get("litellm_params") - litellm_params_dict: dict[str, object] = raw_params if isinstance(raw_params, dict) else {} - - if not _check_wildcard_routing(model_name): - expanded.append(deployment) - continue - - try: - litellm_params = LiteLLM_Params.model_validate(litellm_params_dict) if litellm_params_dict else None - except Exception: - expanded.append(deployment) - continue - listings = _rows_served_by_route(model_name, litellm_params, pattern_router, exact_model_names) + for deployment, candidates in zip(deployments, rows_by_deployment): + listings = tuple( + candidate.listing + for candidate in candidates + if not (candidate.shadowed and candidate.listing.listed_id in served_ids) + ) if not listings: expanded.append(deployment) continue diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index e5249d1506e8..668be7938fc8 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -5,13 +5,15 @@ import copy import re from fnmatch import fnmatchcase +from functools import reduce +from itertools import accumulate from re import Match from typing import Final from pydantic import BaseModel, ConfigDict, TypeAdapter from litellm._logging import verbose_router_logger -from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider +from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider, model_name_at_provider from litellm.types.router import LiteLLM_Params @@ -33,6 +35,31 @@ def matches_from_start(regex: str, request: str) -> bool: return fnmatchcase(request, "*".join(literals) + "*") +def greedy_captures(regex: str, request: str) -> tuple[str, ...] | None: + segments: Final = tuple(_ESCAPED_CHARACTER.sub(r"\1", segment) for segment in regex.split("(.*)")) + prefix: Final = segments[0] + literals: Final = segments[1:] + starts: Final = tuple( + accumulate( + reversed(literals), + lambda ceiling, literal: -1 if ceiling < 0 else request.rfind(literal, len(prefix), ceiling), + initial=len(request), + ) + )[:0:-1] + if not request.startswith(prefix) or -1 in starts: + return None + ends: Final = (len(prefix), *(start + len(literal) for start, literal in zip(starts, literals))) + return tuple(request[end:start] for end, start in zip(ends, starts)) + + +def fill_wildcards(segments: tuple[str, ...], deployment_model: str, request: str) -> str: + if "*" not in deployment_model: + return deployment_model + if len(segments) > deployment_model.count("*"): + return request + return reduce(lambda model, segment: model.replace("*", segment, 1), segments, deployment_model) + + class PatternUtils: @staticmethod def calculate_pattern_specificity(pattern: str) -> tuple[int, int]: @@ -191,12 +218,17 @@ def shadowing_regex(self, request: str, wildcard: str) -> str | None: def routes_elsewhere(self, request: str, wildcard: str, routed_model: str) -> bool: serving_regex: Final = self.shadowing_regex(request, wildcard) - matched: Final = re.match(serving_regex, request) if serving_regex is not None else None - if serving_regex is None or matched is None: + captures: Final = greedy_captures(serving_regex, request) if serving_regex is not None else None + if serving_regex is None or captures is None: return False deployments: Final = _ROUTED_DEPLOYMENTS.validate_python(self.patterns[serving_regex]) + listed_model: Final = model_name_at_provider(routed_model, None) return all( - self.set_deployment_model_name(matched, deployment.litellm_params.model) != routed_model + model_name_at_provider( + fill_wildcards(captures, deployment.litellm_params.model, request), + deployment.litellm_params.custom_llm_provider, + ) + != listed_model for deployment in deployments ) @@ -240,22 +272,7 @@ def set_deployment_model_name( if model_name = "hello-world-meta.llama3-70b" -> model = "bedrock/meta.llama3-70b" """ - ## BASE CASE: if the deployment model name does not contain a wildcard, return the deployment model name - if "*" not in litellm_deployment_litellm_model: - return litellm_deployment_litellm_model - - wildcard_count: Final = litellm_deployment_litellm_model.count("*") - - # Extract all dynamic segments from the request - dynamic_segments: Final = matched_pattern.groups() - - if len(dynamic_segments) > wildcard_count: - return matched_pattern.string # default to the user input, if unable to map based on wildcards. - # Replace the corresponding wildcards in the litellm model pattern with extracted segments - for segment in dynamic_segments: - litellm_deployment_litellm_model = litellm_deployment_litellm_model.replace("*", segment, 1) - - return litellm_deployment_litellm_model + return fill_wildcards(matched_pattern.groups(), litellm_deployment_litellm_model, matched_pattern.string) def get_pattern(self, model: str | None, custom_llm_provider: str | None = None) -> list[dict] | None: """ diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index bcd99ab63167..57804b960eb9 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -8,6 +8,7 @@ from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest import respx @@ -19,6 +20,7 @@ default_health_check_mode, native_health_check_mode, _wildcard_health_check_models, + _wildcard_probe_models, ) from litellm.main import ahealth_check from litellm.proxy._types import UserAPIKeyAuth @@ -1580,6 +1582,53 @@ def test_wildcard_health_check_models_probes_through_the_route_type_prefix( assert _wildcard_health_check_models(wildcard_suffix, custom_llm_provider, candidates) == expected +def test_wildcard_probe_models_probe_each_provider_model_once_for_a_full_wildcard(): + candidates: Final = ("openai/gpt-4o-mini", "gpt-4.1-nano", "o3-mini", "gpt-5-nano") + + assert _wildcard_probe_models("*", "openai", candidates) == ( + "openai/gpt-4o-mini", + "openai/gpt-4.1-nano", + "openai/o3-mini", + ) + + +@pytest.mark.asyncio +async def test_ahealth_check_partial_wildcard_is_healthy_when_only_a_provider_model_answers( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + def answer(request: httpx.Request) -> httpx.Response: + model: Final = json.loads(request.content)["model"] + if model.startswith("gpt-4o"): + return httpx.Response(404, json={"error": {"message": "not found", "type": "invalid_request_error"}}) + return httpx.Response( + 200, + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 0, + "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + upstream: Final = respx_mock.post("https://oai.test/v1/chat/completions").mock(side_effect=answer) + + result: Final = await ahealth_check( + model_params={"model": "openai/gpt-4o*", "api_key": "x", "api_base": "https://oai.test/v1"}, + mode="chat", + ) + + probed: Final = [json.loads(call.request.content)["model"] for call in upstream.calls] + assert "error" not in result, result + assert probed[0].startswith("gpt-4o") + assert not probed[-1].startswith("gpt-4o") + + def test_wildcard_health_check_models_partial_prefix_matching_nothing_keeps_cheapest_candidates(): assert _wildcard_health_check_models("ft:*", "openai", ["gpt-4o-mini", "gpt-4.1-nano"]) == ( "gpt-4o-mini", diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 96d8ad6309ec..458d95759ff4 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -1225,6 +1225,56 @@ def test_key_scoped_to_a_wide_route_lists_ids_a_narrower_route_sends_to_the_same assert sorted(listed) == ["openai/gpt-4.1-mini", "openai/gpt-4.1-nano", "openai/gpt-4o"] +def _wide_route_beside_a_narrower_route_to_another_provider(monkeypatch: pytest.MonkeyPatch): + import litellm + from litellm import Router + + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + monkeypatch.setitem(litellm.models_by_provider, "openai", ["gpt-4o", "gpt-4o-mini", "gpt-4.1-mini"]) + monkeypatch.setitem(litellm.models_by_provider, "azure", []) + return Router( + model_list=[ + {"model_name": "az/*", "litellm_params": {"model": "openai/*"}, "model_info": {"id": "wide"}}, + {"model_name": "az/gpt-4o*", "litellm_params": {"model": "azure/gpt-4o*"}, "model_info": {"id": "narrow"}}, + ] + ) + + +def test_listing_keeps_ids_a_narrower_route_sends_to_the_same_model_at_another_provider( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.auth.model_checks import get_complete_model_list + + router: Final = _wide_route_beside_a_narrower_route_to_another_provider(monkeypatch) + + listed: Final = get_complete_model_list( + key_models=[], + team_models=[], + proxy_model_list=["az/*", "az/gpt-4o*"], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + + assert {"az/gpt-4o", "az/gpt-4o-mini", "az/gpt-4.1-mini"} <= set(listed) + + +def test_model_info_keeps_rows_a_narrower_route_cannot_list_but_sends_to_the_same_model( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info + + router: Final = _wide_route_beside_a_narrower_route_to_another_provider(monkeypatch) + + rows: Final = expand_wildcard_deployments_for_model_info(router.get_model_list(), router.pattern_router) + + assert sorted(row["model_name"] for row in rows if row["model_info"]["id"] == "wide") == [ + "az/gpt-4.1-mini", + "az/gpt-4o", + "az/gpt-4o-mini", + ] + + def test_expand_wildcard_concrete_model_name_with_wildcard_litellm_params(): """Concrete model_name must not be overwritten when only litellm_params.model is wildcard.""" from litellm.proxy.auth.model_checks import ( diff --git a/tests/unit/router_utils/test_pattern_match_deployments.py b/tests/unit/router_utils/test_pattern_match_deployments.py index 4a063d0baa04..13ab50314990 100644 --- a/tests/unit/router_utils/test_pattern_match_deployments.py +++ b/tests/unit/router_utils/test_pattern_match_deployments.py @@ -14,7 +14,13 @@ from litellm import Router from litellm.litellm_core_utils import get_llm_provider_logic from litellm.router import Deployment, LiteLLM_Params -from litellm.router_utils.pattern_match_deployments import PatternMatchRouter, PatternUtils, matches_from_start +from litellm.router_utils.pattern_match_deployments import ( + PatternMatchRouter, + PatternUtils, + fill_wildcards, + greedy_captures, + matches_from_start, +) from litellm.types.router import ModelInfo import json from unittest.mock import MagicMock @@ -428,6 +434,45 @@ def test_shadowing_regex_answers_promptly_beside_a_route_with_many_stars() -> No assert pattern_router.shadowing_regex(f"{long_id}!", "stars/*") == pattern_router.pattern_to_regex("stars/*******!") +@pytest.mark.parametrize( + ("pattern", "deployment_model", "request_model"), + [ + ("openai/*", "openai/*", "openai/gpt-4o"), + ("openai/*", "openai/*", "azure/gpt-4o"), + ("llmengine/fo::*::static::*", "openai/fo::*::static::*", "llmengine/fo::bar::static::baz::static::qux"), + ("*meta.llama3*", "bedrock/meta.llama3*", "hello-world-meta.llama3-70b"), + ("a*b", "x/*", "abab"), + ("*aa*", "p/*aa*", "aaaa"), + ("stars/********g", "openai/********g", "stars/gpt-4o-mini"), + ("stars/*******!", "openai/*******!", "stars/abc"), + ("odd?[name]/*", "openai/*", "odd?[name]/x"), + ], +) +def test_greedy_captures_map_a_request_to_the_model_routing_sends_it_to( + pattern: str, deployment_model: str, request_model: str +) -> None: + pattern_router: Final = PatternMatchRouter() + pattern_router.add_pattern(pattern, {"model_name": pattern, "litellm_params": {"model": deployment_model}}) + routed: Final = _matched_models(pattern_router.route(request_model)) + captures: Final = greedy_captures(pattern_router.pattern_to_regex(pattern), request_model) + mapped: Final = [] if captures is None else [fill_wildcards(captures, deployment_model, request_model)] + + assert mapped == routed + + +@pytest.mark.timeout(10) +def test_routes_elsewhere_answers_promptly_when_a_route_with_many_stars_serves_the_id() -> None: + pattern_router: Final = PatternMatchRouter() + pattern_router.add_pattern("stars/*", {"model_name": "stars/*", "litellm_params": {"model": "openai/*"}}) + pattern_router.add_pattern( + "stars/********g", {"model_name": "stars/********g", "litellm_params": {"model": "openai/********g"}} + ) + long_id: Final = "g" + "x" * 40 + + assert pattern_router.routes_elsewhere(f"stars/{long_id}", "stars/*", f"openai/{long_id}") is True + assert pattern_router.routes_elsewhere(f"stars/{long_id}", "stars/*", "openai/g") is False + + @pytest.mark.parametrize( ("request_model", "wildcard", "routed_model", "routes_elsewhere"), [ @@ -448,6 +493,39 @@ def test_routes_elsewhere_only_when_the_serving_route_sends_another_model( assert pattern_router.routes_elsewhere(request_model, wildcard, routed_model) is routes_elsewhere +@pytest.mark.parametrize( + ("wide_model", "narrow_route", "narrow_model", "custom_llm_provider", "listed_id", "routes_elsewhere"), + [ + ("openai/*", "wide/gpt-4o*", "openai/responses/gpt-4o*", None, "wide/gpt-4o-mini", False), + ("bedrock/*", "wide/us.anthropic*", "bedrock/converse/us.anthropic*", None, "wide/us.anthropic.claude-v2", False), + ("openai/*", "wide/gpt-4.1*", "gpt-4.1*", "openai", "wide/gpt-4.1-mini", False), + ("openai/*", "wide/gpt-4o*", "azure/gpt-4o*", None, "wide/gpt-4o-mini", False), + ("openai/*", "wide/gpt-4.1*", "openai/gpt-4o-mini", None, "wide/gpt-4.1-mini", True), + ("openai/*", "wide/gpt-4o*", "openai/responses/gpt-4o-mini", None, "wide/gpt-4o", True), + ], +) +def test_routes_elsewhere_compares_the_model_each_route_reaches_at_its_provider( + wide_model: str, + narrow_route: str, + narrow_model: str, + custom_llm_provider: str | None, + listed_id: str, + routes_elsewhere: bool, +) -> None: + pattern_router: Final = PatternMatchRouter() + pattern_router.add_pattern("wide/*", {"model_name": "wide/*", "litellm_params": {"model": wide_model}}) + pattern_router.add_pattern( + narrow_route, + { + "model_name": narrow_route, + "litellm_params": {"model": narrow_model, "custom_llm_provider": custom_llm_provider}, + }, + ) + routed_model: Final = wide_model.replace("*", listed_id.removeprefix("wide/")) + + assert pattern_router.routes_elsewhere(listed_id, "wide/*", routed_model) is routes_elsewhere + + def test_sorted_patterns(): """ Tests that the pattern specificity is calculated correctly From 29887b52b336b8ec02f4adcf59c2cbca45552216 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:53:18 -0700 Subject: [PATCH 17/22] test(proxy): cover same-model narrower routes by infix, provider and custom_llm_provider, /model/info agreement, health probe fallback, and team file credentials beside a narrower wildcard The key listing test now also checks that /model/info lists the narrowly served id exactly when the call reaches the model it names, across responses/, bare-model-with-provider and other-provider narrower routes. New cells cover a wildcard health check whose pattern models all fail while a provider model answers, and a team file listing that takes the credentials of a narrower Azure wildcard route. --- .../test_wildcard_partial_prefix_expansion.py | 103 ++++++++++++++++-- 1 file changed, 95 insertions(+), 8 deletions(-) diff --git a/tests/integration/routing/test_wildcard_partial_prefix_expansion.py b/tests/integration/routing/test_wildcard_partial_prefix_expansion.py index 8122ccccf539..c2f76962176d 100644 --- a/tests/integration/routing/test_wildcard_partial_prefix_expansion.py +++ b/tests/integration/routing/test_wildcard_partial_prefix_expansion.py @@ -11,7 +11,7 @@ import httpx import pytest from integration._support.client import JSON_OBJECT, Gateway, Scenario, list_value, object_value, string_value -from integration._support.openai_wire import answering_model_discovery, chat_reply, responses_reply +from integration._support.openai_wire import answering_model_discovery, chat_reply, openai_error, responses_reply from integration._support.timing import assert_within_factor_of_control, timed from integration._support.wire import Reply, Request, Wire, wire_server from pydantic import JsonValue @@ -36,13 +36,18 @@ LISTING_TIMEOUT_SECONDS: Final = 120 -def _register(scenario: Scenario, model_name: str, model: str, api_base: str) -> str: +def _register(scenario: Scenario, model_name: str, model: str, api_base: str, **params: JsonValue) -> str: created: Final = object_value( scenario.gateway.post( "/model/new", { "model_name": model_name, - "litellm_params": {"model": model, "api_key": "integration-wildcard-key", "api_base": api_base}, + "litellm_params": { + "model": model, + "api_key": "integration-wildcard-key", + "api_base": api_base, + **params, + }, }, ) ) @@ -88,6 +93,19 @@ def _openai_reply(request: Request) -> Reply: OPENAI_UPSTREAM: Final = answering_model_discovery(_openai_reply) +def _reply_failing_the_gpt_4o_family(request: Request) -> Reply: + if string_value(JSON_OBJECT.validate_json(request.body)["model"]).startswith("gpt-4o"): + return openai_error(404) + return _openai_reply(request) + + +GPT_4O_FAILING_UPSTREAM: Final = answering_model_discovery(_reply_failing_the_gpt_4o_family) + + +def _empty_file_list(request: Request) -> Reply: + return Reply(body=b'{"object": "list", "data": [], "has_more": false}') + + def _call(gateway: Gateway, model: str) -> httpx.Response: response: Final = gateway.request( "POST", @@ -215,10 +233,24 @@ def test_each_wildcard_row_names_the_deployment_its_id_routes_to( @pytest.mark.parametrize( - "narrow_model", ("openai/gpt-4.1*", "openai/gpt-4o-mini"), ids=("same_model", "different_model") + ("narrow_model", "narrow_params", "reached_model"), + ( + ("openai/gpt-4.1*", MappingProxyType({}), "gpt-4.1"), + ("openai/gpt-4o-mini", MappingProxyType({}), "gpt-4o-mini"), + ("openai/responses/gpt-4.1*", MappingProxyType({}), "gpt-4.1"), + ("gpt-4.1*", MappingProxyType({"custom_llm_provider": "openai"}), "gpt-4.1"), + ("azure/gpt-4.1*", MappingProxyType({"api_version": "2024-10-21"}), "gpt-4.1"), + ), + ids=( + "same_model", + "different_model", + "same_model_through_responses_route", + "same_model_as_bare_model_with_provider", + "same_model_on_another_provider", + ), ) def test_key_scoped_to_a_wildcard_lists_an_id_a_narrower_route_serves_only_if_it_reaches_the_named_model( - gateway: Gateway, narrow_model: str + gateway: Gateway, narrow_model: str, narrow_params: MappingProxyType[str, JsonValue], reached_model: str ) -> None: namespace: Final = f"keyscope-{uuid.uuid4().hex[:12]}" named_model: Final = "gpt-4.1" @@ -226,9 +258,14 @@ def test_key_scoped_to_a_wildcard_lists_an_id_a_narrower_route_serves_only_if_it with wire_server(OPENAI_UPSTREAM) as wire, gateway.scenario() as scenario: _register(scenario, f"{namespace}/*", "openai/*", f"{wire.url}/v1") - _register(scenario, f"{namespace}/{named_model}*", narrow_model, f"{wire.url}/v1") + _register(scenario, f"{namespace}/{named_model}*", narrow_model, f"{wire.url}/v1", **narrow_params) key: Final = scenario.key(models=[f"{namespace}/*"]) listing: Final = gateway.request("GET", "/v1/models", key=key) + info_models: Final = tuple( + string_value(object_value(row["litellm_params"])["model"]) + for row in _rows(gateway, "/model/info") + if row["model_name"] == narrowly_served + ) wire.drain() call: Final = gateway.request( "POST", @@ -239,15 +276,18 @@ def test_key_scoped_to_a_wildcard_lists_an_id_a_narrower_route_serves_only_if_it upstream_model: Final = _upstream_model(wire) assert call.status_code == 200, call.text - assert upstream_model == narrow_model.removeprefix("openai/").removesuffix("*") + assert upstream_model == reached_model assert listing.status_code == 200, listing.text listed: Final = tuple( string_value(object_value(entry)["id"]) for entry in list_value(JSON_OBJECT.validate_json(listing.content)["data"]) ) in_namespace: Final = sorted(model_id for model_id in listed if model_id.startswith(namespace)) + reaches_named_model: Final = upstream_model == named_model assert f"{namespace}/gpt-4o-mini" in listed, in_namespace - assert (narrowly_served in listed) is (upstream_model == named_model), in_namespace + assert (narrowly_served in listed) is reaches_named_model, in_namespace + assert bool(info_models) is reaches_named_model, info_models + assert all(model.rsplit("/", 1)[-1] == upstream_model for model in info_models), info_models def test_overlapping_deployments_of_one_wildcard_list_each_id_once(gateway: Gateway) -> None: @@ -284,6 +324,53 @@ def test_wildcard_health_check_probes_a_model_its_pattern_names( assert probe.startswith(pattern.removesuffix("*")) is probes_match, probe +def test_wildcard_health_check_stays_healthy_when_every_model_its_pattern_names_fails_but_the_provider_answers( + gateway: Gateway, +) -> None: + model_name: Final = f"health-{uuid.uuid4().hex[:12]}/gpt-4o*" + + with wire_server(GPT_4O_FAILING_UPSTREAM) as wire, gateway.scenario() as scenario: + _register(scenario, model_name, "openai/gpt-4o*", f"{wire.url}/v1") + wire.drain() + health: Final = gateway.get("/health", {"model": model_name}) + probes: Final = tuple( + string_value(JSON_OBJECT.validate_json(request.body)["model"]) + for request in wire.drain() + if request.method == "POST" + ) + + assert (health["healthy_count"], health["unhealthy_count"]) == (1, 0), (health, probes) + assert probes and not probes[-1].startswith("gpt-4o"), probes + + +def test_team_file_listing_takes_the_credentials_of_a_narrower_wildcard_route_on_the_requested_provider( + gateway: Gateway, +) -> None: + namespace: Final = f"files-{uuid.uuid4().hex[:12]}" + narrow_api_key: Final = f"integration-azure-{uuid.uuid4().hex[:12]}" + + with wire_server(answering_model_discovery(_empty_file_list)) as wire, gateway.scenario() as scenario: + _register(scenario, f"{namespace}/*", "openai/*", UNREACHABLE_API_BASE) + _register( + scenario, + f"{namespace}/gpt-4o*", + "azure/gpt-4o*", + wire.url, + api_key=narrow_api_key, + api_version="2024-10-21", + ) + team: Final = scenario.team(models=[f"{namespace}/*"]) + key: Final = scenario.key(team_id=team) + wire.drain() + listing: Final = gateway.request("GET", "/v1/files", key=key, params={"custom_llm_provider": "azure"}) + received: Final = tuple(request for request in wire.drain() if "/openai/files" in request.target) + + assert listing.status_code == 200, listing.text + assert [ + {name.lower(): value for name, value in request.headers.items()}.get("api-key") for request in received + ] == [narrow_api_key], [request.target for request in received] + + def _namespace_listing_seconds(gateway: Gateway, path: str, namespace: str) -> float: listing: Final = timed( lambda: gateway.client.get( From be0d53da73723a3868db03e8c6519be9eb43e746 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:49:33 -0700 Subject: [PATCH 18/22] test(proxy): cover a provider-scoped key listing beside a catch-all wildcard deployment --- .../test_wildcard_model_access.py | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/tests/integration/authorization/test_wildcard_model_access.py b/tests/integration/authorization/test_wildcard_model_access.py index f63c2aac84a4..603c7e5cdbe1 100644 --- a/tests/integration/authorization/test_wildcard_model_access.py +++ b/tests/integration/authorization/test_wildcard_model_access.py @@ -6,11 +6,19 @@ import httpx import pytest import yaml -from integration._support.client import Gateway, gateway_from_environment +from integration._support.client import ( + JSON_OBJECT, + Gateway, + gateway_from_environment, + list_value, + object_value, + string_value, +) from integration._support.process import owned_proxy from pydantic import JsonValue pytestmark: Final = pytest.mark.timeout(180) +SAMPLED_LISTED_IDS: Final = 3 def _deployment(model_name: str, model: str, upstream_url: str) -> dict[str, JsonValue]: @@ -57,6 +65,13 @@ def _observed_models(upstream: httpx.Client) -> list[JsonValue]: return [request["body"]["model"] for request in upstream.get("/__observations").json()["requests"]] +def _listed_names(gateway: Gateway, path: str, field: str, key: str) -> list[str]: + response: Final = gateway.request("GET", path, key=key) + assert response.status_code == 200, response.text + rows: Final = list_value(JSON_OBJECT.validate_json(response.content)["data"]) + return [string_value(object_value(row)[field]) for row in rows] + + def test_an_all_models_key_reaches_a_model_served_only_by_the_catch_all_deployment( candidate: Gateway, upstream: httpx.Client ) -> None: @@ -88,3 +103,16 @@ def test_a_key_without_models_inherits_the_users_exact_and_wildcard_grants( assert _observed_models(upstream) == [] assert _chat(candidate, f"groq/{wildcard_model}", candidate.key).status_code == 200 assert _observed_models(upstream) == [wildcard_model] + + +def test_a_key_scoped_to_a_provider_wildcard_on_the_catch_all_lists_only_ids_that_reach_the_upstream_as_named( + candidate: Gateway, upstream: httpx.Client +) -> None: + with candidate.scenario() as scenario: + key: Final = scenario.key(models=["openai/*"]) + listed: Final = _listed_names(candidate, "/v1/models", "id", key) + sampled: Final = listed[:SAMPLED_LISTED_IDS] + for model in sampled: + _chat(candidate, model, key) + assert _observed_models(upstream) == [model.removeprefix("openai/") for model in sampled] + assert sorted(_listed_names(candidate, "/model/info", "model_name", key)) == sorted(listed) From 743790b2a2f0e81682d2b596c966b3c256013a40 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:33:12 -0700 Subject: [PATCH 19/22] fix(proxy): list ids a narrower route serves for keys scoped to a wider wildcard, and treat an aggregator's vendor-namespaced model as the same model --- litellm/proxy/auth/model_checks.py | 51 +++++++--- .../router_utils/pattern_match_deployments.py | 6 +- .../test_wildcard_partial_prefix_expansion.py | 7 +- tests/unit/proxy/auth/test_model_checks.py | 99 +++++++++++++++++-- .../test_pattern_match_deployments.py | 2 + 5 files changed, 138 insertions(+), 27 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 39c05dc60661..4d73b5fbc916 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -427,24 +427,49 @@ def expand_wildcard_deployments_for_model_info( return expanded -def _models_listed_for_wildcard_route( - model: str, - return_wildcard_route: bool, - llm_router: Router | None, - team_id: str | None, -) -> tuple[str, ...]: - model_list: Final = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router else None +def _ids_listed_by_route(route: str, llm_router: Router | None, team_id: str | None) -> tuple[str, ...]: + model_list: Final = llm_router.get_model_list(model_name=route, team_id=team_id) if llm_router else None pattern_router: Final = llm_router.pattern_router if llm_router else None listings: Final = ( chain.from_iterable( - _listings_served_by_route(model, LiteLLM_Params(**router_model["litellm_params"]), pattern_router) + _listings_served_by_route(route, LiteLLM_Params(**router_model["litellm_params"]), pattern_router) for router_model in model_list ) if model_list - else _listings_served_by_route(model, None, pattern_router) + else _listings_served_by_route(route, None, pattern_router) + ) + return tuple(listing.listed_id for listing in listings) + + +def _routes_can_list_same_ids(route: str, other_route: str) -> bool: + prefix: Final = route.partition("*")[0] + other_prefix: Final = other_route.partition("*")[0] + return prefix.startswith(other_prefix) or other_prefix.startswith(prefix) + + +def _ids_other_routes_serve_under( + wildcards: tuple[str, ...], llm_router: Router | None, team_id: str | None +) -> tuple[str, ...]: + if llm_router is None or not wildcards: + return () + other_routes: Final = tuple( + dict.fromkeys( + route + for route in llm_router.get_model_names(team_id=team_id) + if _check_wildcard_routing(route) and route not in wildcards + ) + ) + overlapping_routes: Final = ( + route for route in other_routes if any(_routes_can_list_same_ids(route, wildcard) for wildcard in wildcards) + ) + other_ids: Final = chain.from_iterable( + _ids_listed_by_route(route, llm_router, team_id) for route in overlapping_routes + ) + return tuple( + listed_id + for listed_id in other_ids + if any(model_matches_wildcard(listed_id, wildcard) for wildcard in wildcards) ) - expanded: Final = tuple(listing.listed_id for listing in listings) - return (model, *expanded) if return_wildcard_route else expanded def _get_wildcard_models( @@ -457,10 +482,10 @@ def _get_wildcard_models( for model in wildcard_routes: unique_models.remove(model) listed: Final = chain.from_iterable( - _models_listed_for_wildcard_route(model, bool(return_wildcard_routes), llm_router, team_id) + (*((model,) if return_wildcard_routes else ()), *_ids_listed_by_route(model, llm_router, team_id)) for model in wildcard_routes ) - return list(dict.fromkeys(listed)) + return list(dict.fromkeys(chain(listed, _ids_other_routes_serve_under(wildcard_routes, llm_router, team_id)))) def get_all_fallbacks( diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index 668be7938fc8..fa8002416a01 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -223,14 +223,16 @@ def routes_elsewhere(self, request: str, wildcard: str, routed_model: str) -> bo return False deployments: Final = _ROUTED_DEPLOYMENTS.validate_python(self.patterns[serving_regex]) listed_model: Final = model_name_at_provider(routed_model, None) - return all( + served_models: Final = ( model_name_at_provider( fill_wildcards(captures, deployment.litellm_params.model, request), deployment.litellm_params.custom_llm_provider, ) - != listed_model for deployment in deployments ) + return not any( + served_model == listed_model or served_model.endswith(f"/{listed_model}") for served_model in served_models + ) def routes_to_provider(self, request: str, custom_llm_provider: str) -> bool: deployments: Final = _ROUTED_DEPLOYMENTS.validate_python(self.route(request) or ()) diff --git a/tests/integration/routing/test_wildcard_partial_prefix_expansion.py b/tests/integration/routing/test_wildcard_partial_prefix_expansion.py index c2f76962176d..88d9e5340745 100644 --- a/tests/integration/routing/test_wildcard_partial_prefix_expansion.py +++ b/tests/integration/routing/test_wildcard_partial_prefix_expansion.py @@ -240,6 +240,7 @@ def test_each_wildcard_row_names_the_deployment_its_id_routes_to( ("openai/responses/gpt-4.1*", MappingProxyType({}), "gpt-4.1"), ("gpt-4.1*", MappingProxyType({"custom_llm_provider": "openai"}), "gpt-4.1"), ("azure/gpt-4.1*", MappingProxyType({"api_version": "2024-10-21"}), "gpt-4.1"), + ("openrouter/openai/gpt-4.1*", MappingProxyType({}), "openai/gpt-4.1"), ), ids=( "same_model", @@ -247,6 +248,7 @@ def test_each_wildcard_row_names_the_deployment_its_id_routes_to( "same_model_through_responses_route", "same_model_as_bare_model_with_provider", "same_model_on_another_provider", + "same_model_under_a_vendor_namespace", ), ) def test_key_scoped_to_a_wildcard_lists_an_id_a_narrower_route_serves_only_if_it_reaches_the_named_model( @@ -283,11 +285,12 @@ def test_key_scoped_to_a_wildcard_lists_an_id_a_narrower_route_serves_only_if_it for entry in list_value(JSON_OBJECT.validate_json(listing.content)["data"]) ) in_namespace: Final = sorted(model_id for model_id in listed if model_id.startswith(namespace)) - reaches_named_model: Final = upstream_model == named_model + reached_name: Final = upstream_model.rsplit("/", 1)[-1] + reaches_named_model: Final = reached_name == named_model assert f"{namespace}/gpt-4o-mini" in listed, in_namespace assert (narrowly_served in listed) is reaches_named_model, in_namespace assert bool(info_models) is reaches_named_model, info_models - assert all(model.rsplit("/", 1)[-1] == upstream_model for model in info_models), info_models + assert all(model.rsplit("/", 1)[-1] == reached_name for model in info_models), info_models def test_overlapping_deployments_of_one_wildcard_list_each_id_once(gateway: Gateway) -> None: diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 458d95759ff4..7794a50ba681 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -700,7 +700,9 @@ def test_wildcard_lists_no_id_that_a_more_specific_route_serves( from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list monkeypatch.setattr(litellm, "check_provider_endpoint", False) - monkeypatch.setitem(litellm.models_by_provider, "openrouter", ["openrouter/openai/gpt-4o", "meta-llama/llama-3-8b"]) + monkeypatch.setitem( + litellm.models_by_provider, "openrouter", ["openrouter/openai/gpt-4o-via-openrouter", "meta-llama/llama-3-8b"] + ) router: Final = Router( model_list=[ {"model_name": "*", "litellm_params": {"model": "openrouter/*", "api_key": "key"}}, @@ -711,19 +713,22 @@ def test_wildcard_lists_no_id_that_a_more_specific_route_serves( def served_by(model_id: str) -> str: return str((router.pattern_router.route(model_id) or [{}])[0]["litellm_params"]["model"]) - listed_by_star: Final = get_complete_model_list( - key_models=["*"], - team_models=[], - proxy_model_list=[], - user_model=None, - infer_model_from_keys=False, - llm_router=router, - ) + def listed_for(key_models: list[str]) -> list[str]: + return get_complete_model_list( + key_models=key_models, + team_models=[], + proxy_model_list=[], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + + listed_by_star: Final = listed_for(["*"]) model_info_rows: Final = expand_wildcard_deployments_for_model_info( copy.deepcopy(router.model_list), pattern_router=router.pattern_router ) - assert listed_by_star == ["meta-llama/llama-3-8b"] + assert sorted(listed_by_star) == sorted(["meta-llama/llama-3-8b", *listed_for(["openai/*"])]) assert "openai/gpt-4o" in [row["model_name"] for row in model_info_rows] assert [ row["model_name"] for row in model_info_rows if served_by(row["model_name"]) != row["litellm_params"]["model"] @@ -1225,6 +1230,80 @@ def test_key_scoped_to_a_wide_route_lists_ids_a_narrower_route_sends_to_the_same assert sorted(listed) == ["openai/gpt-4.1-mini", "openai/gpt-4.1-nano", "openai/gpt-4o"] +@pytest.mark.parametrize( + ("key_models", "expected"), + [ + ([], ["wide/gpt-4.1", "wide/gpt-4.1-mini", "wide/gpt-4o", "wide/gpt-4o-mini", "wide/gpt-5"]), + (["wide/*"], ["wide/gpt-4.1", "wide/gpt-4.1-mini", "wide/gpt-4o", "wide/gpt-4o-mini", "wide/gpt-5"]), + (["wide/gpt-4.1*"], ["wide/gpt-4.1", "wide/gpt-4.1-mini"]), + (["wide/gpt-4o*"], ["wide/gpt-4o", "wide/gpt-4o-mini"]), + (["other/*"], []), + ], + ids=("admin", "key_on_wide_route", "key_on_narrow_route", "key_pattern_without_a_route", "key_elsewhere"), +) +def test_key_scoped_to_a_wildcard_lists_the_ids_every_route_serves_under_its_pattern( + monkeypatch: pytest.MonkeyPatch, key_models: list[str], expected: list[str] +) -> None: + import litellm + from litellm import Router + from litellm.proxy.auth.model_checks import get_complete_model_list + + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + monkeypatch.setitem(litellm.models_by_provider, "openai", ["gpt-4o", "gpt-4o-mini", "gpt-4.1", "gpt-4.1-mini", "gpt-5"]) + router: Final = Router( + model_list=[ + {"model_name": "wide/*", "litellm_params": {"model": "openai/*"}}, + {"model_name": "wide/gpt-4.1*", "litellm_params": {"model": "openai/gpt-4o*"}}, + ] + ) + + listed: Final = get_complete_model_list( + key_models=key_models, + team_models=[], + proxy_model_list=["wide/*", "wide/gpt-4.1*"], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + + assert sorted(listed) == expected + + +def test_key_scoped_to_a_wildcard_lists_only_its_own_teams_narrower_routes(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + from litellm import Router + from litellm.proxy.auth.model_checks import get_complete_model_list + + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + monkeypatch.setitem(litellm.models_by_provider, "openai", ["gpt-4o", "gpt-4o-mini", "gpt-4.1", "gpt-5"]) + router: Final = Router( + model_list=[ + {"model_name": "wide/*", "litellm_params": {"model": "openai/*"}}, + { + "model_name": "team-b-narrow", + "litellm_params": {"model": "openai/gpt-4o*"}, + "model_info": {"team_id": "team-b", "team_public_model_name": "wide/gpt-4.1*"}, + }, + ] + ) + + def listed_for(team_id: str) -> list[str]: + return sorted( + get_complete_model_list( + key_models=["wide/*"], + team_models=[], + proxy_model_list=["wide/*"], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + team_id=team_id, + ) + ) + + assert "wide/gpt-4.1-mini" in listed_for("team-b") + assert "wide/gpt-4.1-mini" not in listed_for("team-a") + + def _wide_route_beside_a_narrower_route_to_another_provider(monkeypatch: pytest.MonkeyPatch): import litellm from litellm import Router diff --git a/tests/unit/router_utils/test_pattern_match_deployments.py b/tests/unit/router_utils/test_pattern_match_deployments.py index 13ab50314990..3f2ecb8842dc 100644 --- a/tests/unit/router_utils/test_pattern_match_deployments.py +++ b/tests/unit/router_utils/test_pattern_match_deployments.py @@ -502,6 +502,8 @@ def test_routes_elsewhere_only_when_the_serving_route_sends_another_model( ("openai/*", "wide/gpt-4o*", "azure/gpt-4o*", None, "wide/gpt-4o-mini", False), ("openai/*", "wide/gpt-4.1*", "openai/gpt-4o-mini", None, "wide/gpt-4.1-mini", True), ("openai/*", "wide/gpt-4o*", "openai/responses/gpt-4o-mini", None, "wide/gpt-4o", True), + ("openai/*", "wide/gpt-4o*", "openrouter/openai/gpt-4o*", None, "wide/gpt-4o-mini", False), + ("openai/*", "wide/gpt-4o*", "openrouter/openai/chatgpt-4o-latest", None, "wide/gpt-4o-latest", True), ], ) def test_routes_elsewhere_compares_the_model_each_route_reaches_at_its_provider( From bb2b5b101325ea245a0fe03e6b3aee54e1d1ca26 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:22:26 -0700 Subject: [PATCH 20/22] fix(proxy): list bare-model wildcard ids under their provider and skip team route ids a proxy-wide route serves first --- litellm/proxy/auth/model_checks.py | 24 ++++++- .../router_utils/pattern_match_deployments.py | 6 ++ tests/unit/proxy/auth/test_model_checks.py | 70 +++++++++++++++++-- .../test_pattern_match_deployments.py | 30 ++++++++ 4 files changed, 123 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 69db593da235..0dfcc26c6b36 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -305,11 +305,21 @@ def _same_pattern_listing( ) +def _deployment_provider_and_path(deployment_model: str, custom_llm_provider: str | None) -> tuple[str, str] | None: + provider, provider_sep, deployment_path = deployment_model.partition("/") + if provider_sep: + return provider, deployment_path + return (custom_llm_provider, deployment_model) if custom_llm_provider else None + + def _wildcard_listings(wildcard_model: str, litellm_params: LiteLLM_Params | None) -> tuple[_WildcardListing, ...]: deployment_model: Final = litellm_params.model if litellm_params is not None else wildcard_model - provider, provider_sep, deployment_path = deployment_model.partition("/") - if not provider_sep: + provider_and_path: Final = _deployment_provider_and_path( + deployment_model, litellm_params.custom_llm_provider if litellm_params is not None else None + ) + if provider_and_path is None: return () + provider, deployment_path = provider_and_path route_type_prefix, model_pattern = split_route_type_prefix(deployment_path, provider) if wildcard_model.count("*") == 1 and "*" not in model_pattern: @@ -427,6 +437,11 @@ def expand_wildcard_deployments_for_model_info( return expanded +def _is_team_only_route(route: str, llm_router: Router, team_id: str | None) -> bool: + team_router: Final = llm_router.team_pattern_routers.get(team_id) if team_id is not None else None + return team_router is not None and team_router.registers(route) and not llm_router.pattern_router.registers(route) + + def _ids_listed_by_route(route: str, llm_router: Router | None, team_id: str | None) -> tuple[str, ...]: model_list: Final = llm_router.get_model_list(model_name=route, team_id=team_id) if llm_router else None pattern_router: Final = llm_router.pattern_router if llm_router else None @@ -438,7 +453,10 @@ def _ids_listed_by_route(route: str, llm_router: Router | None, team_id: str | N if model_list else _listings_served_by_route(route, None, pattern_router) ) - return tuple(listing.listed_id for listing in listings) + listed_ids: Final = (listing.listed_id for listing in listings) + if llm_router is None or not _is_team_only_route(route, llm_router, team_id): + return tuple(listed_ids) + return tuple(listed_id for listed_id in listed_ids if not llm_router.pattern_router.serves(listed_id)) def _routes_can_list_same_ids(route: str, other_route: str) -> bool: diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index fa8002416a01..b02fd9b3cc39 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -216,6 +216,12 @@ def shadowing_regex(self, request: str, wildcard: str) -> str | None: serving_regex: Final = next((regex for regex in self.patterns if matches_from_start(regex, request)), own_regex) return None if serving_regex == own_regex else serving_regex + def serves(self, request: str) -> bool: + return any(matches_from_start(regex, request) for regex in self.patterns) + + def registers(self, route: str) -> bool: + return self.pattern_to_regex(route) in self.patterns + def routes_elsewhere(self, request: str, wildcard: str, routed_model: str) -> bool: serving_regex: Final = self.shadowing_regex(request, wildcard) captures: Final = greedy_captures(serving_regex, request) if serving_regex is not None else None diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 17b86972bdf8..1eaae3b0b3e5 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -630,6 +630,33 @@ def test_azure_route_type_infix_lists_the_same_ids_as_the_route_without_it( ) +@pytest.mark.parametrize("model_pattern", ["*", "gpt-4*"]) +def test_bare_deployment_model_with_custom_llm_provider_lists_the_same_ids_as_the_provider_prefixed_model( + monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, model_pattern: str +) -> None: + import litellm + from litellm.proxy.auth.model_checks import ( + expand_wildcard_deployments_for_model_info, + get_known_models_from_wildcard, + ) + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + bare_deployment: Final = { + "model_name": "openai/*", + "litellm_params": {"model": model_pattern, "custom_llm_provider": "openai"}, + } + + prefixed: Final = get_known_models_from_wildcard("openai/*", LiteLLM_Params(model=f"openai/{model_pattern}")) + + assert prefixed + assert ( + get_known_models_from_wildcard("openai/*", LiteLLM_Params(model=model_pattern, custom_llm_provider="openai")) + == prefixed + ) + assert [row["model_name"] for row in expand_wildcard_deployments_for_model_info([bare_deployment])] == prefixed + + @pytest.mark.parametrize( ("model_name", "deployment_model"), [ @@ -1281,7 +1308,7 @@ def test_key_scoped_to_a_wildcard_lists_only_its_own_teams_narrower_routes(monke { "model_name": "team-b-narrow", "litellm_params": {"model": "openai/gpt-4o*"}, - "model_info": {"team_id": "team-b", "team_public_model_name": "wide/gpt-4.1*"}, + "model_info": {"team_id": "team-b", "team_public_model_name": "tw/gpt-4.1*"}, }, ] ) @@ -1290,7 +1317,7 @@ def test_key_scoped_to_a_wildcard_lists_only_its_own_teams_narrower_routes(monke def listed_for(team_id: str) -> list[str]: return sorted( get_complete_model_list( - key_models=["wide/*"], + key_models=["tw/*"], team_models=[], proxy_model_list=["wide/*"], user_model=None, @@ -1300,8 +1327,43 @@ def listed_for(team_id: str) -> list[str]: ) ) - assert "wide/gpt-4.1-mini" in listed_for("team-b") - assert "wide/gpt-4.1-mini" not in listed_for("team-a") + assert "tw/gpt-4.1-mini" in listed_for("team-b") + assert "tw/gpt-4.1-mini" not in listed_for("team-a") + + +@pytest.mark.parametrize(("key_models", "team_models"), [(["wide/*"], []), ([], ["wide/*", "wide/claude-*"])]) +def test_team_key_does_not_list_team_route_ids_a_proxy_wide_route_catches_first( + monkeypatch: pytest.MonkeyPatch, key_models: list[str], team_models: list[str] +) -> None: + import litellm + from litellm import Router + from litellm.proxy.auth.model_checks import get_complete_model_list + + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + team_route: Final = { + "model_name": "team-b-claude", + "litellm_params": {"model": "anthropic/claude-*"}, + "model_info": {"team_id": "team-b", "team_public_model_name": "wide/claude-*"}, + } + router: Final = Router(model_list=[{"model_name": "wide/*", "litellm_params": {"model": "openai/*"}}, team_route]) + monkeypatch.setitem(litellm.models_by_provider, "openai", ["gpt-4o", "gpt-4o-mini"]) + monkeypatch.setitem(litellm.models_by_provider, "anthropic", ["claude-haiku-4-5", "claude-sonnet-4-5"]) + unshadowed_router: Final = Router(model_list=[team_route]) + + def listed(llm_router: Router) -> list[str]: + return get_complete_model_list( + key_models=key_models, + team_models=team_models, + proxy_model_list=["wide/*"], + user_model=None, + infer_model_from_keys=False, + llm_router=llm_router, + team_id="team-b", + ) + + assert "wide/claude-haiku-4-5" in listed(unshadowed_router) + assert not [model for model in listed(router) if model.startswith("wide/claude-")] + assert "wide/gpt-4o" in listed(router) def _wide_route_beside_a_narrower_route_to_another_provider(monkeypatch: pytest.MonkeyPatch): diff --git a/tests/unit/router_utils/test_pattern_match_deployments.py b/tests/unit/router_utils/test_pattern_match_deployments.py index 3f2ecb8842dc..dc215e865e5c 100644 --- a/tests/unit/router_utils/test_pattern_match_deployments.py +++ b/tests/unit/router_utils/test_pattern_match_deployments.py @@ -385,6 +385,36 @@ def test_pattern_matching_router_uses_default_wildcard_for_unmatched_model() -> assert tuple(deployment["model_name"] for deployment in deployments) == ("*",) +@pytest.mark.parametrize( + "request_model", ["openai/gpt-4o", "team/claude-haiku-4-5", "teamx/claude", "odd?[name]/x", "oddX[name]/x"] +) +def test_serves_agrees_with_whether_routing_finds_a_pattern(request_model: str) -> None: + pattern_router: Final = PatternMatchRouter() + pattern_router.add_pattern("openai/*", _wildcard_deployment("openai/*")) + pattern_router.add_pattern("team/claude-*", _wildcard_deployment("team/claude-*")) + pattern_router.add_pattern("odd?[name]/*", _wildcard_deployment("odd?[name]/*")) + + assert pattern_router.serves(request_model) is (pattern_router.route(request_model) is not None) + + +@pytest.mark.parametrize( + ("route", "registered"), + [ + ("openai/*", True), + ("odd?[name]/*", True), + ("openai/gpt-*", False), + ("odd?[name]/x*", False), + ("oddX[name]/*", False), + ], +) +def test_registers_only_routes_that_were_added(route: str, registered: bool) -> None: + pattern_router: Final = PatternMatchRouter() + pattern_router.add_pattern("openai/*", _wildcard_deployment("openai/*")) + pattern_router.add_pattern("odd?[name]/*", _wildcard_deployment("odd?[name]/*")) + + assert pattern_router.registers(route) is registered + + @pytest.mark.parametrize( ("request_model", "wildcard", "shadowed"), [ From 67488f8b0d093ec809481254620b03503204d367 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 16 Oct 2026 21:32:27 -0700 Subject: [PATCH 21/22] fix(proxy): skip ids a narrower route sends to another model for keys granted a wildcard that is not a route --- .../router_utils/pattern_match_deployments.py | 4 +-- tests/unit/proxy/auth/test_model_checks.py | 29 +++++++++++++++++++ .../test_pattern_match_deployments.py | 4 +-- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index b02fd9b3cc39..f9c847f16719 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -211,9 +211,7 @@ def route(self, request: str | None, filtered_model_names: list[str] | None = No def shadowing_regex(self, request: str, wildcard: str) -> str | None: own_regex: Final = self.pattern_to_regex(wildcard) - if own_regex not in self.patterns: - return None - serving_regex: Final = next((regex for regex in self.patterns if matches_from_start(regex, request)), own_regex) + serving_regex: Final = next((regex for regex in self.patterns if matches_from_start(regex, request)), None) return None if serving_regex == own_regex else serving_regex def serves(self, request: str) -> bool: diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 1eaae3b0b3e5..4e474d3573e2 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -1257,6 +1257,35 @@ def test_key_scoped_to_a_wide_route_lists_ids_a_narrower_route_sends_to_the_same assert sorted(listed) == ["openai/gpt-4.1-mini", "openai/gpt-4.1-nano", "openai/gpt-4o"] +def test_key_granted_a_pattern_that_is_not_a_route_skips_ids_a_narrower_route_sends_to_another_model( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm import Router + from litellm.proxy.auth.model_checks import get_complete_model_list + + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + router: Final = Router( + model_list=[ + {"model_name": "openai/*", "litellm_params": {"model": "openai/*"}}, + {"model_name": "openai/gpt-4o*", "litellm_params": {"model": "anthropic/claude-sonnet-4*"}}, + ] + ) + monkeypatch.setitem(litellm.models_by_provider, "openai", ["gpt-4o", "gpt-4o-mini", "gpt-4.1"]) + monkeypatch.setitem(litellm.models_by_provider, "anthropic", ["claude-sonnet-4-5"]) + + listed: Final = get_complete_model_list( + key_models=["openai/gpt-4*"], + team_models=[], + proxy_model_list=["openai/*", "openai/gpt-4o*"], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + + assert sorted(listed) == ["openai/gpt-4.1", "openai/gpt-4o-5"] + + @pytest.mark.parametrize( ("key_models", "expected"), [ diff --git a/tests/unit/router_utils/test_pattern_match_deployments.py b/tests/unit/router_utils/test_pattern_match_deployments.py index dc215e865e5c..4e59715697f8 100644 --- a/tests/unit/router_utils/test_pattern_match_deployments.py +++ b/tests/unit/router_utils/test_pattern_match_deployments.py @@ -421,10 +421,10 @@ def test_registers_only_routes_that_were_added(route: str, registered: bool) -> ("openai/gpt-4o", "*", True), ("openai/gpt-4o", "openai/*", False), ("meta-llama/llama-3-8b", "*", False), - ("openai/gpt-4o", "team/*", False), + ("openai/gpt-4o", "openai/gpt-*", True), ], ) -def test_shadowing_regex_only_when_a_more_specific_registered_pattern_serves_the_request( +def test_shadowing_regex_only_when_another_registered_pattern_serves_the_request( request_model: str, wildcard: str, shadowed: bool ) -> None: pattern_router: Final = PatternMatchRouter() From bc4d050fb16a15909572f8c9d403055e3544ae74 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:02:55 -0700 Subject: [PATCH 22/22] fix(proxy): keep exact deployments and same-provider team ids in wildcard listings, and list ns/* over a bare * --- litellm/proxy/auth/model_checks.py | 85 +++++++++++++++++----- tests/unit/proxy/auth/test_model_checks.py | 77 ++++++++++++++++++-- 2 files changed, 140 insertions(+), 22 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 0dfcc26c6b36..c2289afb6d78 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -10,6 +10,7 @@ from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.get_llm_provider_logic import ( + inferred_provider, model_matches_wildcard, split_route_type_prefix, strip_provider_family_prefix, @@ -24,9 +25,7 @@ from litellm.utils import ProviderConfigManager, get_valid_models _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) - - -_CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) +_KNOWN_PROVIDER_NAMES: Final = frozenset(provider.value for provider in LlmProviders) def _check_wildcard_routing(model: str) -> bool: @@ -312,13 +311,35 @@ def _deployment_provider_and_path(deployment_model: str, custom_llm_provider: st return (custom_llm_provider, deployment_model) if custom_llm_provider else None +def _model_id_litellm_infers_provider_for(model: str) -> str: + provider, _, model_id = model.partition("/") + return model_id if provider in _KNOWN_PROVIDER_NAMES and model_id and "/" not in model_id else model + + +def _namespaced_passthrough_listings( + wildcard_model: str, litellm_params: LiteLLM_Params | None +) -> tuple[_WildcardListing, ...]: + if wildcard_model == "*" or wildcard_model.count("*") != 1: + return () + provider_models: Final = get_provider_models( + provider="*", litellm_params=_hydrate_litellm_credential_name(litellm_params) + ) + model_ids: Final = (_model_id_litellm_infers_provider_for(model) for model in provider_models or ()) + return tuple( + dict.fromkeys( + _WildcardListing(listed_id=wildcard_model.replace("*", model_id, 1), routed_model=model_id) + for model_id in model_ids + ) + ) + + def _wildcard_listings(wildcard_model: str, litellm_params: LiteLLM_Params | None) -> tuple[_WildcardListing, ...]: deployment_model: Final = litellm_params.model if litellm_params is not None else wildcard_model provider_and_path: Final = _deployment_provider_and_path( deployment_model, litellm_params.custom_llm_provider if litellm_params is not None else None ) if provider_and_path is None: - return () + return _namespaced_passthrough_listings(wildcard_model, litellm_params) if deployment_model == "*" else () provider, deployment_path = provider_and_path route_type_prefix, model_pattern = split_route_type_prefix(deployment_path, provider) @@ -350,7 +371,10 @@ def _wildcard_listings(wildcard_model: str, litellm_params: LiteLLM_Params | Non def _listings_served_by_route( - wildcard_model: str, litellm_params: LiteLLM_Params | None, pattern_router: PatternMatchRouter | None + wildcard_model: str, + litellm_params: LiteLLM_Params | None, + pattern_router: PatternMatchRouter | None, + exact_model_names: frozenset[str], ) -> tuple[_WildcardListing, ...]: listings: Final = _wildcard_listings(wildcard_model, litellm_params) if pattern_router is None: @@ -358,7 +382,8 @@ def _listings_served_by_route( return tuple( listing for listing in listings - if not pattern_router.routes_elsewhere(listing.listed_id, wildcard_model, listing.routed_model) + if listing.listed_id in exact_model_names + or not pattern_router.routes_elsewhere(listing.listed_id, wildcard_model, listing.routed_model) ) @@ -387,7 +412,7 @@ def _model_info_rows( shadowed=pattern_router is not None and pattern_router.shadowing_regex(listing.listed_id, model_name) is not None, ) - for listing in _listings_served_by_route(model_name, litellm_params, pattern_router) + for listing in _listings_served_by_route(model_name, litellm_params, pattern_router, exact_model_names) if listing.listed_id not in exact_model_names ) @@ -442,21 +467,38 @@ def _is_team_only_route(route: str, llm_router: Router, team_id: str | None) -> return team_router is not None and team_router.registers(route) and not llm_router.pattern_router.registers(route) -def _ids_listed_by_route(route: str, llm_router: Router | None, team_id: str | None) -> tuple[str, ...]: +def _proxy_wide_route_sends_to_another_provider(listing: _WildcardListing, pattern_router: PatternMatchRouter) -> bool: + provider: Final = inferred_provider(listing.routed_model) + return ( + provider is not None + and pattern_router.serves(listing.listed_id) + and not pattern_router.routes_to_provider(listing.listed_id, provider) + ) + + +def _ids_listed_by_route( + route: str, llm_router: Router | None, team_id: str | None, exact_model_names: frozenset[str] +) -> tuple[str, ...]: model_list: Final = llm_router.get_model_list(model_name=route, team_id=team_id) if llm_router else None pattern_router: Final = llm_router.pattern_router if llm_router else None listings: Final = ( chain.from_iterable( - _listings_served_by_route(route, LiteLLM_Params(**router_model["litellm_params"]), pattern_router) + _listings_served_by_route( + route, LiteLLM_Params(**router_model["litellm_params"]), pattern_router, exact_model_names + ) for router_model in model_list ) if model_list - else _listings_served_by_route(route, None, pattern_router) + else _listings_served_by_route(route, None, pattern_router, exact_model_names) ) - listed_ids: Final = (listing.listed_id for listing in listings) if llm_router is None or not _is_team_only_route(route, llm_router, team_id): - return tuple(listed_ids) - return tuple(listed_id for listed_id in listed_ids if not llm_router.pattern_router.serves(listed_id)) + return tuple(listing.listed_id for listing in listings) + return tuple( + listing.listed_id + for listing in listings + if listing.listed_id in exact_model_names + or not _proxy_wide_route_sends_to_another_provider(listing, llm_router.pattern_router) + ) def _routes_can_list_same_ids(route: str, other_route: str) -> bool: @@ -466,7 +508,7 @@ def _routes_can_list_same_ids(route: str, other_route: str) -> bool: def _ids_other_routes_serve_under( - wildcards: tuple[str, ...], llm_router: Router | None, team_id: str | None + wildcards: tuple[str, ...], llm_router: Router | None, team_id: str | None, exact_model_names: frozenset[str] ) -> tuple[str, ...]: if llm_router is None or not wildcards: return () @@ -481,7 +523,7 @@ def _ids_other_routes_serve_under( route for route in other_routes if any(_routes_can_list_same_ids(route, wildcard) for wildcard in wildcards) ) other_ids: Final = chain.from_iterable( - _ids_listed_by_route(route, llm_router, team_id) for route in overlapping_routes + _ids_listed_by_route(route, llm_router, team_id, exact_model_names) for route in overlapping_routes ) return tuple( listed_id @@ -499,11 +541,20 @@ def _get_wildcard_models( wildcard_routes: Final = tuple(model for model in unique_models if _check_wildcard_routing(model=model)) for model in wildcard_routes: unique_models.remove(model) + exact_model_names: Final = ( + frozenset(name for name in llm_router.get_model_names(team_id=team_id) if not _check_wildcard_routing(name)) + if llm_router + else frozenset[str]() + ) listed: Final = chain.from_iterable( - (*((model,) if return_wildcard_routes else ()), *_ids_listed_by_route(model, llm_router, team_id)) + ( + *((model,) if return_wildcard_routes else ()), + *_ids_listed_by_route(model, llm_router, team_id, exact_model_names), + ) for model in wildcard_routes ) - return list(dict.fromkeys(chain(listed, _ids_other_routes_serve_under(wildcard_routes, llm_router, team_id)))) + other_ids: Final = _ids_other_routes_serve_under(wildcard_routes, llm_router, team_id, exact_model_names) + return list(dict.fromkeys(chain(listed, other_ids))) def get_all_fallbacks( diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 4e474d3573e2..e43e15552c6c 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -486,6 +486,36 @@ def test_wildcard_custom_prefix_keeps_org_segment_for_non_provider_first_segment assert result == ["my_hf/meta-llama/Llama-3-8B"] +def test_namespaced_route_over_a_bare_star_lists_ids_that_route_to_the_inferred_provider( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.auth import model_checks + from litellm.proxy.auth.model_checks import ( + expand_wildcard_deployments_for_model_info, + get_known_models_from_wildcard, + ) + from litellm.types.router import LiteLLM_Params + + catalog: Final = ["gpt-4o-mini", "gemini/gemini-2.5-flash", "together_ai/meta-llama/Llama-3-8B"] + monkeypatch.setattr( + model_checks, + "get_provider_models", + lambda provider, litellm_params=None: catalog if provider == "*" else None, + ) + deployment: Final = {"model_name": "bp/*", "litellm_params": {"model": "*"}} + + assert get_known_models_from_wildcard(wildcard_model="bp/*", litellm_params=LiteLLM_Params(model="*")) == [ + "bp/gpt-4o-mini", + "bp/gemini-2.5-flash", + "bp/together_ai/meta-llama/Llama-3-8B", + ] + assert get_known_models_from_wildcard(wildcard_model="*", litellm_params=LiteLLM_Params(model="*")) == [] + rows: Final = expand_wildcard_deployments_for_model_info([deployment]) + assert [row["litellm_params"]["model"] for row in rows] == [ + row["model_name"].removeprefix("bp/") for row in rows + ] + + def test_cloudflare_partial_wildcard_matches_provider_stripped_cost_map_ids( monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None ) -> None: @@ -1257,8 +1287,9 @@ def test_key_scoped_to_a_wide_route_lists_ids_a_narrower_route_sends_to_the_same assert sorted(listed) == ["openai/gpt-4.1-mini", "openai/gpt-4.1-nano", "openai/gpt-4o"] -def test_key_granted_a_pattern_that_is_not_a_route_skips_ids_a_narrower_route_sends_to_another_model( - monkeypatch: pytest.MonkeyPatch, +@pytest.mark.parametrize("key_models", [["openai/*"], ["openai/gpt-4*"]], ids=("route", "pattern_without_a_route")) +def test_key_on_a_wildcard_skips_ids_a_narrower_route_sends_to_another_model_but_keeps_exact_deployments( + monkeypatch: pytest.MonkeyPatch, key_models: list[str] ) -> None: import litellm from litellm import Router @@ -1269,21 +1300,22 @@ def test_key_granted_a_pattern_that_is_not_a_route_skips_ids_a_narrower_route_se model_list=[ {"model_name": "openai/*", "litellm_params": {"model": "openai/*"}}, {"model_name": "openai/gpt-4o*", "litellm_params": {"model": "anthropic/claude-sonnet-4*"}}, + {"model_name": "openai/gpt-4o-mini", "litellm_params": {"model": "openai/gpt-4o-mini"}}, ] ) monkeypatch.setitem(litellm.models_by_provider, "openai", ["gpt-4o", "gpt-4o-mini", "gpt-4.1"]) monkeypatch.setitem(litellm.models_by_provider, "anthropic", ["claude-sonnet-4-5"]) listed: Final = get_complete_model_list( - key_models=["openai/gpt-4*"], + key_models=key_models, team_models=[], - proxy_model_list=["openai/*", "openai/gpt-4o*"], + proxy_model_list=["openai/*", "openai/gpt-4o*", "openai/gpt-4o-mini"], user_model=None, infer_model_from_keys=False, llm_router=router, ) - assert sorted(listed) == ["openai/gpt-4.1", "openai/gpt-4o-5"] + assert sorted(listed) == ["openai/gpt-4.1", "openai/gpt-4o-5", "openai/gpt-4o-mini"] @pytest.mark.parametrize( @@ -1393,6 +1425,41 @@ def listed(llm_router: Router) -> list[str]: assert "wide/claude-haiku-4-5" in listed(unshadowed_router) assert not [model for model in listed(router) if model.startswith("wide/claude-")] assert "wide/gpt-4o" in listed(router) + exact: Final = {"model_name": "wide/claude-haiku-4-5", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}} + router_with_exact: Final = Router( + model_list=[{"model_name": "wide/*", "litellm_params": {"model": "openai/*"}}, exact, team_route] + ) + assert [model for model in listed(router_with_exact) if model.startswith("wide/claude-")] == ["wide/claude-haiku-4-5"] + + +def test_team_key_lists_its_route_ids_a_proxy_wide_catch_all_sends_to_the_same_model( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm import Router + from litellm.proxy.auth.model_checks import get_complete_model_list + + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + catch_all: Final = {"model_name": "*", "litellm_params": {"model": "*", "custom_llm_provider": "openai"}} + team_route: Final = { + "model_name": "team-a-openai", + "litellm_params": {"model": "openai/*"}, + "model_info": {"team_id": "team-a", "team_public_model_name": "openai/*"}, + } + router: Final = Router(model_list=[catch_all, team_route]) + monkeypatch.setitem(litellm.models_by_provider, "openai", ["gpt-4o-mini", "gpt-4.1-mini"]) + + listed: Final = get_complete_model_list( + key_models=["openai/*"], + team_models=[], + proxy_model_list=["*"], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + team_id="team-a", + ) + + assert sorted(listed) == ["openai/gpt-4.1-mini", "openai/gpt-4o-mini"] def _wide_route_beside_a_narrower_route_to_another_provider(monkeypatch: pytest.MonkeyPatch):