Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions litellm/model_prices_and_context_window_backup.json
Original file line number Diff line number Diff line change
Expand Up @@ -43574,6 +43574,7 @@
"supports_vision": true
},
"bedrock_mantle/xai.grok-4.3": {
"use_openai_responses_path": true,
"input_cost_per_token": 1.25e-06,
"output_cost_per_token": 2.5e-06,
"cache_read_input_token_cost": 2e-07,
Expand Down
9 changes: 9 additions & 0 deletions litellm/proxy/route_llm_request.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import litellm
from litellm.proxy._types import UserAPIKeyAuth
from litellm.router_utils.common_utils import _is_proxy_admin_request

# Router-internal mock_testing_* flag names — kept in sync with
# ``litellm.types.router.MockRouterTestingParams`` by the test
Expand Down Expand Up @@ -363,6 +364,7 @@ async def route_request(

team_id = get_team_id_from_data(data)
router_model_names = llm_router.model_names if llm_router is not None else []
is_proxy_admin_without_team = team_id is None and _is_proxy_admin_request(data)

# Preprocess Google GenAI generate content requests
if route_type in ["agenerate_content", "agenerate_content_stream"]:
Expand Down Expand Up @@ -517,6 +519,13 @@ async def route_request(
data["model"] = team_model_name
return getattr(llm_router, f"{route_type}")(**data)

elif (
is_proxy_admin_without_team
and data["model"] not in router_model_names
and data["model"] in llm_router.team_public_model_names
):
return getattr(llm_router, f"{route_type}")(**data)

elif data["model"] in router_model_names or llm_router.has_model_id(data["model"]):
return getattr(llm_router, f"{route_type}")(**data)

Expand Down
44 changes: 42 additions & 2 deletions litellm/router.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
AsyncGenerator,
Callable,
Dict,
FrozenSet,
Generator,
List,
Literal,
Expand Down Expand Up @@ -108,6 +109,7 @@
is_clientside_credential,
)
from litellm.router_utils.common_utils import (
_is_proxy_admin_request,
filter_team_based_models,
filter_web_search_deployments,
)
Expand Down Expand Up @@ -490,6 +492,7 @@ def __init__(
self.model_name_to_deployment_indices: Dict[str, List[int]] = {}
# Maps (team_id, team_public_model_name) -> list of indices in model_list
self.team_model_to_deployment_indices: Dict[Tuple[str, str], List[int]] = {}
self.team_public_model_names: FrozenSet[str] = frozenset()

# Initialize cache attributes that ``_invalidate_model_group_info_cache``
# touches *before* the first ``set_model_list`` below (which calls
Expand Down Expand Up @@ -7777,6 +7780,7 @@ def set_model_list(self, model_list: list):
self.model_id_to_deployment_index_map = {} # Reset the index
self.model_name_to_deployment_indices = {} # Reset the model_name index
self.team_model_to_deployment_indices = {} # Reset the team_model index
self.team_public_model_names = frozenset()
# Reset per-strategy router registries so hot-reload doesn't leave
# stale routers pointing at the old model_list.
self.quality_routers = {}
Expand Down Expand Up @@ -8128,6 +8132,9 @@ def _update_deployment_indices_after_removal(self, model_id: str, removal_idx: i
self.team_model_to_deployment_indices[key] = updated_indices
else:
del self.team_model_to_deployment_indices[key]
self.team_public_model_names = frozenset(
public_model_name for _, public_model_name in self.team_model_to_deployment_indices
)

def _update_team_model_index(self, model: dict, idx: int) -> None:
"""
Expand All @@ -8141,6 +8148,7 @@ def _update_team_model_index(self, model: dict, idx: int) -> None:
team_public_model_name = (model.get("model_info") or {}).get("team_public_model_name")
if team_id and team_public_model_name:
key = (team_id, team_public_model_name)
self.team_public_model_names = self.team_public_model_names | frozenset({team_public_model_name})
if key not in self.team_model_to_deployment_indices:
self.team_model_to_deployment_indices[key] = []
if idx not in self.team_model_to_deployment_indices[key]:
Expand Down Expand Up @@ -9095,6 +9103,7 @@ def _build_model_name_index(self, model_list: list) -> None:
"""
self.model_name_to_deployment_indices.clear()
self.team_model_to_deployment_indices.clear()
self.team_public_model_names = frozenset()

for idx, model in enumerate(model_list):
model_name = model.get("model_name")
Expand Down Expand Up @@ -10003,7 +10012,10 @@ def _get_deployment_by_litellm_model(self, model: str) -> List:
return [m for m in self.model_list if m["litellm_params"]["model"] == model]

def _try_early_resolve_deployments_for_model_not_in_names(
self, model: str, request_team_id: Optional[str]
self,
model: str,
request_team_id: Optional[str],
include_team_models: bool = False,
) -> Optional[Tuple[str, Union[List, Dict]]]:
"""
When ``model`` is not in ``self.model_names``, try team routes, pattern routes,
Expand All @@ -10018,6 +10030,30 @@ def _try_early_resolve_deployments_for_model_not_in_names(
team_deployments = self._get_all_deployments(model_name=model, team_id=request_team_id)
if team_deployments:
return model, team_deployments
elif include_team_models:
team_deployments = [
self.model_list[index]
for (_, public_model_name), indices in self.team_model_to_deployment_indices.items()
if public_model_name == model
for index in indices
]
team_ids = {
team_id
for deployment in team_deployments
for team_id in [(deployment.get("model_info") or {}).get("team_id")]
if team_id is not None
}
if len(team_ids) > 1:
raise litellm.BadRequestError(
message=(
f"Model name '{model}' matches deployments from multiple teams. "
"Specify the deployment ID directly to disambiguate."
),
model=model,
llm_provider="",
)
Comment on lines +10047 to +10054

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Proxy-specific error message inside router.py

The added error message explicitly refers to "teams" and proxy-level routing guidance ("Specify the deployment ID directly to disambiguate"), which are proxy concepts. Per the project rule, proxy-facing error messages should not live in router.py because it is also an SDK component. While the code is only reachable today when include_team_models=True (a flag only the proxy sets), placing the messaging here still embeds proxy semantics in the SDK layer and would affect SDK users who independently reach this path in the future.

Rule Used: Don’t add stuff to router.py that is error message... (source)

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!

if team_deployments:
return model, team_deployments

pattern_deployments = self.pattern_router.get_deployments_by_pattern(
model=model,
Expand Down Expand Up @@ -10082,7 +10118,11 @@ def _common_checks_available_deployment(
if _model_from_alias is not None:
model = _model_from_alias

early = self._try_early_resolve_deployments_for_model_not_in_names(model=model, request_team_id=request_team_id)
early = self._try_early_resolve_deployments_for_model_not_in_names(
model=model,
request_team_id=request_team_id,
include_team_models=_is_proxy_admin_request(request_kwargs),
)
if early is not None:
return early

Expand Down
47 changes: 47 additions & 0 deletions litellm/router_utils/common_utils.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,27 @@
import hashlib
import json
from collections.abc import Mapping
from typing import TYPE_CHECKING, Dict, List, Optional, Union

if TYPE_CHECKING:
from litellm.types.llms.openai import OpenAIFileObject

from litellm.exceptions import BadRequestError
from litellm.types.router import CredentialLiteLLMParams
from litellm._logging import verbose_logger


def _is_proxy_admin_request(request_kwargs: Optional[Mapping[str, object]]) -> bool:
if request_kwargs is None:
return False
metadata_value = request_kwargs.get("metadata")
litellm_metadata_value = request_kwargs.get("litellm_metadata")
metadata = metadata_value if isinstance(metadata_value, Mapping) else {}
litellm_metadata = litellm_metadata_value if isinstance(litellm_metadata_value, Mapping) else {}
user_api_key_auth = metadata.get("user_api_key_auth") or litellm_metadata.get("user_api_key_auth")
return getattr(user_api_key_auth, "user_role", None) == "proxy_admin"


def get_litellm_params_sensitive_credential_hash(litellm_params: dict) -> str:
"""
Hash of the credential params, used for mapping the file id to the right model
Expand Down Expand Up @@ -59,6 +72,40 @@ def filter_team_based_models(
metadata = request_kwargs.get("metadata") or {}
litellm_metadata = request_kwargs.get("litellm_metadata") or {}
request_team_id = metadata.get("user_api_key_team_id") or litellm_metadata.get("user_api_key_team_id")
if request_team_id is None and _is_proxy_admin_request(request_kwargs) and isinstance(healthy_deployments, list):
requested_model = (
request_kwargs.get("model") or metadata.get("model_group") or litellm_metadata.get("model_group")
)
candidate_deployments = tuple(
(deployment.get("model_name"), deployment.get("model_info") or {}) for deployment in healthy_deployments
)
team_ids = frozenset(
team_id
for _, model_info in candidate_deployments
for team_id in [model_info.get("team_id")]
if team_id is not None
)
matches_requested_model = (
isinstance(requested_model, str)
and bool(candidate_deployments)
and all(
model_info.get("team_id") is not None
and (model_name == requested_model or model_info.get("team_public_model_name") == requested_model)
for model_name, model_info in candidate_deployments
)
)
if matches_requested_model and len(team_ids) > 1:
raise BadRequestError(
message=(
f"Model name '{requested_model}' matches deployments from multiple teams. "
"Specify the deployment ID directly to disambiguate."
),
model=requested_model,
llm_provider="",
)
if matches_requested_model:
return healthy_deployments

ids_to_remove = set()
if isinstance(healthy_deployments, dict):
return healthy_deployments
Expand Down
1 change: 1 addition & 0 deletions model_prices_and_context_window.json
Original file line number Diff line number Diff line change
Expand Up @@ -43806,6 +43806,7 @@
"supports_vision": true
},
"bedrock_mantle/xai.grok-4.3": {
"use_openai_responses_path": true,
"input_cost_per_token": 1.25e-06,
"output_cost_per_token": 2.5e-06,
"cache_read_input_token_cost": 2e-07,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -460,7 +460,12 @@ def test_registry_returns_native_config_for_xai_grok(self, local_cost_map):
model="xai.grok-4.3",
)
assert isinstance(cfg, BedrockMantleResponsesAPIConfig)
assert cfg.use_openai_path is False
# grok-4.3 is a third-party frontier model on Bedrock Mantle, served on the
# /openai/v1 base (like gpt-5.x / gemma-4), not the standard /v1 path used by
# open-weights models such as gpt-oss. The standard /v1 base returns
# "Berm is not enabled for this account", so the price-map entry carries
# use_openai_responses_path=true.
assert cfg.use_openai_path is True

def test_unmapped_frontier_model_falls_through_to_none(self, restore_model_cost):
# The gate is data-driven, not name-based: an unseen model not yet in the
Expand Down
Loading
Loading