Skip to content
Merged
Show file tree
Hide file tree
Changes from 10 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions litellm/proxy/_experimental/mcp_server/sampling_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -771,6 +771,7 @@ async def _check_model_access(model: str, user_api_key_auth: "UserAPIKeyAuth | N

try:
import litellm
from litellm.proxy._types import ModelAccessDeniedProxyException
from litellm.proxy.auth.auth_checks import (
_check_team_member_model_access,
can_key_call_model,
Expand Down Expand Up @@ -884,11 +885,14 @@ async def _check_model_access(model: str, user_api_key_auth: "UserAPIKeyAuth | N
)
return None
except Exception as access_err:
verbose_logger.warning(
"MCP sampling: model access denied for model=%s: %s",
model,
access_err,
)
if isinstance(access_err, ModelAccessDeniedProxyException):
verbose_logger.warning(
"MCP sampling: model access denied for model=%s: %s",
model,
access_err.sanitized_internal_message(),
)
return ErrorData(code=-1, message=access_err.message)
verbose_logger.warning("MCP sampling: model access denied for model=%s: %s", model, access_err)
return ErrorData(
code=-1,
message=(f"Model access denied: the API key is not authorized to use model '{model}'. {access_err}"),
Expand Down
16 changes: 16 additions & 0 deletions litellm/proxy/_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -4030,6 +4030,22 @@ def to_dict(self) -> dict:
return error_dict


class ModelAccessDeniedProxyException(ProxyException):
def __init__(
self,
message: str,
internal_message: str,
type: str,
param: str | None,
code: int | str | None,
) -> None:
super().__init__(message=message, type=type, param=param, code=code)
self.internal_message: Final = internal_message

def sanitized_internal_message(self) -> str:
return self.internal_message.replace("\r", "").replace("\n", "")


class CommonProxyErrors(str, enum.Enum):
db_not_connected_error = (
"DB not connected. This endpoint needs a database; set DATABASE_URL to a "
Expand Down
29 changes: 23 additions & 6 deletions litellm/proxy/auth/auth_checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
LiteLLM_UserTable,
LiteLLMRoutes,
LitellmUserRoles,
ModelAccessDeniedProxyException,
NewTeamRequest,
ProxyErrorTypes,
ProxyException,
Expand All @@ -71,6 +72,7 @@
budget_throttle_percentage,
should_throttle_budget_exceeded,
)
from litellm.proxy.auth.model_access_denied import model_access_denied_client_message
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import publish_auth_cache_invalidation
from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec
Expand Down Expand Up @@ -4170,8 +4172,13 @@ def _can_object_call_model(
):
return True

raise ProxyException(
message=f"{object_type} not allowed to access model. This {object_type} can only access models={models}. Tried to access {model}",
internal_message: Final = (
f"{object_type} not allowed to access model. This {object_type} can only access models={models}. "
f"Tried to access {model}"
)
raise ModelAccessDeniedProxyException(
message=model_access_denied_client_message(model=model),
Comment on lines +4175 to +4180

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Unconditional Error Contract Change

These model-access denials now always replace the established client-visible details with a fixed generic message. The repository requires backward-incompatible behavior changes to have a user-controlled flag, so an opt-in setting or another compatibility path must be provided before merging. The same unconditional change also affects the related user, team-member, and JWT denial paths.

Rule Used: What: avoid backwards-incompatible changes without user-controlled flags Why: This breaks current behaviour for users using existing functionality Example of BAD: this PR (#22164) introduced run_post_custom... (source)

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!

internal_message=internal_message,
type=ProxyErrorTypes.get_model_access_error_type_for_object(object_type=object_type),
param="model",
code=status.HTTP_403_FORBIDDEN,
Expand Down Expand Up @@ -4796,8 +4803,13 @@ async def can_user_call_model(
return True

if SpecialModelNames.no_default_models.value in user_object.models:
raise ProxyException(
message=f"User not allowed to access model. No default model access, only team models allowed. Tried to access {model}",
internal_message: Final = (
f"User not allowed to access model. No default model access, only team models allowed. "
f"Tried to access {model}"
)
raise ModelAccessDeniedProxyException(
message=model_access_denied_client_message(model=model),
internal_message=internal_message,
type=ProxyErrorTypes.key_model_access_denied,
param="model",
code=status.HTTP_403_FORBIDDEN,
Expand Down Expand Up @@ -5398,8 +5410,13 @@ async def _check_team_member_model_access(
team_id=team_object.team_id,
)
except ProxyException:
raise ProxyException(
message=f"Team member not allowed to access model. User={valid_token.user_id}, Team={team_object.team_id}, Model={model}. Allowed member models = {member_allowed_models}",
internal_message: Final = (
f"Team member not allowed to access model. User={valid_token.user_id}, Team={team_object.team_id}, "
f"Model={model}. Allowed member models = {member_allowed_models}"
)
raise ModelAccessDeniedProxyException(
message=model_access_denied_client_message(model=model),
internal_message=internal_message,
type=ProxyErrorTypes.team_model_access_denied,
param="model",
code=status.HTTP_403_FORBIDDEN,
Expand Down
10 changes: 10 additions & 0 deletions litellm/proxy/auth/auth_exception_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from litellm.litellm_core_utils.core_helpers import is_expected_client_error
from litellm.proxy._types import (
LitellmUserRoles,
ModelAccessDeniedProxyException,
ProxyErrorTypes,
ProxyException,
UserAPIKeyAuth,
Expand All @@ -25,6 +26,7 @@
mark_invalid_virtual_key_error,
normalize_request_route,
)
from litellm.proxy.auth.model_access_denied import ModelAccessDeniedHTTPException
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
from litellm.types.services import ServiceTypes

Expand All @@ -51,6 +53,14 @@ def _as_proxy_exception(e: Exception) -> ProxyException:
param=None,
code=getattr(e, "status_code", status.HTTP_429_TOO_MANY_REQUESTS),
)
if isinstance(e, ModelAccessDeniedHTTPException):
return ModelAccessDeniedProxyException(
message=str(e.detail),
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
internal_message=e.internal_message,
type=ProxyErrorTypes.auth_error,
param="None",
code=e.status_code,
)
if isinstance(e, HTTPException):
return ProxyException(
message=getattr(e, "detail", f"Authentication Error({e})"),
Expand Down
18 changes: 14 additions & 4 deletions litellm/proxy/auth/handle_jwt.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@
UserAPIKeyAuth,
)
from litellm.proxy.auth.auth_checks import can_team_access_model
from litellm.proxy.auth.model_access_denied import (
ModelAccessDeniedHTTPException,
model_access_denied_client_message,
)
from litellm.proxy.auth.resolvers.grants import GrantResolver, UserLookup, canonical_user_id
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.auth.team_grants import team_model_aliases
Expand Down Expand Up @@ -1337,9 +1341,13 @@ def can_rbac_role_call_model(
return True

if model not in role_based_models:
raise HTTPException(
internal_message: Final = (
f"Role={rbac_role} not allowed to call model={model}. Allowed models={role_based_models}"
)
raise ModelAccessDeniedHTTPException(
internal_message=internal_message,
status_code=403,
detail=f"Role={rbac_role} not allowed to call model={model}. Allowed models={role_based_models}",
detail=model_access_denied_client_message(model=model),
)

return True
Expand Down Expand Up @@ -1368,9 +1376,11 @@ def check_scope_based_access(
return

if requested_model not in allowed_models:
raise HTTPException(
internal_message: Final = f"model={requested_model} not allowed. Allowed_models={allowed_models}"
raise ModelAccessDeniedHTTPException(
internal_message=internal_message,
status_code=403,
detail={"error": f"model={requested_model} not allowed. Allowed_models={allowed_models}"},
detail={"error": model_access_denied_client_message(model=requested_model)},
)
return

Expand Down
18 changes: 18 additions & 0 deletions litellm/proxy/auth/model_access_denied.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
from typing import Final

from fastapi import HTTPException

MODEL_ACCESS_DENIED_CLIENT_MESSAGE: Final = (
"The requested model '{model}' is not available for this API key, or the model name is invalid. "
"Check the models available to you and try again."
)


def model_access_denied_client_message(model: str | list[str]) -> str:
return MODEL_ACCESS_DENIED_CLIENT_MESSAGE.format(model=model)


class ModelAccessDeniedHTTPException(HTTPException):
def __init__(self, internal_message: str, status_code: int, detail: str | dict[str, str]) -> None:
super().__init__(status_code=status_code, detail=detail)
self.internal_message: Final = internal_message
9 changes: 9 additions & 0 deletions litellm/proxy/proxy_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@
LiteLLM_TeamTableCachedObj,
LiteLLM_UserTable,
LitellmUserRoles,
ModelAccessDeniedProxyException,
PassThroughGenericEndpoint,
ProxyErrorTypes,
ProxyException,
Expand Down Expand Up @@ -1668,6 +1669,7 @@ class UserAPIKeyCacheTTLEnum(enum.Enum):
@app.exception_handler(ProxyException)
async def openai_exception_handler(request: Request, exc: ProxyException):
# NOTE: DO NOT MODIFY THIS, its crucial to map to Openai exceptions
_log_model_access_denial(exc)
headers: Final = exc.headers
error_dict: Final = exc.to_dict()
status_code: Final = int(exc.code) if exc.code else status.HTTP_500_INTERNAL_SERVER_ERROR
Expand All @@ -1679,6 +1681,12 @@ async def openai_exception_handler(request: Request, exc: ProxyException):
)


def _log_model_access_denial(exc: ProxyException) -> None:
if not isinstance(exc, ModelAccessDeniedProxyException):
return
verbose_proxy_logger.warning(exc.sanitized_internal_message())


def _close_dangling_otel_server_span(request: Request, status_code: int, exc: Exception | None = None) -> None:
parent_otel_span: Final[_Span | None] = getattr(request.state, "parent_otel_span", None)
if parent_otel_span is None:
Expand Down Expand Up @@ -11978,6 +11986,7 @@ async def realtime_websocket_endpoint(
llm_router=llm_router,
)
except ProxyException as e:
_log_model_access_denial(e)
await _reject_realtime_session(websocket, user_api_key_dict, code=1008, reason=e.message[:120])
return
await websocket.accept(**accept_kwargs)
Expand Down
8 changes: 3 additions & 5 deletions tests/otel_tests/test_e2e_model_access.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ async def test_model_access_patterns(key_models, test_model, expect_success):
assert _error_body["type"] == "key_model_access_denied"
assert _error_body["param"] == "model"
assert _error_body["code"] == "403"
assert "key not allowed to access model" in _error_body["message"]
assert "is not available for this API key" in _error_body["message"]


@pytest.mark.asyncio
Expand Down Expand Up @@ -299,7 +299,5 @@ def _validate_model_access_exception(
assert _error_body["type"] == expected_type
assert _error_body["param"] == "model"
assert _error_body["code"] == "403"
if expected_type == "key_model_access_denied":
assert "key not allowed to access model" in _error_body["message"]
elif expected_type == "team_model_access_denied":
assert "eam not allowed to access model" in _error_body["message"]
assert "is not available for this API key" in _error_body["message"]
assert "not allowed to access model" not in _error_body["message"]
4 changes: 2 additions & 2 deletions tests/proxy_unit_tests/test_auth_checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ async def test_can_key_call_model(model, expect_to_work):
if expect_to_work:
await can_key_call_model(**args)
else:
with pytest.raises(Exception, match='key not allowed to access model\\. This key can only access') as e:
with pytest.raises(Exception, match='is not available for this API key') as e:
await can_key_call_model(**args)

print(e)
Expand Down Expand Up @@ -943,7 +943,7 @@ async def test_can_key_call_model_with_aliases(model, alias_map, expect_to_work)
llm_router=router,
)
else:
with pytest.raises(Exception, match='key not allowed to access model\\. This key can only access') as e:
with pytest.raises(Exception, match='is not available for this API key') as e:
await can_key_call_model(
model=model,
llm_model_list=llm_model_list,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,22 @@ async def test_should_deny_expensive_model_requested_by_malicious_server(self):
assert result.code == -1
assert "claude-3-opus-20240229" in result.message

@pytest.mark.asyncio
async def test_should_log_internal_denial_reason_and_hide_allowlist_from_client(self, caplog):
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.model_access_denied import model_access_denied_client_message

auth = UserAPIKeyAuth(api_key="sk-test-key", models=["gpt-3.5-turbo"])

with caplog.at_level("WARNING", logger="LiteLLM"):
result = await _check_model_access("gpt-4o\r\nforged", user_api_key_auth=auth)

assert result is not None
assert result.message == model_access_denied_client_message(model="gpt-4o\r\nforged")
denial_records = [r for r in caplog.records if "gpt-3.5-turbo" in r.getMessage()]
assert len(denial_records) == 1
assert "Tried to access gpt-4oforged" in denial_records[0].getMessage()

@pytest.mark.asyncio
async def test_should_deny_empty_oauth_passthrough_placeholder(self):
"""Regression: process_mcp_request() returns an empty UserAPIKeyAuth()
Expand Down
Loading
Loading