Skip to content
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions litellm/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,7 @@ def _dev_env_hot_reload_enabled() -> bool:
# major release; opt in early with `litellm.expose_router_debug_in_errors
# = False`.
expose_router_debug_in_errors: bool = True
model_access_denied_message: str | None = None
filter_invalid_headers: Optional[bool] = False
add_user_information_to_llm_headers: Optional[bool] = (
None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers
Expand Down
3 changes: 3 additions & 0 deletions litellm/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@
REDACTED_BY_LITELLM: Final = "redacted-by-litellm"
# in-memory stand-in handed to provider converters for redacted arguments; never stored
REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}"
MODEL_ACCESS_DENIED_MESSAGE_MODEL_PLACEHOLDER: Final = "{model}"

MAX_STRING_LENGTH_STDOUT_LOG: Final = get_env_int("MAX_STRING_LENGTH_STDOUT_LOG", 4096)

Expand Down Expand Up @@ -1800,6 +1801,8 @@
"max_ui_session_budget",
"budget_rollover",
"mcp_tool_search",
"model_access_denied_message",
"expose_router_debug_in_errors",
]
SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"]
DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60))
Expand Down
5 changes: 4 additions & 1 deletion litellm/proxy/_experimental/mcp_server/sampling_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -771,6 +771,7 @@ async def _check_model_access(model: str, user_api_key_auth: "UserAPIKeyAuth | N

try:
import litellm
from litellm.proxy._types import ModelAccessDeniedProxyException
from litellm.proxy.auth.auth_checks import (
_check_team_member_model_access,
can_key_call_model,
Expand Down Expand Up @@ -887,7 +888,9 @@ async def _check_model_access(model: str, user_api_key_auth: "UserAPIKeyAuth | N
verbose_logger.warning(
"MCP sampling: model access denied for model=%s: %s",
model,
access_err,
access_err.sanitized_internal_message()
if isinstance(access_err, ModelAccessDeniedProxyException)
else access_err,
)
return ErrorData(
code=-1,
Expand Down
16 changes: 16 additions & 0 deletions litellm/proxy/_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -4030,6 +4030,22 @@ def to_dict(self) -> dict:
return error_dict


class ModelAccessDeniedProxyException(ProxyException):
def __init__(
self,
message: str,
internal_message: str,
type: str,
param: str | None,
code: int | str | None,
) -> None:
super().__init__(message=message, type=type, param=param, code=code)
self.internal_message: Final = internal_message

def sanitized_internal_message(self) -> str:
return self.internal_message.replace("\r", "").replace("\n", "")


class CommonProxyErrors(str, enum.Enum):
db_not_connected_error = (
"DB not connected. This endpoint needs a database; set DATABASE_URL to a "
Expand Down
29 changes: 23 additions & 6 deletions litellm/proxy/auth/auth_checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
LiteLLM_UserTable,
LiteLLMRoutes,
LitellmUserRoles,
ModelAccessDeniedProxyException,
NewTeamRequest,
ProxyErrorTypes,
ProxyException,
Expand All @@ -71,6 +72,7 @@
budget_throttle_percentage,
should_throttle_budget_exceeded,
)
from litellm.proxy.auth.model_access_denied import client_facing_model_access_denied_message
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import publish_auth_cache_invalidation
from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec
Expand Down Expand Up @@ -4170,8 +4172,13 @@ def _can_object_call_model(
):
return True

raise ProxyException(
message=f"{object_type} not allowed to access model. This {object_type} can only access models={models}. Tried to access {model}",
internal_message: Final = (
f"{object_type} not allowed to access model. This {object_type} can only access models={models}. "
f"Tried to access {model}"
)
raise ModelAccessDeniedProxyException(
message=client_facing_model_access_denied_message(internal_message=internal_message, model=model),
internal_message=internal_message,
type=ProxyErrorTypes.get_model_access_error_type_for_object(object_type=object_type),
param="model",
code=status.HTTP_403_FORBIDDEN,
Expand Down Expand Up @@ -4796,8 +4803,13 @@ async def can_user_call_model(
return True

if SpecialModelNames.no_default_models.value in user_object.models:
raise ProxyException(
message=f"User not allowed to access model. No default model access, only team models allowed. Tried to access {model}",
internal_message: Final = (
f"User not allowed to access model. No default model access, only team models allowed. "
f"Tried to access {model}"
)
raise ModelAccessDeniedProxyException(
message=client_facing_model_access_denied_message(internal_message=internal_message, model=model),
internal_message=internal_message,
type=ProxyErrorTypes.key_model_access_denied,
param="model",
code=status.HTTP_403_FORBIDDEN,
Expand Down Expand Up @@ -5398,8 +5410,13 @@ async def _check_team_member_model_access(
team_id=team_object.team_id,
)
except ProxyException:
raise ProxyException(
message=f"Team member not allowed to access model. User={valid_token.user_id}, Team={team_object.team_id}, Model={model}. Allowed member models = {member_allowed_models}",
internal_message: Final = (
f"Team member not allowed to access model. User={valid_token.user_id}, Team={team_object.team_id}, "
f"Model={model}. Allowed member models = {member_allowed_models}"
)
raise ModelAccessDeniedProxyException(
message=client_facing_model_access_denied_message(internal_message=internal_message, model=model),
internal_message=internal_message,
type=ProxyErrorTypes.team_model_access_denied,
param="model",
code=status.HTTP_403_FORBIDDEN,
Expand Down
10 changes: 10 additions & 0 deletions litellm/proxy/auth/auth_exception_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from litellm.litellm_core_utils.core_helpers import is_expected_client_error
from litellm.proxy._types import (
LitellmUserRoles,
ModelAccessDeniedProxyException,
ProxyErrorTypes,
ProxyException,
UserAPIKeyAuth,
Expand All @@ -25,6 +26,7 @@
mark_invalid_virtual_key_error,
normalize_request_route,
)
from litellm.proxy.auth.model_access_denied import ModelAccessDeniedHTTPException
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
from litellm.types.services import ServiceTypes

Expand All @@ -51,6 +53,14 @@ def _as_proxy_exception(e: Exception) -> ProxyException:
param=None,
code=getattr(e, "status_code", status.HTTP_429_TOO_MANY_REQUESTS),
)
if isinstance(e, ModelAccessDeniedHTTPException):
return ModelAccessDeniedProxyException(
message=str(e.detail),
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
internal_message=e.internal_message,
type=ProxyErrorTypes.auth_error,
param="None",
code=e.status_code,
)
if isinstance(e, HTTPException):
return ProxyException(
message=getattr(e, "detail", f"Authentication Error({e})"),
Expand Down
22 changes: 18 additions & 4 deletions litellm/proxy/auth/handle_jwt.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@
UserAPIKeyAuth,
)
from litellm.proxy.auth.auth_checks import can_team_access_model
from litellm.proxy.auth.model_access_denied import (
ModelAccessDeniedHTTPException,
client_facing_model_access_denied_message,
)
from litellm.proxy.auth.resolvers.grants import GrantResolver, UserLookup, canonical_user_id
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.auth.team_grants import team_model_aliases
Expand Down Expand Up @@ -1337,9 +1341,13 @@ def can_rbac_role_call_model(
return True

if model not in role_based_models:
raise HTTPException(
internal_message: Final = (
f"Role={rbac_role} not allowed to call model={model}. Allowed models={role_based_models}"
)
raise ModelAccessDeniedHTTPException(
internal_message=internal_message,
status_code=403,
detail=f"Role={rbac_role} not allowed to call model={model}. Allowed models={role_based_models}",
detail=client_facing_model_access_denied_message(internal_message=internal_message, model=model),
)

return True
Expand Down Expand Up @@ -1368,9 +1376,15 @@ def check_scope_based_access(
return

if requested_model not in allowed_models:
raise HTTPException(
internal_message: Final = f"model={requested_model} not allowed. Allowed_models={allowed_models}"
raise ModelAccessDeniedHTTPException(
internal_message=internal_message,
status_code=403,
detail={"error": f"model={requested_model} not allowed. Allowed_models={allowed_models}"},
detail={
"error": client_facing_model_access_denied_message(
internal_message=internal_message, model=requested_model
)
},
)
return

Expand Down
19 changes: 19 additions & 0 deletions litellm/proxy/auth/model_access_denied.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
from typing import Final

from fastapi import HTTPException

import litellm
from litellm.constants import MODEL_ACCESS_DENIED_MESSAGE_MODEL_PLACEHOLDER


def client_facing_model_access_denied_message(internal_message: str, model: str | list[str]) -> str:
template: Final = litellm.model_access_denied_message
if not template:
return internal_message
return template.replace(MODEL_ACCESS_DENIED_MESSAGE_MODEL_PLACEHOLDER, str(model))


class ModelAccessDeniedHTTPException(HTTPException):
def __init__(self, internal_message: str, status_code: int, detail: str | dict[str, str]) -> None:
super().__init__(status_code=status_code, detail=detail)
self.internal_message: Final = internal_message
46 changes: 41 additions & 5 deletions litellm/proxy/proxy_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@
LiteLLM_TeamTableCachedObj,
LiteLLM_UserTable,
LitellmUserRoles,
ModelAccessDeniedProxyException,
PassThroughGenericEndpoint,
ProxyErrorTypes,
ProxyException,
Expand Down Expand Up @@ -1668,6 +1669,7 @@ class UserAPIKeyCacheTTLEnum(enum.Enum):
@app.exception_handler(ProxyException)
async def openai_exception_handler(request: Request, exc: ProxyException):
# NOTE: DO NOT MODIFY THIS, its crucial to map to Openai exceptions
_log_model_access_denial(exc)
headers: Final = exc.headers
error_dict: Final = exc.to_dict()
status_code: Final = int(exc.code) if exc.code else status.HTTP_500_INTERNAL_SERVER_ERROR
Expand All @@ -1679,6 +1681,12 @@ async def openai_exception_handler(request: Request, exc: ProxyException):
)


def _log_model_access_denial(exc: ProxyException) -> None:
if not litellm.model_access_denied_message or not isinstance(exc, ModelAccessDeniedProxyException):
return
verbose_proxy_logger.warning(exc.sanitized_internal_message())


def _close_dangling_otel_server_span(request: Request, status_code: int, exc: Exception | None = None) -> None:
parent_otel_span: Final[_Span | None] = getattr(request.state, "parent_otel_span", None)
if parent_otel_span is None:
Expand Down Expand Up @@ -11978,6 +11986,7 @@ async def realtime_websocket_endpoint(
llm_router=llm_router,
)
except ProxyException as e:
_log_model_access_denial(e)
await _reject_realtime_session(websocket, user_api_key_dict, code=1008, reason=e.message[:120])
return
await websocket.accept(**accept_kwargs)
Expand Down Expand Up @@ -17446,11 +17455,13 @@ async def get_config_general_settings(


class GeneralSettingsUILiteLLMFieldSpec(TypedDict):
type: Literal["Float", "Dollar", "Boolean", "Select"]
description: str
options: NotRequired[tuple[str, ...]]
tab: NotRequired[str] # Admin UI sub-tab this field renders under; None groups it with the rest
default: NotRequired[float] # reset/clear restores this instead of None; fields whose None means fail-open set it
type: ReadOnly[Literal["Float", "Dollar", "Boolean", "Select", "String"]]
description: ReadOnly[str]
options: ReadOnly[NotRequired[tuple[str, ...]]]
tab: ReadOnly[NotRequired[str]] # Admin UI sub-tab this field renders under; None groups it with the rest
default: ReadOnly[
NotRequired[float | bool]
] # reset/clear restores this instead of None; fields whose None means fail-open set it


_GENERAL_SETTINGS_UI_LITELLM_FIELDS: Final[dict[str, GeneralSettingsUILiteLLMFieldSpec]] = {
Expand Down Expand Up @@ -17502,6 +17513,24 @@ class GeneralSettingsUILiteLLMFieldSpec(TypedDict):
"with this budget. Clearing restores the $1 default."
),
},
"model_access_denied_message": {
"type": "String",
"description": (
"Client-facing error message returned when a key, team, user, org or project is not allowed "
"to call the requested model. {model} is replaced with the requested model name. The full "
"denial reason (allowed models and access groups) is still written to the proxy logs. "
"Leave empty to return the detailed message to clients."
),
},
"expose_router_debug_in_errors": {
"type": "Boolean",
"default": True,
"description": (
"Append router debug details (model group, configured fallbacks, fallback errors, cooldown "
"info) to error messages returned to clients. Turn off to keep those details in the proxy "
"logs only."
),
},
}


Expand Down Expand Up @@ -17549,6 +17578,13 @@ def _validate_general_settings_ui_litellm_value(field_name: str, value: object)
detail={"error": f"{field_name} must be a positive dollar amount or empty"},
)
return float(value)
case "String":
if not isinstance(value, str):
raise HTTPException(
status_code=400,
detail={"error": f"{field_name} must be a string or empty"},
)
return value
case _:
assert_never(field_type)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,24 @@ async def test_should_deny_expensive_model_requested_by_malicious_server(self):
assert result.code == -1
assert "claude-3-opus-20240229" in result.message

@pytest.mark.asyncio
async def test_should_log_internal_denial_reason_when_client_message_is_configured(self, monkeypatch, caplog):
import litellm
from litellm.proxy._types import UserAPIKeyAuth

monkeypatch.setattr(litellm, "model_access_denied_message", "The model `{model}` is unavailable for this API key.")
auth = UserAPIKeyAuth(api_key="sk-test-key", models=["gpt-3.5-turbo"])

with caplog.at_level("WARNING", logger="LiteLLM"):
result = await _check_model_access("gpt-4o\r\nforged", user_api_key_auth=auth)

assert result is not None
assert "gpt-4o\r\nforged" in result.message
assert "gpt-3.5-turbo" not in result.message
denial_records = [r for r in caplog.records if "gpt-3.5-turbo" in r.getMessage()]
assert len(denial_records) == 1
assert "Tried to access gpt-4oforged" in denial_records[0].getMessage()

@pytest.mark.asyncio
async def test_should_deny_empty_oauth_passthrough_placeholder(self):
"""Regression: process_mcp_request() returns an empty UserAPIKeyAuth()
Expand Down
Loading
Loading