diff --git a/litellm/__init__.py b/litellm/__init__.py index 00f67ea0ff5f..e95b553c5d4f 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -453,6 +453,7 @@ def _dev_env_hot_reload_enabled() -> bool: # backwards compatibility — arbitrary client-supplied identifiers still # pass through unchanged. validate_end_user_id_in_db: bool = False +block_requests_for_models_without_pricing: bool = False disable_end_user_cost_tracking: Optional[bool] = None disable_end_user_cost_tracking_prometheus_only: Optional[bool] = None enable_end_user_cost_tracking_prometheus_only: Optional[bool] = None diff --git a/litellm/constants.py b/litellm/constants.py index a845b1a49ae4..f8a277f223e1 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1580,6 +1580,7 @@ "public_model_groups_links", "cost_discount_config", "cost_margin_config", + "block_requests_for_models_without_pricing", "budget_exceeded_throttle_percentage", # Every field editable from the Admin UI (proxy_server._GENERAL_SETTINGS_UI_LITELLM_FIELDS) # must be listed here so a DB write from one worker overrides the live litellm attribute on diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 05fc6e071769..525171459827 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3721,6 +3721,8 @@ class ProxyErrorTypes(str, enum.Enum): Project does not have access to the model """ + model_cost_map_missing = "model_cost_map_missing" + expired_key = "expired_key" """ Key has expired diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 8708f96339ff..7d3246aef0ac 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -464,6 +464,103 @@ def _is_cost_explicitly_configured(model: str, llm_router: "Router") -> bool: return False +_EMPTY_COST_ENTRY: Final[Mapping[str, object]] = MappingProxyType({}) + + +def _is_positive_cost(value: object) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) and value > 0 + + +def _entry_has_priced_metric(entry: Mapping[str, object]) -> bool: + if entry.get("tiered_pricing") is not None: + return True + for key, value in entry.items(): + if "cost_per" not in key: + continue + if _is_positive_cost(value): + return True + if isinstance(value, dict) and any(_is_positive_cost(nested) for nested in value.values()): + return True + return False + + +def _entry_declares_price(entry: Mapping[str, object]) -> bool: + return any("cost_per" in key or key == "tiered_pricing" for key in entry) + + +def _model_group_has_pricing(model: str, llm_router: "Router") -> bool: + """ + A model group counts as priced when a deployment overrides any *cost_per* field or + tiered_pricing in its litellm_params, even at zero, or when its resolved model info carries + tiered_pricing or a positive price on any billed metric (tokens, characters, seconds, pages, + images, queries, ...), so models billed by a non-token metric are not treated as unpriced. + """ + for deployment in llm_router.get_model_list(model_name=model) or (): + litellm_params = deployment.get("litellm_params") or _EMPTY_COST_ENTRY + if _entry_declares_price(litellm_params): + return True + + model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id") + if model_id is None: + continue + + model_info = llm_router.get_deployment_model_info( + model_id=model_id, model_name=litellm_params.get("model") or "" + ) + if model_info is not None and _entry_has_priced_metric(model_info): + return True + + return False + + +def _group_declares_explicit_cost(model: str, llm_router: "Router") -> bool: + """ + Alias-aware counterpart to ``_is_cost_explicitly_configured``, which resolves the model group + the same way ``_model_group_has_pricing`` does. A deployment that prices itself through its + ``model_info`` block lands in the cost map under its deployment id rather than in its + litellm_params, and reaching that entry through the router's own resolution keeps an alias + pointing at such a group from being read as unpriced. + """ + for deployment in llm_router.get_model_list(model_name=model) or (): + model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id") + if model_id is None: + continue + raw_entry = litellm.model_cost.get(model_id, _EMPTY_COST_ENTRY) + if "input_cost_per_token" in raw_entry or "output_cost_per_token" in raw_entry: + return True + return False + + +def model_has_no_cost_mapping(model: str | None, llm_router: Router | None) -> bool: + if not model or llm_router is None: + return False + + if llm_router.get_model_group_info(model_group=model) is None: + return False + + if _model_group_has_pricing(model=model, llm_router=llm_router): + return False + + return not _group_declares_explicit_cost(model=model, llm_router=llm_router) + + +def _unpriced_models_in_request(model: str | list[str] | None, llm_router: Router | None) -> tuple[str, ...]: + candidates: Final = (model,) if isinstance(model, str) else tuple(model or ()) + return tuple( + candidate for candidate in candidates if model_has_no_cost_mapping(model=candidate, llm_router=llm_router) + ) + + +def _unpriced_models_block_message(models: tuple[str, ...]) -> str: + names: Final = ", ".join(f"'{model}'" for model in models) + subject: Final = f"Model {names} has" if len(models) == 1 else f"Models {names} have" + return ( + f"{subject} no pricing in the cost map, so litellm cannot price the request. " + "Requests for unpriced models are blocked because 'block_requests_for_models_without_pricing' " + "is enabled. Add pricing (input_cost_per_token/output_cost_per_token) to allow the request." + ) + + async def _run_project_checks( project_object: LiteLLM_ProjectTableCachedObj | None, _model: str | list[str] | None, @@ -734,6 +831,19 @@ async def common_checks( and (route in MODEL_DISCOVERY_ROUTES or not RouteChecks.is_llm_api_route(route=route)) ) + unpriced_models: Final = ( + _unpriced_models_in_request(model=_model, llm_router=llm_router) + if litellm.block_requests_for_models_without_pricing and RouteChecks.is_llm_api_route(route=route) + else () + ) + if unpriced_models: + raise ProxyException( + message=_unpriced_models_block_message(unpriced_models), + type=ProxyErrorTypes.model_cost_map_missing, + param="model", + code=status.HTTP_403_FORBIDDEN, + ) + # 1. If team is blocked if team_object is not None and team_object.blocked is True: raise Exception(f"Team={team_object.team_id} is blocked. Update via `/team/unblock` if you're an admin.") diff --git a/litellm/proxy/management_endpoints/cost_tracking_settings.py b/litellm/proxy/management_endpoints/cost_tracking_settings.py index 56439172b63a..204051c37157 100644 --- a/litellm/proxy/management_endpoints/cost_tracking_settings.py +++ b/litellm/proxy/management_endpoints/cost_tracking_settings.py @@ -15,6 +15,7 @@ from typing import Final from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel import litellm from litellm._logging import verbose_proxy_logger @@ -439,6 +440,76 @@ async def update_cost_margin_config( ) +class BlockUnpricedModelsRequest(BaseModel): + enabled: bool + + +class BlockUnpricedModelsResponse(BaseModel): + enabled: bool + + +@router.get( + "/config/block_requests_for_models_without_pricing", + tags=("Cost Tracking",), + dependencies=(Depends(user_api_key_auth),), + response_model=BlockUnpricedModelsResponse, +) +async def get_block_requests_for_models_without_pricing() -> BlockUnpricedModelsResponse: + return BlockUnpricedModelsResponse(enabled=bool(litellm.block_requests_for_models_without_pricing)) + + +@router.patch( + "/config/block_requests_for_models_without_pricing", + tags=("Cost Tracking",), + dependencies=(Depends(user_api_key_auth),), + response_model=BlockUnpricedModelsResponse, +) +async def update_block_requests_for_models_without_pricing( + request: BlockUnpricedModelsRequest, +) -> BlockUnpricedModelsResponse: + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_config, + store_model_in_db, + ) + + if prisma_client is None: + raise HTTPException( + status_code=500, + detail={ # mutable-ok: HTTPException detail must be a plain mapping + "error": CommonProxyErrors.db_not_connected_error.value + }, + ) + + if store_model_in_db is not True: + raise HTTPException( + status_code=500, + detail={ # mutable-ok: HTTPException detail must be a plain mapping + "error": "Set `'STORE_MODEL_IN_DB='True'` in your env to enable this feature." + }, + ) + + try: + config = await proxy_config.get_config() + if "litellm_settings" not in config: + config["litellm_settings"] = {} # mutable-ok: config is a plain-dict payload for save_config + config["litellm_settings"]["block_requests_for_models_without_pricing"] = request.enabled + await proxy_config.save_config(new_config=config) + + litellm.block_requests_for_models_without_pricing = request.enabled + verbose_proxy_logger.info("Updated block_requests_for_models_without_pricing: %s", request.enabled) + + return BlockUnpricedModelsResponse(enabled=request.enabled) + except Exception as e: # noqa: BLE001 # any config persistence failure must surface as a 500 response, not a crash + verbose_proxy_logger.error("Error updating block_requests_for_models_without_pricing: %s", e) + raise HTTPException( + status_code=500, + detail={ # mutable-ok: HTTPException detail must be a plain mapping + "error": f"Failed to update setting: {e!s}" + }, + ) + + @router.post( "/cost/estimate", tags=["Cost Tracking"], diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 12d0f13ebdd0..db3bb52984a6 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6824,6 +6824,20 @@ async def _init_non_llm_objects_in_db(self, prisma_client: PrismaClient): if self._should_load_db_object(object_type="config_overrides"): await self._init_hashicorp_vault_config_override(prisma_client=prisma_client) + await self._apply_safe_litellm_settings_overrides_from_db(prisma_client=prisma_client) + + async def _apply_safe_litellm_settings_overrides_from_db(self, prisma_client: PrismaClient) -> None: + config_record: Final = await get_config_param(prisma_client, "litellm_settings") + if config_record is None or config_record.param_value is None: + return + raw_settings: Final = config_record.param_value + litellm_settings: Final = json.loads(raw_settings) if isinstance(raw_settings, str) else raw_settings + if not isinstance(litellm_settings, dict): + return + for key, value in litellm_settings.items(): + if key in LITELLM_SETTINGS_SAFE_DB_OVERRIDES: + setattr(litellm, key, value) + async def _init_semantic_filter_settings_in_db(self, prisma_client: PrismaClient): """ Initialize MCP semantic filter settings from database. diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 7a3288dec373..840899220eaa 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -3,8 +3,12 @@ import os import sys from types import SimpleNamespace +from typing import TYPE_CHECKING, Optional from unittest.mock import AsyncMock, MagicMock, patch +if TYPE_CHECKING: + from litellm.router import Router + sys.path.insert( 0, os.path.abspath("../../..") ) # Adds the parent directory to the system path @@ -6554,3 +6558,284 @@ def test_can_object_call_model_team_scoped_wildcard_accepts_bare_model_name(): ) is True ) + + +UNPRICED_UNDERLYING_MODEL = "openai/unpriced-model-lit4984-xyz" + + +def _router_with_priced_and_unpriced_models() -> "Router": + from litellm.router import Router + + return Router( + model_list=[ + { + "model_name": "priced-group", + "litellm_params": {"model": "gpt-3.5-turbo", "api_key": "sk-test"}, + }, + { + "model_name": "unpriced-group", + "litellm_params": {"model": UNPRICED_UNDERLYING_MODEL, "api_key": "sk-test"}, + }, + ] + ) + + +def test_model_has_no_cost_mapping_priced_model_is_false(): + from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping + + router = _router_with_priced_and_unpriced_models() + + assert model_has_no_cost_mapping(model="priced-group", llm_router=router) is False + + +def test_model_has_no_cost_mapping_unpriced_model_is_true(): + from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping + + router = _router_with_priced_and_unpriced_models() + + assert model_has_no_cost_mapping(model="unpriced-group", llm_router=router) is True + + +def test_model_has_no_cost_mapping_no_model_or_router_is_false(): + from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping + + router = _router_with_priced_and_unpriced_models() + + assert model_has_no_cost_mapping(model=None, llm_router=router) is False + assert model_has_no_cost_mapping(model="unpriced-group", llm_router=None) is False + + +@pytest.mark.parametrize( + "underlying_model", + [ + "azure/speech/azure-tts", + "mistral/mistral-ocr-latest", + "vertex_ai/imagen-3.0-generate-001", + "dashscope/qwen-flash", + ], +) +def test_model_has_no_cost_mapping_non_token_priced_model_is_false(underlying_model): + from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping + from litellm.router import Router + + router = Router( + model_list=[ + { + "model_name": "non-token-priced-group", + "litellm_params": {"model": underlying_model, "api_key": "sk-test"}, + } + ] + ) + + assert model_has_no_cost_mapping(model="non-token-priced-group", llm_router=router) is False + + +def test_model_has_no_cost_mapping_non_token_price_from_litellm_params_is_false(): + from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping + from litellm.router import Router + + router = Router( + model_list=[ + { + "model_name": "custom-tts", + "litellm_params": { + "model": f"{UNPRICED_UNDERLYING_MODEL}-per-second", + "api_key": "sk-test", + "input_cost_per_second": 0.0001, + }, + } + ] + ) + + assert model_has_no_cost_mapping(model="custom-tts", llm_router=router) is False + + +@pytest.mark.parametrize("cost_field", ["input_cost_per_second", "input_cost_per_token"]) +def test_model_has_no_cost_mapping_explicit_zero_price_is_false(cost_field): + from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping + from litellm.router import Router + + router = Router( + model_list=[ + { + "model_name": "free-group", + "litellm_params": { + "model": f"{UNPRICED_UNDERLYING_MODEL}-{cost_field}", + "api_key": "sk-test", + cost_field: 0, + }, + } + ] + ) + + assert model_has_no_cost_mapping(model="free-group", llm_router=router) is False + + +def test_model_has_no_cost_mapping_tiered_pricing_only_is_false(): + from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping + from litellm.router import Router + + router = Router( + model_list=[ + { + "model_name": "tiered-group", + "litellm_params": { + "model": f"{UNPRICED_UNDERLYING_MODEL}-tiered", + "api_key": "sk-test", + "tiered_pricing": [ + {"range": [0, 128000], "input_cost_per_token": 2e-7, "output_cost_per_token": 6e-7}, + {"range": [128000, 256000], "input_cost_per_token": 4e-7, "output_cost_per_token": 12e-7}, + ], + }, + } + ] + ) + + assert model_has_no_cost_mapping(model="tiered-group", llm_router=router) is False + + +async def _run_common_checks( + model: Optional[str], llm_router: Optional["Router"], route: str = "/chat/completions" +) -> bool: + from fastapi import Request + + from litellm.proxy.auth.auth_checks import common_checks + + return await common_checks( + request_body={"model": model, "messages": [{"role": "user", "content": "hi"}]}, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings={}, + route=route, + llm_router=llm_router, + proxy_logging_obj=MagicMock(), + valid_token=UserAPIKeyAuth(token="test-token"), + request=MagicMock(spec=Request), + ) + + +@pytest.mark.asyncio +async def test_common_checks_blocks_unpriced_model_when_enabled(monkeypatch): + monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", True) + router = _router_with_priced_and_unpriced_models() + + with pytest.raises(ProxyException) as exc_info: + await _run_common_checks(model="unpriced-group", llm_router=router) + + assert exc_info.value.code == "403" + assert exc_info.value.type == ProxyErrorTypes.model_cost_map_missing + assert exc_info.value.param == "model" + assert "unpriced-group" in exc_info.value.message + assert "pricing" in exc_info.value.message.lower() + + +@pytest.mark.asyncio +async def test_common_checks_allows_unpriced_model_when_disabled(monkeypatch): + monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", False) + router = _router_with_priced_and_unpriced_models() + + result = await _run_common_checks(model="unpriced-group", llm_router=router) + + assert result is True + + +@pytest.mark.asyncio +async def test_common_checks_allows_priced_model_when_enabled(monkeypatch): + monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", True) + router = _router_with_priced_and_unpriced_models() + + result = await _run_common_checks(model="priced-group", llm_router=router) + + assert result is True + + +@pytest.mark.asyncio +async def test_common_checks_ignores_non_llm_route_when_enabled(monkeypatch): + monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", True) + router = _router_with_priced_and_unpriced_models() + + result = await _run_common_checks( + model="unpriced-group", llm_router=router, route="/model/new" + ) + + assert result is True + + +@pytest.mark.asyncio +async def test_common_checks_blocks_alias_resolving_to_unpriced_model(monkeypatch): + from litellm.router import Router + + monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", True) + router = Router( + model_list=[ + { + "model_name": "billed-underlying-group", + "litellm_params": {"model": UNPRICED_UNDERLYING_MODEL, "api_key": "sk-test"}, + } + ], + model_group_alias={"public-alias": "billed-underlying-group"}, + ) + + with pytest.raises(ProxyException) as exc_info: + await _run_common_checks(model="public-alias", llm_router=router) + + assert exc_info.value.code == "403" + assert exc_info.value.type == ProxyErrorTypes.model_cost_map_missing + assert "public-alias" in exc_info.value.message + + +@pytest.mark.asyncio +async def test_common_checks_blocks_comma_separated_request_carrying_an_unpriced_model(monkeypatch): + monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", True) + router = _router_with_priced_and_unpriced_models() + + with pytest.raises(ProxyException) as exc_info: + await _run_common_checks(model="priced-group,unpriced-group", llm_router=router) + + assert exc_info.value.code == "403" + assert exc_info.value.type == ProxyErrorTypes.model_cost_map_missing + assert "'unpriced-group'" in exc_info.value.message + assert "'priced-group'" not in exc_info.value.message + + +@pytest.mark.asyncio +async def test_common_checks_allows_comma_separated_request_when_every_model_is_priced(monkeypatch): + monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", True) + router = _router_with_priced_and_unpriced_models() + + result = await _run_common_checks(model="priced-group,priced-group", llm_router=router) + + assert result is True + + +def _router_with_a_group_priced_through_model_info() -> "Router": + from litellm.router import Router + + return Router( + model_list=[ + { + "model_name": "model-info-priced-group", + "litellm_params": {"model": f"{UNPRICED_UNDERLYING_MODEL}-model-info", "api_key": "sk-test"}, + "model_info": {"input_cost_per_token": 0, "output_cost_per_token": 0}, + } + ], + model_group_alias={"model-info-priced-alias": "model-info-priced-group"}, + ) + + +def test_model_has_no_cost_mapping_group_priced_through_model_info_is_false(): + from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping + + router = _router_with_a_group_priced_through_model_info() + + assert model_has_no_cost_mapping(model="model-info-priced-group", llm_router=router) is False + + +def test_model_has_no_cost_mapping_alias_to_a_group_priced_through_model_info_is_false(): + from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping + + router = _router_with_a_group_priced_through_model_info() + + assert model_has_no_cost_mapping(model="model-info-priced-alias", llm_router=router) is False diff --git a/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py b/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py index 7e83180bfcd1..ea86731eba4e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py +++ b/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py @@ -683,3 +683,103 @@ async def test_estimate_cost_litellm_params_pricing_overrides_model_info(self): assert response.cost_per_request == pytest.approx(0.002) assert response.input_cost_per_token == pytest.approx(0.000001) assert response.output_cost_per_token == pytest.approx(0.000002) + + + + +class TestBlockRequestsForModelsWithoutPricing: + """Test suite for the block_requests_for_models_without_pricing toggle endpoints""" + + @pytest.mark.asyncio + async def test_get_reflects_in_memory_flag(self): + with patch.object(litellm, "block_requests_for_models_without_pricing", True): + response = client.get( + "/config/block_requests_for_models_without_pricing", + headers={"Authorization": "Bearer sk-1234"}, + ) + + assert response.status_code == 200 + assert response.json() == {"enabled": True} + + @pytest.mark.asyncio + async def test_patch_persists_and_updates_flag(self): + mock_proxy_config = AsyncMock() + mock_proxy_config.get_config = AsyncMock(return_value={"litellm_settings": {}}) + mock_proxy_config.save_config = AsyncMock() + + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + patch.object(litellm, "block_requests_for_models_without_pricing", False), + ): + response = client.patch( + "/config/block_requests_for_models_without_pricing", + headers={"Authorization": "Bearer sk-1234"}, + json={"enabled": True}, + ) + + assert response.status_code == 200 + assert response.json() == {"enabled": True} + assert litellm.block_requests_for_models_without_pricing is True + + saved_config = mock_proxy_config.save_config.call_args.kwargs["new_config"] + assert saved_config["litellm_settings"]["block_requests_for_models_without_pricing"] is True + + def test_peer_workers_pick_up_persisted_flag_on_config_reload(self): + """A PATCH only mutates the flag on the worker that served it; peer workers must pick the + persisted value up when they reload litellm_settings from the DB.""" + from litellm.proxy.proxy_server import ProxyConfig + + with patch.object(litellm, "block_requests_for_models_without_pricing", False): + ProxyConfig()._update_config_fields( + current_config={}, + param_name="litellm_settings", + db_param_value={"block_requests_for_models_without_pricing": True}, + ) + + assert litellm.block_requests_for_models_without_pricing is True + + @pytest.mark.asyncio + @pytest.mark.parametrize("loads_config_overrides", [True, False]) + async def test_periodic_db_sync_applies_flag_to_peer_worker(self, loads_config_overrides): + """The ~10s reconcile loop runs _init_non_llm_objects_in_db on every worker; it must apply + the persisted flag so peers converge without a restart, including when supported_db_objects + leaves config_overrides out.""" + from types import SimpleNamespace + + from litellm.proxy.proxy_server import ProxyConfig + + config_record = SimpleNamespace( + param_value={"block_requests_for_models_without_pricing": True, "unsafe_key": "x"} + ) + with ( + patch.object(litellm, "block_requests_for_models_without_pricing", False), + patch.object( + ProxyConfig, + "_should_load_db_object", + side_effect=lambda object_type: loads_config_overrides and object_type == "config_overrides", + ), + patch.object(ProxyConfig, "_init_hashicorp_vault_config_override", AsyncMock()), + patch("litellm.proxy.proxy_server.get_config_param", AsyncMock(return_value=config_record)), + ): + await ProxyConfig()._init_non_llm_objects_in_db(prisma_client=MagicMock()) + + assert litellm.block_requests_for_models_without_pricing is True + assert not hasattr(litellm, "unsafe_key") + + @pytest.mark.asyncio + async def test_patch_requires_store_model_in_db(self): + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.proxy_config", AsyncMock()), + patch("litellm.proxy.proxy_server.store_model_in_db", False), + ): + response = client.patch( + "/config/block_requests_for_models_without_pricing", + headers={"Authorization": "Bearer sk-1234"}, + json={"enabled": True}, + ) + + assert response.status_code == 500 + assert "error" in response.json()["detail"] diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx index 3fce928ed67b..52b66edcbe5d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx @@ -11,6 +11,7 @@ import { } from "@/components/ui/alert-dialog"; import { Button } from "@/components/ui/button"; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; +import { Switch } from "@/components/ui/switch"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { CostTrackingSettingsProps } from "./types"; import ProviderDiscountTable from "./provider_discount_table"; @@ -22,6 +23,7 @@ import { DocsMenu } from "@/components/HelpLink"; import HowItWorks from "./how_it_works"; import { useDiscountConfig } from "./use_discount_config"; import { useMarginConfig } from "./use_margin_config"; +import { useBlockUnpricedConfig } from "./use_block_unpriced_config"; import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models"; import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog"; @@ -86,9 +88,16 @@ const CostTrackingSettings: React.FC = ({ userID, use handleMarginChange, } = useMarginConfig({ accessToken }); + const { + blockUnpriced, + isUpdating: isUpdatingBlockUnpriced, + fetchBlockUnpriced, + setBlockUnpriced, + } = useBlockUnpricedConfig({ accessToken }); + useEffect(() => { if (accessToken) { - Promise.all([fetchDiscountConfig(), fetchMarginConfig()]).finally(() => { + Promise.all([fetchDiscountConfig(), fetchMarginConfig(), fetchBlockUnpriced()]).finally(() => { setIsFetching(false); }); @@ -103,7 +112,7 @@ const CostTrackingSettings: React.FC = ({ userID, use }; loadModels(); } - }, [accessToken, fetchDiscountConfig, fetchMarginConfig]); + }, [accessToken, fetchDiscountConfig, fetchMarginConfig, fetchBlockUnpriced]); const handleAddProvider = async () => { const success = await addProvider(selectedProvider, newDiscount); @@ -301,7 +310,35 @@ const CostTrackingSettings: React.FC = ({ userID, use )} - {/* Accordion 3: Pricing Calculator - Available to all roles */} + {/* Accordion 3: Block Unpriced Models - Only for proxy admins */} + {isProxyAdmin && ( + + + +
+
+
+

Block requests for models without pricing

+

+ When enabled, a request whose resolved model has no cost mapping is rejected with a 403 so an + admin can add pricing for it. Off by default +

+
+ setBlockUnpriced(checked)} + /> +
+
+ + + )} + + {/* Accordion 4: Pricing Calculator - Available to all roles */} ({ + apiClient: { + get: vi.fn(), + patch: vi.fn(), + }, +})); + +const ENDPOINT = "/config/block_requests_for_models_without_pricing"; + +describe("useBlockUnpricedConfig", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + describe("fetchBlockUnpriced", () => { + it("reflects the enabled flag returned by the proxy", async () => { + vi.mocked(apiClient.get).mockResolvedValueOnce({ enabled: true }); + + const { result } = renderHook(() => useBlockUnpricedConfig({ accessToken: "test-token" })); + + await act(async () => { + await result.current.fetchBlockUnpriced(); + }); + + expect(apiClient.get).toHaveBeenCalledWith(ENDPOINT, { accessToken: "test-token" }); + expect(result.current.blockUnpriced).toBe(true); + }); + + it("surfaces a toast when the fetch throws", async () => { + const error = new Error("Network error"); + vi.mocked(apiClient.get).mockRejectedValueOnce(error); + + const { result } = renderHook(() => useBlockUnpricedConfig({ accessToken: "test-token" })); + + await act(async () => { + await result.current.fetchBlockUnpriced(); + }); + + expect(toast.fromError).toHaveBeenCalledWith(error); + expect(result.current.blockUnpriced).toBe(false); + }); + + it("does nothing without an access token", async () => { + const { result } = renderHook(() => useBlockUnpricedConfig({ accessToken: null })); + + await act(async () => { + await result.current.fetchBlockUnpriced(); + }); + + expect(apiClient.get).not.toHaveBeenCalled(); + }); + }); + + describe("setBlockUnpriced", () => { + it("persists the new value and confirms it with a toast", async () => { + vi.mocked(apiClient.patch).mockResolvedValueOnce({ enabled: true }); + + const { result } = renderHook(() => useBlockUnpricedConfig({ accessToken: "test-token" })); + + await act(async () => { + await result.current.setBlockUnpriced(true); + }); + + expect(apiClient.patch).toHaveBeenCalledWith(ENDPOINT, { + accessToken: "test-token", + body: { enabled: true }, + }); + expect(result.current.blockUnpriced).toBe(true); + expect(toast.success).toHaveBeenCalledWith(expect.stringMatching(/will now be blocked/i)); + expect(result.current.isUpdating).toBe(false); + }); + + it("confirms turning the block back off", async () => { + vi.mocked(apiClient.patch).mockResolvedValueOnce({ enabled: false }); + + const { result } = renderHook(() => useBlockUnpricedConfig({ accessToken: "test-token" })); + + await act(async () => { + await result.current.setBlockUnpriced(false); + }); + + expect(result.current.blockUnpriced).toBe(false); + expect(toast.success).toHaveBeenCalledWith(expect.stringMatching(/now allowed/i)); + }); + + it("surfaces the proxy error and leaves the flag unchanged when the update fails", async () => { + const error = new Error("Set `'STORE_MODEL_IN_DB='True'` in your env to enable this feature."); + vi.mocked(apiClient.patch).mockRejectedValueOnce(error); + + const { result } = renderHook(() => useBlockUnpricedConfig({ accessToken: "test-token" })); + + await act(async () => { + await result.current.setBlockUnpriced(true); + }); + + expect(toast.fromError).toHaveBeenCalledWith(error); + expect(toast.success).not.toHaveBeenCalled(); + expect(result.current.blockUnpriced).toBe(false); + expect(result.current.isUpdating).toBe(false); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/use_block_unpriced_config.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/use_block_unpriced_config.ts new file mode 100644 index 000000000000..4bf9d5ddacc3 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/use_block_unpriced_config.ts @@ -0,0 +1,65 @@ +import { useState, useCallback } from "react"; +import { apiClient } from "@/components/networking"; +import { toast } from "@/lib/toast"; + +export interface UseBlockUnpricedConfigProps { + accessToken: string | null; +} + +export interface UseBlockUnpricedConfigReturn { + blockUnpriced: boolean; + isUpdating: boolean; + fetchBlockUnpriced: () => Promise; + setBlockUnpriced: (enabled: boolean) => Promise; +} + +interface BlockUnpricedResponse { + enabled: boolean; +} + +const ENDPOINT = "/config/block_requests_for_models_without_pricing"; + +export function useBlockUnpricedConfig({ accessToken }: UseBlockUnpricedConfigProps): UseBlockUnpricedConfigReturn { + const [blockUnpriced, setBlockUnpricedState] = useState(false); + const [isUpdating, setIsUpdating] = useState(false); + + const fetchBlockUnpriced = useCallback(async () => { + if (!accessToken) return; + try { + const data = await apiClient.get(ENDPOINT, { accessToken }); + setBlockUnpricedState(Boolean(data?.enabled)); + } catch (error) { + console.error("Error fetching block-unpriced-models setting:", error); + toast.fromError(error); + } + }, [accessToken]); + + const setBlockUnpriced = useCallback( + async (enabled: boolean) => { + if (!accessToken) return; + setIsUpdating(true); + try { + const data = await apiClient.patch(ENDPOINT, { accessToken, body: { enabled } }); + setBlockUnpricedState(Boolean(data?.enabled)); + toast.success( + enabled + ? "Requests for models without pricing will now be blocked" + : "Requests for models without pricing are now allowed", + ); + } catch (error) { + console.error("Error updating block-unpriced-models setting:", error); + toast.fromError(error); + } finally { + setIsUpdating(false); + } + }, + [accessToken], + ); + + return { + blockUnpriced, + isUpdating, + fetchBlockUnpriced, + setBlockUnpriced, + }; +} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 790ebcdd2271..ba20f75f4c45 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -2145,6 +2145,24 @@ export interface paths { patch?: never; trace?: never; }; + "/config/block_requests_for_models_without_pricing": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Block Requests For Models Without Pricing */ + get: operations["get_block_requests_for_models_without_pricing_config_block_requests_for_models_without_pricing_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + /** Update Block Requests For Models Without Pricing */ + patch: operations["update_block_requests_for_models_without_pricing_config_block_requests_for_models_without_pricing_patch"]; + trace?: never; + }; "/config/callback/delete": { parameters: { query?: never; @@ -22456,6 +22474,16 @@ export interface components { /** Team Id */ team_id: string; }; + /** BlockUnpricedModelsRequest */ + BlockUnpricedModelsRequest: { + /** Enabled */ + enabled: boolean; + }; + /** BlockUnpricedModelsResponse */ + BlockUnpricedModelsResponse: { + /** Enabled */ + enabled: boolean; + }; /** BlockUsers */ BlockUsers: { /** User Ids */ @@ -40157,6 +40185,59 @@ export interface operations { }; }; }; + get_block_requests_for_models_without_pricing_config_block_requests_for_models_without_pricing_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BlockUnpricedModelsResponse"]; + }; + }; + }; + }; + update_block_requests_for_models_without_pricing_config_block_requests_for_models_without_pricing_patch: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["BlockUnpricedModelsRequest"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BlockUnpricedModelsResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; delete_callback_config_callback_delete_post: { parameters: { query?: never;