feat(proxy): fairness under load controls with fair queueing and admin UI

backend/routes/allowlist.py

@@ -54,6 +54,7 @@
     "/cost_tracking",
     "/cost_optimization/",
     "/cost/",
+    "/fairness/",
     "/credentials",
     "/credential",
     "/provider/budgets",

litellm/__init__.py

@@ -513,11 +513,12 @@ def _dev_env_hot_reload_enabled() -> bool:
 # Old format: { "displayName": "url" } (for backward compatibility)
 public_model_groups_links: Dict[str, Union[str, Dict[str, Any]]] = {}
 #### REQUEST PRIORITIZATION #######
-priority_reservation: Optional[Dict[str, Union[float, "PriorityReservationDict"]]] = None
+priority_reservation: Optional[Mapping[str, Union[float, "PriorityReservationDict"]]] = None
 # priority_reservation_settings is lazy-loaded via __getattr__
 # Only declare for type checking - at runtime __getattr__ handles it
 if TYPE_CHECKING:
     priority_reservation_settings: Optional["PriorityReservationSettings"] = None
+fairness_settings: Optional["FairnessSettings"] = None
 
 
 ######## Networking Settings ########
@@ -1558,6 +1559,7 @@ def set_global_gitlab_config(config: Dict[str, Any]) -> None:
 if TYPE_CHECKING:
     from litellm.types.utils import ModelInfo as _ModelInfoType
     from litellm.types.utils import PriorityReservationSettings
+    from litellm.types.proxy.fairness import FairnessSettings
     from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
     from litellm.caching.caching import Cache
 

litellm/constants.py

@@ -1872,6 +1872,7 @@
     "cost_margin_config",
     "block_requests_for_models_without_pricing",
     "budget_exceeded_throttle_percentage",
+    "fairness_settings",
     # Every field editable from the Admin UI (proxy_server._GENERAL_SETTINGS_UI_LITELLM_FIELDS)
     # must be listed here so a DB write from one worker overrides the live litellm attribute on
     # the others when config reloads; otherwise peer workers stay on their startup value.

litellm/proxy/common_utils/active_request.py

@@ -0,0 +1,16 @@
+from contextvars import ContextVar
+from typing import Final
+
+from fastapi import Request
+
+active_request: Final[ContextVar[Request | None]] = ContextVar("litellm_active_request", default=None)
+
+
+async def active_request_disconnected() -> bool:
+    request: Final = active_request.get()
+    if request is None:
+        return False
+    try:
+        return await request.is_disconnected()
+    except Exception:  # noqa: BLE001  # disconnect probing must never fail the request
+        return False