Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 69 additions & 5 deletions litellm/proxy/health_endpoints/_health_endpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -776,6 +776,35 @@ def _strip_admin_only_fields_from_health_result(result: dict) -> dict:
return out


def _resolve_targeted_model_ids(
model_list: list, model: Optional[str], model_id: Optional[str]
) -> Optional[set]:
"""
Resolve a ``/health`` ``model`` / ``model_id`` query param to the set of
deployment IDs the response should be scoped to.

Mirrors the live-path semantics in ``perform_health_check()``: ``model``
matches either the deployment's ``model_name`` alias or its
``litellm_params.model`` provider string. ``model_id`` is taken as-is.

Returns ``None`` when no targeting is requested — callers should treat
that as "no filter."
"""
if not model and not model_id:
return None
if model_id:
return {model_id}
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
target_ids: set = set()
for m in model_list:
deployment_id = (m.get("model_info") or {}).get("id")
if not deployment_id:
continue
litellm_model = (m.get("litellm_params") or {}).get("model")
if m.get("model_name") == model or litellm_model == model:
target_ids.add(deployment_id)
return target_ids


def _filter_health_check_results_by_model_ids(
results: dict, allowed_model_ids: set
) -> dict:
Expand Down Expand Up @@ -926,13 +955,20 @@ async def health_endpoint(
)

is_admin = _is_proxy_admin(user_api_key_dict)
model_specific_request = bool(model or model_id)

def _post_process(result: dict) -> dict:
# api_base / api_version reveal which provider/region/internal host the
# deployment talks to; only proxy admins receive them. Non-admin keys
# still see model/model_id and the healthy/unhealthy status. We also
# set a header so non-admin clients that previously parsed those
# fields can detect the change programmatically.
# When a caller asked about a specific model/model_id and zero
# endpoints came back healthy, surface that as a 503 so monitoring
# systems can rely on the HTTP status instead of having to parse the
# body. The body shape is unchanged.
if model_specific_request and result.get("healthy_count", 0) == 0:
response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
if is_admin:
return result
response.headers["Litellm-Health-Field-Notice"] = (
Expand Down Expand Up @@ -975,16 +1011,29 @@ def _post_process(result: dict) -> dict:
m for m in _llm_model_list if m.get("model_name") in allowed_models
]
if use_background_health_checks:
# The cached background result covers every model. When the
# caller targets a specific model/model_id we have to narrow the
# cache to that deployment before _post_process evaluates
# healthy_count, otherwise an unhealthy "foo" combined with any
# other healthy model would still report healthy_count > 0 and
# the targeted-503 path would never fire.
targeted_ids = _resolve_targeted_model_ids(_llm_model_list, model, model_id)
if len(user_api_key_dict.models) > 0:
allowed_model_ids = {
(m.get("model_info") or {}).get("id")
for m in _llm_model_list
if (m.get("model_info") or {}).get("id")
}
# _llm_model_list is already scoped to the caller's allowed
# model_names above, so targeted_ids is implicitly the
# intersection of "targeted" and "allowed."
filter_ids = (
targeted_ids if targeted_ids is not None else allowed_model_ids
)
filtered = _filter_health_check_results_by_model_ids(
health_check_results, allowed_model_ids
health_check_results, filter_ids
)
if not allowed_model_ids:
if targeted_ids is None and not allowed_model_ids:
# Caller has accessible model_names but none of the
# matching deployments expose a model_info.id, so the
# cache filter (which keys on model_id) drops every
Expand All @@ -1005,6 +1054,15 @@ def _post_process(result: dict) -> dict:
"to populate model_info.id for these models."
]
return _post_process(filtered)
if targeted_ids is not None:
# Admin caller targeting a specific model: filter the cache
# so the response (and the targeted-503 check) reflects only
# that deployment, not the global aggregate.
return _post_process(
_filter_health_check_results_by_model_ids(
health_check_results, targeted_ids
)
)
return _post_process(health_check_results)
else:
router_result = await _perform_health_check_and_save(
Expand Down Expand Up @@ -1384,7 +1442,7 @@ def callback_name(callback):
tags=["health"],
dependencies=[Depends(user_api_key_auth)],
)
async def health_readiness():
async def health_readiness(response: Response):
"""
Unprotected endpoint for checking if worker can receive requests
"""
Expand Down Expand Up @@ -1417,8 +1475,8 @@ async def health_readiness():
try:
index_info = await litellm.cache.cache._index_info()
except Exception as e:
index_info = "index does not exist - error: " + str(e)
cache_type = {"type": cache_type, "index_info": index_info}
index_info = "index does not exist - error: " + str(e) # type: ignore[assignment]
cache_type = {"type": cache_type, "index_info": index_info} # type: ignore[assignment]

# check log level
log_level_name = logging.getLevelName(verbose_logger.getEffectiveLevel())
Expand All @@ -1427,6 +1485,12 @@ async def health_readiness():
# check DB
if prisma_client is not None: # if db passed in, check if it's connected
db_health_status = await _db_health_readiness_check()
# A configured DB that is not reachable means the worker cannot
# serve requests that depend on persisted state (keys, budgets,
# spend logs). Return 503 so orchestrators take this pod out of
# rotation; "Not connected" (no DB configured at all) stays 200.
if db_health_status["status"] != "connected":
response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
Comment on lines +1502 to +1503

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Backwards-incompatible HTTP status change without a user-controlled flag

Both /health?model=foo and /health/readiness now return 503 where they previously returned 200. The PR description acknowledges this breaks existing callers. Per the team's rule, backwards-incompatible behavioural changes must be gated behind a user-controlled flag (e.g., litellm.health_strict_status_codes) so that operators can opt in on their own schedule rather than being broken on upgrade. The same applies to the /health 503 change introduced in _post_process.

Rule Used: What: avoid backwards-incompatible changes without... (source)

return {
"status": "healthy",
"db": db_health_status["status"],
Expand Down
Loading
Loading