Skip to content

fix(proxy): extend WARN/ERROR exception routing to LLM hot path #40

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
songkuan-zheng merged 1 commit into from
Jun 3, 2026
Merged

fix(proxy): extend WARN/ERROR exception routing to LLM hot path #40

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 28 additions & 17 deletions litellm/proxy/auth/auth_checks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
UserAPIKeyAuth,
)
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.common_utils.exception_logging import log_proxy_exception
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
from litellm.proxy.guardrails.tool_name_extraction import (
TOOL_CAPABLE_CALL_TYPES,
Expand Down Expand Up @@ -1212,11 +1213,12 @@ async def get_team_membership(
_response = LiteLLM_TeamMembership(**response.dict())

return _response
except Exception:
verbose_proxy_logger.exception(
"Error getting team membership for user_id: %s, team_id: %s",
user_id,
team_id,
except Exception as e:
log_proxy_exception(
verbose_proxy_logger,
"get_team_membership",
e,
extra={"user_id": user_id, "team_id": team_id},
)
return None

Expand Down Expand Up @@ -1830,9 +1832,11 @@ async def get_access_object(
except HTTPException:
raise
except Exception as e:
verbose_proxy_logger.exception(
"Error getting access group for access_group_id: %s",
access_group_id,
log_proxy_exception(
verbose_proxy_logger,
"access_group_lookup",
e,
extra={"access_group_id": access_group_id},
)
raise HTTPException(
status_code=404,
Expand Down Expand Up @@ -1943,7 +1947,12 @@ async def get_team_object_by_alias(
except HTTPException:
raise
except Exception as e:
verbose_proxy_logger.exception("Error looking up team by alias: %s", team_alias)
log_proxy_exception(
verbose_proxy_logger,
"team_alias_lookup",
e,
extra={"team_alias": team_alias},
)
raise HTTPException(
status_code=500,
detail={
Expand Down Expand Up @@ -2033,8 +2042,11 @@ async def get_org_object_by_alias(
except HTTPException:
raise
except Exception as e:
verbose_proxy_logger.exception(
"Error looking up organization by alias: %s", org_alias
log_proxy_exception(
verbose_proxy_logger,
"organization_alias_lookup",
e,
extra={"org_alias": org_alias},
)
raise HTTPException(
status_code=500,
Expand Down Expand Up @@ -3126,9 +3138,7 @@ async def _virtual_key_max_budget_alert_check(
alert_email_config: Optional[Dict[str, List[str]]] = (
_merge_budget_alert_email_configs(
global_cfg=litellm.default_key_max_budget_alert_emails,
per_key_cfg=(valid_token.metadata or {}).get(
"max_budget_alert_emails"
),
per_key_cfg=(valid_token.metadata or {}).get("max_budget_alert_emails"),
)
)

Expand All @@ -3138,7 +3148,9 @@ async def _virtual_key_max_budget_alert_check(
(int(k) for k in alert_email_config if k.isdigit()),
default=None,
)
if min_pct is None or valid_token.spend < valid_token.max_budget * (min_pct / 100.0):
if min_pct is None or valid_token.spend < valid_token.max_budget * (
min_pct / 100.0
):
return

call_info = CallInfo(
Expand All @@ -3164,8 +3176,7 @@ async def _virtual_key_max_budget_alert_check(
else:
# Old path: existing single 80% threshold — completely unchanged
alert_threshold = (
valid_token.max_budget
* EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE
valid_token.max_budget * EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE
)

if (
Expand Down
11 changes: 8 additions & 3 deletions litellm/proxy/auth/user_api_key_auth.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
from litellm.proxy.auth.oauth2_proxy_hook import handle_oauth2_proxy_request
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.common_utils.cache_coordinator import EventDrivenCacheCoordinator
from litellm.proxy.common_utils.exception_logging import log_proxy_exception
from litellm.proxy.common_utils.http_parsing_utils import (
_read_request_body,
_safe_get_request_headers,
Expand Down Expand Up @@ -295,7 +296,7 @@ async def return_body():
try:
return await user_api_key_auth(request=request, api_key=f"Bearer {api_key}")
except Exception as e:
verbose_proxy_logger.exception(e)
log_proxy_exception(verbose_proxy_logger, "websocket_auth", e)
await websocket.close(code=status.WS_1008_POLICY_VIOLATION)
raise HTTPException(status_code=403, detail=str(e))

Expand Down Expand Up @@ -1698,8 +1699,12 @@ def get_api_key_from_custom_header(
)
)
else:
verbose_proxy_logger.exception(
f"No LiteLLM Virtual Key pass. Please set header={custom_litellm_key_header_name}: Bearer <api_key>"
# Not an exception at all — header just isn't set. Demote from
# .exception() (which used to print "(NoneType: None)" with no
# current exception) to a single WARNING line.
verbose_proxy_logger.warning(
"No LiteLLM Virtual Key pass. Please set header=%s: Bearer <api_key>",
custom_litellm_key_header_name,
)
return api_key

Expand Down
57 changes: 30 additions & 27 deletions litellm/proxy/common_request_processing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
get_logging_caching_headers,
get_remaining_tokens_and_requests_from_request_data,
)
from litellm.proxy.common_utils.exception_logging import log_proxy_exception
from litellm.proxy.dd_span_tagger import DDSpanTagger
from litellm.proxy.route_llm_request import route_request
from litellm.proxy.utils import ProxyLogging
Expand Down Expand Up @@ -249,9 +250,7 @@ async def empty_gen() -> AsyncGenerator[str, None]:
)
except Exception as e:
# Unexpected error consuming first chunk.
verbose_proxy_logger.exception(
f"Error consuming first chunk from generator: {e}"
)
log_proxy_exception(verbose_proxy_logger, "stream[first-chunk]", e)

# Preserve status code from HTTPException (e.g., guardrail blocks)
error_status = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR)
Expand Down Expand Up @@ -1344,9 +1343,7 @@ async def _on_deferred_stream_complete(
try:
_enqueue_fn()
except Exception as e:
verbose_proxy_logger.exception(
"Error firing deferred logging: %s", e
)
log_proxy_exception(verbose_proxy_logger, "deferred_logging", e)

# Streaming cleanup: if an exception occurred AND the deferred
# streaming closure is still set, no streaming route will
Expand All @@ -1372,8 +1369,10 @@ async def _on_deferred_stream_complete(
)
)
except Exception as e:
verbose_proxy_logger.exception(
"Error in orphaned streaming async logging: %s", e
log_proxy_exception(
verbose_proxy_logger,
"orphaned_streaming_async_logging",
e,
)
try:
from litellm.litellm_core_utils.thread_pool_executor import (
Expand All @@ -1388,8 +1387,10 @@ async def _on_deferred_stream_complete(
end_time=None,
)
except Exception as e:
verbose_proxy_logger.exception(
"Error in orphaned streaming sync logging: %s", e
log_proxy_exception(
verbose_proxy_logger,
"orphaned_streaming_sync_logging",
e,
)

# Always return the client-requested model name (not provider-prefixed internal identifiers)
Expand Down Expand Up @@ -1610,10 +1611,15 @@ async def _run_deferred_stream_guardrails(
if guardrail_result is not None:
_response = guardrail_result
except Exception as e:
verbose_proxy_logger.exception(
"Error running post-call guardrail %s on streaming response: %s",
getattr(cb, "guardrail_name", type(cb).__name__),
log_proxy_exception(
verbose_proxy_logger,
"post_call_streaming_guardrail",
e,
extra={
"guardrail": getattr(
cb, "guardrail_name", type(cb).__name__
)
},
)
if isinstance(e, HTTPException) and hasattr(
captured_logging_obj, "model_call_details"
Expand All @@ -1622,8 +1628,9 @@ async def _run_deferred_stream_guardrails(
"metadata", {}
)["guardrail_blocked"] = True
except Exception as e:
verbose_proxy_logger.exception(
"Error in deferred streaming guardrail initialization: %s",
log_proxy_exception(
verbose_proxy_logger,
"deferred_streaming_guardrail_init",
e,
)
finally:
Expand All @@ -1637,8 +1644,9 @@ async def _run_deferred_stream_guardrails(
)
)
except Exception as e:
verbose_proxy_logger.exception(
"Error in deferred streaming async logging: %s",
log_proxy_exception(
verbose_proxy_logger,
"deferred_streaming_async_logging",
e,
)

Expand All @@ -1651,8 +1659,9 @@ async def _run_deferred_stream_guardrails(
end_time=None,
)
except Exception as e:
verbose_proxy_logger.exception(
"Error in deferred streaming sync logging: %s",
log_proxy_exception(
verbose_proxy_logger,
"deferred_streaming_sync_logging",
e,
)

Expand All @@ -1664,9 +1673,7 @@ async def _handle_llm_api_exception(
version: Optional[str] = None,
):
"""Raises ProxyException (OpenAI API compatible) if an exception is raised"""
verbose_proxy_logger.exception(
f"litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - {str(e)}"
)
log_proxy_exception(verbose_proxy_logger, "llm_api[handle-exception]", e)
# Allow callbacks to transform the error response
transformed_exception = await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
Expand Down Expand Up @@ -1956,11 +1963,7 @@ async def async_streaming_data_generator(
)
yield serialize_chunk(chunk)
except Exception as e:
verbose_proxy_logger.exception(
"litellm.proxy.proxy_server.async_data_generator(): Exception occured - {}".format(
str(e)
)
)
log_proxy_exception(verbose_proxy_logger, "async_data_generator[stream]", e)
transformed_exception = await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=e,
Expand Down
18 changes: 12 additions & 6 deletions litellm/proxy/common_utils/http_parsing_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,12 @@ async def _read_request_body(request: Optional[Request]) -> Dict:
try:
parsed_body = json.loads(body_str)
except json.JSONDecodeError:
# If both orjson and json.loads fail, throw a proper error
verbose_proxy_logger.error(
f"Invalid JSON payload received: {str(e)}"
# If both orjson and json.loads fail, throw a proper error.
# This is a client-formatting problem (400), not a server
# bug — WARN, no traceback. The ProxyException raised
# below is the response; this line is purely diagnostic.
verbose_proxy_logger.warning(
"Invalid JSON payload received: %s", str(e)
)
raise ProxyException(
message=f"Invalid JSON payload: {str(e)}",
Expand All @@ -85,11 +88,14 @@ async def _read_request_body(request: Optional[Request]) -> Dict:
return parsed_body

except (json.JSONDecodeError, orjson.JSONDecodeError, ProxyException) as e:
# Re-raise ProxyException as-is
verbose_proxy_logger.error(f"Invalid JSON payload received: {str(e)}")
# Re-raise ProxyException as-is. Same rationale as above — a 400
# response, not a server bug. WARN without traceback.
verbose_proxy_logger.warning("Invalid JSON payload received: %s", str(e))
raise
except Exception as e:
# Catch unexpected errors to avoid crashes
# Catch unexpected errors to avoid crashes. THIS is a real server
# fault path (we expected json/orjson to be the only decoders and
# something else blew up) — keep ERROR + traceback.
verbose_proxy_logger.exception(
"Unexpected error reading request body - {}".format(e)
)
Expand Down
5 changes: 2 additions & 3 deletions litellm/proxy/litellm_pre_call_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
TeamCallbackMetadata,
UserAPIKeyAuth,
)
from litellm.proxy.common_utils.exception_logging import log_proxy_exception
from litellm.proxy.common_utils.http_parsing_utils import _safe_get_request_headers

# Cache special headers as a frozenset for O(1) lookup performance
Expand Down Expand Up @@ -143,9 +144,7 @@ def safe_add_api_version_from_query_params(data: dict, request: Request):
except KeyError:
pass
except Exception as e:
verbose_logger.exception(
"error checking api version in query params: %s", str(e)
)
log_proxy_exception(verbose_proxy_logger, "api_version_query_param_parse", e)


def convert_key_logging_metadata_to_callback(
Expand Down
Loading