Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions litellm/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -463,6 +463,11 @@ def _dev_env_hot_reload_enabled() -> bool:
prometheus_exclude_metrics: Optional[List[str]] = None
prometheus_exclude_labels: Optional[List[str]] = None
prometheus_emit_stream_label: bool = False
prometheus_deployment_and_latency_caller_identity: Literal[
"api_key_alias",
"user_email",
"both",
] = "api_key_alias"
# Opt-in: emit `rate_limit_category` and `rate_limit_type` labels on
# `litellm_proxy_failed_requests_metric`. Off by default to preserve the
# pre-unification label set so existing dashboards / recording rules keyed on
Expand Down
15 changes: 15 additions & 0 deletions litellm/integrations/prometheus.py
Original file line number Diff line number Diff line change
Expand Up @@ -2462,6 +2462,7 @@ def _extract_deployment_failure_label_values(self, request_kwargs: dict) -> dict
else:
_metadata = {
"user_api_key_alias": getattr(_metadata_raw, "user_api_key_alias", None),
"user_api_key_user_email": getattr(_metadata_raw, "user_api_key_user_email", None),
"user_api_key_team_id": getattr(_metadata_raw, "user_api_key_team_id", None),
"user_api_key_team_alias": getattr(_metadata_raw, "user_api_key_team_alias", None),
"user_api_key_hash": getattr(_metadata_raw, "user_api_key_hash", None),
Expand All @@ -2484,6 +2485,17 @@ def _get_api_key_alias() -> str | None:
return getattr(user_api_key_auth, "key_alias", None)
return None

def _get_user_email() -> str | None:
val = _metadata.get("user_api_key_user_email")
if val is not None:
return val
val = _litellm_params_metadata.get("user_api_key_user_email")
if val is not None:
return val
if user_api_key_auth is not None:
return self._safe_get(user_api_key_auth, "user_email")
return None

def _get_team_id() -> str | None:
val = _metadata.get("user_api_key_team_id")
if val is not None:
Expand Down Expand Up @@ -2519,6 +2531,7 @@ def _get_hashed_api_key() -> str | None:

return {
"api_key_alias": _get_api_key_alias(),
"user_email": _get_user_email(),
"team": _get_team_id(),
"team_alias": _get_team_alias(),
"hashed_api_key": _get_hashed_api_key(),
Expand Down Expand Up @@ -2576,6 +2589,7 @@ def set_llm_deployment_failure_metrics(self, request_kwargs: dict):
_metadata: Final = standard_logging_payload.get("metadata", {}) or {}
hashed_api_key: Final = fallback_values.get("hashed_api_key") or _metadata.get("user_api_key_hash")
api_key_alias: Final = fallback_values.get("api_key_alias") or _metadata.get("user_api_key_alias")
user_email: Final = fallback_values.get("user_email")
team: Final = fallback_values.get("team") or _metadata.get("user_api_key_team_id")
team_alias: Final = fallback_values.get("team_alias") or _metadata.get("user_api_key_team_alias")
client_ip: Final = fallback_values.get("client_ip") or _metadata.get("requester_ip_address")
Expand Down Expand Up @@ -2616,6 +2630,7 @@ def set_llm_deployment_failure_metrics(self, request_kwargs: dict):
requested_model=label_requested_model,
hashed_api_key=hashed_api_key,
api_key_alias=api_key_alias,
user_email=user_email,
team=team,
team_alias=team_alias,
tags=standard_logging_payload.get("request_tags", []),
Expand Down
7 changes: 7 additions & 0 deletions litellm/proxy/proxy_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -4852,6 +4852,13 @@ async def load_config(self, router: litellm.Router | None, config_file_path: str
if litellm_settings is None:
litellm_settings = {}
if litellm_settings:
# Prometheus collectors have fixed label schemas. Load this setting
# before processing callbacks so YAML key order cannot construct the
# collectors with the default caller-identity mode.
caller_identity_mode: Final = litellm_settings.get("prometheus_deployment_and_latency_caller_identity")
if caller_identity_mode is not None:
litellm.prometheus_deployment_and_latency_caller_identity = caller_identity_mode

# ANSI escape code for blue text
blue_color_code: Final = "\033[94m"
reset_color_code: Final = "\033[0m"
Expand Down
60 changes: 58 additions & 2 deletions litellm/types/integrations/prometheus.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import re
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from dataclasses import MISSING, dataclass, field, fields
from enum import Enum
from types import MappingProxyType
Expand Down Expand Up @@ -276,6 +276,59 @@ class UserAPIKeyLabelNames(Enum):
]


PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_METRICS: Final[frozenset[str]] = frozenset(
{
"litellm_deployment_total_requests",
"litellm_deployment_success_responses",
"litellm_deployment_failure_responses",
"litellm_request_total_latency_metric",
"litellm_llm_api_latency_metric",
"litellm_llm_api_time_to_first_token_metric",
"litellm_request_queue_time_seconds",
"litellm_overhead_latency_metric",
"litellm_deployment_latency_per_output_token",
}
)

PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_VALUES: Final[tuple[str, ...]] = (
"api_key_alias",
"user_email",
"both",
)


def _resolve_deployment_and_latency_caller_identity_labels(
metric_name: str,
labels: Sequence[object],
) -> list[str]: # mutable-ok: every caller must receive an independently mutable label list
"""Return a fresh label list with the configured caller identity schema."""
if not all(isinstance(label, str) for label in labels):
raise TypeError(f"Prometheus labels for {metric_name} must be strings")
resolved_labels: Final = [label for label in labels if isinstance(label, str)]
if metric_name not in PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_METRICS:
return resolved_labels

caller_identity: Final[object] = getattr(
litellm,
"prometheus_deployment_and_latency_caller_identity",
"api_key_alias",
)
if caller_identity not in PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_VALUES:
accepted_values: Final = ", ".join(PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_VALUES)
raise ValueError(
"Invalid prometheus_deployment_and_latency_caller_identity="
f"{caller_identity!r}. Accepted values: {accepted_values}."
)

alias_index: Final = resolved_labels.index(UserAPIKeyLabelNames.API_KEY_ALIAS.value)
if caller_identity == "user_email":
resolved_labels[alias_index] = UserAPIKeyLabelNames.USER_EMAIL.value
elif caller_identity == "both":
resolved_labels.insert(alias_index + 1, UserAPIKeyLabelNames.USER_EMAIL.value)

return resolved_labels


class PrometheusMetricLabels:
litellm_llm_api_latency_metric = [
UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value,
Expand Down Expand Up @@ -781,7 +834,10 @@ class PrometheusMetricLabels:

@staticmethod
def get_labels(label_name: DEFINED_PROMETHEUS_METRICS) -> list[str]:
default_labels: Final = getattr(PrometheusMetricLabels, label_name)
default_labels: Final = _resolve_deployment_and_latency_caller_identity_labels(
metric_name=label_name,
labels=getattr(PrometheusMetricLabels, label_name),
)
custom_labels: Final = []

# Add custom metadata labels
Expand Down
Loading
Loading