Skip to content

feat(prometheus): add api_provider label to token, latency, request and cache metrics #32126

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yucheng-berri merged 2 commits into from
Jul 4, 2026
+346 −0
Merged

feat(prometheus): add api_provider label to token, latency, request and cache metrics #32126

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9658382
feat(prometheus): add api_provider label to token, latency, request a…
shivijain2323 Jul 3, 2026
be63de1
fix(prometheus): satisfy ruff BLE001 budget and update enterprise lab…
yucheng-berri Jul 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions litellm/integrations/prometheus.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2043,6 +2043,43 @@ def _should_skip_metrics_for_invalid_key(

return False

@staticmethod
def _extract_api_provider_from_request_data(request_data: dict) -> Optional[str]:
"""
Best-effort provider for the client-side failure path.

A request can fail before a deployment is resolved, so the provider is
not always known. Prefer the resolved ``custom_llm_provider`` on
``litellm_params``, then any provider recovered onto a partial
``standard_logging_object`` (e.g. a stream that broke mid-flight), and
finally infer it from the requested model name (e.g. ``gpt-4o-mini`` ->
``openai``) since the proxy's failure ``request_data`` usually carries
only the client-supplied model. Return ``None`` when it cannot be
determined so the label emits empty rather than a guess.
"""
litellm_params = request_data.get("litellm_params") or {}
provider = litellm_params.get("custom_llm_provider")
if provider:
return provider
standard_logging_object = request_data.get("standard_logging_object") or {}
provider = standard_logging_object.get("custom_llm_provider")
if provider:
return provider
model = litellm_params.get("model") or request_data.get("model")
if not model:
return None
try:
return litellm.get_llm_provider(model=model)[1] or None
except litellm.exceptions.BadRequestError:
return None

@cursor cursor Bot Jul 4, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Failure path misses provider fields

Medium Severity

_extract_api_provider_from_request_data only reads custom_llm_provider from nested litellm_params and from a top-level standard_logging_object. On the proxy client failure path, post_call_failure_hook drops litellm_logging_obj without copying that payload onto request_data, so resolved provider data that still lives on the logging object never reaches the extractor and api_provider can stay empty when model-name inference fails.

Fix in Cursor Fix in Web

Reviewed by Cursor Bugbot for commit be63de1. Configure here.

@yucheng-berri yucheng-berri Jul 4, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

out of scope

except Exception as e: # noqa: BLE001 - metrics labeling must never break request/failure handling
verbose_logger.debug(
"prometheus: unexpected error inferring api_provider from model=%s: %s",
model,
e,
)
return None

async def async_post_call_failure_hook(
self,
request_data: dict,
Expand Down Expand Up @@ -2078,6 +2115,7 @@ async def async_post_call_failure_hook(
_metadata = request_data.get("metadata", {}) or {}
model_id = _metadata.get("model_info", {}).get("id") or request_data.get("model_info", {}).get("id")
rate_limit_category, rate_limit_type = self._extract_rate_limit_labels(original_exception)
api_provider = self._extract_api_provider_from_request_data(request_data)
enum_values = UserAPIKeyLabelValues(
end_user=user_api_key_dict.end_user_id,
user=user_api_key_dict.user_id,
Expand All @@ -2099,6 +2137,7 @@ async def async_post_call_failure_hook(
client_ip=_metadata.get("requester_ip_address"),
user_agent=_metadata.get("user_agent"),
model_id=model_id,
api_provider=api_provider,
stream=(str(request_data.get("stream")) if litellm.prometheus_emit_stream_label else None),
)
_label_ctx = PrometheusLabelFactoryContext(enum_values)
Expand Down
10 changes: 10 additions & 0 deletions litellm/types/integrations/prometheus.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.END_USER.value,
UserAPIKeyLabelNames.USER.value,
UserAPIKeyLabelNames.MODEL_ID.value,
Comment on lines 283 to 285

@greptile-apps greptile-apps Bot Jul 4, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Label schema change causes time-series discontinuity in existing deployments

Adding api_provider to metrics that have been emitting without it (e.g. litellm_llm_api_latency_metric, litellm_proxy_total_requests_metric, the token metrics) is a breaking change for any deployment that has been scraping litellm's Prometheus endpoint. Prometheus identifies a time series by its metric name plus its full label set, so every existing series for these metrics will stop receiving new data points after the upgrade and a fresh set of series (with api_provider included) will begin. Dashboards and alert rules that reference the old series will see a data gap at the upgrade boundary. Per the repo's backwards-compatibility policy, a feature flag (e.g. litellm.prometheus_emit_api_provider_label) that defaults to True for new installs but can be set to False for existing ones would let current users opt out of the label addition until they are ready to migrate their dashboards. At minimum, this change should be prominently called out in the changelog as a breaking label-schema change.

Rule Used: What: avoid backwards-incompatible changes without... (source)

UserAPIKeyLabelNames.API_PROVIDER.value,

@greptile-apps greptile-apps Bot Jul 4, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Backward-incompatible label addition without an opt-in flag

Adding api_provider to these established metrics (token, latency, proxy request counters, cache) changes their label cardinality. In Prometheus, a metric's label set is fixed at registration time — upgrading litellm registers new timeseries that carry the extra label, while the old labelset timeseries stop receiving data. Any existing Grafana panel or PromQL alert that queries litellm_llm_api_latency_metric, litellm_total_tokens_metric, litellm_proxy_failed_requests_metric, or the cache metrics by their current label set will show a gap at the upgrade boundary, and aggregations will silently produce incorrect totals until dashboards are updated.

Other label additions in this codebase (stream label, rate-limit labels) were gated behind litellm.prometheus_emit_stream_label / litellm.prometheus_emit_rate_limit_labels precisely to let operators adopt the new cardinality on their own schedule. The same mechanism should apply here — a litellm.prometheus_emit_api_provider_label (defaulting to False) would let get_labels_for_metric include api_provider only when explicitly enabled, keeping existing dashboards intact until users are ready to migrate.

Rule Used: What: avoid backwards-incompatible changes without... (source)

]
Comment on lines 285 to 287

@greptile-apps greptile-apps Bot Jul 4, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Unconditional label addition breaks existing dashboards on upgrade

The project already has an established pattern for exactly this scenario: litellm.prometheus_emit_rate_limit_labels and litellm.prometheus_emit_stream_label are opt-in flags used precisely because "existing dashboards keyed on this metric's historical label set [must] keep matching after upgrade" (see the comment in litellm_proxy_failed_requests_metric). Adding api_provider unconditionally to 10+ long-lived metrics — including litellm_llm_api_latency_metric, litellm_total_tokens_metric, litellm_cache_hits_metric and the others — changes their time-series fingerprint on the first scrape after upgrade. Every Grafana panel, alert rule, and recording rule that references these metrics by label equality will immediately start matching nothing (old series have no api_provider label; new series carry it), causing silent gaps or broken alerting for all existing deployments. The same label addition should be gated on an opt-in flag (e.g. litellm.prometheus_emit_api_provider_label, defaulting to False) and wired through PrometheusMetricLabels.get_labels the same way the rate-limit and stream labels are today.

Rule Used: What: avoid backwards-incompatible changes without... (source)

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!


litellm_llm_api_time_to_first_token_metric = [
Expand All @@ -295,6 +296,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.END_USER.value,
UserAPIKeyLabelNames.USER.value,
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
]

litellm_request_total_latency_metric = [
Expand All @@ -307,6 +309,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.USER.value,
UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value,
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
]

litellm_request_queue_time_seconds = [
Expand All @@ -319,6 +322,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.USER.value,
UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value,
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
]

# Guardrail metrics - these use custom labels (guardrail_name, status, error_type, hook_type)
Expand All @@ -341,6 +345,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.CLIENT_IP.value,
UserAPIKeyLabelNames.USER_AGENT.value,
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
]

litellm_proxy_failed_requests_metric = [
Expand All @@ -362,6 +367,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.CLIENT_IP.value,
UserAPIKeyLabelNames.USER_AGENT.value,
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
]

litellm_deployment_latency_per_output_token = [
Expand Down Expand Up @@ -458,6 +464,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.USER_EMAIL.value,
UserAPIKeyLabelNames.REQUESTED_MODEL.value,
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
]

litellm_total_tokens_metric = [
Expand All @@ -471,6 +478,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.USER_EMAIL.value,
UserAPIKeyLabelNames.REQUESTED_MODEL.value,
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
]

litellm_output_tokens_metric = [
Expand All @@ -484,6 +492,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.USER_EMAIL.value,
UserAPIKeyLabelNames.REQUESTED_MODEL.value,
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
]

# Token-type detail metrics — reuse the same label set as
Expand Down Expand Up @@ -682,6 +691,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.END_USER.value,
UserAPIKeyLabelNames.USER.value,
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
]

litellm_cache_hits_metric = _cache_metric_labels
Expand Down
8 changes: 8 additions & 0 deletions ...s/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,7 @@ def test_increment_token_metrics(prometheus_logger):
requested_model=None,
model="gpt-5-mini",
model_id="model-123",
api_provider="openai",
)
prometheus_logger.litellm_tokens_metric.labels().inc.assert_called_once_with(100)

Expand All @@ -244,6 +245,7 @@ def test_increment_token_metrics(prometheus_logger):
requested_model=None,
model="gpt-5-mini",
model_id="model-123",
api_provider="openai",
)
prometheus_logger.litellm_input_tokens_metric.labels().inc.assert_called_once_with(
50
Expand All @@ -262,6 +264,7 @@ def test_increment_token_metrics(prometheus_logger):
requested_model=None,
model="gpt-5-mini",
model_id="model-123",
api_provider="openai",
)
prometheus_logger.litellm_output_tokens_metric.labels().inc.assert_called_once_with(
50
Expand Down Expand Up @@ -424,6 +427,7 @@ def test_set_latency_metrics(prometheus_logger):
requested_model="openai-gpt",
model="gpt-5-mini",
model_id="model-123",
api_provider="openai",
)
prometheus_logger.litellm_llm_api_time_to_first_token_metric.labels().observe.assert_called_once_with(
0.5
Expand All @@ -442,6 +446,7 @@ def test_set_latency_metrics(prometheus_logger):
requested_model="openai-gpt",
model="gpt-5-mini",
model_id="model-123",
api_provider="openai",
)
prometheus_logger.litellm_llm_api_latency_metric.labels().observe.assert_called_once_with(
1.5
Expand All @@ -460,6 +465,7 @@ def test_set_latency_metrics(prometheus_logger):
requested_model="openai-gpt",
model="gpt-5-mini",
model_id="model-123",
api_provider="openai",
)
prometheus_logger.litellm_request_total_latency_metric.labels().observe.assert_called_once_with(
2.0
Expand Down Expand Up @@ -844,6 +850,7 @@ async def test_async_post_call_failure_hook(prometheus_logger):
model_id=None,
client_ip=None,
user_agent=None,
api_provider="openai",
)
finally:
litellm.prometheus_emit_rate_limit_labels = original_emit
Expand All @@ -867,6 +874,7 @@ async def test_async_post_call_failure_hook(prometheus_logger):
model_id=None,
client_ip=None,
user_agent=None,
api_provider="openai",
)
prometheus_logger.litellm_proxy_total_requests_metric.labels().inc.assert_called_once()

Expand Down
Loading
Loading