Skip to content

feat(prometheus): add call_type label to request lifecycle metrics #34717

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Bungic wants to merge 3 commits into
base: main
Choose a base branch
from
+197 −2
Open

feat(prometheus): add call_type label to request lifecycle metrics #34717

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
12607d2
feat(prometheus): add call_type label to request lifecycle metrics
Bungic Jul 26, 2026
0249d11
test(prometheus): expect call_type in the enterprise label assertions
Bungic Sep 13, 2026
860713b
fix(prometheus): defer call type aliases until first use
Bungic Sep 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions litellm/integrations/prometheus.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import replace
from datetime import datetime, timedelta
from functools import cache
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast

from pydantic import BaseModel
Expand Down Expand Up @@ -151,6 +152,37 @@ def labels(self, *labelvalues: str, **labelkwargs: str) -> MetricWrapperBase:
_MetricLike: TypeAlias = "NoOpMetric | _ExcludedLabelMetric | MetricWrapperBase"


_ASYNC_CALL_TYPE_PREFIXES: Final = ("a_", "a")


def _sync_twin(name: str, values_by_name: Mapping[str, str]) -> str | None:
return next(
(
values_by_name[name.removeprefix(prefix)]
for prefix in _ASYNC_CALL_TYPE_PREFIXES
if name.startswith(prefix) and name.removeprefix(prefix) in values_by_name
),
None,
)


@cache
def _build_async_call_type_aliases() -> Mapping[str, str]:
"""Matches on ``CallTypes`` member names, not values: stripping "a" from values breaks ``add_message``."""
from types import MappingProxyType

from litellm.types.utils import CallTypes

values_by_name: Final = MappingProxyType({member.name: str(member.value) for member in CallTypes})
return MappingProxyType(
{
value: twin
for name, value in values_by_name.items()
if (twin := _sync_twin(name, values_by_name)) is not None
}
)


def _get_budget_metrics_per_request_timeout() -> float:
raw: Final = os.getenv("PROMETHEUS_BUDGET_METRICS_PER_REQUEST_TIMEOUT")
if raw is None:
Expand Down Expand Up @@ -1442,6 +1474,7 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti
model_id=standard_logging_payload["model_id"],
api_base=standard_logging_payload["api_base"],
api_provider=standard_logging_payload["custom_llm_provider"],
call_type=self._normalize_call_type(standard_logging_payload.get("call_type")),
exception_status=None,
exception_class=None,
custom_metadata_labels=get_custom_labels_from_metadata(metadata=combined_metadata),
Expand Down Expand Up @@ -2544,6 +2577,13 @@ def _should_skip_metrics_for_invalid_key(

return False

@staticmethod
def _normalize_call_type(call_type: str | None) -> str | None:
"""Collapse async call types onto their sync twin so the proxy (async) and SDK (sync) share one series."""
if not call_type:
return None
return _build_async_call_type_aliases().get(call_type, call_type)

@staticmethod
def _extract_api_provider_from_request_data(request_data: dict) -> str | None:
"""
Expand Down Expand Up @@ -2639,6 +2679,9 @@ async def async_post_call_failure_hook(
user_agent=_metadata.get("user_agent"),
model_id=model_id,
api_provider=api_provider,
call_type=self._normalize_call_type(
(request_data.get("standard_logging_object") or {}).get("call_type")
),
stream=(str(request_data.get("stream")) if litellm.prometheus_emit_stream_label else None),
)
_label_ctx: Final = PrometheusLabelFactoryContext(enum_values)
Expand Down Expand Up @@ -2867,6 +2910,7 @@ def set_llm_deployment_failure_metrics(self, request_kwargs: dict):
tags=standard_logging_payload.get("request_tags", []),
client_ip=client_ip,
user_agent=user_agent,
call_type=self._normalize_call_type(standard_logging_payload.get("call_type")),
)

"""
Expand Down
18 changes: 18 additions & 0 deletions litellm/types/integrations/prometheus.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,7 @@ class UserAPIKeyLabelNames(Enum):
MCP_SERVER_NAME = "mcp_server_name"
SERVICE_TIER = "service_tier"
INPUT_SEQUENCE_LENGTH = "input_sequence_length"
CALL_TYPE = "call_type"


DEFINED_PROMETHEUS_METRICS = Literal[
Expand Down Expand Up @@ -415,6 +416,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
UserAPIKeyLabelNames.SERVICE_TIER.value,
UserAPIKeyLabelNames.CALL_TYPE.value,
]

litellm_llm_api_time_to_first_token_metric = [
Expand All @@ -429,6 +431,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
UserAPIKeyLabelNames.SERVICE_TIER.value,
UserAPIKeyLabelNames.CALL_TYPE.value,
]

litellm_request_total_latency_metric = [
Expand All @@ -443,6 +446,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
UserAPIKeyLabelNames.SERVICE_TIER.value,
UserAPIKeyLabelNames.CALL_TYPE.value,
]

litellm_request_queue_time_seconds = [
Expand All @@ -456,6 +460,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value,
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
UserAPIKeyLabelNames.CALL_TYPE.value,
]

# Guardrail metrics - these use custom labels (guardrail_name, status, error_type, hook_type)
Expand All @@ -479,6 +484,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.USER_AGENT.value,
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
UserAPIKeyLabelNames.CALL_TYPE.value,
]

litellm_proxy_failed_requests_metric = [
Expand All @@ -501,6 +507,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.USER_AGENT.value,
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
UserAPIKeyLabelNames.CALL_TYPE.value,
]

litellm_deployment_latency_per_output_token = [
Expand All @@ -512,6 +519,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.API_KEY_ALIAS.value,
UserAPIKeyLabelNames.TEAM.value,
UserAPIKeyLabelNames.TEAM_ALIAS.value,
UserAPIKeyLabelNames.CALL_TYPE.value,
]

litellm_overhead_latency_metric = [
Expand All @@ -522,6 +530,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.API_KEY_HASH.value,
UserAPIKeyLabelNames.API_KEY_ALIAS.value,
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.CALL_TYPE.value,
]

litellm_overhead_with_guardrails_latency_metric = [
Expand All @@ -532,6 +541,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.API_KEY_HASH.value,
UserAPIKeyLabelNames.API_KEY_ALIAS.value,
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.CALL_TYPE.value,
]

litellm_remaining_requests_metric = [
Expand Down Expand Up @@ -568,6 +578,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.REQUESTED_MODEL.value,
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
UserAPIKeyLabelNames.CALL_TYPE.value,
]

litellm_spend_metric = [
Expand All @@ -585,6 +596,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
UserAPIKeyLabelNames.SERVICE_TIER.value,
UserAPIKeyLabelNames.CALL_TYPE.value,
]

litellm_input_tokens_metric = [
Expand All @@ -599,6 +611,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.REQUESTED_MODEL.value,
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
UserAPIKeyLabelNames.CALL_TYPE.value,
]

litellm_total_tokens_metric = [
Expand All @@ -613,6 +626,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.REQUESTED_MODEL.value,
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
UserAPIKeyLabelNames.CALL_TYPE.value,
]

litellm_output_tokens_metric = [
Expand All @@ -627,6 +641,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.REQUESTED_MODEL.value,
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
UserAPIKeyLabelNames.CALL_TYPE.value,
]

# Token-type detail metrics — reuse the same label set as
Expand Down Expand Up @@ -764,6 +779,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.TEAM_ALIAS.value,
UserAPIKeyLabelNames.CLIENT_IP.value,
UserAPIKeyLabelNames.USER_AGENT.value,
UserAPIKeyLabelNames.CALL_TYPE.value,
]

litellm_deployment_total_requests = [
Expand All @@ -778,6 +794,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.TEAM_ALIAS.value,
UserAPIKeyLabelNames.CLIENT_IP.value,
UserAPIKeyLabelNames.USER_AGENT.value,
UserAPIKeyLabelNames.CALL_TYPE.value,
]

litellm_deployment_success_responses = litellm_deployment_total_requests
Expand Down Expand Up @@ -1033,6 +1050,7 @@ class UserAPIKeyLabelValues:
model_id: str | None = None
api_base: str | None = None
api_provider: str | None = None
call_type: str | None = None
exception_status: str | None = None
exception_class: str | None = None
rate_limit_category: str | None = None
Expand Down
14 changes: 14 additions & 0 deletions ...s/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,7 @@ def test_increment_token_metrics(prometheus_logger):
model="gpt-5-mini",
model_id="model-123",
api_provider="openai",
call_type="completion",
)
prometheus_logger.litellm_tokens_metric.labels().inc.assert_called_once_with(100)

Expand All @@ -261,6 +262,7 @@ def test_increment_token_metrics(prometheus_logger):
model="gpt-5-mini",
model_id="model-123",
api_provider="openai",
call_type="completion",
)
prometheus_logger.litellm_input_tokens_metric.labels().inc.assert_called_once_with(
50
Expand All @@ -280,6 +282,7 @@ def test_increment_token_metrics(prometheus_logger):
model="gpt-5-mini",
model_id="model-123",
api_provider="openai",
call_type="completion",
)
prometheus_logger.litellm_output_tokens_metric.labels().inc.assert_called_once_with(
50
Expand Down Expand Up @@ -443,6 +446,7 @@ def test_set_latency_metrics(prometheus_logger):
model="gpt-5-mini",
model_id="model-123",
api_provider="openai",
call_type="completion",
service_tier=None,
)
prometheus_logger.litellm_llm_api_time_to_first_token_metric.labels().observe.assert_called_once_with(
Expand All @@ -463,6 +467,7 @@ def test_set_latency_metrics(prometheus_logger):
model="gpt-5-mini",
model_id="model-123",
api_provider="openai",
call_type="completion",
service_tier=None,
)
prometheus_logger.litellm_llm_api_latency_metric.labels().observe.assert_called_once_with(
Expand All @@ -483,6 +488,7 @@ def test_set_latency_metrics(prometheus_logger):
model="gpt-5-mini",
model_id="model-123",
api_provider="openai",
call_type="completion",
service_tier=None,
)
prometheus_logger.litellm_request_total_latency_metric.labels().observe.assert_called_once_with(
Expand Down Expand Up @@ -629,6 +635,7 @@ def test_increment_top_level_request_and_spend_metrics(prometheus_logger):
model="gpt-5-mini",
model_id="model-123",
api_provider="openai",
call_type="completion",
client_ip=None,
user_agent=None,
requested_model=None,
Expand All @@ -649,6 +656,7 @@ def test_increment_top_level_request_and_spend_metrics(prometheus_logger):
model="gpt-5-mini",
model_id="model-123",
api_provider="openai",
call_type="completion",
client_ip=None,
user_agent=None,
requested_model=None,
Expand Down Expand Up @@ -870,6 +878,7 @@ async def test_async_post_call_failure_hook(prometheus_logger, known_model_route
client_ip=None,
user_agent=None,
api_provider="openai",
call_type=None,
)
finally:
litellm.prometheus_emit_rate_limit_labels = original_emit
Expand All @@ -894,6 +903,7 @@ async def test_async_post_call_failure_hook(prometheus_logger, known_model_route
client_ip=None,
user_agent=None,
api_provider="openai",
call_type=None,
)
prometheus_logger.litellm_proxy_total_requests_metric.labels().inc.assert_called_once()

Expand Down Expand Up @@ -1030,6 +1040,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger):
model_id="model-123",
api_base="https://api.openai.com",
api_provider="openai",
call_type="completion",
requested_model="my_custom_model_group",
hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"],
api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"],
Expand All @@ -1046,6 +1057,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger):
model_id="model-123",
api_base="https://api.openai.com",
api_provider="openai",
call_type="completion",
requested_model="my_custom_model_group",
hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"],
api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"],
Expand All @@ -1062,6 +1074,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger):
model_id="model-123",
api_base="https://api.openai.com",
api_provider="openai",
call_type="completion",
hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"],
api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"],
team=standard_logging_payload["metadata"]["user_api_key_team_id"],
Expand All @@ -1073,6 +1086,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger):
api_base="https://api.openai.com",
api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"],
api_provider="openai",
call_type="completion",
hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"],
litellm_model_name="gpt-5-mini",
model_group="my_custom_model_group",
Expand Down
Loading
Loading