diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 2528f07f92c7..a721db5e02e3 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -10,6 +10,7 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import replace from datetime import datetime, timedelta +from functools import cache from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast from pydantic import BaseModel @@ -151,6 +152,37 @@ def labels(self, *labelvalues: str, **labelkwargs: str) -> MetricWrapperBase: _MetricLike: TypeAlias = "NoOpMetric | _ExcludedLabelMetric | MetricWrapperBase" +_ASYNC_CALL_TYPE_PREFIXES: Final = ("a_", "a") + + +def _sync_twin(name: str, values_by_name: Mapping[str, str]) -> str | None: + return next( + ( + values_by_name[name.removeprefix(prefix)] + for prefix in _ASYNC_CALL_TYPE_PREFIXES + if name.startswith(prefix) and name.removeprefix(prefix) in values_by_name + ), + None, + ) + + +@cache +def _build_async_call_type_aliases() -> Mapping[str, str]: + """Matches on ``CallTypes`` member names, not values: stripping "a" from values breaks ``add_message``.""" + from types import MappingProxyType + + from litellm.types.utils import CallTypes + + values_by_name: Final = MappingProxyType({member.name: str(member.value) for member in CallTypes}) + return MappingProxyType( + { + value: twin + for name, value in values_by_name.items() + if (twin := _sync_twin(name, values_by_name)) is not None + } + ) + + def _get_budget_metrics_per_request_timeout() -> float: raw: Final = os.getenv("PROMETHEUS_BUDGET_METRICS_PER_REQUEST_TIMEOUT") if raw is None: @@ -1442,6 +1474,7 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti model_id=standard_logging_payload["model_id"], api_base=standard_logging_payload["api_base"], api_provider=standard_logging_payload["custom_llm_provider"], + call_type=self._normalize_call_type(standard_logging_payload.get("call_type")), exception_status=None, exception_class=None, custom_metadata_labels=get_custom_labels_from_metadata(metadata=combined_metadata), @@ -2544,6 +2577,13 @@ def _should_skip_metrics_for_invalid_key( return False + @staticmethod + def _normalize_call_type(call_type: str | None) -> str | None: + """Collapse async call types onto their sync twin so the proxy (async) and SDK (sync) share one series.""" + if not call_type: + return None + return _build_async_call_type_aliases().get(call_type, call_type) + @staticmethod def _extract_api_provider_from_request_data(request_data: dict) -> str | None: """ @@ -2639,6 +2679,9 @@ async def async_post_call_failure_hook( user_agent=_metadata.get("user_agent"), model_id=model_id, api_provider=api_provider, + call_type=self._normalize_call_type( + (request_data.get("standard_logging_object") or {}).get("call_type") + ), stream=(str(request_data.get("stream")) if litellm.prometheus_emit_stream_label else None), ) _label_ctx: Final = PrometheusLabelFactoryContext(enum_values) @@ -2867,6 +2910,7 @@ def set_llm_deployment_failure_metrics(self, request_kwargs: dict): tags=standard_logging_payload.get("request_tags", []), client_ip=client_ip, user_agent=user_agent, + call_type=self._normalize_call_type(standard_logging_payload.get("call_type")), ) """ diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index a024581f600e..703b4af3e1ef 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -222,6 +222,7 @@ class UserAPIKeyLabelNames(Enum): MCP_SERVER_NAME = "mcp_server_name" SERVICE_TIER = "service_tier" INPUT_SEQUENCE_LENGTH = "input_sequence_length" + CALL_TYPE = "call_type" DEFINED_PROMETHEUS_METRICS = Literal[ @@ -415,6 +416,7 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.MODEL_ID.value, UserAPIKeyLabelNames.API_PROVIDER.value, UserAPIKeyLabelNames.SERVICE_TIER.value, + UserAPIKeyLabelNames.CALL_TYPE.value, ] litellm_llm_api_time_to_first_token_metric = [ @@ -429,6 +431,7 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.MODEL_ID.value, UserAPIKeyLabelNames.API_PROVIDER.value, UserAPIKeyLabelNames.SERVICE_TIER.value, + UserAPIKeyLabelNames.CALL_TYPE.value, ] litellm_request_total_latency_metric = [ @@ -443,6 +446,7 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.MODEL_ID.value, UserAPIKeyLabelNames.API_PROVIDER.value, UserAPIKeyLabelNames.SERVICE_TIER.value, + UserAPIKeyLabelNames.CALL_TYPE.value, ] litellm_request_queue_time_seconds = [ @@ -456,6 +460,7 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value, UserAPIKeyLabelNames.MODEL_ID.value, UserAPIKeyLabelNames.API_PROVIDER.value, + UserAPIKeyLabelNames.CALL_TYPE.value, ] # Guardrail metrics - these use custom labels (guardrail_name, status, error_type, hook_type) @@ -479,6 +484,7 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.USER_AGENT.value, UserAPIKeyLabelNames.MODEL_ID.value, UserAPIKeyLabelNames.API_PROVIDER.value, + UserAPIKeyLabelNames.CALL_TYPE.value, ] litellm_proxy_failed_requests_metric = [ @@ -501,6 +507,7 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.USER_AGENT.value, UserAPIKeyLabelNames.MODEL_ID.value, UserAPIKeyLabelNames.API_PROVIDER.value, + UserAPIKeyLabelNames.CALL_TYPE.value, ] litellm_deployment_latency_per_output_token = [ @@ -512,6 +519,7 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.API_KEY_ALIAS.value, UserAPIKeyLabelNames.TEAM.value, UserAPIKeyLabelNames.TEAM_ALIAS.value, + UserAPIKeyLabelNames.CALL_TYPE.value, ] litellm_overhead_latency_metric = [ @@ -522,6 +530,7 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.API_KEY_HASH.value, UserAPIKeyLabelNames.API_KEY_ALIAS.value, UserAPIKeyLabelNames.MODEL_ID.value, + UserAPIKeyLabelNames.CALL_TYPE.value, ] litellm_overhead_with_guardrails_latency_metric = [ @@ -532,6 +541,7 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.API_KEY_HASH.value, UserAPIKeyLabelNames.API_KEY_ALIAS.value, UserAPIKeyLabelNames.MODEL_ID.value, + UserAPIKeyLabelNames.CALL_TYPE.value, ] litellm_remaining_requests_metric = [ @@ -568,6 +578,7 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.REQUESTED_MODEL.value, UserAPIKeyLabelNames.MODEL_ID.value, UserAPIKeyLabelNames.API_PROVIDER.value, + UserAPIKeyLabelNames.CALL_TYPE.value, ] litellm_spend_metric = [ @@ -585,6 +596,7 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.MODEL_ID.value, UserAPIKeyLabelNames.API_PROVIDER.value, UserAPIKeyLabelNames.SERVICE_TIER.value, + UserAPIKeyLabelNames.CALL_TYPE.value, ] litellm_input_tokens_metric = [ @@ -599,6 +611,7 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.REQUESTED_MODEL.value, UserAPIKeyLabelNames.MODEL_ID.value, UserAPIKeyLabelNames.API_PROVIDER.value, + UserAPIKeyLabelNames.CALL_TYPE.value, ] litellm_total_tokens_metric = [ @@ -613,6 +626,7 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.REQUESTED_MODEL.value, UserAPIKeyLabelNames.MODEL_ID.value, UserAPIKeyLabelNames.API_PROVIDER.value, + UserAPIKeyLabelNames.CALL_TYPE.value, ] litellm_output_tokens_metric = [ @@ -627,6 +641,7 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.REQUESTED_MODEL.value, UserAPIKeyLabelNames.MODEL_ID.value, UserAPIKeyLabelNames.API_PROVIDER.value, + UserAPIKeyLabelNames.CALL_TYPE.value, ] # Token-type detail metrics — reuse the same label set as @@ -764,6 +779,7 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.TEAM_ALIAS.value, UserAPIKeyLabelNames.CLIENT_IP.value, UserAPIKeyLabelNames.USER_AGENT.value, + UserAPIKeyLabelNames.CALL_TYPE.value, ] litellm_deployment_total_requests = [ @@ -778,6 +794,7 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.TEAM_ALIAS.value, UserAPIKeyLabelNames.CLIENT_IP.value, UserAPIKeyLabelNames.USER_AGENT.value, + UserAPIKeyLabelNames.CALL_TYPE.value, ] litellm_deployment_success_responses = litellm_deployment_total_requests @@ -1033,6 +1050,7 @@ class UserAPIKeyLabelValues: model_id: str | None = None api_base: str | None = None api_provider: str | None = None + call_type: str | None = None exception_status: str | None = None exception_class: str | None = None rate_limit_category: str | None = None diff --git a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py index 58cde4c8103e..d8520d20378b 100644 --- a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py +++ b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py @@ -244,6 +244,7 @@ def test_increment_token_metrics(prometheus_logger): model="gpt-5-mini", model_id="model-123", api_provider="openai", + call_type="completion", ) prometheus_logger.litellm_tokens_metric.labels().inc.assert_called_once_with(100) @@ -261,6 +262,7 @@ def test_increment_token_metrics(prometheus_logger): model="gpt-5-mini", model_id="model-123", api_provider="openai", + call_type="completion", ) prometheus_logger.litellm_input_tokens_metric.labels().inc.assert_called_once_with( 50 @@ -280,6 +282,7 @@ def test_increment_token_metrics(prometheus_logger): model="gpt-5-mini", model_id="model-123", api_provider="openai", + call_type="completion", ) prometheus_logger.litellm_output_tokens_metric.labels().inc.assert_called_once_with( 50 @@ -443,6 +446,7 @@ def test_set_latency_metrics(prometheus_logger): model="gpt-5-mini", model_id="model-123", api_provider="openai", + call_type="completion", service_tier=None, ) prometheus_logger.litellm_llm_api_time_to_first_token_metric.labels().observe.assert_called_once_with( @@ -463,6 +467,7 @@ def test_set_latency_metrics(prometheus_logger): model="gpt-5-mini", model_id="model-123", api_provider="openai", + call_type="completion", service_tier=None, ) prometheus_logger.litellm_llm_api_latency_metric.labels().observe.assert_called_once_with( @@ -483,6 +488,7 @@ def test_set_latency_metrics(prometheus_logger): model="gpt-5-mini", model_id="model-123", api_provider="openai", + call_type="completion", service_tier=None, ) prometheus_logger.litellm_request_total_latency_metric.labels().observe.assert_called_once_with( @@ -629,6 +635,7 @@ def test_increment_top_level_request_and_spend_metrics(prometheus_logger): model="gpt-5-mini", model_id="model-123", api_provider="openai", + call_type="completion", client_ip=None, user_agent=None, requested_model=None, @@ -649,6 +656,7 @@ def test_increment_top_level_request_and_spend_metrics(prometheus_logger): model="gpt-5-mini", model_id="model-123", api_provider="openai", + call_type="completion", client_ip=None, user_agent=None, requested_model=None, @@ -870,6 +878,7 @@ async def test_async_post_call_failure_hook(prometheus_logger, known_model_route client_ip=None, user_agent=None, api_provider="openai", + call_type=None, ) finally: litellm.prometheus_emit_rate_limit_labels = original_emit @@ -894,6 +903,7 @@ async def test_async_post_call_failure_hook(prometheus_logger, known_model_route client_ip=None, user_agent=None, api_provider="openai", + call_type=None, ) prometheus_logger.litellm_proxy_total_requests_metric.labels().inc.assert_called_once() @@ -1030,6 +1040,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): model_id="model-123", api_base="https://api.openai.com", api_provider="openai", + call_type="completion", requested_model="my_custom_model_group", hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"], api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"], @@ -1046,6 +1057,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): model_id="model-123", api_base="https://api.openai.com", api_provider="openai", + call_type="completion", requested_model="my_custom_model_group", hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"], api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"], @@ -1062,6 +1074,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): model_id="model-123", api_base="https://api.openai.com", api_provider="openai", + call_type="completion", hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"], api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"], team=standard_logging_payload["metadata"]["user_api_key_team_id"], @@ -1073,6 +1086,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): api_base="https://api.openai.com", api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"], api_provider="openai", + call_type="completion", hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"], litellm_model_name="gpt-5-mini", model_group="my_custom_model_group", diff --git a/tests/test_litellm/integrations/test_prometheus_labels.py b/tests/test_litellm/integrations/test_prometheus_labels.py index 859cdd30c117..8d3a98d15192 100644 --- a/tests/test_litellm/integrations/test_prometheus_labels.py +++ b/tests/test_litellm/integrations/test_prometheus_labels.py @@ -629,7 +629,7 @@ async def test_success_hook_emits_api_provider_value_on_token_metric(): payload = { "id": "t", - "call_type": "completion", + "call_type": "acompletion", "response_cost": 0.001, "status": "success", "total_tokens": 30, @@ -677,6 +677,7 @@ async def test_success_hook_emits_api_provider_value_on_token_metric(): ) samples = _collected_samples("litellm_total_tokens_metric_total") assert samples, "expected litellm_total_tokens_metric to be emitted" + assert all(s.labels.get("call_type") == "completion" for s in samples) assert all(s.labels.get("api_provider") == "openai" for s in samples), ( "collected token metric must carry api_provider=openai, got " f"{[s.labels.get('api_provider') for s in samples]}" @@ -702,12 +703,17 @@ async def test_failure_hook_emits_api_provider_value_on_failed_requests_metric() try: logger = PrometheusLogger() await logger.async_post_call_failure_hook( - request_data={"model": "gpt-4o-mini", "metadata": {}}, + request_data={ + "model": "gpt-4o-mini", + "metadata": {}, + "standard_logging_object": {"call_type": "acompletion"}, + }, original_exception=Exception("boom"), user_api_key_dict=UserAPIKeyAuth(token="tok"), ) samples = _collected_samples("litellm_proxy_failed_requests_metric_total") assert samples, "expected litellm_proxy_failed_requests_metric to be emitted" + assert all(s.labels.get("call_type") == "completion" for s in samples) assert any(s.labels.get("api_provider") == "openai" for s in samples), ( "collected failed-requests metric must carry api_provider=openai, got " f"{[s.labels.get('api_provider') for s in samples]}" @@ -730,3 +736,116 @@ async def test_failure_hook_emits_api_provider_value_on_failed_requests_metric() test_prometheus_label_value_sanitization_none() test_prometheus_label_value_sanitization_non_string_types() print("\nāœ… All prometheus label tests passed!") + + +def test_call_type_in_request_lifecycle_metrics(): + call_type_label = UserAPIKeyLabelNames.CALL_TYPE.value + + metrics_with_call_type = [ + "litellm_spend_metric", + "litellm_requests_metric", + "litellm_input_tokens_metric", + "litellm_output_tokens_metric", + "litellm_total_tokens_metric", + "litellm_llm_api_latency_metric", + "litellm_llm_api_time_to_first_token_metric", + "litellm_request_total_latency_metric", + "litellm_request_queue_time_seconds", + "litellm_overhead_latency_metric", + "litellm_overhead_with_guardrails_latency_metric", + "litellm_proxy_total_requests_metric", + "litellm_proxy_failed_requests_metric", + "litellm_deployment_latency_per_output_token", + "litellm_deployment_failure_responses", + "litellm_deployment_total_requests", + "litellm_deployment_success_responses", + ] + + for metric_name in metrics_with_call_type: + labels = PrometheusMetricLabels.get_labels(metric_name) + assert call_type_label in labels, f"Metric {metric_name} should contain call_type" + + +def test_call_type_absent_from_quota_and_deployment_gauges(): + """Headroom belongs to a deployment; splitting it by call type would emit duplicate series for one limit.""" + call_type_label = UserAPIKeyLabelNames.CALL_TYPE.value + + for metric_name in [ + "litellm_remaining_requests_metric", + "litellm_remaining_tokens_metric", + "litellm_deployment_tpm_limit", + "litellm_deployment_rpm_limit", + "litellm_deployment_state", + "litellm_deployment_cooled_down", + ]: + labels = PrometheusMetricLabels.get_labels(metric_name) + assert call_type_label not in labels, f"Metric {metric_name} should not carry call_type" + + +@pytest.mark.parametrize( + "raw, expected", + [ + ("acompletion", "completion"), + ("completion", "completion"), + ("aembedding", "embedding"), + ("aimage_generation", "image_generation"), + ("a_add_message", "add_message"), + ("aanthropic_messages", "anthropic_messages"), + ("add_message", "add_message"), + ("anthropic_messages", "anthropic_messages"), + ("something_new", "something_new"), + (None, None), + ("", None), + ], +) +def test_normalize_call_type(raw, expected): + from litellm.integrations.prometheus import PrometheusLogger + + assert PrometheusLogger._normalize_call_type(raw) == expected + + +def test_async_call_type_aliases_cover_every_async_member(): + from litellm.integrations.prometheus import PrometheusLogger + from litellm.types.utils import CallTypes + + names = {member.name: member.value for member in CallTypes} + for name, value in names.items(): + for prefix in ("a_", "a"): + twin = name[len(prefix) :] if name.startswith(prefix) else None + if twin and twin in names and twin != name: + assert PrometheusLogger._normalize_call_type(value) == names[twin] + break + + +def test_call_type_reaches_rendered_metrics_output(): + from litellm.integrations.prometheus import PrometheusLogger + + _clear_prometheus_registry() + logger = PrometheusLogger() + + standard_logging_payload = { + "model_group": "gpt-4o-group", + "call_type": "acompletion", + "api_base": "https://example.invalid", + "model_id": "deployment-1", + "request_tags": [], + "metadata": { + "user_api_key_hash": "test-key", + "user_api_key_alias": None, + "user_api_key_team_id": None, + "user_api_key_team_alias": None, + }, + } + + logger.set_llm_deployment_failure_metrics( + { + "model": "gpt-4o", + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": standard_logging_payload, + "exception": Exception("boom"), + } + ) + + samples = _collected_samples("litellm_deployment_failure_responses_total") + assert samples, "expected litellm_deployment_failure_responses to be emitted" + assert samples[0].labels.get("call_type") == "completion"