From 560a6cf39f3bad3b33730651f11ce22431be9784 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 31 Oct 2024 18:15:35 +0530 Subject: [PATCH 1/4] fix use failing_model as cache key for failed_tracking_alert --- litellm/integrations/SlackAlerting/slack_alerting.py | 12 +++++++++--- litellm/proxy/proxy_config.yaml | 10 +++++----- litellm/proxy/proxy_server.py | 1 + litellm/proxy/utils.py | 9 +++++++-- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index dbe9e4161da1..85d54a337d16 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -550,8 +550,14 @@ async def response_taking_too_long( alerting_metadata=alerting_metadata, ) - async def failed_tracking_alert(self, error_message: str): - """Raise alert when tracking failed for specific model""" + async def failed_tracking_alert(self, error_message: str, failing_model: str): + """ + Raise alert when tracking failed for specific model + + Args: + error_message (str): Error message + failing_model (str): Model that failed tracking + """ if self.alerting is None or self.alert_types is None: # do nothing if alerting is not switched on return @@ -560,7 +566,7 @@ async def failed_tracking_alert(self, error_message: str): _cache: DualCache = self.internal_usage_cache message = "Failed Tracking Cost for " + error_message - _cache_key = "budget_alerts:failed_tracking:{}".format(message) + _cache_key = "budget_alerts:failed_tracking:{}".format(failing_model) result = await _cache.async_get_cache(key=_cache_key) if result is None: await self.send_alert( diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 95eff095c22e..23834f759701 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,11 +1,11 @@ model_list: - model_name: gpt-4o litellm_params: - model: gpt-4o + model: openai/gpt-5 api_key: os.environ/OPENAI_API_KEY api_base: https://exampleopenaiendpoint-production.up.railway.app/ -litellm_settings: - callbacks: ["prometheus"] - service_callback: ["prometheus_system"] - cache: true + +general_settings: + alerting: ["slack"] + alerting_threshold: 0.001 diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1f0271f34517..402063177a68 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -845,6 +845,7 @@ async def _PROXY_track_cost_callback( asyncio.create_task( proxy_logging_obj.failed_tracking_alert( error_message=error_msg, + failing_model=model, ) ) verbose_proxy_logger.debug("error in tracking cost callback - %s", e) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 656e97a6c5ce..8919da978618 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -667,13 +667,18 @@ async def during_call_hook( raise e return data - async def failed_tracking_alert(self, error_message: str): + async def failed_tracking_alert( + self, + error_message: str, + failing_model: str, + ): if self.alerting is None: return if self.slack_alerting_instance: await self.slack_alerting_instance.failed_tracking_alert( - error_message=error_message + error_message=error_message, + failing_model=failing_model, ) async def budget_alerts( From ea7b8bc6b406f38b83324e9463ef8c9e22e00d19 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 31 Oct 2024 18:30:58 +0530 Subject: [PATCH 2/4] fix use standard logging payload for getting response cost --- litellm/proxy/proxy_server.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 402063177a68..b871df6f9f8a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -265,6 +265,7 @@ def generate_feedback_box(): ) from litellm.types.llms.openai import HttpxBinaryResponseContent from litellm.types.router import RouterGeneralSettings +from litellm.types.utils import StandardLoggingPayload try: from litellm._version import version @@ -775,10 +776,14 @@ async def _PROXY_track_cost_callback( org_id = metadata.get("user_api_key_org_id", None) key_alias = metadata.get("user_api_key_alias", None) end_user_max_budget = metadata.get("user_api_end_user_max_budget", None) - if kwargs.get("response_cost", None) is not None: - response_cost = kwargs["response_cost"] + standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get( + "standard_logging_object", None + ) + if standard_logging_payload is None: + raise ValueError("Standard logging payload is missing from kwargs") + response_cost = standard_logging_payload.get("response_cost") + if response_cost is not None: user_api_key = metadata.get("user_api_key", None) - if kwargs.get("cache_hit", False) is True: response_cost = 0.0 verbose_proxy_logger.info( @@ -838,7 +843,7 @@ async def _PROXY_track_cost_callback( f"Cost tracking failed for model={model}.\nDebug info - {cost_tracking_failure_debug_info}\nAdd custom pricing - https://docs.litellm.ai/docs/proxy/custom_pricing" ) except Exception as e: - error_msg = f"error in tracking cost callback - {traceback.format_exc()}" + error_msg = f"Error in tracking cost callback - {str(e)}\n Traceback:{traceback.format_exc()}" model = kwargs.get("model", "") metadata = kwargs.get("litellm_params", {}).get("metadata", {}) error_msg += f"\n Args to _PROXY_track_cost_callback\n model: {model}\n metadata: {metadata}\n" From 60d65ac791e44afc581583b7b2c04db4270d0d71 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 31 Oct 2024 18:53:58 +0530 Subject: [PATCH 3/4] fix kwargs.get("response_cost") --- litellm/proxy/proxy_server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b871df6f9f8a..f888d094e2d5 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -781,7 +781,9 @@ async def _PROXY_track_cost_callback( ) if standard_logging_payload is None: raise ValueError("Standard logging payload is missing from kwargs") - response_cost = standard_logging_payload.get("response_cost") + response_cost = standard_logging_payload.get("response_cost") or kwargs.get( + "response_cost", None + ) # read from kwargs for backwards compatibility with existing tests if response_cost is not None: user_api_key = metadata.get("user_api_key", None) if kwargs.get("cache_hit", False) is True: From 4e9d183fbf14c6a19702e6d09d643a0a1f5f980d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 31 Oct 2024 20:07:14 +0530 Subject: [PATCH 4/4] fix getting response cost --- litellm/proxy/proxy_server.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f888d094e2d5..ca6befef603b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -776,15 +776,8 @@ async def _PROXY_track_cost_callback( org_id = metadata.get("user_api_key_org_id", None) key_alias = metadata.get("user_api_key_alias", None) end_user_max_budget = metadata.get("user_api_end_user_max_budget", None) - standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get( - "standard_logging_object", None - ) - if standard_logging_payload is None: - raise ValueError("Standard logging payload is missing from kwargs") - response_cost = standard_logging_payload.get("response_cost") or kwargs.get( - "response_cost", None - ) # read from kwargs for backwards compatibility with existing tests - if response_cost is not None: + if kwargs.get("response_cost", None) is not None: + response_cost = kwargs["response_cost"] user_api_key = metadata.get("user_api_key", None) if kwargs.get("cache_hit", False) is True: response_cost = 0.0