Skip to content

(testing) increase prometheus.py test coverage to 90% #6466

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ishaan-jaff merged 5 commits into from
Oct 28, 2024
Merged

(testing) increase prometheus.py test coverage to 90% #6466

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2ed749c
testing for failure events prometheus
ishaan-jaff Oct 28, 2024
bcbc8c7
set set_llm_deployment_failure_metrics
ishaan-jaff Oct 28, 2024
25b3964
test_async_post_call_failure_hook
ishaan-jaff Oct 28, 2024
c261557
unit testing for all prometheus functions
ishaan-jaff Oct 28, 2024
83a80b4
fix linting
ishaan-jaff Oct 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 30 additions & 17 deletions litellm/integrations/prometheus.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -397,7 +397,10 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti

# input, output, total token metrics
self._increment_token_metrics(
standard_logging_payload=standard_logging_payload,
# why type ignore below?
# 1. We just checked if isinstance(standard_logging_payload, dict). Pyright complains.
# 2. Pyright does not allow us to run isinstance(standard_logging_payload, StandardLoggingPayload) <- this would be ideal
standard_logging_payload=standard_logging_payload, # type: ignore
end_user_id=end_user_id,
user_api_key=user_api_key,
user_api_key_alias=user_api_key_alias,
Expand Down Expand Up @@ -432,7 +435,10 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti
user_api_key_alias=user_api_key_alias,
user_api_team=user_api_team,
user_api_team_alias=user_api_team_alias,
standard_logging_payload=standard_logging_payload,
# why type ignore below?
# 1. We just checked if isinstance(standard_logging_payload, dict). Pyright complains.
# 2. Pyright does not allow us to run isinstance(standard_logging_payload, StandardLoggingPayload) <- this would be ideal
standard_logging_payload=standard_logging_payload, # type: ignore
)

# set x-ratelimit headers
Expand Down Expand Up @@ -757,24 +763,31 @@ async def async_post_call_success_hook(
pass

def set_llm_deployment_failure_metrics(self, request_kwargs: dict):
"""
Sets Failure metrics when an LLM API call fails

- mark the deployment as partial outage
- increment deployment failure responses metric
- increment deployment total requests metric

Args:
request_kwargs: dict

"""
try:
verbose_logger.debug("setting remaining tokens requests metric")
standard_logging_payload: StandardLoggingPayload = request_kwargs.get(
"standard_logging_object", {}
)
_response_headers = request_kwargs.get("response_headers")
_litellm_params = request_kwargs.get("litellm_params", {}) or {}
_metadata = _litellm_params.get("metadata", {})
litellm_model_name = request_kwargs.get("model", None)
api_base = _metadata.get("api_base", None)
model_group = _metadata.get("model_group", None)
if api_base is None:
api_base = _litellm_params.get("api_base", None)
llm_provider = _litellm_params.get("custom_llm_provider", None)
_model_info = _metadata.get("model_info") or {}
model_id = _model_info.get("id", None)
model_group = standard_logging_payload.get("model_group", None)
api_base = standard_logging_payload.get("api_base", None)
model_id = standard_logging_payload.get("model_id", None)
exception: Exception = request_kwargs.get("exception", None)

llm_provider = _litellm_params.get("custom_llm_provider", None)

"""
log these labels
["litellm_model_name", "model_id", "api_base", "api_provider"]
Expand Down Expand Up @@ -1061,8 +1074,8 @@ def set_litellm_deployment_state(
self,
state: int,
litellm_model_name: str,
model_id: str,
api_base: str,
model_id: Optional[str],
api_base: Optional[str],
api_provider: str,
):
self.litellm_deployment_state.labels(
Expand All @@ -1083,8 +1096,8 @@ def set_deployment_healthy(
def set_deployment_partial_outage(
self,
litellm_model_name: str,
model_id: str,
api_base: str,
model_id: Optional[str],
api_base: Optional[str],
api_provider: str,
):
self.set_litellm_deployment_state(
Expand All @@ -1094,8 +1107,8 @@ def set_deployment_partial_outage(
def set_deployment_complete_outage(
self,
litellm_model_name: str,
model_id: str,
api_base: str,
model_id: Optional[str],
api_base: Optional[str],
api_provider: str,
):
self.set_litellm_deployment_state(
Expand Down
Loading