diff --git a/litellm/integrations/generic_api/generic_api_callback.py b/litellm/integrations/generic_api/generic_api_callback.py index 9a8060520d6b..2982df8fda23 100644 --- a/litellm/integrations/generic_api/generic_api_callback.py +++ b/litellm/integrations/generic_api/generic_api_callback.py @@ -11,8 +11,9 @@ import os import re import traceback -from typing import Dict, List, Literal, Optional, Union +from typing import Any, Dict, List, Literal, Optional, Union +import httpx import litellm from litellm._logging import verbose_logger from litellm._uuid import uuid @@ -103,6 +104,9 @@ def __init__( event_types: Optional[List[API_EVENT_TYPES]] = None, callback_name: Optional[str] = None, log_format: Optional[LOG_FORMAT_TYPES] = None, + max_retries: int = 0, + retry_delay: float = 1.0, + timeout: Optional[Union[float, httpx.Timeout]] = None, **kwargs, ): """ @@ -114,6 +118,9 @@ def __init__( event_types: Optional[List[API_EVENT_TYPES]] = None, callback_name: Optional[str] = None - If provided, loads config from generic_api_compatible_callbacks.json log_format: Optional[LOG_FORMAT_TYPES] = None - Format for log output: "json_array" (default), "ndjson", or "single" + max_retries: Number of retry attempts after the initial request fails. Defaults to 0. + retry_delay: Initial retry delay in seconds. Retries use exponential backoff. + timeout: Optional timeout to use for Generic API callback requests. """ ######################################################### # Check if callback_name is provided and load config @@ -162,6 +169,10 @@ def __init__( self.endpoint: str = endpoint self.event_types: Optional[List[API_EVENT_TYPES]] = event_types self.callback_name: Optional[str] = callback_name + self.max_retries = max(0, int(max_retries or 0)) + retry_delay_value = 0.0 if retry_delay is None else retry_delay + self.retry_delay = max(0.0, float(retry_delay_value)) + self.timeout = timeout # Validate and store log_format if log_format is not None and log_format not in [ @@ -226,6 +237,53 @@ def _get_headers(self, headers: Optional[dict] = None): return headers_dict + def _should_retry_exception(self, exception: Exception) -> bool: + if isinstance(exception, (litellm.Timeout, httpx.TransportError)): + return True + + if isinstance(exception, httpx.HTTPStatusError): + return exception.response.status_code >= 500 + + return False + + async def _sleep_before_retry(self, attempt: int) -> None: + if self.retry_delay <= 0: + return + + delay = self.retry_delay * (2**attempt) + await asyncio.sleep(delay) + + async def _post_with_retries(self, data: str) -> httpx.Response: + post_kwargs: Dict[str, Any] = { + "url": self.endpoint, + "headers": self.headers, + "data": data, + } + if self.timeout is not None: + post_kwargs["timeout"] = self.timeout + + total_attempts = self.max_retries + 1 + for attempt in range(total_attempts): + try: + return await self.async_httpx_client.post(**post_kwargs) + except Exception as e: + is_last_attempt = attempt == self.max_retries + should_retry = self._should_retry_exception(e) + if is_last_attempt or not should_retry: + raise + + verbose_logger.warning( + "Generic API Logger - retrying request to %s after error: %s " + "(attempt %s/%s)", + self.endpoint, + str(e), + attempt + 1, + total_attempts, + ) + await self._sleep_before_retry(attempt) + + raise RuntimeError("Generic API Logger retry loop exited unexpectedly") + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): """ Async Log success events to Generic API Endpoint @@ -325,11 +383,7 @@ async def async_send_batch(self): # Send each log as individual HTTP request in parallel tasks = [] for log_entry in self.log_queue: - task = self.async_httpx_client.post( - url=self.endpoint, - headers=self.headers, - data=safe_dumps(log_entry), - ) + task = self._post_with_retries(data=safe_dumps(log_entry)) tasks.append(task) # Execute all requests in parallel @@ -356,11 +410,7 @@ async def async_send_batch(self): raise ValueError(f"Unknown log_format: {self.log_format}") # Make POST request - response = await self.async_httpx_client.post( - url=self.endpoint, - headers=self.headers, - data=data, - ) + response = await self._post_with_retries(data=data) verbose_logger.debug( f"Generic API Logger - sent batch to {self.endpoint}, " diff --git a/litellm/litellm_core_utils/logging_callback_manager.py b/litellm/litellm_core_utils/logging_callback_manager.py index c5c150274cc3..6c749118dec4 100644 --- a/litellm/litellm_core_utils/logging_callback_manager.py +++ b/litellm/litellm_core_utils/logging_callback_manager.py @@ -221,6 +221,13 @@ def _add_custom_callback_generic_api_str( headers = callback_config.get("headers") event_types = callback_config.get("event_types") log_format = callback_config.get("log_format") + max_retries = max(0, int(callback_config.get("max_retries", 0) or 0)) + retry_delay_value = callback_config.get("retry_delay") + retry_delay = max( + 0.0, + float(0.0 if retry_delay_value is None else retry_delay_value), + ) + timeout = callback_config.get("timeout") if endpoint is None or headers is None: verbose_logger.warning( @@ -236,6 +243,9 @@ def _add_custom_callback_generic_api_str( and cached_logger.headers == headers and cached_logger.event_types == event_types and cached_logger.log_format == log_format + and cached_logger.max_retries == max_retries + and cached_logger.retry_delay == retry_delay + and cached_logger.timeout == timeout ): return cached_logger @@ -244,6 +254,9 @@ def _add_custom_callback_generic_api_str( headers=headers, event_types=event_types, log_format=log_format, + max_retries=max_retries, + retry_delay=retry_delay, + timeout=timeout, ) _generic_api_logger_cache[callback] = new_logger return new_logger diff --git a/tests/litellm_utils_tests/test_logging_callback_manager.py b/tests/litellm_utils_tests/test_logging_callback_manager.py index 88ae07fd81a5..d9540f8f850b 100644 --- a/tests/litellm_utils_tests/test_logging_callback_manager.py +++ b/tests/litellm_utils_tests/test_logging_callback_manager.py @@ -366,3 +366,40 @@ def test_generic_api_compatible_callbacks_json_unknown_callback(): # Should return the string unchanged assert result == "unknown_callback", "Unknown callback should be returned as-is" assert isinstance(result, str), "Unknown callback should remain a string" + + +@pytest.mark.asyncio +async def test_generic_api_callback_settings_retry_config(): + """ + Test that generic_api callback_settings are passed to GenericAPILogger. + """ + from litellm.integrations.generic_api.generic_api_callback import GenericAPILogger + from litellm.litellm_core_utils.logging_callback_manager import ( + _generic_api_logger_cache, + ) + + callback_name = "test_generic_api_retry_config" + _generic_api_logger_cache.pop(callback_name, None) + litellm.callback_settings[callback_name] = { + "callback_type": "generic_api", + "endpoint": "https://example.com/api/logs", + "headers": {"Content-Type": "application/json"}, + "max_retries": 2, + "retry_delay": 0.5, + "timeout": 3, + } + + try: + result = LoggingCallbackManager._add_custom_callback_generic_api_str( + callback_name + ) + + assert isinstance(result, GenericAPILogger) + assert result.endpoint == "https://example.com/api/logs" + assert result.headers == {"Content-Type": "application/json"} + assert result.max_retries == 2 + assert result.retry_delay == 0.5 + assert result.timeout == 3 + finally: + litellm.callback_settings.pop(callback_name, None) + _generic_api_logger_cache.pop(callback_name, None) diff --git a/tests/logging_callback_tests/test_generic_api_callback.py b/tests/logging_callback_tests/test_generic_api_callback.py index 528a5101df67..6984b6fa00c6 100644 --- a/tests/logging_callback_tests/test_generic_api_callback.py +++ b/tests/logging_callback_tests/test_generic_api_callback.py @@ -8,6 +8,7 @@ import asyncio import litellm import gzip +import httpx import json import logging import time @@ -470,3 +471,96 @@ async def test_generic_api_callback_invalid_log_format(): endpoint=test_endpoint, log_format="invalid_format", # type: ignore # Intentionally invalid for testing ) + + +@pytest.mark.asyncio +async def test_generic_api_callback_retries_timeout_then_succeeds(): + """ + Test that GenericAPILogger retries LiteLLM timeout errors when configured. + """ + test_endpoint = "https://example.com/api/logs" + generic_logger = GenericAPILogger( + endpoint=test_endpoint, + max_retries=1, + retry_delay=0, + timeout=0.2, + ) + + mock_post = AsyncMock() + mock_post.side_effect = [ + litellm.Timeout( + message="Connection timed out", + model="default-model-name", + llm_provider="litellm-httpx-handler", + ), + type("Response", (), {"status_code": 200})(), + ] + generic_logger.async_httpx_client.post = mock_post + generic_logger.log_queue = [{"event": "timeout-retry"}] + + await generic_logger.async_send_batch() + + assert mock_post.call_count == 2 + first_call = mock_post.call_args_list[0][1] + assert first_call["url"] == test_endpoint + assert first_call["timeout"] == 0.2 + assert json.loads(first_call["data"]) == [{"event": "timeout-retry"}] + + +@pytest.mark.asyncio +async def test_generic_api_callback_retries_5xx_then_succeeds(): + """ + Test that GenericAPILogger retries transient HTTP 5xx errors when configured. + """ + test_endpoint = "https://example.com/api/logs" + generic_logger = GenericAPILogger( + endpoint=test_endpoint, + max_retries=1, + retry_delay=0, + ) + + request = httpx.Request("POST", test_endpoint) + response = httpx.Response(status_code=503, request=request) + mock_post = AsyncMock() + mock_post.side_effect = [ + httpx.HTTPStatusError( + "Server error", + request=request, + response=response, + ), + type("Response", (), {"status_code": 200})(), + ] + generic_logger.async_httpx_client.post = mock_post + generic_logger.log_queue = [{"event": "5xx-retry"}] + + await generic_logger.async_send_batch() + + assert mock_post.call_count == 2 + + +@pytest.mark.asyncio +async def test_generic_api_callback_does_not_retry_4xx(): + """ + Test that GenericAPILogger does not retry non-transient HTTP 4xx errors. + """ + test_endpoint = "https://example.com/api/logs" + generic_logger = GenericAPILogger( + endpoint=test_endpoint, + max_retries=2, + retry_delay=0, + ) + + request = httpx.Request("POST", test_endpoint) + response = httpx.Response(status_code=401, request=request) + mock_post = AsyncMock() + mock_post.side_effect = httpx.HTTPStatusError( + "Unauthorized", + request=request, + response=response, + ) + generic_logger.async_httpx_client.post = mock_post + generic_logger.log_queue = [{"event": "4xx-no-retry"}] + + await generic_logger.async_send_batch() + + mock_post.assert_called_once()