Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 61 additions & 11 deletions litellm/integrations/generic_api/generic_api_callback.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,9 @@
import os
import re
import traceback
from typing import Dict, List, Literal, Optional, Union
from typing import Any, Dict, List, Literal, Optional, Union

import httpx
import litellm
from litellm._logging import verbose_logger
from litellm._uuid import uuid
Expand Down Expand Up @@ -103,6 +104,9 @@ def __init__(
event_types: Optional[List[API_EVENT_TYPES]] = None,
callback_name: Optional[str] = None,
log_format: Optional[LOG_FORMAT_TYPES] = None,
max_retries: int = 0,
retry_delay: float = 1.0,
timeout: Optional[Union[float, httpx.Timeout]] = None,
**kwargs,
):
"""
Expand All @@ -114,6 +118,9 @@ def __init__(
event_types: Optional[List[API_EVENT_TYPES]] = None,
callback_name: Optional[str] = None - If provided, loads config from generic_api_compatible_callbacks.json
log_format: Optional[LOG_FORMAT_TYPES] = None - Format for log output: "json_array" (default), "ndjson", or "single"
max_retries: Number of retry attempts after the initial request fails. Defaults to 0.
retry_delay: Initial retry delay in seconds. Retries use exponential backoff.
timeout: Optional timeout to use for Generic API callback requests.
"""
#########################################################
# Check if callback_name is provided and load config
Expand Down Expand Up @@ -162,6 +169,10 @@ def __init__(
self.endpoint: str = endpoint
self.event_types: Optional[List[API_EVENT_TYPES]] = event_types
self.callback_name: Optional[str] = callback_name
self.max_retries = max(0, int(max_retries or 0))
retry_delay_value = 0.0 if retry_delay is None else retry_delay
self.retry_delay = max(0.0, float(retry_delay_value))
self.timeout = timeout

# Validate and store log_format
if log_format is not None and log_format not in [
Expand Down Expand Up @@ -226,6 +237,53 @@ def _get_headers(self, headers: Optional[dict] = None):

return headers_dict

def _should_retry_exception(self, exception: Exception) -> bool:
if isinstance(exception, (litellm.Timeout, httpx.TransportError)):
return True

if isinstance(exception, httpx.HTTPStatusError):
Comment thread
milan-berri marked this conversation as resolved.
Resolved
return exception.response.status_code >= 500

return False

async def _sleep_before_retry(self, attempt: int) -> None:
if self.retry_delay <= 0:
return

delay = self.retry_delay * (2**attempt)
await asyncio.sleep(delay)

async def _post_with_retries(self, data: str) -> httpx.Response:
post_kwargs: Dict[str, Any] = {
"url": self.endpoint,
"headers": self.headers,
"data": data,
}
if self.timeout is not None:
Comment thread
milan-berri marked this conversation as resolved.
Resolved
post_kwargs["timeout"] = self.timeout

total_attempts = self.max_retries + 1
for attempt in range(total_attempts):
try:
return await self.async_httpx_client.post(**post_kwargs)
except Exception as e:
is_last_attempt = attempt == self.max_retries
should_retry = self._should_retry_exception(e)
if is_last_attempt or not should_retry:
raise

verbose_logger.warning(
"Generic API Logger - retrying request to %s after error: %s "
"(attempt %s/%s)",
self.endpoint,
str(e),
attempt + 1,
total_attempts,
)
await self._sleep_before_retry(attempt)

raise RuntimeError("Generic API Logger retry loop exited unexpectedly")

async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
"""
Async Log success events to Generic API Endpoint
Expand Down Expand Up @@ -325,11 +383,7 @@ async def async_send_batch(self):
# Send each log as individual HTTP request in parallel
tasks = []
for log_entry in self.log_queue:
task = self.async_httpx_client.post(
url=self.endpoint,
headers=self.headers,
data=safe_dumps(log_entry),
)
task = self._post_with_retries(data=safe_dumps(log_entry))
tasks.append(task)

# Execute all requests in parallel
Expand All @@ -356,11 +410,7 @@ async def async_send_batch(self):
raise ValueError(f"Unknown log_format: {self.log_format}")

# Make POST request
response = await self.async_httpx_client.post(
url=self.endpoint,
headers=self.headers,
data=data,
)
response = await self._post_with_retries(data=data)

verbose_logger.debug(
f"Generic API Logger - sent batch to {self.endpoint}, "
Expand Down
13 changes: 13 additions & 0 deletions litellm/litellm_core_utils/logging_callback_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,13 @@ def _add_custom_callback_generic_api_str(
headers = callback_config.get("headers")
event_types = callback_config.get("event_types")
log_format = callback_config.get("log_format")
max_retries = max(0, int(callback_config.get("max_retries", 0) or 0))
retry_delay_value = callback_config.get("retry_delay")
retry_delay = max(
0.0,
float(0.0 if retry_delay_value is None else retry_delay_value),
)
timeout = callback_config.get("timeout")

if endpoint is None or headers is None:
verbose_logger.warning(
Expand All @@ -236,6 +243,9 @@ def _add_custom_callback_generic_api_str(
and cached_logger.headers == headers
and cached_logger.event_types == event_types
and cached_logger.log_format == log_format
and cached_logger.max_retries == max_retries
and cached_logger.retry_delay == retry_delay
and cached_logger.timeout == timeout
):
return cached_logger

Expand All @@ -244,6 +254,9 @@ def _add_custom_callback_generic_api_str(
headers=headers,
event_types=event_types,
log_format=log_format,
max_retries=max_retries,
retry_delay=retry_delay,
timeout=timeout,
)
_generic_api_logger_cache[callback] = new_logger
return new_logger
Expand Down
37 changes: 37 additions & 0 deletions tests/litellm_utils_tests/test_logging_callback_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -366,3 +366,40 @@ def test_generic_api_compatible_callbacks_json_unknown_callback():
# Should return the string unchanged
assert result == "unknown_callback", "Unknown callback should be returned as-is"
assert isinstance(result, str), "Unknown callback should remain a string"


@pytest.mark.asyncio
async def test_generic_api_callback_settings_retry_config():
"""
Test that generic_api callback_settings are passed to GenericAPILogger.
"""
from litellm.integrations.generic_api.generic_api_callback import GenericAPILogger
from litellm.litellm_core_utils.logging_callback_manager import (
_generic_api_logger_cache,
)

callback_name = "test_generic_api_retry_config"
_generic_api_logger_cache.pop(callback_name, None)
litellm.callback_settings[callback_name] = {
"callback_type": "generic_api",
"endpoint": "https://example.com/api/logs",
"headers": {"Content-Type": "application/json"},
"max_retries": 2,
"retry_delay": 0.5,
"timeout": 3,
}

try:
result = LoggingCallbackManager._add_custom_callback_generic_api_str(
callback_name
)

assert isinstance(result, GenericAPILogger)
assert result.endpoint == "https://example.com/api/logs"
assert result.headers == {"Content-Type": "application/json"}
assert result.max_retries == 2
assert result.retry_delay == 0.5
assert result.timeout == 3
finally:
litellm.callback_settings.pop(callback_name, None)
_generic_api_logger_cache.pop(callback_name, None)
94 changes: 94 additions & 0 deletions tests/logging_callback_tests/test_generic_api_callback.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import asyncio
import litellm
import gzip
import httpx
import json
import logging
import time
Expand Down Expand Up @@ -470,3 +471,96 @@ async def test_generic_api_callback_invalid_log_format():
endpoint=test_endpoint,
log_format="invalid_format", # type: ignore # Intentionally invalid for testing
)


@pytest.mark.asyncio
async def test_generic_api_callback_retries_timeout_then_succeeds():
"""
Test that GenericAPILogger retries LiteLLM timeout errors when configured.
"""
test_endpoint = "https://example.com/api/logs"
generic_logger = GenericAPILogger(
endpoint=test_endpoint,
max_retries=1,
retry_delay=0,
timeout=0.2,
)

mock_post = AsyncMock()
mock_post.side_effect = [
litellm.Timeout(
message="Connection timed out",
model="default-model-name",
llm_provider="litellm-httpx-handler",
),
type("Response", (), {"status_code": 200})(),
]
generic_logger.async_httpx_client.post = mock_post
generic_logger.log_queue = [{"event": "timeout-retry"}]

await generic_logger.async_send_batch()

assert mock_post.call_count == 2
first_call = mock_post.call_args_list[0][1]
assert first_call["url"] == test_endpoint
assert first_call["timeout"] == 0.2
assert json.loads(first_call["data"]) == [{"event": "timeout-retry"}]


@pytest.mark.asyncio
async def test_generic_api_callback_retries_5xx_then_succeeds():
"""
Test that GenericAPILogger retries transient HTTP 5xx errors when configured.
"""
test_endpoint = "https://example.com/api/logs"
generic_logger = GenericAPILogger(
endpoint=test_endpoint,
max_retries=1,
retry_delay=0,
)

request = httpx.Request("POST", test_endpoint)
response = httpx.Response(status_code=503, request=request)
mock_post = AsyncMock()
mock_post.side_effect = [
httpx.HTTPStatusError(
"Server error",
request=request,
response=response,
),
type("Response", (), {"status_code": 200})(),
]
generic_logger.async_httpx_client.post = mock_post
generic_logger.log_queue = [{"event": "5xx-retry"}]

await generic_logger.async_send_batch()

assert mock_post.call_count == 2


@pytest.mark.asyncio
async def test_generic_api_callback_does_not_retry_4xx():
"""
Test that GenericAPILogger does not retry non-transient HTTP 4xx errors.
"""
test_endpoint = "https://example.com/api/logs"
generic_logger = GenericAPILogger(
endpoint=test_endpoint,
max_retries=2,
retry_delay=0,
)

request = httpx.Request("POST", test_endpoint)
response = httpx.Response(status_code=401, request=request)
mock_post = AsyncMock()
mock_post.side_effect = httpx.HTTPStatusError(
"Unauthorized",
request=request,
response=response,
)
generic_logger.async_httpx_client.post = mock_post
generic_logger.log_queue = [{"event": "4xx-no-retry"}]

await generic_logger.async_send_batch()

mock_post.assert_called_once()
Loading