LiteLLM Minor Fixes & Improvements (10/28/2024)

docs/my-website/docs/exception_mapping.md

@@ -2,18 +2,33 @@
 
 LiteLLM maps exceptions across all providers to their OpenAI counterparts.
 
-| Status Code | Error Type               |
-|-------------|--------------------------|
-| 400         | BadRequestError          |
-| 401         | AuthenticationError      |
-| 403         | PermissionDeniedError    |
-| 404         | NotFoundError            |
-| 422         | UnprocessableEntityError |
-| 429         | RateLimitError           |
-| >=500       | InternalServerError      |
-| N/A         | ContextWindowExceededError|
-| 400         | ContentPolicyViolationError|
-| 500         | APIConnectionError       |
+All exceptions can be imported from `litellm` - e.g. `from litellm import BadRequestError`
+
+## LiteLLM Exceptions
+
+| Status Code | Error Type               | Inherits from | Description |
+|-------------|--------------------------|---------------|-------------|
+| 400         | BadRequestError          | openai.BadRequestError |
+| 400 | UnsupportedParamsError | litellm.BadRequestError | Raised when unsupported params are passed |
+| 400         | ContextWindowExceededError| litellm.BadRequestError | Special error type for context window exceeded error messages - enables context window fallbacks |
+| 400         | ContentPolicyViolationError| litellm.BadRequestError | Special error type for content policy violation error messages - enables content policy fallbacks |
+| 400 | InvalidRequestError | openai.BadRequestError | Deprecated error, use BadRequestError instead |
+| 401         | AuthenticationError      | openai.AuthenticationError |
+| 403         | PermissionDeniedError    | openai.PermissionDeniedError |
+| 404         | NotFoundError            | openai.NotFoundError | raise when invalid models passed, example gpt-8 |
+| 408 | Timeout | openai.APITimeoutError | Raised when a timeout occurs |
+| 422         | UnprocessableEntityError | openai.UnprocessableEntityError |
+| 429         | RateLimitError           | openai.RateLimitError |
+| 500         | APIConnectionError       | openai.APIConnectionError | If any unmapped error is returned, we return this error |
+| 500         | APIError | openai.APIError | Generic 500-status code error | 
+| 503 | ServiceUnavailableError | openai.APIStatusError | If provider returns a service unavailable error, this error is raised |
+| >=500       | InternalServerError      | openai.InternalServerError | If any unmapped 500-status code error is returned, this error is raised |
+| N/A         | APIResponseValidationError | openai.APIResponseValidationError | If Rules are used, and request/response fails a rule, this error is raised |
+| N/A | BudgetExceededError | Exception | Raised for proxy, when budget is exceeded |
+| N/A | JSONSchemaValidationError | litellm.APIResponseValidationError | Raised when response does not match expected json schema - used if `response_schema` param passed in with `enforce_validation=True` |
+| N/A | MockException | Exception | Internal exception, raised by mock_completion class. Do not use directly | 
+| N/A | OpenAIError | openai.OpenAIError | Deprecated internal exception, inherits from openai.OpenAIError. |
+
 
 
 Base case we return APIConnectionError

docs/my-website/docs/providers/anthropic.md

@@ -721,6 +721,37 @@ except Exception as e:
 
 s/o @[Shekhar Patnaik](https://www.linkedin.com/in/patnaikshekhar) for requesting this!
 
+### Computer Tools
+
+```python
+from litellm import completion
+
+tools = [
+    {
+        "type": "computer_20241022",
+        "function": {
+            "name": "computer",
+            "parameters": {
+                "display_height_px": 100,
+                "display_width_px": 100,
+                "display_number": 1,
+            },
+        },
+    }
+]
+model = "claude-3-5-sonnet-20241022"
+messages = [{"role": "user", "content": "Save a picture of a cat to my desktop."}]
+
+resp = completion(
+    model=model,
+    messages=messages,
+    tools=tools,
+    # headers={"anthropic-beta": "computer-use-2024-10-22"},
+)
+
+print(resp)
+```
+
 ## Usage - Vision 
 
 ```python

litellm/_service_logger.py

@@ -1,3 +1,4 @@
+import asyncio
 from datetime import datetime, timedelta
 from typing import TYPE_CHECKING, Any, Optional, Union
 
@@ -32,14 +33,62 @@ def __init__(self, mock_testing: bool = False) -> None:
             self.prometheusServicesLogger = PrometheusServicesLogger()
 
     def service_success_hook(
-        self, service: ServiceTypes, duration: float, call_type: str
+        self,
+        service: ServiceTypes,
+        duration: float,
+        call_type: str,
+        parent_otel_span: Optional[Span] = None,
+        start_time: Optional[Union[datetime, float]] = None,
+        end_time: Optional[Union[float, datetime]] = None,
     ):
         """
-        [TODO] Not implemented for sync calls yet. V0 is focused on async monitoring (used by proxy).
+        Handles both sync and async monitoring by checking for existing event loop.
         """
+
         if self.mock_testing:
             self.mock_testing_sync_success_hook += 1
 
+        try:
+            # Try to get the current event loop
+            loop = asyncio.get_event_loop()
+            # Check if the loop is running
+            if loop.is_running():
+                # If we're in a running loop, create a task
+                loop.create_task(
+                    self.async_service_success_hook(
+                        service=service,
+                        duration=duration,
+                        call_type=call_type,
+                        parent_otel_span=parent_otel_span,
+                        start_time=start_time,
+                        end_time=end_time,
+                    )
+                )
+            else:
+                # Loop exists but not running, we can use run_until_complete
+                loop.run_until_complete(
+                    self.async_service_success_hook(
+                        service=service,
+                        duration=duration,
+                        call_type=call_type,
+                        parent_otel_span=parent_otel_span,
+                        start_time=start_time,
+                        end_time=end_time,
+                    )
+                )
+        except RuntimeError:
+            # No event loop exists, create a new one and run
+            asyncio.run(
+                self.async_service_success_hook(
+                    service=service,
+                    duration=duration,
+                    call_type=call_type,
+                    parent_otel_span=parent_otel_span,
+                    start_time=start_time,
+                    end_time=end_time,
+                )
+            )
+
     def service_failure_hook(
         self, service: ServiceTypes, duration: float, error: Exception, call_type: str
     ):
@@ -62,6 +111,7 @@ async def async_service_success_hook(
         """
         - For counting if the redis, postgres call is successful
         """
+
         if self.mock_testing:
             self.mock_testing_async_success_hook += 1
 

litellm/caching/base_cache.py

@@ -8,7 +8,14 @@
     - async_get_cache
 """
 
-from typing import Optional
+from typing import TYPE_CHECKING, Any, Optional
+
+if TYPE_CHECKING:
+    from opentelemetry.trace import Span as _Span
+
+    Span = _Span
+else:
+    Span = Any
 
 
 class BaseCache:

litellm/caching/disk_cache.py

@@ -1,10 +1,17 @@
 import json
-from typing import Optional
+from typing import TYPE_CHECKING, Any, Optional
 
 from litellm._logging import print_verbose
 
 from .base_cache import BaseCache
 
+if TYPE_CHECKING:
+    from opentelemetry.trace import Span as _Span
+
+    Span = _Span
+else:
+    Span = Any
+
 
 class DiskCache(BaseCache):
     def __init__(self, disk_cache_dir: Optional[str] = None):

litellm/caching/dual_cache.py

@@ -9,7 +9,7 @@
 """
 
 import traceback
-from typing import List, Optional
+from typing import TYPE_CHECKING, Any, List, Optional
 
 import litellm
 from litellm._logging import print_verbose, verbose_logger
@@ -18,6 +18,13 @@
 from .in_memory_cache import InMemoryCache
 from .redis_cache import RedisCache
 
+if TYPE_CHECKING:
+    from opentelemetry.trace import Span as _Span
+
+    Span = _Span
+else:
+    Span = Any
+
 
 class DualCache(BaseCache):
     """
@@ -90,7 +97,13 @@ def increment_cache(
             verbose_logger.error(f"LiteLLM Cache: Excepton async add_cache: {str(e)}")
             raise e
 
-    def get_cache(self, key, local_only: bool = False, **kwargs):
+    def get_cache(
+        self,
+        key,
+        parent_otel_span: Optional[Span] = None,
+        local_only: bool = False,
+        **kwargs,
+    ):
         # Try to fetch from in-memory cache first
         try:
             result = None
@@ -102,7 +115,9 @@ def get_cache(self, key, local_only: bool = False, **kwargs):
 
             if result is None and self.redis_cache is not None and local_only is False:
                 # If not found in in-memory cache, try fetching from Redis
-                redis_result = self.redis_cache.get_cache(key, **kwargs)
+                redis_result = self.redis_cache.get_cache(
+                    key, parent_otel_span=parent_otel_span
+                )
 
                 if redis_result is not None:
                     # Update in-memory cache with the value from Redis
@@ -115,7 +130,13 @@ def get_cache(self, key, local_only: bool = False, **kwargs):
         except Exception:
             verbose_logger.error(traceback.format_exc())
 
-    def batch_get_cache(self, keys: list, local_only: bool = False, **kwargs):
+    def batch_get_cache(
+        self,
+        keys: list,
+        parent_otel_span: Optional[Span],
+        local_only: bool = False,
+        **kwargs,
+    ):
         try:
             result = [None for _ in range(len(keys))]
             if self.in_memory_cache is not None:
@@ -133,7 +154,9 @@ def batch_get_cache(self, keys: list, local_only: bool = False, **kwargs):
                     key for key, value in zip(keys, result) if value is None
                 ]
                 # If not found in in-memory cache, try fetching from Redis
-                redis_result = self.redis_cache.batch_get_cache(sublist_keys, **kwargs)
+                redis_result = self.redis_cache.batch_get_cache(
+                    sublist_keys, parent_otel_span=parent_otel_span
+                )
                 if redis_result is not None:
                     # Update in-memory cache with the value from Redis
                     for key in redis_result:
@@ -147,7 +170,13 @@ def batch_get_cache(self, keys: list, local_only: bool = False, **kwargs):
         except Exception:
             verbose_logger.error(traceback.format_exc())
 
-    async def async_get_cache(self, key, local_only: bool = False, **kwargs):
+    async def async_get_cache(
+        self,
+        key,
+        parent_otel_span: Optional[Span] = None,
+        local_only: bool = False,
+        **kwargs,
+    ):
         # Try to fetch from in-memory cache first
         try:
             print_verbose(
@@ -165,7 +194,9 @@ async def async_get_cache(self, key, local_only: bool = False, **kwargs):
 
             if result is None and self.redis_cache is not None and local_only is False:
                 # If not found in in-memory cache, try fetching from Redis
-                redis_result = await self.redis_cache.async_get_cache(key, **kwargs)
+                redis_result = await self.redis_cache.async_get_cache(
+                    key, parent_otel_span=parent_otel_span
+                )
 
                 if redis_result is not None:
                     # Update in-memory cache with the value from Redis
@@ -181,7 +212,11 @@ async def async_get_cache(self, key, local_only: bool = False, **kwargs):
             verbose_logger.error(traceback.format_exc())
 
     async def async_batch_get_cache(
-        self, keys: list, local_only: bool = False, **kwargs
+        self,
+        keys: list,
+        parent_otel_span: Optional[Span] = None,
+        local_only: bool = False,
+        **kwargs,
     ):
         try:
             result = [None for _ in range(len(keys))]
@@ -202,7 +237,7 @@ async def async_batch_get_cache(
                 ]
                 # If not found in in-memory cache, try fetching from Redis
                 redis_result = await self.redis_cache.async_batch_get_cache(
-                    sublist_keys, **kwargs
+                    sublist_keys, parent_otel_span=parent_otel_span
                 )
 
                 if redis_result is not None:
@@ -260,7 +295,12 @@ async def async_batch_set_cache(
             )
 
     async def async_increment_cache(
-        self, key, value: float, local_only: bool = False, **kwargs
+        self,
+        key,
+        value: float,
+        parent_otel_span: Optional[Span],
+        local_only: bool = False,
+        **kwargs,
     ) -> float:
         """
         Key - the key in cache
@@ -277,7 +317,12 @@ async def async_increment_cache(
                 )
 
             if self.redis_cache is not None and local_only is False:
-                result = await self.redis_cache.async_increment(key, value, **kwargs)
+                result = await self.redis_cache.async_increment(
+                    key,
+                    value,
+                    parent_otel_span=parent_otel_span,
+                    ttl=kwargs.get("ttl", None),
+                )
 
             return result
         except Exception as e:
@@ -303,7 +348,7 @@ async def async_set_cache_sadd(
 
             if self.redis_cache is not None and local_only is False:
                 _ = await self.redis_cache.async_set_cache_sadd(
-                    key, value, ttl=kwargs.get("ttl", None) ** kwargs
+                    key, value, ttl=kwargs.get("ttl", None)
                 )
 
             return None

litellm/caching/in_memory_cache.py

@@ -63,7 +63,7 @@ def set_cache(self, key, value, **kwargs):
             self.evict_cache()
 
         self.cache_dict[key] = value
-        if "ttl" in kwargs:
+        if "ttl" in kwargs and kwargs["ttl"] is not None:
             self.ttl_dict[key] = time.time() + kwargs["ttl"]
         else:
             self.ttl_dict[key] = time.time() + self.default_ttl