LiteLLM Minor Fixes & Improvements (10/30/2024)

.gitignore

@@ -65,3 +65,4 @@ litellm/tests/log.txt
 litellm/tests/langfuse.log
 litellm/tests/langfuse.log
 litellm/proxy/google-cloud-sdk/*
+tests/llm_translation/log.txt

docs/my-website/docs/providers/xai.md

@@ -0,0 +1,146 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# XAI
+
+https://docs.x.ai/docs
+
+:::tip
+
+**We support ALL XAI models, just set `model=xai/<any-model-on-xai>` as a prefix when sending litellm requests**
+
+:::
+
+## API Key
+```python
+# env variable
+os.environ['XAI_API_KEY']
+```
+
+## Sample Usage
+```python
+from litellm import completion
+import os
+
+os.environ['XAI_API_KEY'] = ""
+response = completion(
+    model="xai/grok-beta",
+    messages=[
+        {
+            "role": "user",
+            "content": "What's the weather like in Boston today in Fahrenheit?",
+        }
+    ],
+    max_tokens=10,
+    response_format={ "type": "json_object" },
+    seed=123,
+    stop=["\n\n"],
+    temperature=0.2,
+    top_p=0.9,
+    tool_choice="auto",
+    tools=[],
+    user="user",
+)
+print(response)
+```
+
+## Sample Usage - Streaming
+```python
+from litellm import completion
+import os
+
+os.environ['XAI_API_KEY'] = ""
+response = completion(
+    model="xai/grok-beta",
+    messages=[
+        {
+            "role": "user",
+            "content": "What's the weather like in Boston today in Fahrenheit?",
+        }
+    ],
+    stream=True,
+    max_tokens=10,
+    response_format={ "type": "json_object" },
+    seed=123,
+    stop=["\n\n"],
+    temperature=0.2,
+    top_p=0.9,
+    tool_choice="auto",
+    tools=[],
+    user="user",
+)
+
+for chunk in response:
+    print(chunk)
+```
+
+
+## Usage with LiteLLM Proxy Server
+
+Here's how to call a XAI model with the LiteLLM Proxy Server
+
+1. Modify the config.yaml 
+
+  ```yaml
+  model_list:
+    - model_name: my-model
+      litellm_params:
+        model: xai/<your-model-name>  # add xai/ prefix to route as XAI provider
+        api_key: api-key                 # api key to send your model
+  ```
+
+
+2. Start the proxy 
+
+  ```bash
+  $ litellm --config /path/to/config.yaml
+  ```
+
+3. Send Request to LiteLLM Proxy Server
+
+  <Tabs>
+
+  <TabItem value="openai" label="OpenAI Python v1.0.0+">
+
+  ```python
+  import openai
+  client = openai.OpenAI(
+      api_key="sk-1234",             # pass litellm proxy key, if you're using virtual keys
+      base_url="http://0.0.0.0:4000" # litellm-proxy-base url
+  )
+
+  response = client.chat.completions.create(
+      model="my-model",
+      messages = [
+          {
+              "role": "user",
+              "content": "what llm are you"
+          }
+      ],
+  )
+
+  print(response)
+  ```
+  </TabItem>
+
+  <TabItem value="curl" label="curl">
+
+  ```shell
+  curl --location 'http://0.0.0.0:4000/chat/completions' \
+      --header 'Authorization: Bearer sk-1234' \
+      --header 'Content-Type: application/json' \
+      --data '{
+      "model": "my-model",
+      "messages": [
+          {
+          "role": "user",
+          "content": "what llm are you"
+          }
+      ],
+  }'
+  ```
+  </TabItem>
+
+  </Tabs>
+
+

docs/my-website/sidebars.js

@@ -155,6 +155,7 @@ const sidebars = {
         "providers/watsonx",
         "providers/predibase",
         "providers/nvidia_nim", 
+        "providers/xai",
         "providers/cerebras", 
         "providers/volcano", 
         "providers/triton-inference-server",

litellm/__init__.py

@@ -490,6 +490,7 @@ def add_known_models():
     "app.empower.dev/api/v1",
     "inference.friendli.ai/v1",
     "api.sambanova.ai/v1",
+    "api.x.ai/v1",
 ]
 
 # this is maintained for Exception Mapping
@@ -507,6 +508,7 @@ def add_known_models():
     "deepinfra",
     "perplexity",
     "xinference",
+    "xai",
     "together_ai",
     "fireworks_ai",
     "empower",
@@ -717,6 +719,7 @@ class LlmProviders(str, Enum):
     OPENAI = "openai"
     OPENAI_LIKE = "openai_like"  # embedding only
     JINA_AI = "jina_ai"
+    XAI = "xai"
     CUSTOM_OPENAI = "custom_openai"
     TEXT_COMPLETION_OPENAI = "text-completion-openai"
     COHERE = "cohere"
@@ -1021,6 +1024,7 @@ class LlmProviders(str, Enum):
     FireworksAIEmbeddingConfig,
 )
 from .llms.jina_ai.embedding.transformation import JinaAIEmbeddingConfig
+from .llms.xai.chat.xai_transformation import XAIChatConfig
 from .llms.volcengine import VolcEngineConfig
 from .llms.text_completion_codestral import MistralTextCompletionConfig
 from .llms.AzureOpenAI.azure import (

litellm/_logging.py

@@ -35,6 +35,9 @@ def format(self, record):
             "timestamp": self.formatTime(record),
         }
 
+        if record.exc_info:
+            json_record["stacktrace"] = self.formatException(record.exc_info)
+
         return json.dumps(json_record)
 
 

litellm/integrations/SlackAlerting/slack_alerting.py

@@ -550,8 +550,14 @@ async def response_taking_too_long(
                     alerting_metadata=alerting_metadata,
                 )
 
-    async def failed_tracking_alert(self, error_message: str):
-        """Raise alert when tracking failed for specific model"""
+    async def failed_tracking_alert(self, error_message: str, failing_model: str):
+        """
+        Raise alert when tracking failed for specific model
+
+        Args:
+            error_message (str): Error message
+            failing_model (str): Model that failed tracking
+        """
         if self.alerting is None or self.alert_types is None:
             # do nothing if alerting is not switched on
             return
@@ -560,7 +566,7 @@ async def failed_tracking_alert(self, error_message: str):
 
         _cache: DualCache = self.internal_usage_cache
         message = "Failed Tracking Cost for " + error_message
-        _cache_key = "budget_alerts:failed_tracking:{}".format(message)
+        _cache_key = "budget_alerts:failed_tracking:{}".format(failing_model)
         result = await _cache.async_get_cache(key=_cache_key)
         if result is None:
             await self.send_alert(

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -480,6 +480,13 @@ def _get_openai_compatible_provider_info(  # noqa: PLR0915
         ) = litellm.JinaAIEmbeddingConfig()._get_openai_compatible_provider_info(
             api_base, api_key
         )
+    elif custom_llm_provider == "xai":
+        (
+            api_base,
+            dynamic_api_key,
+        ) = litellm.XAIChatConfig()._get_openai_compatible_provider_info(
+            api_base, api_key
+        )
     elif custom_llm_provider == "voyage":
         # voyage is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.voyageai.com/v1
         api_base = (

litellm/llms/anthropic/chat/transformation.py

@@ -123,7 +123,7 @@ def get_anthropic_headers(
         return headers
 
     def _map_tool_choice(
-        self, tool_choice: Optional[str], disable_parallel_tool_use: Optional[bool]
+        self, tool_choice: Optional[str], parallel_tool_use: Optional[bool]
     ) -> Optional[AnthropicMessagesToolChoice]:
         _tool_choice: Optional[AnthropicMessagesToolChoice] = None
         if tool_choice == "auto":
@@ -138,13 +138,15 @@ def _map_tool_choice(
             if _tool_name is not None:
                 _tool_choice["name"] = _tool_name
 
-        if disable_parallel_tool_use is not None:
+        if parallel_tool_use is not None:
+            # Anthropic uses 'disable_parallel_tool_use' flag to determine if parallel tool use is allowed
+            # this is the inverse of the openai flag.
             if _tool_choice is not None:
-                _tool_choice["disable_parallel_tool_use"] = disable_parallel_tool_use
+                _tool_choice["disable_parallel_tool_use"] = not parallel_tool_use
             else:  # use anthropic defaults and make sure to send the disable_parallel_tool_use flag
                 _tool_choice = AnthropicMessagesToolChoice(
                     type="auto",
-                    disable_parallel_tool_use=disable_parallel_tool_use,
+                    disable_parallel_tool_use=not parallel_tool_use,
                 )
         return _tool_choice
 
@@ -255,9 +257,7 @@ def map_openai_params(
                 _tool_choice: Optional[AnthropicMessagesToolChoice] = (
                     self._map_tool_choice(
                         tool_choice=non_default_params.get("tool_choice"),
-                        disable_parallel_tool_use=non_default_params.get(
-                            "parallel_tool_calls"
-                        ),
+                        parallel_tool_use=non_default_params.get("parallel_tool_calls"),
                     )
                 )
 

litellm/llms/prompt_templates/factory.py

@@ -2552,18 +2552,20 @@ def _bedrock_converse_messages_pt(  # noqa: PLR0915
                                 BedrockContentBlock(image=assistants_part)  # type: ignore
                             )
                 assistant_content.extend(assistants_parts)
-            elif messages[msg_i].get(
-                "tool_calls", []
-            ):  # support assistant tool invoke convertion
-                assistant_content.extend(
-                    _convert_to_bedrock_tool_call_invoke(messages[msg_i]["tool_calls"])
-                )
-            else:
+            elif messages[msg_i].get("content", None) is not None and isinstance(
+                messages[msg_i]["content"], str
+            ):
                 assistant_text = (
                     messages[msg_i].get("content") or ""
                 )  # either string or none
                 if assistant_text:
                     assistant_content.append(BedrockContentBlock(text=assistant_text))
+            if messages[msg_i].get(
+                "tool_calls", []
+            ):  # support assistant tool invoke convertion [TODO]:
+                assistant_content.extend(
+                    _convert_to_bedrock_tool_call_invoke(messages[msg_i]["tool_calls"])
+                )
 
             msg_i += 1
 

litellm/llms/vertex_ai_and_google_ai_studio/context_caching/transformation.py

@@ -11,9 +11,9 @@
 from litellm.utils import is_cached_message
 
 from ..common_utils import VertexAIError, get_supports_system_message
-from ..gemini.transformation import _transform_system_message
-from ..gemini.vertex_and_google_ai_studio_gemini import (
+from ..gemini.transformation import (
     _gemini_convert_messages_with_history,
+    _transform_system_message,
 )