Stable branch nov12

litellm/llms/anthropic/chat/handler.py

@@ -58,6 +58,7 @@ def validate_environment(
     user_headers,
     model,
     messages: List[AllMessageValues],
+    is_vertex_request: bool,
     tools: Optional[List[AllAnthropicToolsValues]],
     anthropic_version: Optional[str] = None,
 ):
@@ -78,6 +79,7 @@ def validate_environment(
         prompt_caching_set=prompt_caching_set,
         pdf_used=pdf_used,
         api_key=api_key,
+        is_vertex_request=is_vertex_request,
     )
 
     if user_headers is not None and isinstance(user_headers, dict):
@@ -446,19 +448,20 @@ def completion(
         headers={},
         client=None,
     ):
+        optional_params = copy.deepcopy(optional_params)
+        stream = optional_params.pop("stream", None)
+        json_mode: bool = optional_params.pop("json_mode", False)
+        is_vertex_request: bool = optional_params.pop("is_vertex_request", False)
+        _is_function_call = False
+        messages = copy.deepcopy(messages)
         headers = validate_environment(
             api_key,
             headers,
             model,
             messages=messages,
             tools=optional_params.get("tools"),
+            is_vertex_request=is_vertex_request,
         )
-        _is_function_call = False
-        messages = copy.deepcopy(messages)
-        optional_params = copy.deepcopy(optional_params)
-        stream = optional_params.pop("stream", None)
-        json_mode: bool = optional_params.pop("json_mode", False)
-        is_vertex_request: bool = optional_params.pop("is_vertex_request", False)
 
         data = AnthropicConfig()._transform_request(
             model=model,

litellm/llms/anthropic/chat/transformation.py

@@ -7,6 +7,7 @@
     AllAnthropicToolsValues,
     AnthropicComputerTool,
     AnthropicHostedTools,
+    AnthropicInputSchema,
     AnthropicMessageRequestBase,
     AnthropicMessagesRequest,
     AnthropicMessagesTool,
@@ -105,6 +106,7 @@ def get_anthropic_headers(
         computer_tool_used: bool = False,
         prompt_caching_set: bool = False,
         pdf_used: bool = False,
+        is_vertex_request: bool = False,
     ) -> dict:
         import json
 
@@ -121,8 +123,12 @@ def get_anthropic_headers(
             "accept": "application/json",
             "content-type": "application/json",
         }
-        if len(betas) > 0:
+
+        if is_vertex_request is True:
+            pass
+        elif len(betas) > 0:
             headers["anthropic-beta"] = ",".join(betas)
+
         return headers
 
     def _map_tool_choice(
@@ -159,15 +165,17 @@ def _map_tool_helper(
         returned_tool: Optional[AllAnthropicToolsValues] = None
 
         if tool["type"] == "function" or tool["type"] == "custom":
+            _input_schema: dict = tool["function"].get(
+                "parameters",
+                {
+                    "type": "object",
+                    "properties": {},
+                },
+            )
+            input_schema: AnthropicInputSchema = AnthropicInputSchema(**_input_schema)
             _tool = AnthropicMessagesTool(
                 name=tool["function"]["name"],
-                input_schema=tool["function"].get(
-                    "parameters",
-                    {
-                        "type": "object",
-                        "properties": {},
-                    },
-                ),
+                input_schema=input_schema,
             )
 
             _description = tool["function"].get("description")
@@ -304,17 +312,10 @@ def map_openai_params(
                 - You should set tool_choice (see Forcing tool use) to instruct the model to explicitly use that tool
                 - Remember that the model will pass the input to the tool, so the name of the tool and description should be from the model’s perspective.
                 """
-                _tool_choice = None
                 _tool_choice = {"name": "json_tool_call", "type": "tool"}
-
-                _tool = AnthropicMessagesTool(
-                    name="json_tool_call",
-                    input_schema={
-                        "type": "object",
-                        "properties": {"values": json_schema},  # type: ignore
-                    },
+                _tool = self._create_json_tool_call_for_response_format(
+                    json_schema=json_schema,
                 )
-
                 optional_params["tools"] = [_tool]
                 optional_params["tool_choice"] = _tool_choice
                 optional_params["json_mode"] = True
@@ -341,6 +342,34 @@ def map_openai_params(
 
         return optional_params
 
+    def _create_json_tool_call_for_response_format(
+        self,
+        json_schema: Optional[dict] = None,
+    ) -> AnthropicMessagesTool:
+        """
+        Handles creating a tool call for getting responses in JSON format.
+
+        Args:
+            json_schema (Optional[dict]): The JSON schema the response should be in
+
+        Returns:
+            AnthropicMessagesTool: The tool call to send to Anthropic API to get responses in JSON format
+        """
+        _input_schema: AnthropicInputSchema = AnthropicInputSchema(
+            type="object",
+        )
+
+        if json_schema is None:
+            # Anthropic raises a 400 BadRequest error if properties is passed as None
+            # see usage with additionalProperties (Example 5) https://github.com/anthropics/anthropic-cookbook/blob/main/tool_use/extracting_structured_json.ipynb
+            _input_schema["additionalProperties"] = True
+            _input_schema["properties"] = {}
+        else:
+            _input_schema["properties"] = json_schema
+
+        _tool = AnthropicMessagesTool(name="json_tool_call", input_schema=_input_schema)
+        return _tool
+
     def is_cache_control_set(self, messages: List[AllMessageValues]) -> bool:
         """
         Return if {"cache_control": ..} in message content block

litellm/types/llms/anthropic.py

@@ -12,10 +12,16 @@ class AnthropicMessagesToolChoice(TypedDict, total=False):
     disable_parallel_tool_use: bool  # default is false
 
 
+class AnthropicInputSchema(TypedDict, total=False):
+    type: Optional[str]
+    properties: Optional[dict]
+    additionalProperties: Optional[bool]
+
+
 class AnthropicMessagesTool(TypedDict, total=False):
     name: Required[str]
     description: str
-    input_schema: Required[dict]
+    input_schema: Optional[AnthropicInputSchema]
     type: Literal["custom"]
     cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
 

tests/llm_translation/base_llm_unit_tests.py

@@ -53,6 +53,32 @@ def test_message_with_name(self):
         response = litellm.completion(**base_completion_call_args, messages=messages)
         assert response is not None
 
+    def test_json_response_format(self):
+        """
+        Test that the JSON response format is supported by the LLM API
+        """
+        base_completion_call_args = self.get_base_completion_call_args()
+        litellm.set_verbose = True
+
+        messages = [
+            {
+                "role": "system",
+                "content": "Your output should be a JSON object with no additional properties.  ",
+            },
+            {
+                "role": "user",
+                "content": "Respond with this in json. city=San Francisco, state=CA, weather=sunny, temp=60",
+            },
+        ]
+
+        response = litellm.completion(
+            **base_completion_call_args,
+            messages=messages,
+            response_format={"type": "json_object"},
+        )
+
+        print(response)
+
     @pytest.fixture
     def pdf_messages(self):
         import base64

tests/llm_translation/test_anthropic_completion.py

@@ -627,6 +627,38 @@ def test_anthropic_tool_helper(cache_control_location):
     assert tool["cache_control"] == {"type": "ephemeral"}
 
 
+def test_create_json_tool_call_for_response_format():
+    """
+    tests using response_format=json with anthropic
+
+    A tool call to anthropic is made when response_format=json is used.
+
+    """
+    # Initialize AnthropicConfig
+    config = AnthropicConfig()
+
+    # Test case 1: No schema provided
+    # See Anthropics Example 5 on how to handle cases when no schema is provided https://github.com/anthropics/anthropic-cookbook/blob/main/tool_use/extracting_structured_json.ipynb
+    tool = config._create_json_tool_call_for_response_format()
+    assert tool["name"] == "json_tool_call"
+    _input_schema = tool.get("input_schema")
+    assert _input_schema is not None
+    assert _input_schema.get("type") == "object"
+    assert _input_schema.get("additionalProperties") is True
+    assert _input_schema.get("properties") == {}
+
+    # Test case 2: With custom schema
+    # reference: https://github.com/anthropics/anthropic-cookbook/blob/main/tool_use/extracting_structured_json.ipynb
+    custom_schema = {"name": {"type": "string"}, "age": {"type": "integer"}}
+    tool = config._create_json_tool_call_for_response_format(json_schema=custom_schema)
+    assert tool["name"] == "json_tool_call"
+    _input_schema = tool.get("input_schema")
+    assert _input_schema is not None
+    assert _input_schema.get("type") == "object"
+    assert _input_schema.get("properties") == custom_schema
+    assert "additionalProperties" not in _input_schema
+
+
 from litellm import completion
 
 

tests/local_testing/test_router_fallbacks.py

@@ -1138,9 +1138,9 @@ async def test_router_content_policy_fallbacks(
     router = Router(
         model_list=[
             {
-                "model_name": "claude-2",
+                "model_name": "claude-2.1",
                 "litellm_params": {
-                    "model": "claude-2",
+                    "model": "claude-2.1",
                     "api_key": "",
                     "mock_response": mock_response,
                 },
@@ -1164,22 +1164,22 @@ async def test_router_content_policy_fallbacks(
             {
                 "model_name": "my-general-model",
                 "litellm_params": {
-                    "model": "claude-2",
+                    "model": "claude-2.1",
                     "api_key": "",
                     "mock_response": Exception("Should not have called this."),
                 },
             },
             {
                 "model_name": "my-context-window-model",
                 "litellm_params": {
-                    "model": "claude-2",
+                    "model": "claude-2.1",
                     "api_key": "",
                     "mock_response": Exception("Should not have called this."),
                 },
             },
         ],
         content_policy_fallbacks=(
-            [{"claude-2": ["my-fallback-model"]}]
+            [{"claude-2.1": ["my-fallback-model"]}]
             if fallback_type == "model-specific"
             else None
         ),
@@ -1190,12 +1190,12 @@ async def test_router_content_policy_fallbacks(
 
     if sync_mode is True:
         response = router.completion(
-            model="claude-2",
+            model="claude-2.1",
             messages=[{"role": "user", "content": "Hey, how's it going?"}],
         )
     else:
         response = await router.acompletion(
-            model="claude-2",
+            model="claude-2.1",
             messages=[{"role": "user", "content": "Hey, how's it going?"}],
         )
 

tests/local_testing/test_streaming.py

@@ -718,7 +718,7 @@ async def test_acompletion_claude_2_stream():
     try:
         litellm.set_verbose = True
         response = await litellm.acompletion(
-            model="claude-2",
+            model="claude-2.1",
             messages=[{"role": "user", "content": "hello from litellm"}],
             stream=True,
         )