diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 2d119a28f278..7fc69d3cf749 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -58,6 +58,7 @@ def validate_environment( user_headers, model, messages: List[AllMessageValues], + is_vertex_request: bool, tools: Optional[List[AllAnthropicToolsValues]], anthropic_version: Optional[str] = None, ): @@ -78,6 +79,7 @@ def validate_environment( prompt_caching_set=prompt_caching_set, pdf_used=pdf_used, api_key=api_key, + is_vertex_request=is_vertex_request, ) if user_headers is not None and isinstance(user_headers, dict): @@ -446,19 +448,20 @@ def completion( headers={}, client=None, ): + optional_params = copy.deepcopy(optional_params) + stream = optional_params.pop("stream", None) + json_mode: bool = optional_params.pop("json_mode", False) + is_vertex_request: bool = optional_params.pop("is_vertex_request", False) + _is_function_call = False + messages = copy.deepcopy(messages) headers = validate_environment( api_key, headers, model, messages=messages, tools=optional_params.get("tools"), + is_vertex_request=is_vertex_request, ) - _is_function_call = False - messages = copy.deepcopy(messages) - optional_params = copy.deepcopy(optional_params) - stream = optional_params.pop("stream", None) - json_mode: bool = optional_params.pop("json_mode", False) - is_vertex_request: bool = optional_params.pop("is_vertex_request", False) data = AnthropicConfig()._transform_request( model=model, diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 18c53b696b5c..15ae1eb69f86 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -7,6 +7,7 @@ AllAnthropicToolsValues, AnthropicComputerTool, AnthropicHostedTools, + AnthropicInputSchema, AnthropicMessageRequestBase, AnthropicMessagesRequest, AnthropicMessagesTool, @@ -105,6 +106,7 @@ def get_anthropic_headers( computer_tool_used: bool = False, prompt_caching_set: bool = False, pdf_used: bool = False, + is_vertex_request: bool = False, ) -> dict: import json @@ -121,8 +123,12 @@ def get_anthropic_headers( "accept": "application/json", "content-type": "application/json", } - if len(betas) > 0: + + if is_vertex_request is True: + pass + elif len(betas) > 0: headers["anthropic-beta"] = ",".join(betas) + return headers def _map_tool_choice( @@ -159,15 +165,17 @@ def _map_tool_helper( returned_tool: Optional[AllAnthropicToolsValues] = None if tool["type"] == "function" or tool["type"] == "custom": + _input_schema: dict = tool["function"].get( + "parameters", + { + "type": "object", + "properties": {}, + }, + ) + input_schema: AnthropicInputSchema = AnthropicInputSchema(**_input_schema) _tool = AnthropicMessagesTool( name=tool["function"]["name"], - input_schema=tool["function"].get( - "parameters", - { - "type": "object", - "properties": {}, - }, - ), + input_schema=input_schema, ) _description = tool["function"].get("description") @@ -304,17 +312,10 @@ def map_openai_params( - You should set tool_choice (see Forcing tool use) to instruct the model to explicitly use that tool - Remember that the model will pass the input to the tool, so the name of the tool and description should be from the model’s perspective. """ - _tool_choice = None _tool_choice = {"name": "json_tool_call", "type": "tool"} - - _tool = AnthropicMessagesTool( - name="json_tool_call", - input_schema={ - "type": "object", - "properties": {"values": json_schema}, # type: ignore - }, + _tool = self._create_json_tool_call_for_response_format( + json_schema=json_schema, ) - optional_params["tools"] = [_tool] optional_params["tool_choice"] = _tool_choice optional_params["json_mode"] = True @@ -341,6 +342,34 @@ def map_openai_params( return optional_params + def _create_json_tool_call_for_response_format( + self, + json_schema: Optional[dict] = None, + ) -> AnthropicMessagesTool: + """ + Handles creating a tool call for getting responses in JSON format. + + Args: + json_schema (Optional[dict]): The JSON schema the response should be in + + Returns: + AnthropicMessagesTool: The tool call to send to Anthropic API to get responses in JSON format + """ + _input_schema: AnthropicInputSchema = AnthropicInputSchema( + type="object", + ) + + if json_schema is None: + # Anthropic raises a 400 BadRequest error if properties is passed as None + # see usage with additionalProperties (Example 5) https://github.com/anthropics/anthropic-cookbook/blob/main/tool_use/extracting_structured_json.ipynb + _input_schema["additionalProperties"] = True + _input_schema["properties"] = {} + else: + _input_schema["properties"] = json_schema + + _tool = AnthropicMessagesTool(name="json_tool_call", input_schema=_input_schema) + return _tool + def is_cache_control_set(self, messages: List[AllMessageValues]) -> bool: """ Return if {"cache_control": ..} in message content block diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index b0a3780b8627..55e37ad97109 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -12,10 +12,16 @@ class AnthropicMessagesToolChoice(TypedDict, total=False): disable_parallel_tool_use: bool # default is false +class AnthropicInputSchema(TypedDict, total=False): + type: Optional[str] + properties: Optional[dict] + additionalProperties: Optional[bool] + + class AnthropicMessagesTool(TypedDict, total=False): name: Required[str] description: str - input_schema: Required[dict] + input_schema: Optional[AnthropicInputSchema] type: Literal["custom"] cache_control: Optional[Union[dict, ChatCompletionCachedContent]] diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index 18ac7216f380..acb764ba14b6 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -53,6 +53,32 @@ def test_message_with_name(self): response = litellm.completion(**base_completion_call_args, messages=messages) assert response is not None + def test_json_response_format(self): + """ + Test that the JSON response format is supported by the LLM API + """ + base_completion_call_args = self.get_base_completion_call_args() + litellm.set_verbose = True + + messages = [ + { + "role": "system", + "content": "Your output should be a JSON object with no additional properties. ", + }, + { + "role": "user", + "content": "Respond with this in json. city=San Francisco, state=CA, weather=sunny, temp=60", + }, + ] + + response = litellm.completion( + **base_completion_call_args, + messages=messages, + response_format={"type": "json_object"}, + ) + + print(response) + @pytest.fixture def pdf_messages(self): import base64 diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index 9d7c9af73851..c399c3a470da 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -627,6 +627,38 @@ def test_anthropic_tool_helper(cache_control_location): assert tool["cache_control"] == {"type": "ephemeral"} +def test_create_json_tool_call_for_response_format(): + """ + tests using response_format=json with anthropic + + A tool call to anthropic is made when response_format=json is used. + + """ + # Initialize AnthropicConfig + config = AnthropicConfig() + + # Test case 1: No schema provided + # See Anthropics Example 5 on how to handle cases when no schema is provided https://github.com/anthropics/anthropic-cookbook/blob/main/tool_use/extracting_structured_json.ipynb + tool = config._create_json_tool_call_for_response_format() + assert tool["name"] == "json_tool_call" + _input_schema = tool.get("input_schema") + assert _input_schema is not None + assert _input_schema.get("type") == "object" + assert _input_schema.get("additionalProperties") is True + assert _input_schema.get("properties") == {} + + # Test case 2: With custom schema + # reference: https://github.com/anthropics/anthropic-cookbook/blob/main/tool_use/extracting_structured_json.ipynb + custom_schema = {"name": {"type": "string"}, "age": {"type": "integer"}} + tool = config._create_json_tool_call_for_response_format(json_schema=custom_schema) + assert tool["name"] == "json_tool_call" + _input_schema = tool.get("input_schema") + assert _input_schema is not None + assert _input_schema.get("type") == "object" + assert _input_schema.get("properties") == custom_schema + assert "additionalProperties" not in _input_schema + + from litellm import completion diff --git a/tests/local_testing/test_router_fallbacks.py b/tests/local_testing/test_router_fallbacks.py index cad640a54ec7..7f4d318bf17f 100644 --- a/tests/local_testing/test_router_fallbacks.py +++ b/tests/local_testing/test_router_fallbacks.py @@ -1138,9 +1138,9 @@ async def test_router_content_policy_fallbacks( router = Router( model_list=[ { - "model_name": "claude-2", + "model_name": "claude-2.1", "litellm_params": { - "model": "claude-2", + "model": "claude-2.1", "api_key": "", "mock_response": mock_response, }, @@ -1164,7 +1164,7 @@ async def test_router_content_policy_fallbacks( { "model_name": "my-general-model", "litellm_params": { - "model": "claude-2", + "model": "claude-2.1", "api_key": "", "mock_response": Exception("Should not have called this."), }, @@ -1172,14 +1172,14 @@ async def test_router_content_policy_fallbacks( { "model_name": "my-context-window-model", "litellm_params": { - "model": "claude-2", + "model": "claude-2.1", "api_key": "", "mock_response": Exception("Should not have called this."), }, }, ], content_policy_fallbacks=( - [{"claude-2": ["my-fallback-model"]}] + [{"claude-2.1": ["my-fallback-model"]}] if fallback_type == "model-specific" else None ), @@ -1190,12 +1190,12 @@ async def test_router_content_policy_fallbacks( if sync_mode is True: response = router.completion( - model="claude-2", + model="claude-2.1", messages=[{"role": "user", "content": "Hey, how's it going?"}], ) else: response = await router.acompletion( - model="claude-2", + model="claude-2.1", messages=[{"role": "user", "content": "Hey, how's it going?"}], ) diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py index fcdc6b60d4f9..7b3dbd8d63d2 100644 --- a/tests/local_testing/test_streaming.py +++ b/tests/local_testing/test_streaming.py @@ -718,7 +718,7 @@ async def test_acompletion_claude_2_stream(): try: litellm.set_verbose = True response = await litellm.acompletion( - model="claude-2", + model="claude-2.1", messages=[{"role": "user", "content": "hello from litellm"}], stream=True, )