diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index 627d0ae80726..8812efc3bf61 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -199,19 +199,49 @@ def _normalize_response_format(selected_params: Dict, vendor: OCIVendors) -> Non rf = selected_params.get("responseFormat") if not isinstance(rf, dict) or "type" not in rf: return - rf_payload = dict(rf) - selected_params["responseFormat"] = rf_payload - response_type = rf_payload["type"] - if "json_schema" in rf_payload: - raw_schema = rf_payload.pop("json_schema") - rf_payload["jsonSchema"] = ( - dict(raw_schema) if isinstance(raw_schema, dict) else raw_schema - ) + + rf_type = str(rf["type"]).lower() + raw_schema = rf.get("json_schema") + json_schema = raw_schema if isinstance(raw_schema, dict) else None + + if rf_type == "text": + selected_params["responseFormat"] = {"type": "TEXT"} + return + if vendor == OCIVendors.COHERE: - rf_payload["type"] = response_type - else: - fmt = response_type.upper() - rf_payload["type"] = "JSON_OBJECT" if fmt == "JSON" else fmt + # OCI Cohere has no JSON_SCHEMA type; a schema rides on JSON_OBJECT. + payload: Dict[str, Any] = {"type": "JSON_OBJECT"} + if json_schema is not None and json_schema.get("schema") is not None: + payload["schema"] = json_schema["schema"] + selected_params["responseFormat"] = payload + return + + if rf_type == "json_schema": + if json_schema is None: + raise OCIError( + status_code=400, + message="response_format type 'json_schema' requires a 'json_schema' object", + ) + # OCI's ResponseJsonSchema accepts only name/description/schema/isStrict. + # OpenAI sends `strict` instead of `isStrict`; forwarding it (or any + # other extra key) makes OCI reject the whole request with HTTP 400. + oci_schema: Dict[str, Any] = {"name": json_schema.get("name") or "response"} + if json_schema.get("description") is not None: + oci_schema["description"] = json_schema["description"] + if json_schema.get("schema") is not None: + oci_schema["schema"] = json_schema["schema"] + if json_schema.get("strict") is not None: + oci_schema["isStrict"] = json_schema["strict"] + selected_params["responseFormat"] = { + "type": "JSON_SCHEMA", + "jsonSchema": oci_schema, + } + return + + fmt = rf_type.upper() + selected_params["responseFormat"] = { + "type": "JSON_OBJECT" if fmt == "JSON" else fmt + } def get_vendor_from_model(model: str) -> OCIVendors: diff --git a/litellm/types/llms/oci.py b/litellm/types/llms/oci.py index df551d8a8c60..621f40aa31dc 100644 --- a/litellm/types/llms/oci.py +++ b/litellm/types/llms/oci.py @@ -291,25 +291,6 @@ class CohereToolResult(BaseModel): outputs: List[Dict[str, Any]] -class CohereResponseFormat(BaseModel): - """Response format for Cohere.""" - - type: str - - -class CohereResponseTextFormat(CohereResponseFormat): - """Text response format for Cohere.""" - - type: Literal["text"] = "text" - - -class CohereResponseJSONSchemaFormat(CohereResponseFormat): - """JSON schema response format for Cohere.""" - - type: Literal["json_schema"] = "json_schema" - jsonSchema: Dict[str, Any] - - class CohereChatRequest(BaseModel): """Cohere chat request model.""" @@ -336,13 +317,10 @@ class CohereChatRequest(BaseModel): # ``OCIChatConfig.openai_to_oci_cohere_param_map`` which marks # ``tool_choice`` as unsupported. The field is intentionally absent here # so it isn't silently dropped or surfaced as a supported feature. - responseFormat: Optional[ - Union[ - CohereResponseTextFormat, - CohereResponseJSONSchemaFormat, - CohereResponseFormat, - ] - ] = None + # OCI Cohere responseFormat is {"type": "TEXT" | "JSON_OBJECT", "schema"?: ...}; + # there is no JSON_SCHEMA type. The shape is built in + # OCIChatConfig._normalize_response_format. + responseFormat: Optional[Dict[str, Any]] = None preambleOverride: Optional[str] = None documents: Optional[List[Dict[str, Any]]] = None searchQueriesOnly: Optional[bool] = None diff --git a/tests/integration/test_oci_proxy_integration.py b/tests/integration/test_oci_proxy_integration.py index 67e40ca814a4..a3ba844cb8e8 100644 --- a/tests/integration/test_oci_proxy_integration.py +++ b/tests/integration/test_oci_proxy_integration.py @@ -30,6 +30,7 @@ from __future__ import annotations +import json import os import socket import subprocess @@ -41,7 +42,6 @@ import httpx import pytest - # --------------------------------------------------------------------------- # Skip gate # --------------------------------------------------------------------------- @@ -79,7 +79,9 @@ def _wait_for_health(base_url: str, proc: subprocess.Popen, deadline: float) -> except httpx.HTTPError: pass time.sleep(0.5) - raise RuntimeError(f"litellm proxy did not become ready within {STARTUP_TIMEOUT_S}s") + raise RuntimeError( + f"litellm proxy did not become ready within {STARTUP_TIMEOUT_S}s" + ) def _oci_env_from_profile() -> dict[str, str]: @@ -206,9 +208,7 @@ def test_chat_completion_via_proxy(proxy_url: str, model: str) -> None: # Reasoning models may return empty content if their budget covers only # the thinking turn — accept either text or a non-empty reasoning field. has_content = bool(msg.get("content")) - has_reasoning = bool(msg.get("reasoning_content")) or bool( - msg.get("reasoning") - ) + has_reasoning = bool(msg.get("reasoning_content")) or bool(msg.get("reasoning")) assert has_content or has_reasoning, f"empty assistant message for {model}: {msg}" usage = body.get("usage") or {} assert usage.get("total_tokens", 0) > 0 @@ -232,7 +232,7 @@ def test_chat_completion_streaming_via_proxy(proxy_url: str, model: str) -> None continue if not line.startswith("data:"): continue - payload = line[len("data:"):].strip() + payload = line[len("data:") :].strip() if payload == "[DONE]": saw_done = True break @@ -274,9 +274,54 @@ def test_model_list_advertises_oci_models(proxy_url: str) -> None: assert expected in advertised, f"{expected} missing from /v1/models: {advertised}" +@pytest.mark.parametrize("model", ["oci-cohere-command", "oci-llama"]) +def test_response_format_json_schema_via_proxy(proxy_url: str, model: str) -> None: + """A response_format json_schema succeeds through the gateway for both a + Cohere and a generic OCI model. + Regression for the HTTP 400 ``Please pass in correct format of request`` + that rejected every json_schema request (which MLflow LLM judges always + send): generic models choke on OpenAI's ``strict`` key, and Cohere has no + JSON_SCHEMA type. + """ + r = httpx.post( + f"{proxy_url}/v1/chat/completions", + headers=_auth_headers(), + json={ + "model": model, + "messages": [ + { + "role": "user", + "content": "Rate the answer 4 to 2+2. Give an integer score and a short rationale.", + } + ], + "max_tokens": 200, + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "judgment", + "strict": True, + "schema": { + "type": "object", + "properties": { + "score": {"type": "integer"}, + "rationale": {"type": "string"}, + }, + "required": ["score", "rationale"], + "additionalProperties": False, + }, + }, + }, + }, + timeout=REQUEST_TIMEOUT_S, + ) + assert r.status_code == 200, f"{model} json_schema -> {r.status_code}: {r.text}" + content = r.json()["choices"][0]["message"]["content"] + assert content is not None + assert "score" in json.loads(content) + + def test_omitted_max_tokens_not_truncated(proxy_url: str) -> None: """A request that omits max_tokens completes instead of being cut off. - Regression for OCI's tiny server-side maxTokens default (~20 tokens): without an injected default, a request that doesn't set max_tokens came back with finish_reason "length" after ~19 tokens, so structured outputs (e.g. MLflow @@ -308,4 +353,4 @@ def test_omitted_max_tokens_not_truncated(proxy_url: str) -> None: assert content.strip(), f"empty content: {choice}" # The ~20-token server default truncated well before this; a complete # four-to-five sentence answer comfortably exceeds it. - assert body["usage"]["completion_tokens"] > 50, body["usage"] + assert body["usage"]["completion_tokens"] > 50, body["usage"] \ No newline at end of file diff --git a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py index 94c8b23def39..aec1d9ed1a91 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py @@ -364,6 +364,137 @@ def test_transform_request_response_format_json_shorthand(self): rf = transformed_request["chatRequest"]["responseFormat"] assert rf["type"] == "JSON_OBJECT" + def test_transform_request_response_format_json_schema_generic(self): + """A GENERIC json_schema must become OCI's JSON_SCHEMA shape with the + OpenAI ``strict`` key renamed to ``isStrict``. + + OCI's ResponseJsonSchema rejects ``strict`` (and any other extra key) + with HTTP 400 "Please pass in correct format of request", so the raw + OpenAI body must not be forwarded. + """ + config = OCIChatConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "judgment", + "description": "a score and rationale", + "strict": True, + "schema": { + "type": "object", + "properties": {"score": {"type": "integer"}}, + "required": ["score"], + }, + }, + }, + } + transformed_request = config.transform_request( + model=TEST_MODEL_NAME, # xai.grok-4 -> GENERIC + messages=TEST_MESSAGES, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + rf = transformed_request["chatRequest"]["responseFormat"] + assert rf["type"] == "JSON_SCHEMA" + assert "strict" not in rf["jsonSchema"] + assert rf["jsonSchema"]["isStrict"] is True + assert rf["jsonSchema"]["name"] == "judgment" + assert rf["jsonSchema"]["description"] == "a score and rationale" + assert rf["jsonSchema"]["schema"]["properties"]["score"]["type"] == "integer" + + def test_transform_request_response_format_json_schema_generic_no_strict(self): + """A GENERIC json_schema without ``strict`` must omit ``isStrict``.""" + config = OCIChatConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "response_format": { + "type": "json_schema", + "json_schema": {"name": "j", "schema": {"type": "object"}}, + }, + } + transformed_request = config.transform_request( + model=TEST_MODEL_NAME, + messages=TEST_MESSAGES, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + rf = transformed_request["chatRequest"]["responseFormat"] + assert rf["type"] == "JSON_SCHEMA" + assert "isStrict" not in rf["jsonSchema"] + + def test_transform_request_response_format_json_schema_cohere(self): + """A Cohere json_schema must fold the schema onto JSON_OBJECT. + + OCI Cohere has no JSON_SCHEMA type; sending one yields HTTP 400. + """ + config = OCIChatConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "judgment", + "strict": True, + "schema": { + "type": "object", + "properties": {"score": {"type": "integer"}}, + }, + }, + }, + } + transformed_request = config.transform_request( + model="cohere.command-latest", + messages=TEST_MESSAGES, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + rf = transformed_request["chatRequest"]["responseFormat"] + assert rf["type"] == "JSON_OBJECT" + assert "jsonSchema" not in rf + assert rf["schema"]["properties"]["score"]["type"] == "integer" + + def test_transform_request_response_format_cohere_json_object(self): + """Cohere json_object without a schema stays a bare JSON_OBJECT.""" + config = OCIChatConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "response_format": {"type": "json_object"}, + } + transformed_request = config.transform_request( + model="cohere.command-latest", + messages=TEST_MESSAGES, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + rf = transformed_request["chatRequest"]["responseFormat"] + assert rf == {"type": "JSON_OBJECT"} + + def test_transform_request_json_schema_without_body_raises_generic(self): + """A GENERIC json_schema with no ``json_schema`` body must raise an early + 400, not silently emit {"type": "JSON_SCHEMA"} (which OCI rejects).""" + from litellm.llms.oci.common_utils import OCIError + + config = OCIChatConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "response_format": {"type": "json_schema"}, + } + with pytest.raises(OCIError) as exc_info: + config.transform_request( + model=TEST_MODEL_NAME, # GENERIC + messages=TEST_MESSAGES, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + assert exc_info.value.status_code == 400 + assert "json_schema" in str(exc_info.value) + def test_transform_response_without_token_details(self): """ Tests that responses missing completionTokensDetails and promptTokensDetails diff --git a/tests/test_litellm/llms/oci/chat/test_oci_cohere_tool_calls.py b/tests/test_litellm/llms/oci/chat/test_oci_cohere_tool_calls.py index bee1e033502e..5dd44d72d682 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_cohere_tool_calls.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_cohere_tool_calls.py @@ -237,25 +237,30 @@ def test_cohere_response_with_tool_calls(self): assert result.usage.completion_tokens == 22 assert result.usage.total_tokens == 48 - def test_cohere_request_preserves_json_schema_response_format(self): - """Ensure Cohere requests retain JSON schema payloads in responseFormat.""" + def test_cohere_request_folds_json_schema_into_json_object(self): + """A Cohere json_schema must fold the schema onto JSON_OBJECT. + + OCI Cohere has no JSON_SCHEMA type; sending {"type": "JSON_SCHEMA", ...} + (or the raw lowercase "json_schema" with a jsonSchema body) is rejected + with HTTP 400. The schema rides on JSON_OBJECT instead. + """ config = OCIChatConfig() messages = [{"role": "user", "content": "Return structured info"}] - response_format = { - "type": "json_schema", - "json_schema": { - "name": "test_schema", - "strict": True, - "schema": { - "type": "object", - "properties": {"foo": {"type": "string"}}, - "required": ["foo"], - }, - }, + schema = { + "type": "object", + "properties": {"foo": {"type": "string"}}, + "required": ["foo"], } optional_params = { "oci_compartment_id": TEST_COMPARTMENT_ID, - "response_format": response_format, + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "test_schema", + "strict": True, + "schema": schema, + }, + }, } transformed_request = config.transform_request( @@ -266,18 +271,14 @@ def test_cohere_request_preserves_json_schema_response_format(self): headers={}, ) - chat_request = transformed_request["chatRequest"] - assert chat_request["apiFormat"] == "COHERE" - assert "responseFormat" in chat_request - - cohere_response_format = chat_request["responseFormat"] - assert cohere_response_format["type"] == "json_schema" + cohere_response_format = transformed_request["chatRequest"]["responseFormat"] + assert cohere_response_format["type"] == "JSON_OBJECT" + assert "jsonSchema" not in cohere_response_format assert "json_schema" not in cohere_response_format - assert "jsonSchema" in cohere_response_format - assert cohere_response_format["jsonSchema"] == response_format["json_schema"] + assert cohere_response_format["schema"] == schema - def test_cohere_request_response_format_text_stays_lowercase(self): - """Ensure Cohere keeps response_format type lowercase (e.g. 'text' not 'TEXT').""" + def test_cohere_request_response_format_text_is_uppercased(self): + """Cohere response_format type 'text' maps to OCI's canonical 'TEXT'.""" config = OCIChatConfig() messages = [{"role": "user", "content": "Hello"}] optional_params = { @@ -293,10 +294,7 @@ def test_cohere_request_response_format_text_stays_lowercase(self): headers={}, ) - chat_request = transformed_request["chatRequest"] - assert chat_request["apiFormat"] == "COHERE" - assert "responseFormat" in chat_request - assert chat_request["responseFormat"]["type"] == "text" + assert transformed_request["chatRequest"]["responseFormat"] == {"type": "TEXT"} def test_cohere_tool_call_only_message_no_text(self): """Test chat history with an assistant message that has tool calls but no text content."""