From 5b3e0b479b6bdd86769d98aaebcfee32abdc87db Mon Sep 17 00:00:00 2001 From: Federico Kamelhar Date: Thu, 4 Jun 2026 13:35:47 -0400 Subject: [PATCH 1/3] fix(oci): translate response_format json_schema to OCI's accepted shape OCI GenAI rejected every json_schema response_format with HTTP 400 "Please pass in correct format of request", which broke structured-output callers such as MLflow LLM judges (they always send a json_schema). The provider forwarded OpenAI's raw json_schema body unchanged. For GENERIC models OCI's ResponseJsonSchema accepts only name/description/schema/isStrict, so OpenAI's `strict` key (and any other extra) 400s the request; the key must be renamed to isStrict and the body whitelisted. For Cohere models there is no JSON_SCHEMA type at all; the schema has to ride on JSON_OBJECT as {"type": "JSON_OBJECT", "schema": ...}. Cohere type values must also be the canonical uppercase TEXT/JSON_OBJECT. _normalize_response_format now branches by vendor and emits the exact shape each one accepts (verified live against OCI GenAI for Cohere, Meta, Gemini and Grok). Drops the unused, incorrect Cohere response-format pydantic models. Two existing tests asserted the broken behavior (lowercase type, raw jsonSchema on Cohere); they are rewritten to assert the corrected shape, and generic/Cohere json_schema regression tests are added. --- litellm/llms/oci/chat/transformation.py | 49 ++++++-- litellm/types/llms/oci.py | 30 +---- .../oci/chat/test_oci_chat_transformation.py | 110 ++++++++++++++++++ .../oci/chat/test_oci_cohere_tool_calls.py | 54 +++++---- 4 files changed, 177 insertions(+), 66 deletions(-) diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index f050f9eea36e..9ce2e8acc7e2 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -193,19 +193,44 @@ def _normalize_response_format(selected_params: Dict, vendor: OCIVendors) -> Non rf = selected_params.get("responseFormat") if not isinstance(rf, dict) or "type" not in rf: return - rf_payload = dict(rf) - selected_params["responseFormat"] = rf_payload - response_type = rf_payload["type"] - if "json_schema" in rf_payload: - raw_schema = rf_payload.pop("json_schema") - rf_payload["jsonSchema"] = ( - dict(raw_schema) if isinstance(raw_schema, dict) else raw_schema - ) + + rf_type = str(rf["type"]).lower() + raw_schema = rf.get("json_schema") + json_schema = raw_schema if isinstance(raw_schema, dict) else None + + if rf_type == "text": + selected_params["responseFormat"] = {"type": "TEXT"} + return + if vendor == OCIVendors.COHERE: - rf_payload["type"] = response_type - else: - fmt = response_type.upper() - rf_payload["type"] = "JSON_OBJECT" if fmt == "JSON" else fmt + # OCI Cohere has no JSON_SCHEMA type; a schema rides on JSON_OBJECT. + payload: Dict[str, Any] = {"type": "JSON_OBJECT"} + if json_schema is not None and json_schema.get("schema") is not None: + payload["schema"] = json_schema["schema"] + selected_params["responseFormat"] = payload + return + + if rf_type == "json_schema" and json_schema is not None: + # OCI's ResponseJsonSchema accepts only name/description/schema/isStrict. + # OpenAI sends `strict` instead of `isStrict`; forwarding it (or any + # other extra key) makes OCI reject the whole request with HTTP 400. + oci_schema: Dict[str, Any] = {"name": json_schema.get("name") or "response"} + if json_schema.get("description") is not None: + oci_schema["description"] = json_schema["description"] + if json_schema.get("schema") is not None: + oci_schema["schema"] = json_schema["schema"] + if json_schema.get("strict") is not None: + oci_schema["isStrict"] = json_schema["strict"] + selected_params["responseFormat"] = { + "type": "JSON_SCHEMA", + "jsonSchema": oci_schema, + } + return + + fmt = rf_type.upper() + selected_params["responseFormat"] = { + "type": "JSON_OBJECT" if fmt == "JSON" else fmt + } def get_vendor_from_model(model: str) -> OCIVendors: diff --git a/litellm/types/llms/oci.py b/litellm/types/llms/oci.py index df551d8a8c60..621f40aa31dc 100644 --- a/litellm/types/llms/oci.py +++ b/litellm/types/llms/oci.py @@ -291,25 +291,6 @@ class CohereToolResult(BaseModel): outputs: List[Dict[str, Any]] -class CohereResponseFormat(BaseModel): - """Response format for Cohere.""" - - type: str - - -class CohereResponseTextFormat(CohereResponseFormat): - """Text response format for Cohere.""" - - type: Literal["text"] = "text" - - -class CohereResponseJSONSchemaFormat(CohereResponseFormat): - """JSON schema response format for Cohere.""" - - type: Literal["json_schema"] = "json_schema" - jsonSchema: Dict[str, Any] - - class CohereChatRequest(BaseModel): """Cohere chat request model.""" @@ -336,13 +317,10 @@ class CohereChatRequest(BaseModel): # ``OCIChatConfig.openai_to_oci_cohere_param_map`` which marks # ``tool_choice`` as unsupported. The field is intentionally absent here # so it isn't silently dropped or surfaced as a supported feature. - responseFormat: Optional[ - Union[ - CohereResponseTextFormat, - CohereResponseJSONSchemaFormat, - CohereResponseFormat, - ] - ] = None + # OCI Cohere responseFormat is {"type": "TEXT" | "JSON_OBJECT", "schema"?: ...}; + # there is no JSON_SCHEMA type. The shape is built in + # OCIChatConfig._normalize_response_format. + responseFormat: Optional[Dict[str, Any]] = None preambleOverride: Optional[str] = None documents: Optional[List[Dict[str, Any]]] = None searchQueriesOnly: Optional[bool] = None diff --git a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py index e0911e1ef31d..1a2258b8288a 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py @@ -362,6 +362,116 @@ def test_transform_request_response_format_json_shorthand(self): rf = transformed_request["chatRequest"]["responseFormat"] assert rf["type"] == "JSON_OBJECT" + def test_transform_request_response_format_json_schema_generic(self): + """A GENERIC json_schema must become OCI's JSON_SCHEMA shape with the + OpenAI ``strict`` key renamed to ``isStrict``. + + OCI's ResponseJsonSchema rejects ``strict`` (and any other extra key) + with HTTP 400 "Please pass in correct format of request", so the raw + OpenAI body must not be forwarded. + """ + config = OCIChatConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "judgment", + "description": "a score and rationale", + "strict": True, + "schema": { + "type": "object", + "properties": {"score": {"type": "integer"}}, + "required": ["score"], + }, + }, + }, + } + transformed_request = config.transform_request( + model=TEST_MODEL_NAME, # xai.grok-4 -> GENERIC + messages=TEST_MESSAGES, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + rf = transformed_request["chatRequest"]["responseFormat"] + assert rf["type"] == "JSON_SCHEMA" + assert "strict" not in rf["jsonSchema"] + assert rf["jsonSchema"]["isStrict"] is True + assert rf["jsonSchema"]["name"] == "judgment" + assert rf["jsonSchema"]["description"] == "a score and rationale" + assert rf["jsonSchema"]["schema"]["properties"]["score"]["type"] == "integer" + + def test_transform_request_response_format_json_schema_generic_no_strict(self): + """A GENERIC json_schema without ``strict`` must omit ``isStrict``.""" + config = OCIChatConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "response_format": { + "type": "json_schema", + "json_schema": {"name": "j", "schema": {"type": "object"}}, + }, + } + transformed_request = config.transform_request( + model=TEST_MODEL_NAME, + messages=TEST_MESSAGES, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + rf = transformed_request["chatRequest"]["responseFormat"] + assert rf["type"] == "JSON_SCHEMA" + assert "isStrict" not in rf["jsonSchema"] + + def test_transform_request_response_format_json_schema_cohere(self): + """A Cohere json_schema must fold the schema onto JSON_OBJECT. + + OCI Cohere has no JSON_SCHEMA type; sending one yields HTTP 400. + """ + config = OCIChatConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "judgment", + "strict": True, + "schema": { + "type": "object", + "properties": {"score": {"type": "integer"}}, + }, + }, + }, + } + transformed_request = config.transform_request( + model="cohere.command-latest", + messages=TEST_MESSAGES, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + rf = transformed_request["chatRequest"]["responseFormat"] + assert rf["type"] == "JSON_OBJECT" + assert "jsonSchema" not in rf + assert rf["schema"]["properties"]["score"]["type"] == "integer" + + def test_transform_request_response_format_cohere_json_object(self): + """Cohere json_object without a schema stays a bare JSON_OBJECT.""" + config = OCIChatConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "response_format": {"type": "json_object"}, + } + transformed_request = config.transform_request( + model="cohere.command-latest", + messages=TEST_MESSAGES, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + rf = transformed_request["chatRequest"]["responseFormat"] + assert rf == {"type": "JSON_OBJECT"} + def test_transform_response_without_token_details(self): """ Tests that responses missing completionTokensDetails and promptTokensDetails diff --git a/tests/test_litellm/llms/oci/chat/test_oci_cohere_tool_calls.py b/tests/test_litellm/llms/oci/chat/test_oci_cohere_tool_calls.py index cc914a22eeb4..0690293d3fb6 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_cohere_tool_calls.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_cohere_tool_calls.py @@ -236,25 +236,30 @@ def test_cohere_response_with_tool_calls(self): assert result.usage.completion_tokens == 22 assert result.usage.total_tokens == 48 - def test_cohere_request_preserves_json_schema_response_format(self): - """Ensure Cohere requests retain JSON schema payloads in responseFormat.""" + def test_cohere_request_folds_json_schema_into_json_object(self): + """A Cohere json_schema must fold the schema onto JSON_OBJECT. + + OCI Cohere has no JSON_SCHEMA type; sending {"type": "JSON_SCHEMA", ...} + (or the raw lowercase "json_schema" with a jsonSchema body) is rejected + with HTTP 400. The schema rides on JSON_OBJECT instead. + """ config = OCIChatConfig() messages = [{"role": "user", "content": "Return structured info"}] - response_format = { - "type": "json_schema", - "json_schema": { - "name": "test_schema", - "strict": True, - "schema": { - "type": "object", - "properties": {"foo": {"type": "string"}}, - "required": ["foo"], - }, - }, + schema = { + "type": "object", + "properties": {"foo": {"type": "string"}}, + "required": ["foo"], } optional_params = { "oci_compartment_id": TEST_COMPARTMENT_ID, - "response_format": response_format, + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "test_schema", + "strict": True, + "schema": schema, + }, + }, } transformed_request = config.transform_request( @@ -265,18 +270,14 @@ def test_cohere_request_preserves_json_schema_response_format(self): headers={}, ) - chat_request = transformed_request["chatRequest"] - assert chat_request["apiFormat"] == "COHERE" - assert "responseFormat" in chat_request - - cohere_response_format = chat_request["responseFormat"] - assert cohere_response_format["type"] == "json_schema" + cohere_response_format = transformed_request["chatRequest"]["responseFormat"] + assert cohere_response_format["type"] == "JSON_OBJECT" + assert "jsonSchema" not in cohere_response_format assert "json_schema" not in cohere_response_format - assert "jsonSchema" in cohere_response_format - assert cohere_response_format["jsonSchema"] == response_format["json_schema"] + assert cohere_response_format["schema"] == schema - def test_cohere_request_response_format_text_stays_lowercase(self): - """Ensure Cohere keeps response_format type lowercase (e.g. 'text' not 'TEXT').""" + def test_cohere_request_response_format_text_is_uppercased(self): + """Cohere response_format type 'text' maps to OCI's canonical 'TEXT'.""" config = OCIChatConfig() messages = [{"role": "user", "content": "Hello"}] optional_params = { @@ -292,10 +293,7 @@ def test_cohere_request_response_format_text_stays_lowercase(self): headers={}, ) - chat_request = transformed_request["chatRequest"] - assert chat_request["apiFormat"] == "COHERE" - assert "responseFormat" in chat_request - assert chat_request["responseFormat"]["type"] == "text" + assert transformed_request["chatRequest"]["responseFormat"] == {"type": "TEXT"} def test_cohere_tool_call_only_message_no_text(self): """Test chat history with an assistant message that has tool calls but no text content.""" From 9a81127c0bfb820b12afd0f44909d298b93ad537 Mon Sep 17 00:00:00 2001 From: Federico Kamelhar Date: Thu, 4 Jun 2026 14:34:59 -0400 Subject: [PATCH 2/3] fix(oci): raise early on json_schema response_format with no body A GENERIC model request with {"type": "json_schema"} and no json_schema object fell through to the JSON_OBJECT branch and emitted a bodyless {"type": "JSON_SCHEMA"}, which OCI rejects with an opaque HTTP 400. Raise a descriptive 400 at translation time instead. Cohere is unaffected since it always maps to JSON_OBJECT. --- litellm/llms/oci/chat/transformation.py | 7 ++++++- .../oci/chat/test_oci_chat_transformation.py | 21 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index 9ce2e8acc7e2..5cf9129f6544 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -210,7 +210,12 @@ def _normalize_response_format(selected_params: Dict, vendor: OCIVendors) -> Non selected_params["responseFormat"] = payload return - if rf_type == "json_schema" and json_schema is not None: + if rf_type == "json_schema": + if json_schema is None: + raise OCIError( + status_code=400, + message="response_format type 'json_schema' requires a 'json_schema' object", + ) # OCI's ResponseJsonSchema accepts only name/description/schema/isStrict. # OpenAI sends `strict` instead of `isStrict`; forwarding it (or any # other extra key) makes OCI reject the whole request with HTTP 400. diff --git a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py index 1a2258b8288a..2655796d1d67 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py @@ -472,6 +472,27 @@ def test_transform_request_response_format_cohere_json_object(self): rf = transformed_request["chatRequest"]["responseFormat"] assert rf == {"type": "JSON_OBJECT"} + def test_transform_request_json_schema_without_body_raises_generic(self): + """A GENERIC json_schema with no ``json_schema`` body must raise an early + 400, not silently emit {"type": "JSON_SCHEMA"} (which OCI rejects).""" + from litellm.llms.oci.common_utils import OCIError + + config = OCIChatConfig() + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "response_format": {"type": "json_schema"}, + } + with pytest.raises(OCIError) as exc_info: + config.transform_request( + model=TEST_MODEL_NAME, # GENERIC + messages=TEST_MESSAGES, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + assert exc_info.value.status_code == 400 + assert "json_schema" in str(exc_info.value) + def test_transform_response_without_token_details(self): """ Tests that responses missing completionTokensDetails and promptTokensDetails From 151c5280bd21d10dbb4db09eb1761f7dd1a6ad1b Mon Sep 17 00:00:00 2001 From: Federico Kamelhar Date: Thu, 4 Jun 2026 21:45:32 -0400 Subject: [PATCH 3/3] test(oci): gateway integration test for response_format json_schema Added to tests/integration/ (the real-network integration suite) reusing the existing OCI proxy harness, not tests/llm_translation/ which is mock-only. --- .../integration/test_oci_proxy_integration.py | 63 ++++++++++++++++--- 1 file changed, 56 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_oci_proxy_integration.py b/tests/integration/test_oci_proxy_integration.py index 8bfcdd904862..711a93ce3c4f 100644 --- a/tests/integration/test_oci_proxy_integration.py +++ b/tests/integration/test_oci_proxy_integration.py @@ -30,6 +30,7 @@ from __future__ import annotations +import json import os import socket import subprocess @@ -41,7 +42,6 @@ import httpx import pytest - # --------------------------------------------------------------------------- # Skip gate # --------------------------------------------------------------------------- @@ -79,7 +79,9 @@ def _wait_for_health(base_url: str, proc: subprocess.Popen, deadline: float) -> except httpx.HTTPError: pass time.sleep(0.5) - raise RuntimeError(f"litellm proxy did not become ready within {STARTUP_TIMEOUT_S}s") + raise RuntimeError( + f"litellm proxy did not become ready within {STARTUP_TIMEOUT_S}s" + ) def _oci_env_from_profile() -> dict[str, str]: @@ -206,9 +208,7 @@ def test_chat_completion_via_proxy(proxy_url: str, model: str) -> None: # Reasoning models may return empty content if their budget covers only # the thinking turn — accept either text or a non-empty reasoning field. has_content = bool(msg.get("content")) - has_reasoning = bool(msg.get("reasoning_content")) or bool( - msg.get("reasoning") - ) + has_reasoning = bool(msg.get("reasoning_content")) or bool(msg.get("reasoning")) assert has_content or has_reasoning, f"empty assistant message for {model}: {msg}" usage = body.get("usage") or {} assert usage.get("total_tokens", 0) > 0 @@ -232,7 +232,7 @@ def test_chat_completion_streaming_via_proxy(proxy_url: str, model: str) -> None continue if not line.startswith("data:"): continue - payload = line[len("data:"):].strip() + payload = line[len("data:") :].strip() if payload == "[DONE]": saw_done = True break @@ -271,4 +271,53 @@ def test_model_list_advertises_oci_models(proxy_url: str) -> None: assert r.status_code == 200, r.text advertised = {row["id"] for row in r.json()["data"]} for expected in CHAT_MODELS + ["oci-embed"]: - assert expected in advertised, f"{expected} missing from /v1/models: {advertised}" + assert ( + expected in advertised + ), f"{expected} missing from /v1/models: {advertised}" + + +@pytest.mark.parametrize("model", ["oci-cohere-command", "oci-llama"]) +def test_response_format_json_schema_via_proxy(proxy_url: str, model: str) -> None: + """A response_format json_schema succeeds through the gateway for both a + Cohere and a generic OCI model. + + Regression for the HTTP 400 ``Please pass in correct format of request`` + that rejected every json_schema request (which MLflow LLM judges always + send): generic models choke on OpenAI's ``strict`` key, and Cohere has no + JSON_SCHEMA type. + """ + r = httpx.post( + f"{proxy_url}/v1/chat/completions", + headers=_auth_headers(), + json={ + "model": model, + "messages": [ + { + "role": "user", + "content": "Rate the answer 4 to 2+2. Give an integer score and a short rationale.", + } + ], + "max_tokens": 200, + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "judgment", + "strict": True, + "schema": { + "type": "object", + "properties": { + "score": {"type": "integer"}, + "rationale": {"type": "string"}, + }, + "required": ["score", "rationale"], + "additionalProperties": False, + }, + }, + }, + }, + timeout=REQUEST_TIMEOUT_S, + ) + assert r.status_code == 200, f"{model} json_schema -> {r.status_code}: {r.text}" + content = r.json()["choices"][0]["message"]["content"] + assert content is not None + assert "score" in json.loads(content)