Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 36 additions & 1 deletion litellm/llms/oci/chat/transformation.py
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,7 @@ def __init__(
"parallel_tool_calls": False,
"audio": False,
"web_search_options": False,
"response_format": "responseFormat",
}

# Cohere and Gemini use the same parameter mapping as GENERIC
Expand Down Expand Up @@ -269,6 +270,9 @@ def map_openai_params(

adapted_params[alias] = value

if alias == "responseFormat":
adapted_params["response_format"] = value

return adapted_params

def _sign_with_oci_signer(
Expand Down Expand Up @@ -673,6 +677,36 @@ def _get_optional_params(self, vendor: OCIVendors, optional_params: dict) -> Dic
selected_params["tools"] = adapt_tool_definition_to_oci_standard( # type: ignore[assignment]
selected_params["tools"], vendor # type: ignore[arg-type]
)

# Transform response_format type to OCI uppercase format
if "responseFormat" in selected_params:
rf = selected_params["responseFormat"]
if isinstance(rf, dict) and "type" in rf:
rf_payload = dict(rf)
selected_params["responseFormat"] = rf_payload

response_type = rf_payload["type"]
schema_payload: Optional[Any] = None

if "json_schema" in rf_payload:
raw_schema_payload = rf_payload.pop("json_schema")
if isinstance(raw_schema_payload, dict):
schema_payload = dict(raw_schema_payload)
else:
schema_payload = raw_schema_payload

if schema_payload is not None:
rf_payload["jsonSchema"] = schema_payload

if vendor == OCIVendors.COHERE:
# Cohere expects lower-case type values
rf_payload["type"] = response_type
else:
format_type = response_type.upper()
if format_type == "JSON":
format_type = "JSON_OBJECT"
rf_payload["type"] = format_type

return selected_params

def adapt_messages_to_cohere_standard(self, messages: List[AllMessageValues]) -> List[CohereMessage]:
Expand Down Expand Up @@ -806,11 +840,12 @@ def transform_request(


# Create Cohere-specific chat request
optional_cohere_params = self._get_optional_params(OCIVendors.COHERE, optional_params)
chat_request = CohereChatRequest(
apiFormat="COHERE",
message=self._extract_text_content(user_messages[-1]["content"]),
chatHistory=self.adapt_messages_to_cohere_standard(messages),
**self._get_optional_params(OCIVendors.COHERE, optional_params)
**optional_cohere_params
)

data = OCICompletionPayload(
Expand Down
58 changes: 32 additions & 26 deletions litellm/types/llms/oci.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ class OCIChatRequestPayload(BaseModel):
seed: Optional[int] = None
frequencyPenalty: Optional[float] = None
presencePenalty: Optional[float] = None
responseFormat: Optional[Dict[str, Any]] = None


class OCIServingMode(BaseModel):
Expand All @@ -125,14 +126,14 @@ class OCICompletionPayload(BaseModel):
class OCICompletionTokenDetails(BaseModel):
"""Completion token details in the OCI response."""

acceptedPredictionTokens: int
reasoningTokens: int
acceptedPredictionTokens: Optional[int] = None
reasoningTokens: Optional[int] = None


class OCIPromptTokensDetails(BaseModel):
"""Prompt token details in the OCI response."""

cachedTokens: int
cachedTokens: Optional[int] = None


class OCIResponseUsage(BaseModel):
Expand Down Expand Up @@ -205,87 +206,93 @@ class CohereStreamChunk(BaseModel):

class CohereMessage(BaseModel):
"""Base model for Cohere messages."""

role: str
message: str
message: Optional[str] = None
toolCalls: Optional[List[CohereToolCall]] = None


class CohereUserMessage(CohereMessage):
"""User message in Cohere chat."""

role: Literal["USER"] = "USER"


class CohereChatBotMessage(CohereMessage):
"""Chatbot message in Cohere chat."""

role: Literal["CHATBOT"] = "CHATBOT"


class CohereSystemMessage(CohereMessage):
"""System message in Cohere chat."""

role: Literal["SYSTEM"] = "SYSTEM"


class CohereToolMessage(CohereMessage):
"""Tool message in Cohere chat."""

role: Literal["TOOL"] = "TOOL"
toolCallId: str


class CohereParameterDefinition(BaseModel):
"""Parameter definition for Cohere tools."""

description: str
type: str
isRequired: bool = False


class CohereTool(BaseModel):
"""Tool definition for Cohere."""

name: str
description: str
parameterDefinitions: Dict[str, CohereParameterDefinition]


class CohereToolCall(BaseModel):
"""Tool call made by Cohere model."""

name: str
parameters: Dict[str, Any]


class CohereToolResult(BaseModel):
"""Result of a tool call."""

callId: str
result: str


class CohereResponseFormat(BaseModel):
"""Response format for Cohere."""

type: str


class CohereResponseTextFormat(CohereResponseFormat):
"""Text response format for Cohere."""

type: Literal["text"] = "text"


class CohereResponseJSONSchemaFormat(CohereResponseFormat):
"""JSON schema response format for Cohere."""

type: Literal["json_schema"] = "json_schema"
jsonSchema: Dict[str, Any]


class CohereChatRequest(BaseModel):
"""Cohere chat request model."""

# Required fields
message: str
apiFormat: Literal["COHERE"] = "COHERE"

# Optional fields
chatHistory: Optional[List[CohereMessage]] = None
maxTokens: Optional[int] = None
Expand All @@ -298,7 +305,7 @@ class CohereChatRequest(BaseModel):
seed: Optional[int] = None
tools: Optional[List[CohereTool]] = None
toolChoice: Optional[Union[str, Dict[str, Any]]] = None
responseFormat: Optional[CohereResponseFormat] = None
responseFormat: Optional[Union[CohereResponseTextFormat, CohereResponseJSONSchemaFormat, CohereResponseFormat]] = None
preambleOverride: Optional[str] = None
documents: Optional[List[Dict[str, Any]]] = None
searchQueriesOnly: Optional[bool] = None
Expand All @@ -318,7 +325,7 @@ class CohereChatRequest(BaseModel):

class CohereUsage(BaseModel):
"""Usage information for Cohere response."""

promptTokens: int
completionTokens: int
totalTokens: int
Expand All @@ -328,7 +335,7 @@ class CohereUsage(BaseModel):

class CohereCitation(BaseModel):
"""Citation in Cohere response."""

start: int
end: int
text: str
Expand All @@ -337,19 +344,19 @@ class CohereCitation(BaseModel):

class CohereSearchQuery(BaseModel):
"""Search query generated by Cohere."""

text: str
generation_id: str


class CohereChatResponse(BaseModel):
"""Cohere chat response model."""

# Required fields
text: str
apiFormat: Literal["COHERE"] = "COHERE"
finishReason: Literal["COMPLETE", "ERROR_TOXIC", "ERROR_LIMIT", "ERROR", "USER_CANCEL", "MAX_TOKENS"]

# Optional fields
chatHistory: Optional[List[CohereMessage]] = None
citations: Optional[List[CohereCitation]] = None
Expand All @@ -364,16 +371,15 @@ class CohereChatResponse(BaseModel):

class CohereChatDetails(BaseModel):
"""Chat details for Cohere request."""

compartmentId: str
servingMode: OCIServingMode
chatRequest: CohereChatRequest


class CohereChatResult(BaseModel):
"""Complete Cohere chat result."""

modelId: str
modelVersion: str
chatResponse: CohereChatResponse

108 changes: 108 additions & 0 deletions tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,114 @@ def test_transform_request_dedicated_cohere_with_endpoint_id(self):
# Verify the message content
assert transformed_request["chatRequest"]["message"] == "What is quantum computing?"

def test_transform_request_response_format_json_object(self):
"""
Tests that response_format type 'json_object' is uppercased to 'JSON_OBJECT' for generic OCI models.
"""
config = OCIChatConfig()
optional_params = {
"oci_compartment_id": TEST_COMPARTMENT_ID,
"response_format": {"type": "json_object"},
}
transformed_request = config.transform_request(
model=TEST_MODEL_NAME,
messages=TEST_MESSAGES, # type: ignore
optional_params=optional_params,
litellm_params={},
headers={},
)
rf = transformed_request["chatRequest"]["responseFormat"]
assert rf["type"] == "JSON_OBJECT"

def test_transform_request_response_format_text(self):
"""
Tests that response_format type 'text' is uppercased to 'TEXT' for generic OCI models.
"""
config = OCIChatConfig()
optional_params = {
"oci_compartment_id": TEST_COMPARTMENT_ID,
"response_format": {"type": "text"},
}
transformed_request = config.transform_request(
model=TEST_MODEL_NAME,
messages=TEST_MESSAGES, # type: ignore
optional_params=optional_params,
litellm_params={},
headers={},
)
rf = transformed_request["chatRequest"]["responseFormat"]
assert rf["type"] == "TEXT"

def test_transform_request_response_format_json_shorthand(self):
"""
Tests that response_format type 'json' is mapped to 'JSON_OBJECT' for generic OCI models.
"""
config = OCIChatConfig()
optional_params = {
"oci_compartment_id": TEST_COMPARTMENT_ID,
"response_format": {"type": "json"},
}
transformed_request = config.transform_request(
model=TEST_MODEL_NAME,
messages=TEST_MESSAGES, # type: ignore
optional_params=optional_params,
litellm_params={},
headers={},
)
rf = transformed_request["chatRequest"]["responseFormat"]
assert rf["type"] == "JSON_OBJECT"

def test_transform_response_without_token_details(self):
"""
Tests that responses missing completionTokensDetails and promptTokensDetails
are handled correctly (fields are optional).
"""
config = OCIChatConfig()
created_time = datetime.datetime.now(datetime.timezone.utc).isoformat().replace("+00:00", "Z")
mock_oci_response = {
"modelId": TEST_MODEL_NAME,
"modelVersion": "1.0",
"chatResponse": {
"apiFormat": "GENERIC",
"choices": [
{
"index": 0,
"message": {
"role": "ASSISTANT",
"content": [{"type": "TEXT", "text": "Hello!"}],
},
"finishReason": "STOP",
}
],
"timeCreated": created_time,
"usage": {
"promptTokens": 5,
"completionTokens": 10,
"totalTokens": 15,
},
},
}
response = httpx.Response(
status_code=200, json=mock_oci_response, headers={"Content-Type": "application/json"}
)
result = config.transform_response(
model=TEST_MODEL_NAME,
raw_response=response,
model_response=ModelResponse(),
logging_obj={}, # type: ignore
request_data={},
messages=[],
optional_params={},
litellm_params={},
encoding={},
)

assert isinstance(result, ModelResponse)
assert result.choices[0].message.content == "Hello!"
assert result.usage.prompt_tokens == 5 # type: ignore
assert result.usage.completion_tokens == 10 # type: ignore
assert result.usage.total_tokens == 15 # type: ignore

def test_transform_response_simple_text(self):
"""
Tests if a simple text response is transformed correctly.
Expand Down
Loading
Loading