Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -362,6 +362,7 @@ For MCP OAuth, an upstream may advertise dynamic client registration but refuse
| [Recraft (`recraft`)](https://docs.litellm.ai/docs/providers/recraft) | | | | | ✅ | | | | | |
| [Replicate (`replicate`)](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ | | | | | | | |
| [Sagemaker Chat (`sagemaker_chat`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | | | | | | | |
| [Sail (`sail`)](https://docs.litellm.ai/docs/providers/sail) | ✅ | ✅ | ✅ | | | | | | | |
| [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ | | | | | | | |
| [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ | | | | | | | |
| [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | |
Expand Down
9 changes: 9 additions & 0 deletions litellm-rust/crates/model-catalog/src/model_info.rs
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,9 @@ pub struct ModelInfo {
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_512k_tokens: Option<f64>,
/// Balanced service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_balanced: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_batches: Option<f64>,
/// Flex service-tier rate for the same-named base field.
Expand Down Expand Up @@ -195,6 +198,9 @@ pub struct ModelInfo {
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_512k_tokens: Option<f64>,
/// Balanced service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_balanced: Option<f64>,
/// USD per prompt token via the provider's batch API.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_batches: Option<f64>,
Expand Down Expand Up @@ -318,6 +324,9 @@ pub struct ModelInfo {
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_512k_tokens: Option<f64>,
/// Balanced service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_balanced: Option<f64>,
/// USD per generated token via the provider's batch API.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_batches: Option<f64>,
Expand Down
2 changes: 2 additions & 0 deletions litellm/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -945,6 +945,7 @@
"https://api.libertai.io/v1",
"https://pinstripes.io/v1",
"https://api.meta.ai/v1",
"https://api.sailresearch.com/v1",
"https://api.cognition.ai/v1",
"https://api.scx.ai/v1",
"https://gigachat.devices.sberbank.ru/api/v1",
Expand Down Expand Up @@ -1020,6 +1021,7 @@
"meta", # Meta Model API (Muse Spark) - JSON-configured provider
"cognition",
"scx-ai",
"sail",
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
]

OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai"} | frozenset(openai_compatible_providers))
Expand Down
3 changes: 2 additions & 1 deletion litellm/litellm_core_utils/llm_cost_calc/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
_SERVICE_TIER_TO_COST_KEY_SUFFIX: Final[Mapping[str, str]] = MappingProxyType(
{
ServiceTier.FLEX.value: ServiceTier.FLEX.value,
ServiceTier.BALANCED.value: ServiceTier.BALANCED.value,
ServiceTier.PRIORITY.value: ServiceTier.PRIORITY.value,
ServiceTier.FAST.value: ServiceTier.PRIORITY.value,
ServiceTier.ULTRAFAST.value: ServiceTier.ULTRAFAST.value,
Expand Down Expand Up @@ -252,7 +253,7 @@ def _get_service_tier_cost_key(base_key: str, service_tier: str | None) -> str:

Args:
base_key: The base cost key (e.g., "input_cost_per_token")
service_tier: The service tier ("flex", "priority", "fast", "ultrafast", or None for standard)
service_tier: The service tier ("flex", "balanced", "priority", "fast", "ultrafast", or None for standard)

Returns:
str: The cost key to use (e.g., "input_cost_per_token_flex" or "input_cost_per_token")
Expand Down
11 changes: 10 additions & 1 deletion litellm/llms/base_llm/chat/transformation.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import types
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator, Iterator
from collections.abc import AsyncIterator, Iterator, Mapping
from typing import TYPE_CHECKING, Any, Final, Union

import httpx
Expand Down Expand Up @@ -255,6 +255,15 @@ def validate_environment(
) -> dict:
pass

def transform_extra_body(
self,
extra_body: Mapping[str, object],
request: Mapping[str, object],
model: str,
litellm_params: Mapping[str, object],
) -> Mapping[str, object]:
return extra_body

def sign_request(
self,
headers: dict,
Expand Down
10 changes: 10 additions & 0 deletions litellm/llms/base_llm/responses/transformation.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import types
from abc import ABC, abstractmethod
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, cast

import httpx
Expand Down Expand Up @@ -364,6 +365,15 @@ def strip_custom_tool_call_namespace_from_responses_input(
out.append(item)
return cast(ResponseInputParam, out)

def transform_extra_body(
self,
extra_body: Mapping[str, object],
request: Mapping[str, object],
model: str,
litellm_params: GenericLiteLLMParams,
) -> Mapping[str, object]:
return extra_body

@staticmethod
def normalize_responses_api_request_dict(data: dict[str, Any]) -> dict[str, Any]:
"""Apply provider-agnostic fixes to an outbound Responses API request dict."""
Expand Down
23 changes: 20 additions & 3 deletions litellm/llms/custom_httpx/llm_http_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -649,7 +649,16 @@ def completion(
def sign_and_log(
transformed: dict[str, object], # mutable-ok: async_completion takes dict
) -> tuple[dict[str, object], dict[str, object], bytes | None]: # mutable-ok: async_completion takes dict
data: Final = {**transformed, **extra_body} if extra_body is not None else transformed
data: Final = (
{
**transformed,
**provider_config.transform_extra_body(
extra_body=extra_body, request=transformed, model=model, litellm_params=litellm_params
),
}
if extra_body is not None
else transformed
)
signed: Final = cast( # cast-ok: sign_request is declared as a bare dict
"tuple[dict[str, object], bytes | None]",
provider_config.sign_request(
Expand Down Expand Up @@ -2421,7 +2430,11 @@ def response_api_handler(
data = BaseResponsesAPIConfig.normalize_responses_api_request_dict(data)

if extra_body:
data.update(extra_body)
data.update(
responses_api_provider_config.transform_extra_body(
extra_body=extra_body, request=data, model=model, litellm_params=litellm_params
)
)
stream = bool(stream or data.get("stream"))

# Preserve the OpenAI-style request context (not sent to the provider) for streaming
Expand Down Expand Up @@ -2609,7 +2622,11 @@ async def async_response_api_handler(
data = BaseResponsesAPIConfig.normalize_responses_api_request_dict(data)

if extra_body:
data.update(extra_body)
data.update(
responses_api_provider_config.transform_extra_body(
extra_body=extra_body, request=data, model=model, litellm_params=litellm_params
)
)
stream = bool(stream or data.get("stream"))

# Preserve the OpenAI-style request context (not sent to the provider) for streaming
Expand Down
7 changes: 5 additions & 2 deletions litellm/llms/openai_like/dynamic_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"""

from collections.abc import Coroutine
from typing import Any, Final, Literal, overload
from typing import TYPE_CHECKING, Any, Final, Literal, overload

from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.common_utils import (
Expand All @@ -16,6 +16,9 @@

from .json_loader import SimpleProviderConfig

if TYPE_CHECKING:
from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig


def create_config_class(provider: SimpleProviderConfig):
"""Generate config class dynamically from JSON configuration"""
Expand Down Expand Up @@ -173,7 +176,7 @@ def custom_llm_provider(self) -> str | None:
_responses_config_cache: Final[dict] = {}


def create_responses_config_class(provider: SimpleProviderConfig):
def create_responses_config_class(provider: SimpleProviderConfig) -> "type[OpenAILikeResponsesConfig]":
"""Generate a Responses API config class dynamically from JSON configuration.

Parallel to create_config_class() but for /v1/responses endpoints.
Expand Down
6 changes: 6 additions & 0 deletions litellm/llms/openai_like/providers.json
Original file line number Diff line number Diff line change
Expand Up @@ -200,5 +200,11 @@
"temperature_max": 1.99
},
"supported_endpoints": ["/v1/chat/completions"]
},
"sail": {
"base_url": "https://api.sailresearch.com/v1",
"api_key_env": "SAIL_API_KEY",
"api_base_env": "SAIL_API_BASE",
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
}
}
72 changes: 72 additions & 0 deletions litellm/llms/sail/chat/transformation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
from collections.abc import Mapping
from typing import Final

from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.llms.sail.common_utils import (
chat_request_for_sail,
completion_window_for_service_tier,
extra_body_for_sail,
json_body,
)
from litellm.types.llms.openai import AllMessageValues

_REJECTED_BY_SAIL: Final = frozenset(
{"stop", "seed", "frequency_penalty", "presence_penalty", "logit_bias", "logprobs", "top_logprobs"}
)
_ACCEPTED_BY_SAIL: Final = ("reasoning_effort", "user")


class SailChatConfig(OpenAIGPTConfig):
def get_supported_openai_params(self, model: str) -> list: # mutable-ok: return type fixed by the base interface
inherited: Final = tuple(
param for param in super().get_supported_openai_params(model) if param not in _REJECTED_BY_SAIL
)
added: Final = tuple(param for param in _ACCEPTED_BY_SAIL if param not in inherited)
return [*inherited, *added] # mutable-ok: the base interface returns a list

def map_openai_params(
self,
non_default_params: dict, # mutable-ok: signature fixed by the base interface
optional_params: dict, # mutable-ok: signature fixed by the base interface
model: str,
drop_params: bool,
) -> dict: # mutable-ok: return type fixed by the base interface
completion_window_for_service_tier(non_default_params.get("service_tier"), model=model, drop_params=drop_params)
return super().map_openai_params(
non_default_params=non_default_params,
optional_params=optional_params,
model=model,
drop_params=drop_params,
)

def transform_request(
self,
model: str,
messages: list[AllMessageValues], # mutable-ok: signature fixed by the base interface
optional_params: dict, # mutable-ok: signature fixed by the base interface
litellm_params: dict, # mutable-ok: signature fixed by the base interface
headers: dict, # mutable-ok: signature fixed by the base interface
) -> dict: # mutable-ok: return type fixed by the base interface
request: Final = chat_request_for_sail(
super().transform_request(
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
headers=headers,
),
model=model,
drop_params=bool(litellm_params.get("drop_params")),
)
return json_body(request)

def transform_extra_body(
self,
extra_body: Mapping[str, object],
request: Mapping[str, object],
model: str,
litellm_params: Mapping[str, object],
) -> Mapping[str, object]:
return extra_body_for_sail(
extra_body, request.get("metadata"), model=model, drop_params=bool(litellm_params.get("drop_params"))
)
Loading
Loading