Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions litellm/cost_calculator.py
Original file line number Diff line number Diff line change
Expand Up @@ -2375,6 +2375,11 @@ def default_image_cost_calculator(
model_name_without_custom_llm_provider = model.replace(f"{custom_llm_provider}/", "")
base_model_name = f"{custom_llm_provider}/{size_str}/{model_name_without_custom_llm_provider}"
model_name_with_quality: Final = f"{quality}/{base_model_name}" if quality else base_model_name
provider_first_model_name_with_quality: Final = (
f"{custom_llm_provider}/{quality}/{size_str}/{model_name_without_custom_llm_provider or model}"
if quality and custom_llm_provider
else None
)

# gpt-image-1 models use low, medium, high quality. If user did not specify quality, use medium fot gpt-image-1 model family
model_name_with_v2_quality: Final = f"{ImageGenerationRequestQuality.HIGH.value}/{base_model_name}"
Expand All @@ -2386,6 +2391,7 @@ def default_image_cost_calculator(

models_to_check: Final = (
model_name_with_quality,
provider_first_model_name_with_quality,
base_model_name,
model_name_with_v2_quality,
model_with_quality_without_provider,
Expand Down
30 changes: 30 additions & 0 deletions litellm/llms/bedrock/chat/invoke_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,12 @@ async def make_call(
sync_stream=False,
)
completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=stream_chunk_size))
elif bedrock_invoke_provider == "moonshot":
decoder = AmazonOpenAICompatibleStreamDecoder(
model=model,
sync_stream=False,
)
completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=stream_chunk_size))
else:
decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode)
completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=stream_chunk_size))
Expand Down Expand Up @@ -329,6 +335,12 @@ def make_sync_call(
sync_stream=True,
)
completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
elif bedrock_invoke_provider == "moonshot":
decoder = AmazonOpenAICompatibleStreamDecoder(
model=model,
sync_stream=True,
)
completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
else:
decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode)
completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
Expand Down Expand Up @@ -795,6 +807,24 @@ def _chunk_parser(self, chunk_data: dict) -> GChunk | ModelResponseStream | dict
return self.deepseek_model_response_iterator.chunk_parser(chunk=chunk_data)


class AmazonOpenAICompatibleStreamDecoder(AWSEventStreamDecoder):
def __init__(
self,
model: str,
sync_stream: bool,
) -> None:
super().__init__(model=model)
from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler

self.openai_model_response_iterator = OpenAIChatCompletionStreamingHandler(
streaming_response=None,
sync_stream=sync_stream,
)

def _chunk_parser(self, chunk_data: dict[str, object]) -> ModelResponseStream:
return self.openai_model_response_iterator.chunk_parser(chunk=chunk_data)


class MockResponseIterator: # for returning ai21 streaming responses
def __init__(self, model_response, json_mode: bool | None = False):
self.model_response = model_response
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,10 @@ def _is_gemini_3_or_newer(model: str) -> bool:
Check if the model is Gemini 3 or newer.
"""
model_name = model.split("/")[-1].lower()
if not model_name:
is_vertex_fine_tuned_model: Final = model_name.isdigit() or (
model.startswith("gemini/") and not model_name.startswith("gemini-")
)
if not model_name or is_vertex_fine_tuned_model or model_name.startswith("gemma-"):
return False
# Pre-Gemini 3 models: gemini-1.x, gemini-2.x, gemini-pro, gemini-flash, gemini-exp
if re.match(r"^gemini-(?:[12](?:\.\d+)?|exp|(?:pro|flash)(?!-(?:lite-)?latest$))(?:-|$)", model_name):
Expand Down
Loading
Loading