Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions litellm/cost_calculator.py
Original file line number Diff line number Diff line change
Expand Up @@ -2375,6 +2375,11 @@ def default_image_cost_calculator(
model_name_without_custom_llm_provider = model.replace(f"{custom_llm_provider}/", "")
base_model_name = f"{custom_llm_provider}/{size_str}/{model_name_without_custom_llm_provider}"
model_name_with_quality: Final = f"{quality}/{base_model_name}" if quality else base_model_name
provider_first_model_name_with_quality: Final = (
f"{custom_llm_provider}/{quality}/{size_str}/{model_name_without_custom_llm_provider or model}"
if quality and custom_llm_provider
else None
)

# gpt-image-1 models use low, medium, high quality. If user did not specify quality, use medium fot gpt-image-1 model family
model_name_with_v2_quality: Final = f"{ImageGenerationRequestQuality.HIGH.value}/{base_model_name}"
Expand All @@ -2386,6 +2391,7 @@ def default_image_cost_calculator(

models_to_check: Final = (
model_name_with_quality,
provider_first_model_name_with_quality,
base_model_name,
model_name_with_v2_quality,
model_with_quality_without_provider,
Expand Down
30 changes: 30 additions & 0 deletions litellm/llms/bedrock/chat/invoke_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,12 @@ async def make_call(
sync_stream=False,
)
completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=stream_chunk_size))
elif bedrock_invoke_provider == "moonshot":
decoder = AmazonOpenAICompatibleStreamDecoder(
model=model,
sync_stream=False,
)
completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=stream_chunk_size))
else:
decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode)
completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=stream_chunk_size))
Expand Down Expand Up @@ -329,6 +335,12 @@ def make_sync_call(
sync_stream=True,
)
completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
elif bedrock_invoke_provider == "moonshot":
decoder = AmazonOpenAICompatibleStreamDecoder(
model=model,
sync_stream=True,
)
completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
else:
decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode)
completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
Expand Down Expand Up @@ -795,6 +807,24 @@ def _chunk_parser(self, chunk_data: dict) -> GChunk | ModelResponseStream | dict
return self.deepseek_model_response_iterator.chunk_parser(chunk=chunk_data)


class AmazonOpenAICompatibleStreamDecoder(AWSEventStreamDecoder):
def __init__(
self,
model: str,
sync_stream: bool,
) -> None:
super().__init__(model=model)
from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler

self.openai_model_response_iterator = OpenAIChatCompletionStreamingHandler(
streaming_response=None,
sync_stream=sync_stream,
)

def _chunk_parser(self, chunk_data: dict[str, object]) -> ModelResponseStream:
return self.openai_model_response_iterator.chunk_parser(chunk=chunk_data)


class MockResponseIterator: # for returning ai21 streaming responses
def __init__(self, model_response, json_mode: bool | None = False):
self.model_response = model_response
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,10 @@ def _is_gemini_3_or_newer(model: str) -> bool:
Check if the model is Gemini 3 or newer.
"""
model_name = model.split("/")[-1].lower()
if not model_name:
is_vertex_fine_tuned_model: Final = model_name.isdigit() or (
model.startswith("gemini/") and not model_name.startswith("gemini-")
)
if not model_name or is_vertex_fine_tuned_model or model_name.startswith("gemma-"):
return False
# Pre-Gemini 3 models: gemini-1.x, gemini-2.x, gemini-pro, gemini-flash, gemini-exp
if re.match(r"^gemini-(?:[12](?:\.\d+)?|exp|(?:pro|flash)(?!-(?:lite-)?latest$))(?:-|$)", model_name):
Expand Down
Loading
Loading