Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions litellm/litellm_core_utils/streaming_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -1128,6 +1128,32 @@ def chunk_creator(self, chunk: Any): # type: ignore # noqa: PLR0915
completion_obj: Dict[str, Any] = {"content": ""}
from litellm.types.utils import GenericStreamingChunk as GChunk

if (
isinstance(chunk, ModelResponseStream)
and self.custom_llm_provider is not None
and self.custom_llm_provider in litellm._custom_providers
):
_has_content = bool(
chunk.choices
and chunk.choices[0].delta is not None
and (
chunk.choices[0].delta.content
or chunk.choices[0].delta.tool_calls
)
)
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
if self.received_finish_reason is not None:
if not _has_content:
raise StopIteration
if chunk.choices and chunk.choices[0].finish_reason:
self.received_finish_reason = chunk.choices[0].finish_reason
if not _has_content:
return None
# Strip finish_reason from the content chunk so it appears
# only on the trailing empty-delta chunk (OpenAI spec).
# finish_reason_handler() will emit the proper terminal chunk.
chunk.choices[0].finish_reason = None # type: ignore[assignment]
return chunk

if (
isinstance(chunk, dict)
and generic_chunk_has_all_required_fields(
Expand Down
88 changes: 87 additions & 1 deletion tests/local_testing/test_custom_llm.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,14 @@
image_generation,
)
from litellm.utils import ModelResponseIterator
from litellm.types.utils import ImageResponse, ImageObject, EmbeddingResponse
from litellm.types.utils import (
ImageResponse,
ImageObject,
EmbeddingResponse,
ModelResponseStream,
StreamingChoices,
Delta,
)
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler


Expand Down Expand Up @@ -644,3 +651,82 @@ async def test_simple_aembedding():
"embedding": [0.1, 0.2, 0.3],
"index": 1,
}


# ── Tests for ModelResponseStream passthrough in custom providers (issue #27389) ──


class ModelResponseStreamLLM(MyCustomLLM):
"""Subclass that overrides streaming/astreaming to yield ModelResponseStream directly."""

def __init__(self, finish_reason: str = "stop"):
self._finish_reason = finish_reason

def streaming(self, *args, **kwargs) -> Iterator[ModelResponseStream]: # type: ignore
yield ModelResponseStream(
id="test-stream-id",
choices=[
StreamingChoices(
index=0,
delta=Delta(content="Hello world"),
finish_reason=self._finish_reason,
)
],
)

async def astreaming(self, *args, **kwargs) -> AsyncIterator[ModelResponseStream]: # type: ignore
yield ModelResponseStream(
id="test-stream-id",
choices=[
StreamingChoices(
index=0,
delta=Delta(content="Hello world"),
finish_reason=self._finish_reason,
)
],
)


@pytest.mark.parametrize(
"finish_reason", ["stop", "tool_calls", "length", "content_filter"]
)
def test_custom_llm_streaming_model_response_stream(finish_reason):
my_custom_llm = ModelResponseStreamLLM(finish_reason=finish_reason)
litellm.custom_provider_map = [
{"provider": "custom_llm", "custom_handler": my_custom_llm}
]
resp = completion(
model="custom_llm/my-fake-model",
messages=[{"role": "user", "content": "Hello world!"}],
stream=True,
)

for chunk in resp:
print(chunk)
if chunk.choices[0].finish_reason is None:
assert isinstance(chunk.choices[0].delta.content, str)
else:
assert chunk.choices[0].finish_reason == finish_reason


@pytest.mark.asyncio
@pytest.mark.parametrize(
"finish_reason", ["stop", "tool_calls", "length", "content_filter"]
)
async def test_custom_llm_astreaming_model_response_stream(finish_reason):
my_custom_llm = ModelResponseStreamLLM(finish_reason=finish_reason)
litellm.custom_provider_map = [
{"provider": "custom_llm", "custom_handler": my_custom_llm}
]
resp = await litellm.acompletion(
model="custom_llm/my-fake-model",
messages=[{"role": "user", "content": "Hello world!"}],
stream=True,
)

async for chunk in resp:
print(chunk)
if chunk.choices[0].finish_reason is None:
assert isinstance(chunk.choices[0].delta.content, str)
else:
assert chunk.choices[0].finish_reason == finish_reason
183 changes: 174 additions & 9 deletions tests/test_litellm/litellm_core_utils/test_streaming_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -2036,23 +2036,19 @@ async def test_azure_streaming_role_preserved_with_include_usage(sync_mode: bool
chunks.append(chunk)

# The prompt_filter chunk should be forwarded with choices=[]
assert len(chunks[0].choices) == 0, (
f"Expected prompt_filter chunk with choices=[], got {len(chunks[0].choices)} choices"
)
assert (
len(chunks[0].choices) == 0
), f"Expected prompt_filter chunk with choices=[], got {len(chunks[0].choices)} choices"

# At least one chunk must have role='assistant' in its delta
has_role = any(
len(c.choices) > 0
and getattr(c.choices[0].delta, "role", None) == "assistant"
len(c.choices) > 0 and getattr(c.choices[0].delta, "role", None) == "assistant"
for c in chunks
)
assert has_role, (
"No chunk contained role='assistant' in delta (issue #24221). "
"Chunk deltas: "
+ str([
c.choices[0].delta if c.choices else "no choices"
for c in chunks
])
+ str([c.choices[0].delta if c.choices else "no choices" for c in chunks])
)


Expand Down Expand Up @@ -2124,3 +2120,172 @@ def test_gemini_legacy_vertex_tool_calls_finish_reason_with_stop_enum():
f"Expected 'tool_calls' but got {final.choices[0].finish_reason!r}. "
"STOP enum was not normalised through map_finish_reason()."
)


@pytest.mark.parametrize(
"finish_reason", ["stop", "tool_calls", "length", "content_filter"]
)
def test_chunk_creator_passes_through_model_response_stream(
initialized_custom_stream_wrapper: CustomStreamWrapper,
finish_reason: str,
):
"""
chunk_creator must pass ModelResponseStream chunks from custom providers
straight through and preserve finish_reason exactly — not force-cast to GChunk.
Regression test for issue #27389.
"""
initialized_custom_stream_wrapper.custom_llm_provider = "my-custom-provider"
litellm._custom_providers.append("my-custom-provider")

chunk = ModelResponseStream(
id="test-id",
choices=[
StreamingChoices(
index=0,
delta=Delta(content="Hello", role="assistant"),
finish_reason=finish_reason,
)
],
)

result = initialized_custom_stream_wrapper.chunk_creator(chunk=chunk)

litellm._custom_providers.remove("my-custom-provider")

assert result is not None
assert initialized_custom_stream_wrapper.received_finish_reason == finish_reason


def test_chunk_creator_drops_empty_finish_chunk(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):
"""
A ModelResponseStream chunk with finish_reason but no content should return
None so finish_reason_handler() synthesises the final chunk — mirrors GChunk
behaviour via is_chunk_non_empty.
"""
initialized_custom_stream_wrapper.custom_llm_provider = "my-custom-provider"
litellm._custom_providers.append("my-custom-provider")

chunk = ModelResponseStream(
id="test-id",
choices=[
StreamingChoices(
index=0,
delta=Delta(content=""),
finish_reason="stop",
)
],
)

result = initialized_custom_stream_wrapper.chunk_creator(chunk=chunk)

litellm._custom_providers.remove("my-custom-provider")

assert result is None
assert initialized_custom_stream_wrapper.received_finish_reason == "stop"


def test_chunk_creator_stops_iteration_on_trailing_chunk(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):
"""
After received_finish_reason is set, any empty trailing chunk (e.g. provider
metadata flush) must raise StopIteration to end the stream cleanly.
"""
initialized_custom_stream_wrapper.custom_llm_provider = "my-custom-provider"
initialized_custom_stream_wrapper.received_finish_reason = "stop"
litellm._custom_providers.append("my-custom-provider")

trailing_chunk = ModelResponseStream(
id="test-id",
choices=[
StreamingChoices(
index=0,
delta=Delta(content=None),
finish_reason="stop",
)
],
)

with pytest.raises(StopIteration):
initialized_custom_stream_wrapper.chunk_creator(chunk=trailing_chunk)

litellm._custom_providers.remove("my-custom-provider")


def test_chunk_creator_strips_finish_reason_from_content_chunk(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):
"""
When content and finish_reason arrive in the same chunk, finish_reason must be
stripped so finish_reason_handler() emits it on the synthetic terminal chunk —
preventing two terminal chunks (double finish_reason bug).
"""
initialized_custom_stream_wrapper.custom_llm_provider = "my-custom-provider"
litellm._custom_providers.append("my-custom-provider")

chunk = ModelResponseStream(
id="test-id",
choices=[
StreamingChoices(
index=0,
delta=Delta(content="Hello"),
finish_reason="stop",
)
],
)

result = initialized_custom_stream_wrapper.chunk_creator(chunk=chunk)

litellm._custom_providers.remove("my-custom-provider")

assert result is not None
assert (
result.choices[0].finish_reason is None
), "finish_reason must be stripped from content chunks to avoid double terminal chunks"
assert initialized_custom_stream_wrapper.received_finish_reason == "stop"


def test_chunk_creator_tool_calls_not_dropped_on_finish(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):
"""
A terminal chunk with finish_reason="tool_calls" and delta.tool_calls must NOT
be silently dropped — tool_calls counts as content so the chunk is passed through
(with finish_reason stripped) rather than returning None.
"""
from litellm.types.utils import ChatCompletionDeltaToolCall, Function

initialized_custom_stream_wrapper.custom_llm_provider = "my-custom-provider"
litellm._custom_providers.append("my-custom-provider")

chunk = ModelResponseStream(
id="test-id",
choices=[
StreamingChoices(
index=0,
delta=Delta(
content=None,
tool_calls=[
ChatCompletionDeltaToolCall(
id="call_abc",
function=Function(name="get_weather", arguments='{"city":"NYC"}'),
type="function",
index=0,
)
],
),
finish_reason="tool_calls",
)
],
)

result = initialized_custom_stream_wrapper.chunk_creator(chunk=chunk)

litellm._custom_providers.remove("my-custom-provider")

assert result is not None, "tool_calls chunk must not be dropped"
assert result.choices[0].delta.tool_calls is not None
assert result.choices[0].finish_reason is None
assert initialized_custom_stream_wrapper.received_finish_reason == "tool_calls"
Loading