Skip to content

fix: emit input_json_delta for tool args bundled in first streaming chunk #25533

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
krrish-berri-2 merged 5 commits into from
Apr 14, 2026
Merged

fix: emit input_json_delta for tool args bundled in first streaming chunk #25533

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
fb0cc44
fix: emit input_json_delta for tool args bundled in first streaming c…
krisyang1125 Apr 10, 2026
f797b16
test: add tests for input_json_delta emission on bundled tool args
krisyang1125 Apr 10, 2026
ad2c039
style: apply Black formatting to streaming_iterator.py
krisyang1125 Apr 10, 2026
a9906f4
fix: mirror input_json_delta fix to sync __next__ and add sync tests
krisyang1125 Apr 10, 2026
6164735
test: make no_extra_delta tests assert explicitly instead of passing …
krisyang1125 Apr 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 44 additions & 10 deletions litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,21 +129,40 @@ def __next__(self):

if should_start_new_block and not self.sent_content_block_finish:
# Queue the sequence: content_block_stop -> content_block_start
# The trigger chunk itself is not emitted as a delta since the
# content_block_start already carries the relevant information.
# For text blocks the trigger chunk is not emitted as a separate
# delta because content_block_start carries the information.
# For tool_use blocks we must also emit the trigger chunk's delta
# when it carries input_json_delta data, because some providers
# (e.g. xAI, Gemini) include tool arguments in the same streaming
# chunk as the function name/id.

# 1. Stop current content block
self.chunk_queue.append(
{
"type": "content_block_stop",
"index": max(self.current_content_block_index - 1, 0),
}
)

# 2. Start new content block
self.chunk_queue.append(
{
"type": "content_block_start",
"index": self.current_content_block_index,
"content_block": self.current_content_block_start,
}
)

# 3. If the trigger chunk carries tool argument data, queue it
# so the input_json_delta is not silently dropped.
if (
processed_chunk.get("type") == "content_block_delta"
and isinstance(processed_chunk.get("delta"), dict)
and processed_chunk["delta"].get("type") == "input_json_delta"
and processed_chunk["delta"].get("partial_json")
):
self.chunk_queue.append(processed_chunk)

self.sent_content_block_finish = False
return self.chunk_queue.popleft()

Expand Down Expand Up @@ -282,16 +301,16 @@ async def __anext__(self): # noqa: PLR0915
hasattr(chunk.usage, "_cache_creation_input_tokens")
and chunk.usage._cache_creation_input_tokens > 0
):
usage_dict[
"cache_creation_input_tokens"
] = chunk.usage._cache_creation_input_tokens
usage_dict["cache_creation_input_tokens"] = (
chunk.usage._cache_creation_input_tokens
)
if (
hasattr(chunk.usage, "_cache_read_input_tokens")
and chunk.usage._cache_read_input_tokens > 0
):
usage_dict[
"cache_read_input_tokens"
] = chunk.usage._cache_read_input_tokens
usage_dict["cache_read_input_tokens"] = (
chunk.usage._cache_read_input_tokens
)
merged_chunk["usage"] = usage_dict

# Queue the merged chunk and reset
Expand All @@ -305,8 +324,12 @@ async def __anext__(self): # noqa: PLR0915
if not self.queued_usage_chunk:
if should_start_new_block and not self.sent_content_block_finish:
# Queue the sequence: content_block_stop -> content_block_start
# The trigger chunk itself is not emitted as a delta since the
# content_block_start already carries the relevant information.
# For text blocks the trigger chunk is not emitted as a separate
# delta because content_block_start carries the information.
# For tool_use blocks we must also emit the trigger chunk's delta
# when it carries input_json_delta data, because some providers
# (e.g. xAI, Gemini) include tool arguments in the same streaming
# chunk as the function name/id.

# 1. Stop current content block
self.chunk_queue.append(
Expand All @@ -325,6 +348,17 @@ async def __anext__(self): # noqa: PLR0915
}
)

# 3. If the trigger chunk carries tool argument data, queue it
# so the input_json_delta is not silently dropped.
if (
processed_chunk.get("type") == "content_block_delta"
and isinstance(processed_chunk.get("delta"), dict)
and processed_chunk["delta"].get("type")
== "input_json_delta"
and processed_chunk["delta"].get("partial_json")
):
self.chunk_queue.append(processed_chunk)

# Reset state for new block
self.sent_content_block_finish = False

Expand Down
Loading
Loading