Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 21 additions & 2 deletions litellm/llms/anthropic/chat/handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -585,6 +585,14 @@ def __init__(
self._current_server_tool_id: Optional[str] = None
self._container_id: Optional[str] = None

# Accumulate streamed reasoning ("thinking") text so we can estimate
# reasoning_tokens in the final usage object. Anthropic's streaming
# message_delta only reports total output_tokens, so without this
# accumulator completion_tokens_details.reasoning_tokens is always 0
# for streaming responses even when extended thinking is enabled.
# See: https://github.com/BerriAI/litellm/issues/<TBD>

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Placeholder issue URL left in comment

The comment references https://github.com/BerriAI/litellm/issues/<TBD>, which is an unfilled placeholder. This will be confusing to any contributor who tries to follow the link. Either replace it with the real issue number or remove the See: line entirely.

self.accumulated_reasoning_content: str = ""

def check_empty_tool_call_args(self) -> bool:
"""
Check if the tool call block so far has been an empty string
Expand All @@ -609,9 +617,13 @@ def check_empty_tool_call_args(self) -> bool:
return False

def _handle_usage(self, anthropic_usage_chunk: Union[dict, UsageDelta]) -> Usage:
# Pass the accumulated thinking text (if any) so calculate_usage can
# estimate reasoning_tokens for streaming responses. Falsy when no
# thinking deltas were observed, which preserves the previous behavior
# for non-thinking streams.
return AnthropicConfig().calculate_usage(
usage_object=cast(dict, anthropic_usage_chunk),
reasoning_content=None,
reasoning_content=self.accumulated_reasoning_content or None,
speed=self.speed,
)

Expand Down Expand Up @@ -658,14 +670,21 @@ def _content_block_delta_helper(self, chunk: dict) -> Tuple[
"thinking" in content_block["delta"]
or "signature" in content_block["delta"]
):
thinking_text = content_block["delta"].get("thinking") or ""
thinking_blocks = [
ChatCompletionThinkingBlock(
type="thinking",
thinking=content_block["delta"].get("thinking") or "",
thinking=thinking_text,
signature=str(content_block["delta"].get("signature") or ""),
)
]
provider_specific_fields["thinking_blocks"] = thinking_blocks
# Accumulate so the final usage chunk can report reasoning_tokens.
# Anthropic's streaming message_delta only carries total
# output_tokens; without this accumulator, _handle_usage() would
# always pass reasoning_content=None and yield reasoning_tokens=0.
if thinking_text:
self.accumulated_reasoning_content += thinking_text
elif (
"content" in content_block["delta"]
and content_block["delta"].get("type") == "compaction_delta"
Expand Down
145 changes: 145 additions & 0 deletions tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -1614,3 +1614,148 @@ def test_non_bash_tool_result_skipped():
assert (
len(code_results) == 0
), f"Expected 0 code_interpreter_results for text_editor result, got {len(code_results)}"


def test_streaming_accumulates_reasoning_tokens_in_usage():
"""Reasoning tokens should be reported in the final usage chunk for streaming
responses with extended thinking.

Regression test for the bug where streaming Anthropic responses always
returned reasoning_tokens=0 because thinking deltas were forwarded but never
accumulated, so the message_delta -> calculate_usage path passed
reasoning_content=None.
"""
chunks = [
{
"type": "message_start",
"message": {
"id": "msg_123",
"type": "message",
"role": "assistant",
"content": [],
"usage": {"input_tokens": 10, "output_tokens": 1},
},
},
{
"type": "content_block_start",
"index": 0,
"content_block": {"type": "thinking", "thinking": ""},
},
{
"type": "content_block_delta",
"index": 0,
"delta": {
"type": "thinking_delta",
"thinking": "Let me think carefully about this problem. ",
},
},
{
"type": "content_block_delta",
"index": 0,
"delta": {
"type": "thinking_delta",
"thinking": "I need to consider every possibility before answering.",
},
},
{
"type": "content_block_delta",
"index": 0,
"delta": {"type": "signature_delta", "signature": "sig_abc"},
},
{"type": "content_block_stop", "index": 0},
{
"type": "content_block_start",
"index": 1,
"content_block": {"type": "text", "text": ""},
},
{
"type": "content_block_delta",
"index": 1,
"delta": {"type": "text_delta", "text": "The answer is 42."},
},
{"type": "content_block_stop", "index": 1},
{
"type": "message_delta",
"delta": {"stop_reason": "end_turn", "stop_sequence": None},
"usage": {"output_tokens": 50},
},
]

iterator = ModelResponseIterator(
streaming_response=MagicMock(), sync_stream=True, json_mode=False
)

final_usage = None
for chunk in chunks:
parsed = iterator.chunk_parser(chunk)
if parsed.usage is not None:
final_usage = parsed.usage

expected_thinking = (
"Let me think carefully about this problem. "
"I need to consider every possibility before answering."
)
assert iterator.accumulated_reasoning_content == expected_thinking

assert final_usage is not None
assert final_usage.completion_tokens == 50

completion_details = final_usage.completion_tokens_details
assert completion_details is not None
assert completion_details.reasoning_tokens > 0, (
"Expected reasoning_tokens > 0 in streaming usage when thinking deltas "
f"were observed; got {completion_details.reasoning_tokens}. This is the "
"Anthropic streaming reasoning_tokens=0 regression."
)
# text_tokens is the remainder; it should not absorb the reasoning tokens.
assert completion_details.text_tokens == (50 - completion_details.reasoning_tokens)


def test_streaming_no_thinking_leaves_reasoning_tokens_zero():
"""Streams without any thinking deltas should still report
reasoning_tokens=0 (no false positives from the new accumulator)."""
chunks = [
{
"type": "message_start",
"message": {
"id": "msg_456",
"type": "message",
"role": "assistant",
"content": [],
"usage": {"input_tokens": 5, "output_tokens": 1},
},
},
{
"type": "content_block_start",
"index": 0,
"content_block": {"type": "text", "text": ""},
},
{
"type": "content_block_delta",
"index": 0,
"delta": {"type": "text_delta", "text": "Hello world"},
},
{"type": "content_block_stop", "index": 0},
{
"type": "message_delta",
"delta": {"stop_reason": "end_turn", "stop_sequence": None},
"usage": {"output_tokens": 12},
},
]

iterator = ModelResponseIterator(
streaming_response=MagicMock(), sync_stream=True, json_mode=False
)

final_usage = None
for chunk in chunks:
parsed = iterator.chunk_parser(chunk)
if parsed.usage is not None:
final_usage = parsed.usage

assert iterator.accumulated_reasoning_content == ""
assert final_usage is not None
completion_details = final_usage.completion_tokens_details
assert completion_details is not None
assert completion_details.reasoning_tokens == 0
assert completion_details.text_tokens == 12
Loading