diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index eccca70fde1d..3901eeba6956 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -1,8 +1,12 @@ import time import uuid -from collections.abc import Sequence +from collections.abc import Mapping, Sequence +from itertools import accumulate from typing import Any, Final, cast +from openai.types.responses import ResponseFunctionToolCall, ResponseFunctionWebSearch, ResponseReasoningItem +from pydantic import BaseModel + import litellm from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params from litellm.main import stream_chunk_builder @@ -12,7 +16,10 @@ is_custom_tool_call, serialize_tool_call_arguments, ) -from litellm.responses.litellm_completion_transformation.reasoning_items import mint_reasoning_item_id +from litellm.responses.litellm_completion_transformation.reasoning_items import ( + encode_thinking_blocks, + mint_reasoning_item_id, +) from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) @@ -21,6 +28,7 @@ from litellm.types.llms.openai import ( PART_UNION_TYPES, BaseLiteLLMOpenAIResponseObject, + ChatCompletionAnnotation, ContentPartAddedEvent, ContentPartDoneEvent, ContentPartDonePartOutputText, @@ -47,6 +55,13 @@ WebSearchCallInProgressEvent, WebSearchCallSearchingEvent, ) +from litellm.types.responses.main import ( + CustomToolCallOutputItem, + GenericResponseOutputItem, + GenericResponseOutputItemContentAnnotation, + OutputFunctionToolCall, + OutputText, +) from litellm.types.utils import Delta as ChatCompletionDelta from litellm.types.utils import ( ModelResponse, @@ -76,7 +91,7 @@ def _output_items_with_id(items: tuple[Any, ...], item_type: str, item_id: str | ) -def _delta_has_signed_thinking_block(delta: object) -> bool: +def _delta_has_thinking_block(delta: object) -> bool: blocks: Final = getattr(delta, "thinking_blocks", None) or () return any(isinstance(b, dict) and (b.get("signature") or b.get("data")) for b in blocks) @@ -130,7 +145,7 @@ def __init__( self._tool_item_id_by_call_id: dict[str, str] = {} # mutable-ok: filled per call id as tool call events stream self._tool_call_id_by_index: dict[int, str] = {} self._ambiguous_tool_call_indexes: set[int] = set() - self._next_tool_output_index: int = 1 # output_index=0 reserved for the message item + self._next_tool_output_index: int = 1 self._final_tool_events_queued: bool = False self._sequence_number: int = 0 self._cached_reasoning_item_id: str | None = None @@ -142,6 +157,16 @@ def __init__( self._reasoning_active = False self._reasoning_done_emitted = False self._reasoning_item_id: str | None = None + self._reasoning_output_index: int = 0 + self._reasoning_start_chunk_index: int = 0 + self._completed_reasoning_items: tuple[tuple[int, ResponseReasoningItem], ...] = () + self._pending_thinking_start_index: int | None = None + self._message_active: bool = False + self._message_start_chunk_index: int = 0 + self._closed_message_text_length: int = 0 + self._completed_message_items: tuple[tuple[int, GenericResponseOutputItem], ...] = () + self._final_message_events_queued: bool = False + self._pending_annotation_events: list[BaseLiteLLMOpenAIResponseObject] = [] self._accumulated_reasoning_content_parts: list[str] = [] self._accumulated_provider_specific_fields: dict[str, object] = {} self._custom_tool_names: set[str] = extract_custom_tool_names(self.responses_api_request.get("tools")) @@ -197,7 +222,7 @@ def _is_reasoning_end(self, chunk): return delta.content or delta.function_call or delta.tool_calls or chunk.choices[0].finish_reason is not None - def _reserve_web_search_indexes(self, provider_fields: object) -> None: + def _reserve_web_search_indexes(self, provider_fields: object, *, eager: bool = False) -> None: if not isinstance(provider_fields, dict): return calls: Final = provider_fields.get("web_search_calls") @@ -212,12 +237,24 @@ def _reserve_web_search_indexes(self, provider_fields: object) -> None: if call_id: output_index = self._get_or_assign_tool_output_index(call_id) self._web_search_calls[call_id] = item - if status == "in_progress": + if not eager and status == "in_progress": self._pending_tool_events = [ event for event in self._pending_tool_events if getattr(event, "output_index", None) != output_index ] + if ( + eager + and status in ("in_progress", "searching", "completed", "failed") + and call_id not in self._queued_web_search_call_ids + ): + self._queue_web_search_events( + (search_call := ResponseFunctionWebSearch.model_validate(cast(object, item))).id.removeprefix( + "ws_" + ), + search_call, + finalize=False, + ) def _tool_call_id(self, tool_call: object) -> str: index: Final = self._normalize_tool_call_index(tool_call) @@ -335,7 +372,6 @@ def _queue_final_tool_call_done_events(self, litellm_complete_object: ModelRespo if web_search_call is not None: if call_id not in self._queued_web_search_call_ids: self._queue_web_search_events(call_id, web_search_call) - self._queued_web_search_call_ids.add(call_id) continue # Track if this is a new tool call that wasn't streamed @@ -398,32 +434,36 @@ def _queue_final_tool_call_done_events(self, litellm_complete_object: ModelRespo ) self._pending_tool_events.append(item_done_event) - def _queue_web_search_events(self, call_id: str, web_search_call: object) -> None: - from openai.types.responses import ResponseFunctionWebSearch - + def _queue_web_search_events(self, call_id: str, web_search_call: object, *, finalize: bool = True) -> None: item: Final = ( web_search_call if isinstance(web_search_call, ResponseFunctionWebSearch) else ResponseFunctionWebSearch.model_validate(web_search_call) ) output_index: Final = self._get_or_assign_tool_output_index(call_id) - self._sequence_number += 1 - added: Final = OutputItemAddedEvent( - type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, - output_index=output_index, - item=BaseLiteLLMOpenAIResponseObject( - **{ - "id": item.id, - "type": item.type, - "status": "in_progress", - "action": None, - } - ), - ) - added.__dict__["sequence_number"] = self._sequence_number - self._pending_tool_events.append(added) + if self._tool_item_id_by_call_id.get(call_id) != item.id: + self._pending_tool_events = [ + event for event in self._pending_tool_events if getattr(event, "output_index", None) != output_index + ] + self._tool_item_id_by_call_id[call_id] = item.id + self._sequence_number += 1 + added: Final = OutputItemAddedEvent( + type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, + output_index=output_index, + item=BaseLiteLLMOpenAIResponseObject( + **{"id": item.id, "type": item.type, "status": "in_progress", "action": None} + ), + ) + added.__dict__["sequence_number"] = self._sequence_number + self._pending_tool_events.append(added) + self._sequence_number += 1 + in_progress: Final = WebSearchCallInProgressEvent( + type=ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, output_index=output_index, item_id=item.id + ).model_copy(update={"sequence_number": self._sequence_number}) + self._pending_tool_events.append(in_progress) + if not finalize and item.status in ("in_progress", "searching"): + return for event_type, event_class in ( - (ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, WebSearchCallInProgressEvent), (ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, WebSearchCallSearchingEvent), (ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent), ): @@ -440,6 +480,7 @@ def _queue_web_search_events(self, call_id: str, web_search_call: object) -> Non item=BaseLiteLLMOpenAIResponseObject(**item.model_dump()), ) ) + self._queued_web_search_call_ids.add(call_id) def _adopt_response_id_from_chunk(self, chunk: ModelResponseStream) -> None: if self._cached_response_id is not None: @@ -602,15 +643,29 @@ def create_content_part_added_event(self) -> ContentPartAddedEvent: return event def _queue_message_item_added_events(self) -> None: - if self._cached_item_id is None: + if self._cached_item_id is None or self._completed_message_items: self._cached_item_id = f"msg_{uuid.uuid4()}" + self._message_active = True + self.sent_output_text_done_event = False + self.sent_output_content_part_done_event = False + self.sent_output_item_done_event = False self.sent_message_item_added_event = True self.sent_content_part_added_event = True - if self._cached_reasoning_item_id is not None: + if ( + self._cached_reasoning_item_id is not None + or self._completed_message_items + or 0 in self._tool_output_index_by_call_id.values() + ): self._message_output_index = self._next_tool_output_index self._next_tool_output_index += 1 + self._pending_response_events.extend(self._pending_tool_events) + self._pending_tool_events.clear() else: self._message_output_index = 0 + self._pending_annotation_events = [ + event.model_copy(update={"item_id": self._cached_item_id, "output_index": self._message_output_index}) + for event in self._pending_annotation_events + ] self._sequence_number += 1 event: Final = OutputItemAddedEvent( type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, @@ -641,11 +696,11 @@ def _merge_provider_specific_fields(self, src: dict) -> None: for key, val in src.items(): self._accumulated_provider_specific_fields[key] = val - def create_litellm_model_response(self) -> ModelResponse | None: + def create_litellm_model_response(self, *, chunk_start: int = 0) -> ModelResponse | None: response: Final = cast( ModelResponse | None, stream_chunk_builder( - chunks=self.collected_chat_completion_chunks, + chunks=self.collected_chat_completion_chunks[chunk_start:], logging_obj=self.litellm_logging_obj, ), ) @@ -657,6 +712,15 @@ def create_litellm_model_response(self) -> ModelResponse | None: provider_specific_fields.update(self._accumulated_provider_specific_fields) return response + def _encoded_thinking_blocks(self) -> str | None: + response: Final = self.create_litellm_model_response(chunk_start=self._reasoning_start_chunk_index) + if response is None: + return None + thinking_blocks: Final[Sequence[Mapping[str, object]]] = ( + getattr(response.choices[0].message, "thinking_blocks", None) or () + ) + return encode_thinking_blocks(thinking_blocks) + @staticmethod def _snapshot_chunk_for_stream_chunk_builder( chunk: ModelResponseStream, @@ -699,7 +763,7 @@ def create_reasoning_summary_text_done_event( return ReasoningSummaryTextDoneEvent( type=ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DONE, item_id=reasoning_item_id, - output_index=0, + output_index=self._reasoning_output_index, sequence_number=sequence_number, summary_index=0, text=reasoning_content, @@ -730,7 +794,7 @@ def create_reasoning_summary_part_done_event( return ReasoningSummaryPartDoneEvent( type=ResponsesAPIStreamEvents.REASONING_SUMMARY_PART_DONE, item_id=reasoning_item_id, - output_index=0, + output_index=self._reasoning_output_index, sequence_number=sequence_number, summary_index=0, part=BaseLiteLLMOpenAIResponseObject( @@ -760,11 +824,7 @@ def create_output_content_part_done_event(self, litellm_complete_object: ModelRe text: Final = getattr(litellm_complete_object.choices[0].message, "content", "") or "" annotations: Final = getattr(litellm_complete_object.choices[0].message, "annotations", None) - response_annotations: Final = ( - LiteLLMCompletionResponsesConfig.transform_chat_completion_annotations_to_response_output_annotations( - annotations=annotations - ) - ) + response_annotations: Final = self._message_annotations(annotations) part: Final[PART_UNION_TYPES] = ContentPartDonePartOutputText( type="output_text", text=text, @@ -784,14 +844,10 @@ def create_output_item_done_event(self, litellm_complete_object: ModelResponse) if self._cached_item_id is None: self._cached_item_id = f"msg_{uuid.uuid4()}" - text: Final = self.litellm_model_response.choices[0].message.content or "" - annotations = getattr(self.litellm_model_response.choices[0].message, "annotations", None) + text: Final = litellm_complete_object.choices[0].message.content or "" + annotations = getattr(litellm_complete_object.choices[0].message, "annotations", None) - response_annotations: Final = ( - LiteLLMCompletionResponsesConfig.transform_chat_completion_annotations_to_response_output_annotations( - annotations=annotations - ) - ) + response_annotations: Final = self._message_annotations(annotations) return OutputItemDoneEvent( type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, output_index=self._message_output_index, @@ -799,7 +855,9 @@ def create_output_item_done_event(self, litellm_complete_object: ModelResponse) item=BaseLiteLLMOpenAIResponseObject( **{ "id": self._cached_item_id, - "status": "completed", + "status": LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status( + litellm_complete_object.choices[0].finish_reason or "stop" + ), "type": "message", "role": "assistant", "content": [ @@ -813,6 +871,138 @@ def create_output_item_done_event(self, litellm_complete_object: ModelResponse) ), ) + def _message_annotations( + self, + annotations: list[ChatCompletionAnnotation] | None, + *, + start: int | None = None, + end: int | None = None, + positioned_only: bool = False, + ) -> list[GenericResponseOutputItemContentAnnotation]: + offset: Final = self._closed_message_text_length if start is None else start + return [ + annotation.model_copy( + update={ + key: value - offset + for key, value in (("start_index", annotation.start_index), ("end_index", annotation.end_index)) + if value is not None + } + ) + for annotation in LiteLLMCompletionResponsesConfig.transform_chat_completion_annotations_to_response_output_annotations( + annotations=annotations + ) + if (annotation.start_index is None and end is None and not positioned_only) + or ( + annotation.start_index is not None + and annotation.start_index >= offset + and (end is None or annotation.start_index < end) + ) + ] + + def _store_message_segment(self) -> None: + snapshot: Final = self.create_litellm_model_response(chunk_start=self._message_start_chunk_index) + if snapshot is None: + return + item_done: Final = self.create_output_item_done_event(snapshot) + self._completed_message_items += ( + (self._message_output_index, GenericResponseOutputItem.model_validate(item_done.item.model_dump())), + ) + self._closed_message_text_length += len(snapshot.choices[0].message.content or "") + self._message_start_chunk_index = len(self.collected_chat_completion_chunks) + self._message_active = False + self.sent_annotation_events = False + self.sent_output_text_done_event = True + self.sent_output_content_part_done_event = True + self.sent_output_item_done_event = True + + def _messages_with_final_annotations( + self, response: ModelResponse + ) -> tuple[tuple[int, GenericResponseOutputItem], ...]: + annotations: Final[list[ChatCompletionAnnotation] | None] = getattr( + response.choices[0].message, "annotations", None + ) + offsets: Final = tuple( + accumulate((len(item.content[0].text or "") for _, item in self._completed_message_items), initial=0) + ) + return tuple( + ( + index, + item.model_copy( + update={ + "content": [ + item.content[0].model_copy( + update={ + "annotations": self._message_annotations( + annotations, + start=offsets[position], + end=offsets[position + 1] + if position + 1 < len(self._completed_message_items) + else None, + positioned_only=True, + ) + + self._unpositioned_message_annotations(position) + } + ) + ] + } + ), + ) + for position, (index, item) in enumerate(self._completed_message_items) + ) + + def _unpositioned_message_annotations(self, position: int) -> list[GenericResponseOutputItemContentAnnotation]: + return [ + annotation + for annotation in self._completed_message_items[position][1].content[0].annotations or () + if annotation.start_index is None + ] + + def _queue_final_message_done_events(self, response: ModelResponse) -> None: + if self._final_message_events_queued: + return + self._final_message_events_queued = True + self._completed_message_items = self._messages_with_final_annotations(response) + for index, item in self._completed_message_items: + for annotation_index, annotation in enumerate(item.content[0].annotations or ()): + self._sequence_number += 1 + self._pending_response_events.append( + OutputTextAnnotationAddedEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED, + item_id=item.id, + output_index=index, + content_index=0, + annotation_index=annotation_index, + annotation=annotation.model_dump(exclude_none=True), + ).model_copy(update={"sequence_number": self._sequence_number}) + ) + self._sequence_number += 3 + self._pending_response_events.extend( + ( + OutputTextDoneEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE, + item_id=item.id, + output_index=index, + content_index=0, + text=item.content[0].text or "", + ).model_copy(update={"sequence_number": self._sequence_number - 2}), + ContentPartDoneEvent( + type=ResponsesAPIStreamEvents.CONTENT_PART_DONE, + item_id=item.id, + output_index=index, + content_index=0, + part=ContentPartDonePartOutputText.model_validate( + {**item.content[0].model_dump(), "logprobs": None} + ), + ).model_copy(update={"sequence_number": self._sequence_number - 1}), + OutputItemDoneEvent( + type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, + output_index=index, + item=BaseLiteLLMOpenAIResponseObject(**item.model_dump(exclude_none=True)), + sequence_number=self._sequence_number, + ), + ) + ) + def create_reasoning_output_item_done_event( self, reasoning_item_id: str, @@ -841,12 +1031,13 @@ def create_reasoning_output_item_done_event( """ return OutputItemDoneEvent( type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, - output_index=0, + output_index=self._reasoning_output_index, sequence_number=sequence_number, item=BaseLiteLLMOpenAIResponseObject( **{ "id": reasoning_item_id, "type": "reasoning", + "encrypted_content": self._encoded_thinking_blocks(), "summary": [ { "type": "summary_text", @@ -857,9 +1048,36 @@ def create_reasoning_output_item_done_event( ), ) + def _queue_reasoning_done_events(self) -> None: + reasoning_content: Final = "".join(self._accumulated_reasoning_content_parts) + reasoning_item_id: Final = self._reasoning_item_id or self._cached_reasoning_item_id or mint_reasoning_item_id() + self._sequence_number += 1 + text_done: Final = self.create_reasoning_summary_text_done_event( + reasoning_item_id, reasoning_content, self._sequence_number + ) + self._sequence_number += 1 + part_done: Final = self.create_reasoning_summary_part_done_event( + reasoning_item_id, reasoning_content, self._sequence_number + ) + self._sequence_number += 1 + item_done: Final = self.create_reasoning_output_item_done_event( + reasoning_item_id, reasoning_content, self._sequence_number + ) + self._completed_reasoning_items += ( + (self._reasoning_output_index, ResponseReasoningItem.model_validate(item_done.item.model_dump())), + ) + self._pending_response_events.extend((text_done, part_done, item_done)) + self._reasoning_done_emitted = True + self._reasoning_active = False + def return_default_done_events( - self, litellm_complete_object: ModelResponse + self, litellm_complete_object: ModelResponse, *, store_message: bool = False ) -> BaseLiteLLMOpenAIResponseObject | None: + if store_message and self._message_active: + self._store_message_segment() + if store_message and self._completed_message_items: + self._queue_final_message_done_events(litellm_complete_object) + return self._pending_response_events.pop(0) if self._pending_response_events else None if self.sent_message_item_added_event is False: final_content: Final = litellm_complete_object.choices[0].message.content or "" if not final_content: @@ -910,7 +1128,9 @@ def common_done_event_logic(self, sync_mode: bool = True) -> BaseLiteLLMOpenAIRe if self._pending_tool_events: return self._pending_tool_events.pop(0) - done_event: Final = self.return_default_done_events(self.litellm_model_response) + done_event: Final = self.return_default_done_events( + self.litellm_model_response, store_message=not sync_mode + ) if done_event: return done_event else: @@ -932,27 +1152,66 @@ def common_done_event_logic(self, sync_mode: bool = True) -> BaseLiteLLMOpenAIRe else: raise StopAsyncIteration - def _ensure_output_item_for_chunk(self, chunk: ModelResponseStream) -> None: + def _ensure_output_item_for_chunk( + self, chunk: ModelResponseStream, *, allow_reasoning_resumption: bool = False + ) -> None: # Change: Never return a value, just enqueue output item events - if self.sent_output_item_added_event: + if self.sent_output_item_added_event and not allow_reasoning_resumption: return if not chunk.choices: return delta: Final = chunk.choices[0].delta - self._sequence_number += 1 - self.sent_output_item_added_event = True + if allow_reasoning_resumption and not ( + getattr(delta, "reasoning_content", None) or _delta_has_thinking_block(delta) + ): + if ( + getattr(delta, "thinking_blocks", None) + and not delta.content + and not delta.tool_calls + and chunk.choices[0].finish_reason is None + ): + self._pending_thinking_start_index = ( + len(self.collected_chat_completion_chunks) + if self._pending_thinking_start_index is None + else self._pending_thinking_start_index + ) + return + if ( + self._pending_thinking_start_index is not None + and not delta.content + and not delta.tool_calls + and chunk.choices[0].finish_reason is None + ): + return + self._pending_thinking_start_index = None # Reasoning-first - if (hasattr(delta, "reasoning_content") and delta.reasoning_content) or _delta_has_signed_thinking_block(delta): + if (hasattr(delta, "reasoning_content") and delta.reasoning_content) or _delta_has_thinking_block(delta): + if self._reasoning_active: + return + if self.sent_output_item_added_event: + self._pending_response_events.extend(self._pending_tool_events) + self._pending_tool_events.clear() + self._reasoning_output_index = self._next_tool_output_index + self._next_tool_output_index += 1 self._reasoning_active = True - if self._cached_reasoning_item_id is None: - self._cached_reasoning_item_id = mint_reasoning_item_id() + self._reasoning_done_emitted = False + self._reasoning_start_chunk_index = ( + self._pending_thinking_start_index + if self._pending_thinking_start_index is not None + else len(self.collected_chat_completion_chunks) + ) + self._pending_thinking_start_index = None + self._accumulated_reasoning_content_parts = [] + self._cached_reasoning_item_id = mint_reasoning_item_id() self._reasoning_item_id = self._cached_reasoning_item_id + self.sent_output_item_added_event = True + self._sequence_number += 1 event = OutputItemAddedEvent( type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, - output_index=0, + output_index=self._reasoning_output_index, item=BaseLiteLLMOpenAIResponseObject( **{ "id": self._cached_reasoning_item_id, @@ -966,6 +1225,11 @@ def _ensure_output_item_for_chunk(self, chunk: ModelResponseStream) -> None: self._pending_response_events.append(event) return + if self.sent_output_item_added_event: + return + self.sent_output_item_added_event = True + self._sequence_number += 1 + # Tool-first if hasattr(delta, "tool_calls") and delta.tool_calls: # Tool calls already handled via _queue_tool_call_delta_events @@ -1003,6 +1267,25 @@ async def __anext__( chunk = await self.litellm_custom_stream_wrapper.__anext__() if chunk is not None: chunk = cast(ModelResponseStream, chunk) + if ( + self._message_active + and chunk.choices + and ( + chunk.choices[0].delta.tool_calls + or getattr(chunk.choices[0].delta, "reasoning_content", None) + or _delta_has_thinking_block(chunk.choices[0].delta) + ) + ): + self._store_message_segment() + if ( + not self.sent_output_item_added_event + and chunk.choices + and chunk.choices[0].delta.tool_calls + and not chunk.choices[0].delta.content + and not getattr(chunk.choices[0].delta, "reasoning_content", None) + and not _delta_has_thinking_block(chunk.choices[0].delta) + ): + self._next_tool_output_index = 0 for src in ( getattr(chunk, "provider_specific_fields", None), getattr( @@ -1013,8 +1296,8 @@ async def __anext__( ): if src and isinstance(src, dict): self._merge_provider_specific_fields(src) - self._reserve_web_search_indexes(src) - self._ensure_output_item_for_chunk(chunk) + self._reserve_web_search_indexes(src, eager=True) + self._ensure_output_item_for_chunk(chunk, allow_reasoning_resumption=True) # Proceed to transformation self.collected_chat_completion_chunks.append( self._snapshot_chunk_for_stream_chunk_builder(chunk) @@ -1026,49 +1309,11 @@ async def __anext__( if delta and hasattr(delta, "reasoning_content") and delta.reasoning_content: self._accumulated_reasoning_content_parts.append(delta.reasoning_content) if self._is_reasoning_end(chunk): - reasoning_content = "".join(self._accumulated_reasoning_content_parts) - - # Ensure we have a valid reasoning_item_id - self._cached_reasoning_item_id = ( - self._reasoning_item_id - or self._cached_reasoning_item_id - or mint_reasoning_item_id() - ) - reasoning_item_id = self._cached_reasoning_item_id - - # Create text.done event first with its own sequence number - self._sequence_number += 1 - text_done_event = self.create_reasoning_summary_text_done_event( - reasoning_item_id=reasoning_item_id, - reasoning_content=reasoning_content, - sequence_number=self._sequence_number, - ) - - # Create part.done event second with its own sequence number - self._sequence_number += 1 - part_done_event = self.create_reasoning_summary_part_done_event( - reasoning_item_id=reasoning_item_id, - reasoning_content=reasoning_content, - sequence_number=self._sequence_number, - ) - - self._sequence_number += 1 - reasoning_output_item_done_event = self.create_reasoning_output_item_done_event( - reasoning_item_id=reasoning_item_id, - reasoning_content=reasoning_content, - sequence_number=self._sequence_number, - ) - self._pending_response_events.extend( - [ - text_done_event, - part_done_event, - reasoning_output_item_done_event, - ] - ) - self._reasoning_done_emitted = True - self._reasoning_active = False - - response_api_chunk = self._transform_chat_completion_chunk_to_response_api_chunk(chunk) + self._queue_reasoning_done_events() + + response_api_chunk = self._transform_chat_completion_chunk_to_response_api_chunk( + chunk, allow_reasoning_resumption=True + ) if response_api_chunk: self._pending_response_events.append(response_api_chunk) @@ -1076,6 +1321,9 @@ async def __anext__( return self._pending_response_events.pop(0) except StopAsyncIteration: + if self._reasoning_active: + self._queue_reasoning_done_events() + return self._pending_response_events.pop(0) return self.common_done_event_logic(sync_mode=False) except Exception as e: @@ -1144,7 +1392,7 @@ def __next__( raise e def _transform_chat_completion_chunk_to_response_api_chunk( - self, chunk: ModelResponseStream + self, chunk: ModelResponseStream, *, allow_reasoning_resumption: bool = False ) -> ResponsesAPIStreamingResponse | None: """ Transform a chat completion chunk to a response API chunk. @@ -1161,27 +1409,22 @@ def _transform_chat_completion_chunk_to_response_api_chunk( # This ensures we detect and queue annotation events from the annotation chunk if chunk.choices and hasattr(chunk.choices[0].delta, "annotations"): annotations: Final = chunk.choices[0].delta.annotations - if annotations and self.sent_annotation_events is False: - self.sent_annotation_events = True + if annotations and self.sent_annotation_events is False and not allow_reasoning_resumption: # Store annotation events to emit them one by one - if not hasattr(self, "_pending_annotation_events"): - response_annotations = LiteLLMCompletionResponsesConfig.transform_chat_completion_annotations_to_response_output_annotations( - annotations=annotations + response_annotations = self._message_annotations(annotations) + self.sent_annotation_events = bool(response_annotations) + self._pending_annotation_events = [] + for idx, annotation in enumerate(response_annotations): + annotation_dict = annotation.model_dump() if hasattr(annotation, "model_dump") else dict(annotation) + event = OutputTextAnnotationAddedEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED, + item_id=item_id, + output_index=self._message_output_index, + content_index=0, + annotation_index=idx, + annotation=annotation_dict, ) - self._pending_annotation_events = [] - for idx, annotation in enumerate(response_annotations): - annotation_dict = ( - annotation.model_dump() if hasattr(annotation, "model_dump") else dict(annotation) - ) - event = OutputTextAnnotationAddedEvent( - type=ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED, - item_id=item_id, - output_index=self._message_output_index, - content_index=0, - annotation_index=idx, - annotation=annotation_dict, - ) - self._pending_annotation_events.append(event) + self._pending_annotation_events.append(event) # Priority 1: Handle reasoning content (highest priority) if ( chunk.choices @@ -1193,22 +1436,27 @@ def _transform_chat_completion_chunk_to_response_api_chunk( if self._cached_reasoning_item_id is None: self._cached_reasoning_item_id = mint_reasoning_item_id() - return ReasoningSummaryTextDeltaEvent( + reasoning_event: Final = ReasoningSummaryTextDeltaEvent( type=ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DELTA, item_id=self._cached_reasoning_item_id, - output_index=0, + output_index=self._reasoning_output_index, delta=reasoning_content, ) + if not allow_reasoning_resumption or not self._get_delta_string_from_streaming_choices(chunk.choices): + return reasoning_event + self._pending_response_events.append(reasoning_event) # Priority 2: Handle text deltas delta_content: Final = self._get_delta_string_from_streaming_choices(chunk.choices) if delta_content: - if not self.sent_message_item_added_event: + if allow_reasoning_resumption and self._reasoning_active: + self._queue_reasoning_done_events() + if not self.sent_message_item_added_event or (self._completed_message_items and not self._message_active): self._queue_message_item_added_events() self._sequence_number += 1 text_delta_event: Final = OutputTextDeltaEvent( type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, - item_id=item_id, + item_id=self._cached_item_id or item_id, output_index=self._message_output_index, content_index=0, delta=delta_content, @@ -1226,7 +1474,7 @@ def _transform_chat_completion_chunk_to_response_api_chunk( # Priority 4: If we have pending annotation events, emit the next one # This happens when the current chunk has no text/reasoning content - if hasattr(self, "_pending_annotation_events") and self._pending_annotation_events: + if self._message_active and self._pending_annotation_events: event = self._pending_annotation_events.pop(0) return event @@ -1261,13 +1509,102 @@ def _output_with_streamed_item_ids(self, responses_api_response: ResponsesAPIRes "message", self._cached_item_id, ) + has_streamed_layout: Final = ( + bool(self._completed_reasoning_items) or 0 in self._tool_output_index_by_call_id.values() + ) + streamed_output: Final[tuple[object, ...]] = tuple( + item + for item in message_aligned + if not has_streamed_layout + or self.sent_message_item_added_event + or not isinstance(item, GenericResponseOutputItem) + or item.type != "message" + or item.phase is not None + or any(part.text or part.annotations for part in item.content) + ) + template: Final = next( + ( + item + for item in streamed_output + if isinstance(item, GenericResponseOutputItem) and item.type == "reasoning" + ), + None, + ) + message_template: Final = next( + ( + item + for item in streamed_output + if isinstance(item, GenericResponseOutputItem) and item.type == "message" + ), + None, + ) + if (self._completed_reasoning_items and template is not None) or self._completed_message_items: + indexed: Final = ( + *( + tuple( + ( + index, + template.model_copy( + update={ + "id": item.id, + "encrypted_content": item.encrypted_content, + "content": [ + OutputText(type="output_text", text=part.text, annotations=[]) + for part in item.summary + if part.text + ], + } + ), + ) + for index, item in self._completed_reasoning_items + ) + if template is not None + else () + ), + *( + ( + index, + (message_template or item).model_copy( + update={"id": item.id, "content": item.content, "status": item.status} + ), + ) + for index, item in self._completed_message_items + ), + *( + (self._streamed_output_index(item), item) + for item in streamed_output + if not ( + isinstance(item, GenericResponseOutputItem) + and ( + (item.type == "reasoning" and self._completed_reasoning_items) + or (item.type == "message" and self._completed_message_items) + ) + ) + ), + ) + return tuple(item for _, item in sorted(indexed, key=lambda entry: entry[0])) reasoning_aligned: Final = _output_items_with_id( - message_aligned, + streamed_output, "reasoning", self._cached_reasoning_item_id, ) + if 0 in self._tool_output_index_by_call_id.values(): + return tuple(sorted(reasoning_aligned, key=self._streamed_output_index)) return reasoning_aligned + def _streamed_output_index(self, item: object) -> int: + if isinstance(item, GenericResponseOutputItem) and item.type == "message": + return self._message_output_index + if isinstance(item, (ResponseFunctionToolCall, OutputFunctionToolCall, CustomToolCallOutputItem)): + return self._tool_output_index_by_call_id.get(item.call_id or "", self._next_tool_output_index) + if isinstance(item, ResponseFunctionWebSearch): + return self._tool_output_index_by_call_id.get(item.id.removeprefix("ws_"), self._next_tool_output_index) + if isinstance(item, BaseModel): + item_id: Final[object] = item.model_dump(include={"id"}).get("id") + if isinstance(item_id, str): + return self._tool_output_index_by_call_id.get(item_id, self._next_tool_output_index) + return self._next_tool_output_index + def _emit_terminal_response_event( self, litellm_model_response: ModelResponse ) -> ResponseCompletedEvent | ResponseIncompleteEvent | None: diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 2090ff3c9fa4..ae36ab42642b 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -3934,34 +3934,34 @@ class TestEnsureOutputItemContentPartAdded: def _make_iterator(self): """Create a minimal LiteLLMCompletionStreamingIterator for testing.""" + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import Logging from litellm.responses.litellm_completion_transformation.streaming_iterator import ( LiteLLMCompletionStreamingIterator, ) - iterator = LiteLLMCompletionStreamingIterator.__new__( - LiteLLMCompletionStreamingIterator - ) - iterator.sent_output_item_added_event = False - iterator.sent_content_part_added_event = False - iterator._sequence_number = 0 - iterator._cached_item_id = None - iterator._cached_reasoning_item_id = None - iterator._reasoning_active = False - iterator._pending_response_events = [] - iterator._pending_tool_events = [] - iterator._tool_output_index_by_call_id = {} - iterator._tool_args_by_call_id = {} - iterator._tool_item_id_by_call_id = {} - iterator._tool_call_id_by_index = {} - iterator._ambiguous_tool_call_indexes = set() - iterator._next_tool_output_index = 1 - iterator._final_tool_events_queued = False - iterator._custom_tool_names = set() - iterator.responses_api_request = {} - iterator._namespace_tool_names = LiteLLMCompletionResponsesConfig.namespace_tool_name_map(None) - iterator._web_search_calls = {} - iterator._queued_web_search_call_ids = set() - return iterator + return LiteLLMCompletionStreamingIterator( + model="test-model", + litellm_custom_stream_wrapper=litellm.CustomStreamWrapper( + completion_stream=iter(()), + model="test-model", + custom_llm_provider="cached_response", + logging_obj=Logging( + model="test-model", + messages=[], + stream=True, + call_type="completion", + start_time=datetime(2025, 1, 1), + litellm_call_id="test-call", + function_id="test-function", + litellm_trace_id="test-trace", + supports_correlation_logging=False, + ), + ), + request_input="test", + responses_api_request={}, + ) def _make_text_chunk(self): """Create a mock ModelResponseStream with a text delta.""" diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index a42eb74b0fa2..e8e3f62d46b5 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -11,20 +11,27 @@ """ import json +from itertools import chain from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest +from openai.types.responses import ResponseReasoningItem from litellm.responses.litellm_completion_transformation.streaming_iterator import ( LiteLLMCompletionStreamingIterator, ) +from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, +) from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( BaseLiteLLMOpenAIResponseObject, + ChatCompletionRedactedThinkingBlock, + ChatCompletionThinkingBlock, ResponsesAPIStreamEvents, ) -from litellm.types.responses.main import build_web_search_call +from litellm.types.responses.main import OutputCodeInterpreterCall, build_web_search_call from litellm.types.utils import ( Delta, ModelResponse, @@ -1171,3 +1178,625 @@ async def test_plain_text_stream_announces_exactly_one_message_item(sync_mode: b ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE, ): assert event.item_id == message_item_adds[0].item.id + + +@pytest.mark.parametrize( + "thinking_text,include_redacted", + [("echoecho", False), ("", False), (None, True), ("echoecho", True)], + ids=["visible", "signature-only", "redacted-only", "mixed"], +) +@pytest.mark.parametrize("end_with_tool", [False, True], ids=["text", "tool"]) +@pytest.mark.asyncio +async def test_reasoning_done_payload_replays_signed_blocks( + thinking_text: str | None, include_redacted: bool, end_with_tool: bool +) -> None: + fragments: Final[tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...]] = ( + *( + ChatCompletionThinkingBlock(type="thinking", thinking=part) + for part in (("echo", "echo") if thinking_text else ()) + ), + *( + (ChatCompletionThinkingBlock(type="thinking", thinking="", signature="test-signature"),) + if thinking_text is not None + else () + ), + *( + (ChatCompletionRedactedThinkingBlock(type="redacted_thinking", data="test-redacted-data"),) + if include_redacted + else () + ), + ) + chunks: Final = tuple( + ModelResponseStream( + id=CHAT_COMPLETION_ID, + model="test-model", + choices=[ + StreamingChoices( + index=0, + delta=Delta( + reasoning_content=fragment.get("thinking", ""), + thinking_blocks=[fragment], + provider_specific_fields={"thinking_blocks": [fragment]}, + ), + ) + ], + ) + for fragment in fragments + ) + ending: Final = ( + _tool_call_chunk(finish_reason="tool_calls") if end_with_tool else _chunk("answer", finish_reason="stop") + ) + iterator: Final = _build_iterator([*chunks, ending]) + + events: Final = [event async for event in iterator] + done: Final = next( + event.item + for event in events + if event.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE and event.item.type == "reasoning" + ) + completed: Final = next( + event.response for event in events if event.type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ) + reasoning: Final = next(item for item in completed.output if item.type == "reasoning") + payload: Final = done.model_dump().get("encrypted_content") + expected: Final = [ + *( + [ChatCompletionThinkingBlock(type="thinking", thinking=thinking_text, signature="test-signature")] + if thinking_text is not None + else [] + ), + *( + [ChatCompletionRedactedThinkingBlock(type="redacted_thinking", data="test-redacted-data")] + if include_redacted + else [] + ), + ] + + assert isinstance(payload, str) + assert json.loads(payload) == expected + assert payload == reasoning.encrypted_content + messages: Final = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( + input_item=done.model_dump(exclude_none=True), replay_reasoning=True + ) + assert messages[0]["thinking_blocks"] == expected + + +def test_reasoning_done_without_a_response_snapshot_preserves_summary() -> None: + iterator: Final = _build_iterator([]) + + event: Final = iterator.create_reasoning_output_item_done_event( + reasoning_item_id="rs_pending", + reasoning_content="The response snapshot is not available yet.", + sequence_number=7, + ) + + assert event.type == "response.output_item.done" + assert event.sequence_number == 7 + assert event.item.model_dump(exclude_none=True) == { + "id": "rs_pending", + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "The response snapshot is not available yet."}], + } + + +def _reasoning_block_chunk( + block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, +) -> ModelResponseStream: + return ModelResponseStream( + id=CHAT_COMPLETION_ID, + model="test-model", + choices=[ + StreamingChoices( + index=0, + delta=Delta( + reasoning_content=block.get("thinking", ""), + thinking_blocks=[block], + ), + ) + ], + ) + + +@pytest.mark.parametrize("separator", ["text", "tool", "web_search", "web_search_error"]) +@pytest.mark.parametrize("reasoning_kind", ["visible", "signature-only", "redacted"]) +@pytest.mark.parametrize("finish_with_text", [True, False], ids=["text-ending", "upstream-exhausted"]) +@pytest.mark.asyncio +async def test_resumed_reasoning_items_keep_complete_replay_payloads( + separator: str, reasoning_kind: str, finish_with_text: bool +) -> None: + call_id: Final = "srvtoolu_resume" + blocks: Final = tuple( + ChatCompletionRedactedThinkingBlock(type="redacted_thinking", data=f"{phase}-redacted") + if reasoning_kind == "redacted" + else ChatCompletionThinkingBlock( + type="thinking", thinking=phase if reasoning_kind == "visible" else "", signature=f"{phase}-signature" + ) + for phase in ("first", "second") + ) + middle: Final = ( + (_chunk("interim"),) + if separator == "text" + else (_tool_call_chunk(),) + if separator == "tool" + else tuple( + ModelResponseStream( + id=CHAT_COMPLETION_ID, + model="test-model", + choices=[ + StreamingChoices( + index=0, + delta=Delta( + tool_calls=[ + { + "id": call_id, + "index": 0, + "type": "function", + "function": {"name": "web_search", "arguments": '{"query":"example"}'}, + } + ] + if status == "in_progress" + else None, + provider_specific_fields={ + "web_search_calls": [ + build_web_search_call(call_id, {"query": "example"}, {"content": []}, status=status) + ] + }, + ), + ) + ], + ) + for status in ("in_progress", "failed" if separator == "web_search_error" else "completed") + ) + ) + iterator: Final = _build_iterator( + [ + _reasoning_block_chunk(blocks[0]), + *middle, + _reasoning_block_chunk(blocks[1]), + *([_chunk("answer", finish_reason="stop")] if finish_with_text else []), + ] + ) + events: Final = [json.loads(event.model_dump_json(exclude_none=True)) async for event in iterator] + added: Final = [event for event in events if event["type"] == "response.output_item.added"] + done: Final = [ + event + for event in events + if event["type"] == "response.output_item.done" and event["item"]["type"] == "reasoning" + ] + completed: Final = events[-1]["response"] + assert len(done) == 2 + assert len({event["item"]["id"] for event in done}) == 2 + assert [event["output_index"] for event in added] == list(range(len(added))) + assert [item["id"] for item in completed["output"]] == [event["item"]["id"] for event in added] + + for event, block in zip(done, blocks): + item: Final = event["item"] + expected: Final = [block] + text: Final = block.get("thinking", "") + assert json.loads(item["encrypted_content"]) == expected + assert item["summary"] == [{"type": "summary_text", "text": text}] + assert completed["output"][event["output_index"]]["encrypted_content"] == item["encrypted_content"] + assert completed["output"][event["output_index"]]["content"] == ( + [{"type": "output_text", "text": text, "annotations": []}] if text else [] + ) + parsed: Final = ResponseReasoningItem.model_validate_json(json.dumps(item)) + assert parsed.encrypted_content == item["encrypted_content"] + replay: Final = LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( + input_item=item, replay_reasoning=True + ) + assert replay[0]["thinking_blocks"] == expected + + announced: Final = {event["item"]["id"]: event["output_index"] for event in added} + for event in events: + if event["type"].startswith("response.reasoning_summary"): + assert event["output_index"] == announced[event["item_id"]] + + +@pytest.mark.parametrize("combine_result_and_reasoning", [True, False]) +@pytest.mark.parametrize("visible", [True, False], ids=["visible", "signature-only"]) +@pytest.mark.asyncio +async def test_reasoning_after_initial_server_tool_keeps_item_indexes( + combine_result_and_reasoning: bool, visible: bool +) -> None: + call_id: Final = "srvtoolu_first" + block: Final = ChatCompletionThinkingBlock( + type="thinking", thinking="after search" if visible else "", signature="after-search-signature" + ) + start: Final = Delta( + tool_calls=[ + { + "id": call_id, + "index": 0, + "type": "function", + "function": {"name": "web_search", "arguments": '{"query":"example"}'}, + } + ], + provider_specific_fields={ + "web_search_calls": [ + build_web_search_call(call_id, {"query": "example"}, {"content": []}, status="in_progress") + ] + }, + ) + result_fields: Final = {"web_search_calls": [build_web_search_call(call_id, {"query": "example"}, {"content": []})]} + deltas: Final = ( + start, + *(() if combine_result_and_reasoning else (Delta(provider_specific_fields=result_fields),)), + Delta( + reasoning_content=block["thinking"], + thinking_blocks=[block], + provider_specific_fields=result_fields if combine_result_and_reasoning else None, + ), + Delta(content="answer"), + ) + iterator: Final = _build_iterator( + [ + ModelResponseStream( + id=CHAT_COMPLETION_ID, + model="test-model", + choices=[ + StreamingChoices(index=0, delta=delta, finish_reason="stop" if index == len(deltas) - 1 else None) + ], + ) + for index, delta in enumerate(deltas) + ] + ) + events: Final = [json.loads(event.model_dump_json(exclude_none=True)) async for event in iterator] + added: Final = [event for event in events if event["type"] == "response.output_item.added"] + assert [event["output_index"] for event in added] == [0, 1, 2] + assert [event["item"]["type"] for event in added] == ["web_search_call", "reasoning", "message"] + assert next( + index for index, event in enumerate(events) if event["type"] == "response.web_search_call.completed" + ) < next( + index + for index, event in enumerate(events) + if event["type"] == "response.output_item.added" and event["item"]["type"] == "reasoning" + ) + completed: Final = events[-1]["response"]["output"] + assert [item["id"] for item in completed] == [event["item"]["id"] for event in added] + done: Final = next( + event + for event in events + if event["type"] == "response.output_item.done" and event["item"]["type"] == "reasoning" + ) + assert done["output_index"] == 1 + assert json.loads(done["item"]["encrypted_content"]) == [block] + assert done["item"]["encrypted_content"] == completed[1]["encrypted_content"] + + +@pytest.mark.parametrize("interstitial", ["none", "empty", "usage", "metadata"]) +@pytest.mark.asyncio +async def test_resumed_thinking_blocks_without_reasoning_content_preserve_text(interstitial: str) -> None: + expected: Final = [ + {"type": "thinking", "thinking": "first", "signature": "first-signature"}, + {"type": "thinking", "thinking": "second", "signature": "second-signature"}, + ] + gap: Final = ModelResponseStream( + id=CHAT_COMPLETION_ID, + model="test-model", + choices=[StreamingChoices(index=0, delta=Delta())], + usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2) if interstitial == "usage" else None, + provider_specific_fields={"test_marker": "sideband"} if interstitial == "metadata" else None, + ) + groups: Final = tuple( + ( + ModelResponseStream( + id=CHAT_COMPLETION_ID, + model="test-model", + choices=[ + StreamingChoices( + index=0, delta=Delta(thinking_blocks=[{"type": "thinking", "thinking": block["thinking"]}]) + ) + ], + ), + *((gap,) if interstitial != "none" else ()), + _signature_only_thinking_chunk(block["signature"]), + _chunk("interim"), + ) + for block in expected + ) + chunks: Final = (*chain.from_iterable(groups), _chunk("answer", finish_reason="stop")) + events: Final = [json.loads(event.model_dump_json(exclude_none=True)) async for event in _build_iterator(chunks)] + done: Final = [ + event["item"] + for event in events + if event["type"] == "response.output_item.done" and event["item"]["type"] == "reasoning" + ] + completed: Final = [item for item in events[-1]["response"]["output"] if item["type"] == "reasoning"] + assert len(done) == 2 + assert [json.loads(item["encrypted_content"]) for item in done] == [[block] for block in expected] + assert [item["encrypted_content"] for item in done] == [item["encrypted_content"] for item in completed] + assert [item["type"] for item in events[-1]["response"]["output"]] == [ + "reasoning", + "message", + "reasoning", + "message", + ] + + +@pytest.mark.parametrize("resume_reasoning", [False, True]) +@pytest.mark.asyncio +async def test_signed_tool_response_has_no_unannounced_empty_message(resume_reasoning: bool) -> None: + chunks: Final = ( + _reasoning_block_chunk( + ChatCompletionThinkingBlock(type="thinking", thinking="first", signature="first-signature") + ), + _tool_call_chunk(), + *( + ( + _reasoning_block_chunk( + ChatCompletionThinkingBlock(type="thinking", thinking="second", signature="second-signature") + ), + ) + if resume_reasoning + else () + ), + _chunk("", finish_reason="tool_calls"), + ) + events: Final = [json.loads(event.model_dump_json(exclude_none=True)) async for event in _build_iterator(chunks)] + added: Final = [event for event in events if event["type"] == "response.output_item.added"] + completed: Final = events[-1]["response"]["output"] + assert [item["type"] for item in completed] == ["reasoning", "function_call"] + ( + ["reasoning"] if resume_reasoning else [] + ) + assert [item["id"] for item in completed] == [event["item"]["id"] for event in added] + for event in events: + if event["type"] == "response.output_item.done": + assert completed[event["output_index"]]["id"] == event["item"]["id"] + + +@pytest.mark.parametrize("with_answer", [False, True]) +@pytest.mark.asyncio +async def test_unsigned_thinking_without_reasoning_content_does_not_announce_an_item(with_answer: bool) -> None: + pending: Final = ModelResponseStream( + id=CHAT_COMPLETION_ID, + model="test-model", + choices=[ + StreamingChoices(index=0, delta=Delta(thinking_blocks=[{"type": "thinking", "thinking": "unverified"}])) + ], + ) + iterator: Final = _build_iterator([pending, _chunk("answer" if with_answer else "", finish_reason="stop")]) + events: Final = [json.loads(event.model_dump_json(exclude_none=True)) async for event in iterator] + assert all(event.get("item", {}).get("type") != "reasoning" for event in events) + completed: Final = events[-1]["response"]["output"] + assert all(item["type"] != "reasoning" for item in completed) + for event in events: + if event["type"] == "response.output_item.done": + assert completed[event["output_index"]]["id"] == event["item"]["id"] + + +@pytest.mark.parametrize("include_tool", [False, True]) +@pytest.mark.asyncio +async def test_resumed_reasoning_preserves_message_chronology(include_tool: bool) -> None: + iterator: Final = _build_iterator( + [ + _reasoning_block_chunk( + ChatCompletionThinkingBlock(type="thinking", thinking="first", signature="sig-first") + ), + _chunk("interim"), + *([_tool_call_chunk()] if include_tool else []), + _reasoning_block_chunk( + ChatCompletionThinkingBlock(type="thinking", thinking="second", signature="sig-second") + ), + _chunk("answer", finish_reason="stop"), + ] + ) + events: Final = [json.loads(event.model_dump_json(exclude_none=True)) async for event in iterator] + output: Final = events[-1]["response"]["output"] + assert [item["type"] for item in output] == ["reasoning", "message"] + ( + ["function_call"] if include_tool else [] + ) + ["reasoning", "message"] + assert [item["content"][0]["text"] for item in output if item["type"] == "message"] == ["interim", "answer"] + for event in events: + if event["type"] == "response.output_item.done": + assert output[event["output_index"]]["id"] == event["item"]["id"] + if event["item"]["type"] == "message": + assert output[event["output_index"]]["content"] == event["item"]["content"] + + +@pytest.mark.asyncio +async def test_message_segments_survive_tool_calls_without_reasoning() -> None: + iterator: Final = _build_iterator([_chunk("interim"), _tool_call_chunk(), _chunk("answer", finish_reason="stop")]) + events: Final = [json.loads(event.model_dump_json(exclude_none=True)) async for event in iterator] + output: Final = events[-1]["response"]["output"] + assert [item["type"] for item in output] == ["message", "function_call", "message"] + assert [item["content"][0]["text"] for item in output if item["type"] == "message"] == ["interim", "answer"] + + +@pytest.mark.asyncio +async def test_mixed_reasoning_and_text_chunk_preserves_both_segments() -> None: + mixed: Final = ModelResponseStream( + id=CHAT_COMPLETION_ID, + model="test-model", + choices=[ + StreamingChoices( + index=0, + delta=Delta( + content="answer", + reasoning_content="analysis", + thinking_blocks=[{"type": "thinking", "thinking": "analysis", "signature": "sig"}], + ), + finish_reason="stop", + ) + ], + ) + iterator: Final = _build_iterator([_chunk("interim"), mixed]) + events: Final = [json.loads(event.model_dump_json(exclude_none=True)) async for event in iterator] + output: Final = events[-1]["response"]["output"] + assert [item["type"] for item in output] == ["message", "reasoning", "message"] + assert [item["content"][0]["text"] for item in output if item["type"] == "message"] == ["interim", "answer"] + + +@pytest.mark.parametrize("late_annotations", [False, True]) +@pytest.mark.asyncio +async def test_message_annotations_are_relative_to_each_segment(late_annotations: bool) -> None: + intro: Final = ModelResponseStream( + id=CHAT_COMPLETION_ID, + model="test-model", + choices=[ + StreamingChoices( + index=0, + delta=Delta( + content="intro", + annotations=[ + { + "type": "url_citation", + "url_citation": { + "start_index": 0, + "end_index": 5, + "title": "first", + "url": "https://example.com/first", + }, + } + ], + ), + ) + ], + ) + answer: Final = ModelResponseStream( + id=CHAT_COMPLETION_ID, + model="test-model", + choices=[ + StreamingChoices( + index=0, + delta=Delta( + content="answer", + annotations=[ + { + "type": "url_citation", + "url_citation": { + "start_index": 5, + "end_index": 11, + "title": "second", + "url": "https://example.com/second", + }, + } + ], + ), + finish_reason="stop", + ) + ], + ) + delayed: Final = ModelResponseStream( + id=CHAT_COMPLETION_ID, + model="test-model", + choices=[ + StreamingChoices( + index=0, + delta=Delta(annotations=[*intro.choices[0].delta.annotations, *answer.choices[0].delta.annotations]), + finish_reason="stop", + ) + ], + ) + iterator: Final = _build_iterator( + [ + _chunk("intro") if late_annotations else intro, + _reasoning_block_chunk(ChatCompletionThinkingBlock(type="thinking", thinking="analysis", signature="sig")), + *([_chunk("answer"), delayed] if late_annotations else [answer]), + ] + ) + events: Final = [json.loads(event.model_dump_json(exclude_none=True)) async for event in iterator] + messages: Final = [item for item in events[-1]["response"]["output"] if item["type"] == "message"] + assert [item["content"][0]["text"] for item in messages] == ["intro", "answer"] + assert [item["content"][0]["annotations"][0]["start_index"] for item in messages] == [0, 0] + assert [item["content"][0]["annotations"][0]["end_index"] for item in messages] == [5, 6] + done_by_id: Final = {event["item"]["id"]: event for event in events if event["type"] == "response.output_item.done"} + annotation_by_id: Final = { + event["item_id"]: event for event in events if event["type"] == "response.output_text.annotation.added" + } + for message in messages: + assert done_by_id[message["id"]]["item"]["content"] == message["content"] + assert annotation_by_id[message["id"]]["annotation"] == message["content"][0]["annotations"][0] + assert events.index(annotation_by_id[message["id"]]) < events.index(done_by_id[message["id"]]) + + +@pytest.mark.asyncio +async def test_converted_code_tool_keeps_its_streamed_output_index() -> None: + call_id: Final = "srvtoolu_code" + code_result: Final = OutputCodeInterpreterCall( + type="code_interpreter_call", + id=call_id, + code="print(1)", + container_id="container_test", + status="completed", + outputs=[], + ) + tool: Final = ModelResponseStream( + id=CHAT_COMPLETION_ID, + model="test-model", + choices=[ + StreamingChoices( + index=0, + delta=Delta( + tool_calls=[ + { + "id": call_id, + "type": "function", + "index": 0, + "function": {"name": "bash_code_execution", "arguments": '{"command":"print(1)"}'}, + } + ], + provider_specific_fields={"code_interpreter_results": [code_result]}, + ), + ) + ], + ) + iterator: Final = _build_iterator( + [ + _reasoning_block_chunk( + ChatCompletionThinkingBlock(type="thinking", thinking="first", signature="first-sig") + ), + tool, + _reasoning_block_chunk( + ChatCompletionThinkingBlock(type="thinking", thinking="second", signature="second-sig") + ), + _chunk("answer", finish_reason="stop"), + ] + ) + events: Final = [json.loads(event.model_dump_json(exclude_none=True)) async for event in iterator] + tool_added: Final = next( + event + for event in events + if event["type"] == "response.output_item.added" and event["item"].get("call_id") == call_id + ) + output: Final = events[-1]["response"]["output"] + assert [item["type"] for item in output] == ["reasoning", "code_interpreter_call", "reasoning", "message"] + assert output[tool_added["output_index"]]["id"] == call_id + assert output[tool_added["output_index"]]["code"] == "print(1)" + + +@pytest.mark.asyncio +async def test_positionless_citations_stay_with_their_original_message() -> None: + citation: Final = {"type": "url_citation", "url_citation": {"title": "source", "url": "https://example.com"}} + intro: Final = ModelResponseStream( + id=CHAT_COMPLETION_ID, + model="test-model", + choices=[StreamingChoices(index=0, delta=Delta(content="intro", annotations=[citation]))], + ) + answer: Final = ModelResponseStream( + id=CHAT_COMPLETION_ID, + model="test-model", + choices=[StreamingChoices(index=0, delta=Delta(content="answer"), finish_reason="stop")], + ) + iterator: Final = _build_iterator( + [ + intro, + _reasoning_block_chunk(ChatCompletionThinkingBlock(type="thinking", thinking="analysis", signature="sig")), + answer, + ] + ) + events: Final = [json.loads(event.model_dump_json(exclude_none=True)) async for event in iterator] + messages: Final = [item for item in events[-1]["response"]["output"] if item["type"] == "message"] + assert [item["content"][0]["text"] for item in messages] == ["intro", "answer"] + assert messages[0]["content"][0]["annotations"] == [ + {"type": "url_citation", "title": "source", "url": "https://example.com"} + ] + assert messages[1]["content"][0]["annotations"] == [] + done_messages: Final = [ + event["item"] + for event in events + if event["type"] == "response.output_item.done" and event["item"]["type"] == "message" + ] + assert [item["content"] for item in done_messages] == [item["content"] for item in messages] + annotations: Final = [event for event in events if event["type"] == "response.output_text.annotation.added"] + assert len(annotations) == 1 + assert annotations[0]["item_id"] == messages[0]["id"]