Skip to content

fix(chatgpt): preserve responses routing and recover empty output #26219

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
krrish-berri-2 merged 1 commit into from
Apr 22, 2026
Merged

fix(chatgpt): preserve responses routing and recover empty output #26219

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 137 additions & 4 deletions litellm/completion_extras/litellm_responses_transformation/transformation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import litellm
from litellm import ModelResponse
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.bridges.completion_transformation import (
CompletionTransformationBridge,
Expand Down Expand Up @@ -97,7 +98,7 @@ def _build_reasoning_item(


def _reasoning_item_to_response_input(
r_item: Union[ChatCompletionReasoningItem, Dict[str, Any]]
r_item: Union[ChatCompletionReasoningItem, Dict[str, Any]],
) -> Dict[str, Any]:
"""Convert a stored ChatCompletionReasoningItem back to a Responses API input item."""
r_input: Dict[str, Any] = {
Expand Down Expand Up @@ -583,6 +584,125 @@ def _convert_response_output_to_choices(

return choices

@classmethod
def _recover_output_items_from_raw_sse(
cls, raw_sse: Optional[str]
) -> List[Dict[str, Any]]:
if not raw_sse or not isinstance(raw_sse, str):
return []

recovered_output_items: Dict[int, Dict[str, Any]] = {}
recovered_text_only_items: Dict[int, Dict[str, Any]] = {}

for chunk in raw_sse.splitlines():
stripped_chunk = (
CustomStreamWrapper._strip_sse_data_from_chunk(chunk.strip()) or ""
).strip()
if (
not stripped_chunk
or stripped_chunk == "[DONE]"
or stripped_chunk.startswith("event:")
):
continue

try:
parsed_chunk = json.loads(stripped_chunk)
except json.JSONDecodeError:
continue

if not isinstance(parsed_chunk, dict):
continue

event_type = parsed_chunk.get("type")

if event_type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED:
response_payload = parsed_chunk.get("response")
if isinstance(response_payload, dict):
response_output = response_payload.get("output")
if isinstance(response_output, list) and len(response_output) > 0:
return cast(List[Dict[str, Any]], response_output)
continue

if event_type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE:
item = parsed_chunk.get("item")
if not isinstance(item, dict):
continue
try:
output_index = int(parsed_chunk.get("output_index"))
except (TypeError, ValueError):
output_index = len(recovered_output_items)
recovered_output_items[output_index] = item
continue

if event_type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE:
text = parsed_chunk.get("text")
if not isinstance(text, str):
continue

try:
output_index = int(parsed_chunk.get("output_index"))
except (TypeError, ValueError):
output_index = len(recovered_text_only_items)

item = recovered_output_items.get(
output_index
) or recovered_text_only_items.get(output_index)
if item is None:
item = {
"type": "message",
"id": parsed_chunk.get("item_id") or f"msg_{output_index}",
"role": "assistant",
"status": "completed",
"content": [],
}
recovered_text_only_items[output_index] = item

content = item.setdefault("content", [])
if not isinstance(content, list):
continue

try:
content_index = int(parsed_chunk.get("content_index"))
except (TypeError, ValueError):
content_index = len(content)

while len(content) <= content_index:
content.append(
{
"type": "output_text",
"text": "",
"annotations": [],
}
)

content_item = content[content_index]
if not isinstance(content_item, dict):
content_item = {}
content[content_index] = content_item

content_item["type"] = "output_text"
content_item["text"] = text
if parsed_chunk.get("annotations") is not None:
content_item["annotations"] = parsed_chunk["annotations"]
else:
content_item.setdefault("annotations", [])

if recovered_output_items:
return [item for _, item in sorted(recovered_output_items.items())]

if recovered_text_only_items:
return [item for _, item in sorted(recovered_text_only_items.items())]

return []
Comment on lines +587 to +696

@greptile-apps greptile-apps Bot Apr 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Duplicated SSE-parsing logic between layers

_recover_output_items_from_raw_sse re-implements much of the SSE-scanning logic that was just added to ChatGPTResponsesAPIConfig in litellm/llms/chatgpt/responses/transformation.py. When the ChatGPT layer successfully recovers items, raw_response.output is already populated and this fallback is never reached—making the completion_extras/ copy purely belt-and-suspenders for non-ChatGPT Responses-API providers. If that's the intent, a brief comment explaining which providers need this second-pass fallback and why would help future readers avoid silently removing one layer thinking it is dead code.


@classmethod
def _recover_output_items_from_logging(
cls, logging_obj: "LiteLLMLoggingObj"
) -> List[Dict[str, Any]]:
model_call_details = getattr(logging_obj, "model_call_details", {}) or {}
original_response = model_call_details.get("original_response")
return cls._recover_output_items_from_raw_sse(original_response)

def transform_response( # noqa: PLR0915
self,
model: str,
Expand All @@ -607,9 +727,22 @@ def transform_response( # noqa: PLR0915
if raw_response.error is not None:
raise ValueError(f"Error in response: {raw_response.error}")

output_items = raw_response.output
if len(output_items) == 0:
recovered_output_items = self._recover_output_items_from_logging(
logging_obj
)
if recovered_output_items:
output_items = recovered_output_items
raw_response.output = recovered_output_items

@greptile-apps greptile-apps Bot Apr 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Mutation of the input raw_response object

raw_response.output = recovered_output_items silently mutates the caller-owned ResponsesAPIResponse object. Any code that holds a reference to raw_response after transform_response returns will see the injected output, which may be unexpected. Assigning to a local output_items variable (already done two lines earlier) is sufficient for the local path; the raw_response mutation is unnecessary for correctness here.

verbose_logger.warning(
"Recovered empty Responses API output from raw SSE for model=%s",
model,
)

# Convert response output to choices using the static helper
choices = self._convert_response_output_to_choices(
output_items=raw_response.output,
output_items=output_items,
handle_raw_dict_callback=self._handle_raw_dict_response_item,
)

Expand All @@ -623,7 +756,7 @@ def transform_response( # noqa: PLR0915
)
else:
raise ValueError(
f"Unknown items in responses API response: {raw_response.output}"
f"Unknown items in responses API response: {output_items}"
)

setattr(model_response, "choices", choices)
Expand Down Expand Up @@ -1211,7 +1344,7 @@ def translate_responses_chunk_to_openai_stream( # noqa: PLR0915
raise ValueError(
f"Chat provider: Invalid function argument delta {parsed_chunk}"
)
elif event_type == "response.output_item.done":
elif event_type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE:
# New output item added
output_item = parsed_chunk.get("item", {})
if output_item.get("type") == "function_call":
Expand Down
17 changes: 16 additions & 1 deletion litellm/llms/chatgpt/responses/transformation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import json
from typing import Any, Optional
from typing import Any, Dict, Optional

from litellm.constants import STREAM_SSE_DONE_STRING
from litellm.exceptions import AuthenticationError
Expand Down Expand Up @@ -134,6 +134,7 @@ def transform_response_api_response(

completed_response = None
error_message = None
streamed_output_items: Dict[int, dict] = {}
for chunk in body_text.splitlines():
stripped_chunk = CustomStreamWrapper._strip_sse_data_from_chunk(chunk)
if not stripped_chunk:
Expand All @@ -150,10 +151,24 @@ def transform_response_api_response(
if not isinstance(parsed_chunk, dict):
continue
event_type = parsed_chunk.get("type")
if event_type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE:
item = parsed_chunk.get("item")
output_index = parsed_chunk.get("output_index")
if isinstance(item, dict):
try:
index = int(output_index)
except (TypeError, ValueError):
index = len(streamed_output_items)
streamed_output_items[index] = item
continue
if event_type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED:
response_payload = parsed_chunk.get("response")
if isinstance(response_payload, dict):
response_payload = dict(response_payload)
if not response_payload.get("output") and streamed_output_items:
response_payload["output"] = [
item for _, item in sorted(streamed_output_items.items())
]
if "created_at" in response_payload:
response_payload["created_at"] = _safe_convert_created_field(
response_payload["created_at"]
Expand Down
12 changes: 12 additions & 0 deletions litellm/router.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6760,6 +6760,18 @@ def _create_deployment(
_shared_model_info = {
k: v for k, v in _model_info.items() if k not in _custom_pricing_fields
}
_existing_shared_mode = (
cast(Optional[dict], litellm.model_cost.get(_model_name, {})) or {}
).get("mode")
if (
_existing_shared_mode is not None
and _shared_model_info.get("mode") != _existing_shared_mode
):
# Keep the built-in bridge mode stable for shared backend keys.
# Multiple aliases can point at the same provider/model backend,
# but their deployment-level overrides should not downgrade the
# backend from responses -> chat via last-write-wins registration.
_shared_model_info.pop("mode", None)
Comment on lines +6766 to +6774

@greptile-apps greptile-apps Bot Apr 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Mode preservation is bidirectional, not just anti-downgrade

The guard pops mode from _shared_model_info whenever the incoming value differs from the established one—regardless of direction (e.g. chat → responses is also suppressed). This means a deployment that legitimately wants to upgrade an existing mode: chat shared-backend key to mode: responses would silently have its override dropped. The comment says "downgrade … responses → chat" but the code also blocks the reverse.

For the ChatGPT use-case (built-in key already has mode: responses) this is harmless, but it may surprise future users who try to use model_info.mode to correct a shared key that was first-registered with the wrong mode.

litellm.register_model(
model_cost={
_model_name: _shared_model_info,
Expand Down
Loading
Loading