From a4ce0f2ced0aca420f5d7ec4339d19010df2c060 Mon Sep 17 00:00:00 2001 From: vivekvar-dl Date: Sat, 8 Aug 2026 13:47:50 +0530 Subject: [PATCH 1/3] fix(proxy): deterministically close upstream streams for /v1/responses and /v1/messages The proxy's streaming cleanup closes the upstream response only when the stream object exposes aclose. LiteLLMCompletionStreamingIterator and BaseResponsesAPIStreamingIterator had none, and the /v1/messages passthrough generator never closed its httpx response, so releasing the provider connection depended on GC timing instead of an explicit close. The chat completions route got the explicit contract in #30245; this extends it to the two remaining streaming routes BaseResponsesAPIStreamingIterator gains a null-safe aclose (some subclasses bypass its constructor and carry no response), the bridge iterator delegates to its CustomStreamWrapper, and chunk_processor closes the upstream response in its finally after spend logging is scheduled Fixes #36123 --- .../streaming_handler.py | 4 + .../streaming_iterator.py | 3 + litellm/responses/streaming_iterator.py | 6 + .../test_streaming_handler.py | 114 ++++++++++++++++++ ...t_litellm_completion_streaming_iterator.py | 44 +++++++ .../responses/test_streaming_iterator.py | 47 ++++++++ 6 files changed, 218 insertions(+) create mode 100644 tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler.py create mode 100644 tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_streaming_iterator.py diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index ff1c12d08d76..f77fcb98fbdf 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -117,6 +117,10 @@ async def chunk_processor( ) except Exception as e: verbose_proxy_logger.error("Error scheduling chunk_processor logging: %s", e) + try: + await response.aclose() + except Exception as e: + verbose_proxy_logger.debug("Error closing passthrough upstream response: %s", e) @staticmethod async def _route_streaming_logging_to_handler( diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index ddd050757632..1a289fc4444f 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -106,6 +106,9 @@ def __init__( self._accumulated_provider_specific_fields: dict[str, Any] = {} self._custom_tool_names: set[str] = extract_custom_tool_names(self.responses_api_request.get("tools")) + async def aclose(self) -> None: + await self.litellm_custom_stream_wrapper.aclose() + def _get_or_assign_tool_output_index(self, call_id: str) -> int: existing: Final = self._tool_output_index_by_call_id.get(call_id) if existing is not None: diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 820839fc6bfc..fbce84444550 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -147,6 +147,12 @@ class BaseResponsesAPIStreamingIterator: This class contains shared logic for both synchronous and asynchronous iterators. """ + response: httpx.Response | None = None + + async def aclose(self) -> None: + if self.response is not None: + await self.response.aclose() + def __init__( self, response: httpx.Response, diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler.py new file mode 100644 index 000000000000..7648491e81c5 --- /dev/null +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler.py @@ -0,0 +1,114 @@ +"""Regression tests: chunk_processor must close the upstream httpx.Response +when the stream ends, including on client disconnect (generator aclose), so +the provider connection is released and backends like vLLM stop generating. +""" + +from collections.abc import AsyncIterator +from datetime import datetime, timezone +from typing import Final +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import ( + AnthropicMessagesStreamHiddenParams, + AnthropicMessagesStreamingResponse, +) +from litellm.proxy.pass_through_endpoints.streaming_handler import ( + PassThroughStreamingHandler, +) +from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType + + +class _TrackingStream(httpx.AsyncByteStream): + def __init__(self, chunks: list[bytes]): + self._chunks = chunks + self.aclose_called = False + + async def __aiter__(self) -> AsyncIterator[bytes]: + for chunk in self._chunks: + yield chunk + + async def aclose(self) -> None: + self.aclose_called = True + + +def _streaming_response(chunks: list[bytes]) -> tuple[httpx.Response, _TrackingStream]: + stream: Final = _TrackingStream(chunks) + response: Final = httpx.Response( + 200, + stream=stream, + request=httpx.Request("POST", "http://upstream.test/v1/messages"), + ) + return response, stream + + +def _logging_obj_stub() -> MagicMock: + logging_obj: Final = MagicMock() + logging_obj.completion_start_time = None + logging_obj.model_call_details = {} + return logging_obj + + +def _chunk_processor(response: httpx.Response): + return PassThroughStreamingHandler.chunk_processor( + response=response, + request_body={"model": "claude-sonnet-4-5"}, + litellm_logging_obj=_logging_obj_stub(), + endpoint_type=EndpointType.ANTHROPIC, + start_time=datetime.now(timezone.utc), + passthrough_success_handler_obj=MagicMock(), + url_route="/v1/messages", + ) + + +def _drop_scheduled_coroutine(async_coroutine) -> None: + async_coroutine.close() + + +@pytest.mark.asyncio +async def test_chunk_processor_closes_response_on_client_disconnect(): + response, stream = _streaming_response([b"event: message_start\n\n", b"event: message_stop\n\n"]) + generator: Final = _chunk_processor(response) + + with patch("litellm.proxy.pass_through_endpoints.streaming_handler.GLOBAL_LOGGING_WORKER") as worker: + worker.ensure_initialized_and_enqueue.side_effect = _drop_scheduled_coroutine + first_chunk: Final = await generator.__anext__() + assert first_chunk == b"event: message_start\n\n" + await generator.aclose() + + assert stream.aclose_called + assert response.is_closed + + +@pytest.mark.asyncio +async def test_chunk_processor_closes_response_on_natural_completion(): + response, stream = _streaming_response([b"event: message_start\n\n", b"event: message_stop\n\n"]) + generator: Final = _chunk_processor(response) + + with patch("litellm.proxy.pass_through_endpoints.streaming_handler.GLOBAL_LOGGING_WORKER") as worker: + worker.ensure_initialized_and_enqueue.side_effect = _drop_scheduled_coroutine + collected: Final = [chunk async for chunk in generator] + + assert collected == [b"event: message_start\n\n", b"event: message_stop\n\n"] + assert stream.aclose_called + assert response.is_closed + + +@pytest.mark.asyncio +async def test_anthropic_messages_streaming_response_aclose_closes_upstream_response(): + response, stream = _streaming_response([b"event: message_start\n\n"]) + wrapped: Final = AnthropicMessagesStreamingResponse( + completion_stream=_chunk_processor(response), + hidden_params=AnthropicMessagesStreamHiddenParams(additional_headers={}), + ) + + with patch("litellm.proxy.pass_through_endpoints.streaming_handler.GLOBAL_LOGGING_WORKER") as worker: + worker.ensure_initialized_and_enqueue.side_effect = _drop_scheduled_coroutine + first_chunk: Final = await wrapped.__anext__() + assert first_chunk == b"event: message_start\n\n" + await wrapped.aclose() + + assert stream.aclose_called + assert response.is_closed diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_streaming_iterator.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_streaming_iterator.py new file mode 100644 index 000000000000..ed8af6323a76 --- /dev/null +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_streaming_iterator.py @@ -0,0 +1,44 @@ +"""Regression test: the /v1/responses bridge iterator must expose aclose() +so the proxy's streaming cleanup can release the upstream provider +connection on client disconnect. +""" + +from typing import Final +from unittest.mock import MagicMock + +import pytest + +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.responses.litellm_completion_transformation.streaming_iterator import ( + LiteLLMCompletionStreamingIterator, +) + + +class _TrackingCompletionStream: + def __init__(self): + self.aclose_called = False + + async def aclose(self) -> None: + self.aclose_called = True + + +@pytest.mark.asyncio +async def test_aclose_closes_wrapped_completion_stream(): + completion_stream: Final = _TrackingCompletionStream() + wrapper: Final = CustomStreamWrapper( + completion_stream=completion_stream, + model="gpt-4o-mini", + logging_obj=MagicMock(), + custom_llm_provider="openai", + ) + iterator: Final = LiteLLMCompletionStreamingIterator( + model="gpt-4o-mini", + litellm_custom_stream_wrapper=wrapper, + request_input="hello", + responses_api_request={}, + ) + + await iterator.aclose() + + assert completion_stream.aclose_called + assert wrapper.completion_stream is None diff --git a/tests/test_litellm/responses/test_streaming_iterator.py b/tests/test_litellm/responses/test_streaming_iterator.py index 5b0f40fdf27e..b840728c5016 100644 --- a/tests/test_litellm/responses/test_streaming_iterator.py +++ b/tests/test_litellm/responses/test_streaming_iterator.py @@ -235,3 +235,50 @@ def test_sync_transport_error_before_completed_event_raises(): with pytest.raises(httpx.ReadError): for _ in iterator: pass + + +class _TrackingByteStream(httpx.AsyncByteStream): + def __init__(self, chunks: list[bytes]): + self._chunks = chunks + self.aclose_called = False + + async def __aiter__(self): + for chunk in self._chunks: + yield chunk + + async def aclose(self) -> None: + self.aclose_called = True + + +@pytest.mark.asyncio +async def test_aclose_closes_underlying_httpx_response(): + stream = _TrackingByteStream([_sse_event({"type": "response.output_text.delta", "delta": "hi"})]) + response = httpx.Response( + 200, + stream=stream, + request=httpx.Request("POST", "http://upstream.test/v1/responses"), + ) + + iterator = ResponsesAPIStreamingIterator( + response=response, + model="gpt-4o-mini", + responses_api_provider_config=_mock_config(), + logging_obj=_logging_obj_stub(), + litellm_metadata={}, + custom_llm_provider="openai", + ) + + await iterator.__anext__() + await iterator.aclose() + + assert stream.aclose_called + assert response.is_closed + + +@pytest.mark.asyncio +async def test_aclose_is_a_noop_for_iterators_without_a_response(): + from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator + + iterator = BaseResponsesAPIStreamingIterator.__new__(BaseResponsesAPIStreamingIterator) + + await iterator.aclose() From e8e7db248b2382aa3bc06724ecf993b39d221a14 Mon Sep 17 00:00:00 2001 From: vivekvar-dl Date: Sat, 8 Aug 2026 13:54:57 +0530 Subject: [PATCH 2/3] fix(proxy): suppress BLE001 on upstream close guard to stay within strict budget --- litellm/proxy/pass_through_endpoints/streaming_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index f77fcb98fbdf..15f4a81f3b4d 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -119,7 +119,7 @@ async def chunk_processor( verbose_proxy_logger.error("Error scheduling chunk_processor logging: %s", e) try: await response.aclose() - except Exception as e: + except Exception as e: # noqa: BLE001 # closing a possibly-broken transport must not mask stream teardown verbose_proxy_logger.debug("Error closing passthrough upstream response: %s", e) @staticmethod From 62ca98a3f7b5eea61616d7765acebe79a74316f3 Mon Sep 17 00:00:00 2001 From: vivekvar-dl Date: Sat, 8 Aug 2026 13:57:22 +0530 Subject: [PATCH 3/3] fix(responses): propagate aclose through MCPEnhancedStreamingIterator and type test helpers Greptile review: the MCP wrapper stores its upstream-owning iterator in base_iterator, so the inherited response-based aclose was a no-op there. Delegate to the base iterator's aclose when it has one. Also add the missing parameter and return annotations on the new passthrough test helpers --- .../responses/mcp/mcp_streaming_iterator.py | 5 ++++ .../test_streaming_handler.py | 6 ++--- .../mcp/test_mcp_streaming_iterator.py | 26 +++++++++++++++++++ 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index e4cc36de06ca..e4cf858b49bd 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -326,6 +326,11 @@ def __init__( self._error_event_emitted = False self._last_sequence_number = 0 + async def aclose(self) -> None: + base_aclose: Final = getattr(self.base_iterator, "aclose", None) + if base_aclose is not None: + await base_aclose() + def _extract_mcp_headers_from_params(self) -> None: """Extract MCP headers from original request params to pass to tool calls""" diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler.py index 7648491e81c5..f03f4312a1a4 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler.py @@ -3,7 +3,7 @@ the provider connection is released and backends like vLLM stop generating. """ -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Coroutine from datetime import datetime, timezone from typing import Final from unittest.mock import MagicMock, patch @@ -51,7 +51,7 @@ def _logging_obj_stub() -> MagicMock: return logging_obj -def _chunk_processor(response: httpx.Response): +def _chunk_processor(response: httpx.Response) -> AsyncIterator[bytes]: return PassThroughStreamingHandler.chunk_processor( response=response, request_body={"model": "claude-sonnet-4-5"}, @@ -63,7 +63,7 @@ def _chunk_processor(response: httpx.Response): ) -def _drop_scheduled_coroutine(async_coroutine) -> None: +def _drop_scheduled_coroutine(async_coroutine: Coroutine[object, object, object]) -> None: async_coroutine.close() diff --git a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py index 24edf12fffe9..8ec38fd4a8b0 100644 --- a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py +++ b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py @@ -257,3 +257,29 @@ async def test_initial_call_failure_is_stashed_for_eager_reraise(monkeypatch): assert iterator._initial_creation_error is not None assert "initial boom" in str(iterator._initial_creation_error) + + +@pytest.mark.asyncio +async def test_aclose_closes_base_iterator(): + class _ClosableStream: + def __init__(self): + self.aclose_called = False + + async def aclose(self) -> None: + self.aclose_called = True + + iterator = _make_iterator([]) + closable = _ClosableStream() + iterator.base_iterator = closable + + await iterator.aclose() + + assert closable.aclose_called + + +@pytest.mark.asyncio +async def test_aclose_is_a_noop_without_base_iterator(): + iterator = _make_iterator([]) + iterator.base_iterator = None + + await iterator.aclose()