From 8d6b8d2ce94b712b941fd117e1279454f880db50 Mon Sep 17 00:00:00 2001 From: LHMQ878 <72402929@cityu-dg.edu.cn> Date: Fri, 7 Aug 2026 10:47:42 +0800 Subject: [PATCH 1/9] fix(proxy): register WebSocket passthrough for OpenAI prefixes create_websocket_passthrough_route existed but /openai and /openai_passthrough only registered HTTP methods, so WS upgrades were rejected at routing. Add catch-all websocket routes mirroring the HTTP passthrough target construction. Fixes #36088 --- .../llm_passthrough_endpoints.py | 47 ++++++++++++++++++- .../test_openai_ws_passthrough_routes.py | 15 ++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 40c49df26cf8..f9409ab366f2 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -27,7 +27,7 @@ from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.proxy._types import * from litellm.proxy.auth.route_checks import RouteChecks -from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth, user_api_key_auth_websocket from litellm.proxy.common_utils.http_parsing_utils import ( _read_request_body, _safe_get_request_headers, @@ -1934,6 +1934,51 @@ async def openai_proxy_route( ) +@router.websocket("/openai_passthrough/{endpoint:path}") +@router.websocket("/openai/{endpoint:path}") +async def openai_websocket_proxy_route( + websocket: WebSocket, + endpoint: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth_websocket), +): + """WebSocket passthrough for OpenAI prefixes (realtime / responses.connect).""" + base_target_url = os.getenv("OPENAI_API_BASE") or "https://api.openai.com/" + openai_api_key = passthrough_endpoint_router.get_credentials( + custom_llm_provider=litellm.LlmProviders.OPENAI.value, + region_name=None, + ) + if openai_api_key is None: + await websocket.close(code=1011) + raise Exception("Required 'OPENAI_API_KEY' in environment to make pass-through calls to OpenAI.") + + encoded_endpoint = httpx.URL(endpoint).path + if not encoded_endpoint.startswith("/"): + encoded_endpoint = "/" + encoded_endpoint + base_url = httpx.URL(base_target_url) + updated_url = BaseOpenAIPassThroughHandler._join_url_paths( + base_url=base_url, + path=encoded_endpoint, + custom_llm_provider=litellm.LlmProviders.OPENAI, + ) + # HTTP(S) base -> WS(S) target for the upgrade. + if updated_url.startswith("https://"): + wss_target = "wss://" + updated_url[len("https://") :] + elif updated_url.startswith("http://"): + wss_target = "ws://" + updated_url[len("http://") :] + else: + wss_target = updated_url + + return await websocket_passthrough_request( + websocket=websocket, + target=wss_target, + custom_headers={"Authorization": f"Bearer {openai_api_key}"}, + user_api_key_dict=user_api_key_dict, + forward_headers=True, + endpoint=f"/openai/{endpoint}", + accept_websocket=True, + ) + + class BaseOpenAIPassThroughHandler: @staticmethod async def _base_openai_pass_through_handler( diff --git a/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py b/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py new file mode 100644 index 000000000000..e0184c6c4284 --- /dev/null +++ b/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py @@ -0,0 +1,15 @@ +"""OpenAI passthrough must register WebSocket catch-all routes (#36088).""" + +from starlette.routing import WebSocketRoute + +from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import router + + +def test_openai_websocket_passthrough_routes_registered(): + ws_paths = { + route.path + for route in router.routes + if isinstance(route, WebSocketRoute) + } + assert "/openai/{endpoint:path}" in ws_paths + assert "/openai_passthrough/{endpoint:path}" in ws_paths From 55b52970eac10e36d6563cbff775c0840aacb117 Mon Sep 17 00:00:00 2001 From: LHMQ878 <72402929@cityu-dg.edu.cn> Date: Fri, 7 Aug 2026 11:09:27 +0800 Subject: [PATCH 2/9] fix(proxy): preserve OpenAI WS query params and provider auth Forward realtime model query string, keep OPENAI_API_KEY (forward_headers=False), satisfy ruff strict gates, sync dashboard OpenAPI types, and cover the behavior in tests. --- .../llm_passthrough_endpoints.py | 18 +++-- .../test_openai_ws_passthrough_routes.py | 43 ++++++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 76 +++++++++++++++++++ 3 files changed, 128 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index f9409ab366f2..31a8836abf8c 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -9,7 +9,7 @@ import json import os import re -from typing import Any, Final, cast +from typing import Annotated, Any, Final, cast import httpx from fastapi import APIRouter, Depends, HTTPException, Request, Response, WebSocket @@ -1939,8 +1939,8 @@ async def openai_proxy_route( async def openai_websocket_proxy_route( websocket: WebSocket, endpoint: str, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth_websocket), -): + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth_websocket)], +) -> None: """WebSocket passthrough for OpenAI prefixes (realtime / responses.connect).""" base_target_url = os.getenv("OPENAI_API_BASE") or "https://api.openai.com/" openai_api_key = passthrough_endpoint_router.get_credentials( @@ -1949,7 +1949,7 @@ async def openai_websocket_proxy_route( ) if openai_api_key is None: await websocket.close(code=1011) - raise Exception("Required 'OPENAI_API_KEY' in environment to make pass-through calls to OpenAI.") + raise ValueError("Required 'OPENAI_API_KEY' in environment to make pass-through calls to OpenAI.") encoded_endpoint = httpx.URL(endpoint).path if not encoded_endpoint.startswith("/"): @@ -1960,7 +1960,6 @@ async def openai_websocket_proxy_route( path=encoded_endpoint, custom_llm_provider=litellm.LlmProviders.OPENAI, ) - # HTTP(S) base -> WS(S) target for the upgrade. if updated_url.startswith("https://"): wss_target = "wss://" + updated_url[len("https://") :] elif updated_url.startswith("http://"): @@ -1968,12 +1967,17 @@ async def openai_websocket_proxy_route( else: wss_target = updated_url - return await websocket_passthrough_request( + query_string = websocket.url.query + if query_string: + separator = "&" if "?" in wss_target else "?" + wss_target = f"{wss_target}{separator}{query_string}" + + await websocket_passthrough_request( websocket=websocket, target=wss_target, custom_headers={"Authorization": f"Bearer {openai_api_key}"}, user_api_key_dict=user_api_key_dict, - forward_headers=True, + forward_headers=False, endpoint=f"/openai/{endpoint}", accept_websocket=True, ) diff --git a/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py b/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py index e0184c6c4284..9101cd4b7806 100644 --- a/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py +++ b/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py @@ -1,8 +1,14 @@ -"""OpenAI passthrough must register WebSocket catch-all routes (#36088).""" +"""OpenAI passthrough must register WebSocket catch-all routes (#36088).""" from starlette.routing import WebSocketRoute +from unittest.mock import AsyncMock, MagicMock, patch -from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import router +import pytest + +from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + openai_websocket_proxy_route, + router, +) def test_openai_websocket_passthrough_routes_registered(): @@ -13,3 +19,36 @@ def test_openai_websocket_passthrough_routes_registered(): } assert "/openai/{endpoint:path}" in ws_paths assert "/openai_passthrough/{endpoint:path}" in ws_paths + + +@pytest.mark.asyncio +async def test_openai_websocket_forwards_query_and_keeps_provider_auth(): + websocket = MagicMock() + websocket.url.query = "model=gpt-4o-realtime-preview" + websocket.close = AsyncMock() + user = MagicMock() + + with ( + patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router.get_credentials", + return_value="sk-provider", + ), + patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.BaseOpenAIPassThroughHandler._join_url_paths", + return_value="https://api.openai.com/v1/realtime", + ), + patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.websocket_passthrough_request", + new_callable=AsyncMock, + ) as mock_ws, + ): + await openai_websocket_proxy_route( + websocket=websocket, + endpoint="v1/realtime", + user_api_key_dict=user, + ) + + kwargs = mock_ws.await_args.kwargs + assert kwargs["target"] == "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview" + assert kwargs["custom_headers"] == {"Authorization": "Bearer sk-provider"} + assert kwargs["forward_headers"] is False diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 752572f9863b..d175f94c634f 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -8483,6 +8483,26 @@ export interface paths { patch?: never; trace?: never; }; + "/openai/": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * WebSocket: openai_websocket_proxy_route + * @description WebSocket connection endpoint + */ + get: operations["websocket_openai_websocket_proxy_route_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/openai/deployments/{model}/chat/completions": { parameters: { query?: never; @@ -8983,6 +9003,26 @@ export interface paths { patch: operations["openai_proxy_route_openai__endpoint__patch"]; trace?: never; }; + "/openai_passthrough/": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * WebSocket: openai_websocket_proxy_route + * @description WebSocket connection endpoint + */ + get: operations["websocket_openai_websocket_proxy_route_get_2"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/openai_passthrough/{endpoint}": { parameters: { query?: never; @@ -46427,6 +46467,24 @@ export interface operations { }; }; }; + websocket_openai_websocket_proxy_route_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description WebSocket Protocol Switched */ + 101: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; chat_completion_openai_deployments__model__chat_completions_post: { parameters: { query?: never; @@ -47286,6 +47344,24 @@ export interface operations { }; }; }; + websocket_openai_websocket_proxy_route_get_2: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description WebSocket Protocol Switched */ + 101: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; openai_proxy_route_openai_passthrough__endpoint__get: { parameters: { query?: never; From ae4ee365902478c5141f166f94e739a8860674ff Mon Sep 17 00:00:00 2001 From: LHMQ878 <72402929@cityu-dg.edu.cn> Date: Fri, 7 Aug 2026 11:21:25 +0800 Subject: [PATCH 3/9] fix(proxy): satisfy type-discipline Final/mutable rules on OpenAI WS route --- .../llm_passthrough_endpoints.py | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 31a8836abf8c..9ff71c771301 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -1942,8 +1942,8 @@ async def openai_websocket_proxy_route( user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth_websocket)], ) -> None: """WebSocket passthrough for OpenAI prefixes (realtime / responses.connect).""" - base_target_url = os.getenv("OPENAI_API_BASE") or "https://api.openai.com/" - openai_api_key = passthrough_endpoint_router.get_credentials( + base_target_url: Final = os.getenv("OPENAI_API_BASE") or "https://api.openai.com/" + openai_api_key: Final = passthrough_endpoint_router.get_credentials( custom_llm_provider=litellm.LlmProviders.OPENAI.value, region_name=None, ) @@ -1951,31 +1951,33 @@ async def openai_websocket_proxy_route( await websocket.close(code=1011) raise ValueError("Required 'OPENAI_API_KEY' in environment to make pass-through calls to OpenAI.") - encoded_endpoint = httpx.URL(endpoint).path - if not encoded_endpoint.startswith("/"): - encoded_endpoint = "/" + encoded_endpoint - base_url = httpx.URL(base_target_url) - updated_url = BaseOpenAIPassThroughHandler._join_url_paths( + raw_path: Final = httpx.URL(endpoint).path + encoded_endpoint: Final = raw_path if raw_path.startswith("/") else f"/{raw_path}" + base_url: Final = httpx.URL(base_target_url) + updated_url: Final = BaseOpenAIPassThroughHandler._join_url_paths( base_url=base_url, path=encoded_endpoint, custom_llm_provider=litellm.LlmProviders.OPENAI, ) - if updated_url.startswith("https://"): - wss_target = "wss://" + updated_url[len("https://") :] - elif updated_url.startswith("http://"): - wss_target = "ws://" + updated_url[len("http://") :] - else: - wss_target = updated_url - - query_string = websocket.url.query - if query_string: - separator = "&" if "?" in wss_target else "?" - wss_target = f"{wss_target}{separator}{query_string}" + wss_base: Final = ( + "wss://" + updated_url[len("https://") :] + if updated_url.startswith("https://") + else "ws://" + updated_url[len("http://") :] + if updated_url.startswith("http://") + else updated_url + ) + query_string: Final = websocket.url.query + wss_target: Final = ( + f"{wss_base}{'&' if '?' in wss_base else '?'}{query_string}" if query_string else wss_base + ) + custom_headers: Final = { + "Authorization": f"Bearer {openai_api_key}" + } # mutable-ok: websocket_passthrough_request requires a plain dict of upstream headers await websocket_passthrough_request( websocket=websocket, target=wss_target, - custom_headers={"Authorization": f"Bearer {openai_api_key}"}, + custom_headers=custom_headers, user_api_key_dict=user_api_key_dict, forward_headers=False, endpoint=f"/openai/{endpoint}", From 7a9c38ed0f1302f9064962ed3f951a41ca6a9bd3 Mon Sep 17 00:00:00 2001 From: LHMQ878 <72402929@cityu-dg.edu.cn> Date: Fri, 7 Aug 2026 11:21:46 +0800 Subject: [PATCH 4/9] fix(proxy): place mutable-ok on OpenAI WS headers dict literal --- .../proxy/pass_through_endpoints/llm_passthrough_endpoints.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 9ff71c771301..9ef5f22ec4be 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -1970,9 +1970,9 @@ async def openai_websocket_proxy_route( wss_target: Final = ( f"{wss_base}{'&' if '?' in wss_base else '?'}{query_string}" if query_string else wss_base ) - custom_headers: Final = { + custom_headers: Final = { # mutable-ok: websocket_passthrough_request requires a plain dict of upstream headers "Authorization": f"Bearer {openai_api_key}" - } # mutable-ok: websocket_passthrough_request requires a plain dict of upstream headers + } await websocket_passthrough_request( websocket=websocket, From 02d4f8d6a86eae6d02393d29f99e72f0712eb234 Mon Sep 17 00:00:00 2001 From: LHMQ878 <72402929@cityu-dg.edu.cn> Date: Fri, 7 Aug 2026 11:45:21 +0800 Subject: [PATCH 5/9] style(proxy): ruff-format OpenAI websocket passthrough route --- .../proxy/pass_through_endpoints/llm_passthrough_endpoints.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 9ef5f22ec4be..216d966b800f 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -1967,9 +1967,7 @@ async def openai_websocket_proxy_route( else updated_url ) query_string: Final = websocket.url.query - wss_target: Final = ( - f"{wss_base}{'&' if '?' in wss_base else '?'}{query_string}" if query_string else wss_base - ) + wss_target: Final = f"{wss_base}{'&' if '?' in wss_base else '?'}{query_string}" if query_string else wss_base custom_headers: Final = { # mutable-ok: websocket_passthrough_request requires a plain dict of upstream headers "Authorization": f"Bearer {openai_api_key}" } From a258b2b1309eaa100efc9fddfbdbe588fa12052c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 16 Aug 2026 13:51:50 -0700 Subject: [PATCH 6/9] fix(proxy): harden OpenAI websocket passthrough - decode upstream first frame as utf-8 instead of ascii - reject model-restricted keys at connect to match HTTP model enforcement - log the actual request path for /openai_passthrough traffic --- .../llm_passthrough_endpoints.py | 23 ++++- .../pass_through_endpoints.py | 4 +- .../test_pass_through_endpoints.py | 78 ++++++++++++++++ .../test_openai_ws_passthrough_routes.py | 89 ++++++++++++++++--- 4 files changed, 179 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 216d966b800f..035e6fb7e55b 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -1934,6 +1934,20 @@ async def openai_proxy_route( ) +_OPENAI_WS_ALL_MODEL_ACCESS: Final = frozenset( + { + SpecialModelNames.all_proxy_models.value, + SpecialModelNames.all_team_models.value, + "*", + } +) + + +def _key_has_model_restrictions(user_api_key_dict: UserAPIKeyAuth) -> bool: + scoped_models: Final = (*user_api_key_dict.models, *user_api_key_dict.team_models) + return any(str(model) not in _OPENAI_WS_ALL_MODEL_ACCESS for model in scoped_models) + + @router.websocket("/openai_passthrough/{endpoint:path}") @router.websocket("/openai/{endpoint:path}") async def openai_websocket_proxy_route( @@ -1942,6 +1956,13 @@ async def openai_websocket_proxy_route( user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth_websocket)], ) -> None: """WebSocket passthrough for OpenAI prefixes (realtime / responses.connect).""" + if _key_has_model_restrictions(user_api_key_dict): + await websocket.close( + code=1008, + reason="Keys with model restrictions cannot use OpenAI websocket passthrough", + ) + return + base_target_url: Final = os.getenv("OPENAI_API_BASE") or "https://api.openai.com/" openai_api_key: Final = passthrough_endpoint_router.get_credentials( custom_llm_provider=litellm.LlmProviders.OPENAI.value, @@ -1978,7 +1999,7 @@ async def openai_websocket_proxy_route( custom_headers=custom_headers, user_api_key_dict=user_api_key_dict, forward_headers=False, - endpoint=f"/openai/{endpoint}", + endpoint=websocket.url.path, accept_websocket=True, ) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 8a526fcd6cb7..f0845940ce8a 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -2085,8 +2085,8 @@ async def forward_upstream_to_client() -> None: raw_response = await upstream_ws.recv(decode=False) # Ensure raw_response is bytes before decoding if isinstance(raw_response, str): - raw_response = raw_response.encode("ascii") - setup_response: Final = json.loads(raw_response.decode("ascii")) + raw_response = raw_response.encode("utf-8") + setup_response: Final = json.loads(raw_response.decode("utf-8")) verbose_proxy_logger.debug("Setup response: %s", setup_response) # Extract model and provider from setup response for Vertex AI Live diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 9bddeda07239..fe470aa65009 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -30,6 +30,7 @@ pass_through_request, resolve_pass_through_request_timeout, resolve_llm_passthrough_timeout, + websocket_passthrough_request, ) from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import UserAPIKeyAuth @@ -4877,3 +4878,80 @@ async def test_unusable_upstream_cost_records_zero_not_the_flat_estimate(): assert len(payloads) == 1 assert payloads[0]["response_cost"] == 0.0 assert payloads[0]["total_tokens"] == 1874 + + +class FakeUpstreamWebSocket: + def __init__(self, first_frame: bytes): + self._first_frame = first_frame + self.close = AsyncMock() + + async def recv(self, decode: bool = True): + return self._first_frame + + def __aiter__(self): + return self + + async def __anext__(self): + raise StopAsyncIteration + + +class FakeUpstreamConnect: + def __init__(self, upstream_ws: FakeUpstreamWebSocket): + self._upstream_ws = upstream_ws + + async def __aenter__(self): + return self._upstream_ws + + async def __aexit__(self, exc_type, exc, tb): + return False + + +@pytest.mark.asyncio +async def test_websocket_passthrough_forwards_non_ascii_first_frame(): + from starlette.websockets import WebSocketState + + first_frame = json.dumps( + {"type": "session.created", "session": {"instructions": "Hablas español, ¿sí?"}}, + ensure_ascii=False, + ).encode("utf-8") + upstream_ws = FakeUpstreamWebSocket(first_frame) + + websocket = MagicMock() + websocket.accept = AsyncMock() + websocket.send_text = AsyncMock() + websocket.send_bytes = AsyncMock() + websocket.close = AsyncMock() + websocket.receive = AsyncMock(return_value={"type": "websocket.disconnect"}) + websocket.headers = {} + websocket.client_state = WebSocketState.CONNECTED + + with ( + patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging, + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.connect", + return_value=FakeUpstreamConnect(upstream_ws), + ), + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.GLOBAL_LOGGING_WORKER" + ) as mock_worker, + ): + mock_proxy_logging.pre_call_hook = AsyncMock(return_value={}) + mock_proxy_logging.post_call_success_hook = AsyncMock() + mock_proxy_logging.post_call_failure_hook = AsyncMock() + mock_worker.ensure_initialized_and_enqueue = MagicMock( + side_effect=lambda async_coroutine: async_coroutine.close() + ) + await websocket_passthrough_request( + websocket=websocket, + target="wss://api.openai.com/v1/realtime?model=gpt-realtime", + custom_headers={"Authorization": "Bearer sk-test"}, + user_api_key_dict=UserAPIKeyAuth(), + forward_headers=False, + endpoint="/openai/v1/realtime", + accept_websocket=True, + ) + + websocket.send_text.assert_awaited_once() + forwarded = json.loads(websocket.send_text.await_args.args[0]) + assert forwarded["session"]["instructions"] == "Hablas español, ¿sí?" + assert all(call.kwargs.get("code") != 1011 for call in websocket.close.await_args_list) diff --git a/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py b/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py index 9101cd4b7806..8ca09ec59cfe 100644 --- a/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py +++ b/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py @@ -1,10 +1,11 @@ -"""OpenAI passthrough must register WebSocket catch-all routes (#36088).""" +"""OpenAI passthrough must register WebSocket catch-all routes (#36088).""" -from starlette.routing import WebSocketRoute from unittest.mock import AsyncMock, MagicMock, patch import pytest +from starlette.routing import WebSocketRoute +from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( openai_websocket_proxy_route, router, @@ -12,21 +13,23 @@ def test_openai_websocket_passthrough_routes_registered(): - ws_paths = { - route.path - for route in router.routes - if isinstance(route, WebSocketRoute) - } + ws_paths = {route.path for route in router.routes if isinstance(route, WebSocketRoute)} assert "/openai/{endpoint:path}" in ws_paths assert "/openai_passthrough/{endpoint:path}" in ws_paths -@pytest.mark.asyncio -async def test_openai_websocket_forwards_query_and_keeps_provider_auth(): +def _mock_websocket(path: str, query: str) -> MagicMock: websocket = MagicMock() - websocket.url.query = "model=gpt-4o-realtime-preview" + websocket.url.path = path + websocket.url.query = query websocket.close = AsyncMock() - user = MagicMock() + return websocket + + +@pytest.mark.asyncio +@pytest.mark.parametrize("prefix", ["openai", "openai_passthrough"]) +async def test_openai_websocket_forwards_query_and_keeps_provider_auth(prefix): + websocket = _mock_websocket(f"/{prefix}/v1/realtime", "model=gpt-4o-realtime-preview") with ( patch( @@ -45,10 +48,72 @@ async def test_openai_websocket_forwards_query_and_keeps_provider_auth(): await openai_websocket_proxy_route( websocket=websocket, endpoint="v1/realtime", - user_api_key_dict=user, + user_api_key_dict=UserAPIKeyAuth(), ) kwargs = mock_ws.await_args.kwargs assert kwargs["target"] == "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview" assert kwargs["custom_headers"] == {"Authorization": "Bearer sk-provider"} assert kwargs["forward_headers"] is False + assert kwargs["endpoint"] == f"/{prefix}/v1/realtime" + websocket.close.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "user_api_key_dict", + [ + UserAPIKeyAuth(models=["gpt-4o"]), + UserAPIKeyAuth(team_models=["gpt-4o-realtime-preview"]), + UserAPIKeyAuth(models=["all-team-models"], team_models=["gpt-4o"]), + ], +) +async def test_openai_websocket_rejects_model_restricted_keys(user_api_key_dict): + websocket = _mock_websocket("/openai/v1/realtime", "model=gpt-4o-realtime-preview") + + with patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.websocket_passthrough_request", + new_callable=AsyncMock, + ) as mock_ws: + await openai_websocket_proxy_route( + websocket=websocket, + endpoint="v1/realtime", + user_api_key_dict=user_api_key_dict, + ) + + websocket.close.assert_awaited_once() + assert websocket.close.await_args.kwargs["code"] == 1008 + mock_ws.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "user_api_key_dict", + [ + UserAPIKeyAuth(), + UserAPIKeyAuth(models=["all-proxy-models"]), + UserAPIKeyAuth(models=["*"]), + UserAPIKeyAuth(models=["all-team-models"], team_models=["all-proxy-models"]), + ], +) +async def test_openai_websocket_allows_unrestricted_keys(user_api_key_dict): + websocket = _mock_websocket("/openai/v1/responses", "") + + with ( + patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router.get_credentials", + return_value="sk-provider", + ), + patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.websocket_passthrough_request", + new_callable=AsyncMock, + ) as mock_ws, + ): + await openai_websocket_proxy_route( + websocket=websocket, + endpoint="v1/responses", + user_api_key_dict=user_api_key_dict, + ) + + mock_ws.assert_awaited_once() + websocket.close.assert_not_awaited() From 862f33bbaacffaa5ca6129de363ec60815150064 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 16 Aug 2026 14:11:12 -0700 Subject: [PATCH 7/9] fix(proxy): negotiate client subprotocol on OpenAI websocket passthrough --- .../llm_passthrough_endpoints.py | 9 ++++- .../test_openai_ws_passthrough_routes.py | 38 ++++++++++++++++++- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 035e6fb7e55b..22c2a2a8f16d 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -1993,6 +1993,13 @@ async def openai_websocket_proxy_route( "Authorization": f"Bearer {openai_api_key}" } + requested_subprotocols: Final = tuple( + protocol.strip() + for protocol in (websocket.headers.get("sec-websocket-protocol") or "").split(",") + if protocol.strip() + ) + await websocket.accept(subprotocol=requested_subprotocols[0] if requested_subprotocols else None) + await websocket_passthrough_request( websocket=websocket, target=wss_target, @@ -2000,7 +2007,7 @@ async def openai_websocket_proxy_route( user_api_key_dict=user_api_key_dict, forward_headers=False, endpoint=websocket.url.path, - accept_websocket=True, + accept_websocket=False, ) diff --git a/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py b/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py index 8ca09ec59cfe..d59468dcad6e 100644 --- a/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py +++ b/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py @@ -18,10 +18,12 @@ def test_openai_websocket_passthrough_routes_registered(): assert "/openai_passthrough/{endpoint:path}" in ws_paths -def _mock_websocket(path: str, query: str) -> MagicMock: +def _mock_websocket(path: str, query: str, headers: dict[str, str] | None = None) -> MagicMock: websocket = MagicMock() websocket.url.path = path websocket.url.query = query + websocket.headers = headers or {} + websocket.accept = AsyncMock() websocket.close = AsyncMock() return websocket @@ -56,6 +58,39 @@ async def test_openai_websocket_forwards_query_and_keeps_provider_auth(prefix): assert kwargs["custom_headers"] == {"Authorization": "Bearer sk-provider"} assert kwargs["forward_headers"] is False assert kwargs["endpoint"] == f"/{prefix}/v1/realtime" + assert kwargs["accept_websocket"] is False + websocket.accept.assert_awaited_once_with(subprotocol=None) + websocket.close.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_openai_websocket_accepts_first_client_subprotocol(): + websocket = _mock_websocket( + "/openai/v1/realtime", + "model=gpt-4o-realtime-preview", + headers={ + "sec-websocket-protocol": "realtime, openai-insecure-api-key.sk-abc, openai-beta.realtime-v1" + }, + ) + + with ( + patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router.get_credentials", + return_value="sk-provider", + ), + patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.websocket_passthrough_request", + new_callable=AsyncMock, + ) as mock_ws, + ): + await openai_websocket_proxy_route( + websocket=websocket, + endpoint="v1/realtime", + user_api_key_dict=UserAPIKeyAuth(), + ) + + websocket.accept.assert_awaited_once_with(subprotocol="realtime") + assert mock_ws.await_args.kwargs["accept_websocket"] is False websocket.close.assert_not_awaited() @@ -83,6 +118,7 @@ async def test_openai_websocket_rejects_model_restricted_keys(user_api_key_dict) websocket.close.assert_awaited_once() assert websocket.close.await_args.kwargs["code"] == 1008 + websocket.accept.assert_not_awaited() mock_ws.assert_not_awaited() From 4ba9d6b136fc30ff997c322a0e36b107406e9201 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 16 Aug 2026 14:29:21 -0700 Subject: [PATCH 8/9] fix(proxy): expose url join helper at module level for websocket route --- .../pass_through_endpoints/llm_passthrough_endpoints.py | 6 +----- .../test_llm_pass_through_endpoints.py | 9 +++++---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index d9c76aaeae39..96e32ccf827c 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -2086,7 +2086,7 @@ async def _base_openai_pass_through_handler( # Construct the full target URL by properly joining the base URL and endpoint path base_url: Final = httpx.URL(base_target_url) - updated_url: Final = BaseOpenAIPassThroughHandler._join_url_paths( + updated_url: Final = _join_url_paths( base_url=base_url, path=encoded_endpoint, custom_llm_provider=custom_llm_provider, @@ -2145,10 +2145,6 @@ def _assemble_headers(api_key: str | None, request: Request, extra_headers: dict request=request, ) - @staticmethod - def _join_url_paths(base_url: httpx.URL, path: str, custom_llm_provider: litellm.LlmProviders) -> str: - return _join_url_paths(base_url=base_url, path=path, custom_llm_provider=custom_llm_provider) - @router.api_route( "/cursor/{endpoint:path}", diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 8080ca71773a..9e6a2d427570 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -19,6 +19,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( BaseOpenAIPassThroughHandler, RouteChecks, + _join_url_paths, azure_proxy_route, bedrock_llm_proxy_route, create_pass_through_route, @@ -74,7 +75,7 @@ def test_join_url_paths(self): # Test joining base URL with no path and a path base_url = httpx.URL("https://api.example.com") path = "/v1/chat/completions" - result = BaseOpenAIPassThroughHandler._join_url_paths( + result = _join_url_paths( base_url, path, litellm.LlmProviders.OPENAI.value ) print(f"Base URL with no path: '{base_url}' + '{path}' → '{result}'") @@ -83,7 +84,7 @@ def test_join_url_paths(self): # Test joining base URL with path and another path base_url = httpx.URL("https://api.example.com/v1") path = "/chat/completions" - result = BaseOpenAIPassThroughHandler._join_url_paths( + result = _join_url_paths( base_url, path, litellm.LlmProviders.OPENAI.value ) print(f"Base URL with path: '{base_url}' + '{path}' → '{result}'") @@ -92,7 +93,7 @@ def test_join_url_paths(self): # Test with path not starting with slash base_url = httpx.URL("https://api.example.com/v1") path = "chat/completions" - result = BaseOpenAIPassThroughHandler._join_url_paths( + result = _join_url_paths( base_url, path, litellm.LlmProviders.OPENAI.value ) print(f"Path without leading slash: '{base_url}' + '{path}' → '{result}'") @@ -101,7 +102,7 @@ def test_join_url_paths(self): # Test with base URL having trailing slash base_url = httpx.URL("https://api.example.com/v1/") path = "/chat/completions" - result = BaseOpenAIPassThroughHandler._join_url_paths( + result = _join_url_paths( base_url, path, litellm.LlmProviders.OPENAI.value ) print(f"Base URL with trailing slash: '{base_url}' + '{path}' → '{result}'") From 5965648547b016e5993cf8bb2ec4d1b3c447d2c8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 16 Aug 2026 14:40:37 -0700 Subject: [PATCH 9/9] fix(proxy): close websocket cleanly when OpenAI credentials are missing --- .../llm_passthrough_endpoints.py | 7 +++-- .../test_openai_ws_passthrough_routes.py | 26 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 96e32ccf827c..8cdcdc07547a 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -2025,8 +2025,11 @@ async def openai_websocket_proxy_route( region_name=None, ) if openai_api_key is None: - await websocket.close(code=1011) - raise ValueError("Required 'OPENAI_API_KEY' in environment to make pass-through calls to OpenAI.") + await websocket.close( + code=1011, + reason="Required 'OPENAI_API_KEY' in environment to make pass-through calls to OpenAI.", + ) + return raw_path: Final = httpx.URL(endpoint).path encoded_endpoint: Final = raw_path if raw_path.startswith("/") else f"/{raw_path}" diff --git a/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py b/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py index 82422d35b9f1..b22e202d9e05 100644 --- a/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py +++ b/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py @@ -94,6 +94,32 @@ async def test_openai_websocket_accepts_first_client_subprotocol(): websocket.close.assert_not_awaited() +@pytest.mark.asyncio +async def test_openai_websocket_closes_cleanly_when_provider_credentials_missing(): + websocket = _mock_websocket("/openai/v1/realtime", "model=gpt-4o-realtime-preview") + + with ( + patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router.get_credentials", + return_value=None, + ), + patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.websocket_passthrough_request", + new_callable=AsyncMock, + ) as mock_ws, + ): + await openai_websocket_proxy_route( + websocket=websocket, + endpoint="v1/realtime", + user_api_key_dict=UserAPIKeyAuth(), + ) + + websocket.close.assert_awaited_once() + assert websocket.close.await_args.kwargs["code"] == 1011 + websocket.accept.assert_not_awaited() + mock_ws.assert_not_awaited() + + @pytest.mark.asyncio @pytest.mark.parametrize( "user_api_key_dict",