Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions tests/e2e/coverage_registry/llm_conversational.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@
- {id: llm.responses.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Streaming via /v1/responses"}
- {id: llm.responses.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "response_api_endpoints/endpoints.py:26", rationale: "Cost logged on responses"}
- {id: llm.responses.openai.passthrough.stream.cost_logged, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [cost_logged], source: "test_passthrough_e2e.py", rationale: "A streamed POST /openai_passthrough/v1/responses is costed and keyed by the provider response id; it used to log a zero-cost row under a random id (GitHub issue #36523)"}
- {id: llm.responses.openai.passthrough_websocket.stream.works, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [works], fail_before_fix: proven, source: "test_passthrough_e2e.py", rationale: "A websocket upgrade on /openai/v1/responses is accepted, so a responses.connect client reaches OpenAI through the same prefix its HTTP traffic uses; the prefix carried no websocket route and refused the upgrade with a 403 (GitHub issue #36088)"}
- {id: llm.responses.openai.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Tool calls via Responses API"}
- {id: llm.responses.openai.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Vision via Responses API"}
- {id: llm.responses.anthropic.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Responses w/ Anthropic translation (smoke)"}
Expand Down
1 change: 1 addition & 0 deletions tests/e2e/coverage_registry/llm_nonconversational.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
- {id: llm.google_native.gemini.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: google_native, route: gemini, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "LIT-4076 / proxy/google_endpoints/endpoints.py", fail_before_fix: proven, rationale: "google-native generateContent must stamp x-litellm-response-cost so SDK traffic reconciles against spend"}
- {id: llm.google_native.gemini.basic.stream.works, module: llm, tier: P0, subject_endpoint: google_native, route: gemini, capability: basic, streaming: stream, assertions: [works], source: "PR #28213 / proxy/proxy_server.py async_data_generator", fail_before_fix: proven, rationale: "streamGenerateContent must relay single-prefixed SSE frames with no [DONE] sentinel; doubled data: prefixes and the OpenAI terminator both break the Vertex Java SDK"}
- {id: llm.realtime.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: realtime, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "vendor strategy §9.19 / LIT-4778", rationale: "HTTP /v1/realtime/client_secrets returns an ephemeral credential"}
- {id: llm.realtime.openai.passthrough.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: openai, capability: basic, streaming: stream, assertions: [works], fail_before_fix: proven, source: "test_passthrough_e2e.py", rationale: "A websocket upgrade on /openai_passthrough/v1/realtime is accepted and relayed to OpenAI; only HTTP routes were registered under the prefix, so realtime clients were refused with a 403 before a socket existed (GitHub issue #36088)"}
- {id: llm.vector_stores.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: vector_stores, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "vendor strategy §9.17 / LIT-4778", rationale: "Vector store create/list/retrieve/delete lifecycle"}
- {id: llm.vector_stores.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: vector_stores, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.17 / LIT-4778", rationale: "Vector store search and invalid id errors"}
- {id: llm.bedrock_native.bedrock_converse.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock native converse happy path"}
Expand Down
9 changes: 9 additions & 0 deletions tests/e2e/e2e_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,15 @@
)


def ws_base_url() -> str:
"""PROXY_BASE_URL with its scheme swapped for the websocket one, so a suite
opening a socket points at the same proxy every HTTP suite uses."""
for scheme, ws_scheme in (("https://", "wss://"), ("http://", "ws://")):
if PROXY_BASE_URL.startswith(scheme):
return ws_scheme + PROXY_BASE_URL[len(scheme) :]
return PROXY_BASE_URL


def datadog_mcp_url(*, toolsets: str = "core") -> str:
"""Regional Datadog remote MCP endpoint for this process's DD_SITE.

Expand Down
58 changes: 58 additions & 0 deletions tests/e2e/llm_translation/passthrough_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,13 @@
from __future__ import annotations

from dataclasses import dataclass
from urllib.parse import urlencode

from pydantic import BaseModel, Field
from websockets.exceptions import InvalidStatus
from websockets.sync.client import connect

from e2e_config import ws_base_url
from proxy_client import ProxyClient
from e2e_http import FileUploadForm, Headers, NoBody, Result, StreamingResponse
from models import ChatMessage
Expand Down Expand Up @@ -175,6 +179,26 @@ class OpenAIEmbeddingBody(BaseModel):
input: str


class WebsocketEnvelope(BaseModel):
"""The one field every provider event carries, so the first frame off a
passthrough socket identifies itself without the suite parsing raw dicts."""

type: str


class WebsocketHandshake(BaseModel):
"""What the proxy did with a websocket upgrade on a passthrough prefix.

`rejected_status` is the HTTP status of a refused upgrade: a prefix carrying no
websocket route answers 403, before any socket exists. `first_event_type` is the
type of the first frame an accepted socket delivered, which is None when the
provider waits for the client to speak first.
"""

rejected_status: int | None = None
first_event_type: str | None = None


class PassthroughBatchList(BaseModel):
"""OpenAI's own batch page, relayed verbatim. `object` is required so a body
that is not an OpenAI list fails validation instead of passing vacuously."""
Expand Down Expand Up @@ -339,5 +363,39 @@ def openai_chat(
),
)

# ---- OpenAI websocket passthrough ----------------------------------
#
# The same prefixes over an upgrade instead of a POST, for the provider APIs
# that only speak websocket (realtime, responses.connect).

def openai_passthrough_websocket(
self,
key: str,
path: str,
*,
model: str | None = None,
open_timeout: float = 30.0,
first_event_timeout: float = 30.0,
) -> WebsocketHandshake:
query = f"?{urlencode({'model': model})}" if model is not None else ""
try:
connection = connect(
f"{ws_base_url()}{path}{query}",
additional_headers={"Authorization": f"Bearer {key}"},
open_timeout=open_timeout,
)
except InvalidStatus as rejected:
return WebsocketHandshake(rejected_status=rejected.response.status_code)
with connection:
try:
frame = connection.recv(timeout=first_event_timeout)
except TimeoutError:
return WebsocketHandshake()
text = frame.decode("utf-8") if isinstance(frame, bytes) else frame
return WebsocketHandshake(
first_event_type=WebsocketEnvelope.model_validate_json(text).type
)


def build_client(proxy: ProxyClient) -> PassthroughClient:
return PassthroughClient(proxy=proxy)
9 changes: 1 addition & 8 deletions tests/e2e/llm_translation/realtime/realtime_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,20 +21,13 @@
from websockets.sync.client import connect
from websockets.sync.connection import Connection

from e2e_config import PROXY_BASE_URL, unique_marker
from e2e_config import unique_marker, ws_base_url
from proxy_client import ProxyClient
from models import LiteLLMParamsBody

_M = TypeVar("_M", bound=BaseModel)


def ws_base_url() -> str:
for scheme, ws_scheme in (("https://", "wss://"), ("http://", "ws://")):
if PROXY_BASE_URL.startswith(scheme):
return ws_scheme + PROXY_BASE_URL[len(scheme) :]
return PROXY_BASE_URL


def realtime_ws_url(model: str) -> str:
return f"{ws_base_url()}/v1/realtime?{urlencode({'model': model})}"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@

import pytest

from e2e_config import ws_base_url
from realtime_client import (
PROVIDERS,
RealtimeProvider,
ws_base_url,
realtime_model,
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@

import pytest

from e2e_config import ws_base_url
from realtime_client import (
PROVIDERS,
RealtimeProvider,
ws_base_url,
realtime_model,
)

Expand Down
52 changes: 52 additions & 0 deletions tests/e2e/llm_translation/test_passthrough_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@
)

EMBEDDING_MODEL = "text-embedding-3-small"
# Relayed to OpenAI untranslated, so this is OpenAI's own realtime model name
# rather than a gateway deployment alias.
REALTIME_MODEL = "gpt-realtime-2"

pytestmark = pytest.mark.e2e

Expand Down Expand Up @@ -339,3 +342,52 @@ def test_embeddings_call_logs_its_cost(
f"the embeddings row logged no prompt tokens, so whatever cost it carries "
f"was not computed from the real usage: {row}"
)


class TestOpenAIPassthroughWebsocket:
"""The OpenAI passthrough prefixes must answer a websocket upgrade, not only a POST.

The customer points realtime and responses.connect clients at the same prefixes
their HTTP traffic already uses. Only HTTP routes were registered under those
prefixes, so every upgrade was refused before a socket existed and those clients
could not reach the gateway at all. A refused upgrade is an HTTP response, not a
close frame, which is why these assert on the handshake rather than a close code.
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
"""

@pytest.mark.covers("llm.realtime.openai.passthrough.stream.works")
def test_realtime_upgrade_reaches_openai_through_the_passthrough_prefix(
self, client: PassthroughClient, scoped_key: str
) -> None:
"""Pins GitHub issue #36088: /openai_passthrough/v1/realtime accepts the
upgrade and relays OpenAI's own session, instead of rejecting it with a 403."""
handshake = client.openai_passthrough_websocket(
scoped_key, "/openai_passthrough/v1/realtime", model=REALTIME_MODEL
)

assert handshake.rejected_status is None, (
f"/openai_passthrough/v1/realtime refused the websocket upgrade with HTTP "
f"{handshake.rejected_status}, so a realtime client cannot connect through "
"the gateway at all"
)
assert handshake.first_event_type == "session.created", (
"the accepted socket never carried OpenAI's opening session event, so the "
f"upgrade was not relayed upstream; the first frame was "
f"{handshake.first_event_type}"
)

@pytest.mark.covers("llm.responses.openai.passthrough_websocket.stream.works")
def test_responses_upgrade_is_accepted_on_the_openai_prefix(
self, client: PassthroughClient, scoped_key: str
) -> None:
"""Pins GitHub issue #36088 on the second prefix: /openai/v1/responses upgrades
as well. A responses.connect socket waits for the client to speak first, so the
accepted handshake is the whole signal here."""
handshake = client.openai_passthrough_websocket(
scoped_key, "/openai/v1/responses", first_event_timeout=2.0
)

assert handshake.rejected_status is None, (
f"/openai/v1/responses refused the websocket upgrade with HTTP "
f"{handshake.rejected_status}; the prefix relays this route over HTTP but "
"drops a responses.connect client before the socket opens"
)
Loading