Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion python/sglang/srt/entrypoints/anthropic/serving.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
import uuid
from typing import TYPE_CHECKING, Any, AsyncGenerator, Optional, Union

from fastapi import Request
from fastapi import HTTPException, Request
from fastapi.responses import JSONResponse, StreamingResponse
from pydantic import BaseModel, ValidationError

Expand Down Expand Up @@ -757,6 +757,15 @@ async def _handle_non_streaming(
)
except asyncio.CancelledError:
raise
except HTTPException as e:
logger.warning(
"Non-streaming request rejected: %s %s", e.status_code, e.detail
)
return self._error_response(
status_code=e.status_code,
error_type=ERROR_TYPE_MAP.get(e.status_code, "api_error"),
message=_scrub_error_message(str(e.detail), e.status_code),
)
except Exception as e:
logger.exception("Error processing Anthropic request: %s", e)
return self._error_response(
Expand Down
116 changes: 96 additions & 20 deletions test/registered/unit/entrypoints/anthropic/test_serving.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
import asyncio
import json
import unittest
from http import HTTPStatus
from types import SimpleNamespace

from sglang.test.test_utils import maybe_stub_sgl_kernel

maybe_stub_sgl_kernel() # must precede imports that may pull in sgl_kernel

from fastapi import HTTPException # noqa: E402
from fastapi.responses import JSONResponse # noqa: E402
from jinja2 import Environment # noqa: E402

Expand Down Expand Up @@ -56,14 +58,15 @@ def wrap_reasoning_history(self, text):


class _FakeNonStreamingErrorOpenAI:
"""Returns a configurable error response from the OpenAI handler."""
"""Returns an error response or raises an exception from the OpenAI handler."""

supports_inline_system = False

def __init__(self, status_code=400, body=None, content=None):
def __init__(self, status_code=400, body=None, content=None, exception=None):
self._status_code = status_code
self._body = body
self._content = content
self._exception = exception

def _validate_request(self, chat_request):
return None
Expand All @@ -74,6 +77,8 @@ def _convert_to_internal_request(self, chat_request, raw_request):
async def _handle_non_streaming_request(
self, adapted_request, processed_request, raw_request
):
if self._exception is not None:
raise self._exception
if self._body is not None:
# Build a response object exposing raw bytes via `.body`.
return SimpleNamespace(status_code=self._status_code, body=self._body)
Expand All @@ -89,24 +94,6 @@ async def _handle_non_streaming_request(
)


class _FakeNonStreamingOpenAI:
"""Returns a configurable ChatCompletionResponse from the OpenAI handler."""

def __init__(self, response):
self._response = response

def _validate_request(self, chat_request):
return None

def _convert_to_internal_request(self, chat_request, raw_request):
return SimpleNamespace(), chat_request

async def _handle_non_streaming_request(
self, adapted_request, processed_request, raw_request
):
return self._response


def _chunk(choices=None, usage=None):
data = {
"id": "chatcmpl-test",
Expand Down Expand Up @@ -179,6 +166,12 @@ def _anthropic_request(self, **overrides):
data.update(overrides)
return AnthropicMessagesRequest.model_validate(data)

def _non_streaming_exception_response(self, exception):
serving = AnthropicServing(_FakeNonStreamingErrorOpenAI(exception=exception))
return asyncio.run(
serving.handle_messages(self._anthropic_request(stream=False), object())
)

def _tool_result_request(self, content, tools=None):
overrides = {
"stream": False,
Expand Down Expand Up @@ -551,6 +544,89 @@ def test_non_streaming_openai_error_response_is_forwarded(self):
self.assertEqual(payload["error"]["type"], "invalid_request_error")
self.assertEqual(payload["error"]["message"], "context length exceeded")

def test_non_streaming_queue_full_http_exception_preserves_503(self):
for status_code in (503, HTTPStatus.SERVICE_UNAVAILABLE):
with self.subTest(status_code=status_code, code_type=type(status_code)):
response = self._non_streaming_exception_response(
HTTPException(
status_code=status_code,
detail="The request queue is full: sensitive internals /opt/secret",
)
)
self.assertEqual(response.status_code, 503)
self.assertEqual(
json.loads(response.body),
{
"type": "error",
"error": {
"type": "overloaded_error",
"message": "Internal server error",
},
},
)

def test_non_streaming_http_exception_4xx_preserves_status_and_message(self):
for status_code, expected_type in [
(400, "invalid_request_error"),
(418, "api_error"),
]:
with self.subTest(status_code=status_code):
response = self._non_streaming_exception_response(
HTTPException(status_code=status_code, detail="Request rejected")
)
self.assertEqual(response.status_code, status_code)
self.assertEqual(
json.loads(response.body)["error"],
{"type": expected_type, "message": "Request rejected"},
)

def test_non_streaming_http_exception_non_str_detail_is_stringified(self):
response = self._non_streaming_exception_response(
HTTPException(status_code=400, detail={"message": "Bad request"})
)
self.assertEqual(response.status_code, 400)
self.assertEqual(
json.loads(response.body)["error"],
{
"type": "invalid_request_error",
"message": "{'message': 'Bad request'}",
},
)

def test_non_streaming_http_exception_logs_original_detail(self):
detail = "worker rejected request: sensitive internals /opt/secret"
with self.assertLogs(
"sglang.srt.entrypoints.anthropic.serving", level="WARNING"
) as captured:
response = self._non_streaming_exception_response(
HTTPException(status_code=503, detail=detail)
)
self.assertEqual(len(captured.records), 1)
message = captured.records[0].getMessage()
self.assertIn(detail, message)
self.assertIn("503", message)
self.assertEqual(response.status_code, 503)
self.assertEqual(
json.loads(response.body)["error"],
{"type": "overloaded_error", "message": "Internal server error"},
)

def test_non_streaming_unexpected_exception_returns_generic_500(self):
response = self._non_streaming_exception_response(
RuntimeError("sensitive internals: /opt/secret")
)
self.assertEqual(response.status_code, 500)
self.assertEqual(
json.loads(response.body)["error"],
{"type": "api_error", "message": "Internal server error"},
)

def test_non_streaming_cancelled_error_propagates(self):
exception = asyncio.CancelledError()
with self.assertRaises(asyncio.CancelledError) as captured:
self._non_streaming_exception_response(exception)
self.assertIs(captured.exception, exception)

# ------------------------------------------------------------------
# Edge-case coverage added in the review-fix pass
# ------------------------------------------------------------------
Expand Down
Loading