diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 7dcb01d3e597..6b0b4db1b18c 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -3,7 +3,7 @@ import time from collections.abc import AsyncIterator, Mapping from types import MappingProxyType -from typing import Any, cast +from typing import TYPE_CHECKING, Any, NamedTuple, cast, get_args from uuid import uuid4 import fastapi @@ -19,9 +19,12 @@ user_api_key_auth_websocket, ) from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing -from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse +from litellm.types.llms.openai import REASONING_EFFORT, ResponseAPIUsage, ResponsesAPIResponse from litellm.types.responses.main import DeleteResponseResult +if TYPE_CHECKING: + from litellm.router import Router + router = APIRouter() _user_api_key_auth_dep = Depends(user_api_key_auth) @@ -93,6 +96,58 @@ def _is_chat_completions_body(data: Mapping[str, Any]) -> bool: return "messages" in data and "input" not in data +_CURSOR_THINKING_SEPARATOR = "-thinking-" +_CURSOR_FAST_SUFFIX = "-fast" +_CURSOR_THINKING_LEVELS: frozenset[str] = frozenset(get_args(REASONING_EFFORT)) + + +class _CursorModelVariant(NamedTuple): + base_model: str + reasoning_effort: str | None + + +def _parse_cursor_model_variant(model: str) -> _CursorModelVariant: + stripped = model.removesuffix(_CURSOR_FAST_SUFFIX) + base, separator, level = stripped.rpartition(_CURSOR_THINKING_SEPARATOR) + if separator and base and level in _CURSOR_THINKING_LEVELS: + return _CursorModelVariant(base, level) + return _CursorModelVariant(stripped, None) + + +def _router_can_serve(model: str, llm_router: "Router | None") -> bool: + if llm_router is None: + return False + if model in llm_router.model_names or model in llm_router.model_group_alias: + return True + if model in llm_router.team_public_model_names: + return True + return bool(llm_router.pattern_router.get_pattern(model)) + + +def _resolve_cursor_model_variant( + data: dict, llm_router: "Router | None" +) -> dict: # mutable-ok: the parsed request body contract is a plain dict + model = data.get("model") + if not isinstance(model, str) or _router_can_serve(model, llm_router): + return data + variant = _parse_cursor_model_variant(model) + if variant.base_model == model or not _router_can_serve(variant.base_model, llm_router): + return data + resolved = {**data, "model": variant.base_model} # mutable-ok: plain body dict + if variant.reasoning_effort is None: + return resolved + if _is_chat_completions_body(data): + if "reasoning_effort" in data: + return resolved + return {**resolved, "reasoning_effort": variant.reasoning_effort} # mutable-ok: plain body dict + reasoning = data.get("reasoning") + if isinstance(reasoning, dict): + if reasoning.get("effort"): + return resolved + return {**resolved, "reasoning": {**reasoning, "effort": variant.reasoning_effort}} # mutable-ok: same + return {**resolved, "reasoning": {"effort": variant.reasoning_effort}} # mutable-ok: plain body dict + + @router.post( "/v1/responses", dependencies=[Depends(user_api_key_auth)], @@ -440,7 +495,8 @@ async def cursor_chat_completions( from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ModelResponse - data = await _read_request_body(request=request) + raw_body = await _read_request_body(request=request) + data = _resolve_cursor_model_variant(raw_body, llm_router) if _is_chat_completions_body(data): # Genuine chat completions body (Cursor sends these for models whose BYOK it @@ -448,7 +504,7 @@ async def cursor_chat_completions( # Keyed on messages CONTENT, not key presence: Cursor can send a null or # empty messages stub alongside a real agent-mode input array normalized = _normalize_tool_dialect(data, to_chat=True) - if normalized is not data: + if normalized is not raw_body: _safe_set_request_parsed_body(request=request, parsed_body=normalized) return await chat_completion( request=request, diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index 00ac8ca386a9..60168e7f9121 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -1340,3 +1340,231 @@ async def test_null_messages_stub_with_input_reaches_responses_arm(self): assert response.status_code == 200 assert mock_router.aresponses.call_args is not None assert mock_router.aresponses.call_args.kwargs["input"] == [{"role": "user", "content": "hello"}] + + +class TestParseCursorModelVariant: + @pytest.mark.parametrize( + "model,expected_base,expected_effort", + [ + ("claude-opus-5-thinking-high", "claude-opus-5", "high"), + ("claude-opus-5-thinking-xhigh-fast", "claude-opus-5", "xhigh"), + ("gemini-3.0-pro-thinking-low", "gemini-3.0-pro", "low"), + ("claude-opus-5-fast", "claude-opus-5", None), + ("gpt-5.6-sol", "gpt-5.6-sol", None), + ("foo-thinking-ultra-fast", "foo-thinking-ultra", None), + ("-thinking-high", "-thinking-high", None), + ], + ) + def test_parse_matrix(self, model, expected_base, expected_effort): + from litellm.proxy.response_api_endpoints.endpoints import _parse_cursor_model_variant + + variant = _parse_cursor_model_variant(model) + assert variant.base_model == expected_base + assert variant.reasoning_effort == expected_effort + + +class TestResolveCursorModelVariant: + @pytest.fixture(scope="class") + def wildcard_router(self): + from litellm import Router + + return Router( + model_list=[ + {"model_name": "anthropic/*", "litellm_params": {"model": "anthropic/*", "api_key": "fake"}}, + {"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": "fake"}}, + { + "model_name": "explicit-alias-thinking-high", + "litellm_params": {"model": "anthropic/claude-opus-5", "api_key": "fake"}, + }, + ] + ) + + def test_chat_body_suffix_stripped_into_reasoning_effort(self, wildcard_router): + from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant + + body = { + "model": "claude-opus-5-thinking-xhigh-fast", + "messages": [{"role": "user", "content": "hi"}], + } + resolved = _resolve_cursor_model_variant(body, wildcard_router) + assert resolved["model"] == "claude-opus-5" + assert resolved["reasoning_effort"] == "xhigh" + assert resolved["messages"] == body["messages"] + assert body["model"] == "claude-opus-5-thinking-xhigh-fast" + + def test_responses_body_suffix_stripped_into_reasoning_dict(self, wildcard_router): + from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant + + body = {"model": "claude-opus-5-thinking-high", "input": [{"role": "user", "content": "hi"}]} + resolved = _resolve_cursor_model_variant(body, wildcard_router) + assert resolved["model"] == "claude-opus-5" + assert resolved["reasoning"] == {"effort": "high"} + + def test_responses_body_merges_effort_into_existing_reasoning(self, wildcard_router): + from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant + + body = { + "model": "claude-opus-5-thinking-high", + "input": [{"role": "user", "content": "hi"}], + "reasoning": {"summary": "auto"}, + } + resolved = _resolve_cursor_model_variant(body, wildcard_router) + assert resolved["model"] == "claude-opus-5" + assert resolved["reasoning"] == {"summary": "auto", "effort": "high"} + + def test_existing_reasoning_effort_wins_but_model_still_rewritten(self, wildcard_router): + from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant + + chat_body = { + "model": "claude-opus-5-thinking-high", + "messages": [{"role": "user", "content": "hi"}], + "reasoning_effort": "low", + } + resolved_chat = _resolve_cursor_model_variant(chat_body, wildcard_router) + assert resolved_chat["model"] == "claude-opus-5" + assert resolved_chat["reasoning_effort"] == "low" + + responses_body = { + "model": "claude-opus-5-thinking-high", + "input": [{"role": "user", "content": "hi"}], + "reasoning": {"effort": "low"}, + } + resolved_responses = _resolve_cursor_model_variant(responses_body, wildcard_router) + assert resolved_responses["model"] == "claude-opus-5" + assert resolved_responses["reasoning"] == {"effort": "low"} + + def test_fast_only_suffix_strips_without_reasoning(self, wildcard_router): + from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant + + body = {"model": "claude-opus-5-fast", "messages": [{"role": "user", "content": "hi"}]} + resolved = _resolve_cursor_model_variant(body, wildcard_router) + assert resolved["model"] == "claude-opus-5" + assert "reasoning_effort" not in resolved + + def test_explicitly_configured_suffixed_name_untouched(self, wildcard_router): + from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant + + body = {"model": "explicit-alias-thinking-high", "messages": [{"role": "user", "content": "hi"}]} + assert _resolve_cursor_model_variant(body, wildcard_router) is body + + def test_provider_inferable_bare_name_untouched(self, wildcard_router): + from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant + + body = {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]} + assert _resolve_cursor_model_variant(body, wildcard_router) is body + + def test_unservable_base_untouched(self, wildcard_router): + from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant + + body = {"model": "totally-unknown-thinking-high", "messages": [{"role": "user", "content": "hi"}]} + assert _resolve_cursor_model_variant(body, wildcard_router) is body + + def test_no_router_untouched(self): + from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant + + body = {"model": "claude-opus-5-thinking-high", "messages": [{"role": "user", "content": "hi"}]} + assert _resolve_cursor_model_variant(body, None) is body + + def test_missing_or_non_string_model_untouched(self, wildcard_router): + from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant + + no_model = {"messages": [{"role": "user", "content": "hi"}]} + assert _resolve_cursor_model_variant(no_model, wildcard_router) is no_model + null_model = {"model": None, "messages": [{"role": "user", "content": "hi"}]} + assert _resolve_cursor_model_variant(null_model, wildcard_router) is null_model + + +def _router_serving_only(base_model: str) -> MagicMock: + mock_router = MagicMock() + mock_router.model_names = set() + mock_router.model_group_alias = {} + mock_router.team_public_model_names = frozenset() + mock_router.pattern_router.get_pattern.side_effect = ( + lambda model: [{"model_name": "anthropic/*"}] if model == base_model else None + ) + return mock_router + + +class TestCursorModelSuffixResolutionEndToEnd: + @pytest.mark.asyncio + async def test_chat_arm_rewrites_suffixed_model_before_delegation(self): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + seen = {} + + async def fake_chat_completion(request, fastapi_response, model, user_api_key_dict): + from litellm.proxy.common_utils.http_parsing_utils import _read_request_body + + seen["body"] = await _read_request_body(request=request) + return {"id": "chatcmpl-fake", "object": "chat.completion", "choices": []} + + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-1234") + try: + with ( + patch("litellm.proxy.proxy_server.llm_router", new=_router_serving_only("claude-opus-5")), + patch("litellm.proxy.proxy_server.chat_completion", new=fake_chat_completion), + ): + client = TestClient(app) + response = client.post( + "/cursor/chat/completions", + json={ + "model": "claude-opus-5-thinking-xhigh-fast", + "messages": [{"role": "user", "content": "hi"}], + }, + headers={"Authorization": "Bearer sk-1234"}, + ) + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 200 + assert seen["body"]["model"] == "claude-opus-5" + assert seen["body"]["reasoning_effort"] == "xhigh" + assert seen["body"]["messages"] == [{"role": "user", "content": "hi"}] + + @pytest.mark.asyncio + async def test_responses_arm_rewrites_suffixed_model_before_routing(self): + from openai.types.responses import ResponseOutputMessage, ResponseOutputText + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.types.llms.openai import ResponsesAPIResponse + + mock_response = ResponsesAPIResponse( + id="resp_suffix1", + created_at=1234567890, + model="claude-opus-5", + object="response", + output=[ + ResponseOutputMessage( + id="msg_suffix1", + type="message", + role="assistant", + status="completed", + content=[ResponseOutputText(type="output_text", text="ok", annotations=[])], + ) + ], + ) + + mock_router = _router_serving_only("claude-opus-5") + mock_router.aresponses = AsyncMock(return_value=mock_response) + + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-1234") + try: + with patch("litellm.proxy.proxy_server.llm_router", new=mock_router): + client = TestClient(app) + response = client.post( + "/cursor/chat/completions", + json={ + "model": "claude-opus-5-thinking-high", + "input": [{"role": "user", "content": "hello"}], + }, + headers={"Authorization": "Bearer sk-1234"}, + ) + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 200 + assert mock_router.aresponses.call_args is not None + assert mock_router.aresponses.call_args.kwargs["model"] == "claude-opus-5" + assert mock_router.aresponses.call_args.kwargs["reasoning"] == {"effort": "high"}