From 65ce090a9b6cc20692b3e28152716b08240f8f6d Mon Sep 17 00:00:00 2001 From: "li.wang" Date: Thu, 14 May 2026 16:12:42 +0800 Subject: [PATCH 1/2] fix(api-server): honor chat completions request model --- gateway/platforms/api_server.py | 21 +++++++++++-- tests/gateway/test_api_server.py | 51 ++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py index 8b53db3a99f3..8fc6281c1d6c 100644 --- a/gateway/platforms/api_server.py +++ b/gateway/platforms/api_server.py @@ -799,6 +799,7 @@ def _create_agent( self, ephemeral_system_prompt: Optional[str] = None, session_id: Optional[str] = None, + requested_model: Optional[str] = None, stream_delta_callback=None, tool_progress_callback=None, tool_start_callback=None, @@ -826,7 +827,8 @@ def _create_agent( runtime_kwargs = _resolve_runtime_agent_kwargs() reasoning_config = GatewayRunner._load_reasoning_config() - model = _resolve_gateway_model() + runtime_model = runtime_kwargs.pop("model", None) + model = requested_model or runtime_model or _resolve_gateway_model() user_config = _load_gateway_config() enabled_toolsets = sorted(_get_platform_tools(user_config, "api_server")) @@ -858,6 +860,17 @@ def _create_agent( ) return agent + def _resolve_request_model(self, body: Dict[str, Any]): + raw_model = body.get("model") + if not isinstance(raw_model, str): + return self._model_name, None + model = raw_model.strip() + if not model: + return self._model_name, None + if model == self._model_name: + return model, None + return model, model + # ------------------------------------------------------------------ # HTTP Handlers # ------------------------------------------------------------------ @@ -1082,7 +1095,7 @@ async def _handle_chat_completions(self, request: "web.Request") -> "web.Respons # history already set from request body above completion_id = f"chatcmpl-{uuid.uuid4().hex[:29]}" - model_name = body.get("model", self._model_name) + model_name, requested_model = self._resolve_request_model(body) created = int(time.time()) if stream: @@ -1167,6 +1180,7 @@ def _on_tool_complete(tool_call_id, function_name, function_args, function_resul tool_complete_callback=_on_tool_complete, agent_ref=agent_ref, gateway_session_key=gateway_session_key, + requested_model=requested_model, )) # Ensure SSE drain loops can terminate without relying on polling # agent_task.done(), which can race with queue timeout checks. @@ -1186,6 +1200,7 @@ async def _compute_completion(): ephemeral_system_prompt=system_prompt, session_id=session_id, gateway_session_key=gateway_session_key, + requested_model=requested_model, ) idempotency_key = request.headers.get("Idempotency-Key") @@ -2690,6 +2705,7 @@ async def _run_agent( tool_complete_callback=None, agent_ref: Optional[list] = None, gateway_session_key: Optional[str] = None, + requested_model: Optional[str] = None, ) -> tuple: """ Create an agent and run a conversation in a thread executor. @@ -2713,6 +2729,7 @@ def _run(): tool_start_callback=tool_start_callback, tool_complete_callback=tool_complete_callback, gateway_session_key=gateway_session_key, + requested_model=requested_model, ) if agent_ref is not None: agent_ref[0] = agent diff --git a/tests/gateway/test_api_server.py b/tests/gateway/test_api_server.py index 66b304fff516..5adb9629c9f7 100644 --- a/tests/gateway/test_api_server.py +++ b/tests/gateway/test_api_server.py @@ -282,6 +282,36 @@ def __init__(self, **kwargs): assert isinstance(agent, FakeAgent) assert captured["reasoning_config"] == {"enabled": True, "effort": "xhigh"} + def test_create_agent_uses_requested_model(self, monkeypatch): + captured = {} + + class FakeAgent: + def __init__(self, **kwargs): + captured.update(kwargs) + + monkeypatch.setattr("run_agent.AIAgent", FakeAgent) + monkeypatch.setattr( + "gateway.run._resolve_runtime_agent_kwargs", + lambda: { + "provider": "openai", + "base_url": "https://example.test/v1", + "api_mode": "chat_completions", + }, + ) + monkeypatch.setattr("gateway.run._resolve_gateway_model", lambda: "configured-model") + monkeypatch.setattr("gateway.run._load_gateway_config", lambda: {}) + monkeypatch.setattr("gateway.run.GatewayRunner._load_reasoning_config", staticmethod(lambda: None)) + monkeypatch.setattr("gateway.run.GatewayRunner._load_fallback_model", staticmethod(lambda: None)) + monkeypatch.setattr("hermes_cli.tools_config._get_platform_tools", lambda *_: set()) + + adapter = APIServerAdapter(PlatformConfig(enabled=True)) + monkeypatch.setattr(adapter, "_ensure_session_db", lambda: None) + + agent = adapter._create_agent(requested_model="request-model") + + assert isinstance(agent, FakeAgent) + assert captured["model"] == "request-model" + # --------------------------------------------------------------------------- # Auth checking @@ -1092,6 +1122,27 @@ async def test_successful_completion(self, adapter): assert data["choices"][0]["finish_reason"] == "stop" assert "usage" in data + @pytest.mark.asyncio + async def test_chat_completion_model_is_passed_to_agent(self, adapter): + mock_result = {"final_response": "ok", "messages": [], "api_calls": 1} + + app = _create_app(adapter) + async with TestClient(TestServer(app)) as cli: + with patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run: + mock_run.return_value = (mock_result, {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}) + resp = await cli.post( + "/v1/chat/completions", + json={ + "model": "gpt-requested", + "messages": [{"role": "user", "content": "Hello"}], + }, + ) + + assert resp.status == 200 + data = await resp.json() + assert data["model"] == "gpt-requested" + assert mock_run.call_args.kwargs["requested_model"] == "gpt-requested" + @pytest.mark.asyncio async def test_system_prompt_extracted(self, adapter): """System messages from the client are passed as ephemeral_system_prompt.""" From 29b61e71a76df27899552ab50a827aa3d30e7358 Mon Sep 17 00:00:00 2001 From: "li.wang" Date: Fri, 15 May 2026 10:47:32 +0800 Subject: [PATCH 2/2] fix(api-server): surface upstream model id in chat completion response MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a routing gateway (e.g. an alias like "high-accuracy" that fans out to "high-accuracy-m1/m2/m3") is configured as the inference backend, clients had no way to tell which concrete model actually served a given request — the response.model field always echoed the alias the client sent. Capture response.model from every successful upstream API call on AIAgent (_last_response_model) and surface it through run_conversation() so the api_server adapter can put the concrete backend id into the chat.completion response. Falls back to the client-supplied name when the upstream omits a model field, preserving the existing behavior for non-routing providers. Adds a regression test that verifies the alias is still used for provider selection (requested_model kwarg) while the response advertises the upstream-reported concrete backend. --- gateway/platforms/api_server.py | 9 ++++++++- run_agent.py | 19 +++++++++++++++++++ tests/gateway/test_api_server.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py index 8fc6281c1d6c..84259658d1c5 100644 --- a/gateway/platforms/api_server.py +++ b/gateway/platforms/api_server.py @@ -1267,11 +1267,18 @@ async def _compute_completion(): # Soft-partial path: we have *some* text but the run did not complete # (e.g. truncation with partial buffered output). Still 200 but signal # truncation via finish_reason="length" + Hermes-specific extras. + # + # Prefer the upstream-reported model id when present so clients see + # the concrete backend a routing gateway actually served (e.g. an + # alias "high-accuracy" → "high-accuracy-m3"). Fall back to the + # client-supplied name when the upstream did not advertise one. + upstream_model = result.get("last_response_model") + response_model = upstream_model if isinstance(upstream_model, str) and upstream_model else model_name response_data = { "id": completion_id, "object": "chat.completion", "created": created, - "model": model_name, + "model": response_model, "choices": [ { "index": 0, diff --git a/run_agent.py b/run_agent.py index d995c607de67..eec631bcc19d 100644 --- a/run_agent.py +++ b/run_agent.py @@ -1396,6 +1396,14 @@ def __init__( self._interrupt_message = None # Optional message that triggered interrupt self._execution_thread_id: int | None = None # Set at run_conversation() start self._interrupt_thread_signal_pending = False + + # Tracks the upstream-reported model id from the last API response. + # Routing gateways (e.g. an alias like "high-accuracy" that fans out + # to "high-accuracy-m1/m2/m3") populate ``response.model`` with the + # concrete backend they actually served. We surface this through + # ``run_conversation()`` so callers can report it to the end user + # instead of echoing the alias the client originally sent. + self._last_response_model: Optional[str] = None self._client_lock = threading.RLock() # /steer mechanism — inject a user note into the next tool result @@ -14495,6 +14503,16 @@ def _stop_spinner(): self._persist_session(messages, conversation_history) break + # Capture the upstream-reported model id (e.g. routing gateways + # that turn "high-accuracy" into "high-accuracy-m3"). We stamp + # this on every successful response so the last assignment + # reflects the final turn; falsy values are ignored so a + # provider that omits the field doesn't blank out an earlier + # known value. + _resp_model = getattr(response, "model", None) + if isinstance(_resp_model, str) and _resp_model: + self._last_response_model = _resp_model + try: _transport = self._get_transport() _normalize_kwargs = {} @@ -15576,6 +15594,7 @@ def _stop_spinner(): "interrupted": interrupted, "response_previewed": getattr(self, "_response_was_previewed", False), "model": self.model, + "last_response_model": self._last_response_model, "provider": self.provider, "base_url": self.base_url, "input_tokens": self.session_input_tokens, diff --git a/tests/gateway/test_api_server.py b/tests/gateway/test_api_server.py index 5adb9629c9f7..1bc897057a3e 100644 --- a/tests/gateway/test_api_server.py +++ b/tests/gateway/test_api_server.py @@ -1143,6 +1143,38 @@ async def test_chat_completion_model_is_passed_to_agent(self, adapter): assert data["model"] == "gpt-requested" assert mock_run.call_args.kwargs["requested_model"] == "gpt-requested" + @pytest.mark.asyncio + async def test_chat_completion_reports_upstream_model(self, adapter): + """When the upstream surfaces a concrete model (routing-gateway alias + like ``high-accuracy`` → ``high-accuracy-m3``), the response should + advertise the concrete backend instead of echoing the alias the + client originally sent. + """ + mock_result = { + "final_response": "ok", + "messages": [], + "api_calls": 1, + "last_response_model": "high-accuracy-m3", + } + + app = _create_app(adapter) + async with TestClient(TestServer(app)) as cli: + with patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run: + mock_run.return_value = (mock_result, {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}) + resp = await cli.post( + "/v1/chat/completions", + json={ + "model": "high-accuracy", + "messages": [{"role": "user", "content": "Hello"}], + }, + ) + + assert resp.status == 200 + data = await resp.json() + assert data["model"] == "high-accuracy-m3" + # The alias still drives provider selection / agent kwargs. + assert mock_run.call_args.kwargs["requested_model"] == "high-accuracy" + @pytest.mark.asyncio async def test_system_prompt_extracted(self, adapter): """System messages from the client are passed as ephemeral_system_prompt."""