From 557bb9d3a1ffca35c0cea6df1e969f18b2ab36dc Mon Sep 17 00:00:00 2001 From: Halldrix <12357213+Halldrix@users.noreply.github.com> Date: Sat, 19 Sep 2026 15:11:25 -0500 Subject: [PATCH 1/2] fix(gateway): distinguish interrupted, unreachable and ambiguous model connections (#116323) --- gateway/run.py | 34 ++++++- .../test_local_model_connection_reply.py | 98 ++++++++++++++++++- 2 files changed, 125 insertions(+), 7 deletions(-) diff --git a/gateway/run.py b/gateway/run.py index f919aec51e8aa..462ff1bcd6026 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -388,12 +388,31 @@ def _gateway_surface_passes_raw_text(platform: Any) -> bool: # Connection-failure markers: the first 8 also anchor the provider-failure envelope shape below. _CONNECTION_ERROR_MARKERS = ( - r"(?:\w+\.)?(?:api\s*)?connection\s*(?:error|timeout)", r"(?:\w+\.)?connect\s*(?:error|timeout)", + r"(?:\w+\.)?connection\s*(?:error|timeout)", r"(?:\w+\.)?connect\s*(?:error|timeout)", r"connection\s+refused", r"connection\s+reset", r"connection\s+aborted", r"actively\s+refused", r"winerror\s+10061", r"errno\s+111", r"no\s+route\s+to\s+host", r"network\s+is\s+unreachable", r"cannot\s+connect", r"failed\s+to\s+establish", r"could\s+not\s+connect") _GATEWAY_CONNECTION_ERROR_RE = re.compile("(" + "|".join(_CONNECTION_ERROR_MARKERS) + ")", re.IGNORECASE) +# An ESTABLISHED connection died mid-transfer. Says nothing about whether the endpoint is up: +# an earlier call in the same turn may already have been answered by it (#26339). +_CONNECTION_INTERRUPTED_MARKERS = ( + r"connection\s+reset", r"connection\s+aborted", r"errno\s+104", r"broken\s+pipe", + r"server\s+disconnected", r"peer\s+closed\s+connection", r"connection\s+was\s+closed", + r"network\s+connection\s+lost", r"unexpected\s+eof", r"incomplete\s+chunked\s+read", + r"response\s+ended\s+prematurely", r"socket\s+hang\s+up", r"(?:\w+\.)?remoteprotocolerror", + r"(?:\w+\.)?readerror") +_GATEWAY_CONNECTION_INTERRUPTED_RE = re.compile( + "(" + "|".join(_CONNECTION_INTERRUPTED_MARKERS) + ")", re.IGNORECASE) + +# Nothing accepted the connection / no path to the host: "the endpoint is not up" IS the diagnosis. +_ENDPOINT_UNREACHABLE_MARKERS = ( + r"connection\s+refused", r"actively\s+refused", r"winerror\s+10061", r"errno\s+111", + r"no\s+route\s+to\s+host", r"network\s+is\s+unreachable", r"cannot\s+connect", + r"failed\s+to\s+establish", r"could\s+not\s+connect") +_GATEWAY_ENDPOINT_UNREACHABLE_RE = re.compile( + "(" + "|".join(_ENDPOINT_UNREACHABLE_MARKERS) + ")", re.IGNORECASE) + def _ensure_windows_gateway_venv_imports() -> None: """Make detached Windows gateway runs see the Hermes venv packages. @@ -595,9 +614,16 @@ def _format_exec_approval_fallback( "or ask whoever runs this bot to run `hermes doctor` on the host."), (_GATEWAY_PROVIDER_POLICY_RE, "⚠️ The AI model service rejected this request. Try rephrasing your " "message, or use /model to switch models."), - (_GATEWAY_CONNECTION_ERROR_RE, "⚠️ The AI model service isn't reachable right now — the configured model " - "endpoint is not running or is unreachable. Wait a moment and use /retry; " - "if it persists, run `hermes doctor` on the host.")) + (_GATEWAY_CONNECTION_INTERRUPTED_RE, "⚠️ The connection to the AI model service was interrupted. " + "Use /retry to try again; if it keeps happening, run " + "`hermes doctor` on the host."), + (_GATEWAY_ENDPOINT_UNREACHABLE_RE, "⚠️ The AI model service isn't reachable right now — the configured model " + "endpoint is not running or is unreachable. Wait a moment and use /retry; " + "if it persists, run `hermes doctor` on the host."), + (_GATEWAY_CONNECTION_ERROR_RE, "⚠️ The request to the AI model service couldn't be completed over the " + "network — either the connection was interrupted or the endpoint isn't " + "reachable. Use /retry to try again; if it keeps happening, run " + "`hermes doctor` on the host.")) # Shared by the failed-turn normalizer and ``run_turn._hmwa_agent_error_reply``; canonical diff --git a/tests/gateway/test_local_model_connection_reply.py b/tests/gateway/test_local_model_connection_reply.py index 5ff1bdad8c679..11d2e0676fc73 100644 --- a/tests/gateway/test_local_model_connection_reply.py +++ b/tests/gateway/test_local_model_connection_reply.py @@ -1,4 +1,4 @@ -"""Regression tests for #86570: gateway provider error connection messaging.""" +"""Regression tests for #86570 and #116323: gateway provider error connection messaging.""" import pytest @@ -6,13 +6,53 @@ _GATEWAY_CONNECTION_ERROR_RE, _gateway_provider_error_reply, _looks_like_gateway_provider_error, + _sanitize_gateway_final_response, ) +# An ESTABLISHED connection died mid-transfer. Whether the endpoint is up is unknowable from these +# (the same turn may already have been answered by it, cf. #26339). +INTERRUPTED_ENVELOPES = ( + "API call failed after 3 retries: httpx.ReadError: [Errno 104] Connection reset by peer", + "API call failed after 3 retries: ConnectionResetError: [Errno 104] Connection reset by peer", + "API call failed after 3 retries: httpx.RemoteProtocolError: peer closed connection " + "without sending complete message body", + "API call failed after 3 retries: httpx.ReadError: server disconnected without sending a response", +) + +# Nothing accepted the connection / no path to the host: "the endpoint is not up" IS the diagnosis +# here, and it is the case the #86570 wording was written for. +UNREACHABLE_ENVELOPES = ( + "API call failed after 3 retries: httpx.ConnectError: [Errno 111] Connection refused", + "API call failed after 3 retries: ConnectionError: [WinError 10061] No connection could " + "be made because the target machine actively refused it", + "API call failed after 3 retries: httpx.ConnectError: [Errno 113] No route to host", +) + +# Connection-shaped, but the SDK flattened the cause away; neither diagnosis is supported. +AMBIGUOUS_ENVELOPES = ( + "API call failed after 3 retries: openai.APIConnectionError: Connection error.", + "❌ API failed after 3 retries — openai.APIConnectionError: Connection error.", +) + +# Claims that the configured endpoint stopped/never came up. Asserting one of these about a +# mid-transfer reset sends the user to debug a server that is answering. +_ENDPOINT_DOWN_CLAIMS = ("not running", "not responding", "unreachable", "not started", "is down") + + +def _claims_the_endpoint_is_down(reply: str) -> bool: + return any(claim in reply.lower() for claim in _ENDPOINT_DOWN_CLAIMS) + + class TestGatewayConnectionErrorReply: def test_connection_error_strings_produce_specific_reply(self): + """A connect that was REFUSED/unroutable is the local-endpoint-down case of #86570. + + A bare ``openai.APIConnectionError`` is not: the SDK kept no cause, so it is equally a + dropped response from a live endpoint. It stays a recognised provider envelope, but the + endpoint-down diagnosis is no longer asserted for it (see the distinct-categories test). + """ samples = [ - "openai.APIConnectionError", "httpx.ConnectError: connection refused", "ConnectionError: [WinError 10061] No connection could be made", "Errno 111 Connection refused", @@ -33,6 +73,58 @@ def test_broad_connection_phrases_still_map_once_classified(self): reply = _gateway_provider_error_reply(text) assert "not running or is unreachable" in reply, text + def test_three_connection_causes_are_three_distinct_categories(self): + """A dropped response, a refused connect and a cause-free error are different failures. + + Contract, not wording: each cause gets ONE reply, the three replies differ, and only the + refused/unroutable one may claim the endpoint is down (that claim is pinned to the + refusal samples by ``test_connection_error_strings_produce_specific_reply`` above). + """ + interrupted = {_gateway_provider_error_reply(e) for e in INTERRUPTED_ENVELOPES} + unreachable = {_gateway_provider_error_reply(e) for e in UNREACHABLE_ENVELOPES} + ambiguous = {_gateway_provider_error_reply(e) for e in AMBIGUOUS_ENVELOPES} + + assert len(interrupted) == 1, interrupted + assert len(unreachable) == 1, unreachable + assert len(ambiguous) == 1, ambiguous + assert len(interrupted | unreachable | ambiguous) == 3 + + for reply in interrupted | ambiguous: + assert reply.strip() + assert not _claims_the_endpoint_is_down(reply), reply + + @pytest.mark.parametrize("platform", ["telegram", "slack", "feishu"]) + def test_interrupted_connection_delivery_keeps_precedence_and_redaction(self, platform): + """The new category rides the real chat path, and takes nothing from the other rows.""" + raw_reset = ( + "API call failed after 3 retries: httpx.ReadError: [Errno 104] Connection reset " + "by peer (Authorization: Bearer ***" + ) + + sanitized = _sanitize_gateway_final_response(platform, raw_reset) + + assert sanitized.strip() + assert "sk-ABCDEF" not in sanitized + assert "Errno 104" not in sanitized + assert not _claims_the_endpoint_is_down(sanitized), sanitized + assert sanitized != _gateway_provider_error_reply(UNREACHABLE_ENVELOPES[0]) + + # Auth beats policy beats rate-limit beats connection: an envelope carrying BOTH its own + # marker and connection wording keeps the category it had before the connection split. + for tainted, clean in ( + ("API call failed after 3 retries: HTTP 401 incorrect api key provided; " + "connection reset by peer on the retry", "provider authentication failed"), + ("API call failed after 3 retries: HTTP 400 request was blocked under the provider " + "safety policy; connection reset by peer", "request was blocked under the safety policy"), + ("API call failed after 3 retries: HTTP 429 rate limit exceeded for this model; " + "connection reset by peer", "rate limited after 3 retries"), + ): + assert _sanitize_gateway_final_response(platform, tainted) == ( + _gateway_provider_error_reply(clean)), tainted + + # Programmatic consumers still get the bottom exception, byte for byte. + assert _sanitize_gateway_final_response("local", raw_reset) == raw_reset + def test_prose_cannot_connect_is_not_a_provider_error(self): text = ( "cannot connect to the office VPN from this cafe, " @@ -119,4 +211,4 @@ def _resolve(**_kwargs): session_key="slack:C1", user_config={}, message="Hi") result = TurnRunner(runner, ctx).run_sync() reply = result["final_response"] - assert "/login" not in reply and "resets in ~33h" in reply and result["api_calls"] == 0 + assert "/login" not in reply and "resets in ~33h" in reply and result["api_calls"] == 0 \ No newline at end of file From 1586916857a240324e8e4a3ed0d5a59ab7c71ede Mon Sep 17 00:00:00 2001 From: Halldrix <12357213+Halldrix@users.noreply.github.com> Date: Sat, 19 Sep 2026 16:19:25 -0500 Subject: [PATCH 2/2] fix(gateway): anchor errno/winerror connection markers with word boundaries --- gateway/run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gateway/run.py b/gateway/run.py index 462ff1bcd6026..42594c9cb220e 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -390,14 +390,14 @@ def _gateway_surface_passes_raw_text(platform: Any) -> bool: _CONNECTION_ERROR_MARKERS = ( r"(?:\w+\.)?connection\s*(?:error|timeout)", r"(?:\w+\.)?connect\s*(?:error|timeout)", r"connection\s+refused", r"connection\s+reset", r"connection\s+aborted", r"actively\s+refused", - r"winerror\s+10061", r"errno\s+111", r"no\s+route\s+to\s+host", r"network\s+is\s+unreachable", + r"winerror\s+10061\b", r"errno\s+111\b", r"no\s+route\s+to\s+host", r"network\s+is\s+unreachable", r"cannot\s+connect", r"failed\s+to\s+establish", r"could\s+not\s+connect") _GATEWAY_CONNECTION_ERROR_RE = re.compile("(" + "|".join(_CONNECTION_ERROR_MARKERS) + ")", re.IGNORECASE) # An ESTABLISHED connection died mid-transfer. Says nothing about whether the endpoint is up: # an earlier call in the same turn may already have been answered by it (#26339). _CONNECTION_INTERRUPTED_MARKERS = ( - r"connection\s+reset", r"connection\s+aborted", r"errno\s+104", r"broken\s+pipe", + r"connection\s+reset", r"connection\s+aborted", r"errno\s+104\b", r"broken\s+pipe", r"server\s+disconnected", r"peer\s+closed\s+connection", r"connection\s+was\s+closed", r"network\s+connection\s+lost", r"unexpected\s+eof", r"incomplete\s+chunked\s+read", r"response\s+ended\s+prematurely", r"socket\s+hang\s+up", r"(?:\w+\.)?remoteprotocolerror", @@ -407,7 +407,7 @@ def _gateway_surface_passes_raw_text(platform: Any) -> bool: # Nothing accepted the connection / no path to the host: "the endpoint is not up" IS the diagnosis. _ENDPOINT_UNREACHABLE_MARKERS = ( - r"connection\s+refused", r"actively\s+refused", r"winerror\s+10061", r"errno\s+111", + r"connection\s+refused", r"actively\s+refused", r"winerror\s+10061\b", r"errno\s+111\b", r"no\s+route\s+to\s+host", r"network\s+is\s+unreachable", r"cannot\s+connect", r"failed\s+to\s+establish", r"could\s+not\s+connect") _GATEWAY_ENDPOINT_UNREACHABLE_RE = re.compile(