fix(gateway): distinguish interrupted, unreachable and ambiguous model connections (#116323)

gateway/run.py

@@ -388,12 +388,31 @@ def _gateway_surface_passes_raw_text(platform: Any) -> bool:
 
 # Connection-failure markers: the first 8 also anchor the provider-failure envelope shape below.
 _CONNECTION_ERROR_MARKERS = (
-    r"(?:\w+\.)?(?:api\s*)?connection\s*(?:error|timeout)", r"(?:\w+\.)?connect\s*(?:error|timeout)",
+    r"(?:\w+\.)?connection\s*(?:error|timeout)", r"(?:\w+\.)?connect\s*(?:error|timeout)",
     r"connection\s+refused", r"connection\s+reset", r"connection\s+aborted", r"actively\s+refused",
-    r"winerror\s+10061", r"errno\s+111", r"no\s+route\s+to\s+host", r"network\s+is\s+unreachable",
+    r"winerror\s+10061\b", r"errno\s+111\b", r"no\s+route\s+to\s+host", r"network\s+is\s+unreachable",
     r"cannot\s+connect", r"failed\s+to\s+establish", r"could\s+not\s+connect")
 _GATEWAY_CONNECTION_ERROR_RE = re.compile("(" + "|".join(_CONNECTION_ERROR_MARKERS) + ")", re.IGNORECASE)
 
+# An ESTABLISHED connection died mid-transfer. Says nothing about whether the endpoint is up:
+# an earlier call in the same turn may already have been answered by it (#26339).
+_CONNECTION_INTERRUPTED_MARKERS = (
+    r"connection\s+reset", r"connection\s+aborted", r"errno\s+104\b", r"broken\s+pipe",
+    r"server\s+disconnected", r"peer\s+closed\s+connection", r"connection\s+was\s+closed",
+    r"network\s+connection\s+lost", r"unexpected\s+eof", r"incomplete\s+chunked\s+read",
+    r"response\s+ended\s+prematurely", r"socket\s+hang\s+up", r"(?:\w+\.)?remoteprotocolerror",
+    r"(?:\w+\.)?readerror")
+_GATEWAY_CONNECTION_INTERRUPTED_RE = re.compile(
+    "(" + "|".join(_CONNECTION_INTERRUPTED_MARKERS) + ")", re.IGNORECASE)
+
+# Nothing accepted the connection / no path to the host: "the endpoint is not up" IS the diagnosis.
+_ENDPOINT_UNREACHABLE_MARKERS = (
+    r"connection\s+refused", r"actively\s+refused", r"winerror\s+10061\b", r"errno\s+111\b",
+    r"no\s+route\s+to\s+host", r"network\s+is\s+unreachable", r"cannot\s+connect",
+    r"failed\s+to\s+establish", r"could\s+not\s+connect")
+_GATEWAY_ENDPOINT_UNREACHABLE_RE = re.compile(
+    "(" + "|".join(_ENDPOINT_UNREACHABLE_MARKERS) + ")", re.IGNORECASE)
+
 def _ensure_windows_gateway_venv_imports() -> None:
     """Make detached Windows gateway runs see the Hermes venv packages.
 
@@ -595,9 +614,16 @@ def _format_exec_approval_fallback(
                              "or ask whoever runs this bot to run `hermes doctor` on the host."),
     (_GATEWAY_PROVIDER_POLICY_RE, "⚠️ The AI model service rejected this request. Try rephrasing your "
                                   "message, or use /model to switch models."),
-    (_GATEWAY_CONNECTION_ERROR_RE, "⚠️ The AI model service isn't reachable right now — the configured model "
-                                   "endpoint is not running or is unreachable. Wait a moment and use /retry; "
-                                   "if it persists, run `hermes doctor` on the host."))
+    (_GATEWAY_CONNECTION_INTERRUPTED_RE, "⚠️ The connection to the AI model service was interrupted. "
+                                         "Use /retry to try again; if it keeps happening, run "
+                                         "`hermes doctor` on the host."),
+    (_GATEWAY_ENDPOINT_UNREACHABLE_RE, "⚠️ The AI model service isn't reachable right now — the configured model "
+                                       "endpoint is not running or is unreachable. Wait a moment and use /retry; "
+                                       "if it persists, run `hermes doctor` on the host."),
+    (_GATEWAY_CONNECTION_ERROR_RE, "⚠️ The request to the AI model service couldn't be completed over the "
+                                   "network — either the connection was interrupted or the endpoint isn't "
+                                   "reachable. Use /retry to try again; if it keeps happening, run "
+                                   "`hermes doctor` on the host."))
 
 
 # Shared by the failed-turn normalizer and ``run_turn._hmwa_agent_error_reply``; canonical

tests/gateway/test_local_model_connection_reply.py

@@ -1,18 +1,58 @@
-"""Regression tests for #86570: gateway provider error connection messaging."""
+"""Regression tests for #86570 and #116323: gateway provider error connection messaging."""
 
 import pytest
 
 from gateway.run import (
     _GATEWAY_CONNECTION_ERROR_RE,
     _gateway_provider_error_reply,
     _looks_like_gateway_provider_error,
+    _sanitize_gateway_final_response,
 )
 
 
+# An ESTABLISHED connection died mid-transfer. Whether the endpoint is up is unknowable from these
+# (the same turn may already have been answered by it, cf. #26339).
+INTERRUPTED_ENVELOPES = (
+    "API call failed after 3 retries: httpx.ReadError: [Errno 104] Connection reset by peer",
+    "API call failed after 3 retries: ConnectionResetError: [Errno 104] Connection reset by peer",
+    "API call failed after 3 retries: httpx.RemoteProtocolError: peer closed connection "
+    "without sending complete message body",
+    "API call failed after 3 retries: httpx.ReadError: server disconnected without sending a response",
+)
+
+# Nothing accepted the connection / no path to the host: "the endpoint is not up" IS the diagnosis
+# here, and it is the case the #86570 wording was written for.
+UNREACHABLE_ENVELOPES = (
+    "API call failed after 3 retries: httpx.ConnectError: [Errno 111] Connection refused",
+    "API call failed after 3 retries: ConnectionError: [WinError 10061] No connection could "
+    "be made because the target machine actively refused it",
+    "API call failed after 3 retries: httpx.ConnectError: [Errno 113] No route to host",
+)
+
+# Connection-shaped, but the SDK flattened the cause away; neither diagnosis is supported.
+AMBIGUOUS_ENVELOPES = (
+    "API call failed after 3 retries: openai.APIConnectionError: Connection error.",
+    "❌ API failed after 3 retries — openai.APIConnectionError: Connection error.",
+)
+
+# Claims that the configured endpoint stopped/never came up. Asserting one of these about a
+# mid-transfer reset sends the user to debug a server that is answering.
+_ENDPOINT_DOWN_CLAIMS = ("not running", "not responding", "unreachable", "not started", "is down")
+
+
+def _claims_the_endpoint_is_down(reply: str) -> bool:
+    return any(claim in reply.lower() for claim in _ENDPOINT_DOWN_CLAIMS)
+
+
 class TestGatewayConnectionErrorReply:
     def test_connection_error_strings_produce_specific_reply(self):
+        """A connect that was REFUSED/unroutable is the local-endpoint-down case of #86570.
+
+        A bare ``openai.APIConnectionError`` is not: the SDK kept no cause, so it is equally a
+        dropped response from a live endpoint. It stays a recognised provider envelope, but the
+        endpoint-down diagnosis is no longer asserted for it (see the distinct-categories test).
+        """
         samples = [
-            "openai.APIConnectionError",
             "httpx.ConnectError: connection refused",
             "ConnectionError: [WinError 10061] No connection could be made",
             "Errno 111 Connection refused",
@@ -33,6 +73,58 @@ def test_broad_connection_phrases_still_map_once_classified(self):
             reply = _gateway_provider_error_reply(text)
             assert "not running or is unreachable" in reply, text
 
+    def test_three_connection_causes_are_three_distinct_categories(self):
+        """A dropped response, a refused connect and a cause-free error are different failures.
+
+        Contract, not wording: each cause gets ONE reply, the three replies differ, and only the
+        refused/unroutable one may claim the endpoint is down (that claim is pinned to the
+        refusal samples by ``test_connection_error_strings_produce_specific_reply`` above).
+        """
+        interrupted = {_gateway_provider_error_reply(e) for e in INTERRUPTED_ENVELOPES}
+        unreachable = {_gateway_provider_error_reply(e) for e in UNREACHABLE_ENVELOPES}
+        ambiguous = {_gateway_provider_error_reply(e) for e in AMBIGUOUS_ENVELOPES}
+
+        assert len(interrupted) == 1, interrupted
+        assert len(unreachable) == 1, unreachable
+        assert len(ambiguous) == 1, ambiguous
+        assert len(interrupted | unreachable | ambiguous) == 3
+
+        for reply in interrupted | ambiguous:
+            assert reply.strip()
+            assert not _claims_the_endpoint_is_down(reply), reply
+
+    @pytest.mark.parametrize("platform", ["telegram", "slack", "feishu"])
+    def test_interrupted_connection_delivery_keeps_precedence_and_redaction(self, platform):
+        """The new category rides the real chat path, and takes nothing from the other rows."""
+        raw_reset = (
+            "API call failed after 3 retries: httpx.ReadError: [Errno 104] Connection reset "
+            "by peer (Authorization: Bearer ***"
+        )
+
+        sanitized = _sanitize_gateway_final_response(platform, raw_reset)
+
+        assert sanitized.strip()
+        assert "sk-ABCDEF" not in sanitized
+        assert "Errno 104" not in sanitized
+        assert not _claims_the_endpoint_is_down(sanitized), sanitized
+        assert sanitized != _gateway_provider_error_reply(UNREACHABLE_ENVELOPES[0])
+
+        # Auth beats policy beats rate-limit beats connection: an envelope carrying BOTH its own
+        # marker and connection wording keeps the category it had before the connection split.
+        for tainted, clean in (
+            ("API call failed after 3 retries: HTTP 401 incorrect api key provided; "
+             "connection reset by peer on the retry", "provider authentication failed"),
+            ("API call failed after 3 retries: HTTP 400 request was blocked under the provider "
+             "safety policy; connection reset by peer", "request was blocked under the safety policy"),
+            ("API call failed after 3 retries: HTTP 429 rate limit exceeded for this model; "
+             "connection reset by peer", "rate limited after 3 retries"),
+        ):
+            assert _sanitize_gateway_final_response(platform, tainted) == (
+                _gateway_provider_error_reply(clean)), tainted
+
+        # Programmatic consumers still get the bottom exception, byte for byte.
+        assert _sanitize_gateway_final_response("local", raw_reset) == raw_reset
+
     def test_prose_cannot_connect_is_not_a_provider_error(self):
         text = (
             "cannot connect to the office VPN from this cafe, "
@@ -119,4 +211,4 @@ def _resolve(**_kwargs):
                           session_key="slack:C1", user_config={}, message="Hi")
         result = TurnRunner(runner, ctx).run_sync()
         reply = result["final_response"]
-        assert "/login" not in reply and "resets in ~33h" in reply and result["api_calls"] == 0
+        assert "/login" not in reply and "resets in ~33h" in reply and result["api_calls"] == 0