fix(gateway): distinguish interrupted and unreachable model connections

gateway/run.py

@@ -392,6 +392,25 @@ def _gateway_surface_passes_raw_text(platform: Any) -> bool:
     r"cannot\s+connect", r"failed\s+to\s+establish", r"could\s+not\s+connect")
 _GATEWAY_CONNECTION_ERROR_RE = re.compile("(" + "|".join(_CONNECTION_ERROR_MARKERS) + ")", re.IGNORECASE)
 
+# An ESTABLISHED connection died mid-transfer. Says nothing about whether the endpoint is up:
+# an earlier call in the same turn may already have been answered by it (#26339).
+_CONNECTION_INTERRUPTED_MARKERS = (
+    r"connection\s+reset", r"connection\s+aborted", r"errno\s+104", r"errno\s+103",
+    r"broken\s+pipe", r"server\s+disconnected", r"peer\s+closed\s+connection",
+    r"connection\s+was\s+closed", r"network\s+connection\s+lost", r"unexpected\s+eof",
+    r"incomplete\s+chunked\s+read", r"response\s+ended\s+prematurely", r"socket\s+hang\s+up",
+    r"(?:\w+\.)?remoteprotocolerror", r"(?:\w+\.)?readerror")
+_GATEWAY_CONNECTION_INTERRUPTED_RE = re.compile(
+    "(" + "|".join(_CONNECTION_INTERRUPTED_MARKERS) + ")", re.IGNORECASE)
+
+# Nothing accepted the connection / no path to the host: "the endpoint is not up" IS the diagnosis.
+_ENDPOINT_UNREACHABLE_MARKERS = (
+    r"connection\s+refused", r"actively\s+refused", r"winerror\s+10061", r"errno\s+111",
+    r"no\s+route\s+to\s+host", r"network\s+is\s+unreachable", r"cannot\s+connect",
+    r"failed\s+to\s+establish", r"could\s+not\s+connect", r"(?:\w+\.)?connect\s*(?:error|timeout)")
+_GATEWAY_ENDPOINT_UNREACHABLE_RE = re.compile(
+    "(" + "|".join(_ENDPOINT_UNREACHABLE_MARKERS) + ")", re.IGNORECASE)
+
 _GATEWAY_SECRET_PATTERNS = (
     re.compile(r"\bsk-[A-Za-z0-9][A-Za-z0-9_\-]{12,}\b"),
     re.compile(r"\bgh[pousr]_[A-Za-z0-9_]{20,}\b"), re.compile(r"\bxapp-\d+-[A-Za-z0-9\-]{20,}\b"),
@@ -601,14 +620,30 @@ def _format_exec_approval_fallback(
         + ", ".join(choices[:-1]) + f", or {choices[-1]}.")
 
 # Ordered: auth beats policy beats rate-limit beats connection; first match wins.
+#
+# The three connection rows are NOT interchangeable, and collapsing them onto the unreachable
+# wording tells a user whose endpoint answered an earlier call in the same turn to go restart it:
+#   * interrupted  — an established connection died mid-transfer. Whether the endpoint is up is
+#                    unknown from this alone, so we must not guess.
+#   * unreachable  — nothing accepted the connection at all; "not running / unreachable" is the
+#                    actual diagnosis, and this is the case that wording was written for (#86570).
+#   * ambiguous    — connection-shaped but the cause was flattened away (an SDK-wrapped
+#                    ``APIConnectionError: Connection error.`` keeps neither). Name both
+#                    possibilities; assert neither.
 _PROVIDER_ERROR_REPLIES = (
     (_GATEWAY_AUTH_ERROR_RE, "⚠️ Provider authentication failed. Check the configured credentials; "
                              "raw provider details are in the gateway logs."),
     (_GATEWAY_PROVIDER_POLICY_RE, "⚠️ The model provider rejected the request. I kept the raw provider "
                                   "error out of chat; check gateway logs for details or try rephrasing."),
     (_GATEWAY_RATE_LIMIT_RE, "⏱️ The model provider is rate-limiting requests. Please wait a moment and try again."),
-    (_GATEWAY_CONNECTION_ERROR_RE, "⚠️ The model server is not responding — it looks like the configured "
-                                   "model endpoint is not running or is unreachable."))
+    (_GATEWAY_CONNECTION_INTERRUPTED_RE, "⚠️ The connection to the model provider was interrupted. Please try "
+                                         "again; the transport details are in the gateway logs."),
+    (_GATEWAY_ENDPOINT_UNREACHABLE_RE, "⚠️ The model server is not responding — it looks like the configured "
+                                       "model endpoint is not running or is unreachable."),
+    (_GATEWAY_CONNECTION_ERROR_RE, "⚠️ The model request could not be completed over the network — the "
+                                   "connection was either never established or was interrupted. Please try "
+                                   "again; if it keeps happening, check that the configured model endpoint "
+                                   "is reachable. Details are in the gateway logs."))
 
 
 def _gateway_provider_error_reply(text: str) -> str:

tests/gateway/test_local_model_connection_reply.py

@@ -2,17 +2,57 @@
 
 import pytest
 
+from gateway.config import Platform
 from gateway.run import (
     _GATEWAY_CONNECTION_ERROR_RE,
     _gateway_provider_error_reply,
     _looks_like_gateway_provider_error,
+    _sanitize_gateway_final_response,
 )
 
+# Terminal-path envelopes for an ESTABLISHED connection that died mid-transfer. Whether the
+# endpoint is up is unknowable from these (the same turn may already have been answered by it).
+INTERRUPTED_ENVELOPES = (
+    "API call failed after 3 retries: httpx.ReadError: [Errno 104] Connection reset by peer",
+    "API call failed after 3 retries: ConnectionResetError: [Errno 104] Connection reset by peer",
+    "API call failed after 3 retries: httpx.RemoteProtocolError: peer closed connection "
+    "without sending complete message body",
+    "API call failed after 3 retries: httpx.ReadError: server disconnected without sending a response",
+)
+
+# Nothing accepted the connection / no path to the host: "the endpoint is not up" IS the diagnosis
+# here, and it is the case the #86570 wording was written for.
+UNREACHABLE_ENVELOPES = (
+    "API call failed after 3 retries: httpx.ConnectError: [Errno 111] Connection refused",
+    "API call failed after 3 retries: ConnectionError: [WinError 10061] No connection could "
+    "be made because the target machine actively refused it",
+    "API call failed after 3 retries: httpx.ConnectError: [Errno 113] No route to host",
+)
+
+# Connection-shaped, but the SDK flattened the cause away; neither diagnosis is supported.
+AMBIGUOUS_ENVELOPES = (
+    "API call failed after 3 retries: openai.APIConnectionError: Connection error.",
+    "❌ API failed after 3 retries — openai.APIConnectionError: Connection error.",
+)
+
+# Claims that the configured endpoint stopped/never came up. Asserting one of these about a
+# mid-transfer reset sends the user to debug a server that is answering.
+_ENDPOINT_DOWN_CLAIMS = ("not running", "not responding", "unreachable", "not started", "is down")
+
+
+def _claims_the_endpoint_is_down(reply: str) -> bool:
+    return any(claim in reply.lower() for claim in _ENDPOINT_DOWN_CLAIMS)
+
 
 class TestGatewayConnectionErrorReply:
     def test_connection_error_strings_produce_specific_reply(self):
+        """A connect that was REFUSED/unroutable is the local-endpoint-down case of #86570.
+
+        A bare ``openai.APIConnectionError`` is not: the SDK kept no cause, so it is equally a
+        dropped response from a live endpoint. It stays a recognised provider envelope, but the
+        endpoint-down diagnosis is no longer asserted for it (see the distinct-categories test).
+        """
         samples = [
-            "openai.APIConnectionError",
             "httpx.ConnectError: connection refused",
             "ConnectionError: [WinError 10061] No connection could be made",
             "Errno 111 Connection refused",
@@ -24,6 +64,8 @@ def test_connection_error_strings_produce_specific_reply(self):
             assert "not responding" in reply.lower(), text
             assert "not running or is unreachable" in reply, text
 
+        assert _looks_like_gateway_provider_error("openai.APIConnectionError")
+
     def test_broad_connection_phrases_still_map_once_classified(self):
         """Reply selector keeps the full phrase set; the gate does not."""
         for text in (
@@ -61,3 +103,55 @@ def test_auth_and_rate_limit_preserved(self):
         assert "rate-limiting" in _gateway_provider_error_reply(
             "rate limited after 3 retries"
         ).lower()
+
+    def test_three_connection_causes_are_three_distinct_categories(self):
+        """A dropped response, a refused connect and a cause-free error are different failures.
+
+        Contract, not wording: each cause gets ONE reply, the three replies differ, and only the
+        refused/unroutable one may claim the endpoint is down (that claim is pinned to the
+        refusal samples by ``test_connection_error_strings_produce_specific_reply`` above).
+        """
+        interrupted = {_gateway_provider_error_reply(e) for e in INTERRUPTED_ENVELOPES}
+        unreachable = {_gateway_provider_error_reply(e) for e in UNREACHABLE_ENVELOPES}
+        ambiguous = {_gateway_provider_error_reply(e) for e in AMBIGUOUS_ENVELOPES}
+
+        assert len(interrupted) == 1, interrupted
+        assert len(unreachable) == 1, unreachable
+        assert len(ambiguous) == 1, ambiguous
+        assert len(interrupted | unreachable | ambiguous) == 3
+
+        for reply in interrupted | ambiguous:
+            assert reply.strip()
+            assert not _claims_the_endpoint_is_down(reply), reply
+
+    @pytest.mark.parametrize("platform", [Platform.TELEGRAM, "slack", "feishu"])
+    def test_interrupted_connection_delivery_keeps_precedence_and_redaction(self, platform):
+        """The new category rides the real chat path, and takes nothing from the other rows."""
+        raw_reset = (
+            "API call failed after 3 retries: httpx.ReadError: [Errno 104] Connection reset "
+            "by peer (Authorization: Bearer sk-ABCDEF0123456789abcdef0123)"
+        )
+
+        sanitized = _sanitize_gateway_final_response(platform, raw_reset)
+
+        assert sanitized.strip()
+        assert "sk-ABCDEF" not in sanitized
+        assert "Errno 104" not in sanitized
+        assert not _claims_the_endpoint_is_down(sanitized), sanitized
+        assert sanitized != _gateway_provider_error_reply(UNREACHABLE_ENVELOPES[0])
+
+        # Auth beats policy beats rate-limit beats connection: an envelope carrying BOTH its own
+        # marker and connection wording keeps the category it had before the connection split.
+        for tainted, clean in (
+            ("API call failed after 3 retries: HTTP 401 incorrect api key provided; "
+             "connection reset by peer on the retry", "provider authentication failed"),
+            ("API call failed after 3 retries: HTTP 400 request was blocked under the provider "
+             "safety policy; connection reset by peer", "request was blocked under the safety policy"),
+            ("API call failed after 3 retries: HTTP 429 rate limit exceeded for this model; "
+             "connection reset by peer", "rate limited after 3 retries"),
+        ):
+            assert _sanitize_gateway_final_response(platform, tainted) == (
+                _gateway_provider_error_reply(clean)), tainted
+
+        # Programmatic consumers still get the bottom exception, byte for byte.
+        assert _sanitize_gateway_final_response("local", raw_reset) == raw_reset