From 46e681817a1a27ad639239140c56a6297e356fa2 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Sun, 13 Sep 2026 06:36:38 +0000 Subject: [PATCH 1/2] fix(gateway): distinguish interrupted connections from an unreachable model endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _gateway_provider_error_reply() answered every connection-shaped provider failure with "the configured model endpoint is not running or is unreachable". The marker tuple behind that single row mixes three different failures, and only one of them supports the sentence: * interrupted — an ESTABLISHED connection died mid-transfer (ReadError / ECONNRESET / RemoteProtocolError). Something accepted and answered, so whether the endpoint is up is unknown from this alone; an earlier call in the same turn may have succeeded against it. * unreachable — nothing accepted the connection (ECONNREFUSED, WinError 10061, no route). Here "not running or unreachable" IS the diagnosis, and it is the case the wording was written for (#86570, merged in #86729). Unchanged. * ambiguous — connection-shaped with the cause flattened away by the SDK (openai.APIConnectionError: Connection error.). Neither diagnosis is supported. Split the one connection row into three ordered rows so the first and third stop asserting that the model server stopped. Neither new reply claims a retry count, which this layer cannot know. _CONNECTION_ERROR_MARKERS is deliberately untouched: its first 8 entries are sliced into _GATEWAY_PROVIDER_ERROR_SHAPE_RE, so editing it would change WHICH texts are treated as provider envelopes rather than what they are called. Auth/policy/rate-limit precedence, secret redaction and raw passthrough for programmatic surfaces are unchanged. Refs #26339 (a real errno-104 reset against a live endpoint; its payload-side cause is not fixed here, only the way it is described). Co-authored-by: Lei-k <11388531+Lei-k@users.noreply.github.com> --- gateway/run.py | 40 +++++++- .../test_local_model_connection_reply.py | 96 ++++++++++++++++++- 2 files changed, 133 insertions(+), 3 deletions(-) diff --git a/gateway/run.py b/gateway/run.py index b615dc9a8027..3a8c6e2482c4 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -392,6 +392,25 @@ def _gateway_surface_passes_raw_text(platform: Any) -> bool: r"cannot\s+connect", r"failed\s+to\s+establish", r"could\s+not\s+connect") _GATEWAY_CONNECTION_ERROR_RE = re.compile("(" + "|".join(_CONNECTION_ERROR_MARKERS) + ")", re.IGNORECASE) +# An ESTABLISHED connection died mid-transfer. Says nothing about whether the endpoint is up: +# an earlier call in the same turn may already have been answered by it (#26339). +_CONNECTION_INTERRUPTED_MARKERS = ( + r"connection\s+reset", r"connection\s+aborted", r"errno\s+104", r"errno\s+103", + r"broken\s+pipe", r"server\s+disconnected", r"peer\s+closed\s+connection", + r"connection\s+was\s+closed", r"network\s+connection\s+lost", r"unexpected\s+eof", + r"incomplete\s+chunked\s+read", r"response\s+ended\s+prematurely", r"socket\s+hang\s+up", + r"(?:\w+\.)?remoteprotocolerror", r"(?:\w+\.)?readerror") +_GATEWAY_CONNECTION_INTERRUPTED_RE = re.compile( + "(" + "|".join(_CONNECTION_INTERRUPTED_MARKERS) + ")", re.IGNORECASE) + +# Nothing accepted the connection / no path to the host: "the endpoint is not up" IS the diagnosis. +_ENDPOINT_UNREACHABLE_MARKERS = ( + r"connection\s+refused", r"actively\s+refused", r"winerror\s+10061", r"errno\s+111", + r"no\s+route\s+to\s+host", r"network\s+is\s+unreachable", r"cannot\s+connect", + r"failed\s+to\s+establish", r"could\s+not\s+connect", r"(?:\w+\.)?connect\s*(?:error|timeout)") +_GATEWAY_ENDPOINT_UNREACHABLE_RE = re.compile( + "(" + "|".join(_ENDPOINT_UNREACHABLE_MARKERS) + ")", re.IGNORECASE) + _GATEWAY_SECRET_PATTERNS = ( re.compile(r"\bsk-[A-Za-z0-9][A-Za-z0-9_\-]{12,}\b"), re.compile(r"\bgh[pousr]_[A-Za-z0-9_]{20,}\b"), re.compile(r"\bxapp-\d+-[A-Za-z0-9\-]{20,}\b"), @@ -601,14 +620,31 @@ def _format_exec_approval_fallback( + ", ".join(choices[:-1]) + f", or {choices[-1]}.") # Ordered: auth beats policy beats rate-limit beats connection; first match wins. +# +# The three connection rows are NOT interchangeable, and collapsing them onto the unreachable +# wording tells a user whose endpoint answered an earlier call in the same turn to go restart it: +# * interrupted — an established connection died mid-transfer. Whether the endpoint is up is +# unknown from this alone, so we must not guess. +# * unreachable — nothing accepted the connection at all; "not running / unreachable" is the +# actual diagnosis, and this is the case that wording was written for (#86570). +# * ambiguous — connection-shaped but the cause was flattened away (an SDK-wrapped +# ``APIConnectionError: Connection error.`` keeps neither). Name both +# possibilities; assert neither. _PROVIDER_ERROR_REPLIES = ( (_GATEWAY_AUTH_ERROR_RE, "⚠️ Provider authentication failed. Check the configured credentials; " "raw provider details are in the gateway logs."), (_GATEWAY_PROVIDER_POLICY_RE, "⚠️ The model provider rejected the request. I kept the raw provider " "error out of chat; check gateway logs for details or try rephrasing."), (_GATEWAY_RATE_LIMIT_RE, "⏱️ The model provider is rate-limiting requests. Please wait a moment and try again."), - (_GATEWAY_CONNECTION_ERROR_RE, "⚠️ The model server is not responding — it looks like the configured " - "model endpoint is not running or is unreachable.")) + (_GATEWAY_CONNECTION_INTERRUPTED_RE, "⚠️ The connection to the model provider was interrupted before the " + "reply arrived, and the retries hit the same problem. Please try " + "again; the transport details are in the gateway logs."), + (_GATEWAY_ENDPOINT_UNREACHABLE_RE, "⚠️ The model server is not responding — it looks like the configured " + "model endpoint is not running or is unreachable."), + (_GATEWAY_CONNECTION_ERROR_RE, "⚠️ The model request could not be completed over the network after " + "retries — the connection was either never established or dropped before " + "the reply arrived. Please try again; if it keeps happening, check that the " + "configured model endpoint is reachable. Details are in the gateway logs.")) def _gateway_provider_error_reply(text: str) -> str: diff --git a/tests/gateway/test_local_model_connection_reply.py b/tests/gateway/test_local_model_connection_reply.py index ae52e318a71c..cda875f74645 100644 --- a/tests/gateway/test_local_model_connection_reply.py +++ b/tests/gateway/test_local_model_connection_reply.py @@ -2,17 +2,57 @@ import pytest +from gateway.config import Platform from gateway.run import ( _GATEWAY_CONNECTION_ERROR_RE, _gateway_provider_error_reply, _looks_like_gateway_provider_error, + _sanitize_gateway_final_response, ) +# Terminal-path envelopes for an ESTABLISHED connection that died mid-transfer. Whether the +# endpoint is up is unknowable from these (the same turn may already have been answered by it). +INTERRUPTED_ENVELOPES = ( + "API call failed after 3 retries: httpx.ReadError: [Errno 104] Connection reset by peer", + "API call failed after 3 retries: ConnectionResetError: [Errno 104] Connection reset by peer", + "API call failed after 3 retries: httpx.RemoteProtocolError: peer closed connection " + "without sending complete message body", + "API call failed after 3 retries: httpx.ReadError: server disconnected without sending a response", +) + +# Nothing accepted the connection / no path to the host: "the endpoint is not up" IS the diagnosis +# here, and it is the case the #86570 wording was written for. +UNREACHABLE_ENVELOPES = ( + "API call failed after 3 retries: httpx.ConnectError: [Errno 111] Connection refused", + "API call failed after 3 retries: ConnectionError: [WinError 10061] No connection could " + "be made because the target machine actively refused it", + "API call failed after 3 retries: httpx.ConnectError: [Errno 113] No route to host", +) + +# Connection-shaped, but the SDK flattened the cause away; neither diagnosis is supported. +AMBIGUOUS_ENVELOPES = ( + "API call failed after 3 retries: openai.APIConnectionError: Connection error.", + "❌ API failed after 3 retries — openai.APIConnectionError: Connection error.", +) + +# Claims that the configured endpoint stopped/never came up. Asserting one of these about a +# mid-transfer reset sends the user to debug a server that is answering. +_ENDPOINT_DOWN_CLAIMS = ("not running", "not responding", "unreachable", "not started", "is down") + + +def _claims_the_endpoint_is_down(reply: str) -> bool: + return any(claim in reply.lower() for claim in _ENDPOINT_DOWN_CLAIMS) + class TestGatewayConnectionErrorReply: def test_connection_error_strings_produce_specific_reply(self): + """A connect that was REFUSED/unroutable is the local-endpoint-down case of #86570. + + A bare ``openai.APIConnectionError`` is not: the SDK kept no cause, so it is equally a + dropped response from a live endpoint. It stays a recognised provider envelope, but the + endpoint-down diagnosis is no longer asserted for it (see the distinct-categories test). + """ samples = [ - "openai.APIConnectionError", "httpx.ConnectError: connection refused", "ConnectionError: [WinError 10061] No connection could be made", "Errno 111 Connection refused", @@ -24,6 +64,8 @@ def test_connection_error_strings_produce_specific_reply(self): assert "not responding" in reply.lower(), text assert "not running or is unreachable" in reply, text + assert _looks_like_gateway_provider_error("openai.APIConnectionError") + def test_broad_connection_phrases_still_map_once_classified(self): """Reply selector keeps the full phrase set; the gate does not.""" for text in ( @@ -61,3 +103,55 @@ def test_auth_and_rate_limit_preserved(self): assert "rate-limiting" in _gateway_provider_error_reply( "rate limited after 3 retries" ).lower() + + def test_three_connection_causes_are_three_distinct_categories(self): + """A dropped response, a refused connect and a cause-free error are different failures. + + Contract, not wording: each cause gets ONE reply, the three replies differ, and only the + refused/unroutable one may claim the endpoint is down (that claim is pinned to the + refusal samples by ``test_connection_error_strings_produce_specific_reply`` above). + """ + interrupted = {_gateway_provider_error_reply(e) for e in INTERRUPTED_ENVELOPES} + unreachable = {_gateway_provider_error_reply(e) for e in UNREACHABLE_ENVELOPES} + ambiguous = {_gateway_provider_error_reply(e) for e in AMBIGUOUS_ENVELOPES} + + assert len(interrupted) == 1, interrupted + assert len(unreachable) == 1, unreachable + assert len(ambiguous) == 1, ambiguous + assert len(interrupted | unreachable | ambiguous) == 3 + + for reply in interrupted | ambiguous: + assert reply.strip() + assert not _claims_the_endpoint_is_down(reply), reply + + @pytest.mark.parametrize("platform", [Platform.TELEGRAM, "slack", "feishu"]) + def test_interrupted_connection_delivery_keeps_precedence_and_redaction(self, platform): + """The new category rides the real chat path, and takes nothing from the other rows.""" + raw_reset = ( + "API call failed after 3 retries: httpx.ReadError: [Errno 104] Connection reset " + "by peer (Authorization: Bearer sk-ABCDEF0123456789abcdef0123)" + ) + + sanitized = _sanitize_gateway_final_response(platform, raw_reset) + + assert sanitized.strip() + assert "sk-ABCDEF" not in sanitized + assert "Errno 104" not in sanitized + assert not _claims_the_endpoint_is_down(sanitized), sanitized + assert sanitized != _gateway_provider_error_reply(UNREACHABLE_ENVELOPES[0]) + + # Auth beats policy beats rate-limit beats connection: an envelope carrying BOTH its own + # marker and connection wording keeps the category it had before the connection split. + for tainted, clean in ( + ("API call failed after 3 retries: HTTP 401 incorrect api key provided; " + "connection reset by peer on the retry", "provider authentication failed"), + ("API call failed after 3 retries: HTTP 400 request was blocked under the provider " + "safety policy; connection reset by peer", "request was blocked under the safety policy"), + ("API call failed after 3 retries: HTTP 429 rate limit exceeded for this model; " + "connection reset by peer", "rate limited after 3 retries"), + ): + assert _sanitize_gateway_final_response(platform, tainted) == ( + _gateway_provider_error_reply(clean)), tainted + + # Programmatic consumers still get the bottom exception, byte for byte. + assert _sanitize_gateway_final_response("local", raw_reset) == raw_reset From ab7b0f54bbc5466a064f7f885ac3c13590a0f261 Mon Sep 17 00:00:00 2001 From: Lei-k <11388531+Lei-k@users.noreply.github.com> Date: Sun, 13 Sep 2026 06:44:42 +0000 Subject: [PATCH 2/2] fix(gateway): distinguish interrupted and unreachable model connections Selectively adapt the connection-wording portion of Lei-k/hermes-agent#16 (497cc47556b10eba94c65147cceedeba622aa087). Preserve endpoint-down guidance for refused connections without asserting that a reset means the endpoint stopped. Leave retry recovery and status/final deduplication to existing upstream contributions. --- gateway/run.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/gateway/run.py b/gateway/run.py index 3a8c6e2482c4..5bb1787ebbb4 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -636,15 +636,14 @@ def _format_exec_approval_fallback( (_GATEWAY_PROVIDER_POLICY_RE, "⚠️ The model provider rejected the request. I kept the raw provider " "error out of chat; check gateway logs for details or try rephrasing."), (_GATEWAY_RATE_LIMIT_RE, "⏱️ The model provider is rate-limiting requests. Please wait a moment and try again."), - (_GATEWAY_CONNECTION_INTERRUPTED_RE, "⚠️ The connection to the model provider was interrupted before the " - "reply arrived, and the retries hit the same problem. Please try " + (_GATEWAY_CONNECTION_INTERRUPTED_RE, "⚠️ The connection to the model provider was interrupted. Please try " "again; the transport details are in the gateway logs."), (_GATEWAY_ENDPOINT_UNREACHABLE_RE, "⚠️ The model server is not responding — it looks like the configured " "model endpoint is not running or is unreachable."), - (_GATEWAY_CONNECTION_ERROR_RE, "⚠️ The model request could not be completed over the network after " - "retries — the connection was either never established or dropped before " - "the reply arrived. Please try again; if it keeps happening, check that the " - "configured model endpoint is reachable. Details are in the gateway logs.")) + (_GATEWAY_CONNECTION_ERROR_RE, "⚠️ The model request could not be completed over the network — the " + "connection was either never established or was interrupted. Please try " + "again; if it keeps happening, check that the configured model endpoint " + "is reachable. Details are in the gateway logs.")) def _gateway_provider_error_reply(text: str) -> str: