From 88d1fa347faed8233437fde5baba1ced078fc3c0 Mon Sep 17 00:00:00 2001 From: Kyzcreig <9063726+Kyzcreig@users.noreply.github.com> Date: Fri, 25 Sep 2026 10:50:44 -0700 Subject: [PATCH 1/2] Revert "fix(failover): keep Tailscale errors terse (#622)" This reverts commit 738c5bb6993ab5b2beac172847f85b61963a2740. --- agent/conversation_loop.py | 5 ++++- agent/shared_transport_guard.py | 8 +++++--- hermes_cli/model_switch.py | 6 +++++- tests/agent/test_shared_transport_guard.py | 9 +++++++-- tests/hermes_cli/test_user_providers_model_switch.py | 5 ++++- tests/run_agent/test_shared_transport_fallback.py | 8 +++++--- 6 files changed, 30 insertions(+), 11 deletions(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index c956fbaf833a..fed618e2c46f 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -3680,7 +3680,10 @@ def _body_budget_failure(error_message: str) -> Dict[str, Any]: # the same Tailscale diagnosis twice; route count/session # detail remains in the WARNING diagnostic. agent._clear_status_buffer() - _transport_error = "Tailscale down" + _transport_error = ( + "Tailscale is down on the gateway host. Reconnect " + "Tailscale or use a non-tailnet provider." + ) agent._persist_session(messages, conversation_history) return { "final_response": _transport_error, diff --git a/agent/shared_transport_guard.py b/agent/shared_transport_guard.py index fe166ac3233d..cc879efc6647 100644 --- a/agent/shared_transport_guard.py +++ b/agent/shared_transport_guard.py @@ -188,9 +188,11 @@ def emit_unavailable_summary(agent, *, evidence: str) -> None: agent._shared_transport_summary_emitted = True count = len(routes) - # Chat gets the terse causal label. Route count, affected routes, session, - # evidence, and remediation remain available in the diagnostic below. - agent._buffer_status("⚠️ Tailscale down") + noun = "route" if count == 1 else "routes" + agent._buffer_status( + f"⚠️ Model routing degraded (Tailscale down): {count} tailnet {noun} " + "affected. Reconnect Tailscale or use a non-tailnet provider." + ) logger.warning( "Shared transport unavailable: transport=tailscale evidence=%s " "affected_routes=%d routes=%s session=%s remediation=%s", diff --git a/hermes_cli/model_switch.py b/hermes_cli/model_switch.py index e923898d9948..288d18d6cbf4 100644 --- a/hermes_cli/model_switch.py +++ b/hermes_cli/model_switch.py @@ -2416,7 +2416,11 @@ def switch_model( target_provider=target_provider, provider_label=provider_label, is_global=is_global, - error_message="Tailscale down", + error_message=( + "Tailscale is down on the gateway host; " + f"`{target_provider}/{new_model}` is unavailable. " + "Reconnect Tailscale or use a non-tailnet provider." + ), ) except Exception: logger.debug("Model-switch Tailscale preflight failed open", exc_info=True) diff --git a/tests/agent/test_shared_transport_guard.py b/tests/agent/test_shared_transport_guard.py index 956d58c4c9e3..555b76703591 100644 --- a/tests/agent/test_shared_transport_guard.py +++ b/tests/agent/test_shared_transport_guard.py @@ -96,7 +96,7 @@ def _run(argv, **kwargs): assert observed["kwargs"]["timeout"] == guard._TAILSCALE_STATUS_TIMEOUT_S -def test_summary_is_one_terse_causal_label(): +def test_summary_is_one_actionable_line_with_count_and_remediation(): agent = MagicMock() agent.session_id = "session-1" agent._shared_transport_affected_routes = set() @@ -108,4 +108,9 @@ def test_summary_is_one_terse_causal_label(): guard.emit_unavailable_summary(agent, evidence="backend_state=Stopped") guard.emit_unavailable_summary(agent, evidence="backend_state=Stopped") - agent._buffer_status.assert_called_once_with("⚠️ Tailscale down") + agent._buffer_status.assert_called_once() + message = agent._buffer_status.call_args.args[0] + assert "Tailscale down" in message + assert "3 tailnet routes" in message + assert "Reconnect Tailscale or use a non-tailnet provider" in message + assert "\n" not in message diff --git a/tests/hermes_cli/test_user_providers_model_switch.py b/tests/hermes_cli/test_user_providers_model_switch.py index 163adc83af6e..67d45e0cfc89 100644 --- a/tests/hermes_cli/test_user_providers_model_switch.py +++ b/tests/hermes_cli/test_user_providers_model_switch.py @@ -449,7 +449,10 @@ def test_switch_model_rejects_known_tailscale_route_while_client_is_stopped( ) assert result.success is False - assert result.error_message == "Tailscale down" + assert result.error_message == ( + "Tailscale is down on the gateway host; `claude-apr/claude-fable-5` " + "is unavailable. Reconnect Tailscale or use a non-tailnet provider." + ) validation.assert_not_called() diff --git a/tests/run_agent/test_shared_transport_fallback.py b/tests/run_agent/test_shared_transport_fallback.py index 74a3ad966519..0e7c191426e0 100644 --- a/tests/run_agent/test_shared_transport_fallback.py +++ b/tests/run_agent/test_shared_transport_fallback.py @@ -77,8 +77,7 @@ def test_confirmed_tailscale_stop_skips_every_tailnet_rung_and_uses_cross_transp assert all(client.close.called for client in clients[:3]) buffered = [message for _kind, message in agent._retry_status_buffer] assert sum("Tailscale down" in message for message in buffered) == 1 - assert buffered.count("⚠️ Tailscale down") == 1 - assert not any("tailnet routes" in message for message in buffered) + assert any("4 tailnet routes" in message for message in buffered) assert not any("Primary model failed" in message for message in buffered) @@ -165,7 +164,10 @@ def test_confirmed_stop_fails_before_any_model_request(monkeypatch): agent.client.chat.completions.create.assert_not_called() agent._anthropic_client.messages.stream.assert_not_called() assert agent._retry_status_buffer == [] - assert result["final_response"] == "Tailscale down" + assert result["final_response"] == ( + "Tailscale is down on the gateway host. Reconnect Tailscale or use a " + "non-tailnet provider." + ) def test_repeat_outage_reports_once_per_turn_with_fresh_route_count(monkeypatch): From 13ec95a4c33591f881eb41d71675fa83cf7fa895 Mon Sep 17 00:00:00 2001 From: Kyzcreig <9063726+Kyzcreig@users.noreply.github.com> Date: Fri, 25 Sep 2026 11:19:20 -0700 Subject: [PATCH 2/2] Revert "[verified] fix(failover): bound Tailscale transport outages (#621)" This reverts commit 7a4455f7dd, stacked on the clean #622 revert (6c344e319a). Hand-resolved (audit t_712c13f2, DROP): - agent/chat_completion_helpers.py, agent/conversation_loop.py: drop the Tailscale preflight blocks (HEAD carried only cosmetic later edits to them); inline _POOLED_PROVIDERS = frozenset({claude-apr, claude-bpr}) since #913 imported _TAILSCALE_RELAY_PROVIDERS from the deleted module. - tests/agent/test_error_classifier.py: drop tailscale_down from the enum list, keep the later pool_stalled member. Verified: test-gate pytest tests/agent/test_error_classifier.py tests/agent/test_fallback_reason_surfacing.py tests/agent/test_fallback_reason_threading.py -> 166 passed; tests/hermes_cli/test_user_providers_model_switch.py tests/run_agent/test_primary_runtime_restore.py -> 63 passed, 2 failed (same 2 node ids fail on clean origin/main 59102f4a05: inherited). --- agent/chat_completion_helpers.py | 71 +----- agent/conversation_loop.py | 71 ------ agent/error_classifier.py | 4 - agent/shared_transport_guard.py | 204 ----------------- agent/turn_context.py | 6 - hermes_cli/model_switch.py | 29 --- tests/agent/test_error_classifier.py | 2 +- tests/agent/test_fallback_reason_surfacing.py | 3 - tests/agent/test_fallback_reason_threading.py | 16 +- tests/agent/test_shared_transport_guard.py | 116 ---------- .../test_user_providers_model_switch.py | 61 ----- .../test_shared_transport_fallback.py | 208 ------------------ 12 files changed, 10 insertions(+), 781 deletions(-) delete mode 100644 agent/shared_transport_guard.py delete mode 100644 tests/agent/test_shared_transport_guard.py delete mode 100644 tests/run_agent/test_shared_transport_fallback.py diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index 07bdac5438af..aa7d91b4ab54 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -55,7 +55,6 @@ from agent.fork_ext.relay_headers import _pool_affinity_headers, _pool_lane, _pool_lane_src from agent.reasoning_summaries import separate_glued_reasoning_blocks from agent.stream_single_writer import claim_stream_writer, stream_writer_is_current -from agent.shared_transport_guard import _TAILSCALE_RELAY_PROVIDERS from tools.terminal_tool import is_persistent_env from utils import base_url_host_matches, base_url_hostname, env_float, env_int @@ -69,7 +68,8 @@ "x-pool-route-id", "x-pool-unreachable", ) -_POOLED_PROVIDERS = _TAILSCALE_RELAY_PROVIDERS +# Relay-pool providers whose responses carry the x-pool-* attribution headers. +_POOLED_PROVIDERS = frozenset({"claude-apr", "claude-bpr"}) _PINNED_PROVIDER_KEYS = { "xai-oauth": "supergrok", "gemini-bridge": "gemini", @@ -3008,7 +3008,6 @@ def _side(prov, mdl, eff): "overloaded": "provider overloaded", "server_error": "provider error", "timeout": "connection dropped", - "tailscale_down": "Tailscale down", "stream_parse": "malformed stream", "decode_error": "corrupt response", "ssl_cert_verification": "TLS error", @@ -3607,17 +3606,6 @@ def try_activate_fallback( backoff_count, backoff_seconds, backoff_seconds / 60, backoff_count + 1, ) if agent._fallback_index >= len(agent._fallback_chain): - try: - from agent.shared_transport_guard import emit_unavailable_summary - - emit_unavailable_summary( - agent, - evidence=getattr( - agent, "_shared_transport_evidence", "backend_state=Stopped" - ), - ) - except Exception: - logger.debug("Could not emit shared-transport fallback summary", exc_info=True) # Chain exhausted. If we actually walked a non-empty chain and the # failure was NOT a rate-limit/billing event (those already armed # their own 60s cooldown above), arm a short cooldown so the next @@ -3781,61 +3769,6 @@ def try_activate_fallback( # not pin api_mode explicitly. An explicit fb.api_mode (even # "chat_completions") must never be overridden here. fb_base_url = str(fb_client.base_url) - try: - from agent.shared_transport_guard import ( - emit_unavailable_summary, - record_unavailable_route, - route_uses_tailscale, - tailscale_status_down, - ) - - if route_uses_tailscale(fb_provider, fb_base_url): - tailscale_down, tailscale_evidence = tailscale_status_down() - if tailscale_down is True: - source_uses_tailscale = route_uses_tailscale( - getattr(agent, "provider", ""), - getattr(agent, "base_url", ""), - ) - if source_uses_tailscale: - record_unavailable_route( - agent, - getattr(agent, "provider", ""), - getattr(agent, "model", ""), - ) - record_unavailable_route(agent, fb_provider, fb_model) - agent._shared_transport_evidence = tailscale_evidence - unavailable.add(fb_key) - try: - fb_client.close() - except Exception: - pass - logger.warning( - "Fallback skip: %s/%s shares unavailable Tailscale " - "transport (%s); suppressing for this session", - fb_provider, - fb_model, - tailscale_evidence, - ) - next_reason = ( - FailoverReason.tailscale_down - if source_uses_tailscale - or reason == FailoverReason.tailscale_down - else reason - ) - return agent._try_activate_fallback( - next_reason, error_context=error_context, - display_reason=display_reason if next_reason == reason else None, - ) - emit_unavailable_summary( - agent, - evidence=getattr( - agent, "_shared_transport_evidence", "backend_state=Stopped" - ), - ) - except Exception: - # Availability detection is an optimization. An unavailable or - # malformed local Tailscale CLI must preserve historical routing. - logger.debug("Shared-transport fallback preflight failed open", exc_info=True) _fb_is_azure = agent._is_azure_openai_url(fb_base_url) if not fb_api_mode_explicit and fb_api_mode == "chat_completions": diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index fed618e2c46f..c4931e566717 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -3629,77 +3629,6 @@ def _body_budget_failure(error_message: str) -> Dict[str, Any]: effective_task_id=effective_task_id, close_tail=True, ) - # ── Shared host-transport preflight ─────────────────────── - # APR/BPR and direct 100.64/10 routes all depend on the local - # Tailscale client. If the host-local CLI explicitly says it is - # stopped, do not issue a request that can only consume the relay's - # full upstream timeout, then walk every route sharing the same - # prerequisite. Unknown/ambiguous CLI state fails open, so a lone - # provider timeout retains the normal retry and reason label. - try: - from agent.shared_transport_guard import ( - emit_unavailable_summary, - record_unavailable_route, - route_uses_tailscale, - tailscale_status_down, - ) - - _route_uses_tailscale = route_uses_tailscale( - getattr(agent, "provider", ""), - getattr(agent, "base_url", ""), - ) - _tailscale_down, _tailscale_evidence = ( - tailscale_status_down() - if _route_uses_tailscale - else (None, "not_applicable") - ) - if _route_uses_tailscale and _tailscale_down is True: - record_unavailable_route(agent, agent.provider, agent.model) - agent._shared_transport_evidence = _tailscale_evidence - if agent._try_activate_fallback( - reason=FailoverReason.tailscale_down - ): - active_system_prompt = _sync_failover_system_message( - agent, api_messages, active_system_prompt - ) - retry_count = 0 - compression_attempts = 0 - _retry.primary_recovery_attempted = False - # parity 2026-08-30: break to the restart-with-rebuilt- - # messages handler (not a bare continue) so the pre-API - # preflight re-runs against the fallback context window - # (#84733 restart discipline; upstream guard enforces). - _retry.restart_with_rebuilt_messages = True - break - - emit_unavailable_summary( - agent, evidence=_tailscale_evidence - ) - # The terminal result below is the single human-facing - # error. Drop the buffered aggregate here rather than send - # the same Tailscale diagnosis twice; route count/session - # detail remains in the WARNING diagnostic. - agent._clear_status_buffer() - _transport_error = ( - "Tailscale is down on the gateway host. Reconnect " - "Tailscale or use a non-tailnet provider." - ) - agent._persist_session(messages, conversation_history) - return { - "final_response": _transport_error, - "messages": messages, - "completed": False, - "api_calls": api_call_count, - "error": _transport_error, - "partial": True, - "failed": True, - "shared_transport_unavailable": "tailscale", - } - except Exception: - logger.debug( - "Shared-transport request preflight failed open", - exc_info=True, - ) # ── Nous Portal rate limit guard ────────────────────── # If another session already recorded that Nous is rate- # limited, skip the API call entirely. Each attempt diff --git a/agent/error_classifier.py b/agent/error_classifier.py index 3bd53f2563ad..96c281eb6c2d 100644 --- a/agent/error_classifier.py +++ b/agent/error_classifier.py @@ -79,10 +79,6 @@ class FailoverReason(enum.Enum): # Transport timeout = "timeout" # Connection/read timeout — rebuild client + retry - # Local Tailscale client is explicitly stopped. This is not inferred from - # an API timeout: the host-local CLI must confirm the shared prerequisite - # is down before fallback routing may use this reason. - tailscale_down = "tailscale_down" # An HTTP response stream opened successfully but contained malformed # JSON/SSE data. Retry because another provider box may return valid bytes, # then fail over with an honest user-facing reason if the fault persists. diff --git a/agent/shared_transport_guard.py b/agent/shared_transport_guard.py deleted file mode 100644 index cc879efc6647..000000000000 --- a/agent/shared_transport_guard.py +++ /dev/null @@ -1,204 +0,0 @@ -"""Bound fallback cascades when a route's local Tailscale prerequisite is down. - -The gateway process multiplexes many sessions, but Tailscale state belongs to the -host, so the short-lived status cache is deliberately process-global. Unknown -or ambiguous CLI results fail open: a provider timeout alone must never be -relabeled as a Tailscale outage. -""" - -from __future__ import annotations - -import ipaddress -import json -import logging -import os -from dataclasses import dataclass -from pathlib import Path -import subprocess # noqa: S404 # nosec B404 -- fixed absolute CLI allowlist -import sys -import threading -import time -from typing import Optional -from urllib.parse import urlsplit - -logger = logging.getLogger(__name__) - -_TAILSCALE_NETWORK = ipaddress.ip_network("100.64.0.0/10") -_TAILSCALE_RELAY_PROVIDERS = frozenset({"claude-apr", "claude-bpr"}) -_TAILSCALE_STATUS_TIMEOUT_S = 1.0 -_TAILSCALE_UP_CACHE_TTL_S = 5.0 -_TAILSCALE_DOWN_CACHE_TTL_S = 2.0 -_TAILSCALE_UNKNOWN_CACHE_TTL_S = 1.0 - -@dataclass -class _TailscaleStatusCache: - expires_at: float = 0.0 - value: Optional[bool] = None - evidence: str = "not_checked" - - -_CACHE_LOCK = threading.Lock() -_CACHE = _TailscaleStatusCache() - - -def route_uses_tailscale(provider: str, base_url: str) -> bool: - """Return whether a route has a known local Tailscale prerequisite.""" - normalized_provider = str(provider or "").strip().lower() - if normalized_provider in _TAILSCALE_RELAY_PROVIDERS: - return True - - try: - host = urlsplit(str(base_url or "")).hostname - address = ipaddress.ip_address(host or "") - except (TypeError, ValueError): - return False - return isinstance(address, ipaddress.IPv4Address) and address in _TAILSCALE_NETWORK - - -def _parse_tailscale_status( - returncode: int, stdout: str, stderr: str -) -> Optional[bool]: - """Parse an explicit stopped/running signal; return None when ambiguous.""" - if "tailscale is stopped" in str(stderr or "").lower(): - return True - - try: - payload = json.loads(stdout or "") - except (TypeError, ValueError): - # Some CLI versions return the stopped marker on stdout instead of - # stderr. Only inspect it after JSON parsing fails: peer names live in - # the status JSON and must not be able to spoof host state. - if "tailscale is stopped" in str(stdout or "").lower(): - return True - return None - if not isinstance(payload, dict): - return None - - backend_state = str(payload.get("BackendState") or "").strip().lower() - if backend_state == "stopped": - return True - if returncode == 0 and backend_state == "running": - return False - return None - - -def _tailscale_binary() -> Optional[str]: - candidates = [ - "/usr/bin/tailscale", - "/usr/local/bin/tailscale", - "/opt/homebrew/bin/tailscale", - ] - if sys.platform == "darwin": - candidates.insert(0, "/Applications/Tailscale.app/Contents/MacOS/Tailscale") - - for candidate in candidates: - try: - if Path(candidate).is_file() and os.access(candidate, os.X_OK): - return candidate - except OSError: - continue - return None - - -def _probe_tailscale_status_uncached() -> tuple[Optional[bool], str]: - binary = _tailscale_binary() - if not binary: - return None, "cli_unavailable" - - try: - completed = subprocess.run( # noqa: S603 # nosec B603 -- allowlisted binary - [binary, "status", "--json"], - capture_output=True, - check=False, - text=True, - timeout=_TAILSCALE_STATUS_TIMEOUT_S, - ) - except subprocess.TimeoutExpired: - return None, "cli_timeout" - except OSError as exc: - return None, f"cli_error={type(exc).__name__}" - - down = _parse_tailscale_status( - completed.returncode, completed.stdout, completed.stderr - ) - if down is True: - evidence = "backend_state=Stopped" - elif down is False: - evidence = "backend_state=Running" - else: - evidence = f"cli_indeterminate_rc={completed.returncode}" - return down, evidence - - -def tailscale_status_down() -> tuple[Optional[bool], str]: - """Return cached host Tailscale state as ``(down, evidence)``. - - The lock intentionally covers the bounded one-second probe: this is a rare - routing preflight and single-flight behavior is preferable to many gateway - sessions spawning the CLI simultaneously during the same outage. - """ - now = time.monotonic() - with _CACHE_LOCK: - if now < _CACHE.expires_at: - return _CACHE.value, _CACHE.evidence - - value, evidence = _probe_tailscale_status_uncached() - checked_at = time.monotonic() - if value is True: - ttl = _TAILSCALE_DOWN_CACHE_TTL_S - elif value is False: - ttl = _TAILSCALE_UP_CACHE_TTL_S - else: - ttl = _TAILSCALE_UNKNOWN_CACHE_TTL_S - _CACHE.value = value - _CACHE.evidence = evidence - _CACHE.expires_at = checked_at + ttl - return value, evidence - - -def _reset_cache_for_tests() -> None: - with _CACHE_LOCK: - _CACHE.expires_at = 0.0 - _CACHE.value = None - _CACHE.evidence = "not_checked" - - -def reset_turn_state(agent) -> None: - """Start a fresh per-turn outage-reporting episode for a reused agent.""" - agent._shared_transport_affected_routes = set() - agent._shared_transport_summary_emitted = False - agent._shared_transport_evidence = "not_checked" - - -def record_unavailable_route(agent, provider: str, model: str) -> None: - routes = getattr(agent, "_shared_transport_affected_routes", None) - if not isinstance(routes, set): - routes = set() - agent._shared_transport_affected_routes = routes - routes.add(f"{provider}/{model}") - - -def emit_unavailable_summary(agent, *, evidence: str) -> None: - """Buffer one actionable aggregate and write route/session detail to logs.""" - if getattr(agent, "_shared_transport_summary_emitted", False): - return - routes = getattr(agent, "_shared_transport_affected_routes", None) - if not isinstance(routes, set) or not routes: - return - - agent._shared_transport_summary_emitted = True - count = len(routes) - noun = "route" if count == 1 else "routes" - agent._buffer_status( - f"⚠️ Model routing degraded (Tailscale down): {count} tailnet {noun} " - "affected. Reconnect Tailscale or use a non-tailnet provider." - ) - logger.warning( - "Shared transport unavailable: transport=tailscale evidence=%s " - "affected_routes=%d routes=%s session=%s remediation=%s", - evidence, - count, - sorted(routes), - getattr(agent, "session_id", None), - "Reconnect Tailscale or use a non-tailnet provider", - ) diff --git a/agent/turn_context.py b/agent/turn_context.py index 369f9b490c70..efde365328e0 100644 --- a/agent/turn_context.py +++ b/agent/turn_context.py @@ -611,12 +611,6 @@ def build_turn_context( reset_quota_gate_turn_state(agent) except Exception: logger.debug("Could not reset quota-gate turn state", exc_info=True) - try: - from agent.shared_transport_guard import reset_turn_state - - reset_turn_state(agent) - except Exception: - logger.debug("Could not reset shared-transport turn state", exc_info=True) # Restore the primary runtime if the previous turn activated fallback. agent._restore_primary_runtime() diff --git a/hermes_cli/model_switch.py b/hermes_cli/model_switch.py index 288d18d6cbf4..243eeb398ebf 100644 --- a/hermes_cli/model_switch.py +++ b/hermes_cli/model_switch.py @@ -2396,35 +2396,6 @@ def switch_model( ) new_model = normalize_model_for_provider(new_model, target_provider) - # A model-list probe cannot make a route usable when its explicit local - # transport prerequisite is stopped. Fail before the probe so /model does - # not spend its timeout and then claim a dead APR/APX route was switched - # "without verification" (2026-08-19 incident). Ambiguous CLI state keeps - # the historical validation path. - try: - from agent.shared_transport_guard import ( - route_uses_tailscale, - tailscale_status_down, - ) - - if route_uses_tailscale(target_provider, base_url): - tailscale_down, _tailscale_evidence = tailscale_status_down() - if tailscale_down is True: - return ModelSwitchResult( - success=False, - new_model=new_model, - target_provider=target_provider, - provider_label=provider_label, - is_global=is_global, - error_message=( - "Tailscale is down on the gateway host; " - f"`{target_provider}/{new_model}` is unavailable. " - "Reconnect Tailscale or use a non-tailnet provider." - ), - ) - except Exception: - logger.debug("Model-switch Tailscale preflight failed open", exc_info=True) - # --- Validate --- try: if not probe_catalog: diff --git a/tests/agent/test_error_classifier.py b/tests/agent/test_error_classifier.py index fb2e7ad6e7ad..5e895fcc5fdc 100644 --- a/tests/agent/test_error_classifier.py +++ b/tests/agent/test_error_classifier.py @@ -59,7 +59,7 @@ def test_enum_members_exist(self): expected = { "auth", "auth_permanent", "account_blocked", "billing", "rate_limit", "upstream_rate_limit", "pool_exhausted", "pool_stalled", - "overloaded", "server_error", "timeout", "tailscale_down", "stream_parse", + "overloaded", "server_error", "timeout", "stream_parse", "ssl_cert_verification", "decode_error", "context_overflow", "body_too_large", "payload_too_large", "image_too_large", diff --git a/tests/agent/test_fallback_reason_surfacing.py b/tests/agent/test_fallback_reason_surfacing.py index 838f9fc40d83..db64e7018127 100644 --- a/tests/agent/test_fallback_reason_surfacing.py +++ b/tests/agent/test_fallback_reason_surfacing.py @@ -55,9 +55,6 @@ def test_timeout_maps_to_connection_dropped(self): # must read as "connection dropped", not the ambiguous bare "timeout". assert _fallback_reason_label(FailoverReason.timeout) == "connection dropped" - def test_explicit_tailscale_stop_maps_to_terse_cause(self): - assert _fallback_reason_label(FailoverReason.tailscale_down) == "Tailscale down" - def test_ssl_cert_maps(self): assert _fallback_reason_label(FailoverReason.ssl_cert_verification) == "TLS error" diff --git a/tests/agent/test_fallback_reason_threading.py b/tests/agent/test_fallback_reason_threading.py index 1206590032a5..7d854a5e0a0d 100644 --- a/tests/agent/test_fallback_reason_threading.py +++ b/tests/agent/test_fallback_reason_threading.py @@ -15,8 +15,7 @@ here. It also asserts the by-design floor sites remain reason-less, so we don't over-fix and fabricate a reason where none is classified. -Authoritative site audit (updated 2026-08-19), 11 sites total: - SHARED TRANSPORT: explicit Tailscale-down preflight +Authoritative site audit (ground-truthed 2026-07-12), 10 sites total: THREAD (knowable reason): 1241 rate_limit · 1991 content_policy_blocked · 3935 classified.reason · 4144 classified.reason ALREADY THREADED: 1820 · 3386 · 3422 @@ -63,11 +62,10 @@ def _is_bare_none(kw) -> bool: return kw is not None and isinstance(kw.value, ast.Constant) and kw.value.value is None -# The five sites that MUST thread a real reason. We key on a nearby structural +# The four sites that MUST thread a real reason. We key on a nearby structural # anchor (a distinctive nearby literal) rather than a bare line number so the # test survives small line drift in the file. _THREAD_ANCHORS = { - "shared_transport_preflight": "Shared host-transport preflight", "nous_rate_limit_guard": "Nous Portal rate limit active", "content_filter_stream_kill": "Content filter terminated stream", "client_error_should_fallback": "Non-retryable error (HTTP", @@ -92,7 +90,7 @@ def _lineno_after_anchor(anchor: str) -> int: class TestKnowableReasonSitesThread: - """Each of the 5 knowable-reason sites passes a non-None reason= (RC1 effect gate).""" + """Each of the 4 knowable-reason sites passes a non-None reason= (RC1 effect gate).""" @pytest.mark.parametrize("name,anchor", list(_THREAD_ANCHORS.items())) def test_site_threads_non_none_reason(self, name, anchor): @@ -136,11 +134,11 @@ def test_floor_site_has_no_reason(self, name, anchor): class TestSiteCountReconciles: - """There are exactly 11 fallback call sites (RC3 reconciliation).""" + """There are exactly 10 fallback call sites (RC3 reconciliation).""" - def test_eleven_sites(self): - assert len(_fallback_calls()) == 11, ( - f"expected 11 _try_activate_fallback sites, found {len(_fallback_calls())} — " + def test_ten_sites(self): + assert len(_fallback_calls()) == 10, ( + f"expected 10 _try_activate_fallback sites, found {len(_fallback_calls())} — " f"the audit table in the PR must be re-reconciled" ) diff --git a/tests/agent/test_shared_transport_guard.py b/tests/agent/test_shared_transport_guard.py deleted file mode 100644 index 555b76703591..000000000000 --- a/tests/agent/test_shared_transport_guard.py +++ /dev/null @@ -1,116 +0,0 @@ -from __future__ import annotations - -from types import SimpleNamespace -from unittest.mock import MagicMock - -import pytest - -from agent import shared_transport_guard as guard - - -@pytest.fixture(autouse=True) -def _reset_tailscale_cache(): - guard._reset_cache_for_tests() - yield - guard._reset_cache_for_tests() - - -@pytest.mark.parametrize( - ("provider", "base_url", "expected"), - [ - ("claude-apr", "http://127.0.0.1:18810/anthropic", True), - ("claude-bpr", "http://127.0.0.1:18811/anthropic", True), - ("custom", "http://100.64.0.1:18801/anthropic", True), - ("custom", "http://100.127.255.254:18801/anthropic", True), - ("custom", "http://100.63.255.255:18801/anthropic", False), - ("custom", "http://100.128.0.1:18801/anthropic", False), - ("custom", "https://api.anthropic.com", False), - ("custom", "http://127.0.0.1:18801", False), - ], -) -def test_route_uses_tailscale_only_for_known_relay_or_cgnat_tailnet( - provider, base_url, expected -): - assert guard.route_uses_tailscale(provider, base_url) is expected - - -@pytest.mark.parametrize( - ("returncode", "stdout", "stderr", "expected"), - [ - (1, "", "Tailscale is stopped.", True), - (0, '{"BackendState":"Stopped"}', "", True), - (0, '{"BackendState":"Running"}', "", False), - (1, '{"BackendState":"NeedsLogin"}', "", None), - (1, "not json", "permission denied", None), - ], -) -def test_parse_tailscale_status_requires_explicit_stopped_evidence( - returncode, stdout, stderr, expected -): - assert guard._parse_tailscale_status(returncode, stdout, stderr) is expected - - -def test_peer_name_cannot_spoof_stopped_marker_inside_running_json(): - payload = ( - '{"BackendState":"Running","Peer":{"node":{"DNSName":' - '"tailscale is stopped.example"}}}' - ) - assert guard._parse_tailscale_status(0, payload, "") is False - - -def test_tailscale_status_is_process_cached(monkeypatch): - calls = [] - - def _probe(): - calls.append(True) - return True, "backend_state=Stopped" - - monkeypatch.setattr(guard, "_probe_tailscale_status_uncached", _probe) - - assert guard.tailscale_status_down() == (True, "backend_state=Stopped") - assert guard.tailscale_status_down() == (True, "backend_state=Stopped") - assert len(calls) == 1 - - -def test_uncached_probe_executes_status_json_and_parses_stopped(monkeypatch): - observed = {} - - def _run(argv, **kwargs): - observed["argv"] = argv - observed["kwargs"] = kwargs - return SimpleNamespace( - returncode=1, - stdout="", - stderr="Tailscale is stopped.", - ) - - monkeypatch.setattr(guard, "_tailscale_binary", lambda: "/fake/tailscale") - monkeypatch.setattr(guard.subprocess, "run", _run) - - assert guard._probe_tailscale_status_uncached() == ( - True, - "backend_state=Stopped", - ) - assert observed["argv"] == ["/fake/tailscale", "status", "--json"] - assert observed["kwargs"]["check"] is False - assert observed["kwargs"]["timeout"] == guard._TAILSCALE_STATUS_TIMEOUT_S - - -def test_summary_is_one_actionable_line_with_count_and_remediation(): - agent = MagicMock() - agent.session_id = "session-1" - agent._shared_transport_affected_routes = set() - agent._shared_transport_summary_emitted = False - - guard.record_unavailable_route(agent, "claude-apr", "claude-fable-5") - guard.record_unavailable_route(agent, "claude-apx-1", "claude-opus-5") - guard.record_unavailable_route(agent, "claude-apx-2", "claude-opus-5") - guard.emit_unavailable_summary(agent, evidence="backend_state=Stopped") - guard.emit_unavailable_summary(agent, evidence="backend_state=Stopped") - - agent._buffer_status.assert_called_once() - message = agent._buffer_status.call_args.args[0] - assert "Tailscale down" in message - assert "3 tailnet routes" in message - assert "Reconnect Tailscale or use a non-tailnet provider" in message - assert "\n" not in message diff --git a/tests/hermes_cli/test_user_providers_model_switch.py b/tests/hermes_cli/test_user_providers_model_switch.py index 67d45e0cfc89..5fe4a74178e0 100644 --- a/tests/hermes_cli/test_user_providers_model_switch.py +++ b/tests/hermes_cli/test_user_providers_model_switch.py @@ -5,10 +5,7 @@ are exposed in the model picker. """ -from unittest.mock import MagicMock - import pytest - from hermes_cli.model_switch import list_authenticated_providers, switch_model from hermes_cli import runtime_provider as rp @@ -398,64 +395,6 @@ def test_switch_model_resolves_user_provider_credentials(monkeypatch, tmp_path): assert result.error_message == "" -def test_switch_model_rejects_known_tailscale_route_while_client_is_stopped( - monkeypatch, tmp_path -): - """A manual /model must not claim success for a route whose explicit local - prerequisite is down. This is the 2026-08-19 ``/model fable`` warning: - model-list verification noise hid the actionable Tailscale condition.""" - import yaml - - config = { - "providers": { - "claude-apr": { - "name": "Claude APR", - "api": "http://127.0.0.1:18810/anthropic", - "default_model": "claude-fable-5", - "transport": "anthropic_messages", - } - } - } - (tmp_path / "config.yaml").write_text(yaml.safe_dump(config), encoding="utf-8") - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - monkeypatch.setattr( - "hermes_cli.runtime_provider.resolve_runtime_provider", - lambda **_kw: { - "api_key": "test-key", - "base_url": "http://127.0.0.1:18810/anthropic", - "api_mode": "anthropic_messages", - }, - ) - monkeypatch.setattr( - "agent.shared_transport_guard.tailscale_status_down", - lambda: (True, "backend_state=Stopped"), - ) - validation = MagicMock( - return_value={ - "accepted": True, - "persist": True, - "recognized": True, - "message": None, - } - ) - monkeypatch.setattr("hermes_cli.models.validate_requested_model", validation) - - result = switch_model( - raw_input="claude-fable-5", - current_provider="claude-apr", - current_model="claude-opus-5", - current_base_url="http://127.0.0.1:18810/anthropic", - user_providers=config["providers"], - ) - - assert result.success is False - assert result.error_message == ( - "Tailscale is down on the gateway host; `claude-apr/claude-fable-5` " - "is unavailable. Reconnect Tailscale or use a non-tailnet provider." - ) - validation.assert_not_called() - - # ============================================================================= # Regression: providers: dict ``transport`` field must be honored # ============================================================================= diff --git a/tests/run_agent/test_shared_transport_fallback.py b/tests/run_agent/test_shared_transport_fallback.py deleted file mode 100644 index 0e7c191426e0..000000000000 --- a/tests/run_agent/test_shared_transport_fallback.py +++ /dev/null @@ -1,208 +0,0 @@ -from __future__ import annotations - -from unittest.mock import MagicMock, patch - -import pytest - -from agent.error_classifier import FailoverReason -from run_agent import AIAgent - - -def _make_agent(fallbacks): - with ( - patch("run_agent.get_tool_definitions", return_value=[]), - patch("run_agent.check_toolset_requirements", return_value={}), - patch("run_agent.OpenAI"), - ): - agent = AIAgent( - api_key="test-key", - base_url="http://127.0.0.1:18810/anthropic", - provider="claude-apr", - model="claude-fable-5", - quiet_mode=True, - skip_context_files=True, - skip_memory=True, - fallback_model=fallbacks, - ) - agent.client = MagicMock() - agent._retry_status_buffer = [] - return agent - - -def _client(base_url): - client = MagicMock() - client.base_url = base_url - client.api_key = "test-key" - return client - - -def _fallbacks(): - return [ - {"provider": "claude-apx-1", "model": "claude-opus-5"}, - {"provider": "claude-apx-2", "model": "claude-opus-5"}, - {"provider": "claude-apx-3", "model": "claude-opus-5"}, - {"provider": "openai-codex", "model": "gpt-5.6-sol"}, - ] - - -def test_confirmed_tailscale_stop_skips_every_tailnet_rung_and_uses_cross_transport( - monkeypatch, -): - from agent import shared_transport_guard as guard - - agent = _make_agent(_fallbacks()) - clients = [ - _client("http://100.84.177.69:18801/anthropic"), - _client("http://100.100.218.3:18801/anthropic"), - _client("http://100.102.189.29:18801/anthropic"), - _client("https://api.openai.com/v1"), - ] - monkeypatch.setattr( - guard, - "tailscale_status_down", - lambda: (True, "backend_state=Stopped"), - ) - - with patch( - "agent.auxiliary_client.resolve_provider_client", - side_effect=[(client, entry["model"]) for client, entry in zip(clients, _fallbacks())], - ): - assert agent._try_activate_fallback(reason=FailoverReason.timeout) is True - - assert agent.provider == "openai-codex" - assert agent.model == "gpt-5.6-sol" - assert agent._fallback_index == 4 - assert agent._last_fallback_event["reason"] == FailoverReason.tailscale_down.value - assert agent._last_fallback_event["reason_label"] == "Tailscale down" - assert all(client.close.called for client in clients[:3]) - buffered = [message for _kind, message in agent._retry_status_buffer] - assert sum("Tailscale down" in message for message in buffered) == 1 - assert any("4 tailnet routes" in message for message in buffered) - assert not any("Primary model failed" in message for message in buffered) - - -def test_tailscale_healthy_preserves_single_provider_connection_drop_semantics(monkeypatch): - from agent import shared_transport_guard as guard - - agent = _make_agent(_fallbacks()[:2]) - monkeypatch.setattr( - guard, - "tailscale_status_down", - lambda: (False, "backend_state=Running"), - ) - - with patch( - "agent.auxiliary_client.resolve_provider_client", - return_value=( - _client("http://100.84.177.69:18801/anthropic"), - "claude-opus-5", - ), - ): - assert agent._try_activate_fallback(reason=FailoverReason.timeout) is True - - assert agent.provider == "claude-apx-1" - assert agent._fallback_index == 1 - event = agent._last_fallback_event - assert event["reason"] == FailoverReason.timeout.value - assert event["reason_label"] == "connection dropped" - assert not any( - "Tailscale" in message for _kind, message in agent._retry_status_buffer - ) - - -def test_tailscale_indeterminate_does_not_infer_outage_from_one_timeout(monkeypatch): - from agent import shared_transport_guard as guard - - agent = _make_agent(_fallbacks()[:1]) - monkeypatch.setattr( - guard, - "tailscale_status_down", - lambda: (None, "cli_unavailable"), - ) - - with patch( - "agent.auxiliary_client.resolve_provider_client", - return_value=( - _client("http://100.84.177.69:18801/anthropic"), - "claude-opus-5", - ), - ): - assert agent._try_activate_fallback(reason=FailoverReason.timeout) is True - - assert agent.provider == "claude-apx-1" - assert agent._last_fallback_event["reason"] == FailoverReason.timeout.value - - -def test_confirmed_stop_fails_before_any_model_request(monkeypatch): - from agent import shared_transport_guard as guard - - agent = _make_agent(_fallbacks()[:1]) - agent.client.chat.completions.create.side_effect = AssertionError( - "model request must not fire while Tailscale is explicitly stopped" - ) - agent._anthropic_client = MagicMock() - agent._anthropic_client.messages.stream.side_effect = AssertionError( - "Anthropic request must not fire while Tailscale is explicitly stopped" - ) - monkeypatch.setattr( - guard, - "tailscale_status_down", - lambda: (True, "backend_state=Stopped"), - ) - - with patch( - "agent.auxiliary_client.resolve_provider_client", - return_value=( - _client("http://100.84.177.69:18801/anthropic"), - "claude-opus-5", - ), - ): - result = agent.run_conversation("hello", system_message="test system") - - assert result["failed"] is True - assert result["shared_transport_unavailable"] == "tailscale" - agent.client.chat.completions.create.assert_not_called() - agent._anthropic_client.messages.stream.assert_not_called() - assert agent._retry_status_buffer == [] - assert result["final_response"] == ( - "Tailscale is down on the gateway host. Reconnect Tailscale or use a " - "non-tailnet provider." - ) - - -def test_repeat_outage_reports_once_per_turn_with_fresh_route_count(monkeypatch): - from agent import shared_transport_guard as guard - - agent = _make_agent(_fallbacks()[:1]) - monkeypatch.setattr( - guard, - "tailscale_status_down", - lambda: (True, "backend_state=Stopped"), - ) - - with ( - patch( - "agent.auxiliary_client.resolve_provider_client", - return_value=( - _client("http://100.84.177.69:18801/anthropic"), - "claude-opus-5", - ), - ), - patch.object(guard.logger, "warning") as warning, - ): - first = agent.run_conversation("first", system_message="test system") - second = agent.run_conversation("second", system_message="test system") - - assert first["shared_transport_unavailable"] == "tailscale" - assert second["shared_transport_unavailable"] == "tailscale" - assert warning.call_count == 2 - assert [call.args[2] for call in warning.call_args_list] == [2, 1] - assert agent._shared_transport_affected_routes == { - "claude-apr/claude-fable-5", - } - - -def test_tailscale_reason_rider_is_two_words(): - from agent.chat_completion_helpers import _fallback_reason_label - - assert _fallback_reason_label(FailoverReason.tailscale_down) == "Tailscale down"