Skip to content
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 74 additions & 13 deletions gateway/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -1703,6 +1703,7 @@ def __init__(self, config: Optional[GatewayConfig] = None):
self._restart_drain_timeout = self._load_restart_drain_timeout()
self._provider_routing = self._load_provider_routing()
self._fallback_model = self._load_fallback_model()
self._platform_resume_delay = self._load_platform_resume_delay()

# Wire process registry into session store for reset protection
from tools.process_registry import process_registry
Expand Down Expand Up @@ -2824,17 +2825,42 @@ def _pause_failed_platform(self, platform, *, reason: str = "") -> None:
auto-pause — retryable (network/DNS) failures keep retrying at the
backoff cap indefinitely so a transient outage self-heals without
manual intervention.

If ``gateway.platform_resume_delay`` (or env
``HERMES_PLATFORM_RESUME_DELAY``) is set to a positive number of
**minutes**, the watcher will auto-resume the platform after that
delay instead of staying paused forever.
"""
info = getattr(self, "_failed_platforms", {}).get(platform)
if info is None:
return
if info.get("paused"):
return
info["paused"] = True
delay_min = getattr(self, "_platform_resume_delay", 0)
info["pause_reason"] = reason or "auto-paused after repeated failures"
# Push next_retry far enough out that even if "paused" is missed
# by a stale code path, the watcher won't fire on it.
info["next_retry"] = float("inf")
if delay_min > 0:
# Schedule auto-resume after the configured delay
info["next_retry"] = time.monotonic() + delay_min * 60
info["auto_resume_at"] = info["next_retry"]
logger.warning(
"%s paused after %d consecutive failures (%s) — "
"will auto-resume in %d min. "
"Run `/platform resume %s` to retry sooner.",
platform.value, info.get("attempts", 0),
info["pause_reason"], delay_min, platform.value,
)
else:
# Push next_retry far enough out that even if "paused" is missed
# by a stale code path, the watcher won't fire on it.
info["next_retry"] = float("inf")
logger.warning(
"%s paused after %d consecutive failures (%s) — "
"fix the underlying issue then run `/platform resume %s` "
"to retry, or `hermes gateway restart` to restart the gateway.",
platform.value, info.get("attempts", 0),
info["pause_reason"], platform.value,
)
try:
self._update_platform_runtime_status(
platform.value,
Expand All @@ -2844,13 +2870,6 @@ def _pause_failed_platform(self, platform, *, reason: str = "") -> None:
)
except Exception:
pass
logger.warning(
"%s paused after %d consecutive failures (%s) — "
"fix the underlying issue then run `/platform resume %s` "
"to retry, or `hermes gateway restart` to restart the gateway.",
platform.value, info.get("attempts", 0),
info["pause_reason"], platform.value,
)

def _resume_paused_platform(self, platform) -> bool:
"""Unpause a platform — reset its attempt counter and schedule an
Expand All @@ -2864,6 +2883,7 @@ def _resume_paused_platform(self, platform) -> bool:
return False
info["paused"] = False
info.pop("pause_reason", None)
info.pop("auto_resume_at", None)
info["attempts"] = 0
info["next_retry"] = time.monotonic() # retry on next watcher tick
try:
Expand Down Expand Up @@ -3111,6 +3131,32 @@ def _load_provider_routing() -> dict:
pass
return {}

@staticmethod
def _load_platform_resume_delay() -> int:
"""Load gateway.platform_resume_delay from config.yaml.

Controls how many **minutes** the gateway waits before automatically
resuming a paused platform (e.g. after Clash/VPN network recovery).

0 = never auto-resume (current default, backward compatible).
Environment variable ``HERMES_PLATFORM_RESUME_DELAY`` overrides.
"""
raw = os.getenv("HERMES_PLATFORM_RESUME_DELAY", "").strip()
if not raw:
try:
cfg = _load_gateway_runtime_config()
raw = str(
cfg_get(cfg, "gateway", "platform_resume_delay", default="0")
or ""
).strip()
except Exception:
pass
try:
val = int(raw)
return max(val, 0)
except (TypeError, ValueError, AttributeError):
return 0

@staticmethod
def _load_fallback_model() -> list | None:
"""Load fallback provider chain from config.yaml.
Expand Down Expand Up @@ -5937,10 +5983,25 @@ async def _platform_reconnect_watcher(self) -> None:
if not self._running:
return
info = self._failed_platforms[platform]
# Skip paused platforms entirely — they need explicit
# /platform resume to come back.
# Skip paused platforms that haven't reached their
# auto-resume time yet (if gateway.platform_resume_delay
# is set). Platforms paused manually via /platform pause
# have no auto_resume_at and stay paused until resumed.
if info.get("paused"):
continue
auto_at = info.get("auto_resume_at")
if auto_at is None or now < auto_at:
continue
# Auto-resume: timeout expired
logger.info(
"%s auto-resuming after pause timeout",
platform.value,
)
self._resume_paused_platform(platform)
if not self._running:
return
info = self._failed_platforms.get(platform)
if info is None or info.get("paused"):
continue
if now < info["next_retry"]:
continue # not time yet

Expand Down