Skip to content

fix(gateway): replace wall-clock agent timeout with inactivity-based timeout #5389

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
teknium1 merged 1 commit into from
Apr 6, 2026
Merged

fix(gateway): replace wall-clock agent timeout with inactivity-based timeout #5389

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 86 additions & 36 deletions gateway/run.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,10 @@ def _ensure_ssl_certs() -> None:
if _agent_cfg and isinstance(_agent_cfg, dict):
if "max_turns" in _agent_cfg:
os.environ["HERMES_MAX_ITERATIONS"] = str(_agent_cfg["max_turns"])
# Bridge agent.gateway_timeout → HERMES_AGENT_TIMEOUT env var.
# Env var from .env takes precedence (already in os.environ).
if "gateway_timeout" in _agent_cfg and "HERMES_AGENT_TIMEOUT" not in os.environ:
os.environ["HERMES_AGENT_TIMEOUT"] = str(_agent_cfg["gateway_timeout"])
# Timezone: bridge config.yaml → HERMES_TIMEZONE env var.
# HERMES_TIMEZONE from .env takes precedence (already in os.environ).
_tz_cfg = _cfg.get("timezone", "")
Expand Down Expand Up @@ -1800,32 +1804,46 @@ async def _handle_message(self, event: MessageEvent) -> Optional[str]:
# simultaneous updates. Do NOT interrupt for photo-only follow-ups here;
# let the adapter-level batching/queueing logic absorb them.

# Staleness eviction: if an entry has been in _running_agents for
# longer than the agent timeout, it's a leaked lock from a hung or
# crashed handler. Evict it so the session isn't permanently stuck.
# Staleness eviction: detect leaked locks from hung/crashed handlers.
# With inactivity-based timeout, active tasks can run for hours, so
# wall-clock age alone isn't sufficient. Evict only when the agent
# has been *idle* beyond the inactivity threshold (or when the agent
# object has no activity tracker and wall-clock age is extreme).
_raw_stale_timeout = float(os.getenv("HERMES_AGENT_TIMEOUT", 1800))
_STALE_TTL = (_raw_stale_timeout + 60) if _raw_stale_timeout > 0 else float("inf")
_stale_ts = self._running_agents_ts.get(_quick_key, 0)
if _quick_key in self._running_agents and _stale_ts and (time.time() - _stale_ts) > _STALE_TTL:
if _quick_key in self._running_agents and _stale_ts:
_stale_age = time.time() - _stale_ts
_stale_agent = self._running_agents.get(_quick_key)
_stale_idle = float("inf") # assume idle if we can't check
_stale_detail = ""
if _stale_agent and hasattr(_stale_agent, "get_activity_summary"):
try:
_sa = _stale_agent.get_activity_summary()
_stale_idle = _sa.get("seconds_since_activity", float("inf"))
_stale_detail = (
f" | last_activity={_sa.get('last_activity_desc', 'unknown')} "
f"({_sa.get('seconds_since_activity', 0):.0f}s ago) "
f"({_stale_idle:.0f}s ago) "
f"| iteration={_sa.get('api_call_count', 0)}/{_sa.get('max_iterations', 0)}"
)
except Exception:
pass
logger.warning(
"Evicting stale _running_agents entry for %s (age: %.0fs, TTL: %.0fs)%s",
_quick_key[:30], _stale_age, _STALE_TTL, _stale_detail,
# Evict if: agent is idle beyond timeout, OR wall-clock age is
# extreme (10x timeout or 2h, whichever is larger — catches
# cases where the agent object was garbage-collected).
_wall_ttl = max(_raw_stale_timeout * 10, 7200) if _raw_stale_timeout > 0 else float("inf")
_should_evict = (
(_raw_stale_timeout > 0 and _stale_idle >= _raw_stale_timeout)
or _stale_age > _wall_ttl
)
del self._running_agents[_quick_key]
self._running_agents_ts.pop(_quick_key, None)
if _should_evict:
logger.warning(
"Evicting stale _running_agents entry for %s "
"(age: %.0fs, idle: %.0fs, timeout: %.0fs)%s",
_quick_key[:30], _stale_age, _stale_idle,
_raw_stale_timeout, _stale_detail,
)
del self._running_agents[_quick_key]
self._running_agents_ts.pop(_quick_key, None)

if _quick_key in self._running_agents:
if event.get_command() == "status":
Expand Down Expand Up @@ -6766,19 +6784,54 @@ async def _notify_long_running():
_notify_task = asyncio.create_task(_notify_long_running())

try:
# Run in thread pool to not block. Cap total execution time
# so a hung API call or runaway tool doesn't permanently lock
# the session. Default 30 minutes; override with env var.
# Set to 0 for no limit (infinite).
# Run in thread pool to not block. Use an *inactivity*-based
# timeout instead of a wall-clock limit: the agent can run for
# hours if it's actively calling tools / receiving stream tokens,
# but a hung API call or stuck tool with no activity for the
# configured duration is caught and killed. (#4815)
#
# Config: agent.gateway_timeout in config.yaml, or
# HERMES_AGENT_TIMEOUT env var (env var takes precedence).
# Default 1800s (30 min inactivity). 0 = unlimited.
_agent_timeout_raw = float(os.getenv("HERMES_AGENT_TIMEOUT", 1800))
_agent_timeout = _agent_timeout_raw if _agent_timeout_raw > 0 else None
loop = asyncio.get_event_loop()
try:
response = await asyncio.wait_for(
loop.run_in_executor(None, run_sync),
timeout=_agent_timeout,
)
except asyncio.TimeoutError:
_executor_task = asyncio.ensure_future(
loop.run_in_executor(None, run_sync)
)

_inactivity_timeout = False
_POLL_INTERVAL = 5.0

if _agent_timeout is None:
# Unlimited — just await the result.
response = await _executor_task
else:
# Poll loop: check the agent's built-in activity tracker
# (updated by _touch_activity() on every tool call, API
# call, and stream delta) every few seconds.
response = None
while True:
done, _ = await asyncio.wait(
{_executor_task}, timeout=_POLL_INTERVAL
)
if done:
response = _executor_task.result()
break
# Agent still running — check inactivity.
_agent_ref = agent_holder[0]
_idle_secs = 0.0
if _agent_ref and hasattr(_agent_ref, "get_activity_summary"):
try:
_act = _agent_ref.get_activity_summary()
_idle_secs = _act.get("seconds_since_activity", 0.0)
except Exception:
pass
if _idle_secs >= _agent_timeout:
_inactivity_timeout = True
break

if _inactivity_timeout:
# Build a diagnostic summary from the agent's activity tracker.
_timed_out_agent = agent_holder[0]
_activity = {}
Expand All @@ -6795,29 +6848,26 @@ async def _notify_long_running():
_iter_max = _activity.get("max_iterations", 0)

logger.error(
"Agent execution timed out after %.0fs for session %s "
"| last_activity=%.0fs ago (%s) | iteration=%s/%s | tool=%s",
_agent_timeout, session_key,
_secs_ago, _last_desc, _iter_n, _iter_max,
"Agent idle for %.0fs (timeout %.0fs) in session %s "
"| last_activity=%s | iteration=%s/%s | tool=%s",
_secs_ago, _agent_timeout, session_key,
_last_desc, _iter_n, _iter_max,
_cur_tool or "none",
)

# Interrupt the agent if it's still running so the thread
# pool worker is freed.
if _timed_out_agent and hasattr(_timed_out_agent, "interrupt"):
_timed_out_agent.interrupt("Execution timed out")
_timed_out_agent.interrupt("Execution timed out (inactivity)")

_timeout_mins = int(_agent_timeout // 60)
_timeout_mins = int(_agent_timeout // 60) or 1

# Construct a user-facing message with diagnostic context.
_diag_lines = [f"⏱️ Request timed out after {_timeout_mins} minutes."]
if _secs_ago < 30:
_diag_lines.append(
f"The agent was actively working when the timeout fired "
f"(last activity: {_last_desc}, {_secs_ago:.0f}s ago, "
f"iteration {_iter_n}/{_iter_max})."
)
elif _cur_tool:
_diag_lines = [
f"⏱️ Agent inactive for {_timeout_mins} min — no tool calls "
f"or API responses."
]
if _cur_tool:
_diag_lines.append(
f"The agent appears stuck on tool `{_cur_tool}` "
f"({_secs_ago:.0f}s since last activity, "
Expand All @@ -6830,7 +6880,7 @@ async def _notify_long_running():
"The agent may have been waiting on an API response."
)
_diag_lines.append(
"To increase the limit, set HERMES_AGENT_TIMEOUT in your .env "
"To increase the limit, set agent.gateway_timeout in config.yaml "
"(value in seconds, 0 = no limit) and restart the gateway.\n"
"Try again, or use /reset to start fresh."
)
Expand Down
5 changes: 5 additions & 0 deletions hermes_cli/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,11 @@ def ensure_hermes_home():
"toolsets": ["hermes-cli"],
"agent": {
"max_turns": 90,
# Inactivity timeout for gateway agent execution (seconds).
# The agent can run indefinitely as long as it's actively calling
# tools or receiving API responses. Only fires when the agent has
# been completely idle for this duration. 0 = unlimited.
"gateway_timeout": 1800,
# Tool-use enforcement: injects system prompt guidance that tells the
# model to actually call tools instead of describing intended actions.
# Values: "auto" (default — applies to gpt/codex models), true/false
Expand Down
Loading