diff --git a/cli-config.yaml.example b/cli-config.yaml.example index 2049b4426a54..09b3d2fa4a11 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -716,6 +716,11 @@ agent: # window on /restart, and keep it well under systemd's TimeoutStopSec. # restart_drain_timeout: 0 + # Idle TTL for cached gateway agents between messages (seconds). + # Increase for long-lived messaging threads; set to 0 to disable idle eviction. + # The hard cache size cap still applies. + # cache_idle_ttl_seconds: 3600 + # Max app-level retry attempts for API errors (connection drops, provider # timeouts, 5xx, etc.) before the agent surfaces the failure. Lower this # to 1 if you use fallback providers and want fast failover on flaky diff --git a/gateway/run.py b/gateway/run.py index b071e4854f39..d131a808a72f 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -30,6 +30,7 @@ import inspect import json import logging +import math import os import re import shlex @@ -56,7 +57,7 @@ from agent.async_utils import consume_detached_task_result, safe_schedule_threadsafe from agent.conversation_loop import INTERRUPT_WAITING_FOR_MODEL_PREFIX from agent.i18n import t -from hermes_cli.config import cfg_get +from hermes_cli.config import cfg_get, load_config from hermes_cli.fallback_config import get_fallback_chain # --- Agent cache tuning --------------------------------------------------- @@ -2001,6 +2002,57 @@ def _own_policy_open_startup_violation(config) -> Optional[str]: return None +def _resolve_agent_cache_idle_ttl_secs(config: Optional[Dict[str, Any]] = None) -> float: + """Return the configured agent-cache idle TTL in seconds. + + ``0`` disables idle eviction. Missing, negative, non-finite, or non-numeric + values keep the historical one-hour default so a bad config cannot + accidentally disable cache cleanup for long-lived gateways. + """ + cfg = config + if cfg is None: + try: + cfg = load_config() + except Exception: + cfg = {} + + raw = cfg_get( + cfg, + "agent", + "cache_idle_ttl_seconds", + default=_AGENT_CACHE_IDLE_TTL_SECS, + ) + if raw is False: + return 0.0 + if raw is True: + return _AGENT_CACHE_IDLE_TTL_SECS + + try: + ttl = float(raw) + except (TypeError, ValueError): + logger.warning( + "Ignoring invalid agent.cache_idle_ttl_seconds=%r; using default %.0fs", + raw, + _AGENT_CACHE_IDLE_TTL_SECS, + ) + return _AGENT_CACHE_IDLE_TTL_SECS + if not math.isfinite(ttl): + logger.warning( + "Ignoring non-finite agent.cache_idle_ttl_seconds=%r; using default %.0fs", + raw, + _AGENT_CACHE_IDLE_TTL_SECS, + ) + return _AGENT_CACHE_IDLE_TTL_SECS + if ttl < 0: + logger.warning( + "Ignoring negative agent.cache_idle_ttl_seconds=%r; using default %.0fs", + raw, + _AGENT_CACHE_IDLE_TTL_SECS, + ) + return _AGENT_CACHE_IDLE_TTL_SECS + return ttl + + # Sentinel placed into _running_agents immediately when a session starts # processing, *before* any await. Prevents a second message for the same # session from bypassing the "already running" guard during the async gap @@ -3102,6 +3154,7 @@ def __init__(self, config: Optional[GatewayConfig] = None): self._busy_input_mode = self._load_busy_input_mode() self._busy_text_mode = self._load_busy_text_mode() self._restart_drain_timeout = self._load_restart_drain_timeout() + self._agent_cache_idle_ttl_secs = _resolve_agent_cache_idle_ttl_secs() self._provider_routing = self._load_provider_routing() self._fallback_model = self._load_fallback_model() @@ -18251,7 +18304,7 @@ def _enforce_agent_cache_cap(self) -> None: ).start() def _sweep_idle_cached_agents(self) -> int: - """Evict cached agents whose AIAgent has been idle > _AGENT_CACHE_IDLE_TTL_SECS. + """Evict cached agents whose AIAgent has been idle past the configured TTL. Safe to call from the session expiry watcher without holding the cache lock — acquires it internally. Returns the number of entries @@ -18265,6 +18318,11 @@ def _sweep_idle_cached_agents(self) -> int: _lock = getattr(self, "_agent_cache_lock", None) if _cache is None or _lock is None: return 0 + idle_ttl_secs = float( + getattr(self, "_agent_cache_idle_ttl_secs", _AGENT_CACHE_IDLE_TTL_SECS) + ) + if idle_ttl_secs <= 0: + return 0 now = time.time() to_evict: List[tuple] = [] running_ids = { @@ -18282,7 +18340,7 @@ def _sweep_idle_cached_agents(self) -> int: last_activity = getattr(agent, "_last_activity_ts", None) if last_activity is None: continue - if (now - last_activity) > _AGENT_CACHE_IDLE_TTL_SECS: + if (now - last_activity) > idle_ttl_secs: # Check whether the session has actually expired in the # session store. If it hasn't (e.g. daily-reset mode # where the reset fires hours after the user's last diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 66a4024809b9..b3652383e2f0 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -1031,6 +1031,9 @@ def _ensure_hermes_home_managed(home: Path): # Set a positive value in config.yaml only if you explicitly want a # grace window on /restart (and keep it well under TimeoutStopSec). "restart_drain_timeout": 0, + # Idle TTL for cached gateway agents between messages. 0 disables + # idle eviction; the hard cache size cap still applies. + "cache_idle_ttl_seconds": 3600, # Max app-level retry attempts for API errors (connection drops, # provider timeouts, 5xx, etc.) before the agent surfaces the # failure. The OpenAI SDK already does its own low-level retries diff --git a/tests/gateway/test_agent_cache.py b/tests/gateway/test_agent_cache.py index 81264fa37c9d..cf8e01cb3f91 100644 --- a/tests/gateway/test_agent_cache.py +++ b/tests/gateway/test_agent_cache.py @@ -781,12 +781,71 @@ def test_idle_ttl_sweep_evicts_stale_agents(self, monkeypatch): assert "stale" not in runner._agent_cache assert "fresh" in runner._agent_cache + def test_idle_ttl_sweep_can_be_disabled(self): + """agent.cache_idle_ttl_seconds=0 disables idle eviction.""" + runner = self._bounded_runner() + runner._agent_cache_idle_ttl_secs = 0 + runner._cleanup_agent_resources = MagicMock() + + import time as _t + stale = self._fake_agent(last_activity=_t.time() - 10.0) + runner._agent_cache["stale"] = (stale, "sig") + + assert runner._sweep_idle_cached_agents() == 0 + assert "stale" in runner._agent_cache + runner._cleanup_agent_resources.assert_not_called() + + def test_idle_ttl_sweep_uses_runner_configured_ttl(self): + """The runner's configured TTL controls stale cache eviction.""" + runner = self._bounded_runner() + runner._agent_cache_idle_ttl_secs = 60.0 + runner._cleanup_agent_resources = MagicMock() + + import time as _t + fresh = self._fake_agent(last_activity=_t.time() - 30.0) + stale = self._fake_agent(last_activity=_t.time() - 90.0) + runner._agent_cache["fresh"] = (fresh, "s1") + runner._agent_cache["stale"] = (stale, "s2") + + assert runner._sweep_idle_cached_agents() == 1 + assert "fresh" in runner._agent_cache + assert "stale" not in runner._agent_cache + + def test_agent_cache_idle_ttl_config_parser(self, monkeypatch): + """TTL config accepts 0/positive values and falls back on invalid input.""" + from gateway import run as gw_run + + assert gw_run._resolve_agent_cache_idle_ttl_secs({ + "agent": {"cache_idle_ttl_seconds": 0} + }) == 0.0 + assert gw_run._resolve_agent_cache_idle_ttl_secs({ + "agent": {"cache_idle_ttl_seconds": "7200"} + }) == 7200.0 + assert gw_run._resolve_agent_cache_idle_ttl_secs({ + "agent": {"cache_idle_ttl_seconds": -1} + }) == gw_run._AGENT_CACHE_IDLE_TTL_SECS + assert gw_run._resolve_agent_cache_idle_ttl_secs({ + "agent": {"cache_idle_ttl_seconds": "not-a-number"} + }) == gw_run._AGENT_CACHE_IDLE_TTL_SECS + for raw in ("nan", "NaN", "inf", "-inf", float("nan"), float("inf")): + assert gw_run._resolve_agent_cache_idle_ttl_secs({ + "agent": {"cache_idle_ttl_seconds": raw} + }) == gw_run._AGENT_CACHE_IDLE_TTL_SECS + + monkeypatch.setattr( + gw_run, + "load_config", + lambda: {"agent": {"cache_idle_ttl_seconds": 123}}, + ) + assert gw_run._resolve_agent_cache_idle_ttl_secs() == 123.0 + def test_idle_sweep_skips_agents_without_activity_ts(self, monkeypatch): """Agents missing _last_activity_ts are left alone (defensive).""" from gateway import run as gw_run monkeypatch.setattr(gw_run, "_AGENT_CACHE_IDLE_TTL_SECS", 0.01) runner = self._bounded_runner() + runner._agent_cache_idle_ttl_secs = 0.01 runner._cleanup_agent_resources = MagicMock() no_ts = MagicMock(spec=[]) # no _last_activity_ts attribute @@ -807,6 +866,7 @@ def test_idle_sweep_keeps_agent_when_session_not_expired(self, monkeypatch): monkeypatch.setattr(gw_run, "_AGENT_CACHE_IDLE_TTL_SECS", 0.01) runner = self._bounded_runner() + runner._agent_cache_idle_ttl_secs = 0.01 runner._cleanup_agent_resources = MagicMock() import time as _t @@ -833,6 +893,7 @@ def test_idle_sweep_evicts_when_session_is_expired(self, monkeypatch): monkeypatch.setattr(gw_run, "_AGENT_CACHE_IDLE_TTL_SECS", 0.01) runner = self._bounded_runner() + runner._agent_cache_idle_ttl_secs = 0.01 runner._cleanup_agent_resources = MagicMock() import time as _t diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index 5297a39f11d3..d8b24e324d30 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -838,6 +838,7 @@ Instead, when the budget is actually exhausted (90/90), Hermes injects one messa ```yaml agent: max_turns: 90 # Max iterations per conversation turn (default: 90) + cache_idle_ttl_seconds: 3600 # Evict idle cached gateway agents after 1h; 0 disables idle eviction api_max_retries: 3 # Retries per provider before fallback engages (default: 3) ``` @@ -845,6 +846,8 @@ When the iteration budget is fully exhausted, the CLI shows a notification to th `agent.api_max_retries` controls how many times Hermes retries a provider API call on transient errors (rate limits, connection drops, 5xx) **before** fallback-provider switching engages. The default is `3` — four attempts total. If you have [fallback providers](/user-guide/features/fallback-providers) configured and want to fail over faster, drop this to `0` so the first transient error on your primary immediately hands off to the fallback instead of churning retries against the flaky endpoint. +`agent.cache_idle_ttl_seconds` controls how long gateway sessions keep an idle in-memory agent between messages. The default is `3600` seconds. Set it higher for long-lived messaging threads that should preserve conversation context across breaks, or set it to `0` to disable idle eviction entirely. The hard cache size cap still applies, so disabling idle TTL does not make the cache unbounded. + ## Standing Goals (`/goal`) When a standing goal is active, Hermes judges whether each assistant response satisfies it. If not, it feeds a continuation prompt back into the same session and keeps working until the goal is done, the turn budget is exhausted, or the user pauses/clears it. The turn budget is the real backstop — judge failures fail **open** (continue) so a flaky judge never wedges progress.