feat(gateway): make agent cache idle TTL configurable

cli-config.yaml.example

@@ -716,6 +716,11 @@ agent:
   # window on /restart, and keep it well under systemd's TimeoutStopSec.
   # restart_drain_timeout: 0
 
+  # Idle TTL for cached gateway agents between messages (seconds).
+  # Increase for long-lived messaging threads; set to 0 to disable idle eviction.
+  # The hard cache size cap still applies.
+  # cache_idle_ttl_seconds: 3600
+
   # Max app-level retry attempts for API errors (connection drops, provider
   # timeouts, 5xx, etc.) before the agent surfaces the failure. Lower this
   # to 1 if you use fallback providers and want fast failover on flaky

gateway/run.py

@@ -30,6 +30,7 @@
 import inspect
 import json
 import logging
+import math
 import os
 import re
 import shlex
@@ -56,7 +57,7 @@
 from agent.async_utils import consume_detached_task_result, safe_schedule_threadsafe
 from agent.conversation_loop import INTERRUPT_WAITING_FOR_MODEL_PREFIX
 from agent.i18n import t
-from hermes_cli.config import cfg_get
+from hermes_cli.config import cfg_get, load_config
 from hermes_cli.fallback_config import get_fallback_chain
 
 # --- Agent cache tuning ---------------------------------------------------
@@ -2001,6 +2002,57 @@ def _own_policy_open_startup_violation(config) -> Optional[str]:
     return None
 
 
+def _resolve_agent_cache_idle_ttl_secs(config: Optional[Dict[str, Any]] = None) -> float:
+    """Return the configured agent-cache idle TTL in seconds.
+
+    ``0`` disables idle eviction. Missing, negative, non-finite, or non-numeric
+    values keep the historical one-hour default so a bad config cannot
+    accidentally disable cache cleanup for long-lived gateways.
+    """
+    cfg = config
+    if cfg is None:
+        try:
+            cfg = load_config()
+        except Exception:
+            cfg = {}
+
+    raw = cfg_get(
+        cfg,
+        "agent",
+        "cache_idle_ttl_seconds",
+        default=_AGENT_CACHE_IDLE_TTL_SECS,
+    )
+    if raw is False:
+        return 0.0
+    if raw is True:
+        return _AGENT_CACHE_IDLE_TTL_SECS
+
+    try:
+        ttl = float(raw)
+    except (TypeError, ValueError):
+        logger.warning(
+            "Ignoring invalid agent.cache_idle_ttl_seconds=%r; using default %.0fs",
+            raw,
+            _AGENT_CACHE_IDLE_TTL_SECS,
+        )
+        return _AGENT_CACHE_IDLE_TTL_SECS
+    if not math.isfinite(ttl):
+        logger.warning(
+            "Ignoring non-finite agent.cache_idle_ttl_seconds=%r; using default %.0fs",
+            raw,
+            _AGENT_CACHE_IDLE_TTL_SECS,
+        )
+        return _AGENT_CACHE_IDLE_TTL_SECS
+    if ttl < 0:
+        logger.warning(
+            "Ignoring negative agent.cache_idle_ttl_seconds=%r; using default %.0fs",
+            raw,
+            _AGENT_CACHE_IDLE_TTL_SECS,
+        )
+        return _AGENT_CACHE_IDLE_TTL_SECS
+    return ttl
+
+
 # Sentinel placed into _running_agents immediately when a session starts
 # processing, *before* any await.  Prevents a second message for the same
 # session from bypassing the "already running" guard during the async gap
@@ -3102,6 +3154,7 @@ def __init__(self, config: Optional[GatewayConfig] = None):
         self._busy_input_mode = self._load_busy_input_mode()
         self._busy_text_mode = self._load_busy_text_mode()
         self._restart_drain_timeout = self._load_restart_drain_timeout()
+        self._agent_cache_idle_ttl_secs = _resolve_agent_cache_idle_ttl_secs()
         self._provider_routing = self._load_provider_routing()
         self._fallback_model = self._load_fallback_model()
 
@@ -18251,7 +18304,7 @@ def _enforce_agent_cache_cap(self) -> None:
                 ).start()
 
     def _sweep_idle_cached_agents(self) -> int:
-        """Evict cached agents whose AIAgent has been idle > _AGENT_CACHE_IDLE_TTL_SECS.
+        """Evict cached agents whose AIAgent has been idle past the configured TTL.
 
         Safe to call from the session expiry watcher without holding the
         cache lock — acquires it internally.  Returns the number of entries
@@ -18265,6 +18318,11 @@ def _sweep_idle_cached_agents(self) -> int:
         _lock = getattr(self, "_agent_cache_lock", None)
         if _cache is None or _lock is None:
             return 0
+        idle_ttl_secs = float(
+            getattr(self, "_agent_cache_idle_ttl_secs", _AGENT_CACHE_IDLE_TTL_SECS)
+        )
+        if idle_ttl_secs <= 0:
+            return 0
         now = time.time()
         to_evict: List[tuple] = []
         running_ids = {
@@ -18282,7 +18340,7 @@ def _sweep_idle_cached_agents(self) -> int:
                 last_activity = getattr(agent, "_last_activity_ts", None)
                 if last_activity is None:
                     continue
-                if (now - last_activity) > _AGENT_CACHE_IDLE_TTL_SECS:
+                if (now - last_activity) > idle_ttl_secs:
                     # Check whether the session has actually expired in the
                     # session store.  If it hasn't (e.g. daily-reset mode
                     # where the reset fires hours after the user's last

hermes_cli/config.py

@@ -1031,6 +1031,9 @@ def _ensure_hermes_home_managed(home: Path):
         # Set a positive value in config.yaml only if you explicitly want a
         # grace window on /restart (and keep it well under TimeoutStopSec).
         "restart_drain_timeout": 0,
+        # Idle TTL for cached gateway agents between messages.  0 disables
+        # idle eviction; the hard cache size cap still applies.
+        "cache_idle_ttl_seconds": 3600,
         # Max app-level retry attempts for API errors (connection drops,
         # provider timeouts, 5xx, etc.) before the agent surfaces the
         # failure.  The OpenAI SDK already does its own low-level retries

tests/gateway/test_agent_cache.py

@@ -781,12 +781,71 @@ def test_idle_ttl_sweep_evicts_stale_agents(self, monkeypatch):
         assert "stale" not in runner._agent_cache
         assert "fresh" in runner._agent_cache
 
+    def test_idle_ttl_sweep_can_be_disabled(self):
+        """agent.cache_idle_ttl_seconds=0 disables idle eviction."""
+        runner = self._bounded_runner()
+        runner._agent_cache_idle_ttl_secs = 0
+        runner._cleanup_agent_resources = MagicMock()
+
+        import time as _t
+        stale = self._fake_agent(last_activity=_t.time() - 10.0)
+        runner._agent_cache["stale"] = (stale, "sig")
+
+        assert runner._sweep_idle_cached_agents() == 0
+        assert "stale" in runner._agent_cache
+        runner._cleanup_agent_resources.assert_not_called()
+
+    def test_idle_ttl_sweep_uses_runner_configured_ttl(self):
+        """The runner's configured TTL controls stale cache eviction."""
+        runner = self._bounded_runner()
+        runner._agent_cache_idle_ttl_secs = 60.0
+        runner._cleanup_agent_resources = MagicMock()
+
+        import time as _t
+        fresh = self._fake_agent(last_activity=_t.time() - 30.0)
+        stale = self._fake_agent(last_activity=_t.time() - 90.0)
+        runner._agent_cache["fresh"] = (fresh, "s1")
+        runner._agent_cache["stale"] = (stale, "s2")
+
+        assert runner._sweep_idle_cached_agents() == 1
+        assert "fresh" in runner._agent_cache
+        assert "stale" not in runner._agent_cache
+
+    def test_agent_cache_idle_ttl_config_parser(self, monkeypatch):
+        """TTL config accepts 0/positive values and falls back on invalid input."""
+        from gateway import run as gw_run
+
+        assert gw_run._resolve_agent_cache_idle_ttl_secs({
+            "agent": {"cache_idle_ttl_seconds": 0}
+        }) == 0.0
+        assert gw_run._resolve_agent_cache_idle_ttl_secs({
+            "agent": {"cache_idle_ttl_seconds": "7200"}
+        }) == 7200.0
+        assert gw_run._resolve_agent_cache_idle_ttl_secs({
+            "agent": {"cache_idle_ttl_seconds": -1}
+        }) == gw_run._AGENT_CACHE_IDLE_TTL_SECS
+        assert gw_run._resolve_agent_cache_idle_ttl_secs({
+            "agent": {"cache_idle_ttl_seconds": "not-a-number"}
+        }) == gw_run._AGENT_CACHE_IDLE_TTL_SECS
+        for raw in ("nan", "NaN", "inf", "-inf", float("nan"), float("inf")):
+            assert gw_run._resolve_agent_cache_idle_ttl_secs({
+                "agent": {"cache_idle_ttl_seconds": raw}
+            }) == gw_run._AGENT_CACHE_IDLE_TTL_SECS
+
+        monkeypatch.setattr(
+            gw_run,
+            "load_config",
+            lambda: {"agent": {"cache_idle_ttl_seconds": 123}},
+        )
+        assert gw_run._resolve_agent_cache_idle_ttl_secs() == 123.0
+
     def test_idle_sweep_skips_agents_without_activity_ts(self, monkeypatch):
         """Agents missing _last_activity_ts are left alone (defensive)."""
         from gateway import run as gw_run
 
         monkeypatch.setattr(gw_run, "_AGENT_CACHE_IDLE_TTL_SECS", 0.01)
         runner = self._bounded_runner()
+        runner._agent_cache_idle_ttl_secs = 0.01
         runner._cleanup_agent_resources = MagicMock()
 
         no_ts = MagicMock(spec=[])  # no _last_activity_ts attribute
@@ -807,6 +866,7 @@ def test_idle_sweep_keeps_agent_when_session_not_expired(self, monkeypatch):
 
         monkeypatch.setattr(gw_run, "_AGENT_CACHE_IDLE_TTL_SECS", 0.01)
         runner = self._bounded_runner()
+        runner._agent_cache_idle_ttl_secs = 0.01
         runner._cleanup_agent_resources = MagicMock()
 
         import time as _t
@@ -833,6 +893,7 @@ def test_idle_sweep_evicts_when_session_is_expired(self, monkeypatch):
 
         monkeypatch.setattr(gw_run, "_AGENT_CACHE_IDLE_TTL_SECS", 0.01)
         runner = self._bounded_runner()
+        runner._agent_cache_idle_ttl_secs = 0.01
         runner._cleanup_agent_resources = MagicMock()
 
         import time as _t

website/docs/user-guide/configuration.md

@@ -838,13 +838,16 @@ Instead, when the budget is actually exhausted (90/90), Hermes injects one messa
 ```yaml
 agent:
   max_turns: 90                # Max iterations per conversation turn (default: 90)
+  cache_idle_ttl_seconds: 3600 # Evict idle cached gateway agents after 1h; 0 disables idle eviction
   api_max_retries: 3           # Retries per provider before fallback engages (default: 3)
 ```
 
 When the iteration budget is fully exhausted, the CLI shows a notification to the user: `⚠ Iteration budget reached (90/90) — response may be incomplete`.
 
 `agent.api_max_retries` controls how many times Hermes retries a provider API call on transient errors (rate limits, connection drops, 5xx) **before** fallback-provider switching engages. The default is `3` — four attempts total. If you have [fallback providers](/user-guide/features/fallback-providers) configured and want to fail over faster, drop this to `0` so the first transient error on your primary immediately hands off to the fallback instead of churning retries against the flaky endpoint.
 
+`agent.cache_idle_ttl_seconds` controls how long gateway sessions keep an idle in-memory agent between messages. The default is `3600` seconds. Set it higher for long-lived messaging threads that should preserve conversation context across breaks, or set it to `0` to disable idle eviction entirely. The hard cache size cap still applies, so disabling idle TTL does not make the cache unbounded.
+
 ## Standing Goals (`/goal`)
 
 When a standing goal is active, Hermes judges whether each assistant response satisfies it. If not, it feeds a continuation prompt back into the same session and keeps working until the goal is done, the turn budget is exhausted, or the user pauses/clears it. The turn budget is the real backstop — judge failures fail **open** (continue) so a flaky judge never wedges progress.