From 3b910aaee7a5b56f38fab2e28f80513e87a33740 Mon Sep 17 00:00:00 2001 From: sjh6457 Date: Fri, 10 Apr 2026 19:27:21 -0400 Subject: [PATCH 1/3] feat(gateway): make still-working heartbeats configurable --- gateway/run.py | 111 +++++++++++++++-- hermes_cli/config.py | 2 + .../test_still_working_notifications.py | 112 ++++++++++++++++++ website/docs/user-guide/configuration.md | 16 +++ website/docs/user-guide/messaging/index.md | 14 +++ 5 files changed, 242 insertions(+), 13 deletions(-) create mode 100644 tests/gateway/test_still_working_notifications.py diff --git a/gateway/run.py b/gateway/run.py index 659ba8013697e..6c12dd3459a51 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -16,6 +16,7 @@ import asyncio import json import logging +import math import os import re import shlex @@ -1029,6 +1030,88 @@ def _load_background_notifications_mode() -> str: return "all" return mode + @staticmethod + def _load_still_working_interval(platform_key: str | None = None) -> float | None: + """Load per-platform still-working heartbeat interval from config. + + Returns seconds as a float, or ``None`` when disabled. + Config keys: + - ``display.still_working_interval`` (global default, seconds) + - ``display.still_working_overrides.`` (per-platform override) + + Values of ``0``, ``false``, or ``off`` disable the heartbeat. + Invalid global values fall back to the default 600 seconds. Invalid + per-platform overrides are ignored so they inherit the global setting. + """ + + def _coerce_interval(raw, *, key_name: str, invalid_fallback) -> float | None | object: + if raw is None or raw == "": + return _MISSING + if raw is False: + return None + if raw is True: + return invalid_fallback + text = str(raw).strip().lower() + if text in {"off", "false", "none"}: + return None + if text in {"on", "true"}: + return invalid_fallback + try: + value = float(raw) + except (TypeError, ValueError): + logger.warning( + "Unknown %s '%s', defaulting to %s", + key_name, + raw, + "inherit" if invalid_fallback is _MISSING else f"{invalid_fallback}s", + ) + return invalid_fallback + if math.isnan(value) or math.isinf(value): + logger.warning( + "Unknown %s '%s', defaulting to %s", + key_name, + raw, + "inherit" if invalid_fallback is _MISSING else f"{invalid_fallback}s", + ) + return invalid_fallback + if value <= 0: + return None + return value + + _MISSING = object() + try: + import yaml as _y + cfg_path = _hermes_home / "config.yaml" + if cfg_path.exists(): + with open(cfg_path, encoding="utf-8") as _f: + cfg = _y.safe_load(_f) or {} + display_cfg = cfg.get("display", {}) if isinstance(cfg, dict) else {} + else: + display_cfg = {} + except Exception: + display_cfg = {} + + raw_value = _coerce_interval( + display_cfg.get("still_working_interval"), + key_name="still_working_interval", + invalid_fallback=600.0, + ) + interval = 600.0 if raw_value is _MISSING else raw_value + + overrides = display_cfg.get("still_working_overrides", {}) + if not isinstance(overrides, dict): + overrides = {} + if platform_key: + override_value = _coerce_interval( + overrides.get(platform_key), + key_name=f"still_working_overrides.{platform_key}", + invalid_fallback=_MISSING, + ) + if override_value is not _MISSING: + interval = override_value + + return interval + @staticmethod def _load_provider_routing() -> dict: """Load OpenRouter provider routing preferences from config.yaml.""" @@ -7327,29 +7410,29 @@ async def monitor_for_interrupt(): interrupt_monitor = asyncio.create_task(monitor_for_interrupt()) # Periodic "still working" notifications for long-running tasks. - # Fires every 10 minutes so the user knows the agent hasn't died. - _NOTIFY_INTERVAL = 600 # 10 minutes + # Default every 10 minutes, configurable via display.still_working_interval + # with optional per-platform overrides in display.still_working_overrides. + _notify_interval = self._load_still_working_interval(platform_key) _notify_start = time.time() async def _notify_long_running(): _notify_adapter = self.adapters.get(source.platform) - if not _notify_adapter: + if not _notify_adapter or _notify_interval is None: return while True: - await asyncio.sleep(_NOTIFY_INTERVAL) + await asyncio.sleep(_notify_interval) _elapsed_mins = int((time.time() - _notify_start) // 60) # Include agent activity context if available. _agent_ref = agent_holder[0] _status_detail = "" if _agent_ref and hasattr(_agent_ref, "get_activity_summary"): try: - _a = _agent_ref.get_activity_summary() - _parts = [f"iteration {_a['api_call_count']}/{_a['max_iterations']}"] - if _a.get("current_tool"): - _parts.append(f"running: {_a['current_tool']}") - else: - _parts.append(_a.get("last_activity_desc", "")) - _status_detail = " — " + ", ".join(_parts) + _act = _agent_ref.get_activity_summary() + _cur_tool = _act.get("current_tool") + _iter_n = _act.get("api_call_count", 0) + _iter_max = _act.get("max_iterations", 0) + if _cur_tool: + _status_detail = f" — iteration {_iter_n}/{_iter_max}, running: {_cur_tool}" except Exception: pass try: @@ -7361,7 +7444,8 @@ async def _notify_long_running(): except Exception as _ne: logger.debug("Long-running notification error: %s", _ne) - _notify_task = asyncio.create_task(_notify_long_running()) + _notify_task = asyncio.create_task(_notify_long_running()) if _notify_interval is not None else None + try: # Run in thread pool to not block. Use an *inactivity*-based @@ -7602,7 +7686,8 @@ async def _notify_long_running(): if progress_task: progress_task.cancel() interrupt_monitor.cancel() - _notify_task.cancel() + if _notify_task: + _notify_task.cancel() # Wait for stream consumer to finish its final edit if stream_task: diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 24fc655a2f622..8be8f85604249 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -443,6 +443,8 @@ def _ensure_hermes_home_managed(home: Path): "skin": "default", "tool_progress_command": False, # Enable /verbose command in messaging gateway "tool_progress_overrides": {}, # Per-platform overrides: {"signal": "off", "telegram": "all"} + "still_working_interval": 600, # Seconds between long-running task heartbeats in the gateway (0/false/off disables) + "still_working_overrides": {}, # Per-platform interval overrides: {"signal": "off", "telegram": 300} "tool_preview_length": 0, # Max chars for tool call previews (0 = no limit, show full paths/commands) }, diff --git a/tests/gateway/test_still_working_notifications.py b/tests/gateway/test_still_working_notifications.py new file mode 100644 index 0000000000000..87bc942e9a25f --- /dev/null +++ b/tests/gateway/test_still_working_notifications.py @@ -0,0 +1,112 @@ +"""Tests for configurable gateway still-working notifications. + +The gateway emits periodic "Still working..." heartbeats for long-running agent +turns. These tests cover the config loader that controls the per-platform +interval or disables heartbeats entirely. +""" + +import pytest + +from gateway.run import GatewayRunner + + +class TestLoadStillWorkingInterval: + def test_defaults_to_600_seconds(self, monkeypatch, tmp_path): + import gateway.run as gw + + monkeypatch.setattr(gw, "_hermes_home", tmp_path) + assert GatewayRunner._load_still_working_interval("telegram") == 600.0 + + def test_reads_global_interval_from_config(self, monkeypatch, tmp_path): + (tmp_path / "config.yaml").write_text( + "display:\n still_working_interval: 300\n", + encoding="utf-8", + ) + import gateway.run as gw + + monkeypatch.setattr(gw, "_hermes_home", tmp_path) + assert GatewayRunner._load_still_working_interval("telegram") == 300.0 + + def test_platform_override_wins(self, monkeypatch, tmp_path): + (tmp_path / "config.yaml").write_text( + "display:\n" + " still_working_interval: 600\n" + " still_working_overrides:\n" + " signal: off\n" + " telegram: 120\n", + encoding="utf-8", + ) + import gateway.run as gw + + monkeypatch.setattr(gw, "_hermes_home", tmp_path) + assert GatewayRunner._load_still_working_interval("signal") is None + assert GatewayRunner._load_still_working_interval("telegram") == 120.0 + + @pytest.mark.parametrize( + "raw_yaml", + [ + "display:\n still_working_interval: 0\n", + "display:\n still_working_interval: false\n", + "display:\n still_working_interval: off\n", + ], + ) + def test_zero_false_or_off_disables_globally(self, monkeypatch, tmp_path, raw_yaml): + (tmp_path / "config.yaml").write_text(raw_yaml, encoding="utf-8") + import gateway.run as gw + + monkeypatch.setattr(gw, "_hermes_home", tmp_path) + assert GatewayRunner._load_still_working_interval("telegram") is None + + def test_invalid_value_defaults_to_600_seconds(self, monkeypatch, tmp_path): + (tmp_path / "config.yaml").write_text( + "display:\n still_working_interval: banana\n", + encoding="utf-8", + ) + import gateway.run as gw + + monkeypatch.setattr(gw, "_hermes_home", tmp_path) + assert GatewayRunner._load_still_working_interval("telegram") == 600.0 + + def test_invalid_platform_override_inherits_global_interval(self, monkeypatch, tmp_path): + (tmp_path / "config.yaml").write_text( + "display:\n" + " still_working_interval: 300\n" + " still_working_overrides:\n" + " signal: banana\n", + encoding="utf-8", + ) + import gateway.run as gw + + monkeypatch.setattr(gw, "_hermes_home", tmp_path) + assert GatewayRunner._load_still_working_interval("signal") == 300.0 + + @pytest.mark.parametrize( + "raw_yaml", + [ + "display:\n still_working_interval: true\n", + "display:\n still_working_interval: on\n", + ], + ) + def test_true_or_on_global_value_maps_to_default_interval(self, monkeypatch, tmp_path, raw_yaml): + (tmp_path / "config.yaml").write_text(raw_yaml, encoding="utf-8") + import gateway.run as gw + + monkeypatch.setattr(gw, "_hermes_home", tmp_path) + assert GatewayRunner._load_still_working_interval("telegram") == 600.0 + + @pytest.mark.parametrize( + "raw_value", + ["true", "on"], + ) + def test_true_or_on_platform_override_inherits_global_interval(self, monkeypatch, tmp_path, raw_value): + (tmp_path / "config.yaml").write_text( + "display:\n" + " still_working_interval: 300\n" + " still_working_overrides:\n" + f" signal: {raw_value}\n", + encoding="utf-8", + ) + import gateway.run as gw + + monkeypatch.setattr(gw, "_hermes_home", tmp_path) + assert GatewayRunner._load_still_working_interval("signal") == 300.0 diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index 6c52645e190f9..f5215f46b4c0a 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -824,6 +824,8 @@ display: tool_progress: all # off | new | all | verbose tool_progress_command: false # Enable /verbose slash command in messaging gateway tool_progress_overrides: {} # Per-platform overrides (see below) + still_working_interval: 600 # Seconds between gateway long-running heartbeats (0/false/off disables) + still_working_overrides: {} # Per-platform interval overrides (see below) skin: default # Built-in or custom CLI skin (see user-guide/features/skins) personality: "kawaii" # Legacy cosmetic field still surfaced in some summaries compact: false # Compact output mode (less whitespace) @@ -859,6 +861,20 @@ display: Platforms without an override fall back to the global `tool_progress` value. Valid platform keys: `telegram`, `discord`, `slack`, `signal`, `whatsapp`, `matrix`, `mattermost`, `email`, `sms`, `homeassistant`, `dingtalk`, `feishu`, `wecom`, `weixin`, `bluebubbles`. +### Long-running heartbeat interval + +The gateway sends a periodic `⏳ Still working...` message during very long agent turns so users know the request has not died. Control the global interval with `display.still_working_interval` (in seconds), and override it per platform with `display.still_working_overrides`: + +```yaml +display: + still_working_interval: 600 # default: every 10 minutes + still_working_overrides: + signal: off # disable noisy heartbeats on Signal + telegram: 300 # every 5 minutes on Telegram +``` + +Values of `0`, `false`, or `off` disable the heartbeat. Platforms without an override use the global interval. + ## Privacy ```yaml diff --git a/website/docs/user-guide/messaging/index.md b/website/docs/user-guide/messaging/index.md index 6ae559ab79935..421cabe541209 100644 --- a/website/docs/user-guide/messaging/index.md +++ b/website/docs/user-guide/messaging/index.md @@ -276,6 +276,20 @@ You can also set this via environment variable: HERMES_BACKGROUND_NOTIFICATIONS=result ``` +### Long-running agent heartbeats + +When an agent turn itself runs for a long time (for example deep research, delegation, or many tool calls), the gateway can send periodic `⏳ Still working...` heartbeats. Control this with `display.still_working_interval`: + +```yaml +display: + still_working_interval: 600 # default: every 10 minutes + still_working_overrides: + signal: off # disable on Signal + telegram: 300 # more frequent on Telegram +``` + +Values of `0`, `false`, or `off` disable the heartbeat. + ### Use Cases - **Server monitoring** — "/background Check the health of all services and alert me if anything is down" From 4694c21417541dad59330ceea1cde24b055dfe8f Mon Sep 17 00:00:00 2001 From: sjh6457 Date: Fri, 10 Apr 2026 21:04:50 -0400 Subject: [PATCH 2/3] fix(gateway): preserve still-working activity detail --- gateway/run.py | 8 +- .../test_still_working_notifications.py | 146 +++++++++++++++++- 2 files changed, 151 insertions(+), 3 deletions(-) diff --git a/gateway/run.py b/gateway/run.py index 6c12dd3459a51..4482db1f63732 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -7431,8 +7431,14 @@ async def _notify_long_running(): _cur_tool = _act.get("current_tool") _iter_n = _act.get("api_call_count", 0) _iter_max = _act.get("max_iterations", 0) + _parts = [f"iteration {_iter_n}/{_iter_max}"] if _cur_tool: - _status_detail = f" — iteration {_iter_n}/{_iter_max}, running: {_cur_tool}" + _parts.append(f"running: {_cur_tool}") + else: + _last_desc = _act.get("last_activity_desc") + if _last_desc: + _parts.append(_last_desc) + _status_detail = " — " + ", ".join(_parts) except Exception: pass try: diff --git a/tests/gateway/test_still_working_notifications.py b/tests/gateway/test_still_working_notifications.py index 87bc942e9a25f..edefb4b1376fe 100644 --- a/tests/gateway/test_still_working_notifications.py +++ b/tests/gateway/test_still_working_notifications.py @@ -1,13 +1,21 @@ """Tests for configurable gateway still-working notifications. The gateway emits periodic "Still working..." heartbeats for long-running agent -turns. These tests cover the config loader that controls the per-platform -interval or disables heartbeats entirely. +turns. These tests cover both the config loader and the runtime heartbeat +messages that _run_agent emits. """ +import sys +import time +import types +from types import SimpleNamespace + import pytest +from gateway.config import Platform, PlatformConfig +from gateway.platforms.base import BasePlatformAdapter, SendResult from gateway.run import GatewayRunner +from gateway.session import SessionSource class TestLoadStillWorkingInterval: @@ -110,3 +118,137 @@ def test_true_or_on_platform_override_inherits_global_interval(self, monkeypatch monkeypatch.setattr(gw, "_hermes_home", tmp_path) assert GatewayRunner._load_still_working_interval("signal") == 300.0 + + +class _HeartbeatCaptureAdapter(BasePlatformAdapter): + def __init__(self, platform=Platform.TELEGRAM): + super().__init__(PlatformConfig(enabled=True, token="***"), platform) + self.sent = [] + + async def connect(self) -> bool: + return True + + async def disconnect(self) -> None: + return None + + async def send(self, chat_id, content, reply_to=None, metadata=None) -> SendResult: + self.sent.append( + { + "chat_id": chat_id, + "content": content, + "reply_to": reply_to, + "metadata": metadata, + } + ) + return SendResult(success=True, message_id="heartbeat-1") + + async def edit_message(self, chat_id, message_id, content) -> SendResult: + return SendResult(success=True, message_id=message_id) + + async def send_typing(self, chat_id, metadata=None) -> None: + return None + + async def get_chat_info(self, chat_id: str): + return {"id": chat_id} + + +def _make_runner(adapter): + import gateway.run as gateway_run + + runner = object.__new__(gateway_run.GatewayRunner) + runner.adapters = {adapter.platform: adapter} + runner._voice_mode = {} + runner._prefill_messages = [] + runner._ephemeral_system_prompt = "" + runner._reasoning_config = None + runner._provider_routing = {} + runner._fallback_model = None + runner._session_db = None + runner._running_agents = {} + runner._session_model_overrides = {} + runner.hooks = SimpleNamespace(loaded_hooks=False) + return runner + + +def _install_fake_run_agent(monkeypatch, *, activity_summary, run_duration=0.05): + class _FakeStillWorkingAgent: + def __init__(self, **kwargs): + self._activity_summary = dict(activity_summary) + + def get_activity_summary(self): + return dict(self._activity_summary) + + def run_conversation(self, message, conversation_history=None, task_id=None): + time.sleep(run_duration) + return {"final_response": "done", "messages": [], "api_calls": 1} + + fake_run_agent = types.ModuleType("run_agent") + fake_run_agent.AIAgent = _FakeStillWorkingAgent + monkeypatch.setitem(sys.modules, "run_agent", fake_run_agent) + + +async def _run_agent_with_heartbeat(monkeypatch, tmp_path, config_text, *, activity_summary, platform=Platform.TELEGRAM): + (tmp_path / "config.yaml").write_text(config_text, encoding="utf-8") + + import gateway.run as gateway_run + + monkeypatch.setattr(gateway_run, "_hermes_home", tmp_path) + monkeypatch.setattr(gateway_run, "_resolve_runtime_agent_kwargs", lambda: {"api_key": "***"}) + monkeypatch.setenv("HERMES_TOOL_PROGRESS_MODE", "off") + monkeypatch.setenv("HERMES_AGENT_TIMEOUT", "0") + _install_fake_run_agent(monkeypatch, activity_summary=activity_summary) + + adapter = _HeartbeatCaptureAdapter(platform=platform) + runner = _make_runner(adapter) + source = SessionSource(platform=platform, chat_id="123", chat_type="dm") + result = await runner._run_agent( + message="hello", + context_prompt="", + history=[], + source=source, + session_id="sess-1", + session_key=f"agent:main:{platform.value}:dm:123", + ) + return adapter, result + + +class TestStillWorkingRuntime: + @pytest.mark.asyncio + async def test_runtime_heartbeat_preserves_last_activity_context(self, monkeypatch, tmp_path): + adapter, result = await _run_agent_with_heartbeat( + monkeypatch, + tmp_path, + "display:\n still_working_interval: 0.01\n", + activity_summary={ + "current_tool": None, + "last_activity_desc": "waiting on provider response", + "api_call_count": 2, + "max_iterations": 90, + "seconds_since_activity": 0.0, + }, + ) + + assert result["final_response"] == "done" + assert adapter.sent + assert any( + "iteration 2/90" in msg["content"] and "waiting on provider response" in msg["content"] + for msg in adapter.sent + ) + + @pytest.mark.asyncio + async def test_runtime_heartbeat_disabled_emits_no_messages(self, monkeypatch, tmp_path): + adapter, result = await _run_agent_with_heartbeat( + monkeypatch, + tmp_path, + "display:\n still_working_interval: off\n", + activity_summary={ + "current_tool": "terminal", + "last_activity_desc": "tool_call", + "api_call_count": 1, + "max_iterations": 90, + "seconds_since_activity": 0.0, + }, + ) + + assert result["final_response"] == "done" + assert adapter.sent == [] From 68798df259d0ac43e37e6072cb8ff6fdb7235ea3 Mon Sep 17 00:00:00 2001 From: sjh6457 Date: Sat, 18 Apr 2026 07:12:12 -0400 Subject: [PATCH 3/3] test: sync config migration assertion after main merge --- tests/hermes_cli/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/hermes_cli/test_config.py b/tests/hermes_cli/test_config.py index 1f4538346cf93..7bf20b2f877ca 100644 --- a/tests/hermes_cli/test_config.py +++ b/tests/hermes_cli/test_config.py @@ -639,7 +639,7 @@ def test_migrate_to_v17_also_moves_legacy_compression_summary_settings(self, tmp migrate_config(interactive=False, quiet=True) raw = yaml.safe_load(config_path.read_text(encoding="utf-8")) - assert raw["_config_version"] == 18 + assert raw["_config_version"] == DEFAULT_CONFIG["_config_version"] assert raw["display"]["still_working_interval"] == 600 assert raw["display"]["still_working_overrides"] == {} assert raw["compression"]["target_ratio"] == 0.2