feat: auto-refresh MCP tools for slow servers via async late-binding

hermes_cli/cli_agent_setup_mixin.py

@@ -407,6 +407,27 @@ def _init_agent(self, *, model_override: str = None, runtime_override: dict = No
                 seed_credits_at_session_start(self.agent)
             except Exception:
                 pass
+            # Spawn background thread to auto-refresh MCP tools once slow
+            # servers finish connecting (lark ~8s, redis ~10s, ssh ~4s).
+            # Non-blocking -- agent starts immediately with whatever tools
+            # were ready; late arrivals are merged in automatically.
+            try:
+                from hermes_cli.mcp_startup import spawn_late_mcp_refresh
+                from model_tools import get_tool_definitions as _gtd
+                from cli import logger as _cli_logger
+
+                def _on_cli_mcp_refreshed(added: int, total: int) -> None:
+                    from cli import _cprint as _cp
+                    _cp(f'  MCP late refresh: {added} new tool(s) loaded ({total} total)')
+
+                spawn_late_mcp_refresh(
+                    agent=self.agent,
+                    logger=_cli_logger,
+                    get_tool_definitions_fn=_gtd,
+                    on_refreshed=_on_cli_mcp_refreshed,
+                )
+            except Exception:
+                pass
             self._active_agent_route_signature = (
                 effective_model,
                 runtime.get("provider"),

hermes_cli/mcp_startup.py

@@ -9,6 +9,12 @@
 _mcp_discovery_started = False
 _mcp_discovery_thread: Optional[threading.Thread] = None
 
+# Lock for serializing agent tool updates.  The late-refresh background
+# thread and the main thread both mutate ``agent.tools`` /
+# ``agent.valid_tool_names``; without a lock a concurrent read during
+# tool iteration could see a half-written list.
+_agent_tools_lock = threading.Lock()
+
 
 def _has_configured_mcp_servers() -> bool:
     """Cheap config probe so non-MCP users avoid importing the MCP stack."""
@@ -52,8 +58,141 @@ def _discover() -> None:
 
 
 def wait_for_mcp_discovery(timeout: float = 0.75) -> None:
-    """Briefly wait for background MCP discovery before the first tool snapshot."""
+    """Briefly wait for background MCP discovery before the first tool snapshot.
+
+    A short bounded wait lets fast servers land before the agent is built.
+    Slow servers (lark ~8s, redis ~10s, ssh ~4s) are handled by the
+    late-binding refresh thread that auto-merges their tools once discovery
+    completes (see ``spawn_late_mcp_refresh``).
+    """
     thread = _mcp_discovery_thread
     if thread is None or not thread.is_alive():
         return
     thread.join(timeout=timeout)
+
+
+# ── Late-binding MCP tool refresh ─────────────────────────────────────
+# After the agent is built (with whatever tools were ready at the time),
+# spawn a background thread that waits for MCP discovery to finish.  When
+# it does, check whether new tools appeared and auto-refresh the agent's
+# tool list.  This is the same logic as ``/reload-mcp`` but triggered
+# automatically so slow MCP servers (lark ~8s, redis ~10s) land in the
+# agent without blocking startup or requiring manual intervention.
+
+_mcp_late_refresh_thread: Optional[threading.Thread] = None
+
+# Default timeout (seconds) for waiting on MCP discovery to complete
+# before giving up on the late-refresh path.
+_LATE_REFRESH_DISCOVERY_TIMEOUT_S = 30.0
+
+
+def _update_agent_tools(agent, new_defs, new_tool_names, logger, on_refreshed, added):
+    """Thread-safe helper to swap agent tools in-place."""
+    with _agent_tools_lock:
+        agent.tools = new_defs
+        agent.valid_tool_names = new_tool_names
+
+    logger.info(
+        "MCP late refresh: %d new tool(s) added (%s)",
+        len(added),
+        ", ".join(sorted(added)[:5]) + ("..." if len(added) > 5 else ""),
+    )
+
+    if on_refreshed:
+        try:
+            on_refreshed(len(added), len(new_tool_names))
+        except Exception:
+            pass
+
+
+def spawn_late_mcp_refresh(
+    *,
+    agent,
+    logger,
+    get_tool_definitions_fn: callable,
+    on_refreshed: "callable | None" = None,
+) -> None:
+    """Spawn a background thread that auto-refreshes MCP tools once discovery finishes.
+
+    Args:
+        agent: The AIAgent instance whose ``.tools`` and ``.valid_tool_names``
+               will be updated in-place.
+        logger: Logger instance for debug/warning output.
+        get_tool_definitions_fn: A callable that returns the current tool
+                                 definitions list (e.g. ``model_tools.get_tool_definitions``).
+        on_refreshed: Optional callback invoked after tools are refreshed.
+                      Receives ``(added_count: int, total_count: int)``.
+    """
+    global _mcp_late_refresh_thread
+
+    # Only spawn if discovery is still running
+    thread = _mcp_discovery_thread
+    if thread is None or not thread.is_alive():
+        # Discovery thread is None or already finished.  The agent was built
+        # with whatever tools were ready at wait_for_mcp_discovery time.
+        # Slow servers may have connected since then — do one inline refresh
+        # check right now.
+        try:
+            with _agent_tools_lock:
+                current_tools = set()
+                if hasattr(agent, "tools") and agent.tools:
+                    current_tools = {t["function"]["name"] for t in agent.tools}
+            new_defs = get_tool_definitions_fn(quiet_mode=True)
+            new_tool_names = {t["function"]["name"] for t in new_defs} if new_defs else set()
+            added = new_tool_names - current_tools
+            if added:
+                _update_agent_tools(agent, new_defs, new_tool_names, logger, on_refreshed, added)
+        except Exception:
+            pass
+        return
+
+    # Avoid spawning multiple late-refresh threads (checked inside the lock
+    # so two concurrent callers can't both pass the guard).
+    with _mcp_discovery_lock:
+        if _mcp_late_refresh_thread is not None and _mcp_late_refresh_thread.is_alive():
+            return
+
+    def _refresh() -> None:
+        try:
+            # Wait for discovery to fully complete
+            discovery_thread = _mcp_discovery_thread
+            if discovery_thread is not None:
+                discovery_thread.join(timeout=_LATE_REFRESH_DISCOVERY_TIMEOUT_S)
+
+            if discovery_thread and discovery_thread.is_alive():
+                logger.debug(
+                    "MCP discovery still running after %.0fs, skipping late refresh",
+                    _LATE_REFRESH_DISCOVERY_TIMEOUT_S,
+                )
+                return
+
+            # Snapshot current tools (read under lock)
+            with _agent_tools_lock:
+                current_tools = set()
+                if hasattr(agent, "tools") and agent.tools:
+                    current_tools = {t["function"]["name"] for t in agent.tools}
+
+            # Get fresh tool definitions from the registry
+            new_defs = get_tool_definitions_fn(quiet_mode=True)
+            new_tool_names = {t["function"]["name"] for t in new_defs} if new_defs else set()
+
+            # Only update if new tools appeared
+            added = new_tool_names - current_tools
+            if not added:
+                logger.debug("MCP late refresh: no new tools discovered")
+                return
+
+            # Update agent in-place (same as /reload-mcp)
+            _update_agent_tools(agent, new_defs, new_tool_names, logger, on_refreshed, added)
+
+        except Exception as exc:
+            logger.debug("MCP late refresh failed: %s", exc)
+
+    with _mcp_discovery_lock:
+        late_thread = threading.Thread(
+            target=_refresh,
+            name="mcp-late-refresh",
+            daemon=True,
+        )
+        _mcp_late_refresh_thread = late_thread
+        late_thread.start()

tests/hermes_cli/test_mcp_startup.py

@@ -19,16 +19,22 @@
 def _reset_mcp_startup_state():
     saved_started = mcp_startup._mcp_discovery_started
     saved_thread = mcp_startup._mcp_discovery_thread
+    saved_late = getattr(mcp_startup, "_mcp_late_refresh_thread", None)
     try:
         mcp_startup._mcp_discovery_started = False
         mcp_startup._mcp_discovery_thread = None
+        mcp_startup._mcp_late_refresh_thread = None
         yield
     finally:
         thread = mcp_startup._mcp_discovery_thread
         if thread is not None and thread.is_alive():
             thread.join(timeout=1.0)
+        late = mcp_startup._mcp_late_refresh_thread
+        if late is not None and late.is_alive():
+            late.join(timeout=2.0)
         mcp_startup._mcp_discovery_started = saved_started
         mcp_startup._mcp_discovery_thread = saved_thread
+        mcp_startup._mcp_late_refresh_thread = saved_late
 
 
 def _agent_args(**overrides) -> Namespace:
@@ -164,3 +170,180 @@ def _fake_agent(*_a, **_k):
     monkeypatch.setattr(cli_mod, "AIAgent", _fake_agent)
 
     assert cli._init_agent() is True
+
+
+# ── spawn_late_mcp_refresh tests ──────────────────────────────────────
+
+
+class _FakeAgent:
+    """Minimal agent stub for late-refresh tests."""
+
+    def __init__(self, tools=None, valid_tool_names=None):
+        self.tools = tools or []
+        self.valid_tool_names = valid_tool_names or set()
+
+
+def _make_tool_def(name):
+    return {"function": {"name": name, "parameters": {}, "description": ""}}
+
+
+def test_late_refresh_inline_when_discovery_already_done():
+    """When discovery thread is None/finished, do an inline refresh check."""
+    agent = _FakeAgent(tools=[_make_tool_def("existing")], valid_tool_names={"existing"})
+    new_tools = [_make_tool_def("existing"), _make_tool_def("new_tool")]
+    callback_calls = []
+
+    mcp_startup._mcp_discovery_thread = None
+
+    mcp_startup.spawn_late_mcp_refresh(
+        agent=agent,
+        logger=types.SimpleNamespace(debug=lambda *a, **k: None, info=lambda *a, **k: None),
+        get_tool_definitions_fn=lambda quiet_mode=False: new_tools,
+        on_refreshed=lambda added, total: callback_calls.append((added, total)),
+    )
+
+    assert "new_tool" in agent.valid_tool_names
+    assert len(agent.tools) == 2
+    assert callback_calls == [(1, 2)]
+
+
+def test_late_refresh_inline_no_new_tools():
+    """When no new tools appeared, agent is not modified."""
+    agent = _FakeAgent(tools=[_make_tool_def("a")], valid_tool_names={"a"})
+    original_tools = agent.tools
+
+    mcp_startup._mcp_discovery_thread = None
+
+    mcp_startup.spawn_late_mcp_refresh(
+        agent=agent,
+        logger=types.SimpleNamespace(debug=lambda *a, **k: None, info=lambda *a, **k: None),
+        get_tool_definitions_fn=lambda quiet_mode=False: [_make_tool_def("a")],
+    )
+
+    assert agent.tools is original_tools  # not replaced
+
+
+def test_late_refresh_background_thread_adds_tools():
+    """When discovery is still running, a background thread waits and refreshes."""
+    stop = threading.Event()
+
+    def _slow_discover():
+        stop.wait(timeout=5)
+
+    discover_thread = threading.Thread(target=_slow_discover, daemon=True)
+    discover_thread.start()
+    mcp_startup._mcp_discovery_thread = discover_thread
+
+    agent = _FakeAgent(tools=[_make_tool_def("existing")], valid_tool_names={"existing"})
+    new_tools = [_make_tool_def("existing"), _make_tool_def("slow_tool")]
+    callback_calls = []
+    logged = []
+
+    def _log_info(msg, *args):
+        logged.append(msg % args if args else msg)
+
+    mcp_startup.spawn_late_mcp_refresh(
+        agent=agent,
+        logger=types.SimpleNamespace(
+            debug=lambda *a, **k: None,
+            info=_log_info,
+        ),
+        get_tool_definitions_fn=lambda quiet_mode=False: new_tools,
+        on_refreshed=lambda added, total: callback_calls.append((added, total)),
+    )
+
+    assert mcp_startup._mcp_late_refresh_thread is not None
+
+    # Let discovery finish
+    stop.set()
+    mcp_startup._mcp_late_refresh_thread.join(timeout=5)
+
+    assert "slow_tool" in agent.valid_tool_names
+    assert len(agent.tools) == 2
+    assert callback_calls == [(1, 2)]
+    assert any("slow_tool" in msg for msg in logged)
+
+
+def test_late_refresh_no_duplicate_threads():
+    """Calling spawn_late_mcp_refresh twice only creates one thread."""
+    stop = threading.Event()
+
+    def _slow_discover():
+        stop.wait(timeout=5)
+
+    discover_thread = threading.Thread(target=_slow_discover, daemon=True)
+    discover_thread.start()
+    mcp_startup._mcp_discovery_thread = discover_thread
+
+    agent = _FakeAgent()
+
+    mcp_startup.spawn_late_mcp_refresh(
+        agent=agent,
+        logger=types.SimpleNamespace(debug=lambda *a, **k: None, info=lambda *a, **k: None),
+        get_tool_definitions_fn=lambda quiet_mode=False: [],
+    )
+    first_thread = mcp_startup._mcp_late_refresh_thread
+
+    mcp_startup.spawn_late_mcp_refresh(
+        agent=agent,
+        logger=types.SimpleNamespace(debug=lambda *a, **k: None, info=lambda *a, **k: None),
+        get_tool_definitions_fn=lambda quiet_mode=False: [],
+    )
+    second_thread = mcp_startup._mcp_late_refresh_thread
+
+    assert first_thread is second_thread
+
+    stop.set()
+    first_thread.join(timeout=5)
+
+
+def test_late_refresh_background_timeout_skips():
+    """If discovery hangs beyond timeout, late refresh gives up."""
+    def _hang_forever():
+        time.sleep(999)
+
+    discover_thread = threading.Thread(target=_hang_forever, daemon=True)
+    discover_thread.start()
+    mcp_startup._mcp_discovery_thread = discover_thread
+
+    agent = _FakeAgent(tools=[_make_tool_def("a")], valid_tool_names={"a"})
+    logged = []
+    original_timeout = mcp_startup._LATE_REFRESH_DISCOVERY_TIMEOUT_S
+
+    try:
+        mcp_startup._LATE_REFRESH_DISCOVERY_TIMEOUT_S = 0.1  # 100ms for test speed
+
+        mcp_startup.spawn_late_mcp_refresh(
+            agent=agent,
+            logger=types.SimpleNamespace(
+                debug=lambda msg, *args: logged.append(msg % args if args else msg),
+                info=lambda *a, **k: None,
+            ),
+            get_tool_definitions_fn=lambda quiet_mode=False: [_make_tool_def("a"), _make_tool_def("b")],
+        )
+
+        mcp_startup._mcp_late_refresh_thread.join(timeout=5)
+
+        assert "b" not in agent.valid_tool_names  # not updated
+        assert any("skipping late refresh" in msg for msg in logged)
+    finally:
+        mcp_startup._LATE_REFRESH_DISCOVERY_TIMEOUT_S = original_timeout
+
+
+def test_late_refresh_callback_failure_is_swallowed():
+    """If on_refreshed raises, it doesn't crash the refresh thread."""
+    agent = _FakeAgent(tools=[_make_tool_def("old")], valid_tool_names={"old"})
+    mcp_startup._mcp_discovery_thread = None
+
+    def _bad_callback(added, total):
+        raise RuntimeError("boom")
+
+    # Should not raise
+    mcp_startup.spawn_late_mcp_refresh(
+        agent=agent,
+        logger=types.SimpleNamespace(debug=lambda *a, **k: None, info=lambda *a, **k: None),
+        get_tool_definitions_fn=lambda quiet_mode=False: [_make_tool_def("old"), _make_tool_def("new")],
+        on_refreshed=_bad_callback,
+    )
+
+    assert "new" in agent.valid_tool_names