Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions agent/agent_runtime_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -1554,11 +1554,16 @@ def invoke_tool(agent, function_name: str, function_args: dict, effective_task_i
its own inline invocation for backward-compatible display handling.
"""
# Check plugin hooks for a block directive before executing anything.
# Routed through the latency-aware wrapper so a slow pre_tool_call
# hook surfaces a WARNING in agent.log instead of producing a silent
# multi-second wall-clock gap (#32460).
block_message: Optional[str] = None
if not pre_tool_block_checked:
try:
from hermes_cli.plugins import get_pre_tool_call_block_message
block_message = get_pre_tool_call_block_message(
from agent.tool_dispatch_helpers import (
pre_tool_call_block_message_with_latency,
)
block_message = pre_tool_call_block_message_with_latency(
function_name, function_args, task_id=effective_task_id or "",
)
except Exception:
Expand Down
121 changes: 96 additions & 25 deletions agent/shell_hooks.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,18 @@
ALLOWLIST_FILENAME = "shell-hooks-allowlist.json"
_DEFAULT_BLOCK_MESSAGE = "Blocked by shell hook."

# Slow-hook watchdog (#32460). A ``pre_tool_call`` hook is on the hot
# path between an LLM response and tool execution, so a hook that hangs
# for the full ``DEFAULT_TIMEOUT_SECONDS = 60`` window manifests to the
# user as a silent 50-65s wall-clock gap with no log line at all — only
# a single ``shell hook timed out`` WARNING fires after the timer
# expires. These thresholds make the hang visible immediately: a
# WARNING is emitted while the hook is *still running* once it crosses
# the threshold, then repeated at the interval so the user can identify
# the offending script without waiting for the full timeout.
_SLOW_HOOK_THRESHOLD_SECONDS = 5.0
_SLOW_HOOK_REPEAT_SECONDS = 10.0

# (event, matcher, command) triples that have been wired to the plugin
# manager in the current process. Matcher is part of the key because
# the same script can legitimately register for different matchers under
Expand Down Expand Up @@ -361,6 +373,43 @@ def _parse_single_entry(
_TOP_LEVEL_PAYLOAD_KEYS = {"tool_name", "args", "session_id", "parent_session_id"}


def _watch_slow_hook(
spec: ShellHookSpec,
started_at: float,
stop_event: threading.Event,
) -> None:
"""Emit a WARNING while a shell hook is still running past the threshold.

Runs in a daemon thread spawned by :func:`_spawn`. The first warning
fires once the hook has been running for ``_SLOW_HOOK_THRESHOLD_SECONDS``
and is repeated every ``_SLOW_HOOK_REPEAT_SECONDS`` until the hook
returns or its own timeout elapses. Without this, a hook that hangs
for the full ``spec.timeout`` window manifests as a silent ~60s wall
clock gap between an LLM response and the resulting tool execution —
the symptom reported in #32460.
"""
if not stop_event.wait(_SLOW_HOOK_THRESHOLD_SECONDS):
# Hook still hasn't returned at the threshold — log once.
logger.warning(
"shell hook still running after %.1fs "
"(event=%s command=%s timeout=%ss); "
"this stalls every tool call dispatch until the hook returns",

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

_spawn() is shared by every shell-hook event, including pre_llm_call, so this wording is inaccurate outside pre_tool_call. Either limit this watchdog to pre-tool hooks or describe the specific event that is currently blocked.

time.monotonic() - started_at,
spec.event,
spec.command,
spec.timeout,
)
while not stop_event.wait(_SLOW_HOOK_REPEAT_SECONDS):
logger.warning(
"shell hook still running after %.1fs "
"(event=%s command=%s timeout=%ss)",
time.monotonic() - started_at,
spec.event,
spec.command,
spec.timeout,
)


def _spawn(spec: ShellHookSpec, stdin_json: str) -> Dict[str, Any]:
"""Run ``spec.command`` as a subprocess with ``stdin_json`` on stdin.

Expand All @@ -370,6 +419,11 @@ def _spawn(spec: ShellHookSpec, stdin_json: str) -> Dict[str, Any]:
subprocess is actually invoked — both the live callback path
(:func:`_make_callback`) and the CLI test helper (:func:`run_once`)
go through it.

While the subprocess is running, a watchdog thread emits a WARNING
if the hook crosses ``_SLOW_HOOK_THRESHOLD_SECONDS`` so a hanging
script becomes visible immediately instead of after the full
``spec.timeout`` window (see #32460).
"""
result: Dict[str, Any] = {
"returncode": None,
Expand All @@ -389,34 +443,51 @@ def _spawn(spec: ShellHookSpec, stdin_json: str) -> Dict[str, Any]:
return result

t0 = time.monotonic()
# Slow-hook watchdog (#32460) — fire a WARNING while the hook is
# still running once it crosses ``_SLOW_HOOK_THRESHOLD_SECONDS``, so
# a hanging script is surfaced immediately instead of after the full
# ``spec.timeout`` window. The watchdog is a daemon thread that
# stops as soon as the subprocess returns (success, timeout, or
# spawn error — all paths go through ``finally``).
_watchdog_stop = threading.Event()
_watchdog = threading.Thread(
target=_watch_slow_hook,
args=(spec, t0, _watchdog_stop),
daemon=True,
name=f"shell-hook-watchdog[{spec.event}]",
)
_watchdog.start()
try:
proc = subprocess.run(
argv,
input=stdin_json,
capture_output=True,
timeout=spec.timeout,
text=True,
shell=False,
)
except subprocess.TimeoutExpired:
result["timed_out"] = True
try:
proc = subprocess.run(
argv,
input=stdin_json,
capture_output=True,
timeout=spec.timeout,
text=True,
shell=False,
)
except subprocess.TimeoutExpired:
result["timed_out"] = True
result["elapsed_seconds"] = round(time.monotonic() - t0, 3)
return result
except FileNotFoundError:
result["error"] = "command not found"
return result
except PermissionError:
result["error"] = "command not executable"
return result
except Exception as exc: # pragma: no cover — defensive
result["error"] = str(exc)
return result

result["returncode"] = proc.returncode
result["stdout"] = proc.stdout or ""
result["stderr"] = proc.stderr or ""
result["elapsed_seconds"] = round(time.monotonic() - t0, 3)
return result
except FileNotFoundError:
result["error"] = "command not found"
return result
except PermissionError:
result["error"] = "command not executable"
return result
except Exception as exc: # pragma: no cover — defensive
result["error"] = str(exc)
return result

result["returncode"] = proc.returncode
result["stdout"] = proc.stdout or ""
result["stderr"] = proc.stderr or ""
result["elapsed_seconds"] = round(time.monotonic() - t0, 3)
return result
finally:
_watchdog_stop.set()


def _make_callback(spec: ShellHookSpec) -> Callable[..., Optional[Dict[str, Any]]]:
Expand Down
58 changes: 58 additions & 0 deletions agent/tool_dispatch_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import logging
import os
import re
import time
from pathlib import Path
from typing import Any, Dict, List, Optional

Expand All @@ -36,6 +37,63 @@

logger = logging.getLogger(__name__)

# Pre-tool dispatch latency threshold (#32460). The agent loop calls
# ``get_pre_tool_call_block_message()`` for every tool call before the
# tool starts measuring its own duration, so a slow plugin / shell hook
# here produces a silent wall-clock gap between an LLM response and the
# resulting tool execution. When the call exceeds this threshold, emit
# a single WARNING that names the tool — the operator no longer has to
# guess whether the agent was idle or one of their hooks was hung.
_PRE_TOOL_DISPATCH_SLOW_THRESHOLD_SECONDS = 2.0


def pre_tool_call_block_message_with_latency(
tool_name: str,
function_args: Optional[Dict[str, Any]],
*,
task_id: str = "",
session_id: str = "",
tool_call_id: str = "",
) -> Optional[str]:
"""Call ``get_pre_tool_call_block_message`` and warn on slow dispatch.

Equivalent to invoking
:func:`hermes_cli.plugins.get_pre_tool_call_block_message` directly,
but wraps the call in a ``time.monotonic`` window so dispatch latency
above ``_PRE_TOOL_DISPATCH_SLOW_THRESHOLD_SECONDS`` is reported via a
WARNING that names the offending tool.

The pre-tool dispatch path is silent in the agent log under normal
conditions, so a hook or plugin that blocks here for tens of seconds
appears to the user as the agent "freezing" between an LLM response
and the next tool execution — exactly the symptom reported in
#32460. Surfacing the latency immediately lets the operator chase
down the responsible hook / plugin without having to enable debug
logging or instrument by hand.
"""
from hermes_cli.plugins import get_pre_tool_call_block_message

t0 = time.monotonic()
try:
return get_pre_tool_call_block_message(

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Current main makes get_pre_tool_call_block_message() a deprecated block-only shim (hermes_cli/plugins.py:2201-2223); live dispatch uses resolve_pre_tool_block() to enforce approve directives fail-closed. Salvage this wrapper around resolve_pre_tool_block() and forward the full dispatch metadata, otherwise approval directives can be treated as allow.

tool_name,
function_args,
task_id=task_id,
session_id=session_id,
tool_call_id=tool_call_id,
)
finally:
elapsed = time.monotonic() - t0
if elapsed >= _PRE_TOOL_DISPATCH_SLOW_THRESHOLD_SECONDS:
logger.warning(
"pre_tool_call dispatch took %.1fs for tool %r — a slow "
"shell hook or plugin is stalling the agent loop. "
"Check ~/.hermes/config.yaml 'hooks.pre_tool_call' "
"entries and recently installed plugins.",
elapsed,
tool_name,
)

# Tools that must never run concurrently (interactive / user-facing).
# When any of these appear in a batch, we fall back to sequential execution.
_NEVER_PARALLEL_TOOLS = frozenset({"clarify"})
Expand Down
7 changes: 3 additions & 4 deletions agent/tool_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
_multimodal_text_summary,
_append_subdir_hint_to_multimodal,
make_tool_result_message,
pre_tool_call_block_message_with_latency,
)
from tools.terminal_tool import (
_get_approval_callback,
Expand Down Expand Up @@ -125,8 +126,7 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe
block_result = None
blocked_by_guardrail = False
try:
from hermes_cli.plugins import get_pre_tool_call_block_message
block_message = get_pre_tool_call_block_message(
block_message = pre_tool_call_block_message_with_latency(
function_name, function_args, task_id=effective_task_id or "",
)
except Exception:
Expand Down Expand Up @@ -500,8 +500,7 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe
# Check plugin hooks for a block directive before executing.
_block_msg: Optional[str] = None
try:
from hermes_cli.plugins import get_pre_tool_call_block_message
_block_msg = get_pre_tool_call_block_message(
_block_msg = pre_tool_call_block_message_with_latency(
function_name, function_args, task_id=effective_task_id or "",
)
except Exception:
Expand Down
9 changes: 7 additions & 2 deletions model_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -784,8 +784,13 @@ def handle_function_call(
if not skip_pre_tool_call_hook:
block_message: Optional[str] = None
try:
from hermes_cli.plugins import get_pre_tool_call_block_message
block_message = get_pre_tool_call_block_message(
# Routed through the latency-aware wrapper so a slow
# pre_tool_call hook surfaces a WARNING in agent.log
# instead of a silent wall-clock gap (#32460).
from agent.tool_dispatch_helpers import (
pre_tool_call_block_message_with_latency,
)
block_message = pre_tool_call_block_message_with_latency(
function_name,
function_args,
task_id=task_id or "",
Expand Down
Loading
Loading