Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
151 changes: 151 additions & 0 deletions run_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,9 @@
)


DOOM_LOOP_THRESHOLD = 3


class IterationBudget:
"""Thread-safe shared iteration counter for parent and child agents.

Expand Down Expand Up @@ -263,6 +266,8 @@ def __init__(
self.clarify_callback = clarify_callback
self.step_callback = step_callback
self._last_reported_tool = None # Track for "new tool" mode
self._doom_loop_signature = None
self._doom_loop_count = 0

# Interrupt mechanism for breaking out of tool loops
self._interrupt_requested = False
Expand Down Expand Up @@ -2700,6 +2705,8 @@ def _compress_context(self, messages: list, system_message: str, *, approx_token

def _execute_tool_calls(self, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0) -> None:
"""Execute tool calls from the assistant message and append results to messages."""
blocked_doom_loop_signature: Optional[tuple[str, str]] = None
deferred_doom_loop_recovery_msg: Optional[str] = None
for i, tool_call in enumerate(assistant_message.tool_calls, 1):
# SAFETY: check interrupt BEFORE starting each tool.
# If the user sent "stop" during a previous tool's execution,
Expand Down Expand Up @@ -2734,6 +2741,68 @@ def _execute_tool_calls(self, assistant_message, messages: list, effective_task_
if not isinstance(function_args, dict):
function_args = {}

doom_loop_signature = self._make_doom_loop_signature(function_name, function_args)
if blocked_doom_loop_signature is not None and doom_loop_signature == blocked_doom_loop_signature:
skip_content = (
f"[Tool execution skipped - doom loop detected: '{function_name}' "
"was requested again with the same arguments in this turn]"
)
messages.append(
{
"role": "tool",
"content": skip_content,
"tool_call_id": tool_call.id,
}
)
continue

if self._should_trigger_doom_loop(doom_loop_signature):
decision = self._handle_doom_loop(tool_name=function_name)
if decision == "continue":
pass
elif decision == "stop":
skip_content = (
f"[Tool execution skipped - doom loop detected: '{function_name}' was about to be called "
f"{DOOM_LOOP_THRESHOLD} times in a row with the same arguments]"
)
for skipped_tc in assistant_message.tool_calls[i - 1:]:
messages.append(
{
"role": "tool",
"content": skip_content,
"tool_call_id": skipped_tc.id,
}
)
deferred_doom_loop_recovery_msg = (
f"A doom loop was detected on the '{function_name}' tool, and the user chose to stop. "
"Do not call more tools. Provide a final response summarizing what you found so far."
)
self._doom_loop_signature = None
self._doom_loop_count = 0
break
else:
skip_content = (
f"[Tool execution skipped - doom loop detected: '{function_name}' was about to be called "
f"{DOOM_LOOP_THRESHOLD} times in a row with the same arguments]"
)
messages.append(
{
"role": "tool",
"content": skip_content,
"tool_call_id": tool_call.id,
}
)
blocked_doom_loop_signature = doom_loop_signature
deferred_doom_loop_recovery_msg = (
f"A doom loop was detected on the '{function_name}' tool: you attempted the same call "
f"with the same arguments {DOOM_LOOP_THRESHOLD} times in a row. "
"Do NOT repeat that exact tool call again. Try a different approach, use different "
"arguments, use another tool, or provide a final response if you are blocked."
)
self._doom_loop_signature = None
self._doom_loop_count = 0
continue

if not self.quiet_mode:
args_str = json.dumps(function_args, ensure_ascii=False)
args_preview = args_str[:self.log_prefix_chars] + "..." if len(args_str) > self.log_prefix_chars else args_str
Expand Down Expand Up @@ -2898,6 +2967,7 @@ def _execute_tool_calls(self, assistant_message, messages: list, effective_task_
# Log tool errors to the persistent error log so [error] tags
# in the UI always have a corresponding detailed entry on disk.
_is_error_result, _ = _detect_tool_failure(function_name, function_result)
self._record_tool_call_result(doom_loop_signature, _is_error_result)
if _is_error_result:
logger.warning("Tool %s returned error (%.2fs): %s", function_name, tool_duration, result_preview)

Expand Down Expand Up @@ -2945,6 +3015,9 @@ def _execute_tool_calls(self, assistant_message, messages: list, effective_task_
if self.tool_delay > 0 and i < len(assistant_message.tool_calls):
time.sleep(self.tool_delay)

if deferred_doom_loop_recovery_msg:
messages.append({"role": "user", "content": deferred_doom_loop_recovery_msg})

# ── Budget pressure injection ─────────────────────────────────
# After all tool calls in this turn are processed, check if we're
# approaching max_iterations. If so, inject a warning into the LAST
Expand Down Expand Up @@ -2990,6 +3063,82 @@ def _get_budget_warning(self, api_call_count: int) -> Optional[str]:
)
return None

def _make_doom_loop_signature(
self, tool_name: str, function_args: dict
) -> Optional[tuple[str, str]]:
if self._is_doom_loop_exempt(tool_name, function_args):
return None

normalized_args = json.dumps(
function_args,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
)
args_hash = hashlib.md5(normalized_args.encode("utf-8")).hexdigest()
return tool_name, args_hash

@staticmethod
def _is_doom_loop_exempt(tool_name: str, function_args: dict) -> bool:
if tool_name != "process":
return False

action = str(function_args.get("action", "")).lower()
return action in {"poll", "wait", "log"}

def _should_trigger_doom_loop(self, signature: Optional[tuple[str, str]]) -> bool:
return (
signature is not None
and self._doom_loop_signature == signature
and self._doom_loop_count >= DOOM_LOOP_THRESHOLD - 1
)

def _record_tool_call_result(
self, signature: Optional[tuple[str, str]], _is_error_result: bool
) -> None:
if signature is None:
self._doom_loop_signature = None
self._doom_loop_count = 0
return

if self._doom_loop_signature == signature:
self._doom_loop_count += 1
return

self._doom_loop_signature = signature
self._doom_loop_count = 1

def _handle_doom_loop(self, tool_name: str) -> str:
logger.warning(
"Doom loop detected for %s after %s identical calls",
tool_name,
DOOM_LOOP_THRESHOLD,
)

question = (
f"Hermes detected a doom loop: the agent has tried the same '{tool_name}' "
f"tool call with the same arguments {DOOM_LOOP_THRESHOLD} times in a row. "
"What should it do?"
)
response = ""
if self.platform == "cli" and self.clarify_callback is not None:
try:
response = str(
self.clarify_callback(
question,
["Continue anyway", "Try different approach", "Stop and summarize"],
)
).strip()
except Exception as exc:
logging.warning(f"Doom loop clarify callback failed: {exc}")

normalized_response = response.lower()
if "continue" in normalized_response:
return "continue"
if "stop" in normalized_response:
return "stop"
return "reroute"

def _handle_max_iterations(self, messages: list, api_call_count: int) -> str:
"""Request a summary when max iterations are reached. Returns the final response text."""
print(f"⚠️ Reached maximum iterations ({self.max_iterations}). Requesting summary...")
Expand Down Expand Up @@ -3147,6 +3296,8 @@ def run_conversation(
self._invalid_tool_retries = 0
self._invalid_json_retries = 0
self._empty_content_retries = 0
self._doom_loop_signature = None
self._doom_loop_count = 0
self._incomplete_scratchpad_retries = 0
self._codex_incomplete_retries = 0
self._last_content_with_tools = None
Expand Down
86 changes: 86 additions & 0 deletions tests/test_run_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -648,6 +648,74 @@ def test_result_truncation_over_100k(self, agent):
assert len(messages[0]["content"]) < 150_000
assert "Truncated" in messages[0]["content"]

def test_doom_loop_stops_third_identical_call(self, agent):
t1 = _mock_tool_call(name="web_search", arguments='{"q":"same"}', call_id="c1")
t2 = _mock_tool_call(name="web_search", arguments='{"q":"same"}', call_id="c2")
t3 = _mock_tool_call(name="web_search", arguments='{"q":"same"}', call_id="c3")
mock_msg = _mock_assistant_msg(content="", tool_calls=[t1, t2, t3])
messages = []

with patch("run_agent.handle_function_call", return_value="search result") as mock_hfc:
agent._execute_tool_calls(mock_msg, messages, "task-1")

assert mock_hfc.call_count == 2
assert len(messages) == 4
assert messages[2]["role"] == "tool"
assert "doom loop detected" in messages[2]["content"].lower()
assert messages[3]["role"] == "user"
assert "Do NOT repeat that exact tool call" in messages[3]["content"]

def test_doom_loop_continue_option_allows_execution(self, agent):
t1 = _mock_tool_call(name="web_search", arguments='{"q":"same"}', call_id="c1")
t2 = _mock_tool_call(name="web_search", arguments='{"q":"same"}', call_id="c2")
t3 = _mock_tool_call(name="web_search", arguments='{"q":"same"}', call_id="c3")
mock_msg = _mock_assistant_msg(content="", tool_calls=[t1, t2, t3])
messages = []
agent.platform = "cli"
agent.clarify_callback = lambda _q, _choices: "Continue anyway"

with patch("run_agent.handle_function_call", return_value="search result") as mock_hfc:
agent._execute_tool_calls(mock_msg, messages, "task-1")

assert mock_hfc.call_count == 3
assert len(messages) == 3
assert all(msg["role"] == "tool" for msg in messages)

def test_doom_loop_exempts_process_poll_wait_log(self, agent):
poll_calls = [
_mock_tool_call(name="process", arguments='{"action":"poll","id":"p1"}', call_id="c1"),
_mock_tool_call(name="process", arguments='{"action":"poll","id":"p1"}', call_id="c2"),
_mock_tool_call(name="process", arguments='{"action":"poll","id":"p1"}', call_id="c3"),
_mock_tool_call(name="process", arguments='{"action":"poll","id":"p1"}', call_id="c4"),
]
mock_msg = _mock_assistant_msg(content="", tool_calls=poll_calls)
messages = []

with patch("run_agent.handle_function_call", return_value="ok") as mock_hfc:
agent._execute_tool_calls(mock_msg, messages, "task-1")

assert mock_hfc.call_count == 4
assert len(messages) == 4

def test_doom_loop_does_not_skip_unrelated_remaining_calls(self, agent):
t1 = _mock_tool_call(name="web_search", arguments='{"q":"same"}', call_id="c1")
t2 = _mock_tool_call(name="web_search", arguments='{"q":"same"}', call_id="c2")
t3 = _mock_tool_call(name="web_search", arguments='{"q":"same"}', call_id="c3")
t4 = _mock_tool_call(name="read_file", arguments='{"path":"run_agent.py"}', call_id="c4")
mock_msg = _mock_assistant_msg(content="", tool_calls=[t1, t2, t3, t4])
messages = []

with patch("run_agent.handle_function_call", return_value="ok") as mock_hfc:
agent._execute_tool_calls(mock_msg, messages, "task-1")

called_tools = [call.args[0] for call in mock_hfc.call_args_list]
assert called_tools == ["web_search", "web_search", "read_file"]
assert any(
msg["role"] == "tool" and "doom loop detected" in msg["content"].lower()
for msg in messages
)
assert messages[-1]["role"] == "user"


class TestHandleMaxIterations:
def test_returns_summary(self, agent):
Expand Down Expand Up @@ -714,6 +782,24 @@ def test_tool_calls_then_stop(self, agent):
assert result["final_response"] == "Done searching"
assert result["api_calls"] == 2

def test_resets_doom_loop_state_each_turn(self, agent):
self._setup_agent(agent)
agent._doom_loop_signature = ("web_search", "abc")
agent._doom_loop_count = 99
resp = _mock_response(content="Final answer", finish_reason="stop")
agent.client.chat.completions.create.return_value = resp

with (
patch.object(agent, "_persist_session"),
patch.object(agent, "_save_trajectory"),
patch.object(agent, "_cleanup_task_resources"),
):
result = agent.run_conversation("hello")

assert result["completed"] is True
assert agent._doom_loop_signature is None
assert agent._doom_loop_count == 0

def test_interrupt_breaks_loop(self, agent):
self._setup_agent(agent)

Expand Down
Loading