From d36b447798d83ce4266dd257e11b723dba7fcdcd Mon Sep 17 00:00:00 2001 From: Omar Baradei Date: Sat, 30 May 2026 16:57:40 -0700 Subject: [PATCH 01/13] feat(agent): retry stalled no-tool-call turns on a higher-quality lane dflash (Qwen3.6-27B Q4_K_M, lucebox spec-decode) sometimes ends an agentic decision turn with EOS right after a short action preamble ("Let me check X:") and NO tool_call, stalling the loop. Higher-precision weights (the stock qwen3.6-27b-256k lane on the same host) continue to a real tool call on the identical prompt. This adds agent/stall_retry.py: when a no-tool-call turn looks like that stall (short, announces an action, not a genuine completion) and HERMES_STALL_RETRY_MODEL is set, re-issue the SAME turn once on that lane; if it yields tool calls, adopt it and continue. Same provider/endpoint so only the model name is overridden (no client rebuild). Fires at most once per conversation. No-op unless the env is set, so default behavior is unchanged. Validated: detector 17/17 on real captured stall contents, 0 false positives on genuine completions; live retry recovered real stalls into real tool calls (terminal/execute_code) against the live endpoint; runs correctly under `hermes -z` (probe-confirmed, env propagated). --- agent/conversation_loop.py | 33 +++++++++- agent/stall_retry.py | 127 +++++++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 agent/stall_retry.py diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 743988b03b0fe..5beda5fe1fd18 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -793,6 +793,11 @@ def run_conversation( should_review_memory=_should_review_memory, ) + # Agentic stall-retry guard: ensures the HERMES_STALL_RETRY_MODEL retry + # fires at most once per conversation (avoids loops if the retry lane also + # stalls). See agent/stall_retry.py. + _stall_retry_used = False + while (api_call_count < agent.max_iterations and agent.iteration_budget.remaining > 0) or agent._budget_grace_call: # Reset per-turn checkpoint dedup so each iteration can take one snapshot agent._checkpoint_mgr.new_turn() @@ -3983,7 +3988,33 @@ def _stop_spinner(): else: # No tool calls - this is the final response final_response = assistant_message.content or "" - + + # ── Agentic stall-retry (opt-in via HERMES_STALL_RETRY_MODEL) ── + # dflash Q4 sometimes emits EOS right after an action preamble + # ("Let me check X:") with no tool_call, ending the turn early + # and stalling the agent mid-task. If this no-tool-call turn + # looks like that stall (not a genuine final answer) and a retry + # lane is configured, re-issue the SAME turn once on a + # higher-quality model; if it yields tool calls, adopt it and + # continue the loop instead of stopping. No-op unless the env is + # set, so default behavior is unchanged. + if not _stall_retry_used: + try: + from agent.stall_retry import looks_like_stall, retry_on_stall + if looks_like_stall( + final_response, finish_reason, + bool(getattr(assistant_message, "tool_calls", None)), + int(os.environ.get("HERMES_STALL_RETRY_MAX_CHARS", "400") or 400), + ): + _retried = retry_on_stall(agent, api_messages, finish_reason) + if _retried is not None and getattr(_retried, "tool_calls", None): + _stall_retry_used = True + assistant_message = _retried + finish_reason = "tool_calls" + continue # re-enter loop top; tool-calls path handles it + except Exception: + pass # any failure: keep the original response + # Fix: unmute output when entering the no-tool-call branch # so the user can see empty-response warnings and recovery # status messages. _mute_post_response was set during a diff --git a/agent/stall_retry.py b/agent/stall_retry.py new file mode 100644 index 0000000000000..06edb6ba0b784 --- /dev/null +++ b/agent/stall_retry.py @@ -0,0 +1,127 @@ +""" +Agentic stall-retry (dflash Q4 premature-EOS workaround). + +dflash (Qwen3.6-27B Q4_K_M, lucebox spec-decode) sometimes emits EOS right +after a short action preamble ("Let me check X:") on agentic decision turns, +ending the turn with NO tool_call -> the agent loop treats it as a final +answer and stops mid-task. Higher-precision weights (the stock Q6 lane on the +same host) continue to a real tool call on the identical prompt. + +This module detects that stall signature on a no-tool-call turn and retries +the SAME turn once against a higher-quality model lane. If the retry produces +tool_calls, the loop adopts that response and continues; otherwise the +original response stands (no behavior change). + +Entirely opt-in: does nothing unless ``HERMES_STALL_RETRY_MODEL`` is set +(e.g. ``qwen3.6-27b-256k``). Default-off => zero change to existing behavior. + +Env: + HERMES_STALL_RETRY_MODEL retry lane/model name (required to enable) + HERMES_STALL_RETRY_MAX_CHARS max content length to still count as a stall + (default 400; real final answers are longer) +""" +from __future__ import annotations + +import os +import re + +# Action-preamble signature: the turn announced an action but produced no tool +# call. These end mid-thought, typically with a colon, or open with intent. +_ACTION_RE = re.compile( + r"(let me\b|let's\b|i'?ll\b|i will\b|i'?m going to\b|i am going to\b|" + r"now i\b|first,?\s+i\b|next,?\s+i\b|i need to\b|i should\b|" + r"going to (check|look|run|start|examine|search|read|list|create|write|edit|use))", + re.IGNORECASE, +) +# Genuine completion signature: the model declared it is done / nothing to do. +# These must NOT be retried (they are correct no-tool-call turns). +_COMPLETION_RE = re.compile( + r"(\bdone\b|\bcomplete(d)?\b|nothing to (do|save|change|report|fix)|" + r"no changes?\b|no action\b|already (complete|done|finished)|\bfinished\b|" + r"all set\b|no further\b|nothing left\b|here('?s| is| are)\b|" + r"in summary\b|to summarize\b|the answer is\b)", + re.IGNORECASE, +) + + +def looks_like_stall(content: str, finish_reason: str, has_tool_calls: bool, + max_chars: int) -> bool: + """True when a no-tool-call turn looks like a premature agentic stall + (announced an action, didn't call a tool) rather than a real final answer.""" + if has_tool_calls: + return False + if finish_reason not in ("stop", "length"): + return False + c = (content or "").strip() + # Strip a leading ... block if present; judge the visible tail. + c = re.sub(r"^.*?\s*", "", c, flags=re.IGNORECASE | re.DOTALL).strip() + if not c: + return True # empty visible turn mid-task => stall + if len(c) > max_chars: + return False # long => almost certainly a real answer + if _COMPLETION_RE.search(c): + return False # model said it's done => respect it + if _ACTION_RE.search(c): + return True # announced an action, no tool call => stall + # Short prose that doesn't declare completion and isn't an obvious answer: + # a trailing colon strongly implies "about to do something". + if c.endswith(":"): + return True + return False + + +def retry_on_stall(agent, api_messages, finish_reason): + """If the just-finished no-tool-call turn looks like a stall and a retry + lane is configured, re-issue the SAME turn against that lane (same provider + / client / endpoint — only the model name changes) ONCE. + + Returns the normalized assistant_message from the retry IF it produced tool + calls (caller should adopt it + its finish_reason='tool_calls'), else None. + Never raises into the caller — any failure returns None (original stands). + """ + retry_model = os.environ.get("HERMES_STALL_RETRY_MODEL", "").strip() + if not retry_model: + return None + try: + max_chars = int(os.environ.get("HERMES_STALL_RETRY_MAX_CHARS", "400")) + except ValueError: + max_chars = 400 + + try: + # Build kwargs exactly as the normal turn would, then override only the + # model name. Safe when the retry lane is served by the SAME provider/ + # endpoint as agent.model (e.g. taro serves both dflash and the Q6 lane), + # so no client rebuild is needed. + api_kwargs = agent._build_api_kwargs(api_messages) + orig_model = api_kwargs.get("model") + if retry_model == orig_model: + return None # nothing to gain retrying the same model + api_kwargs = dict(api_kwargs) + api_kwargs["model"] = retry_model + # Force non-streaming for the retry (simpler, we only inspect the result). + api_kwargs.pop("stream", None) + api_kwargs["stream"] = False + + try: + agent._vprint( + f"{getattr(agent, 'log_prefix', '')}↻ stall detected " + f"(no tool call) — retrying turn on '{retry_model}'", + force=True, + ) + except Exception: + pass + + response = agent._interruptible_api_call(api_kwargs) + if response is None: + return None + transport = agent._get_transport() + normalize_kwargs = {} + if getattr(agent, "api_mode", None) == "anthropic_messages": + normalize_kwargs["strip_tool_prefix"] = getattr(agent, "_is_anthropic_oauth", False) + normalized = transport.normalize_response(response, **normalize_kwargs) + if getattr(normalized, "tool_calls", None): + return normalized + return None + except Exception: + # Any error => silently fall back to the original response. + return None From 4d298e9a57448c015940fa5e4b1d18acd73fbc35 Mon Sep 17 00:00:00 2001 From: Omar Baradei Date: Sat, 30 May 2026 18:29:52 -0700 Subject: [PATCH 02/13] fix(agent): execute stall-retry tool calls in-loop --- agent/conversation_loop.py | 112 ++++++++++++++++++++++++-------- agent/stall_retry.py | 8 ++- tests/agent/test_stall_retry.py | 75 +++++++++++++++++++++ 3 files changed, 166 insertions(+), 29 deletions(-) create mode 100644 tests/agent/test_stall_retry.py diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 5beda5fe1fd18..fbbbe5a001572 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -3652,6 +3652,92 @@ def _stop_spinner(): } elif hasattr(agent, "_codex_incomplete_retries"): agent._codex_incomplete_retries = 0 + + # ── Agentic stall-retry (opt-in via HERMES_STALL_RETRY_MODEL) ── + # dflash Q4 can stop right after an action preamble ("Let me + # check X") without producing the promised tool_call. Retry the + # exact same turn on the configured higher-quality lane before + # the final-response branch sees it. If the retry returns tool + # calls, fall through to the normal executor below in this same + # loop iteration. If it still returns no tool call, fail this + # turn as partial instead of persisting the planning-only text as + # a completed assistant message that poisons future "continue" + # turns. + retry_model = os.environ.get("HERMES_STALL_RETRY_MODEL", "").strip() + if ( + retry_model + and not _stall_retry_used + and getattr(agent, "tools", None) + and not getattr(assistant_message, "tool_calls", None) + ): + try: + max_chars = int( + os.environ.get("HERMES_STALL_RETRY_MAX_CHARS", "400") or 400 + ) + except ValueError: + max_chars = 400 + try: + from agent.stall_retry import looks_like_stall, retry_on_stall + + if looks_like_stall( + assistant_message.content or "", + finish_reason, + False, + max_chars, + ): + _stall_retry_used = True + retried = retry_on_stall(agent, api_messages, finish_reason) + if retried is not None and getattr(retried, "tool_calls", None): + assistant_message = retried + finish_reason = getattr(retried, "finish_reason", None) or "tool_calls" + if finish_reason != "tool_calls": + finish_reason = "tool_calls" + else: + _turn_exit_reason = "stall_retry_failed_no_tool_call" + agent._mute_post_response = False + agent._vprint( + ( + f"{agent.log_prefix}❌ Stall retry did not produce " + "tool calls; saving as partial without storing the " + "planning-only assistant turn." + ), + force=True, + ) + agent._cleanup_task_resources(effective_task_id) + agent._persist_session(messages, conversation_history) + return { + "final_response": None, + "messages": messages, + "api_calls": api_call_count, + "completed": False, + "partial": True, + "failed": True, + "error": ( + "Model stopped after an action preamble with no " + "tool call; configured stall retry also produced " + "no tool call." + ), + "failure_subclass": "stall_retry_failed_no_tool_call", + } + except Exception as exc: + _turn_exit_reason = "stall_retry_exception" + agent._mute_post_response = False + agent._vprint( + f"{agent.log_prefix}❌ Stall retry failed before recovery: {exc}", + force=True, + ) + agent._cleanup_task_resources(effective_task_id) + agent._persist_session(messages, conversation_history) + return { + "final_response": None, + "messages": messages, + "api_calls": api_call_count, + "completed": False, + "partial": True, + "failed": True, + "error": f"Stall retry failed before recovery: {exc}", + "failure_subclass": "stall_retry_exception", + } # Check for tool calls if assistant_message.tool_calls: @@ -3989,32 +4075,6 @@ def _stop_spinner(): # No tool calls - this is the final response final_response = assistant_message.content or "" - # ── Agentic stall-retry (opt-in via HERMES_STALL_RETRY_MODEL) ── - # dflash Q4 sometimes emits EOS right after an action preamble - # ("Let me check X:") with no tool_call, ending the turn early - # and stalling the agent mid-task. If this no-tool-call turn - # looks like that stall (not a genuine final answer) and a retry - # lane is configured, re-issue the SAME turn once on a - # higher-quality model; if it yields tool calls, adopt it and - # continue the loop instead of stopping. No-op unless the env is - # set, so default behavior is unchanged. - if not _stall_retry_used: - try: - from agent.stall_retry import looks_like_stall, retry_on_stall - if looks_like_stall( - final_response, finish_reason, - bool(getattr(assistant_message, "tool_calls", None)), - int(os.environ.get("HERMES_STALL_RETRY_MAX_CHARS", "400") or 400), - ): - _retried = retry_on_stall(agent, api_messages, finish_reason) - if _retried is not None and getattr(_retried, "tool_calls", None): - _stall_retry_used = True - assistant_message = _retried - finish_reason = "tool_calls" - continue # re-enter loop top; tool-calls path handles it - except Exception: - pass # any failure: keep the original response - # Fix: unmute output when entering the no-tool-call branch # so the user can see empty-response warnings and recovery # status messages. _mute_post_response was set during a diff --git a/agent/stall_retry.py b/agent/stall_retry.py index 06edb6ba0b784..de500064a31bf 100644 --- a/agent/stall_retry.py +++ b/agent/stall_retry.py @@ -9,8 +9,9 @@ This module detects that stall signature on a no-tool-call turn and retries the SAME turn once against a higher-quality model lane. If the retry produces -tool_calls, the loop adopts that response and continues; otherwise the -original response stands (no behavior change). +tool_calls, the loop adopts that response and continues; otherwise the caller +should fail the turn as partial rather than persist the planning-only text as +a final assistant message. Entirely opt-in: does nothing unless ``HERMES_STALL_RETRY_MODEL`` is set (e.g. ``qwen3.6-27b-256k``). Default-off => zero change to existing behavior. @@ -77,7 +78,8 @@ def retry_on_stall(agent, api_messages, finish_reason): Returns the normalized assistant_message from the retry IF it produced tool calls (caller should adopt it + its finish_reason='tool_calls'), else None. - Never raises into the caller — any failure returns None (original stands). + Never raises into the caller — any failure returns None so the caller can + fail closed without storing the stalled assistant message. """ retry_model = os.environ.get("HERMES_STALL_RETRY_MODEL", "").strip() if not retry_model: diff --git a/tests/agent/test_stall_retry.py b/tests/agent/test_stall_retry.py new file mode 100644 index 0000000000000..c4f201ab5efd0 --- /dev/null +++ b/tests/agent/test_stall_retry.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import inspect +from types import SimpleNamespace + +from agent import conversation_loop +from agent.stall_retry import looks_like_stall, retry_on_stall + + +def test_action_preamble_without_tool_call_is_a_stall() -> None: + assert looks_like_stall( + "Let me check what's on taro and figure out the right approach.", + "stop", + False, + 400, + ) + + +def test_completion_text_is_not_a_stall() -> None: + assert not looks_like_stall( + "Done. The task is complete and no further action is needed.", + "stop", + False, + 400, + ) + + +def test_retry_on_stall_switches_model_and_returns_tool_calls(monkeypatch) -> None: + captured: dict[str, object] = {} + tool_call = SimpleNamespace( + function=SimpleNamespace(name="terminal", arguments='{"cmd":"pwd"}') + ) + normalized = SimpleNamespace( + content="", + tool_calls=[tool_call], + finish_reason="tool_calls", + ) + + def interruptible_api_call(kwargs: dict[str, object]) -> object: + captured["kwargs"] = dict(kwargs) + return normalized + + agent = SimpleNamespace( + api_mode="openai", + _is_anthropic_oauth=False, + log_prefix="", + _build_api_kwargs=lambda messages: { + "model": "dflash", + "messages": messages, + "stream": True, + }, + _interruptible_api_call=interruptible_api_call, + _get_transport=lambda: SimpleNamespace( + normalize_response=lambda response, **_kwargs: response + ), + _vprint=lambda *_args, **_kwargs: None, + ) + + monkeypatch.setenv("HERMES_STALL_RETRY_MODEL", "qwen3.6-27b-256k") + result = retry_on_stall(agent, [{"role": "user", "content": "go"}], "stop") + + assert result is normalized + kwargs = captured["kwargs"] + assert kwargs["model"] == "qwen3.6-27b-256k" + assert kwargs["stream"] is False + + +def test_conversation_loop_adopts_retry_before_tool_call_branch() -> None: + source = inspect.getsource(conversation_loop.run_conversation) + retry_idx = source.index("retried = retry_on_stall") + tool_branch_idx = source.index("# Check for tool calls") + + assert retry_idx < tool_branch_idx + assert "continue # re-enter loop top; tool-calls path handles it" not in source + assert "stall_retry_failed_no_tool_call" in source From 2e8a237abd43df5d52f1f8342a508ea11d55f361 Mon Sep 17 00:00:00 2001 From: Omar B Date: Sat, 30 May 2026 19:27:28 -0700 Subject: [PATCH 03/13] fix(agent): retry incomplete dflash final fragments --- agent/stall_retry.py | 21 +++++++++++++++++++++ tests/agent/test_stall_retry.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/agent/stall_retry.py b/agent/stall_retry.py index de500064a31bf..1d63a4f587048 100644 --- a/agent/stall_retry.py +++ b/agent/stall_retry.py @@ -43,6 +43,20 @@ r"in summary\b|to summarize\b|the answer is\b)", re.IGNORECASE, ) +_NATURAL_END_CHARS = '.!?:)"\']}。！？：）】」』》^' +_MIN_INCOMPLETE_FINAL_CHARS = 80 + + +def _has_natural_response_ending(content: str) -> bool: + stripped = (content or "").rstrip() + if not stripped: + return False + if stripped.endswith("```"): + return True + last = stripped[-1] + if last in _NATURAL_END_CHARS: + return True + return ord(last) >= 0x1F300 def looks_like_stall(content: str, finish_reason: str, has_tool_calls: bool, @@ -68,6 +82,13 @@ def looks_like_stall(content: str, finish_reason: str, has_tool_calls: bool, # a trailing colon strongly implies "about to do something". if c.endswith(":"): return True + # dflash can also stop after a tool result with ordinary-looking prose that + # is simply cut off mid-sentence (for example after a CLI interrupt resumes + # the turn). In an agentic tool loop, a short no-tool stop that declares no + # completion and lacks a natural ending is safer to retry than to persist as + # a final assistant message. + if len(c) >= _MIN_INCOMPLETE_FINAL_CHARS and not _has_natural_response_ending(c): + return True return False diff --git a/tests/agent/test_stall_retry.py b/tests/agent/test_stall_retry.py index c4f201ab5efd0..87df480e14a90 100644 --- a/tests/agent/test_stall_retry.py +++ b/tests/agent/test_stall_retry.py @@ -25,6 +25,37 @@ def test_completion_text_is_not_a_stall() -> None: ) +def test_incomplete_final_fragment_without_action_preamble_is_a_stall() -> None: + assert looks_like_stall( + ( + "I'm on main with a clean tree. The fix I made was on Taro " + "(remote machine), not locally. The change was on Taro's " + "`~/.gitconfig` and the worktree's local git config. These are " + "machine-specific runtime" + ), + "stop", + False, + 400, + ) + + +def test_short_status_answer_without_punctuation_is_not_a_stall() -> None: + assert not looks_like_stall("main", "stop", False, 400) + + +def test_complete_agentic_answer_without_action_preamble_is_not_a_stall() -> None: + assert not looks_like_stall( + ( + "I'm on main with a clean tree. The Taro git identity and SSH " + "push configuration are machine-local runtime settings, so there " + "is no repository diff to publish." + ), + "stop", + False, + 400, + ) + + def test_retry_on_stall_switches_model_and_returns_tool_calls(monkeypatch) -> None: captured: dict[str, object] = {} tool_call = SimpleNamespace( From 04afbb35f83a5532656daf8ebecfad3012a03e64 Mon Sep 17 00:00:00 2001 From: Omar B Date: Sat, 30 May 2026 19:39:05 -0700 Subject: [PATCH 04/13] fix(agent): allow bounded repeated stall retries --- agent/conversation_loop.py | 47 ++++++++++++++++++++++++++++----- agent/stall_retry.py | 1 + tests/agent/test_stall_retry.py | 18 +++++++++++++ 3 files changed, 60 insertions(+), 6 deletions(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index fbbbe5a001572..9f5440d7192f8 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -793,10 +793,18 @@ def run_conversation( should_review_memory=_should_review_memory, ) - # Agentic stall-retry guard: ensures the HERMES_STALL_RETRY_MODEL retry - # fires at most once per conversation (avoids loops if the retry lane also - # stalls). See agent/stall_retry.py. - _stall_retry_used = False + # Agentic stall-retry guard: dflash can stall more than once in a long + # tool loop, so allow a bounded number of successful rescues per user turn. + # If the cap is exhausted, fail partial rather than accept another + # planning-only text response as final. See agent/stall_retry.py. + _stall_retry_count = 0 + try: + _stall_retry_max_per_turn = int( + os.environ.get("HERMES_STALL_RETRY_MAX_PER_TURN", "5") or 5 + ) + except ValueError: + _stall_retry_max_per_turn = 5 + _stall_retry_max_per_turn = max(0, _stall_retry_max_per_turn) while (api_call_count < agent.max_iterations and agent.iteration_budget.remaining > 0) or agent._budget_grace_call: # Reset per-turn checkpoint dedup so each iteration can take one snapshot @@ -3666,7 +3674,6 @@ def _stop_spinner(): retry_model = os.environ.get("HERMES_STALL_RETRY_MODEL", "").strip() if ( retry_model - and not _stall_retry_used and getattr(agent, "tools", None) and not getattr(assistant_message, "tool_calls", None) ): @@ -3685,7 +3692,35 @@ def _stop_spinner(): False, max_chars, ): - _stall_retry_used = True + if _stall_retry_count >= _stall_retry_max_per_turn: + _turn_exit_reason = "stall_retry_limit_exhausted" + agent._mute_post_response = False + agent._vprint( + ( + f"{agent.log_prefix}❌ Stall retry limit " + f"({_stall_retry_max_per_turn}/turn) exhausted; " + "saving as partial without storing the " + "planning-only assistant turn." + ), + force=True, + ) + agent._cleanup_task_resources(effective_task_id) + agent._persist_session(messages, conversation_history) + return { + "final_response": None, + "messages": messages, + "api_calls": api_call_count, + "completed": False, + "partial": True, + "failed": True, + "error": ( + "Model repeatedly stopped after an agentic " + "preamble with no tool call; configured stall " + "retry limit was exhausted." + ), + "failure_subclass": "stall_retry_limit_exhausted", + } + _stall_retry_count += 1 retried = retry_on_stall(agent, api_messages, finish_reason) if retried is not None and getattr(retried, "tool_calls", None): assistant_message = retried diff --git a/agent/stall_retry.py b/agent/stall_retry.py index 1d63a4f587048..1879562a18a1b 100644 --- a/agent/stall_retry.py +++ b/agent/stall_retry.py @@ -18,6 +18,7 @@ Env: HERMES_STALL_RETRY_MODEL retry lane/model name (required to enable) + HERMES_STALL_RETRY_MAX_PER_TURN max retries per user turn (default 5) HERMES_STALL_RETRY_MAX_CHARS max content length to still count as a stall (default 400; real final answers are longer) """ diff --git a/tests/agent/test_stall_retry.py b/tests/agent/test_stall_retry.py index 87df480e14a90..146f3f6ceda41 100644 --- a/tests/agent/test_stall_retry.py +++ b/tests/agent/test_stall_retry.py @@ -16,6 +16,15 @@ def test_action_preamble_without_tool_call_is_a_stall() -> None: ) +def test_followup_action_preamble_after_successful_retry_is_a_stall() -> None: + assert looks_like_stall( + "Let me look at open tasks with high priority that I can actually pick up.", + "stop", + False, + 400, + ) + + def test_completion_text_is_not_a_stall() -> None: assert not looks_like_stall( "Done. The task is complete and no further action is needed.", @@ -104,3 +113,12 @@ def test_conversation_loop_adopts_retry_before_tool_call_branch() -> None: assert retry_idx < tool_branch_idx assert "continue # re-enter loop top; tool-calls path handles it" not in source assert "stall_retry_failed_no_tool_call" in source + + +def test_conversation_loop_allows_bounded_multiple_stall_retries() -> None: + source = inspect.getsource(conversation_loop.run_conversation) + + assert "_stall_retry_used" not in source + assert "_stall_retry_count += 1" in source + assert "HERMES_STALL_RETRY_MAX_PER_TURN" in source + assert "stall_retry_limit_exhausted" in source From c19a9a7445db1c18ff1d74c3966e1c984243d5c5 Mon Sep 17 00:00:00 2001 From: Omar B Date: Sun, 31 May 2026 14:47:19 -0700 Subject: [PATCH 05/13] fix(agent): recover empty post-tool stalls locally --- agent/conversation_loop.py | 52 +++++++++++++++++- agent/stall_retry.py | 37 +++++++++++-- tests/agent/test_stall_retry.py | 94 ++++++++++++++++++++++++++++++++- 3 files changed, 177 insertions(+), 6 deletions(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 9f5440d7192f8..da7aee0fbe44c 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -3684,9 +3684,57 @@ def _stop_spinner(): except ValueError: max_chars = 400 try: - from agent.stall_retry import looks_like_stall, retry_on_stall + from agent.stall_retry import ( + EMPTY_AFTER_TOOL_RETRY_NUDGE, + looks_like_stall, + retry_on_stall, + ) + + _empty_after_tool_result = ( + getattr(agent, "tools", None) + and not getattr(assistant_message, "tool_calls", None) + and not agent._strip_think_blocks( + assistant_message.content or "" + ).strip() + and any( + isinstance(m, dict) and m.get("role") == "tool" + for m in messages[-5:] + ) + ) + if ( + _empty_after_tool_result + and _stall_retry_count < _stall_retry_max_per_turn + ): + _stall_retry_count += 1 + retried = retry_on_stall( + agent, + api_messages, + finish_reason, + accept_content=True, + retry_nudge=EMPTY_AFTER_TOOL_RETRY_NUDGE, + ) + if retried is not None: + assistant_message = retried + finish_reason = ( + getattr(retried, "finish_reason", None) + or ( + "tool_calls" + if getattr(retried, "tool_calls", None) + else "stop" + ) + ) + agent._empty_content_retries = 0 + agent._post_tool_empty_retried = False + else: + logging.warning( + "Stall retry lane did not recover empty " + "post-tool response; continuing " + "empty-response recovery (model=%s provider=%s)", + agent.model, + agent.provider, + ) - if looks_like_stall( + if not _empty_after_tool_result and looks_like_stall( assistant_message.content or "", finish_reason, False, diff --git a/agent/stall_retry.py b/agent/stall_retry.py index 1879562a18a1b..25a88d4c982fc 100644 --- a/agent/stall_retry.py +++ b/agent/stall_retry.py @@ -46,6 +46,12 @@ ) _NATURAL_END_CHARS = '.!?:)"\']}。！？：）】」』》^' _MIN_INCOMPLETE_FINAL_CHARS = 80 +EMPTY_AFTER_TOOL_RETRY_NUDGE = ( + "Your previous assistant response after the tool results was empty. " + "Continue the same task using the tool results above. If the next step " + "requires another tool, call it immediately; otherwise provide the next " + "concise response. Do not summarize or apologize." +) def _has_natural_response_ending(content: str) -> bool: @@ -93,7 +99,25 @@ def looks_like_stall(content: str, finish_reason: str, has_tool_calls: bool, return False -def retry_on_stall(agent, api_messages, finish_reason): +def _retry_messages_with_nudge(api_messages, stalled_content="", retry_nudge=None): + retry_messages = list(api_messages) + visible = (stalled_content or "").strip() + if visible: + retry_messages.append({"role": "assistant", "content": visible}) + if retry_nudge: + retry_messages.append({"role": "user", "content": retry_nudge}) + return retry_messages + + +def retry_on_stall( + agent, + api_messages, + finish_reason, + stalled_content="", + *, + accept_content=False, + retry_nudge=None, +): """If the just-finished no-tool-call turn looks like a stall and a retry lane is configured, re-issue the SAME turn against that lane (same provider / client / endpoint — only the model name changes) ONCE. @@ -116,7 +140,12 @@ def retry_on_stall(agent, api_messages, finish_reason): # model name. Safe when the retry lane is served by the SAME provider/ # endpoint as agent.model (e.g. taro serves both dflash and the Q6 lane), # so no client rebuild is needed. - api_kwargs = agent._build_api_kwargs(api_messages) + retry_messages = _retry_messages_with_nudge( + api_messages, + stalled_content=stalled_content, + retry_nudge=retry_nudge, + ) + api_kwargs = agent._build_api_kwargs(retry_messages) orig_model = api_kwargs.get("model") if retry_model == orig_model: return None # nothing to gain retrying the same model @@ -143,7 +172,9 @@ def retry_on_stall(agent, api_messages, finish_reason): if getattr(agent, "api_mode", None) == "anthropic_messages": normalize_kwargs["strip_tool_prefix"] = getattr(agent, "_is_anthropic_oauth", False) normalized = transport.normalize_response(response, **normalize_kwargs) - if getattr(normalized, "tool_calls", None): + tool_calls = getattr(normalized, "tool_calls", None) + content = getattr(normalized, "content", "") or "" + if tool_calls or (accept_content and content.strip()): return normalized return None except Exception: diff --git a/tests/agent/test_stall_retry.py b/tests/agent/test_stall_retry.py index 146f3f6ceda41..162fae549fd9e 100644 --- a/tests/agent/test_stall_retry.py +++ b/tests/agent/test_stall_retry.py @@ -4,7 +4,11 @@ from types import SimpleNamespace from agent import conversation_loop -from agent.stall_retry import looks_like_stall, retry_on_stall +from agent.stall_retry import ( + EMPTY_AFTER_TOOL_RETRY_NUDGE, + looks_like_stall, + retry_on_stall, +) def test_action_preamble_without_tool_call_is_a_stall() -> None: @@ -105,6 +109,82 @@ def interruptible_api_call(kwargs: dict[str, object]) -> object: assert kwargs["stream"] is False +def test_retry_on_stall_can_accept_visible_content(monkeypatch) -> None: + captured: dict[str, object] = {} + normalized = SimpleNamespace( + content="I processed the tool result and will continue.", + tool_calls=None, + finish_reason="stop", + ) + + def interruptible_api_call(kwargs: dict[str, object]) -> object: + captured["kwargs"] = dict(kwargs) + return normalized + + agent = SimpleNamespace( + api_mode="openai", + _is_anthropic_oauth=False, + log_prefix="", + _build_api_kwargs=lambda messages: { + "model": "dflash", + "messages": messages, + "stream": True, + }, + _interruptible_api_call=interruptible_api_call, + _get_transport=lambda: SimpleNamespace( + normalize_response=lambda response, **_kwargs: response + ), + _vprint=lambda *_args, **_kwargs: None, + ) + + monkeypatch.setenv("HERMES_STALL_RETRY_MODEL", "qwen3.6-27b-256k") + result = retry_on_stall( + agent, + [{"role": "user", "content": "go"}], + "stop", + accept_content=True, + retry_nudge=EMPTY_AFTER_TOOL_RETRY_NUDGE, + ) + + assert result is normalized + kwargs = captured["kwargs"] + assert kwargs["model"] == "qwen3.6-27b-256k" + assert kwargs["stream"] is False + assert kwargs["messages"][-1]["role"] == "user" + assert "after the tool results was empty" in kwargs["messages"][-1]["content"] + + +def test_retry_on_stall_still_rejects_content_without_accept_content(monkeypatch) -> None: + normalized = SimpleNamespace( + content="I processed the tool result and will continue.", + tool_calls=None, + finish_reason="stop", + ) + + agent = SimpleNamespace( + api_mode="openai", + _is_anthropic_oauth=False, + log_prefix="", + _build_api_kwargs=lambda messages: { + "model": "dflash", + "messages": messages, + "stream": True, + }, + _interruptible_api_call=lambda _kwargs: normalized, + _get_transport=lambda: SimpleNamespace( + normalize_response=lambda response, **_kwargs: response + ), + _vprint=lambda *_args, **_kwargs: None, + ) + + monkeypatch.setenv("HERMES_STALL_RETRY_MODEL", "qwen3.6-27b-256k") + assert retry_on_stall( + agent, + [{"role": "user", "content": "go"}], + "stop", + ) is None + + def test_conversation_loop_adopts_retry_before_tool_call_branch() -> None: source = inspect.getsource(conversation_loop.run_conversation) retry_idx = source.index("retried = retry_on_stall") @@ -115,6 +195,18 @@ def test_conversation_loop_adopts_retry_before_tool_call_branch() -> None: assert "stall_retry_failed_no_tool_call" in source +def test_conversation_loop_retries_empty_post_tool_before_generic_stall() -> None: + source = inspect.getsource(conversation_loop.run_conversation) + empty_retry_idx = source.index("EMPTY_AFTER_TOOL_RETRY_NUDGE") + generic_stall_idx = source.index("looks_like_stall(") + tool_branch_idx = source.index("# Check for tool calls") + + assert empty_retry_idx < generic_stall_idx + assert empty_retry_idx < tool_branch_idx + assert "not _empty_after_tool_result and looks_like_stall" in source + assert "accept_content=True" in source + + def test_conversation_loop_allows_bounded_multiple_stall_retries() -> None: source = inspect.getsource(conversation_loop.run_conversation) From 686fe8dcf3495f84ac4edc557d145b59633ec5af Mon Sep 17 00:00:00 2001 From: Omar Baradei Date: Sun, 31 May 2026 15:24:49 -0700 Subject: [PATCH 06/13] fix(agent): resolve configured stall retry provider --- agent/conversation_loop.py | 9 +- agent/stall_retry.py | 488 +++++++++++++++++++++++++++++--- tests/agent/test_stall_retry.py | 302 +++++++++++++++++++- 3 files changed, 756 insertions(+), 43 deletions(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index da7aee0fbe44c..d2ec109c7eaad 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -799,12 +799,11 @@ def run_conversation( # planning-only text response as final. See agent/stall_retry.py. _stall_retry_count = 0 try: - _stall_retry_max_per_turn = int( - os.environ.get("HERMES_STALL_RETRY_MAX_PER_TURN", "5") or 5 - ) - except ValueError: + from agent.stall_retry import get_stall_retry_max_per_turn + + _stall_retry_max_per_turn = get_stall_retry_max_per_turn(agent) + except Exception: _stall_retry_max_per_turn = 5 - _stall_retry_max_per_turn = max(0, _stall_retry_max_per_turn) while (api_call_count < agent.max_iterations and agent.iteration_budget.remaining > 0) or agent._budget_grace_call: # Reset per-turn checkpoint dedup so each iteration can take one snapshot diff --git a/agent/stall_retry.py b/agent/stall_retry.py index 25a88d4c982fc..ede9c985a8774 100644 --- a/agent/stall_retry.py +++ b/agent/stall_retry.py @@ -8,7 +8,8 @@ same host) continue to a real tool call on the identical prompt. This module detects that stall signature on a no-tool-call turn and retries -the SAME turn once against a higher-quality model lane. If the retry produces +the turn against a higher-quality model lane, with a small recovery nudge that +asks the model to emit the tool call it just promised. If the retry produces tool_calls, the loop adopts that response and continues; otherwise the caller should fail the turn as partial rather than persist the planning-only text as a final assistant message. @@ -18,17 +19,30 @@ Env: HERMES_STALL_RETRY_MODEL retry lane/model name (required to enable) + HERMES_STALL_RETRY_PROVIDER optional provider override for the retry lane + HERMES_STALL_RETRY_BASE_URL optional OpenAI-compatible retry endpoint HERMES_STALL_RETRY_MAX_PER_TURN max retries per user turn (default 5) HERMES_STALL_RETRY_MAX_CHARS max content length to still count as a stall - (default 400; real final answers are longer) + (default 400; longer open action preambles + ending in ":" get a bounded exception) + HERMES_STALL_RETRY_NUDGE true/false; add a retry-only continuation nudge + (default true) + HERMES_STALL_RETRY_TELEMETRY true/false; append local NDJSON telemetry + (default true) """ from __future__ import annotations +import json import os import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping # Action-preamble signature: the turn announced an action but produced no tool -# call. These end mid-thought, typically with a colon, or open with intent. +# call. These English phrases match the observed dflash stall corpus; broader +# language-agnostic fallbacks below still catch trailing-colon and incomplete +# final fragments without pretending this regex is multilingual. _ACTION_RE = re.compile( r"(let me\b|let's\b|i'?ll\b|i will\b|i'?m going to\b|i am going to\b|" r"now i\b|first,?\s+i\b|next,?\s+i\b|i need to\b|i should\b|" @@ -36,7 +50,8 @@ re.IGNORECASE, ) # Genuine completion signature: the model declared it is done / nothing to do. -# These must NOT be retried (they are correct no-tool-call turns). +# These English phrases must NOT be retried (they are correct no-tool-call +# turns); other languages still rely on the neutral structural checks below. _COMPLETION_RE = re.compile( r"(\bdone\b|\bcomplete(d)?\b|nothing to (do|save|change|report|fix)|" r"no changes?\b|no action\b|already (complete|done|finished)|\bfinished\b|" @@ -46,6 +61,13 @@ ) _NATURAL_END_CHARS = '.!?:)"\']}。！？：）】」』》^' _MIN_INCOMPLETE_FINAL_CHARS = 80 +_ACTION_TAIL_CHARS = 500 +_STALL_RETRY_NUDGE = ( + "Your previous assistant response ended after describing the next action, " + "but it did not include the required tool call. Continue the same task now " + "by making the tool call immediately. Do not summarize or apologize; call " + "the tool that performs the action you just announced." +) EMPTY_AFTER_TOOL_RETRY_NUDGE = ( "Your previous assistant response after the tool results was empty. " "Continue the same task using the tool results above. If the next step " @@ -54,6 +76,106 @@ ) +def _as_positive_int(value: Any, default: int) -> int: + try: + parsed = int(value) + except (TypeError, ValueError): + return default + return parsed if parsed > 0 else default + + +def _as_bool(value: Any, default: bool) -> bool: + if value is None: + return default + if isinstance(value, bool): + return value + if isinstance(value, (int, float)): + return bool(value) + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in {"1", "true", "yes", "on", "enabled"}: + return True + if lowered in {"0", "false", "no", "off", "disabled"}: + return False + return default + + +def _stall_retry_config(agent: Any | None = None) -> Mapping[str, Any]: + cfg = getattr(agent, "_stall_retry_config", None) + if isinstance(cfg, Mapping): + return cfg + try: + from hermes_cli.config import load_config + + loaded = load_config() + except Exception: + return {} + cfg = loaded.get("stall_retry") if isinstance(loaded, Mapping) else None + return cfg if isinstance(cfg, Mapping) else {} + + +def get_stall_retry_model(agent: Any | None = None) -> str: + """Return the configured retry model, with env taking precedence.""" + env_model = os.environ.get("HERMES_STALL_RETRY_MODEL", "").strip() + if env_model: + return env_model + cfg_model = _stall_retry_config(agent).get("model") + return str(cfg_model or "").strip() + + +def get_stall_retry_provider(agent: Any | None = None) -> str: + """Return an optional provider override for the retry lane.""" + env_provider = os.environ.get("HERMES_STALL_RETRY_PROVIDER", "").strip() + if env_provider: + return env_provider + cfg_provider = _stall_retry_config(agent).get("provider") + return str(cfg_provider or "").strip() + + +def get_stall_retry_base_url(agent: Any | None = None) -> str: + """Return an optional base URL override for the retry lane.""" + env_base_url = os.environ.get("HERMES_STALL_RETRY_BASE_URL", "").strip() + if env_base_url: + return env_base_url + cfg_base_url = _stall_retry_config(agent).get("base_url") + return str(cfg_base_url or "").strip() + + +def get_stall_retry_max_chars(agent: Any | None = None) -> int: + env_value = os.environ.get("HERMES_STALL_RETRY_MAX_CHARS") + if env_value is not None: + return _as_positive_int(env_value, 400) + return _as_positive_int(_stall_retry_config(agent).get("max_chars"), 400) + + +def get_stall_retry_max_per_turn(agent: Any | None = None) -> int: + env_value = os.environ.get("HERMES_STALL_RETRY_MAX_PER_TURN") + if env_value is not None: + try: + return max(0, int(env_value)) + except ValueError: + return 5 + cfg_value = _stall_retry_config(agent).get("max_per_turn") + try: + return max(0, int(cfg_value)) + except (TypeError, ValueError): + return 5 + + +def get_stall_retry_nudge_enabled(agent: Any | None = None) -> bool: + env_value = os.environ.get("HERMES_STALL_RETRY_NUDGE") + if env_value is not None: + return _as_bool(env_value, True) + return _as_bool(_stall_retry_config(agent).get("nudge"), True) + + +def get_stall_retry_telemetry_enabled(agent: Any | None = None) -> bool: + env_value = os.environ.get("HERMES_STALL_RETRY_TELEMETRY") + if env_value is not None: + return _as_bool(env_value, True) + return _as_bool(_stall_retry_config(agent).get("telemetry"), True) + + def _has_natural_response_ending(content: str) -> bool: stripped = (content or "").rstrip() if not stripped: @@ -66,6 +188,262 @@ def _has_natural_response_ending(content: str) -> bool: return ord(last) >= 0x1F300 +def _action_after_completion(content: str) -> bool: + """True when a response says some step is complete, then promises work. + + The dflash phone failure hit this exact shape: + "Onboarding complete. Now let me read the STATUS.md ..." + + The earlier completion word is not a final answer when a later clause is + still announcing the next tool step. + """ + action_matches = list(_ACTION_RE.finditer(content or "")) + if not action_matches: + return False + completion_matches = list(_COMPLETION_RE.finditer(content or "")) + if not completion_matches: + return False + return action_matches[-1].start() > completion_matches[-1].end() + + +def _ends_with_action_promise(content: str) -> bool: + """True when the visible tail promises immediate work but stops there. + + The generic stall heuristic is intentionally length-capped because long + prose is often a real answer. Explicit tail promises are different: a long + diagnostic can still end with "Let me check that:" and no tool call, which + is the exact dflash premature-stop shape this module exists to recover. + """ + tail = (content or "").strip()[-_ACTION_TAIL_CHARS:] + if not tail: + return False + if not _ACTION_RE.search(tail): + return False + return tail.rstrip().endswith(":") + + +def _safe_preview(value: Any, max_chars: int = 240) -> str: + text = value if isinstance(value, str) else str(value or "") + text = re.sub(r"^.*?\s*", "", text, flags=re.IGNORECASE | re.DOTALL) + text = re.sub(r"\s+", " ", text).strip() + if len(text) <= max_chars: + return text + return text[: max_chars - 3].rstrip() + "..." + + +def _jsonable(value: Any) -> Any: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, Mapping): + return {str(k): _jsonable(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_jsonable(v) for v in value] + return str(value) + + +def _stall_retry_log_path(agent: Any | None = None) -> Path: + cfg_path = _stall_retry_config(agent).get("telemetry_path") + if cfg_path: + return Path(str(cfg_path)).expanduser() + try: + from hermes_constants import get_hermes_home + + home = Path(get_hermes_home()) + except Exception: + home = Path(os.environ.get("HERMES_HOME", "~/.hermes")).expanduser() + return home / "logs" / "stall-retry.ndjson" + + +def record_stall_retry_event(agent: Any, event: str, **fields: Any) -> None: + """Record local, bounded stall-retry telemetry.""" + entry: dict[str, Any] = { + "ts": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "event": str(event), + "session_id": str(getattr(agent, "session_id", "") or ""), + "model": str(getattr(agent, "model", "") or ""), + "provider": str(getattr(agent, "provider", "") or ""), + } + content = fields.pop("content", None) + if content is not None: + text = content if isinstance(content, str) else str(content) + entry["content_chars"] = len(text) + entry["content_preview"] = _safe_preview(text) + entry.update({str(k): _jsonable(v) for k, v in fields.items()}) + + events = getattr(agent, "_stall_retry_events", None) + if not isinstance(events, list): + events = [] + try: + setattr(agent, "_stall_retry_events", events) + except Exception: + pass + events.append(entry) + + if not get_stall_retry_telemetry_enabled(agent): + return + try: + path = _stall_retry_log_path(agent) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(entry, ensure_ascii=False, sort_keys=True) + "\n") + except Exception: + return + + +def stall_retry_summary(agent: Any) -> dict[str, Any] | None: + events = getattr(agent, "_stall_retry_events", None) + if not isinstance(events, list) or not events: + return None + counts = { + "detected": 0, + "attempted": 0, + "recovered": 0, + "failed": 0, + "limit_exhausted": 0, + "exceptions": 0, + } + for item in events: + kind = item.get("event") if isinstance(item, Mapping) else None + if kind == "detected": + counts["detected"] += 1 + elif kind == "attempt": + counts["attempted"] += 1 + elif kind == "recovered": + counts["recovered"] += 1 + elif kind in {"failed_no_tool_call", "api_none", "skipped_same_model"}: + counts["failed"] += 1 + elif kind == "limit_exhausted": + counts["limit_exhausted"] += 1 + elif kind == "exception": + counts["exceptions"] += 1 + summary: dict[str, Any] = dict(counts) + summary["events"] = len(events) + if get_stall_retry_telemetry_enabled(agent): + summary["log_path"] = str(_stall_retry_log_path(agent)) + return summary + + +def _retry_messages_with_nudge( + agent: Any, + api_messages: list[dict[str, Any]], + stalled_content: str, + retry_nudge: str | None = None, +) -> list[dict[str, Any]]: + if not get_stall_retry_nudge_enabled(agent): + return api_messages + retry_messages = [msg.copy() if isinstance(msg, dict) else msg for msg in api_messages] + visible = (stalled_content or "").strip() + if visible: + retry_messages.append({"role": "assistant", "content": visible}) + retry_messages.append({"role": "user", "content": retry_nudge or _STALL_RETRY_NUDGE}) + return retry_messages + + +def _load_env_value(name: str) -> str: + env_name = str(name or "").strip() + if not env_name: + return "" + value = os.environ.get(env_name, "").strip() + if value: + return value + try: + from hermes_cli.env_loader import load_hermes_dotenv + from hermes_constants import get_hermes_home + + load_hermes_dotenv(hermes_home=get_hermes_home()) + except Exception: + pass + return os.environ.get(env_name, "").strip() + + +def _base_url_keys(value: Any) -> set[str]: + raw = str(value or "").strip().lower().rstrip("/") + if not raw: + return set() + keys = {raw} + if raw.endswith("/v1"): + keys.add(raw[:-3].rstrip("/")) + else: + keys.add(f"{raw}/v1") + return keys + + +def _custom_provider_entry(agent: Any | None, provider: str, base_url: str) -> Mapping[str, Any]: + try: + from hermes_cli.config import get_compatible_custom_providers, load_config + + entries = get_compatible_custom_providers(load_config()) + except Exception: + entries = getattr(agent, "_custom_providers", []) if agent is not None else [] + if not isinstance(entries, list): + return {} + + provider_name = str(provider or "").strip().lower() + target_keys = _base_url_keys(base_url or getattr(agent, "base_url", "")) + for entry in entries: + if not isinstance(entry, Mapping): + continue + entry_name = str(entry.get("name") or "").strip().lower() + if provider_name and entry_name == provider_name: + return entry + entry_keys = _base_url_keys(entry.get("base_url")) + if target_keys and entry_keys and target_keys.intersection(entry_keys): + return entry + return {} + + +def _configured_retry_api_key(agent: Any, provider: str, base_url: str) -> str: + cfg = _stall_retry_config(agent) + explicit = str(cfg.get("api_key") or "").strip() + if explicit: + return explicit + key_env = str(cfg.get("key_env") or cfg.get("api_key_env") or "").strip() + if key_env: + return _load_env_value(key_env) + + entry = _custom_provider_entry(agent, provider, base_url) + explicit = str(entry.get("api_key") or "").strip() + if explicit: + return explicit + key_env = str(entry.get("key_env") or entry.get("api_key_env") or "").strip() + return _load_env_value(key_env) if key_env else "" + + +def _retry_api_call(agent: Any, api_kwargs: dict[str, Any], retry_model: str) -> Any: + """Execute the retry request, optionally through a configured provider. + + The original implementation reused the active client and only changed the + model name. That is fine when both models are served anonymously by the + same endpoint, but it breaks when the retry lane is a named custom provider + whose auth lives in ``key_env``. Resolve that provider explicitly when the + retry config names one or supplies endpoint/auth details. + """ + retry_provider = get_stall_retry_provider(agent) + retry_base_url = get_stall_retry_base_url(agent) + retry_api_key = _configured_retry_api_key(agent, retry_provider, retry_base_url) + if not retry_base_url and retry_api_key: + retry_base_url = str(getattr(agent, "base_url", "") or "").strip() + + if retry_provider or retry_base_url or retry_api_key: + from agent.auxiliary_client import resolve_provider_client + + provider = retry_provider or str(getattr(agent, "provider", "") or "custom") + client, resolved_model = resolve_provider_client( + provider, + model=retry_model, + raw_codex=True, + explicit_base_url=retry_base_url or None, + explicit_api_key=retry_api_key or None, + ) + if client is None: + raise RuntimeError(f"Could not resolve stall retry provider {provider!r}") + retry_kwargs = dict(api_kwargs) + retry_kwargs["model"] = resolved_model or retry_model + return client.chat.completions.create(**retry_kwargs) + + return agent._interruptible_api_call(api_kwargs) + + def looks_like_stall(content: str, finish_reason: str, has_tool_calls: bool, max_chars: int) -> bool: """True when a no-tool-call turn looks like a premature agentic stall @@ -78,11 +456,17 @@ def looks_like_stall(content: str, finish_reason: str, has_tool_calls: bool, # Strip a leading ... block if present; judge the visible tail. c = re.sub(r"^.*?\s*", "", c, flags=re.IGNORECASE | re.DOTALL).strip() if not c: - return True # empty visible turn mid-task => stall - if len(c) > max_chars: - return False # long => almost certainly a real answer + # Truly empty responses have their own recovery path in the + # conversation loop. Do not let stall retry preempt that machinery. + return False + if _action_after_completion(c): + return True if _COMPLETION_RE.search(c): return False # model said it's done => respect it + if _ends_with_action_promise(c): + return True + if len(c) > max_chars: + return False # long => almost certainly a real answer if _ACTION_RE.search(c): return True # announced an action, no tool call => stall # Short prose that doesn't declare completion and isn't an obvious answer: @@ -99,55 +483,52 @@ def looks_like_stall(content: str, finish_reason: str, has_tool_calls: bool, return False -def _retry_messages_with_nudge(api_messages, stalled_content="", retry_nudge=None): - retry_messages = list(api_messages) - visible = (stalled_content or "").strip() - if visible: - retry_messages.append({"role": "assistant", "content": visible}) - if retry_nudge: - retry_messages.append({"role": "user", "content": retry_nudge}) - return retry_messages - - def retry_on_stall( agent, api_messages, finish_reason, - stalled_content="", + stalled_content: str = "", + retry_index: int | None = None, *, - accept_content=False, - retry_nudge=None, + accept_content: bool = False, + retry_nudge: str | None = None, ): """If the just-finished no-tool-call turn looks like a stall and a retry - lane is configured, re-issue the SAME turn against that lane (same provider - / client / endpoint — only the model name changes) ONCE. + lane is configured, re-issue the turn against that lane (same provider / + client / endpoint — only the model name changes). A retry-only nudge is + appended by default so the fallback model is told to continue with a tool + call instead of repeating the action preamble. Returns the normalized assistant_message from the retry IF it produced tool calls (caller should adopt it + its finish_reason='tool_calls'), else None. Never raises into the caller — any failure returns None so the caller can fail closed without storing the stalled assistant message. """ - retry_model = os.environ.get("HERMES_STALL_RETRY_MODEL", "").strip() + retry_model = get_stall_retry_model(agent) if not retry_model: return None - try: - max_chars = int(os.environ.get("HERMES_STALL_RETRY_MAX_CHARS", "400")) - except ValueError: - max_chars = 400 try: - # Build kwargs exactly as the normal turn would, then override only the - # model name. Safe when the retry lane is served by the SAME provider/ - # endpoint as agent.model (e.g. taro serves both dflash and the Q6 lane), - # so no client rebuild is needed. retry_messages = _retry_messages_with_nudge( + agent, api_messages, - stalled_content=stalled_content, + stalled_content, retry_nudge=retry_nudge, ) + # Build kwargs exactly as the normal turn would, then override only the + # model name. Safe when the retry lane is served by the SAME provider/ + # endpoint as agent.model (e.g. taro serves both dflash and the Q6 lane), + # so no client rebuild is needed. api_kwargs = agent._build_api_kwargs(retry_messages) orig_model = api_kwargs.get("model") if retry_model == orig_model: + record_stall_retry_event( + agent, + "skipped_same_model", + retry_model=retry_model, + finish_reason=finish_reason, + retry_index=retry_index, + ) return None # nothing to gain retrying the same model api_kwargs = dict(api_kwargs) api_kwargs["model"] = retry_model @@ -164,8 +545,24 @@ def retry_on_stall( except Exception: pass - response = agent._interruptible_api_call(api_kwargs) + record_stall_retry_event( + agent, + "attempt", + retry_model=retry_model, + original_model=orig_model, + finish_reason=finish_reason, + retry_index=retry_index, + nudge=get_stall_retry_nudge_enabled(agent), + content=stalled_content, + ) + response = _retry_api_call(agent, api_kwargs, retry_model) if response is None: + record_stall_retry_event( + agent, + "api_none", + retry_model=retry_model, + retry_index=retry_index, + ) return None transport = agent._get_transport() normalize_kwargs = {} @@ -175,8 +572,31 @@ def retry_on_stall( tool_calls = getattr(normalized, "tool_calls", None) content = getattr(normalized, "content", "") or "" if tool_calls or (accept_content and content.strip()): + record_stall_retry_event( + agent, + "recovered", + retry_model=retry_model, + retry_index=retry_index, + tool_call_count=len(tool_calls or []), + content=content, + ) return normalized + record_stall_retry_event( + agent, + "failed_no_tool_call", + retry_model=retry_model, + retry_index=retry_index, + content=getattr(normalized, "content", "") or "", + ) return None - except Exception: + except Exception as exc: + record_stall_retry_event( + agent, + "exception", + retry_model=retry_model, + retry_index=retry_index, + error_type=type(exc).__name__, + error=str(exc)[:300], + ) # Any error => silently fall back to the original response. return None diff --git a/tests/agent/test_stall_retry.py b/tests/agent/test_stall_retry.py index 162fae549fd9e..5761edd55139a 100644 --- a/tests/agent/test_stall_retry.py +++ b/tests/agent/test_stall_retry.py @@ -1,13 +1,20 @@ from __future__ import annotations import inspect +import json from types import SimpleNamespace from agent import conversation_loop from agent.stall_retry import ( EMPTY_AFTER_TOOL_RETRY_NUDGE, + get_stall_retry_max_chars, + get_stall_retry_max_per_turn, + get_stall_retry_model, + get_stall_retry_nudge_enabled, looks_like_stall, + record_stall_retry_event, retry_on_stall, + stall_retry_summary, ) @@ -29,6 +36,49 @@ def test_followup_action_preamble_after_successful_retry_is_a_stall() -> None: ) +def test_mixed_explanation_action_preamble_over_default_limit_is_a_stall() -> None: + content = ( + "Now I have a clear picture. The task is to add a pre-dispatch " + "executable-state gate in subagent_dispatch_register_command that " + "checks if the linked task is in an executable state BEFORE registering " + "a dispatch. Currently the stale guard only runs post-dispatch during " + "drain and settlement. The goal is to prevent non-executable dispatches " + "from being registered in the first place. Let me look at the register " + "command's validation section and the _validate_subagent_dispatch_record " + "function:" + ) + + assert len(content) > 400 + assert looks_like_stall(content, "stop", False, 400) + + +def test_long_diagnostic_ending_with_action_promise_is_a_stall() -> None: + content = ( + "The crash pattern is clear: when the context fills up and llama.cpp " + "tries to allocate a new tensor for the KV cache, it runs out of GPU " + "memory. The first allocation succeeds, but the subsequent data " + "allocation fails and leaves the tensor in an inconsistent state. " + * 4 + ) + content += "Let me look at the exact crash mechanism more carefully:" + + assert len(content) > 400 + assert looks_like_stall(content, "stop", False, 400) + + +def test_long_diagnostic_with_completion_text_is_not_a_stall() -> None: + content = ( + "The crash pattern is clear: when the context fills up and llama.cpp " + "tries to allocate a new tensor for the KV cache, it runs out of GPU " + "memory. " + * 5 + ) + content += "In summary, the task is complete." + + assert len(content) > 400 + assert not looks_like_stall(content, "stop", False, 400) + + def test_completion_text_is_not_a_stall() -> None: assert not looks_like_stall( "Done. The task is complete and no further action is needed.", @@ -38,6 +88,15 @@ def test_completion_text_is_not_a_stall() -> None: ) +def test_completion_then_action_promise_is_still_a_stall() -> None: + assert looks_like_stall( + "Onboarding complete. Now let me read the STATUS.md to find a task I can pick up.", + "stop", + False, + 400, + ) + + def test_incomplete_final_fragment_without_action_preamble_is_a_stall() -> None: assert looks_like_stall( ( @@ -52,6 +111,11 @@ def test_incomplete_final_fragment_without_action_preamble_is_a_stall() -> None: ) +def test_empty_visible_response_uses_empty_response_recovery_not_stall_retry() -> None: + assert not looks_like_stall("", "stop", False, 400) + assert not looks_like_stall("still reasoning", "stop", False, 400) + + def test_short_status_answer_without_punctuation_is_not_a_stall() -> None: assert not looks_like_stall("main", "stop", False, 400) @@ -101,12 +165,27 @@ def interruptible_api_call(kwargs: dict[str, object]) -> object: ) monkeypatch.setenv("HERMES_STALL_RETRY_MODEL", "qwen3.6-27b-256k") - result = retry_on_stall(agent, [{"role": "user", "content": "go"}], "stop") + monkeypatch.setenv("HERMES_STALL_RETRY_TELEMETRY", "0") + result = retry_on_stall( + agent, + [{"role": "user", "content": "go"}], + "stop", + stalled_content="Let me check the repo.", + retry_index=1, + ) assert result is normalized kwargs = captured["kwargs"] assert kwargs["model"] == "qwen3.6-27b-256k" assert kwargs["stream"] is False + assert kwargs["messages"][-2]["role"] == "assistant" + assert kwargs["messages"][-2]["content"] == "Let me check the repo." + assert kwargs["messages"][-1]["role"] == "user" + assert "required tool call" in kwargs["messages"][-1]["content"] + summary = stall_retry_summary(agent) + assert summary is not None + assert summary["attempted"] == 1 + assert summary["recovered"] == 1 def test_retry_on_stall_can_accept_visible_content(monkeypatch) -> None: @@ -138,6 +217,7 @@ def interruptible_api_call(kwargs: dict[str, object]) -> object: ) monkeypatch.setenv("HERMES_STALL_RETRY_MODEL", "qwen3.6-27b-256k") + monkeypatch.setenv("HERMES_STALL_RETRY_TELEMETRY", "0") result = retry_on_stall( agent, [{"role": "user", "content": "go"}], @@ -178,6 +258,7 @@ def test_retry_on_stall_still_rejects_content_without_accept_content(monkeypatch ) monkeypatch.setenv("HERMES_STALL_RETRY_MODEL", "qwen3.6-27b-256k") + monkeypatch.setenv("HERMES_STALL_RETRY_TELEMETRY", "0") assert retry_on_stall( agent, [{"role": "user", "content": "go"}], @@ -185,6 +266,218 @@ def test_retry_on_stall_still_rejects_content_without_accept_content(monkeypatch ) is None +def test_retry_on_stall_uses_agent_config_when_env_is_absent(monkeypatch) -> None: + captured: dict[str, object] = {} + tool_call = SimpleNamespace( + function=SimpleNamespace(name="terminal", arguments='{"cmd":"pwd"}') + ) + normalized = SimpleNamespace( + content="", + tool_calls=[tool_call], + finish_reason="tool_calls", + ) + + def interruptible_api_call(kwargs: dict[str, object]) -> object: + captured["kwargs"] = dict(kwargs) + return normalized + + agent = SimpleNamespace( + api_mode="openai", + _is_anthropic_oauth=False, + log_prefix="", + _stall_retry_config={ + "model": "qwen3.6-27b-256k", + "max_chars": 240, + "max_per_turn": 7, + }, + _build_api_kwargs=lambda messages: { + "model": "dflash", + "messages": messages, + "stream": True, + }, + _interruptible_api_call=interruptible_api_call, + _get_transport=lambda: SimpleNamespace( + normalize_response=lambda response, **_kwargs: response + ), + _vprint=lambda *_args, **_kwargs: None, + ) + + monkeypatch.delenv("HERMES_STALL_RETRY_MODEL", raising=False) + monkeypatch.delenv("HERMES_STALL_RETRY_MAX_CHARS", raising=False) + monkeypatch.delenv("HERMES_STALL_RETRY_MAX_PER_TURN", raising=False) + monkeypatch.setenv("HERMES_STALL_RETRY_TELEMETRY", "0") + + assert get_stall_retry_model(agent) == "qwen3.6-27b-256k" + assert get_stall_retry_max_chars(agent) == 240 + assert get_stall_retry_max_per_turn(agent) == 7 + assert get_stall_retry_nudge_enabled(agent) + + result = retry_on_stall(agent, [{"role": "user", "content": "go"}], "stop") + + assert result is normalized + assert captured["kwargs"]["model"] == "qwen3.6-27b-256k" + assert captured["kwargs"]["stream"] is False + + +def test_retry_on_stall_uses_configured_retry_provider(monkeypatch) -> None: + captured: dict[str, object] = {} + tool_call = SimpleNamespace( + function=SimpleNamespace(name="terminal", arguments='{"cmd":"pwd"}') + ) + normalized = SimpleNamespace( + content="", + tool_calls=[tool_call], + finish_reason="tool_calls", + ) + + class FakeCompletions: + def create(self, **kwargs: object) -> object: + captured["kwargs"] = dict(kwargs) + return normalized + + fake_client = SimpleNamespace( + api_key="retry-key", + base_url="http://taro:8080/v1", + chat=SimpleNamespace(completions=FakeCompletions()), + ) + + def fake_resolve_provider_client(**kwargs: object) -> tuple[object, str]: + captured["resolve_kwargs"] = dict(kwargs) + return fake_client, "qwen3.6-27b-256k" + + agent = SimpleNamespace( + api_mode="openai", + _is_anthropic_oauth=False, + base_url="http://taro:8080/v1", + log_prefix="", + _stall_retry_config={ + "model": "qwen3.6-27b-256k", + "provider": "taro", + "api_key_env": "HERMES_RETRY_TEST_KEY", + }, + _build_api_kwargs=lambda messages: { + "model": "dflash", + "messages": messages, + "stream": True, + }, + _interruptible_api_call=lambda _kwargs: (_ for _ in ()).throw( + AssertionError("same-client retry should not be used") + ), + _get_transport=lambda: SimpleNamespace( + normalize_response=lambda response, **_kwargs: response + ), + _vprint=lambda *_args, **_kwargs: None, + ) + + monkeypatch.delenv("HERMES_STALL_RETRY_MODEL", raising=False) + monkeypatch.setenv("HERMES_RETRY_TEST_KEY", "retry-key") + monkeypatch.setenv("HERMES_STALL_RETRY_TELEMETRY", "0") + monkeypatch.setattr( + "agent.auxiliary_client.resolve_provider_client", + lambda provider, **kwargs: fake_resolve_provider_client(provider=provider, **kwargs), + ) + + result = retry_on_stall(agent, [{"role": "user", "content": "go"}], "stop") + + assert result is normalized + assert captured["resolve_kwargs"]["provider"] == "taro" + assert captured["resolve_kwargs"]["model"] == "qwen3.6-27b-256k" + assert captured["resolve_kwargs"]["explicit_api_key"] == "retry-key" + assert captured["kwargs"]["model"] == "qwen3.6-27b-256k" + assert captured["kwargs"]["stream"] is False + + +def test_retry_on_stall_can_disable_retry_nudge(monkeypatch) -> None: + captured: dict[str, object] = {} + tool_call = SimpleNamespace( + function=SimpleNamespace(name="terminal", arguments='{"cmd":"pwd"}') + ) + normalized = SimpleNamespace( + content="", + tool_calls=[tool_call], + finish_reason="tool_calls", + ) + + def interruptible_api_call(kwargs: dict[str, object]) -> object: + captured["kwargs"] = dict(kwargs) + return normalized + + agent = SimpleNamespace( + api_mode="openai", + _is_anthropic_oauth=False, + log_prefix="", + _stall_retry_config={ + "model": "qwen3.6-27b-256k", + "nudge": False, + "telemetry": False, + }, + _build_api_kwargs=lambda messages: { + "model": "dflash", + "messages": messages, + "stream": True, + }, + _interruptible_api_call=interruptible_api_call, + _get_transport=lambda: SimpleNamespace( + normalize_response=lambda response, **_kwargs: response + ), + _vprint=lambda *_args, **_kwargs: None, + ) + + monkeypatch.delenv("HERMES_STALL_RETRY_MODEL", raising=False) + monkeypatch.delenv("HERMES_STALL_RETRY_NUDGE", raising=False) + monkeypatch.delenv("HERMES_STALL_RETRY_TELEMETRY", raising=False) + + assert not get_stall_retry_nudge_enabled(agent) + + result = retry_on_stall( + agent, + [{"role": "user", "content": "go"}], + "stop", + stalled_content="Let me check.", + ) + + assert result is normalized + assert captured["kwargs"]["messages"] == [{"role": "user", "content": "go"}] + + +def test_stall_retry_telemetry_writes_bounded_local_jsonl(tmp_path) -> None: + log_path = tmp_path / "stall-retry.ndjson" + agent = SimpleNamespace( + session_id="s1", + model="dflash", + provider="nous", + _stall_retry_config={ + "telemetry": True, + "telemetry_path": str(log_path), + }, + ) + + record_stall_retry_event( + agent, + "detected", + retry_model="qwen3.6-27b-256k", + content="Let me check the repo.\n" * 30, + ) + + lines = log_path.read_text(encoding="utf-8").splitlines() + assert len(lines) == 1 + event = json.loads(lines[0]) + assert event["event"] == "detected" + assert event["session_id"] == "s1" + assert event["model"] == "dflash" + assert event["provider"] == "nous" + assert event["retry_model"] == "qwen3.6-27b-256k" + assert event["content_chars"] > len(event["content_preview"]) + assert len(event["content_preview"]) <= 240 + assert "messages" not in event + + summary = stall_retry_summary(agent) + assert summary is not None + assert summary["detected"] == 1 + assert summary["events"] == 1 + assert summary["log_path"] == str(log_path) + + def test_conversation_loop_adopts_retry_before_tool_call_branch() -> None: source = inspect.getsource(conversation_loop.run_conversation) retry_idx = source.index("retried = retry_on_stall") @@ -195,15 +488,16 @@ def test_conversation_loop_adopts_retry_before_tool_call_branch() -> None: assert "stall_retry_failed_no_tool_call" in source -def test_conversation_loop_retries_empty_post_tool_before_generic_stall() -> None: +def test_conversation_loop_retries_empty_post_tool_before_tool_branch() -> None: source = inspect.getsource(conversation_loop.run_conversation) - empty_retry_idx = source.index("EMPTY_AFTER_TOOL_RETRY_NUDGE") + empty_retry_idx = source.index("empty_after_tool_result") generic_stall_idx = source.index("looks_like_stall(") tool_branch_idx = source.index("# Check for tool calls") assert empty_retry_idx < generic_stall_idx assert empty_retry_idx < tool_branch_idx assert "not _empty_after_tool_result and looks_like_stall" in source + assert "EMPTY_AFTER_TOOL_RETRY_NUDGE" in source assert "accept_content=True" in source @@ -212,5 +506,5 @@ def test_conversation_loop_allows_bounded_multiple_stall_retries() -> None: assert "_stall_retry_used" not in source assert "_stall_retry_count += 1" in source - assert "HERMES_STALL_RETRY_MAX_PER_TURN" in source + assert "get_stall_retry_max_per_turn" in source assert "stall_retry_limit_exhausted" in source From e7111504c4564ad44f257bc40632499d51e9509b Mon Sep 17 00:00:00 2001 From: Omar Baradei Date: Sun, 31 May 2026 15:50:26 -0700 Subject: [PATCH 07/13] fix(agent): honor global stall retry config --- agent/stall_retry.py | 22 +++++++++++++++++----- tests/agent/test_stall_retry.py | 27 +++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/agent/stall_retry.py b/agent/stall_retry.py index ede9c985a8774..c5853900f8cd4 100644 --- a/agent/stall_retry.py +++ b/agent/stall_retry.py @@ -101,17 +101,29 @@ def _as_bool(value: Any, default: bool) -> bool: def _stall_retry_config(agent: Any | None = None) -> Mapping[str, Any]: - cfg = getattr(agent, "_stall_retry_config", None) - if isinstance(cfg, Mapping): - return cfg + loaded_cfg: Mapping[str, Any] = {} try: from hermes_cli.config import load_config loaded = load_config() except Exception: - return {} + loaded = {} cfg = loaded.get("stall_retry") if isinstance(loaded, Mapping) else None - return cfg if isinstance(cfg, Mapping) else {} + if isinstance(cfg, Mapping): + loaded_cfg = cfg + + agent_cfg = getattr(agent, "_stall_retry_config", None) + if not isinstance(agent_cfg, Mapping): + return loaded_cfg + if not agent_cfg: + return loaded_cfg + + merged = dict(loaded_cfg) + for key, value in agent_cfg.items(): + if value is None or value == "": + continue + merged[str(key)] = value + return merged def get_stall_retry_model(agent: Any | None = None) -> str: diff --git a/tests/agent/test_stall_retry.py b/tests/agent/test_stall_retry.py index 5761edd55139a..ee66ae2d45c39 100644 --- a/tests/agent/test_stall_retry.py +++ b/tests/agent/test_stall_retry.py @@ -319,6 +319,33 @@ def interruptible_api_call(kwargs: dict[str, object]) -> object: assert captured["kwargs"]["stream"] is False +def test_stall_retry_empty_agent_config_falls_back_to_loaded_config(monkeypatch) -> None: + import hermes_cli.config as config_mod + + monkeypatch.delenv("HERMES_STALL_RETRY_MODEL", raising=False) + monkeypatch.delenv("HERMES_STALL_RETRY_PROVIDER", raising=False) + monkeypatch.delenv("HERMES_STALL_RETRY_MAX_PER_TURN", raising=False) + monkeypatch.setattr( + config_mod, + "load_config", + lambda: { + "stall_retry": { + "max_per_turn": 3, + "model": "qwen3.6-27b-256k", + "provider": "taro", + } + }, + ) + + empty_agent = SimpleNamespace(_stall_retry_config={}) + provider_only_agent = SimpleNamespace(_stall_retry_config={"provider": "ko-mac"}) + + assert get_stall_retry_model(empty_agent) == "qwen3.6-27b-256k" + assert get_stall_retry_max_per_turn(empty_agent) == 3 + assert get_stall_retry_model(provider_only_agent) == "qwen3.6-27b-256k" + assert get_stall_retry_max_per_turn(provider_only_agent) == 3 + + def test_retry_on_stall_uses_configured_retry_provider(monkeypatch) -> None: captured: dict[str, object] = {} tool_call = SimpleNamespace( From 89f16d519b8a742c60bd937dbf9b78a0ee9d0715 Mon Sep 17 00:00:00 2001 From: Omar Baradei Date: Sun, 31 May 2026 16:06:05 -0700 Subject: [PATCH 08/13] fix(agent): retry short incomplete dflash finals --- agent/conversation_loop.py | 43 ++++++++++++++++++++++++--------- agent/stall_retry.py | 35 ++++++++++++++++++++++++++- tests/agent/test_stall_retry.py | 22 +++++++++++++++++ 3 files changed, 88 insertions(+), 12 deletions(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index d2ec109c7eaad..2872ae0041084 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -3670,25 +3670,27 @@ def _stop_spinner(): # turn as partial instead of persisting the planning-only text as # a completed assistant message that poisons future "continue" # turns. - retry_model = os.environ.get("HERMES_STALL_RETRY_MODEL", "").strip() + try: + from agent.stall_retry import get_stall_retry_model + + retry_model = get_stall_retry_model(agent) + except Exception: + retry_model = os.environ.get("HERMES_STALL_RETRY_MODEL", "").strip() if ( retry_model and getattr(agent, "tools", None) and not getattr(assistant_message, "tool_calls", None) ): - try: - max_chars = int( - os.environ.get("HERMES_STALL_RETRY_MAX_CHARS", "400") or 400 - ) - except ValueError: - max_chars = 400 try: from agent.stall_retry import ( EMPTY_AFTER_TOOL_RETRY_NUDGE, + get_stall_retry_max_chars, + looks_like_incomplete_final_fragment, looks_like_stall, retry_on_stall, ) + max_chars = get_stall_retry_max_chars(agent) _empty_after_tool_result = ( getattr(agent, "tools", None) and not getattr(assistant_message, "tool_calls", None) @@ -3709,6 +3711,7 @@ def _stop_spinner(): agent, api_messages, finish_reason, + stalled_content=assistant_message.content or "", accept_content=True, retry_nudge=EMPTY_AFTER_TOOL_RETRY_NUDGE, ) @@ -3733,6 +3736,12 @@ def _stop_spinner(): agent.provider, ) + _retry_accepts_content = looks_like_incomplete_final_fragment( + assistant_message.content or "", + finish_reason, + False, + max_chars, + ) if not _empty_after_tool_result and looks_like_stall( assistant_message.content or "", finish_reason, @@ -3768,11 +3777,23 @@ def _stop_spinner(): "failure_subclass": "stall_retry_limit_exhausted", } _stall_retry_count += 1 - retried = retry_on_stall(agent, api_messages, finish_reason) - if retried is not None and getattr(retried, "tool_calls", None): + retried = retry_on_stall( + agent, + api_messages, + finish_reason, + stalled_content=assistant_message.content or "", + accept_content=_retry_accepts_content, + ) + if retried is not None and ( + getattr(retried, "tool_calls", None) or _retry_accepts_content + ): assistant_message = retried - finish_reason = getattr(retried, "finish_reason", None) or "tool_calls" - if finish_reason != "tool_calls": + finish_reason = getattr(retried, "finish_reason", None) or ( + "tool_calls" + if getattr(retried, "tool_calls", None) + else "stop" + ) + if getattr(retried, "tool_calls", None) and finish_reason != "tool_calls": finish_reason = "tool_calls" else: _turn_exit_reason = "stall_retry_failed_no_tool_call" diff --git a/agent/stall_retry.py b/agent/stall_retry.py index c5853900f8cd4..4de4f2c1824ea 100644 --- a/agent/stall_retry.py +++ b/agent/stall_retry.py @@ -62,6 +62,14 @@ _NATURAL_END_CHARS = '.!?:)"\']}。！？：）】」』》^' _MIN_INCOMPLETE_FINAL_CHARS = 80 _ACTION_TAIL_CHARS = 500 +_INCOMPLETE_TAIL_RE = re.compile( + r"\b(" + r"and|or|but|so|because|while|with|without|for|to|of|in|on|at|by|from|" + r"the|a|an|this|that|these|those|some|any|another|more|other|which|who|" + r"where|when|if|then|also" + r")\s*$", + re.IGNORECASE, +) _STALL_RETRY_NUDGE = ( "Your previous assistant response ended after describing the next action, " "but it did not include the required tool call. Continue the same task now " @@ -200,6 +208,31 @@ def _has_natural_response_ending(content: str) -> bool: return ord(last) >= 0x1F300 +def looks_like_incomplete_final_fragment( + content: str, + finish_reason: str, + has_tool_calls: bool, + max_chars: int, +) -> bool: + """True when a short no-tool final looks cut off mid-sentence. + + dflash can occasionally stop with ordinary visible prose after a tool + result, e.g. ``"... and some"``. That is not an action preamble, but it is + still unsafe to persist as the final answer in a tool loop. + """ + if has_tool_calls or finish_reason not in ("stop", "length"): + return False + c = (content or "").strip() + c = re.sub(r"^.*?\s*", "", c, flags=re.IGNORECASE | re.DOTALL).strip() + if not c or len(c) > max_chars: + return False + if _COMPLETION_RE.search(c) or _has_natural_response_ending(c): + return False + if len(c) >= _MIN_INCOMPLETE_FINAL_CHARS: + return True + return len(c) >= 40 and bool(_INCOMPLETE_TAIL_RE.search(c)) + + def _action_after_completion(content: str) -> bool: """True when a response says some step is complete, then promises work. @@ -490,7 +523,7 @@ def looks_like_stall(content: str, finish_reason: str, has_tool_calls: bool, # the turn). In an agentic tool loop, a short no-tool stop that declares no # completion and lacks a natural ending is safer to retry than to persist as # a final assistant message. - if len(c) >= _MIN_INCOMPLETE_FINAL_CHARS and not _has_natural_response_ending(c): + if looks_like_incomplete_final_fragment(c, finish_reason, False, max_chars): return True return False diff --git a/tests/agent/test_stall_retry.py b/tests/agent/test_stall_retry.py index ee66ae2d45c39..9055c867b8345 100644 --- a/tests/agent/test_stall_retry.py +++ b/tests/agent/test_stall_retry.py @@ -11,6 +11,7 @@ get_stall_retry_max_per_turn, get_stall_retry_model, get_stall_retry_nudge_enabled, + looks_like_incomplete_final_fragment, looks_like_stall, record_stall_retry_event, retry_on_stall, @@ -111,6 +112,13 @@ def test_incomplete_final_fragment_without_action_preamble_is_a_stall() -> None: ) +def test_short_incomplete_connector_tail_is_a_stall() -> None: + content = "I see a lot of discord-res tasks (digest Discord content) and some" + + assert looks_like_incomplete_final_fragment(content, "stop", False, 400) + assert looks_like_stall(content, "stop", False, 400) + + def test_empty_visible_response_uses_empty_response_recovery_not_stall_retry() -> None: assert not looks_like_stall("", "stop", False, 400) assert not looks_like_stall("still reasoning", "stop", False, 400) @@ -528,6 +536,20 @@ def test_conversation_loop_retries_empty_post_tool_before_tool_branch() -> None: assert "accept_content=True" in source +def test_conversation_loop_uses_configured_stall_retry_model() -> None: + source = inspect.getsource(conversation_loop.run_conversation) + + assert "get_stall_retry_model(agent)" in source + assert 'os.environ.get("HERMES_STALL_RETRY_MODEL"' in source + + +def test_conversation_loop_accepts_content_for_incomplete_final_fragment() -> None: + source = inspect.getsource(conversation_loop.run_conversation) + + assert "looks_like_incomplete_final_fragment" in source + assert "accept_content=_retry_accepts_content" in source + + def test_conversation_loop_allows_bounded_multiple_stall_retries() -> None: source = inspect.getsource(conversation_loop.run_conversation) From 238b9ac1ebbd0bea1eb80c1039548e1fb8df9400 Mon Sep 17 00:00:00 2001 From: Omar Baradei Date: Sun, 31 May 2026 16:42:10 -0700 Subject: [PATCH 09/13] fix(agent): keep post-tool empty recovery on local retry lane --- agent/conversation_loop.py | 6 ++---- agent/stall_retry.py | 24 +++++++++++++++++++++++- tests/agent/test_stall_retry.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 2872ae0041084..f8269e7e32ba1 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -3685,6 +3685,7 @@ def _stop_spinner(): from agent.stall_retry import ( EMPTY_AFTER_TOOL_RETRY_NUDGE, get_stall_retry_max_chars, + has_recent_tool_result, looks_like_incomplete_final_fragment, looks_like_stall, retry_on_stall, @@ -3697,10 +3698,7 @@ def _stop_spinner(): and not agent._strip_think_blocks( assistant_message.content or "" ).strip() - and any( - isinstance(m, dict) and m.get("role") == "tool" - for m in messages[-5:] - ) + and has_recent_tool_result(messages) ) if ( _empty_after_tool_result diff --git a/agent/stall_retry.py b/agent/stall_retry.py index 4de4f2c1824ea..13167f3249004 100644 --- a/agent/stall_retry.py +++ b/agent/stall_retry.py @@ -37,7 +37,7 @@ import re from datetime import datetime, timezone from pathlib import Path -from typing import Any, Mapping +from typing import Any, Mapping, Sequence # Action-preamble signature: the turn announced an action but produced no tool # call. These English phrases match the observed dflash stall corpus; broader @@ -108,6 +108,28 @@ def _as_bool(value: Any, default: bool) -> bool: return default +def has_recent_tool_result(messages: Sequence[Any], *, lookback: int = 24) -> bool: + """Return whether the current turn has a recent tool result. + + Stop at the current turn's user message so a tool-heavy previous turn does + not make an unrelated empty first response look like post-tool fallout. + """ + + checked = 0 + for msg in reversed(messages): + if checked >= lookback: + return False + if not isinstance(msg, Mapping): + continue + role = msg.get("role") + if role == "tool": + return True + if role == "user": + return False + checked += 1 + return False + + def _stall_retry_config(agent: Any | None = None) -> Mapping[str, Any]: loaded_cfg: Mapping[str, Any] = {} try: diff --git a/tests/agent/test_stall_retry.py b/tests/agent/test_stall_retry.py index 9055c867b8345..46ed21731fdd9 100644 --- a/tests/agent/test_stall_retry.py +++ b/tests/agent/test_stall_retry.py @@ -11,6 +11,7 @@ get_stall_retry_max_per_turn, get_stall_retry_model, get_stall_retry_nudge_enabled, + has_recent_tool_result, looks_like_incomplete_final_fragment, looks_like_stall, record_stall_retry_event, @@ -124,6 +125,34 @@ def test_empty_visible_response_uses_empty_response_recovery_not_stall_retry() - assert not looks_like_stall("still reasoning", "stop", False, 400) +def test_recent_tool_result_detector_spans_status_scaffolding() -> None: + messages = [ + {"role": "user", "content": "do the onboarding"}, + {"role": "assistant", "tool_calls": [{"id": "call_1"}]}, + {"role": "tool", "content": "Onboarding complete"}, + {"role": "assistant", "content": ""}, + {"role": "assistant", "content": ""}, + {"role": "assistant", "content": ""}, + {"role": "system", "content": "status update"}, + {"role": "assistant", "content": ""}, + ] + + assert has_recent_tool_result(messages) + + +def test_recent_tool_result_detector_stops_at_current_user() -> None: + messages = [ + {"role": "user", "content": "old task"}, + {"role": "assistant", "tool_calls": [{"id": "call_1"}]}, + {"role": "tool", "content": "old result"}, + {"role": "assistant", "content": "done"}, + {"role": "user", "content": "new task"}, + {"role": "assistant", "content": ""}, + ] + + assert not has_recent_tool_result(messages) + + def test_short_status_answer_without_punctuation_is_not_a_stall() -> None: assert not looks_like_stall("main", "stop", False, 400) From f2a9ffcad0cf8ef48d23fc6fe2ab75f302fe59dd Mon Sep 17 00:00:00 2001 From: Omar Baradei Date: Sun, 31 May 2026 17:29:50 -0700 Subject: [PATCH 10/13] fix(agent): promote dflash recovery lane after repeated stalls --- agent/conversation_loop.py | 52 ++++++- agent/stall_retry.py | 242 +++++++++++++++++++++++++++++++- tests/agent/test_stall_retry.py | 96 +++++++++++++ 3 files changed, 382 insertions(+), 8 deletions(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index f8269e7e32ba1..8d904858af10f 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -794,16 +794,25 @@ def run_conversation( ) # Agentic stall-retry guard: dflash can stall more than once in a long - # tool loop, so allow a bounded number of successful rescues per user turn. - # If the cap is exhausted, fail partial rather than accept another - # planning-only text response as final. See agent/stall_retry.py. + # tool loop. Count total attempts for telemetry, but only consume the + # terminal budget when the retry lane fails to produce forward progress. + # A recovered tool call is work advanced, not a reason to kill the turn. + # See agent/stall_retry.py. _stall_retry_count = 0 + _stall_retry_success_count = 0 + _stall_retry_failed_count = 0 + agent._stall_retry_runtime_promoted = False try: - from agent.stall_retry import get_stall_retry_max_per_turn + from agent.stall_retry import ( + get_stall_retry_max_per_turn, + get_stall_retry_promote_after, + ) _stall_retry_max_per_turn = get_stall_retry_max_per_turn(agent) + _stall_retry_promote_after = get_stall_retry_promote_after(agent) except Exception: _stall_retry_max_per_turn = 5 + _stall_retry_promote_after = 2 while (api_call_count < agent.max_iterations and agent.iteration_budget.remaining > 0) or agent._budget_grace_call: # Reset per-turn checkpoint dedup so each iteration can take one snapshot @@ -3684,6 +3693,7 @@ def _stop_spinner(): try: from agent.stall_retry import ( EMPTY_AFTER_TOOL_RETRY_NUDGE, + activate_stall_retry_runtime, get_stall_retry_max_chars, has_recent_tool_result, looks_like_incomplete_final_fragment, @@ -3702,7 +3712,7 @@ def _stop_spinner(): ) if ( _empty_after_tool_result - and _stall_retry_count < _stall_retry_max_per_turn + and _stall_retry_failed_count < _stall_retry_max_per_turn ): _stall_retry_count += 1 retried = retry_on_stall( @@ -3725,7 +3735,21 @@ def _stop_spinner(): ) agent._empty_content_retries = 0 agent._post_tool_empty_retried = False + _stall_retry_success_count += 1 + _stall_retry_failed_count = 0 + if ( + getattr(retried, "tool_calls", None) + and _stall_retry_promote_after > 0 + and _stall_retry_success_count >= _stall_retry_promote_after + ): + activate_stall_retry_runtime( + agent, + retry_model, + promote_after=_stall_retry_promote_after, + successful_retries=_stall_retry_success_count, + ) else: + _stall_retry_failed_count += 1 logging.warning( "Stall retry lane did not recover empty " "post-tool response; continuing " @@ -3746,13 +3770,13 @@ def _stop_spinner(): False, max_chars, ): - if _stall_retry_count >= _stall_retry_max_per_turn: + if _stall_retry_max_per_turn <= 0: _turn_exit_reason = "stall_retry_limit_exhausted" agent._mute_post_response = False agent._vprint( ( f"{agent.log_prefix}❌ Stall retry limit " - f"({_stall_retry_max_per_turn}/turn) exhausted; " + f"({_stall_retry_max_per_turn}/turn) disabled; " "saving as partial without storing the " "planning-only assistant turn." ), @@ -3793,7 +3817,21 @@ def _stop_spinner(): ) if getattr(retried, "tool_calls", None) and finish_reason != "tool_calls": finish_reason = "tool_calls" + _stall_retry_success_count += 1 + _stall_retry_failed_count = 0 + if ( + getattr(retried, "tool_calls", None) + and _stall_retry_promote_after > 0 + and _stall_retry_success_count >= _stall_retry_promote_after + ): + activate_stall_retry_runtime( + agent, + retry_model, + promote_after=_stall_retry_promote_after, + successful_retries=_stall_retry_success_count, + ) else: + _stall_retry_failed_count += 1 _turn_exit_reason = "stall_retry_failed_no_tool_call" agent._mute_post_response = False agent._vprint( diff --git a/agent/stall_retry.py b/agent/stall_retry.py index 13167f3249004..ed99f34a21767 100644 --- a/agent/stall_retry.py +++ b/agent/stall_retry.py @@ -21,7 +21,12 @@ HERMES_STALL_RETRY_MODEL retry lane/model name (required to enable) HERMES_STALL_RETRY_PROVIDER optional provider override for the retry lane HERMES_STALL_RETRY_BASE_URL optional OpenAI-compatible retry endpoint - HERMES_STALL_RETRY_MAX_PER_TURN max retries per user turn (default 5) + HERMES_STALL_RETRY_MAX_PER_TURN max unrecovered retries per user turn + (default 5). Successful retry-lane tool calls + are progress and do not consume this budget. + HERMES_STALL_RETRY_PROMOTE_AFTER successful rescues before routing the rest + of the turn through the retry lane (default 2, + 0 disables) HERMES_STALL_RETRY_MAX_CHARS max content length to still count as a stall (default 400; longer open action preambles ending in ":" get a bounded exception) @@ -204,6 +209,21 @@ def get_stall_retry_max_per_turn(agent: Any | None = None) -> int: return 5 +def get_stall_retry_promote_after(agent: Any | None = None) -> int: + """Return successful retry rescues needed before turn-scoped promotion.""" + env_value = os.environ.get("HERMES_STALL_RETRY_PROMOTE_AFTER") + if env_value is not None: + try: + return max(0, int(env_value)) + except ValueError: + return 2 + cfg_value = _stall_retry_config(agent).get("promote_after") + try: + return max(0, int(cfg_value)) + except (TypeError, ValueError): + return 2 + + def get_stall_retry_nudge_enabled(agent: Any | None = None) -> bool: env_value = os.environ.get("HERMES_STALL_RETRY_NUDGE") if env_value is not None: @@ -511,6 +531,226 @@ def _retry_api_call(agent: Any, api_kwargs: dict[str, Any], retry_model: str) -> return agent._interruptible_api_call(api_kwargs) +def activate_stall_retry_runtime( + agent: Any, + retry_model: str, + *, + promote_after: int, + successful_retries: int, +) -> bool: + """Route the rest of this turn through the retry lane. + + A repeated pattern of primary-model stalls followed by retry-lane tool-call + recovery means the primary is no longer reliable for this agentic turn. This + helper promotes the already-configured retry lane into the active runtime + without updating ``_primary_runtime``; Hermes' normal start-of-turn restore + then switches back to the user's selected primary on the next user turn. + """ + + retry_model = str(retry_model or "").strip() + if not retry_model: + return False + if getattr(agent, "_stall_retry_runtime_promoted", False): + return False + + original_model = str(getattr(agent, "model", "") or "") + original_provider = str(getattr(agent, "provider", "") or "") + original_base_url = str(getattr(agent, "base_url", "") or "") + retry_provider = get_stall_retry_provider(agent) + retry_base_url = get_stall_retry_base_url(agent) + retry_api_key = _configured_retry_api_key(agent, retry_provider, retry_base_url) + if not retry_base_url and retry_api_key: + retry_base_url = original_base_url + + provider = retry_provider or original_provider + base_url = retry_base_url or original_base_url + api_key = retry_api_key or getattr(agent, "api_key", "") + client = None + resolved_model = retry_model + client_kwargs: dict[str, Any] = {} + + try: + if retry_provider or retry_base_url or retry_api_key: + from agent.auxiliary_client import resolve_provider_client + + client, provider_model = resolve_provider_client( + provider or "custom", + model=retry_model, + raw_codex=True, + explicit_base_url=retry_base_url or None, + explicit_api_key=retry_api_key or None, + ) + if client is None: + record_stall_retry_event( + agent, + "promotion_skipped", + retry_model=retry_model, + original_model=original_model, + reason="provider_unavailable", + promote_after=promote_after, + successful_retries=successful_retries, + ) + return False + resolved_model = provider_model or retry_model + base_url = str(getattr(client, "base_url", base_url) or base_url) + api_key = getattr(client, "api_key", api_key) + headers = ( + getattr(client, "_custom_headers", None) + or getattr(client, "default_headers", None) + ) + client_kwargs = { + "api_key": api_key, + "base_url": base_url, + **({"default_headers": dict(headers)} if headers else {}), + } + else: + if retry_model == original_model: + record_stall_retry_event( + agent, + "promotion_skipped", + retry_model=retry_model, + original_model=original_model, + reason="same_model", + promote_after=promote_after, + successful_retries=successful_retries, + ) + return False + client_kwargs = dict(getattr(agent, "_client_kwargs", {}) or {}) + if not client_kwargs: + client_kwargs = { + "api_key": api_key, + "base_url": base_url, + } + + try: + from hermes_cli.providers import determine_api_mode + + api_mode = determine_api_mode(provider, base_url) + except Exception: + api_mode = getattr(agent, "api_mode", "") or "chat_completions" + if api_mode in {"anthropic_messages", "bedrock_converse", "codex_responses"}: + record_stall_retry_event( + agent, + "promotion_skipped", + retry_model=retry_model, + original_model=original_model, + reason=f"unsupported_api_mode:{api_mode}", + promote_after=promote_after, + successful_retries=successful_retries, + ) + return False + + try: + from hermes_cli.timeouts import get_provider_request_timeout + + timeout = get_provider_request_timeout(provider, resolved_model) + if timeout is not None: + client_kwargs["timeout"] = timeout + except Exception: + pass + + agent._config_context_length = None + agent.model = resolved_model + agent.provider = provider + agent.base_url = base_url + agent.api_key = api_key + agent.api_mode = api_mode or "chat_completions" + agent._client_kwargs = client_kwargs + if hasattr(agent, "_transport_cache"): + agent._transport_cache.clear() + if client is not None: + agent.client = client + elif hasattr(agent, "_create_openai_client"): + agent.client = agent._create_openai_client( + dict(client_kwargs), + reason="stall_retry_runtime_promotion", + shared=True, + ) + + try: + agent._use_prompt_caching, agent._use_native_cache_layout = ( + agent._anthropic_prompt_cache_policy( + provider=provider, + base_url=base_url, + api_mode=agent.api_mode, + model=resolved_model, + ) + ) + except Exception: + pass + + if hasattr(agent, "_ensure_lmstudio_runtime_loaded"): + agent._ensure_lmstudio_runtime_loaded() + + context_length = None + if getattr(agent, "context_compressor", None): + try: + from agent.model_metadata import get_model_context_length + + ctx_api_key = api_key if isinstance(api_key, str) else "" + context_length = get_model_context_length( + resolved_model, + base_url=base_url, + api_key=ctx_api_key, + provider=provider, + config_context_length=getattr(agent, "_config_context_length", None), + custom_providers=getattr(agent, "_custom_providers", None), + ) + agent.context_compressor.update_model( + model=resolved_model, + context_length=context_length, + base_url=base_url, + api_key=api_key, + provider=provider, + api_mode=agent.api_mode, + ) + except Exception: + pass + + agent._fallback_activated = True + agent._stall_retry_runtime_promoted = True + agent._stall_retry_promoted_from = original_model + + record_stall_retry_event( + agent, + "runtime_promoted", + retry_model=resolved_model, + original_model=original_model, + original_provider=original_provider, + promote_after=promote_after, + successful_retries=successful_retries, + context_length=context_length, + ) + try: + agent._emit_status( + "↻ dflash stalled repeatedly; using " + f"{resolved_model} for the rest of this turn. " + "Primary model will be restored next turn." + ) + except Exception: + try: + agent._vprint( + f"{getattr(agent, 'log_prefix', '')}↻ dflash stalled " + f"repeatedly; using {resolved_model} for the rest of this turn.", + force=True, + ) + except Exception: + pass + return True + except Exception as exc: + record_stall_retry_event( + agent, + "promotion_exception", + retry_model=retry_model, + original_model=original_model, + error_type=type(exc).__name__, + error=str(exc)[:300], + promote_after=promote_after, + successful_retries=successful_retries, + ) + return False + + def looks_like_stall(content: str, finish_reason: str, has_tool_calls: bool, max_chars: int) -> bool: """True when a no-tool-call turn looks like a premature agentic stall diff --git a/tests/agent/test_stall_retry.py b/tests/agent/test_stall_retry.py index 46ed21731fdd9..bec232231a20a 100644 --- a/tests/agent/test_stall_retry.py +++ b/tests/agent/test_stall_retry.py @@ -7,10 +7,12 @@ from agent import conversation_loop from agent.stall_retry import ( EMPTY_AFTER_TOOL_RETRY_NUDGE, + activate_stall_retry_runtime, get_stall_retry_max_chars, get_stall_retry_max_per_turn, get_stall_retry_model, get_stall_retry_nudge_enabled, + get_stall_retry_promote_after, has_recent_tool_result, looks_like_incomplete_final_fragment, looks_like_stall, @@ -383,6 +385,83 @@ def test_stall_retry_empty_agent_config_falls_back_to_loaded_config(monkeypatch) assert get_stall_retry_max_per_turn(provider_only_agent) == 3 +def test_stall_retry_promote_after_uses_config_and_env(monkeypatch) -> None: + agent = SimpleNamespace(_stall_retry_config={"promote_after": 4}) + + monkeypatch.delenv("HERMES_STALL_RETRY_PROMOTE_AFTER", raising=False) + assert get_stall_retry_promote_after(agent) == 4 + + monkeypatch.setenv("HERMES_STALL_RETRY_PROMOTE_AFTER", "0") + assert get_stall_retry_promote_after(agent) == 0 + + +def test_activate_stall_retry_runtime_promotes_for_current_turn(monkeypatch) -> None: + captured: dict[str, object] = {} + fake_client = SimpleNamespace( + api_key="retry-key", + base_url="http://taro:8080/v1", + _custom_headers={"X-Test": "1"}, + ) + + def fake_resolve_provider_client(**kwargs: object) -> tuple[object, str]: + captured["resolve_kwargs"] = dict(kwargs) + return fake_client, "qwen3.6-27b-256k" + + def update_model(**kwargs: object) -> None: + captured["context_model"] = dict(kwargs) + + agent = SimpleNamespace( + model="dflash", + provider="custom", + base_url="http://primary:8080/v1", + api_key="primary-key", + api_mode="chat_completions", + _client_kwargs={"api_key": "primary-key", "base_url": "http://primary:8080/v1"}, + _stall_retry_config={ + "model": "qwen3.6-27b-256k", + "provider": "taro", + "base_url": "http://taro:8080/v1", + }, + _transport_cache={"chat": object()}, + _config_context_length=262144, + _custom_providers=[], + context_compressor=SimpleNamespace(update_model=update_model), + _anthropic_prompt_cache_policy=lambda **_kwargs: (False, False), + _ensure_lmstudio_runtime_loaded=lambda: None, + _emit_status=lambda message: captured.setdefault("status", message), + _fallback_activated=False, + ) + + monkeypatch.setenv("HERMES_STALL_RETRY_TELEMETRY", "0") + monkeypatch.setattr( + "agent.auxiliary_client.resolve_provider_client", + lambda provider, **kwargs: fake_resolve_provider_client( + provider=provider, **kwargs + ), + ) + monkeypatch.setattr( + "agent.model_metadata.get_model_context_length", + lambda *_args, **_kwargs: 262144, + ) + + assert activate_stall_retry_runtime( + agent, + "qwen3.6-27b-256k", + promote_after=2, + successful_retries=2, + ) + assert agent.model == "qwen3.6-27b-256k" + assert agent.provider == "taro" + assert str(agent.base_url) == "http://taro:8080/v1" + assert agent._fallback_activated is True + assert agent._stall_retry_runtime_promoted is True + assert agent._stall_retry_promoted_from == "dflash" + assert agent._transport_cache == {} + assert captured["resolve_kwargs"]["provider"] == "taro" + assert captured["context_model"]["model"] == "qwen3.6-27b-256k" + assert "rest of this turn" in captured["status"] + + def test_retry_on_stall_uses_configured_retry_provider(monkeypatch) -> None: captured: dict[str, object] = {} tool_call = SimpleNamespace( @@ -586,3 +665,20 @@ def test_conversation_loop_allows_bounded_multiple_stall_retries() -> None: assert "_stall_retry_count += 1" in source assert "get_stall_retry_max_per_turn" in source assert "stall_retry_limit_exhausted" in source + + +def test_conversation_loop_does_not_treat_recovered_stalls_as_failures() -> None: + source = inspect.getsource(conversation_loop.run_conversation) + + assert "_stall_retry_failed_count" in source + assert "if _stall_retry_count >= _stall_retry_max_per_turn" not in source + assert "A recovered tool call is work advanced" in source + + +def test_conversation_loop_promotes_retry_lane_after_repeated_rescues() -> None: + source = inspect.getsource(conversation_loop.run_conversation) + + assert "get_stall_retry_promote_after" in source + assert "_stall_retry_success_count += 1" in source + assert "activate_stall_retry_runtime(" in source + assert "_stall_retry_success_count >= _stall_retry_promote_after" in source From 50212031d3282fdf19e02da3ed4864f123026766 Mon Sep 17 00:00:00 2001 From: Omar Baradei Date: Sun, 31 May 2026 21:05:07 -0700 Subject: [PATCH 11/13] fix(agent): recover failed stall retry loops --- agent/conversation_loop.py | 59 +++++++++++++++-- agent/stall_retry.py | 51 +++++++++++--- tests/agent/test_stall_retry.py | 114 ++++++++++++++++++++++++++++++++ 3 files changed, 207 insertions(+), 17 deletions(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 8d904858af10f..207dd327be283 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -801,17 +801,21 @@ def run_conversation( _stall_retry_count = 0 _stall_retry_success_count = 0 _stall_retry_failed_count = 0 + _stall_retry_no_tool_recovery_count = 0 agent._stall_retry_runtime_promoted = False try: from agent.stall_retry import ( get_stall_retry_max_per_turn, + get_stall_retry_no_tool_recovery_max, get_stall_retry_promote_after, ) _stall_retry_max_per_turn = get_stall_retry_max_per_turn(agent) + _stall_retry_no_tool_recovery_max = get_stall_retry_no_tool_recovery_max(agent) _stall_retry_promote_after = get_stall_retry_promote_after(agent) except Exception: _stall_retry_max_per_turn = 5 + _stall_retry_no_tool_recovery_max = 2 _stall_retry_promote_after = 2 while (api_call_count < agent.max_iterations and agent.iteration_budget.remaining > 0) or agent._budget_grace_call: @@ -3673,12 +3677,10 @@ def _stop_spinner(): # dflash Q4 can stop right after an action preamble ("Let me # check X") without producing the promised tool_call. Retry the # exact same turn on the configured higher-quality lane before - # the final-response branch sees it. If the retry returns tool - # calls, fall through to the normal executor below in this same - # loop iteration. If it still returns no tool call, fail this - # turn as partial instead of persisting the planning-only text as - # a completed assistant message that poisons future "continue" - # turns. + # the final-response branch sees it. If it still returns no tool + # call, feed a bounded corrective continuation back into the same + # turn before finally failing as partial; this prevents one bad + # retry-lane sample from killing long mobile sessions. try: from agent.stall_retry import get_stall_retry_model @@ -3693,11 +3695,13 @@ def _stop_spinner(): try: from agent.stall_retry import ( EMPTY_AFTER_TOOL_RETRY_NUDGE, + FAILED_STALL_RETRY_RECOVERY_NUDGE, activate_stall_retry_runtime, get_stall_retry_max_chars, has_recent_tool_result, looks_like_incomplete_final_fragment, looks_like_stall, + record_stall_retry_event, retry_on_stall, ) @@ -3799,11 +3803,13 @@ def _stop_spinner(): "failure_subclass": "stall_retry_limit_exhausted", } _stall_retry_count += 1 + stalled_content = assistant_message.content or "" retried = retry_on_stall( agent, api_messages, finish_reason, - stalled_content=assistant_message.content or "", + stalled_content=stalled_content, + retry_index=_stall_retry_count, accept_content=_retry_accepts_content, ) if retried is not None and ( @@ -3832,6 +3838,45 @@ def _stop_spinner(): ) else: _stall_retry_failed_count += 1 + if ( + _stall_retry_no_tool_recovery_count + < _stall_retry_no_tool_recovery_max + ): + _stall_retry_no_tool_recovery_count += 1 + record_stall_retry_event( + agent, + "no_tool_recovery_prompt", + finish_reason=finish_reason, + api_call=api_call_count, + retry_count=_stall_retry_count, + failed_count=_stall_retry_failed_count, + recovery_count=_stall_retry_no_tool_recovery_count, + recovery_max=_stall_retry_no_tool_recovery_max, + content=stalled_content, + ) + agent._vprint( + ( + f"{agent.log_prefix}↻ Stall retry still " + "returned no tool call; feeding a bounded " + "same-turn correction back to the model " + f"({_stall_retry_no_tool_recovery_count}/" + f"{_stall_retry_no_tool_recovery_max})." + ), + force=True, + ) + assistant_msg = agent._build_assistant_message( + assistant_message, + finish_reason, + ) + messages.append(assistant_msg) + messages.append( + { + "role": "user", + "content": FAILED_STALL_RETRY_RECOVERY_NUDGE, + } + ) + agent._session_messages = messages + continue _turn_exit_reason = "stall_retry_failed_no_tool_call" agent._mute_post_response = False agent._vprint( diff --git a/agent/stall_retry.py b/agent/stall_retry.py index ed99f34a21767..bad5cc6e1505c 100644 --- a/agent/stall_retry.py +++ b/agent/stall_retry.py @@ -27,6 +27,9 @@ HERMES_STALL_RETRY_PROMOTE_AFTER successful rescues before routing the rest of the turn through the retry lane (default 2, 0 disables) + HERMES_STALL_RETRY_NO_TOOL_RECOVERY_MAX corrective same-turn continuations + after the retry lane also returns no tool call + (default 2, 0 disables) HERMES_STALL_RETRY_MAX_CHARS max content length to still count as a stall (default 400; longer open action preambles ending in ":" get a bounded exception) @@ -87,6 +90,13 @@ "requires another tool, call it immediately; otherwise provide the next " "concise response. Do not summarize or apologize." ) +FAILED_STALL_RETRY_RECOVERY_NUDGE = ( + "The previous assistant response again described an action but did not " + "include a tool call. Continue the same task now. If that action requires " + "a tool, call it immediately. If no tool is needed because the task is " + "complete, state completion explicitly instead of describing another " + "future action." +) def _as_positive_int(value: Any, default: int) -> int: @@ -97,6 +107,14 @@ def _as_positive_int(value: Any, default: int) -> int: return parsed if parsed > 0 else default +def _as_nonnegative_int(value: Any, default: int) -> int: + try: + parsed = int(value) + except (TypeError, ValueError): + return default + return max(0, parsed) + + def _as_bool(value: Any, default: bool) -> bool: if value is None: return default @@ -213,15 +231,19 @@ def get_stall_retry_promote_after(agent: Any | None = None) -> int: """Return successful retry rescues needed before turn-scoped promotion.""" env_value = os.environ.get("HERMES_STALL_RETRY_PROMOTE_AFTER") if env_value is not None: - try: - return max(0, int(env_value)) - except ValueError: - return 2 - cfg_value = _stall_retry_config(agent).get("promote_after") - try: - return max(0, int(cfg_value)) - except (TypeError, ValueError): - return 2 + return _as_nonnegative_int(env_value, 2) + return _as_nonnegative_int(_stall_retry_config(agent).get("promote_after"), 2) + + +def get_stall_retry_no_tool_recovery_max(agent: Any | None = None) -> int: + """Return corrective continuations allowed after retry returns no tools.""" + env_value = os.environ.get("HERMES_STALL_RETRY_NO_TOOL_RECOVERY_MAX") + if env_value is not None: + return _as_nonnegative_int(env_value, 2) + return _as_nonnegative_int( + _stall_retry_config(agent).get("no_tool_recovery_max"), + 2, + ) def get_stall_retry_nudge_enabled(agent: Any | None = None) -> bool: @@ -828,7 +850,8 @@ def retry_on_stall( # so no client rebuild is needed. api_kwargs = agent._build_api_kwargs(retry_messages) orig_model = api_kwargs.get("model") - if retry_model == orig_model: + same_model_retry = retry_model == orig_model + if same_model_retry and not getattr(agent, "_stall_retry_runtime_promoted", False): record_stall_retry_event( agent, "skipped_same_model", @@ -837,6 +860,14 @@ def retry_on_stall( retry_index=retry_index, ) return None # nothing to gain retrying the same model + if same_model_retry: + record_stall_retry_event( + agent, + "same_model_retry_after_promotion", + retry_model=retry_model, + finish_reason=finish_reason, + retry_index=retry_index, + ) api_kwargs = dict(api_kwargs) api_kwargs["model"] = retry_model # Force non-streaming for the retry (simpler, we only inspect the result). diff --git a/tests/agent/test_stall_retry.py b/tests/agent/test_stall_retry.py index bec232231a20a..e3a9aaf586be0 100644 --- a/tests/agent/test_stall_retry.py +++ b/tests/agent/test_stall_retry.py @@ -7,10 +7,12 @@ from agent import conversation_loop from agent.stall_retry import ( EMPTY_AFTER_TOOL_RETRY_NUDGE, + FAILED_STALL_RETRY_RECOVERY_NUDGE, activate_stall_retry_runtime, get_stall_retry_max_chars, get_stall_retry_max_per_turn, get_stall_retry_model, + get_stall_retry_no_tool_recovery_max, get_stall_retry_nudge_enabled, get_stall_retry_promote_after, has_recent_tool_result, @@ -305,6 +307,95 @@ def test_retry_on_stall_still_rejects_content_without_accept_content(monkeypatch ) is None +def test_retry_on_stall_skips_same_model_before_runtime_promotion(monkeypatch) -> None: + captured = {"called": False} + + agent = SimpleNamespace( + api_mode="openai", + _is_anthropic_oauth=False, + log_prefix="", + _build_api_kwargs=lambda messages: { + "model": "qwen3.6-27b-256k", + "messages": messages, + "stream": True, + }, + _interruptible_api_call=lambda _kwargs: captured.__setitem__("called", True), + _get_transport=lambda: SimpleNamespace( + normalize_response=lambda response, **_kwargs: response + ), + _vprint=lambda *_args, **_kwargs: None, + ) + + monkeypatch.setenv("HERMES_STALL_RETRY_MODEL", "qwen3.6-27b-256k") + monkeypatch.setenv("HERMES_STALL_RETRY_TELEMETRY", "0") + + assert retry_on_stall(agent, [{"role": "user", "content": "go"}], "stop") is None + assert captured["called"] is False + summary = stall_retry_summary(agent) + assert summary is not None + assert any( + event.get("event") == "skipped_same_model" + for event in getattr(agent, "_stall_retry_events", []) + ) + + +def test_retry_on_stall_allows_same_model_after_runtime_promotion(monkeypatch) -> None: + captured: dict[str, object] = {} + tool_call = SimpleNamespace( + function=SimpleNamespace(name="terminal", arguments='{"cmd":"pwd"}') + ) + normalized = SimpleNamespace( + content="", + tool_calls=[tool_call], + finish_reason="tool_calls", + ) + + def interruptible_api_call(kwargs: dict[str, object]) -> object: + captured["kwargs"] = dict(kwargs) + return normalized + + agent = SimpleNamespace( + api_mode="openai", + _is_anthropic_oauth=False, + log_prefix="", + _stall_retry_runtime_promoted=True, + _build_api_kwargs=lambda messages: { + "model": "qwen3.6-27b-256k", + "messages": messages, + "stream": True, + }, + _interruptible_api_call=interruptible_api_call, + _get_transport=lambda: SimpleNamespace( + normalize_response=lambda response, **_kwargs: response + ), + _vprint=lambda *_args, **_kwargs: None, + ) + + monkeypatch.setenv("HERMES_STALL_RETRY_MODEL", "qwen3.6-27b-256k") + monkeypatch.setenv("HERMES_STALL_RETRY_TELEMETRY", "0") + + result = retry_on_stall( + agent, + [{"role": "user", "content": "go"}], + "stop", + stalled_content="Let me check the repo.", + retry_index=3, + ) + + assert result is normalized + assert captured["kwargs"]["model"] == "qwen3.6-27b-256k" + assert captured["kwargs"]["stream"] is False + assert captured["kwargs"]["messages"][-1]["role"] == "user" + assert "required tool call" in captured["kwargs"]["messages"][-1]["content"] + summary = stall_retry_summary(agent) + assert summary is not None + assert any( + event.get("event") == "same_model_retry_after_promotion" + for event in getattr(agent, "_stall_retry_events", []) + ) + assert summary["recovered"] == 1 + + def test_retry_on_stall_uses_agent_config_when_env_is_absent(monkeypatch) -> None: captured: dict[str, object] = {} tool_call = SimpleNamespace( @@ -395,6 +486,16 @@ def test_stall_retry_promote_after_uses_config_and_env(monkeypatch) -> None: assert get_stall_retry_promote_after(agent) == 0 +def test_stall_retry_no_tool_recovery_max_uses_config_and_env(monkeypatch) -> None: + agent = SimpleNamespace(_stall_retry_config={"no_tool_recovery_max": 4}) + + monkeypatch.delenv("HERMES_STALL_RETRY_NO_TOOL_RECOVERY_MAX", raising=False) + assert get_stall_retry_no_tool_recovery_max(agent) == 4 + + monkeypatch.setenv("HERMES_STALL_RETRY_NO_TOOL_RECOVERY_MAX", "0") + assert get_stall_retry_no_tool_recovery_max(agent) == 0 + + def test_activate_stall_retry_runtime_promotes_for_current_turn(monkeypatch) -> None: captured: dict[str, object] = {} fake_client = SimpleNamespace( @@ -629,6 +730,10 @@ def test_conversation_loop_adopts_retry_before_tool_call_branch() -> None: assert retry_idx < tool_branch_idx assert "continue # re-enter loop top; tool-calls path handles it" not in source assert "stall_retry_failed_no_tool_call" in source + assert "FAILED_STALL_RETRY_RECOVERY_NUDGE" in source + assert "no_tool_recovery_prompt" in source + assert "agent._session_messages = messages" in source + assert "continue" in source def test_conversation_loop_retries_empty_post_tool_before_tool_branch() -> None: @@ -682,3 +787,12 @@ def test_conversation_loop_promotes_retry_lane_after_repeated_rescues() -> None: assert "_stall_retry_success_count += 1" in source assert "activate_stall_retry_runtime(" in source assert "_stall_retry_success_count >= _stall_retry_promote_after" in source + + +def test_conversation_loop_configures_no_tool_recovery_limit() -> None: + source = inspect.getsource(conversation_loop.run_conversation) + + assert "get_stall_retry_no_tool_recovery_max(agent)" in source + assert "_stall_retry_no_tool_recovery_count" in source + assert "_stall_retry_no_tool_recovery_max" in source + assert FAILED_STALL_RETRY_RECOVERY_NUDGE From e60132cde6af72cbff64f81d53f847137e6099cb Mon Sep 17 00:00:00 2001 From: Omar B Date: Tue, 9 Jun 2026 17:25:36 -0700 Subject: [PATCH 12/13] docs(agent): genericize local-model naming in stall retry comments Co-Authored-By: Claude Fable 5 --- agent/conversation_loop.py | 5 +++-- agent/stall_retry.py | 36 ++++++++++++++++++--------------- tests/agent/test_stall_retry.py | 20 +++++++++--------- 3 files changed, 33 insertions(+), 28 deletions(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 207dd327be283..4fe0a8fa252d2 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -793,7 +793,8 @@ def run_conversation( should_review_memory=_should_review_memory, ) - # Agentic stall-retry guard: dflash can stall more than once in a long + # Agentic stall-retry guard: local quantized models can stall more than + # once in a long # tool loop. Count total attempts for telemetry, but only consume the # terminal budget when the retry lane fails to produce forward progress. # A recovered tool call is work advanced, not a reason to kill the turn. @@ -3674,7 +3675,7 @@ def _stop_spinner(): agent._codex_incomplete_retries = 0 # ── Agentic stall-retry (opt-in via HERMES_STALL_RETRY_MODEL) ── - # dflash Q4 can stop right after an action preamble ("Let me + # Local quantized models can stop right after an action preamble ("Let me # check X") without producing the promised tool_call. Retry the # exact same turn on the configured higher-quality lane before # the final-response branch sees it. If it still returns no tool diff --git a/agent/stall_retry.py b/agent/stall_retry.py index bad5cc6e1505c..b80ac2826cf18 100644 --- a/agent/stall_retry.py +++ b/agent/stall_retry.py @@ -1,11 +1,12 @@ """ -Agentic stall-retry (dflash Q4 premature-EOS workaround). +Agentic stall-retry (local quantized-model premature-EOS workaround). -dflash (Qwen3.6-27B Q4_K_M, lucebox spec-decode) sometimes emits EOS right -after a short action preamble ("Let me check X:") on agentic decision turns, -ending the turn with NO tool_call -> the agent loop treats it as a final -answer and stops mid-task. Higher-precision weights (the stock Q6 lane on the -same host) continue to a real tool call on the identical prompt. +Local quantized models (e.g. aggressive low-bit quants with speculative +decoding) sometimes emit EOS right after a short action preamble ("Let me +check X:") on agentic decision turns, ending the turn with NO tool_call -> +the agent loop treats it as a final answer and stops mid-task. +Higher-precision weights (e.g. a higher-bit lane on the same host) continue +to a real tool call on the identical prompt. This module detects that stall signature on a no-tool-call turn and retries the turn against a higher-quality model lane, with a small recovery nudge that @@ -48,7 +49,8 @@ from typing import Any, Mapping, Sequence # Action-preamble signature: the turn announced an action but produced no tool -# call. These English phrases match the observed dflash stall corpus; broader +# call. These English phrases match the observed local quantized-model stall +# corpus; broader # language-agnostic fallbacks below still catch trailing-colon and incomplete # final fragments without pretending this regex is multilingual. _ACTION_RE = re.compile( @@ -280,8 +282,8 @@ def looks_like_incomplete_final_fragment( ) -> bool: """True when a short no-tool final looks cut off mid-sentence. - dflash can occasionally stop with ordinary visible prose after a tool - result, e.g. ``"... and some"``. That is not an action preamble, but it is + Local quantized models can occasionally stop with ordinary visible prose + after a tool result, e.g. ``"... and some"``. That is not an action preamble, but it is still unsafe to persist as the final answer in a tool loop. """ if has_tool_calls or finish_reason not in ("stop", "length"): @@ -300,7 +302,7 @@ def looks_like_incomplete_final_fragment( def _action_after_completion(content: str) -> bool: """True when a response says some step is complete, then promises work. - The dflash phone failure hit this exact shape: + An observed local quantized-model failure hit this exact shape: "Onboarding complete. Now let me read the STATUS.md ..." The earlier completion word is not a final answer when a later clause is @@ -321,7 +323,8 @@ def _ends_with_action_promise(content: str) -> bool: The generic stall heuristic is intentionally length-capped because long prose is often a real answer. Explicit tail promises are different: a long diagnostic can still end with "Let me check that:" and no tool call, which - is the exact dflash premature-stop shape this module exists to recover. + is the exact local quantized-model premature-stop shape this module + exists to recover. """ tail = (content or "").strip()[-_ACTION_TAIL_CHARS:] if not tail: @@ -745,14 +748,14 @@ def activate_stall_retry_runtime( ) try: agent._emit_status( - "↻ dflash stalled repeatedly; using " + "↻ Primary model stalled repeatedly; using " f"{resolved_model} for the rest of this turn. " "Primary model will be restored next turn." ) except Exception: try: agent._vprint( - f"{getattr(agent, 'log_prefix', '')}↻ dflash stalled " + f"{getattr(agent, 'log_prefix', '')}↻ Primary model stalled " f"repeatedly; using {resolved_model} for the rest of this turn.", force=True, ) @@ -802,7 +805,8 @@ def looks_like_stall(content: str, finish_reason: str, has_tool_calls: bool, # a trailing colon strongly implies "about to do something". if c.endswith(":"): return True - # dflash can also stop after a tool result with ordinary-looking prose that + # Local quantized models can also stop after a tool result with + # ordinary-looking prose that # is simply cut off mid-sentence (for example after a CLI interrupt resumes # the turn). In an agentic tool loop, a short no-tool stop that declares no # completion and lacks a natural ending is safer to retry than to persist as @@ -846,8 +850,8 @@ def retry_on_stall( ) # Build kwargs exactly as the normal turn would, then override only the # model name. Safe when the retry lane is served by the SAME provider/ - # endpoint as agent.model (e.g. taro serves both dflash and the Q6 lane), - # so no client rebuild is needed. + # endpoint as agent.model (e.g. one host serving multiple local + # quantization lanes), so no client rebuild is needed. api_kwargs = agent._build_api_kwargs(retry_messages) orig_model = api_kwargs.get("model") same_model_retry = retry_model == orig_model diff --git a/tests/agent/test_stall_retry.py b/tests/agent/test_stall_retry.py index e3a9aaf586be0..bd10d2c0e65e0 100644 --- a/tests/agent/test_stall_retry.py +++ b/tests/agent/test_stall_retry.py @@ -194,7 +194,7 @@ def interruptible_api_call(kwargs: dict[str, object]) -> object: _is_anthropic_oauth=False, log_prefix="", _build_api_kwargs=lambda messages: { - "model": "dflash", + "model": "local-q4", "messages": messages, "stream": True, }, @@ -246,7 +246,7 @@ def interruptible_api_call(kwargs: dict[str, object]) -> object: _is_anthropic_oauth=False, log_prefix="", _build_api_kwargs=lambda messages: { - "model": "dflash", + "model": "local-q4", "messages": messages, "stream": True, }, @@ -287,7 +287,7 @@ def test_retry_on_stall_still_rejects_content_without_accept_content(monkeypatch _is_anthropic_oauth=False, log_prefix="", _build_api_kwargs=lambda messages: { - "model": "dflash", + "model": "local-q4", "messages": messages, "stream": True, }, @@ -421,7 +421,7 @@ def interruptible_api_call(kwargs: dict[str, object]) -> object: "max_per_turn": 7, }, _build_api_kwargs=lambda messages: { - "model": "dflash", + "model": "local-q4", "messages": messages, "stream": True, }, @@ -512,7 +512,7 @@ def update_model(**kwargs: object) -> None: captured["context_model"] = dict(kwargs) agent = SimpleNamespace( - model="dflash", + model="local-q4", provider="custom", base_url="http://primary:8080/v1", api_key="primary-key", @@ -556,7 +556,7 @@ def update_model(**kwargs: object) -> None: assert str(agent.base_url) == "http://taro:8080/v1" assert agent._fallback_activated is True assert agent._stall_retry_runtime_promoted is True - assert agent._stall_retry_promoted_from == "dflash" + assert agent._stall_retry_promoted_from == "local-q4" assert agent._transport_cache == {} assert captured["resolve_kwargs"]["provider"] == "taro" assert captured["context_model"]["model"] == "qwen3.6-27b-256k" @@ -600,7 +600,7 @@ def fake_resolve_provider_client(**kwargs: object) -> tuple[object, str]: "api_key_env": "HERMES_RETRY_TEST_KEY", }, _build_api_kwargs=lambda messages: { - "model": "dflash", + "model": "local-q4", "messages": messages, "stream": True, }, @@ -656,7 +656,7 @@ def interruptible_api_call(kwargs: dict[str, object]) -> object: "telemetry": False, }, _build_api_kwargs=lambda messages: { - "model": "dflash", + "model": "local-q4", "messages": messages, "stream": True, }, @@ -688,7 +688,7 @@ def test_stall_retry_telemetry_writes_bounded_local_jsonl(tmp_path) -> None: log_path = tmp_path / "stall-retry.ndjson" agent = SimpleNamespace( session_id="s1", - model="dflash", + model="local-q4", provider="nous", _stall_retry_config={ "telemetry": True, @@ -708,7 +708,7 @@ def test_stall_retry_telemetry_writes_bounded_local_jsonl(tmp_path) -> None: event = json.loads(lines[0]) assert event["event"] == "detected" assert event["session_id"] == "s1" - assert event["model"] == "dflash" + assert event["model"] == "local-q4" assert event["provider"] == "nous" assert event["retry_model"] == "qwen3.6-27b-256k" assert event["content_chars"] > len(event["content_preview"]) From bc5de445653100646724f8fa757981f2acb5ab12 Mon Sep 17 00:00:00 2001 From: Omar Baradei Date: Sun, 5 Jul 2026 13:27:20 -0700 Subject: [PATCH 13/13] fix(stall-retry): expose actionable final_response on stall-retry failure returns The stall-retry lane added three run_conversation failure returns (stall_retry_limit_exhausted, stall_retry_failed_no_tool_call, stall_retry_exception) with "final_response": None, tripping the test_run_conversation_dict_returns_include_final_response guard that forbids literal-None final_response on dict returns. Hoist each block's error text into a local message var and set final_response to it, matching the sibling convention where final_response == error (actionable text the caller/UI can surface instead of an empty final response). Co-Authored-By: Claude Opus 4.8 --- agent/conversation_loop.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index d6fb7c7723360..e6afd12810a2c 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -4542,18 +4542,19 @@ def _perform_api_call(next_api_kwargs): ) agent._cleanup_task_resources(effective_task_id) agent._persist_session(messages, conversation_history) + _stall_limit_msg = ( + "Model repeatedly stopped after an agentic " + "preamble with no tool call; configured stall " + "retry limit was exhausted." + ) return { - "final_response": None, + "final_response": _stall_limit_msg, "messages": messages, "api_calls": api_call_count, "completed": False, "partial": True, "failed": True, - "error": ( - "Model repeatedly stopped after an agentic " - "preamble with no tool call; configured stall " - "retry limit was exhausted." - ), + "error": _stall_limit_msg, "failure_subclass": "stall_retry_limit_exhausted", } _stall_retry_count += 1 @@ -4643,18 +4644,19 @@ def _perform_api_call(next_api_kwargs): ) agent._cleanup_task_resources(effective_task_id) agent._persist_session(messages, conversation_history) + _stall_failed_msg = ( + "Model stopped after an action preamble with no " + "tool call; configured stall retry also produced " + "no tool call." + ) return { - "final_response": None, + "final_response": _stall_failed_msg, "messages": messages, "api_calls": api_call_count, "completed": False, "partial": True, "failed": True, - "error": ( - "Model stopped after an action preamble with no " - "tool call; configured stall retry also produced " - "no tool call." - ), + "error": _stall_failed_msg, "failure_subclass": "stall_retry_failed_no_tool_call", } except Exception as exc: @@ -4666,14 +4668,15 @@ def _perform_api_call(next_api_kwargs): ) agent._cleanup_task_resources(effective_task_id) agent._persist_session(messages, conversation_history) + _stall_exc_msg = f"Stall retry failed before recovery: {exc}" return { - "final_response": None, + "final_response": _stall_exc_msg, "messages": messages, "api_calls": api_call_count, "completed": False, "partial": True, "failed": True, - "error": f"Stall retry failed before recovery: {exc}", + "error": _stall_exc_msg, "failure_subclass": "stall_retry_exception", }