From 0e3ff0fa7440a3e6b86cff3561c940d8b796beb4 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 27 May 2026 11:30:05 +0000 Subject: [PATCH 1/2] Auto-deny abandoned tool calls before next LLM request When the LLM requested a tool call requiring approval and the user abandoned it (closed the approval modal or asked a new follow-up question without deciding), the conversation history ended up with an assistant tool_calls message that had no matching tool result. The next LLM call then failed with "tool_use ids were found without tool_result blocks immediately after" on Anthropic/Bedrock. call_stream now resolves any orphaned tool calls by injecting a denial tool result, so the conversation can continue with the user's new request. https://claude.ai/code/session_012djRRHcKAG9NTsYZoQiGVj Signed-off-by: Claude --- holmes/core/tool_calling_llm.py | 74 ++++++++++++++++ tests/test_orphaned_tool_calls.py | 137 ++++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+) create mode 100644 tests/test_orphaned_tool_calls.py diff --git a/holmes/core/tool_calling_llm.py b/holmes/core/tool_calling_llm.py index 8d8d109069..d3e034c198 100644 --- a/holmes/core/tool_calling_llm.py +++ b/holmes/core/tool_calling_llm.py @@ -402,6 +402,70 @@ def _execute_tool_decisions( return messages, events + def _resolve_orphaned_tool_calls( + self, messages: List[Dict[str, Any]] + ) -> tuple[List[Dict[str, Any]], list[StreamMessage]]: + """Inject denial tool results for assistant tool_calls that have no result. + + A tool call is "orphaned" when the assistant requested it but the + conversation never recorded a matching tool result. This happens when a + user abandons a pending tool approval — closing the approval modal or + asking a new follow-up question instead of approving/denying. Without a + matching tool result, the next LLM call fails because providers + (Anthropic/Bedrock) require every tool_use block to be immediately + followed by a tool_result block. + + We treat any such abandoned call as denied so the conversation can + continue with the user's new request. + """ + resolved_ids = { + msg.get("tool_call_id") + for msg in messages + if msg.get("role") == "tool" and msg.get("tool_call_id") + } + + events: list[StreamMessage] = [] + # Walk from the end so insertions don't shift indices we haven't visited. + for i in reversed(range(len(messages))): + msg = messages[i] + if msg.get("role") != "assistant" or not msg.get("tool_calls"): + continue + insert_offset = 1 + for tool_call in msg.get("tool_calls", []): + tool_call_id = tool_call.get("id") + if not tool_call_id or tool_call_id in resolved_ids: + continue + # Drop any stale pending_approval flag so it isn't re-emitted. + tool_call.pop("pending_approval", None) + function = tool_call.get("function") or {} + tool_name = function.get("name") or "unknown" + tool_result = ToolCallResult( + tool_call_id=tool_call_id, + tool_name=tool_name, + description=tool_name, + result=StructuredToolResult( + status=StructuredToolResultStatus.ERROR, + error="Tool execution was cancelled because the user " + "submitted a new request before approving it.", + ), + ) + messages.insert( + i + insert_offset, + tool_result.to_llm_message( + supports_vision=self._supports_vision() + ), + ) + resolved_ids.add(tool_call_id) + insert_offset += 1 + events.append( + StreamMessage( + event=StreamEvents.TOOL_RESULT, + data=tool_result.to_client_dict(), + ) + ) + + return messages, events + @staticmethod def _process_frontend_tool_results( messages: List[Dict[str, Any]], @@ -1018,6 +1082,16 @@ def call_stream( if ev.event == StreamEvents.TOOL_RESULT: all_tool_calls.append(ev.data) + # Deny any tool calls the user abandoned (e.g. closed the approval modal + # or asked a new question without deciding). Otherwise the LLM call fails + # because every tool_use block must be followed by a tool_result block. + if msgs: + msgs, events = self._resolve_orphaned_tool_calls(msgs) + for ev in events: + yield ev + if ev.event == StreamEvents.TOOL_RESULT: + all_tool_calls.append(ev.data) + messages: list[dict] = list(msgs) if msgs else [] tool_calls: list[dict] = [] tools: Optional[list] = self._get_tools() diff --git a/tests/test_orphaned_tool_calls.py b/tests/test_orphaned_tool_calls.py new file mode 100644 index 0000000000..2ede39fdc6 --- /dev/null +++ b/tests/test_orphaned_tool_calls.py @@ -0,0 +1,137 @@ +"""Tests for auto-denying abandoned (orphaned) tool calls. + +When the LLM requests a tool call that requires approval and the user never +decides on it — they close the approval modal or ask a new follow-up question +instead — the conversation history ends up with an assistant `tool_calls` +message that has no matching tool result. The next LLM call then fails because +providers (Anthropic/Bedrock) require every tool_use block to be immediately +followed by a tool_result block: + + `tool_use` ids were found without `tool_result` blocks immediately after + +`_resolve_orphaned_tool_calls` fixes this by injecting a denial tool result for +any such abandoned call so the conversation can continue. +""" + +import json +from unittest.mock import MagicMock + +from holmes.core.tool_calling_llm import ToolCallingLLM +from holmes.utils.stream import StreamEvents + + +def _build_ai() -> ToolCallingLLM: + return ToolCallingLLM( + tool_executor=MagicMock(), + max_steps=5, + llm=MagicMock(), + tool_results_dir=None, + ) + + +def _assistant_tool_call_msg(tool_call_id: str, pending_approval: bool = True) -> dict: + tool_call = { + "id": tool_call_id, + "type": "function", + "function": { + "name": "bash", + "arguments": json.dumps({"command": "kubectl delete pod x"}), + }, + } + if pending_approval: + tool_call["pending_approval"] = True + return { + "role": "assistant", + "content": "I'll run a command", + "tool_calls": [tool_call], + } + + +def test_orphaned_pending_tool_call_gets_denial_result(): + ai = _build_ai() + messages = [ + {"role": "user", "content": "do something"}, + _assistant_tool_call_msg("tc1"), + {"role": "user", "content": "actually, never mind — what's the weather?"}, + ] + + updated, events = ai._resolve_orphaned_tool_calls(messages) + + # A tool result was inserted immediately after the assistant tool_calls msg. + assert updated[2]["role"] == "tool" + assert updated[2]["tool_call_id"] == "tc1" + assert "cancelled" in updated[2]["content"] + # The new user question still follows. + assert updated[3] == { + "role": "user", + "content": "actually, never mind — what's the weather?", + } + # The stale pending_approval flag is cleared. + assert "pending_approval" not in updated[1]["tool_calls"][0] + # A TOOL_RESULT stream event was emitted for the client. + assert any(ev.event == StreamEvents.TOOL_RESULT for ev in events) + + +def test_orphaned_tool_call_without_pending_flag_gets_denial_result(): + # Reproduces the "Approve/Deny then immediately stop" case: the assistant + # tool_calls message has no pending_approval flag but also no result. + ai = _build_ai() + messages = [ + {"role": "user", "content": "do something"}, + _assistant_tool_call_msg("tc1", pending_approval=False), + ] + + updated, events = ai._resolve_orphaned_tool_calls(messages) + + assert updated[2]["role"] == "tool" + assert updated[2]["tool_call_id"] == "tc1" + assert len(events) == 1 + + +def test_resolved_tool_calls_are_left_untouched(): + ai = _build_ai() + messages = [ + {"role": "user", "content": "do something"}, + _assistant_tool_call_msg("tc1", pending_approval=False), + {"role": "tool", "tool_call_id": "tc1", "name": "bash", "content": "done"}, + ] + + updated, events = ai._resolve_orphaned_tool_calls(messages) + + assert updated == messages + assert events == [] + + +def test_multiple_tool_calls_in_one_message_each_get_a_result(): + ai = _build_ai() + assistant_msg = { + "role": "assistant", + "content": "running two commands", + "tool_calls": [ + { + "id": "tc1", + "type": "function", + "function": {"name": "bash", "arguments": json.dumps({"command": "a"})}, + "pending_approval": True, + }, + { + "id": "tc2", + "type": "function", + "function": {"name": "bash", "arguments": json.dumps({"command": "b"})}, + "pending_approval": True, + }, + ], + } + messages = [ + {"role": "user", "content": "do something"}, + assistant_msg, + {"role": "user", "content": "new question"}, + ] + + updated, events = ai._resolve_orphaned_tool_calls(messages) + + # Both denial results inserted, in order, immediately after the assistant msg. + assert updated[2]["role"] == "tool" and updated[2]["tool_call_id"] == "tc1" + assert updated[3]["role"] == "tool" and updated[3]["tool_call_id"] == "tc2" + assert updated[4] == {"role": "user", "content": "new question"} + assert len(events) == 2 From 570b236b24b7fb58ae769a61c2c3fb8780af1c0e Mon Sep 17 00:00:00 2001 From: Naomi Caren Date: Thu, 28 May 2026 12:20:55 +0300 Subject: [PATCH 2/2] override config file --- server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.py b/server.py index adf3db67fd..9178e80b48 100644 --- a/server.py +++ b/server.py @@ -123,7 +123,7 @@ def init_config(): tuple: (config, dal) - The initialized Config object and its DAL instance """ default_config_path = Path(DEFAULT_CONFIG_LOCATION) - if default_config_path.exists(): + if default_config_path.exists() and os.environ.get("LOAD_CONFIG_FROM_ENV", "false").lower() == "false": logging.info(f"Loading config from file: {default_config_path}") config = Config.load_from_file(default_config_path) else: