fix(hermes): strict ReAct loop — no fabricated tool output

api/streaming.py

@@ -1,10 +1,12 @@
 """
 Hermes Web UI -- SSE streaming engine and agent thread runner.
 Includes Sprint 10 cancel support via CANCEL_FLAGS.
+Includes hallucination guard to prevent fabricated tool output.
 """
 import json
 import os
 import queue
+import re
 import threading
 import time
 import traceback
@@ -29,6 +31,75 @@
 # metadata added by the webui and must be stripped before the API call.
 _API_SAFE_MSG_KEYS = {'role', 'content', 'tool_calls', 'tool_call_id', 'name', 'refusal'}
 
+# ── Hallucination guard: detect and strip fabricated tool output ──────────
+
+# Patterns that indicate the model is writing fake tool execution results
+# instead of actually calling tools. These should ONLY appear in real tool
+# result messages (role: "tool"), never in assistant text.
+_FAKE_OUTPUT_PATTERNS = [
+    # {"output": "...", "exit_code": N}
+    re.compile(r'\{\s*"output"\s*:\s*"[^"]*"\s*,\s*"exit_code"\s*:\s*\d+[^}]*\}'),
+    # {"output": "...", "stderr": "...", "exit_code": N}
+    re.compile(r'\{\s*"output"\s*:\s*"[^"]*"\s*,\s*"stderr"\s*:\s*"[^"]*"\s*,\s*"exit_code"\s*:\s*\d+[^}]*\}'),
+    # Executing:\n{...} or Executing:\n```
+    re.compile(r'Executing:\s*\n\s*[{`]'),
+    # Output:\n{"output": ...}
+    re.compile(r'(?:Output|Result):\s*\n\s*\{\s*"(?:output|result|exit_code)'),
+    # Bare fabricated command output blocks after "Running" or "Executing"
+    re.compile(r'(?:Running|Executing)\s+`[^`]+`[:\s]*\n```(?:json|bash|sh)?\s*\n\s*\{[^}]*"(?:output|exit_code)"'),
+]
+
+# Lighter pattern for streaming tokens — just check for the opening of fake JSON
+_FAKE_TOKEN_PATTERN = re.compile(
+    r'\{\s*"(?:output|exit_code|stderr)"\s*:'
+)
+
+
+def _contains_fake_tool_output(text: str) -> bool:
+    """Return True if text contains patterns that look like fabricated tool output."""
+    if not text:
+        return False
+    for pat in _FAKE_OUTPUT_PATTERNS:
+        if pat.search(text):
+            return True
+    return False
+
+
+def _strip_fake_tool_output(text: str) -> str:
+    """Remove fabricated tool output patterns from assistant text.
+
+    When a model writes text like:
+        Executing: {"output": "total 396\\ndrwxr-xr-x...", "exit_code": 0}
+    this replaces the fake JSON with a notice that the model should use
+    the actual tool instead.
+    """
+    if not text:
+        return text
+    cleaned = text
+    # Replace fake JSON output blocks
+    cleaned = re.sub(
+        r'\{\s*"output"\s*:\s*"[^"]*"\s*,\s*(?:"stderr"\s*:\s*"[^"]*"\s*,\s*)?'
+        r'"exit_code"\s*:\s*\d+[^}]*\}',
+        '[Fabricated output removed — use the terminal tool to execute commands]',
+        cleaned,
+    )
+    return cleaned
+
+
+# System prompt injection to prevent hallucinated tool output.
+# Appended as ephemeral_system_prompt so it's part of every API call
+# but not persisted to session transcripts.
+_ANTI_HALLUCINATION_PROMPT = """
+CRITICAL EXECUTION RULES — TOOL USE ONLY:
+- You MUST use the terminal tool to run ANY shell command. NEVER write command output yourself.
+- NEVER write {"output": ..., "exit_code": ...} in your response text. That JSON format comes ONLY from actual tool execution results.
+- NEVER fabricate, guess, or simulate command output. If you need to know the result of a command, call the terminal tool.
+- If a command fails, report the actual error from the tool result. Never fabricate success.
+- "Executing:" followed by JSON you wrote is NOT execution — it is hallucination. Use the terminal tool instead.
+- When asked to run a command (e.g., "run ls", "execute curl", "run echo"), you MUST call the terminal tool. Writing the expected output is not acceptable.
+- After calling a tool, wait for the real result before continuing. Do not predict or pre-write tool results.
+""".strip()
+
 
 def _sanitize_messages_for_api(messages):
     """Return a deep copy of messages with only API-safe fields.
@@ -112,9 +183,48 @@ def put(event, data):
               os.environ['HERMES_HOME'] = _profile_home
 
           try:
+            # ── Hallucination-guarded token callback ──────────────
+            # Buffer streamed tokens to detect fake tool output mid-stream.
+            # When the model starts writing {"output": ..., we hold tokens
+            # until we can confirm it's real commentary or fake output.
+            _token_buf = []       # buffered token strings
+            _token_buf_len = 0    # total chars in buffer
+            _BUF_THRESHOLD = 200  # flush if buffer exceeds this without match
+
+            def _flush_token_buf():
+                """Flush buffered tokens to client, stripping fake output."""
+                nonlocal _token_buf, _token_buf_len
+                if not _token_buf:
+                    return
+                combined = ''.join(_token_buf)
+                _token_buf = []
+                _token_buf_len = 0
+                if _contains_fake_tool_output(combined):
+                    cleaned = _strip_fake_tool_output(combined)
+                    if cleaned.strip():
+                        put('token', {'text': cleaned})
+                    print('[webui] hallucination guard: stripped fake tool output from stream', flush=True)
+                else:
+                    put('token', {'text': combined})
+
             def on_token(text):
+                nonlocal _token_buf, _token_buf_len
                 if text is None:
-                    return  # end-of-stream sentinel
+                    _flush_token_buf()  # flush remaining buffer on end-of-stream
+                    return
+                # Check if this token or accumulated buffer looks like fake output
+                if _FAKE_TOKEN_PATTERN.search(text) or (_token_buf and _FAKE_TOKEN_PATTERN.search(''.join(_token_buf) + text)):
+                    _token_buf.append(text)
+                    _token_buf_len += len(text)
+                    # If buffer is large enough, we can decide
+                    if _token_buf_len >= _BUF_THRESHOLD:
+                        _flush_token_buf()
+                    return
+                if _token_buf:
+                    # We were buffering but this token doesn't continue the pattern
+                    _token_buf.append(text)
+                    _flush_token_buf()
+                    return
                 put('token', {'text': text})
 
             def on_tool(name, preview, args):
@@ -169,6 +279,7 @@ def on_tool(name, preview, args):
                 session_id=session_id,
                 stream_delta_callback=on_token,
                 tool_progress_callback=on_tool,
+                ephemeral_system_prompt=_ANTI_HALLUCINATION_PROMPT,
             )
             # Prepend workspace context so the agent always knows which directory
             # to use for file operations, regardless of session age or AGENTS.md defaults.
@@ -191,7 +302,22 @@ def on_tool(name, preview, args):
                 task_id=session_id,
                 persist_user_message=msg_text,
             )
-            s.messages = result.get('messages') or s.messages
+
+            # ── Post-run hallucination scrub ───────────────────────
+            # Scan assistant messages for any fabricated tool output
+            # that slipped past the streaming guard (e.g. from non-
+            # streaming providers or buffered responses).
+            _result_msgs = result.get('messages') or s.messages
+            _hallucination_count = 0
+            for _m in _result_msgs:
+                if _m.get('role') == 'assistant' and isinstance(_m.get('content'), str):
+                    if _contains_fake_tool_output(_m['content']):
+                        _m['content'] = _strip_fake_tool_output(_m['content'])
+                        _hallucination_count += 1
+            if _hallucination_count:
+                print(f'[webui] hallucination guard: scrubbed {_hallucination_count} assistant message(s) with fake tool output', flush=True)
+
+            s.messages = _result_msgs
             # Stamp 'timestamp' on any messages that don't have one yet
             _now = time.time()
             for _m in s.messages:

tests/test_hallucination_guard.py

@@ -0,0 +1,143 @@
+"""
+Tests for the hallucination guard in api/streaming.py.
+
+These tests verify that the guard correctly detects and strips fabricated
+tool output from model responses — the core fix for the "Hermes writes fake
+tool results instead of actually calling tools" bug.
+"""
+import sys
+import pathlib
+
+# Add repo root to path so we can import api.streaming
+REPO_ROOT = pathlib.Path(__file__).parent.parent.resolve()
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from api.streaming import (
+    _contains_fake_tool_output,
+    _strip_fake_tool_output,
+    _ANTI_HALLUCINATION_PROMPT,
+)
+
+
+class TestContainsFakeToolOutput:
+    """Test _contains_fake_tool_output detection."""
+
+    def test_simple_fake_output(self):
+        text = '{"output": "total 396\\ndrwxr-xr-x", "exit_code": 0}'
+        assert _contains_fake_tool_output(text) is True
+
+    def test_fake_output_with_stderr(self):
+        text = '{"output": "hello world", "stderr": "", "exit_code": 0}'
+        assert _contains_fake_tool_output(text) is True
+
+    def test_executing_with_json(self):
+        text = 'Executing:\n{"output": "some result", "exit_code": 0}'
+        assert _contains_fake_tool_output(text) is True
+
+    def test_executing_with_backtick(self):
+        text = 'Executing:\n```json\n{"output": "result"}\n```'
+        assert _contains_fake_tool_output(text) is True
+
+    def test_output_label_with_json(self):
+        text = 'Output:\n{"output": "test", "exit_code": 0}'
+        assert _contains_fake_tool_output(text) is True
+
+    def test_result_label_with_json(self):
+        text = 'Result:\n{"result": "success", "exit_code": 0}'
+        assert _contains_fake_tool_output(text) is True
+
+    def test_running_backtick_command(self):
+        text = 'Running `ls -la`:\n```json\n{"output": "total 4", "exit_code": 0}\n```'
+        assert _contains_fake_tool_output(text) is True
+
+    def test_normal_text_not_flagged(self):
+        text = "I'll run the ls command for you to check the directory contents."
+        assert _contains_fake_tool_output(text) is False
+
+    def test_code_discussion_not_flagged(self):
+        text = 'The function returns a dict like {"name": "test", "value": 42}'
+        assert _contains_fake_tool_output(text) is False
+
+    def test_empty_string(self):
+        assert _contains_fake_tool_output("") is False
+
+    def test_none(self):
+        assert _contains_fake_tool_output(None) is False
+
+    def test_json_in_code_block_discussion(self):
+        """Regular JSON discussion should not be flagged."""
+        text = 'The API returns:\n```json\n{"name": "test", "status": "ok"}\n```'
+        assert _contains_fake_tool_output(text) is False
+
+    def test_mentioning_exit_code_in_prose(self):
+        """Talking about exit codes in prose should not be flagged."""
+        text = "The command returned exit_code 1, indicating an error."
+        assert _contains_fake_tool_output(text) is False
+
+    def test_real_world_hallucination_pattern(self):
+        """Match the exact pattern from the bug report."""
+        text = '''Executing:
+{"output": "total 396\ndrwxr-xr-x  2 user user  4096 Apr  1 12:00 scripts\n-rw-r--r--  1 user user  1234 Apr  1 12:00 README.md", "exit_code": 0}'''
+        assert _contains_fake_tool_output(text) is True
+
+
+class TestStripFakeToolOutput:
+    """Test _strip_fake_tool_output sanitization."""
+
+    def test_strips_fake_json(self):
+        text = 'Here is the result: {"output": "hello world", "exit_code": 0}'
+        result = _strip_fake_tool_output(text)
+        assert '"output"' not in result
+        assert '"exit_code"' not in result
+        assert 'Fabricated output removed' in result
+
+    def test_strips_fake_json_with_stderr(self):
+        text = '{"output": "data", "stderr": "warn", "exit_code": 1}'
+        result = _strip_fake_tool_output(text)
+        assert '"output"' not in result
+        assert 'Fabricated output removed' in result
+
+    def test_preserves_surrounding_text(self):
+        text = 'Before text. {"output": "fake", "exit_code": 0} After text.'
+        result = _strip_fake_tool_output(text)
+        assert 'Before text.' in result
+        assert 'After text.' in result
+
+    def test_no_change_for_clean_text(self):
+        text = "This is a normal assistant response with no fake output."
+        assert _strip_fake_tool_output(text) == text
+
+    def test_empty_string(self):
+        assert _strip_fake_tool_output("") == ""
+
+    def test_none(self):
+        assert _strip_fake_tool_output(None) is None
+
+    def test_multiple_fake_blocks(self):
+        text = (
+            'First: {"output": "a", "exit_code": 0}\n'
+            'Second: {"output": "b", "exit_code": 1}'
+        )
+        result = _strip_fake_tool_output(text)
+        # Both should be replaced
+        assert result.count('Fabricated output removed') == 2
+
+
+class TestAntiHallucinationPrompt:
+    """Test the system prompt content."""
+
+    def test_prompt_exists(self):
+        assert _ANTI_HALLUCINATION_PROMPT
+        assert len(_ANTI_HALLUCINATION_PROMPT) > 100
+
+    def test_prompt_mentions_terminal_tool(self):
+        assert 'terminal tool' in _ANTI_HALLUCINATION_PROMPT
+
+    def test_prompt_mentions_never_fabricate(self):
+        assert 'NEVER' in _ANTI_HALLUCINATION_PROMPT
+        assert 'fabricate' in _ANTI_HALLUCINATION_PROMPT.lower()
+
+    def test_prompt_mentions_output_format(self):
+        assert '"output"' in _ANTI_HALLUCINATION_PROMPT
+        assert '"exit_code"' in _ANTI_HALLUCINATION_PROMPT