diff --git a/api/streaming.py b/api/streaming.py index 1a17a6698f4..3158f68c0a9 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -1,10 +1,12 @@ """ Hermes Web UI -- SSE streaming engine and agent thread runner. Includes Sprint 10 cancel support via CANCEL_FLAGS. +Includes hallucination guard to prevent fabricated tool output. """ import json import os import queue +import re import threading import time import traceback @@ -29,6 +31,75 @@ # metadata added by the webui and must be stripped before the API call. _API_SAFE_MSG_KEYS = {'role', 'content', 'tool_calls', 'tool_call_id', 'name', 'refusal'} +# ── Hallucination guard: detect and strip fabricated tool output ────────── + +# Patterns that indicate the model is writing fake tool execution results +# instead of actually calling tools. These should ONLY appear in real tool +# result messages (role: "tool"), never in assistant text. +_FAKE_OUTPUT_PATTERNS = [ + # {"output": "...", "exit_code": N} + re.compile(r'\{\s*"output"\s*:\s*"[^"]*"\s*,\s*"exit_code"\s*:\s*\d+[^}]*\}'), + # {"output": "...", "stderr": "...", "exit_code": N} + re.compile(r'\{\s*"output"\s*:\s*"[^"]*"\s*,\s*"stderr"\s*:\s*"[^"]*"\s*,\s*"exit_code"\s*:\s*\d+[^}]*\}'), + # Executing:\n{...} or Executing:\n``` + re.compile(r'Executing:\s*\n\s*[{`]'), + # Output:\n{"output": ...} + re.compile(r'(?:Output|Result):\s*\n\s*\{\s*"(?:output|result|exit_code)'), + # Bare fabricated command output blocks after "Running" or "Executing" + re.compile(r'(?:Running|Executing)\s+`[^`]+`[:\s]*\n```(?:json|bash|sh)?\s*\n\s*\{[^}]*"(?:output|exit_code)"'), +] + +# Lighter pattern for streaming tokens — just check for the opening of fake JSON +_FAKE_TOKEN_PATTERN = re.compile( + r'\{\s*"(?:output|exit_code|stderr)"\s*:' +) + + +def _contains_fake_tool_output(text: str) -> bool: + """Return True if text contains patterns that look like fabricated tool output.""" + if not text: + return False + for pat in _FAKE_OUTPUT_PATTERNS: + if pat.search(text): + return True + return False + + +def _strip_fake_tool_output(text: str) -> str: + """Remove fabricated tool output patterns from assistant text. + + When a model writes text like: + Executing: {"output": "total 396\\ndrwxr-xr-x...", "exit_code": 0} + this replaces the fake JSON with a notice that the model should use + the actual tool instead. + """ + if not text: + return text + cleaned = text + # Replace fake JSON output blocks + cleaned = re.sub( + r'\{\s*"output"\s*:\s*"[^"]*"\s*,\s*(?:"stderr"\s*:\s*"[^"]*"\s*,\s*)?' + r'"exit_code"\s*:\s*\d+[^}]*\}', + '[Fabricated output removed — use the terminal tool to execute commands]', + cleaned, + ) + return cleaned + + +# System prompt injection to prevent hallucinated tool output. +# Appended as ephemeral_system_prompt so it's part of every API call +# but not persisted to session transcripts. +_ANTI_HALLUCINATION_PROMPT = """ +CRITICAL EXECUTION RULES — TOOL USE ONLY: +- You MUST use the terminal tool to run ANY shell command. NEVER write command output yourself. +- NEVER write {"output": ..., "exit_code": ...} in your response text. That JSON format comes ONLY from actual tool execution results. +- NEVER fabricate, guess, or simulate command output. If you need to know the result of a command, call the terminal tool. +- If a command fails, report the actual error from the tool result. Never fabricate success. +- "Executing:" followed by JSON you wrote is NOT execution — it is hallucination. Use the terminal tool instead. +- When asked to run a command (e.g., "run ls", "execute curl", "run echo"), you MUST call the terminal tool. Writing the expected output is not acceptable. +- After calling a tool, wait for the real result before continuing. Do not predict or pre-write tool results. +""".strip() + def _sanitize_messages_for_api(messages): """Return a deep copy of messages with only API-safe fields. @@ -112,9 +183,48 @@ def put(event, data): os.environ['HERMES_HOME'] = _profile_home try: + # ── Hallucination-guarded token callback ────────────── + # Buffer streamed tokens to detect fake tool output mid-stream. + # When the model starts writing {"output": ..., we hold tokens + # until we can confirm it's real commentary or fake output. + _token_buf = [] # buffered token strings + _token_buf_len = 0 # total chars in buffer + _BUF_THRESHOLD = 200 # flush if buffer exceeds this without match + + def _flush_token_buf(): + """Flush buffered tokens to client, stripping fake output.""" + nonlocal _token_buf, _token_buf_len + if not _token_buf: + return + combined = ''.join(_token_buf) + _token_buf = [] + _token_buf_len = 0 + if _contains_fake_tool_output(combined): + cleaned = _strip_fake_tool_output(combined) + if cleaned.strip(): + put('token', {'text': cleaned}) + print('[webui] hallucination guard: stripped fake tool output from stream', flush=True) + else: + put('token', {'text': combined}) + def on_token(text): + nonlocal _token_buf, _token_buf_len if text is None: - return # end-of-stream sentinel + _flush_token_buf() # flush remaining buffer on end-of-stream + return + # Check if this token or accumulated buffer looks like fake output + if _FAKE_TOKEN_PATTERN.search(text) or (_token_buf and _FAKE_TOKEN_PATTERN.search(''.join(_token_buf) + text)): + _token_buf.append(text) + _token_buf_len += len(text) + # If buffer is large enough, we can decide + if _token_buf_len >= _BUF_THRESHOLD: + _flush_token_buf() + return + if _token_buf: + # We were buffering but this token doesn't continue the pattern + _token_buf.append(text) + _flush_token_buf() + return put('token', {'text': text}) def on_tool(name, preview, args): @@ -169,6 +279,7 @@ def on_tool(name, preview, args): session_id=session_id, stream_delta_callback=on_token, tool_progress_callback=on_tool, + ephemeral_system_prompt=_ANTI_HALLUCINATION_PROMPT, ) # Prepend workspace context so the agent always knows which directory # to use for file operations, regardless of session age or AGENTS.md defaults. @@ -191,7 +302,22 @@ def on_tool(name, preview, args): task_id=session_id, persist_user_message=msg_text, ) - s.messages = result.get('messages') or s.messages + + # ── Post-run hallucination scrub ─────────────────────── + # Scan assistant messages for any fabricated tool output + # that slipped past the streaming guard (e.g. from non- + # streaming providers or buffered responses). + _result_msgs = result.get('messages') or s.messages + _hallucination_count = 0 + for _m in _result_msgs: + if _m.get('role') == 'assistant' and isinstance(_m.get('content'), str): + if _contains_fake_tool_output(_m['content']): + _m['content'] = _strip_fake_tool_output(_m['content']) + _hallucination_count += 1 + if _hallucination_count: + print(f'[webui] hallucination guard: scrubbed {_hallucination_count} assistant message(s) with fake tool output', flush=True) + + s.messages = _result_msgs # Stamp 'timestamp' on any messages that don't have one yet _now = time.time() for _m in s.messages: diff --git a/tests/test_hallucination_guard.py b/tests/test_hallucination_guard.py new file mode 100644 index 00000000000..308138a08aa --- /dev/null +++ b/tests/test_hallucination_guard.py @@ -0,0 +1,143 @@ +""" +Tests for the hallucination guard in api/streaming.py. + +These tests verify that the guard correctly detects and strips fabricated +tool output from model responses — the core fix for the "Hermes writes fake +tool results instead of actually calling tools" bug. +""" +import sys +import pathlib + +# Add repo root to path so we can import api.streaming +REPO_ROOT = pathlib.Path(__file__).parent.parent.resolve() +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from api.streaming import ( + _contains_fake_tool_output, + _strip_fake_tool_output, + _ANTI_HALLUCINATION_PROMPT, +) + + +class TestContainsFakeToolOutput: + """Test _contains_fake_tool_output detection.""" + + def test_simple_fake_output(self): + text = '{"output": "total 396\\ndrwxr-xr-x", "exit_code": 0}' + assert _contains_fake_tool_output(text) is True + + def test_fake_output_with_stderr(self): + text = '{"output": "hello world", "stderr": "", "exit_code": 0}' + assert _contains_fake_tool_output(text) is True + + def test_executing_with_json(self): + text = 'Executing:\n{"output": "some result", "exit_code": 0}' + assert _contains_fake_tool_output(text) is True + + def test_executing_with_backtick(self): + text = 'Executing:\n```json\n{"output": "result"}\n```' + assert _contains_fake_tool_output(text) is True + + def test_output_label_with_json(self): + text = 'Output:\n{"output": "test", "exit_code": 0}' + assert _contains_fake_tool_output(text) is True + + def test_result_label_with_json(self): + text = 'Result:\n{"result": "success", "exit_code": 0}' + assert _contains_fake_tool_output(text) is True + + def test_running_backtick_command(self): + text = 'Running `ls -la`:\n```json\n{"output": "total 4", "exit_code": 0}\n```' + assert _contains_fake_tool_output(text) is True + + def test_normal_text_not_flagged(self): + text = "I'll run the ls command for you to check the directory contents." + assert _contains_fake_tool_output(text) is False + + def test_code_discussion_not_flagged(self): + text = 'The function returns a dict like {"name": "test", "value": 42}' + assert _contains_fake_tool_output(text) is False + + def test_empty_string(self): + assert _contains_fake_tool_output("") is False + + def test_none(self): + assert _contains_fake_tool_output(None) is False + + def test_json_in_code_block_discussion(self): + """Regular JSON discussion should not be flagged.""" + text = 'The API returns:\n```json\n{"name": "test", "status": "ok"}\n```' + assert _contains_fake_tool_output(text) is False + + def test_mentioning_exit_code_in_prose(self): + """Talking about exit codes in prose should not be flagged.""" + text = "The command returned exit_code 1, indicating an error." + assert _contains_fake_tool_output(text) is False + + def test_real_world_hallucination_pattern(self): + """Match the exact pattern from the bug report.""" + text = '''Executing: +{"output": "total 396\ndrwxr-xr-x 2 user user 4096 Apr 1 12:00 scripts\n-rw-r--r-- 1 user user 1234 Apr 1 12:00 README.md", "exit_code": 0}''' + assert _contains_fake_tool_output(text) is True + + +class TestStripFakeToolOutput: + """Test _strip_fake_tool_output sanitization.""" + + def test_strips_fake_json(self): + text = 'Here is the result: {"output": "hello world", "exit_code": 0}' + result = _strip_fake_tool_output(text) + assert '"output"' not in result + assert '"exit_code"' not in result + assert 'Fabricated output removed' in result + + def test_strips_fake_json_with_stderr(self): + text = '{"output": "data", "stderr": "warn", "exit_code": 1}' + result = _strip_fake_tool_output(text) + assert '"output"' not in result + assert 'Fabricated output removed' in result + + def test_preserves_surrounding_text(self): + text = 'Before text. {"output": "fake", "exit_code": 0} After text.' + result = _strip_fake_tool_output(text) + assert 'Before text.' in result + assert 'After text.' in result + + def test_no_change_for_clean_text(self): + text = "This is a normal assistant response with no fake output." + assert _strip_fake_tool_output(text) == text + + def test_empty_string(self): + assert _strip_fake_tool_output("") == "" + + def test_none(self): + assert _strip_fake_tool_output(None) is None + + def test_multiple_fake_blocks(self): + text = ( + 'First: {"output": "a", "exit_code": 0}\n' + 'Second: {"output": "b", "exit_code": 1}' + ) + result = _strip_fake_tool_output(text) + # Both should be replaced + assert result.count('Fabricated output removed') == 2 + + +class TestAntiHallucinationPrompt: + """Test the system prompt content.""" + + def test_prompt_exists(self): + assert _ANTI_HALLUCINATION_PROMPT + assert len(_ANTI_HALLUCINATION_PROMPT) > 100 + + def test_prompt_mentions_terminal_tool(self): + assert 'terminal tool' in _ANTI_HALLUCINATION_PROMPT + + def test_prompt_mentions_never_fabricate(self): + assert 'NEVER' in _ANTI_HALLUCINATION_PROMPT + assert 'fabricate' in _ANTI_HALLUCINATION_PROMPT.lower() + + def test_prompt_mentions_output_format(self): + assert '"output"' in _ANTI_HALLUCINATION_PROMPT + assert '"exit_code"' in _ANTI_HALLUCINATION_PROMPT