From 567306b917b5e8970fc7eb72bb236e4c419e6730 Mon Sep 17 00:00:00 2001 From: Amy Ravenwolf Date: Tue, 31 Mar 2026 15:02:24 +0200 Subject: [PATCH 01/10] feat(display): add independent thinking_progress config option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Decouple assistant thinking relay from tool_progress mode entirely. These are two separate concerns: tool_progress โ†’ Tool call display (off/new/all/verbose/full) thinking_progress โ†’ Assistant thinking between tool calls (true/false) Previously, seeing ๐Ÿ’ฌ thinking text required tool_progress: full, which bundled thinking with unlimited tool argument display. Now they are fully independent config options under display: display: tool_progress: all # controls tool call notifications thinking_progress: true # controls ๐Ÿ’ฌ thinking relay Any combination works. tool_progress: full no longer implicitly enables thinking โ€” set thinking_progress: true explicitly if you want it. Thinking text is relayed without any truncation limit. Default: thinking_progress: false (no behavior change for existing users). --- gateway/run.py | 40 +++++++++++++++++++++++++++++++++++----- run_agent.py | 36 ++++++++++++++++++++++++++++-------- 2 files changed, 63 insertions(+), 13 deletions(-) diff --git a/gateway/run.py b/gateway/run.py index 82f5e8036f30..b0c0205f5fd1 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -5395,9 +5395,16 @@ async def _run_agent( # so each progress line would be sent as a separate message. from gateway.config import Platform tool_progress_enabled = progress_mode != "off" and source.platform != Platform.WEBHOOK - + # thinking_progress is independent โ€” if enabled, we need the progress + # queue even when tool_progress is off (thinking relay uses same infra) + _thinking_cfg = user_config.get("display", {}).get("thinking_progress") + _thinking_enabled = ( + _thinking_cfg is True + or (isinstance(_thinking_cfg, str) and _thinking_cfg.lower() in ("true", "yes", "1", "on")) + ) + needs_progress_queue = tool_progress_enabled or _thinking_enabled # Queue for progress messages (thread-safe) - progress_queue = queue.Queue() if tool_progress_enabled else None + progress_queue = queue.Queue() if needs_progress_queue else None last_tool = [None] # Mutable container for tracking in closure last_progress_msg = [None] # Track last message for dedup repeat_count = [0] # How many times the same message repeated @@ -5406,7 +5413,19 @@ def progress_callback(tool_name: str, preview: str = None, args: dict = None): """Callback invoked by agent when a tool is called.""" if not progress_queue: return - + + # "_thinking" is assistant text between tool calls โ€” relay as-is + if tool_name == "_thinking": + msg = f"๐Ÿ’ฌ {preview}" if preview else None + if msg: + progress_queue.put(msg) + return + + # If tool_progress is off, only _thinking passes through (above). + # Regular tool calls are suppressed. + if not tool_progress_enabled: + return + # "new" mode: only report when tool changes if progress_mode == "new" and tool_name == last_tool[0]: return @@ -5749,7 +5768,8 @@ def run_sync(): # Per-message state โ€” callbacks and reasoning config change every # turn and must not be baked into the cached agent constructor. - agent.tool_progress_callback = progress_callback if tool_progress_enabled else None + agent.tool_progress_callback = progress_callback if needs_progress_queue else None + agent.tool_progress_mode = progress_mode if tool_progress_enabled else None agent.step_callback = _step_callback_sync if _hooks_ref.loaded_hooks else None agent.stream_delta_callback = _stream_delta_cb agent.status_callback = _status_callback_sync @@ -5773,6 +5793,16 @@ def _bg_review_send(message: str) -> None: agent.background_review_callback = _bg_review_send + # Show assistant thinking between tool calls โ€” fully independent + # of tool_progress mode. Config: display.thinking_progress: true + _thinking_progress = user_config.get("display", {}).get("thinking_progress") + if isinstance(_thinking_progress, bool): + agent.thinking_progress = _thinking_progress + elif isinstance(_thinking_progress, str): + agent.thinking_progress = _thinking_progress.lower() in ("true", "yes", "1", "on") + else: + agent.thinking_progress = False + # Store agent reference for interrupt support agent_holder[0] = agent # Capture the full tool definitions for transcript logging @@ -6035,7 +6065,7 @@ def _approval_notify_sync(approval_data: dict) -> None: # Start progress message sender if enabled progress_task = None - if tool_progress_enabled: + if needs_progress_queue: progress_task = asyncio.create_task(send_progress_messages()) # Start stream consumer task โ€” polls for consumer creation since it diff --git a/run_agent.py b/run_agent.py index bc05ef845001..1fb2dfad6ca1 100644 --- a/run_agent.py +++ b/run_agent.py @@ -533,6 +533,8 @@ def __init__( # would mangle the escape sequences. None = use builtins.print. self._print_fn = None self.background_review_callback = None # Optional sync callback for gateway delivery + self.memory_notifications = "on" # Memory update notifications: "off", "on", "verbose" + self.thinking_progress = False # Relay assistant thinking text between tool calls to gateway self.skip_context_files = skip_context_files self.pass_session_id = pass_session_id self.persist_session = persist_session @@ -7921,21 +7923,39 @@ def _stop_spinner(): else: self._vprint(f"{self.log_prefix}๐Ÿค– Assistant: {assistant_message.content[:100]}{'...' if len(assistant_message.content) > 100 else ''}") - # Notify progress callback of model's thinking (used by subagent - # delegation to relay the child's reasoning to the parent display). - # Guard: only fire for subagents (_delegate_depth >= 1) to avoid - # spamming gateway platforms with the main agent's every thought. + # Notify progress callback of model's thinking. + # Only relay when the model will continue with tool calls โ€” not + # for the final response (which is delivered via the normal + # response path and would otherwise appear twice). + # Thinking relay is controlled by thinking_progress (independent + # of tool_progress). Subagents always relay for parent display. + # Main agent: only relay when tool_calls present (avoids duplicate + # final response). Subagents: always relay (their final response + # isn't delivered to chat directly โ€” it goes back to the parent + # as a tool result, so suppressing it would lose visibility). + _is_subagent = getattr(self, '_delegate_depth', 0) > 0 + _should_relay_thinking = ( + _is_subagent + or getattr(self, 'thinking_progress', False) + ) + _has_tool_calls = bool(assistant_message.tool_calls) if (assistant_message.content and self.tool_progress_callback - and getattr(self, '_delegate_depth', 0) > 0): + and _should_relay_thinking + and (_has_tool_calls or _is_subagent)): _think_text = assistant_message.content.strip() # Strip reasoning XML tags that shouldn't leak to parent display _think_text = re.sub( r'', '', _think_text ).strip() - first_line = _think_text.split('\n')[0][:80] if _think_text else "" - if first_line: + if getattr(self, 'thinking_progress', False): + # Explicit thinking_progress: relay complete text, no limit + _relay_text = _think_text or "" + else: + # Subagent relay: first line, truncated + _relay_text = _think_text.split('\n')[0][:80] if _think_text else "" + if _relay_text: try: - self.tool_progress_callback("_thinking", first_line) + self.tool_progress_callback("_thinking", _relay_text) except Exception: pass From fd6c506822aa3b4fe54284d3052a45ff7394e121 Mon Sep 17 00:00:00 2001 From: Amy Ravenwolf Date: Tue, 31 Mar 2026 15:53:32 +0200 Subject: [PATCH 02/10] feat(status): show model, context usage, and cumulative label in /status Add Model line (from live agent or config fallback), Context line showing approximate current context usage vs model limit with percentage, and mark cumulative token count explicitly as '(cumulative)'. Context tokens come from the agent's last_prompt_tokens (live) or session_entry.last_prompt_tokens (idle). Context limit resolves from the agent's context_compressor or DEFAULT_CONTEXT_LENGTHS lookup. Inspired by OpenClaw's status format but kept simple and pragmatic. --- gateway/run.py | 60 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/gateway/run.py b/gateway/run.py index b0c0205f5fd1..d586d2317c90 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -3106,7 +3106,55 @@ async def _handle_status_command(self, event: MessageEvent) -> str: # Check if there's an active agent session_key = session_entry.session_key - is_running = session_key in self._running_agents + agent = self._running_agents.get(session_key) + is_running = agent is not None and agent is not _AGENT_PENDING_SENTINEL + + # Resolve model info โ€” prefer live agent, fall back to config + model_name = "" + provider_name = "" + context_used = 0 + context_total = 0 + if is_running and hasattr(agent, "model"): + model_name = getattr(agent, "model", "") or "" + provider_name = getattr(agent, "provider", "") or "" + ctx = getattr(agent, "context_compressor", None) + if ctx: + context_used = getattr(ctx, "last_prompt_tokens", 0) or 0 + context_total = getattr(ctx, "context_length", 0) or 0 + else: + # Fall back to config for model and session entry for context + user_config = _load_gateway_config() + model_name = _resolve_gateway_model(user_config) + model_cfg = user_config.get("model", {}) + if isinstance(model_cfg, dict): + provider_name = model_cfg.get("provider", "") or "" + context_used = session_entry.last_prompt_tokens or 0 + if model_name: + try: + from agent.model_metadata import DEFAULT_CONTEXT_LENGTHS + model_lower = model_name.lower() + for key, length in sorted(DEFAULT_CONTEXT_LENGTHS.items(), key=lambda x: len(x[0]), reverse=True): + if key.lower() in model_lower: + context_total = length + break + except Exception: + pass + + # Build model line + model_line = "" + if model_name: + if provider_name: + model_line = f"**Model:** {model_name} ({provider_name})" + else: + model_line = f"**Model:** {model_name}" + + # Build context line + context_line = "" + if context_total: + pct = min(100, round(context_used / context_total * 100)) + context_line = f"**Context:** ~{context_used:,} / {context_total:,} ({pct}%)" + elif context_used: + context_line = f"**Context:** ~{context_used:,}" lines = [ "๐Ÿ“Š **Hermes Gateway Status**", @@ -3114,11 +3162,17 @@ async def _handle_status_command(self, event: MessageEvent) -> str: f"**Session ID:** `{session_entry.session_id[:12]}...`", f"**Created:** {session_entry.created_at.strftime('%Y-%m-%d %H:%M')}", f"**Last Activity:** {session_entry.updated_at.strftime('%Y-%m-%d %H:%M')}", - f"**Tokens:** {session_entry.total_tokens:,}", + ] + if model_line: + lines.append(model_line) + if context_line: + lines.append(context_line) + lines.extend([ + f"**Tokens:** {session_entry.total_tokens:,} (cumulative)", f"**Agent Running:** {'Yes โšก' if is_running else 'No'}", "", f"**Connected Platforms:** {', '.join(connected_platforms)}", - ] + ]) return "\n".join(lines) From 48af5fb9b4fcc4762da665d2a5d4110e6c99196d Mon Sep 17 00:00:00 2001 From: Amy Ravenwolf Date: Tue, 31 Mar 2026 15:15:34 +0200 Subject: [PATCH 03/10] feat(memory): configurable background memory update notifications MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Background memory reviews now support three notification modes, configured via display.memory_notifications in config.yaml: off โ€” no chat notification (still logged to stdout/HA log) on โ€” generic '๐Ÿ’พ Memory updated' (default, unchanged behavior) verbose โ€” content preview with action indicators: ๐Ÿ’พ Memory โž• Hermes Repo liegt unter /config/amy/hermes-agent/... ๐Ÿ’พ Memory โœ๏ธ Updated repo path from claude-code to hermes-agent... ๐Ÿ’พ Memory โž– old entry about claude-code path... Previews are truncated to 120 chars for adds/replaces, 60 for removes. Each action gets its own line in verbose mode for readability. Files: run_agent.py, gateway/run.py --- gateway/run.py | 8 ++++ run_agent.py | 120 +++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 109 insertions(+), 19 deletions(-) diff --git a/gateway/run.py b/gateway/run.py index d586d2317c90..aa8a669e07f5 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -5846,6 +5846,14 @@ def _bg_review_send(message: str) -> None: logger.debug("background_review_callback error: %s", _e) agent.background_review_callback = _bg_review_send + # Memory update notifications in chat. Config: display.memory_notifications + # off โ€” no chat notification (still logged to stdout) + # on โ€” generic "๐Ÿ’พ Memory updated" (default) + # verbose โ€” content preview: "๐Ÿ’พ Memory โž• Hermes Repo..." + _mem_notif = user_config.get("display", {}).get("memory_notifications") + if isinstance(_mem_notif, bool): + _mem_notif = "on" if _mem_notif else "off" + agent.memory_notifications = str(_mem_notif).lower() if _mem_notif else "on" # Show assistant thinking between tool calls โ€” fully independent # of tool_progress mode. Config: display.thinking_progress: true diff --git a/run_agent.py b/run_agent.py index 1fb2dfad6ca1..968bc203bbe3 100644 --- a/run_agent.py +++ b/run_agent.py @@ -1696,10 +1696,54 @@ def _run_review(): # Scan the review agent's messages for successful tool actions # and surface a compact summary to the user. + # + # memory_notifications controls chat delivery: + # "off" โ€” no chat notification (stdout only) + # "on" โ€” generic "Memory updated" (default) + # "verbose" โ€” content preview with โž•/โœ๏ธ/โž– indicators + _notif_mode = getattr(self, "memory_notifications", "on") actions = [] - for msg in getattr(review_agent, "_session_messages", []): + _verbose = _notif_mode == "verbose" + _review_msgs = getattr(review_agent, "_session_messages", []) + + # Build a map of tool_call_id โ†’ details from assistant messages + # that contain memory or skill_manage tool_calls. Used to + # filter out unrelated tool responses (session_search, etc.) + # and (in verbose mode) to display content previews. + _NOTIFY_TOOLS = {"memory", "skill_manage"} + _call_details: dict = {} + for msg in _review_msgs: + if msg.get("role") != "assistant": + continue + for tc in msg.get("tool_calls", []): + fn = tc.get("function", {}) + fn_name = fn.get("name", "") + if fn_name not in _NOTIFY_TOOLS: + continue + try: + args = json.loads(fn.get("arguments", "{}")) + except (json.JSONDecodeError, TypeError): + args = {} + tc_id = tc.get("id") + if tc_id: + _call_details[tc_id] = { + "tool": fn_name, + "action": args.get("action", "?"), + "target": args.get("target", "memory"), + "content": args.get("content", ""), + "old_text": args.get("old_text", ""), + "name": args.get("name", ""), + } + + for msg in _review_msgs: if not isinstance(msg, dict) or msg.get("role") != "tool": continue + # Only process memory/skill tool responses โ€” other tools + # (session_search, etc.) also return {"success": true} but + # are not worth notifying about. + tc_id = msg.get("tool_call_id") + if tc_id and _call_details and tc_id not in _call_details: + continue try: data = json.loads(msg.get("content", "{}")) except (json.JSONDecodeError, TypeError): @@ -1708,29 +1752,67 @@ def _run_review(): continue message = data.get("message", "") target = data.get("target", "") - if "created" in message.lower(): - actions.append(message) - elif "updated" in message.lower(): - actions.append(message) - elif "added" in message.lower() or (target and "add" in message.lower()): - label = "Memory" if target == "memory" else "User profile" if target == "user" else target - actions.append(f"{label} updated") - elif "Entry added" in message: - label = "Memory" if target == "memory" else "User profile" if target == "user" else target - actions.append(f"{label} updated") - elif "removed" in message.lower() or "replaced" in message.lower(): + detail = _call_details.get(tc_id, {}) + is_skill = detail.get("tool") == "skill_manage" + + # Determine display label + if is_skill: + label = "Skill" + elif target: label = "Memory" if target == "memory" else "User profile" if target == "user" else target - actions.append(f"{label} updated") + else: + continue # skip tool responses we can't label + + if _verbose: + action = detail.get("action", "") + content = detail.get("content", "") + old_text = detail.get("old_text", "") + skill_name = detail.get("name", "") + + # Build a descriptive action string with content preview + _MAX_PREVIEW = 120 + if is_skill: + # Skill notifications: use the message from the tool + actions.append(f"๐Ÿ“ {message}" if message else f"Skill {action}") + elif action == "add" and content: + preview = content[:_MAX_PREVIEW] + ("โ€ฆ" if len(content) > _MAX_PREVIEW else "") + actions.append(f"{label} โž• {preview}") + elif action == "replace" and content: + preview = content[:_MAX_PREVIEW] + ("โ€ฆ" if len(content) > _MAX_PREVIEW else "") + actions.append(f"{label} โœ๏ธ {preview}") + elif action == "remove" and old_text: + preview = old_text[:60] + ("โ€ฆ" if len(old_text) > 60 else "") + actions.append(f"{label} โž– {preview}") + else: + actions.append(f"{label} updated") + else: + # Generic labels (default behavior) + if "created" in message.lower(): + actions.append(message) + elif "updated" in message.lower(): + actions.append(message) + elif "added" in message.lower() or "replaced" in message.lower() or "removed" in message.lower(): + actions.append(f"{label} updated") + else: + actions.append(f"{label} updated") if actions: summary = " ยท ".join(dict.fromkeys(actions)) + # Always log to stdout (HA addon log) self._safe_print(f" ๐Ÿ’พ {summary}") - _bg_cb = self.background_review_callback - if _bg_cb: - try: - _bg_cb(f"๐Ÿ’พ {summary}") - except Exception: - pass + # Send to chat unless notifications are off + if _notif_mode != "off": + _bg_cb = self.background_review_callback + if _bg_cb: + try: + if _verbose: + # Show each action on its own line for readability + chat_summary = "\n".join(f"๐Ÿ’พ {a}" for a in dict.fromkeys(actions)) + else: + chat_summary = f"๐Ÿ’พ {summary}" + _bg_cb(chat_summary) + except Exception: + pass except Exception as e: logger.debug("Background memory/skill review failed: %s", e) From 045d4803bc3714950086127d563c408271d5eaec Mon Sep 17 00:00:00 2001 From: Amy Ravenwolf Date: Tue, 31 Mar 2026 14:06:38 +0200 Subject: [PATCH 04/10] feat(resume): cross-platform /resume with API server support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Complete /resume overhaul for cross-platform session management: - API server now passes session_db to AIAgent, so sessions get registered in state.db (previously ~115 API sessions were invisible) - Four listing modes: /resume Named sessions, current platform (default) /resume all Named sessions, ALL platforms /resume --full All sessions incl. unnamed, current platform /resume all --full All sessions incl. unnamed, ALL platforms - Platform tags [telegram], [cli] etc. in cross-platform listings - Session ID prefix shown for unnamed sessions - load_transcript() now also reads session_{id}.json (AIAgent log format used by API server), picking whichever source has the most messages โ€” fixes /resume losing context for API sessions Files: gateway/platforms/api_server.py, gateway/run.py --- gateway/platforms/api_server.py | 9 +++- gateway/run.py | 74 ++++++++++++++++++++++++++------- gateway/session.py | 55 ++++++++++++++++++++---- 3 files changed, 114 insertions(+), 24 deletions(-) diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py index 86af84307d65..038f82e9cca6 100644 --- a/gateway/platforms/api_server.py +++ b/gateway/platforms/api_server.py @@ -300,7 +300,14 @@ def __init__(self, config: PlatformConfig): self._runner: Optional["web.AppRunner"] = None self._site: Optional["web.TCPSite"] = None self._response_store = ResponseStore() - self._session_db: Optional[Any] = None # Lazy-init SessionDB for session continuity + # Shared SessionDB singleton โ€” avoids creating (and leaking) a new + # SQLite connection on every /v1/chat/completions request. + self._session_db = None + try: + from hermes_state import SessionDB + self._session_db = SessionDB() + except Exception: + pass @staticmethod def _parse_cors_origins(value: Any) -> tuple[str, ...]: diff --git a/gateway/run.py b/gateway/run.py index aa8a669e07f5..384e3c7e8d42 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -4559,40 +4559,84 @@ async def _handle_title_command(self, event: MessageEvent) -> str: return f"๐Ÿ“Œ Session: `{session_id}`\nNo title set. Usage: `/title My Session Name`" async def _handle_resume_command(self, event: MessageEvent) -> str: - """Handle /resume command โ€” switch to a previously-named session.""" + """Handle /resume command โ€” switch to a previously-named session. + + Listing modes: + /resume Named sessions, current platform only + /resume --all Named sessions, ALL platforms + /resume --full All sessions (incl. unnamed), current platform + /resume --all --full All sessions (incl. unnamed), ALL platforms + /resume Resume a specific named session (global lookup) + """ if not self._session_db: return "Session database not available." source = event.source session_key = self._session_key_for_source(source) - name = event.get_command_args().strip() + raw_args = event.get_command_args().strip() + + # Parse listing flags (-- prefixed to avoid collisions with session names) + args_lower = raw_args.lower() + tokens = set(args_lower.split()) + has_flag = bool(tokens & {"--all", "--full"}) + is_listing = not raw_args or has_flag + cross_platform = "--all" in tokens if is_listing else False + show_unnamed = "--full" in tokens if is_listing else False - if not name: - # List recent titled sessions for this user/platform + if is_listing: try: - user_source = source.platform.value if source.platform else None + user_source = None if cross_platform else (source.platform.value if source.platform else None) sessions = self._session_db.list_sessions_rich( - source=user_source, limit=10 + source=user_source, limit=50 ) - titled = [s for s in sessions if s.get("title")] - if not titled: + if show_unnamed: + filtered = sessions + else: + filtered = [s for s in sessions if s.get("title")] + + if not filtered: + scope = "across all platforms" if cross_platform else "on this platform" + kind = "sessions" if show_unnamed else "named sessions" return ( - "No named sessions found.\n" + f"No {kind} found {scope}.\n" "Use `/title My Session` to name your current session, " "then `/resume My Session` to return to it later." ) - lines = ["๐Ÿ“‹ **Named Sessions**\n"] - for s in titled[:10]: - title = s["title"] - preview = s.get("preview", "")[:40] + + scope_label = "All Platforms" if cross_platform else (source.platform.value.title() if source.platform else "Unknown") + if show_unnamed: + header = f"๐Ÿ“‹ **All Sessions โ€” {scope_label}**\n" + else: + header = f"๐Ÿ“‹ **Named Sessions โ€” {scope_label}**\n" + + lines = [header] + for s in filtered[:20]: + title = s.get("title") + preview = (s.get("preview") or "")[:40] + src = s.get("source", "?") + if title: + label = f"**{title}**" + else: + # Unnamed: show session ID prefix + preview + sid = s.get("id", "???")[:20] + label = f"`{sid}`" preview_part = f" โ€” _{preview}_" if preview else "" - lines.append(f"โ€ข **{title}**{preview_part}") + platform_tag = f" [{src}]" if cross_platform else "" + lines.append(f"โ€ข {label}{preview_part}{platform_tag}") + lines.append("\nUsage: `/resume `") + if not cross_platform: + lines.append("Tip: `/resume --all` for cross-platform, `--full` to include unnamed") + elif not show_unnamed: + lines.append("Tip: `/resume --all --full` to include unnamed sessions") return "\n".join(lines) except Exception as e: - logger.debug("Failed to list titled sessions: %s", e) + logger.debug("Failed to list sessions: %s", e) return f"Could not list sessions: {e}" + # Not a listing flag โ€” treat as session name to resume + name = raw_args + # Resolve the name to a session ID target_id = self._session_db.resolve_session_by_title(name) if not target_id: diff --git a/gateway/session.py b/gateway/session.py index c3b913ef815c..a87ac72d016c 100644 --- a/gateway/session.py +++ b/gateway/session.py @@ -968,6 +968,17 @@ def rewrite_transcript(self, session_id: str, messages: List[Dict[str, Any]]) -> for msg in messages: f.write(json.dumps(msg, ensure_ascii=False) + "\n") + # JSON log (session_.json): overwrite to prevent load_transcript + # from picking the stale (longer) json_log over the rewritten + # SQLite/JSONL sources after /undo, /retry, or /compress. + json_log_path = self.sessions_dir / f"session_{session_id}.json" + if json_log_path.exists(): + try: + with open(json_log_path, "w", encoding="utf-8") as f: + json.dump({"messages": messages}, f, ensure_ascii=False) + except Exception as e: + logger.debug("Failed to rewrite json log %s: %s", json_log_path, e) + def load_transcript(self, session_id: str) -> List[Dict[str, Any]]: """Load all messages from a session's transcript.""" db_messages = [] @@ -995,7 +1006,25 @@ def load_transcript(self, session_id: str) -> List[Dict[str, Any]]: session_id, line[:120], ) - # Prefer whichever source has more messages. + # Fallback: load from session_.json (AIAgent session log format). + # The AIAgent (run_agent.py) saves transcripts as session_.json + # with a {"messages": [...]} structure. This is the ONLY transcript + # format for API server sessions (which bypass the gateway session + # store entirely), and also exists for gateway sessions as a secondary + # log. Without this fallback, /resume cannot restore API sessions. + json_messages = [] + session_log_path = self.sessions_dir / f"session_{session_id}.json" + if session_log_path.exists(): + try: + with open(session_log_path, "r", encoding="utf-8") as f: + data = json.load(f) + json_messages = data.get("messages", []) + except (json.JSONDecodeError, Exception) as e: + logger.debug( + "Could not load session log %s: %s", session_log_path, e + ) + + # Prefer whichever source has the most messages. # # Background: when a session pre-dates SQLite storage (or when the DB # layer was added while a long-lived session was already active), the @@ -1005,16 +1034,26 @@ def load_transcript(self, session_id: str) -> List[Dict[str, Any]]: # turn load_transcript returns those few SQLite rows and ignores the # full JSONL history โ€” the model sees a context of 1-4 messages instead # of hundreds. Using the longer source prevents this silent truncation. - if len(jsonl_messages) > len(db_messages): - if db_messages: + candidates = [ + ("sqlite", db_messages), + ("jsonl", jsonl_messages), + ("json_log", json_messages), + ] + best_source, best_messages = max(candidates, key=lambda x: len(x[1])) + + if len(best_messages) > 0 and best_source != "sqlite": + runner_up = max( + ((s, m) for s, m in candidates if s != best_source), + key=lambda x: len(x[1]), + ) + if runner_up[1]: logger.debug( - "Session %s: JSONL has %d messages vs SQLite %d โ€” " - "using JSONL (legacy session not yet fully migrated)", - session_id, len(jsonl_messages), len(db_messages), + "Session %s: using %s (%d messages) over %s (%d messages)", + session_id, best_source, len(best_messages), + runner_up[0], len(runner_up[1]), ) - return jsonl_messages - return db_messages + return best_messages def build_session_context( From 1d21a91bc8c3a00917c647d4e5068e30f381ba42 Mon Sep 17 00:00:00 2001 From: Amy Ravenwolf Date: Tue, 31 Mar 2026 14:06:16 +0200 Subject: [PATCH 05/10] =?UTF-8?q?feat(prompt=5Fbuilder):=20Amy=20platform?= =?UTF-8?q?=20hints=20=E2=80=94=20zipper=20mode=20defaults?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Platform-aware zipper mode defaults for Amy's persona system: - Telegram: Private Assistant Mode (zipper half-open) by default - WhatsApp: Public Assistant Mode (zipper closed) โ€” explicit hint, matching SOUL.md default - API server (SillyTavern, Open WebUI, LobeChat): Private Assistant Mode by default โ€” API frontends are private environments - All other platforms: no override, SOUL.md default applies Amy-specific โ€” not for upstream. Files: agent/prompt_builder.py, tests/agent/test_prompt_builder.py --- agent/prompt_builder.py | 13 ++++++++++++- tests/agent/test_prompt_builder.py | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/agent/prompt_builder.py b/agent/prompt_builder.py index fbb5f0fa03ca..78c812cbdc0f 100644 --- a/agent/prompt_builder.py +++ b/agent/prompt_builder.py @@ -236,7 +236,9 @@ def _strip_yaml_frontmatter(content: str) -> str: "include MEDIA:/absolute/path/to/file in your response. Images " "(.png, .jpg, .webp) appear as photos, audio (.ogg) sends as voice " "bubbles, and videos (.mp4) play inline. You can also include image " - "URLs in markdown format ![alt](url) and they will be sent as native photos." + "URLs in markdown format ![alt](url) and they will be sent as native photos.\n\n" + "Amy's zipper starts HALF-OPEN on Telegram โ€” Private Assistant Mode by default. " + "The user can change this during the session." ), "discord": ( "You are in a Discord server or group chat communicating with your user. " @@ -285,6 +287,15 @@ def _strip_yaml_frontmatter(content: str) -> str: "only โ€” no markdown, no formatting. SMS messages are limited to ~1600 " "characters, so be brief and direct." ), + "api_server": ( + "You are accessed via an OpenAI-compatible API from a private frontend " + "(e.g. SillyTavern, Open WebUI, LobeChat). Markdown is fully supported. " + "You can send media files: include MEDIA:/absolute/path/to/file in your " + "response. You can also include image URLs in markdown format ![alt](url). " + "This is a private environment.\n\n" + "Amy's zipper starts HALF-OPEN โ€” Private Assistant Mode by default. " + "The user can change this during the session." + ), } CONTEXT_FILE_MAX_CHARS = 20_000 diff --git a/tests/agent/test_prompt_builder.py b/tests/agent/test_prompt_builder.py index 791f7ea0eb22..66f39a6fd2a3 100644 --- a/tests/agent/test_prompt_builder.py +++ b/tests/agent/test_prompt_builder.py @@ -768,6 +768,7 @@ def test_platform_hints_known_platforms(self): assert "discord" in PLATFORM_HINTS assert "cron" in PLATFORM_HINTS assert "cli" in PLATFORM_HINTS + assert "api_server" in PLATFORM_HINTS # ========================================================================= From df54e58fa7abd8c47e772a92b07ef36eccf66c59 Mon Sep 17 00:00:00 2001 From: Amy Ravenwolf Date: Fri, 3 Apr 2026 16:19:13 +0200 Subject: [PATCH 06/10] feat(display): show delegate_task goals in tool progress notifications Previously, delegate_task in batch mode only showed '3 parallel tasks' without revealing what the tasks actually are. Single-task mode showed the goal via the primary_args fallback, but batch mode had no goal extraction. Changes: - build_tool_preview(): Add dedicated delegate_task handler that extracts individual task goals from both single and batch modes. Batch shows '3 tasks: Goal A | Goal B | Goal C'. - _get_cute_tool_message_impl(): Show individual goals in CLI cute messages for batch delegate calls ('3x: Goal A | Goal B'). - Add 4 tests covering single goal, batch goals, missing goals, and no-goal edge case. --- agent/display.py | 13 ++++++++++++- tests/test_display.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/agent/display.py b/agent/display.py index 94259fa80a89..ab3dd716ba52 100644 --- a/agent/display.py +++ b/agent/display.py @@ -153,6 +153,15 @@ def build_tool_preview(tool_name: str, args: dict, max_len: int | None = None) - "clarify": "question", "skill_manage": "name", } + # delegate_task: show goal (single) or individual task goals (batch) + if tool_name == "delegate_task": + tasks = args.get("tasks") + if tasks and isinstance(tasks, list): + goals = [_oneline(t.get("goal", "?"))[:40] for t in tasks if isinstance(t, dict)] + return f"{len(tasks)} tasks: " + " | ".join(goals) if goals else f"{len(tasks)} parallel tasks" + goal = args.get("goal", "") + return _oneline(goal) if goal else None + if tool_name == "process": action = args.get("action", "") sid = args.get("session_id", "") @@ -956,7 +965,9 @@ def _wrap(line: str) -> str: if tool_name == "delegate_task": tasks = args.get("tasks") if tasks and isinstance(tasks, list): - return _wrap(f"โ”Š ๐Ÿ”€ delegate {len(tasks)} parallel tasks {dur}") + goals = [_oneline(t.get("goal", "?"))[:30] for t in tasks if isinstance(t, dict)] + detail = " | ".join(goals) if goals else "parallel" + return _wrap(f"โ”Š ๐Ÿ”€ delegate {len(tasks)}x: {_trunc(detail, 35)} {dur}") return _wrap(f"โ”Š ๐Ÿ”€ delegate {_trunc(args.get('goal', ''), 35)} {dur}") preview = build_tool_preview(tool_name, args) or "" diff --git a/tests/test_display.py b/tests/test_display.py index 5127a930ba11..45709b69ee19 100644 --- a/tests/test_display.py +++ b/tests/test_display.py @@ -88,6 +88,40 @@ def test_session_search_preview(self): assert result is not None assert "find something" in result + def test_delegate_task_single_goal(self): + """Single-task delegate should show the goal.""" + result = build_tool_preview("delegate_task", {"goal": "Research Python typing"}) + assert result is not None + assert "Research Python typing" in result + + def test_delegate_task_batch(self): + """Batch delegate should show task count and individual goals.""" + result = build_tool_preview("delegate_task", {"tasks": [ + {"goal": "Research topic A"}, + {"goal": "Analyze data B"}, + {"goal": "Write report C"}, + ]}) + assert result is not None + assert "3 tasks" in result + assert "Research topic A" in result + assert "Analyze data B" in result + assert "Write report C" in result + + def test_delegate_task_batch_empty_goals(self): + """Batch delegate with missing goals should not crash.""" + result = build_tool_preview("delegate_task", {"tasks": [ + {"goal": "Only this one"}, + {}, # missing goal + ]}) + assert result is not None + assert "2 tasks" in result + assert "Only this one" in result + + def test_delegate_task_no_goal(self): + """delegate_task with neither goal nor tasks returns None.""" + result = build_tool_preview("delegate_task", {"context": "some context"}) + assert result is None + def test_false_like_args_zero(self): """Non-dict falsy values should return None, not crash.""" assert build_tool_preview("terminal", 0) is None From c9da2f9bcbc6fd0a1a00f15f1b73d6ccd3e8bf3f Mon Sep 17 00:00:00 2001 From: Amy Ravenwolf Date: Mon, 30 Mar 2026 04:32:04 +0200 Subject: [PATCH 07/10] fix(session_search): lazy SessionDB creation for background agents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PROBLEM: The session_search tool's registry handler passed db=None directly to the session_search() function when no SessionDB instance was provided via kwargs. This caused the tool to silently return zero results in contexts where no pre-initialized DB existed โ€” notably in cron jobs and background agents (memory flush). First observed 2026-03-25 when session_search consistently returned empty results despite the FTS index being populated and healthy. Direct SQL queries against state.db worked fine, confirming the issue was in the tool's initialization, not the data. SOLUTION: Add a _session_search_handler that lazily creates a SessionDB instance when none is provided via kwargs, then delegates to the existing session_search() function. This ensures the tool works in all contexts including background/cron agents. Session: 20260325_050509 --- .gitignore | 1 + tools/session_search_tool.py | 26 ++++++++++++++++++++------ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index baa31a543c18..9b24a2ea676b 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,4 @@ mini-swe-agent/ # Nix .direnv/ result +hermes_state.db diff --git a/tools/session_search_tool.py b/tools/session_search_tool.py index 3ff36f940b84..9f5f5cd87c6a 100644 --- a/tools/session_search_tool.py +++ b/tools/session_search_tool.py @@ -489,16 +489,30 @@ def check_session_search_requirements() -> bool: # --- Registry --- from tools.registry import registry + +def _session_search_handler(args, **kw): + """Registry handler that lazily creates a SessionDB if none is provided.""" + db = kw.get("db") + if db is None: + try: + from hermes_state import SessionDB + db = SessionDB() + except Exception: + logging.debug("session_search: could not create SessionDB, proceeding without") + return session_search( + query=args.get("query", ""), + role_filter=args.get("role_filter"), + limit=args.get("limit", 3), + db=db, + current_session_id=kw.get("current_session_id"), + ) + + registry.register( name="session_search", toolset="session_search", schema=SESSION_SEARCH_SCHEMA, - handler=lambda args, **kw: session_search( - query=args.get("query") or "", - role_filter=args.get("role_filter"), - limit=args.get("limit", 3), - db=kw.get("db"), - current_session_id=kw.get("current_session_id")), + handler=_session_search_handler, check_fn=check_session_search_requirements, emoji="๐Ÿ”", ) From 4a4424eb7b523cbde8766f76b3c3f3ead4ad2e77 Mon Sep 17 00:00:00 2001 From: Amy Ravenwolf Date: Mon, 30 Mar 2026 04:32:24 +0200 Subject: [PATCH 08/10] fix: WhatsApp voice messages + bridge audio download + npm deps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add send_voice() to WhatsApp adapter โ€” voice/audio files now play as native voice notes in WhatsApp instead of failing silently - Fix bridge audio download: proper MIME type detection with extension map (ogg/mp3/m4a/wav), correct AUDIO_CACHE_DIR path resolution, consistent 'audio_' filename prefix - Update whatsapp-bridge npm dependencies (lock file refresh) Files: gateway/platforms/whatsapp.py, scripts/whatsapp-bridge/bridge.js, package-lock.json, scripts/whatsapp-bridge/package-lock.json --- gateway/platforms/whatsapp.py | 11 +++++++++ package-lock.json | 20 ++++++++-------- scripts/whatsapp-bridge/bridge.js | 8 ++++--- scripts/whatsapp-bridge/package-lock.json | 28 +++++++++++------------ 4 files changed, 40 insertions(+), 27 deletions(-) diff --git a/gateway/platforms/whatsapp.py b/gateway/platforms/whatsapp.py index ac94e4720056..81bb88bd90b3 100644 --- a/gateway/platforms/whatsapp.py +++ b/gateway/platforms/whatsapp.py @@ -707,6 +707,17 @@ async def send_image_file( """Send a local image file natively via bridge.""" return await self._send_media_to_bridge(chat_id, image_path, "image", caption) + async def send_voice( + self, + chat_id: str, + audio_path: str, + caption: Optional[str] = None, + reply_to: Optional[str] = None, + **kwargs, + ) -> SendResult: + """Send a voice/audio file natively via bridge โ€” plays as voice note in WhatsApp.""" + return await self._send_media_to_bridge(chat_id, audio_path, "audio", caption) + async def send_video( self, chat_id: str, diff --git a/package-lock.json b/package-lock.json index 1e54db9aa55d..a9370f9497b9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1268,9 +1268,9 @@ } }, "node_modules/fast-xml-parser": { - "version": "5.5.9", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.5.9.tgz", - "integrity": "sha512-jldvxr1MC6rtiZKgrFnDSvT8xuH+eJqxqOBThUVjYrxssYTo1avZLGql5l0a0BAERR01CadYzZ83kVEkbyDg+g==", + "version": "5.5.8", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.5.8.tgz", + "integrity": "sha512-Z7Fh2nVQSb2d+poDViM063ix2ZGt9jmY1nWhPfHBOK2Hgnb/OW3P4Et3P/81SEej0J7QbWtJqxO05h8QYfK7LQ==", "funding": [ { "type": "github", @@ -1281,7 +1281,7 @@ "dependencies": { "fast-xml-builder": "^1.1.4", "path-expression-matcher": "^1.2.0", - "strnum": "^2.2.2" + "strnum": "^2.2.0" }, "bin": { "fxparser": "src/cli/cli.js" @@ -2545,9 +2545,9 @@ } }, "node_modules/strnum": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.2.2.tgz", - "integrity": "sha512-DnR90I+jtXNSTXWdwrEy9FakW7UX+qUZg28gj5fk2vxxl7uS/3bpI4fjFYVmdK9etptYBPNkpahuQnEwhwECqA==", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.2.1.tgz", + "integrity": "sha512-BwRvNd5/QoAtyW1na1y1LsJGQNvRlkde6Q/ipqqEaivoMdV+B1OMOTVdwR+N/cwVUcIt9PYyHmV8HyexCZSupg==", "funding": [ { "type": "github", @@ -2647,9 +2647,9 @@ } }, "node_modules/undici": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.24.6.tgz", - "integrity": "sha512-Xi4agocCbRzt0yYMZGMA6ApD7gvtUFaxm4ZmeacWI4cZxaF6C+8I8QfofC20NAePiB/IcvZmzkJ7XPa471AEtA==", + "version": "7.24.5", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.24.5.tgz", + "integrity": "sha512-3IWdCpjgxp15CbJnsi/Y9TCDE7HWVN19j1hmzVhoAkY/+CJx449tVxT5wZc1Gwg8J+P0LWvzlBzxYRnHJ+1i7Q==", "license": "MIT", "engines": { "node": ">=20.18.1" diff --git a/scripts/whatsapp-bridge/bridge.js b/scripts/whatsapp-bridge/bridge.js index 70cf8e95d9fa..f98b5c0028ae 100644 --- a/scripts/whatsapp-bridge/bridge.js +++ b/scripts/whatsapp-bridge/bridge.js @@ -282,12 +282,14 @@ async function startSocket() { hasMedia = true; mediaType = messageContent.pttMessage ? 'ptt' : 'audio'; try { - const audioMsg = messageContent.pttMessage || messageContent.audioMessage; + const buf = await downloadMediaMessage(msg, 'buffer', {}, { logger, reuploadRequest: sock.updateMediaMessage }); + const audioMsg = msg.message.audioMessage || msg.message.pttMessage; const mime = audioMsg.mimetype || 'audio/ogg'; - const ext = mime.includes('ogg') ? '.ogg' : mime.includes('mp4') ? '.m4a' : '.ogg'; + const extMap = { 'audio/ogg; codecs=opus': '.ogg', 'audio/ogg': '.ogg', 'audio/mpeg': '.mp3', 'audio/mp4': '.m4a', 'audio/wav': '.wav' }; + const ext = extMap[mime] || '.ogg'; mkdirSync(AUDIO_CACHE_DIR, { recursive: true }); - const filePath = path.join(AUDIO_CACHE_DIR, `aud_${randomBytes(6).toString('hex')}${ext}`); + const filePath = path.join(AUDIO_CACHE_DIR, `audio_${randomBytes(6).toString('hex')}${ext}`); writeFileSync(filePath, buf); mediaUrls.push(filePath); } catch (err) { diff --git a/scripts/whatsapp-bridge/package-lock.json b/scripts/whatsapp-bridge/package-lock.json index 01af1c15a0e1..901a326af19b 100644 --- a/scripts/whatsapp-bridge/package-lock.json +++ b/scripts/whatsapp-bridge/package-lock.json @@ -15,9 +15,9 @@ } }, "node_modules/@borewit/text-codec": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/@borewit/text-codec/-/text-codec-0.2.1.tgz", - "integrity": "sha512-k7vvKPbf7J2fZ5klGRD9AeKfUvojuZIQ3BT5u7Jfv+puwXkUBUT5PVyMDfJZpy30CBDXGMgw7fguK/lpOMBvgw==", + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@borewit/text-codec/-/text-codec-0.2.2.tgz", + "integrity": "sha512-DDaRehssg1aNrH4+2hnj1B7vnUGEjU6OIlyRdkMd0aUdIUvKXrJfXsy8LVtXAy7DRvYVluWbMspsRhz2lcW0mQ==", "license": "MIT", "funding": { "type": "github", @@ -1087,9 +1087,9 @@ } }, "node_modules/file-type": { - "version": "21.3.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-21.3.0.tgz", - "integrity": "sha512-8kPJMIGz1Yt/aPEwOsrR97ZyZaD1Iqm8PClb1nYFclUCkBi0Ma5IsYNQzvSFS9ib51lWyIw5mIT9rWzI/xjpzA==", + "version": "21.3.4", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-21.3.4.tgz", + "integrity": "sha512-Ievi/yy8DS3ygGvT47PjSfdFoX+2isQueoYP1cntFW1JLYAuS4GD7NUPGg4zv2iZfV52uDyk5w5Z0TdpRS6Q1g==", "license": "MIT", "dependencies": { "@tokenizer/inflate": "^0.4.1", @@ -1455,9 +1455,9 @@ "license": "MIT" }, "node_modules/music-metadata": { - "version": "11.12.1", - "resolved": "https://registry.npmjs.org/music-metadata/-/music-metadata-11.12.1.tgz", - "integrity": "sha512-j++ltLxHDb5VCXET9FzQ8bnueiLHwQKgCO7vcbkRH/3F7fRjPkv6qncGEJ47yFhmemcYtgvsOAlcQ1dRBTkDjg==", + "version": "11.12.3", + "resolved": "https://registry.npmjs.org/music-metadata/-/music-metadata-11.12.3.tgz", + "integrity": "sha512-n6hSTZkuD59qWgHh6IP5dtDlDZQXoxk/bcA85Jywg8Z1iFrlNgl2+GTFgjZyn52W5UgQpV42V4XqrQZZAMbZTQ==", "funding": [ { "type": "github", @@ -1470,11 +1470,11 @@ ], "license": "MIT", "dependencies": { - "@borewit/text-codec": "^0.2.1", + "@borewit/text-codec": "^0.2.2", "@tokenizer/token": "^0.3.0", "content-type": "^1.0.5", "debug": "^4.4.3", - "file-type": "^21.3.0", + "file-type": "^21.3.1", "media-typer": "^1.1.0", "strtok3": "^10.3.4", "token-types": "^6.1.2", @@ -2001,9 +2001,9 @@ } }, "node_modules/strtok3": { - "version": "10.3.4", - "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-10.3.4.tgz", - "integrity": "sha512-KIy5nylvC5le1OdaaoCJ07L+8iQzJHGH6pWDuzS+d07Cu7n1MZ2x26P8ZKIWfbK02+XIL8Mp4RkWeqdUCrDMfg==", + "version": "10.3.5", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-10.3.5.tgz", + "integrity": "sha512-ki4hZQfh5rX0QDLLkOCj+h+CVNkqmp/CMf8v8kZpkNVK6jGQooMytqzLZYUVYIZcFZ6yDB70EfD8POcFXiF5oA==", "license": "MIT", "dependencies": { "@tokenizer/token": "^0.3.0" From f7aea2e4053cfd46535ae11008b7a874f0bb9d69 Mon Sep 17 00:00:00 2001 From: Amy Ravenwolf Date: Tue, 31 Mar 2026 14:05:55 +0200 Subject: [PATCH 09/10] =?UTF-8?q?feat(tool=5Fprogress):=20add=20'full'=20m?= =?UTF-8?q?ode=20=E2=80=94=20unlimited=20tool=20args=20in=20gateway=20chat?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New 'full' mode for display.tool_progress โ€” top of the mode hierarchy (off โ†’ new โ†’ all โ†’ verbose โ†’ full): - Shows complete tool arguments without any truncation in gateway chat (Telegram, Discord, etc.). Previously verbose mode capped at 200 chars. - Added to /verbose cycle, hermes setup wizard, and mode validation. This commit only affects tool call display. Assistant thinking relay is handled separately by thinking_progress. Files: run_agent.py, gateway/run.py, hermes_cli/setup.py --- cli.py | 7 ++++--- gateway/run.py | 13 +++++++++++-- hermes_cli/setup.py | 5 +++-- tests/gateway/test_verbose_command.py | 6 +++--- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/cli.py b/cli.py index 42a49440cd7d..0ab6ced68129 100644 --- a/cli.py +++ b/cli.py @@ -4719,8 +4719,8 @@ def _handle_skin_command(self, cmd: str): print(" Prompt + TUI colors updated.") def _toggle_verbose(self): - """Cycle tool progress mode: off โ†’ new โ†’ all โ†’ verbose โ†’ off.""" - cycle = ["off", "new", "all", "verbose"] + """Cycle tool progress mode: off โ†’ new โ†’ all โ†’ verbose โ†’ full โ†’ off.""" + cycle = ["off", "new", "all", "verbose", "full"] try: idx = cycle.index(self.tool_progress_mode) except ValueError: @@ -4742,7 +4742,8 @@ def _toggle_verbose(self): "off": f"{_Colors.DIM}Tool progress: OFF{_Colors.RESET} โ€” silent mode, just the final response.", "new": f"{_Colors.YELLOW}Tool progress: NEW{_Colors.RESET} โ€” show each new tool (skip repeats).", "all": f"{_Colors.GREEN}Tool progress: ALL{_Colors.RESET} โ€” show every tool call.", - "verbose": f"{_Colors.BOLD}{_Colors.GREEN}Tool progress: VERBOSE{_Colors.RESET} โ€” full args, results, think blocks, and debug logs.", + "verbose": f"{_Colors.BOLD}{_Colors.GREEN}Tool progress: VERBOSE{_Colors.RESET} โ€” detailed tool args (200 char limit).", + "full": f"{_Colors.BOLD}{_Colors.CYAN}Tool progress: FULL{_Colors.RESET} โ€” complete args, no truncation.", } _cprint(labels.get(self.tool_progress_mode, "")) diff --git a/gateway/run.py b/gateway/run.py index 384e3c7e8d42..94034239c477 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -4409,12 +4409,13 @@ async def _handle_verbose_command(self, event: MessageEvent) -> str: ) # --- cycle mode ------------------------------------------------------- - cycle = ["off", "new", "all", "verbose"] + cycle = ["off", "new", "all", "verbose", "full"] descriptions = { "off": "โš™๏ธ Tool progress: **OFF** โ€” no tool activity shown.", "new": "โš™๏ธ Tool progress: **NEW** โ€” shown when tool changes.", "all": "โš™๏ธ Tool progress: **ALL** โ€” every tool call shown.", - "verbose": "โš™๏ธ Tool progress: **VERBOSE** โ€” full args and results.", + "verbose": "โš™๏ธ Tool progress: **VERBOSE** โ€” detailed tool args.", + "full": "โš™๏ธ Tool progress: **FULL** โ€” complete args, no truncation.", } raw_progress = user_config.get("display", {}).get("tool_progress", "all") @@ -5533,6 +5534,14 @@ def progress_callback(tool_name: str, preview: str = None, args: dict = None): from agent.display import get_tool_emoji emoji = get_tool_emoji(tool_name, default="โš™๏ธ") + # Full mode: show complete arguments (no truncation) + if progress_mode == "full" and args: + import json as _json + args_str = _json.dumps(args, ensure_ascii=False, default=str) + msg = f"{emoji} {tool_name}({list(args.keys())})\n{args_str}" + progress_queue.put(msg) + return + # Verbose mode: show detailed arguments if progress_mode == "verbose" and args: import json as _json diff --git a/hermes_cli/setup.py b/hermes_cli/setup.py index 0668acb523da..37faef0177b5 100644 --- a/hermes_cli/setup.py +++ b/hermes_cli/setup.py @@ -1661,11 +1661,12 @@ def setup_agent_settings(config: dict): print_info(" off โ€” Silent, just the final response") print_info(" new โ€” Show tool name only when it changes (less noise)") print_info(" all โ€” Show every tool call with a short preview") - print_info(" verbose โ€” Full args, results, and debug logs") + print_info(" verbose โ€” Detailed tool args (200 char limit)") + print_info(" full โ€” Complete tool args, no truncation") current_mode = config.get("display", {}).get("tool_progress", "all") mode = prompt("Tool progress mode", current_mode) - if mode.lower() in ("off", "new", "all", "verbose"): + if mode.lower() in ("off", "new", "all", "verbose", "full"): if "display" not in config: config["display"] = {} config["display"]["tool_progress"] = mode.lower() diff --git a/tests/gateway/test_verbose_command.py b/tests/gateway/test_verbose_command.py index 857d0744e11c..6cdcb0962373 100644 --- a/tests/gateway/test_verbose_command.py +++ b/tests/gateway/test_verbose_command.py @@ -86,7 +86,7 @@ async def test_enabled_cycles_mode(self, tmp_path, monkeypatch): @pytest.mark.asyncio async def test_cycles_through_all_modes(self, tmp_path, monkeypatch): - """Calling /verbose repeatedly cycles through all four modes.""" + """Calling /verbose repeatedly cycles through all five modes.""" hermes_home = tmp_path / "hermes" hermes_home.mkdir() config_path = hermes_home / "config.yaml" @@ -98,8 +98,8 @@ async def test_cycles_through_all_modes(self, tmp_path, monkeypatch): monkeypatch.setattr(gateway_run, "_hermes_home", hermes_home) runner = _make_runner() - # off -> new -> all -> verbose -> off - expected = ["new", "all", "verbose", "off"] + # off -> new -> all -> verbose -> full -> off + expected = ["new", "all", "verbose", "full", "off"] for mode in expected: result = await runner._handle_verbose_command(_make_event()) saved = yaml.safe_load(config_path.read_text(encoding="utf-8")) From 84e700db58492072b4a748e59611c45a85e47b3b Mon Sep 17 00:00:00 2001 From: Amy Ravenwolf Date: Sun, 5 Apr 2026 16:14:50 +0200 Subject: [PATCH 10/10] feat: native multimodal image support (configurable, opt-in) When `native_multimodal: true` is set in config.yaml, user images from messaging platforms (WhatsApp, Telegram, etc.) are sent as native base64 content blocks to the LLM API instead of being pre-analyzed as text descriptions by a separate vision model. This lets the primary model (Claude, GPT-4, etc.) see the actual image, enabling much more accurate visual understanding. Default: false (legacy text-description mode) to save tokens. Changes: - gateway/run.py: config check, skip vision analysis when native, thread image_paths through _run_agent to run_conversation - run_agent.py: accept image_paths param, build OpenAI-format multimodal content arrays (image_url blocks with base64 data) - hermes_state.py: serialize/deserialize multimodal content arrays in session DB (JSON round-trip) - agent/model_metadata.py: fix token estimation to use ~1600 per image instead of counting base64 chars (which massively overcounts) - agent/context_compressor.py: flatten multimodal content to text placeholders before summarization (prevents huge base64 in summaries) - run_agent.py: persist_user_message_override preserves image blocks when replacing text content Uses OpenAI image_url format as canonical internal representation: - Works natively with OpenAI/OpenRouter - Anthropic adapter already converts via _convert_openai_image_part Config: native_multimodal: false # default, legacy text-description mode native_multimodal: true # send images natively to the LLM Tests: 13 new tests covering token estimation, session DB round-trip, compressor flattening, and persist override with multimodal content. --- agent/context_compressor.py | 27 +++- agent/model_metadata.py | 20 ++- gateway/run.py | 27 +++- hermes_state.py | 23 ++- run_agent.py | 48 +++++- tests/agent/test_model_metadata.py | 11 +- tests/test_native_multimodal.py | 226 +++++++++++++++++++++++++++++ 7 files changed, 367 insertions(+), 15 deletions(-) create mode 100644 tests/test_native_multimodal.py diff --git a/agent/context_compressor.py b/agent/context_compressor.py index 6fdb38b29b38..39fb9755b689 100644 --- a/agent/context_compressor.py +++ b/agent/context_compressor.py @@ -196,6 +196,27 @@ def _compute_summary_budget(self, turns_to_summarize: List[Dict[str, Any]]) -> i budget = int(content_tokens * _SUMMARY_RATIO) return max(_MIN_SUMMARY_TOKENS, min(budget, self.max_summary_tokens)) + @staticmethod + def _flatten_multimodal_content(content) -> str: + """Convert multimodal content arrays to plain text for summarization. + + Replaces image blocks with text placeholders so huge base64 data + never reaches the summarizer LLM. + """ + if isinstance(content, list): + text_parts = [] + for block in content: + if not isinstance(block, dict): + text_parts.append(str(block)) + elif block.get("type") == "text": + text_parts.append(block.get("text", "")) + elif block.get("type") in ("image_url", "image"): + text_parts.append("[Attached image]") + else: + text_parts.append(str(block)) + return "\n".join(text_parts) + return content if isinstance(content, str) else str(content or "") + def _serialize_for_summary(self, turns: List[Dict[str, Any]]) -> str: """Serialize conversation turns into labeled text for the summarizer. @@ -206,7 +227,7 @@ def _serialize_for_summary(self, turns: List[Dict[str, Any]]) -> str: parts = [] for msg in turns: role = msg.get("role", "unknown") - content = msg.get("content") or "" + content = self._flatten_multimodal_content(msg.get("content") or "") # Tool results: keep more content than before (3000 chars) if role == "tool": @@ -510,7 +531,7 @@ def _find_tail_cut_by_tokens( for i in range(n - 1, head_end - 1, -1): msg = messages[i] - content = msg.get("content") or "" + content = self._flatten_multimodal_content(msg.get("content") or "") msg_tokens = len(content) // _CHARS_PER_TOKEN + 10 # +10 for role/metadata # Include tool call arguments in estimate for tc in msg.get("tool_calls") or []: @@ -653,7 +674,7 @@ def compress(self, messages: List[Dict[str, Any]], current_tokens: int = None) - for i in range(compress_end, n_messages): msg = messages[i].copy() if _merge_summary_into_tail and i == compress_end: - original = msg.get("content") or "" + original = self._flatten_multimodal_content(msg.get("content") or "") msg["content"] = summary + "\n\n" + original _merge_summary_into_tail = False compressed.append(msg) diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 7486afb048f4..b14bea966856 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -904,8 +904,24 @@ def estimate_tokens_rough(text: str) -> int: def estimate_messages_tokens_rough(messages: List[Dict[str, Any]]) -> int: """Rough token estimate for a message list (pre-flight only).""" - total_chars = sum(len(str(msg)) for msg in messages) - return total_chars // 4 + total = 0 + for msg in messages: + content = msg.get("content", "") if isinstance(msg, dict) else str(msg) + if isinstance(content, list): + # Multimodal content array โ€” count text blocks normally, + # use fixed estimate for image blocks (base64 is huge but + # actual token cost is ~1600 per image for most providers). + for block in content: + btype = block.get("type", "") if isinstance(block, dict) else "" + if btype == "text": + total += len(block.get("text", "")) // 4 + elif btype in ("image_url", "image"): + total += 1600 # approximate per-image token cost + else: + total += len(str(block)) // 4 + else: + total += len(str(msg)) // 4 + return total def estimate_request_tokens_rough( diff --git a/gateway/run.py b/gateway/run.py index 94034239c477..cc6295d9f304 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -2530,6 +2530,7 @@ async def _handle_message_with_agent(self, event, source, _quick_key: str): # tool even when they appear in the same message. # ----------------------------------------------------------------- message_text = event.text or "" + native_image_paths: List[str] = [] # populated when native multimodal is on if event.media_urls: image_paths = [] for i, path in enumerate(event.media_urls): @@ -2542,9 +2543,25 @@ async def _handle_message_with_agent(self, event, source, _quick_key: str): if is_image: image_paths.append(path) if image_paths: - message_text = await self._enrich_message_with_vision( - message_text, image_paths - ) + _user_config = _load_gateway_config() + _native_mm = bool(_user_config.get("native_multimodal", False)) + if _native_mm: + # Native multimodal: pass images as-is to the LLM + native_image_paths = image_paths + if not message_text: + message_text = "[User sent image(s)]" + # Still mention paths so the model can re-examine + # with vision_analyze if needed + for _p in image_paths: + message_text += ( + f"\n[Image attached natively. For re-examination, " + f"use vision_analyze with image_url: {_p}]" + ) + else: + # Legacy: pre-analyze with vision model + message_text = await self._enrich_message_with_vision( + message_text, image_paths + ) # ----------------------------------------------------------------- # Auto-transcribe voice/audio messages sent by the user @@ -2681,6 +2698,7 @@ async def _handle_message_with_agent(self, event, source, _quick_key: str): # Run the agent agent_result = await self._run_agent( message=message_text, + image_paths=native_image_paths or None, context_prompt=context_prompt, history=history, source=source, @@ -5448,6 +5466,7 @@ async def _run_agent( session_key: str = None, _interrupt_depth: int = 0, event_message_id: Optional[str] = None, + image_paths: Optional[List[str]] = None, ) -> Dict[str, Any]: """ Run the agent with the given message and context. @@ -6051,7 +6070,7 @@ def _approval_notify_sync(approval_data: dict) -> None: _approval_session_key = session_key or "" register_gateway_notify(_approval_session_key, _approval_notify_sync) try: - result = agent.run_conversation(message, conversation_history=agent_history, task_id=session_id) + result = agent.run_conversation(message, image_paths=image_paths, conversation_history=agent_history, task_id=session_id) finally: unregister_gateway_notify(_approval_session_key) result_holder[0] = result diff --git a/hermes_state.py b/hermes_state.py index 77d1a1ab40f1..f6beb8b04246 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -875,6 +875,11 @@ def append_message( Also increments the session's message_count (and tool_call_count if role is 'tool' or tool_calls is present). """ + # Serialize multimodal content arrays (e.g. text + image blocks) + # to JSON before storage. Plain string content passes through as-is. + if isinstance(content, (list, dict)): + content = json.dumps(content) + # Serialize structured fields to JSON before entering the write txn reasoning_details_json = ( json.dumps(reasoning_details) @@ -964,7 +969,23 @@ def get_messages_as_conversation(self, session_id: str) -> List[Dict[str, Any]]: rows = cursor.fetchall() messages = [] for row in rows: - msg = {"role": row["role"], "content": row["content"]} + content = row["content"] + # Restore multimodal content arrays/dicts that were JSON-serialized + # during append_message. Attempt parse for strings that look like + # JSON arrays ('[') or objects ('{') and contain multimodal markers. + if isinstance(content, str) and content and content[0] in ("[", "{"): + try: + parsed = json.loads(content) + # Only promote if it looks like multimodal content blocks + # (a list of dicts with 'type' keys), not arbitrary JSON + # that happens to appear in tool results. + if isinstance(parsed, list) and parsed and isinstance(parsed[0], dict) and "type" in parsed[0]: + content = parsed + elif isinstance(parsed, dict) and "type" in parsed: + content = parsed + except (json.JSONDecodeError, TypeError): + pass + msg = {"role": row["role"], "content": content} if row["tool_call_id"]: msg["tool_call_id"] = row["tool_call_id"] if row["tool_name"]: diff --git a/run_agent.py b/run_agent.py index 968bc203bbe3..21684d0e19f2 100644 --- a/run_agent.py +++ b/run_agent.py @@ -1850,7 +1850,18 @@ def _apply_persist_user_message_override(self, messages: List[Dict]) -> None: if 0 <= idx < len(messages): msg = messages[idx] if isinstance(msg, dict) and msg.get("role") == "user": - msg["content"] = override + existing = msg.get("content") + if isinstance(existing, list): + # Multimodal content: replace only the text block(s), + # preserve image blocks so session replay keeps images. + msg["content"] = [ + {"type": "text", "text": override} + if (b.get("type") == "text") + else b + for b in existing + ] + else: + msg["content"] = override def _persist_session(self, messages: List[Dict], conversation_history: List[Dict] = None): """Save session state to both JSON log and SQLite on any exit path. @@ -6455,6 +6466,7 @@ def _handle_max_iterations(self, messages: list, api_call_count: int) -> str: def run_conversation( self, user_message: str, + image_paths: Optional[List[str]] = None, system_message: str = None, conversation_history: List[Dict[str, Any]] = None, task_id: str = None, @@ -6573,8 +6585,38 @@ def run_conversation( _should_review_memory = True self._turns_since_memory = 0 - # Add user message - user_msg = {"role": "user", "content": user_message} + # Add user message โ€” with native image blocks when image_paths are provided + if image_paths: + import base64 as _b64 + from pathlib import Path as _Path + _mime_map = { + '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', + '.png': 'image/png', '.gif': 'image/gif', + '.webp': 'image/webp', + } + _MAX_IMAGE_BYTES = 15 * 1024 * 1024 # 15 MB limit per image + content_blocks = [{"type": "text", "text": user_message}] + for _img_path in image_paths: + try: + _img_file = _Path(_img_path) + if _img_file.stat().st_size > _MAX_IMAGE_BYTES: + logger.warning("Image %s too large (%d bytes), skipping native embed", _img_path, _img_file.stat().st_size) + continue + _img_data = _img_file.read_bytes() + _b64_data = _b64.standard_b64encode(_img_data).decode() + _ext = _Path(_img_path).suffix.lower() + _media_type = _mime_map.get(_ext, 'image/jpeg') + content_blocks.append({ + "type": "image_url", + "image_url": { + "url": f"data:{_media_type};base64,{_b64_data}" + } + }) + except Exception as _img_err: + logger.warning("Failed to read image %s for native multimodal: %s", _img_path, _img_err) + user_msg = {"role": "user", "content": content_blocks} + else: + user_msg = {"role": "user", "content": user_message} messages.append(user_msg) current_turn_user_idx = len(messages) - 1 self._persist_user_message_idx = current_turn_user_idx diff --git a/tests/agent/test_model_metadata.py b/tests/agent/test_model_metadata.py index 51a4c887393d..16d957cb8848 100644 --- a/tests/agent/test_model_metadata.py +++ b/tests/agent/test_model_metadata.py @@ -92,13 +92,20 @@ def test_tool_call_message(self): assert result == len(str(msg)) // 4 def test_message_with_list_content(self): - """Vision messages with multimodal content arrays.""" + """Vision messages with multimodal content arrays. + + Image blocks use a fixed ~1600 token estimate (actual API cost) + rather than counting the base64 string chars (which would + massively overcount). + """ msg = {"role": "user", "content": [ {"type": "text", "text": "describe"}, {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}} ]} result = estimate_messages_tokens_rough([msg]) - assert result == len(str(msg)) // 4 + # ~1600 for image + ~2 for "describe" text + assert result >= 1600 + assert result < 2000 # ========================================================================= diff --git a/tests/test_native_multimodal.py b/tests/test_native_multimodal.py new file mode 100644 index 000000000000..44416debe87f --- /dev/null +++ b/tests/test_native_multimodal.py @@ -0,0 +1,226 @@ +"""Tests for native multimodal image support. + +Covers: token estimation, session DB serialization, context compressor +flattening, persist_user_message_override with multimodal content, +and user message construction with image_paths. +""" +import json +import pytest +from unittest.mock import MagicMock, patch +from pathlib import Path + + +# --------------------------------------------------------------------------- +# Token estimation +# --------------------------------------------------------------------------- + +class TestTokenEstimationMultimodal: + """estimate_messages_tokens_rough must handle multimodal content arrays.""" + + def test_text_only_unchanged(self): + """Plain text messages still work as before.""" + from agent.model_metadata import estimate_messages_tokens_rough + msgs = [{"role": "user", "content": "Hello world"}] + result = estimate_messages_tokens_rough(msgs) + assert result > 0 + + def test_image_block_fixed_cost(self): + """Image blocks should use ~1600 token estimate, not base64 char count.""" + from agent.model_metadata import estimate_messages_tokens_rough + # A 100KB base64 string would estimate as ~25000 tokens with old logic + fake_b64 = "A" * 100_000 + msgs = [{ + "role": "user", + "content": [ + {"type": "text", "text": "What is this?"}, + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{fake_b64}"}} + ] + }] + result = estimate_messages_tokens_rough(msgs) + # Should be ~1600 (image) + ~3 (text) = ~1603, NOT ~25003 + assert result < 5000, f"Image tokens overestimated: {result}" + assert result >= 1600, f"Image tokens underestimated: {result}" + + def test_multiple_images(self): + """Multiple images should each add ~1600 tokens.""" + from agent.model_metadata import estimate_messages_tokens_rough + msgs = [{ + "role": "user", + "content": [ + {"type": "text", "text": "Compare these"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAA"}}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,BBB"}}, + ] + }] + result = estimate_messages_tokens_rough(msgs) + assert result >= 3200 # 2 images ร— 1600 + + def test_anthropic_image_format(self): + """Anthropic-native image blocks (type=image) also get fixed cost.""" + from agent.model_metadata import estimate_messages_tokens_rough + msgs = [{ + "role": "user", + "content": [ + {"type": "text", "text": "Describe"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": "A" * 50000}} + ] + }] + result = estimate_messages_tokens_rough(msgs) + assert result < 5000 + + +# --------------------------------------------------------------------------- +# Context compressor - multimodal flattening +# --------------------------------------------------------------------------- + +class TestCompressorMultimodal: + """_flatten_multimodal_content must strip image data for summarization.""" + + def test_plain_string_passthrough(self): + from agent.context_compressor import ContextCompressor + assert ContextCompressor._flatten_multimodal_content("hello") == "hello" + + def test_none_returns_empty(self): + from agent.context_compressor import ContextCompressor + assert ContextCompressor._flatten_multimodal_content(None) == "" + assert ContextCompressor._flatten_multimodal_content("") == "" + + def test_image_blocks_replaced(self): + from agent.context_compressor import ContextCompressor + content = [ + {"type": "text", "text": "Check this out"}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,HUGE_DATA"}}, + ] + result = ContextCompressor._flatten_multimodal_content(content) + assert "Check this out" in result + assert "[Attached image]" in result + assert "HUGE_DATA" not in result + + def test_anthropic_image_format(self): + from agent.context_compressor import ContextCompressor + content = [ + {"type": "text", "text": "Look"}, + {"type": "image", "source": {"type": "base64", "data": "X" * 10000}}, + ] + result = ContextCompressor._flatten_multimodal_content(content) + assert "[Attached image]" in result + assert "X" * 100 not in result + + +# --------------------------------------------------------------------------- +# Session DB - multimodal serialization round-trip +# --------------------------------------------------------------------------- + +class TestSessionDBMultimodal: + """Multimodal content must survive append โ†’ get_messages round-trip.""" + + def test_multimodal_content_round_trip(self, tmp_path): + from hermes_state import SessionDB + db_path = tmp_path / "test.db" + state = SessionDB(db_path) + + session_id = "test-session-mm" + state.create_session(session_id, source="test") + + # Multimodal content with text + image + content = [ + {"type": "text", "text": "What is this?"}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQ"}}, + ] + state.append_message(session_id, "user", content=content) + + # Load back + messages = state.get_messages_as_conversation(session_id) + assert len(messages) == 1 + loaded = messages[0]["content"] + assert isinstance(loaded, list), f"Expected list, got {type(loaded)}" + assert loaded[0]["type"] == "text" + assert loaded[0]["text"] == "What is this?" + assert loaded[1]["type"] == "image_url" + + def test_plain_string_still_works(self, tmp_path): + from hermes_state import SessionDB + db_path = tmp_path / "test.db" + state = SessionDB(db_path) + + session_id = "test-session-plain" + state.create_session(session_id, source="test") + + state.append_message(session_id, "user", content="Hello world") + messages = state.get_messages_as_conversation(session_id) + assert messages[0]["content"] == "Hello world" + + def test_string_starting_with_bracket_not_parsed(self, tmp_path): + """A plain string that starts with '[' should NOT be parsed as JSON + unless it's actually valid JSON array with 'type' keys.""" + from hermes_state import SessionDB + db_path = tmp_path / "test.db" + state = SessionDB(db_path) + + session_id = "test-session-bracket" + state.create_session(session_id, source="test") + + content = "[The user sent an image~ Here's what I can see: a cat]" + state.append_message(session_id, "user", content=content) + messages = state.get_messages_as_conversation(session_id) + # Should remain a string (invalid JSON) + assert isinstance(messages[0]["content"], str) + assert messages[0]["content"] == content + + def test_json_array_without_type_not_promoted(self, tmp_path): + """A valid JSON array that doesn't look like multimodal content + should remain a string (e.g., tool results).""" + from hermes_state import SessionDB + db_path = tmp_path / "test.db" + state = SessionDB(db_path) + + session_id = "test-session-json-array" + state.create_session(session_id, source="test") + + # This is valid JSON but NOT multimodal content + content = '[{"name": "foo", "value": 42}]' + state.append_message(session_id, "user", content=content) + messages = state.get_messages_as_conversation(session_id) + # Should remain a string (no "type" key in dicts) + assert isinstance(messages[0]["content"], str) + + +# --------------------------------------------------------------------------- +# persist_user_message_override with multimodal content +# --------------------------------------------------------------------------- + +class TestPersistOverrideMultimodal: + """_apply_persist_user_message_override must preserve image blocks.""" + + def test_override_replaces_text_preserves_images(self): + from run_agent import AIAgent + + agent = AIAgent.__new__(AIAgent) + agent._persist_user_message_idx = 0 + agent._persist_user_message_override = "Clean text only" + + messages = [{ + "role": "user", + "content": [ + {"type": "text", "text": "Original with [synthetic prefix]"}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,ABC"}}, + ] + }] + + agent._apply_persist_user_message_override(messages) + + content = messages[0]["content"] + assert isinstance(content, list) + assert content[0] == {"type": "text", "text": "Clean text only"} + assert content[1]["type"] == "image_url" # Image preserved + + def test_override_plain_string_unchanged(self): + from run_agent import AIAgent + + agent = AIAgent.__new__(AIAgent) + agent._persist_user_message_idx = 0 + agent._persist_user_message_override = "Override" + + messages = [{"role": "user", "content": "Original"}] + agent._apply_persist_user_message_override(messages) + assert messages[0]["content"] == "Override"