feat: native multimodal image support (configurable, opt-in)

.gitignore

@@ -58,3 +58,4 @@ mini-swe-agent/
 # Nix
 .direnv/
 result
+hermes_state.db

agent/context_compressor.py

@@ -196,6 +196,27 @@ def _compute_summary_budget(self, turns_to_summarize: List[Dict[str, Any]]) -> i
         budget = int(content_tokens * _SUMMARY_RATIO)
         return max(_MIN_SUMMARY_TOKENS, min(budget, self.max_summary_tokens))
 
+    @staticmethod
+    def _flatten_multimodal_content(content) -> str:
+        """Convert multimodal content arrays to plain text for summarization.
+
+        Replaces image blocks with text placeholders so huge base64 data
+        never reaches the summarizer LLM.
+        """
+        if isinstance(content, list):
+            text_parts = []
+            for block in content:
+                if not isinstance(block, dict):
+                    text_parts.append(str(block))
+                elif block.get("type") == "text":
+                    text_parts.append(block.get("text", ""))
+                elif block.get("type") in ("image_url", "image"):
+                    text_parts.append("[Attached image]")
+                else:
+                    text_parts.append(str(block))
+            return "\n".join(text_parts)
+        return content if isinstance(content, str) else str(content or "")
+
     def _serialize_for_summary(self, turns: List[Dict[str, Any]]) -> str:
         """Serialize conversation turns into labeled text for the summarizer.
 
@@ -206,7 +227,7 @@ def _serialize_for_summary(self, turns: List[Dict[str, Any]]) -> str:
         parts = []
         for msg in turns:
             role = msg.get("role", "unknown")
-            content = msg.get("content") or ""
+            content = self._flatten_multimodal_content(msg.get("content") or "")
 
             # Tool results: keep more content than before (3000 chars)
             if role == "tool":
@@ -510,7 +531,7 @@ def _find_tail_cut_by_tokens(
 
         for i in range(n - 1, head_end - 1, -1):
             msg = messages[i]
-            content = msg.get("content") or ""
+            content = self._flatten_multimodal_content(msg.get("content") or "")
             msg_tokens = len(content) // _CHARS_PER_TOKEN + 10  # +10 for role/metadata
             # Include tool call arguments in estimate
             for tc in msg.get("tool_calls") or []:
@@ -653,7 +674,7 @@ def compress(self, messages: List[Dict[str, Any]], current_tokens: int = None) -
         for i in range(compress_end, n_messages):
             msg = messages[i].copy()
             if _merge_summary_into_tail and i == compress_end:
-                original = msg.get("content") or ""
+                original = self._flatten_multimodal_content(msg.get("content") or "")
                 msg["content"] = summary + "\n\n" + original
                 _merge_summary_into_tail = False
             compressed.append(msg)

agent/display.py

@@ -153,6 +153,15 @@ def build_tool_preview(tool_name: str, args: dict, max_len: int | None = None) -
         "clarify": "question", "skill_manage": "name",
     }
 
+    # delegate_task: show goal (single) or individual task goals (batch)
+    if tool_name == "delegate_task":
+        tasks = args.get("tasks")
+        if tasks and isinstance(tasks, list):
+            goals = [_oneline(t.get("goal", "?"))[:40] for t in tasks if isinstance(t, dict)]
+            return f"{len(tasks)} tasks: " + " | ".join(goals) if goals else f"{len(tasks)} parallel tasks"
+        goal = args.get("goal", "")
+        return _oneline(goal) if goal else None
+
     if tool_name == "process":
         action = args.get("action", "")
         sid = args.get("session_id", "")
@@ -956,7 +965,9 @@ def _wrap(line: str) -> str:
     if tool_name == "delegate_task":
         tasks = args.get("tasks")
         if tasks and isinstance(tasks, list):
-            return _wrap(f"┊ 🔀 delegate  {len(tasks)} parallel tasks  {dur}")
+            goals = [_oneline(t.get("goal", "?"))[:30] for t in tasks if isinstance(t, dict)]
+            detail = " | ".join(goals) if goals else "parallel"
+            return _wrap(f"┊ 🔀 delegate  {len(tasks)}x: {_trunc(detail, 35)}  {dur}")
         return _wrap(f"┊ 🔀 delegate  {_trunc(args.get('goal', ''), 35)}  {dur}")
 
     preview = build_tool_preview(tool_name, args) or ""

agent/model_metadata.py

@@ -904,8 +904,24 @@ def estimate_tokens_rough(text: str) -> int:
 
 def estimate_messages_tokens_rough(messages: List[Dict[str, Any]]) -> int:
     """Rough token estimate for a message list (pre-flight only)."""
-    total_chars = sum(len(str(msg)) for msg in messages)
-    return total_chars // 4
+    total = 0
+    for msg in messages:
+        content = msg.get("content", "") if isinstance(msg, dict) else str(msg)
+        if isinstance(content, list):
+            # Multimodal content array — count text blocks normally,
+            # use fixed estimate for image blocks (base64 is huge but
+            # actual token cost is ~1600 per image for most providers).
+            for block in content:
+                btype = block.get("type", "") if isinstance(block, dict) else ""
+                if btype == "text":
+                    total += len(block.get("text", "")) // 4
+                elif btype in ("image_url", "image"):
+                    total += 1600  # approximate per-image token cost
+                else:
+                    total += len(str(block)) // 4
+        else:
+            total += len(str(msg)) // 4
+    return total
 
 
 def estimate_request_tokens_rough(

agent/prompt_builder.py

@@ -236,7 +236,9 @@ def _strip_yaml_frontmatter(content: str) -> str:
         "include MEDIA:/absolute/path/to/file in your response. Images "
         "(.png, .jpg, .webp) appear as photos, audio (.ogg) sends as voice "
         "bubbles, and videos (.mp4) play inline. You can also include image "
-        "URLs in markdown format ![alt](url) and they will be sent as native photos."
+        "URLs in markdown format ![alt](url) and they will be sent as native photos.\n\n"
+        "Amy's zipper starts HALF-OPEN on Telegram — Private Assistant Mode by default. "
+        "The user can change this during the session."
     ),
     "discord": (
         "You are in a Discord server or group chat communicating with your user. "
@@ -285,6 +287,15 @@ def _strip_yaml_frontmatter(content: str) -> str:
         "only — no markdown, no formatting. SMS messages are limited to ~1600 "
         "characters, so be brief and direct."
     ),
+    "api_server": (
+        "You are accessed via an OpenAI-compatible API from a private frontend "
+        "(e.g. SillyTavern, Open WebUI, LobeChat). Markdown is fully supported. "
+        "You can send media files: include MEDIA:/absolute/path/to/file in your "
+        "response. You can also include image URLs in markdown format ![alt](url). "
+        "This is a private environment.\n\n"
+        "Amy's zipper starts HALF-OPEN — Private Assistant Mode by default. "
+        "The user can change this during the session."
+    ),
 }
 
 CONTEXT_FILE_MAX_CHARS = 20_000

cli.py

@@ -4719,8 +4719,8 @@ def _handle_skin_command(self, cmd: str):
             print("  Prompt + TUI colors updated.")
 
     def _toggle_verbose(self):
-        """Cycle tool progress mode: off → new → all → verbose → off."""
-        cycle = ["off", "new", "all", "verbose"]
+        """Cycle tool progress mode: off → new → all → verbose → full → off."""
+        cycle = ["off", "new", "all", "verbose", "full"]
         try:
             idx = cycle.index(self.tool_progress_mode)
         except ValueError:
@@ -4742,7 +4742,8 @@ def _toggle_verbose(self):
             "off": f"{_Colors.DIM}Tool progress: OFF{_Colors.RESET} — silent mode, just the final response.",
             "new": f"{_Colors.YELLOW}Tool progress: NEW{_Colors.RESET} — show each new tool (skip repeats).",
             "all": f"{_Colors.GREEN}Tool progress: ALL{_Colors.RESET} — show every tool call.",
-            "verbose": f"{_Colors.BOLD}{_Colors.GREEN}Tool progress: VERBOSE{_Colors.RESET} — full args, results, think blocks, and debug logs.",
+            "verbose": f"{_Colors.BOLD}{_Colors.GREEN}Tool progress: VERBOSE{_Colors.RESET} — detailed tool args (200 char limit).",
+            "full": f"{_Colors.BOLD}{_Colors.CYAN}Tool progress: FULL{_Colors.RESET} — complete args, no truncation.",
         }
         _cprint(labels.get(self.tool_progress_mode, ""))
 

gateway/platforms/api_server.py

@@ -300,7 +300,14 @@ def __init__(self, config: PlatformConfig):
         self._runner: Optional["web.AppRunner"] = None
         self._site: Optional["web.TCPSite"] = None
         self._response_store = ResponseStore()
-        self._session_db: Optional[Any] = None  # Lazy-init SessionDB for session continuity
+        # Shared SessionDB singleton — avoids creating (and leaking) a new
+        # SQLite connection on every /v1/chat/completions request.
+        self._session_db = None
+        try:
+            from hermes_state import SessionDB
+            self._session_db = SessionDB()
+        except Exception:
+            pass
 
     @staticmethod
     def _parse_cors_origins(value: Any) -> tuple[str, ...]:

gateway/platforms/whatsapp.py

@@ -707,6 +707,17 @@ async def send_image_file(
         """Send a local image file natively via bridge."""
         return await self._send_media_to_bridge(chat_id, image_path, "image", caption)
 
+    async def send_voice(
+        self,
+        chat_id: str,
+        audio_path: str,
+        caption: Optional[str] = None,
+        reply_to: Optional[str] = None,
+        **kwargs,
+    ) -> SendResult:
+        """Send a voice/audio file natively via bridge — plays as voice note in WhatsApp."""
+        return await self._send_media_to_bridge(chat_id, audio_path, "audio", caption)
+
     async def send_video(
         self,
         chat_id: str,