feat(computer-use): draw self-localization action marker on follow-up captures

MANIFEST.in

@@ -7,5 +7,7 @@ graft locales
 # built from the sdist (e.g. Homebrew, downstream packagers). package-data
 # below covers the wheel; this covers the sdist. See #34034 / #28149.
 recursive-include plugins plugin.yaml plugin.yml
+# Gateway assets include images plus YAML catalogs such as status_phrases.yaml.
+recursive-include gateway/assets *
 global-exclude __pycache__
 global-exclude *.py[cod]

acp_adapter/tools.py

@@ -617,7 +617,7 @@ def _format_session_search_result(result: Optional[str]) -> Optional[str]:
         return None
     mode = data.get("mode") or "search"
     query = data.get("query")
-    lines = ["Recent sessions" if mode == "recent" else f"Session search results" + (f" for `{query}`" if query else "")]
+    lines = ["Recent sessions" if mode == "recent" else "Session search results" + (f" for `{query}`" if query else "")]
     if not results:
         lines.append(str(data.get("message") or "No matching sessions found."))
         return "\n".join(lines)

agent/agent_init.py

@@ -1822,6 +1822,8 @@ def _moa_reference_relay(event: str, **kwargs: Any) -> None:
         working_dir=os.getenv("TERMINAL_CWD") or None,
     )
     agent._user_turn_count = 0
+    # Copilot x-initiator flag: first API call of a user turn sends "user" (#3040).
+    agent._is_user_initiated_turn = False
 
     # Cumulative token usage for the session
     agent.session_prompt_tokens = 0

agent/auxiliary_client.py

@@ -2003,6 +2003,76 @@ def _read_main_provider() -> str:
     return ""
 
 
+def _read_main_api_key() -> str:
+    """Read the user's main model API key from the runtime override or config.
+
+    Mirrors ``_read_main_model`` / ``_read_main_provider``: checks the
+    process-local ``_RUNTIME_MAIN_API_KEY`` override first (set by
+    ``set_runtime_main`` when an AIAgent is active), then falls back to
+    ``model.api_key`` in config.yaml.
+
+    Used by the ``custom`` provider fallback chain so that auxiliary tasks
+    configured with an explicit ``base_url`` but empty ``api_key`` inherit
+    the main model's credentials instead of falling to ``no-key-required``
+    (issue #9318).
+    """
+    override = _RUNTIME_MAIN_API_KEY
+    if isinstance(override, str) and override.strip():
+        return override.strip()
+    try:
+        from hermes_cli.config import load_config
+        cfg = load_config()
+        model_cfg = cfg.get("model", {})
+        if isinstance(model_cfg, dict):
+            key = model_cfg.get("api_key", "")
+            if isinstance(key, str) and key.strip():
+                return key.strip()
+    except Exception:
+        pass
+    return ""
+
+
+def _read_main_base_url() -> str:
+    """Read the main model's base_url from the runtime override or config.
+
+    Same override-then-config pattern as ``_read_main_api_key``.
+    """
+    override = _RUNTIME_MAIN_BASE_URL
+    if isinstance(override, str) and override.strip():
+        return override.strip()
+    try:
+        from hermes_cli.config import load_config
+        cfg = load_config()
+        model_cfg = cfg.get("model", {})
+        if isinstance(model_cfg, dict):
+            base = model_cfg.get("base_url", "")
+            if isinstance(base, str) and base.strip():
+                return base.strip()
+    except Exception:
+        pass
+    return ""
+
+
+def _read_main_api_key_if_same_host(aux_base_url: str) -> str:
+    """Return the main api_key only when *aux_base_url* points at the same
+    host as the main model's base_url.
+
+    The #9318 use case is an auxiliary task sharing the main model's
+    self-hosted gateway (same host, different model) with an empty per-task
+    api_key. Inheriting unconditionally would send the main credential to
+    ANY host a misconfigured aux base_url names — a cross-host credential
+    leak. A host mismatch keeps the previous fail-safe behavior
+    (``no-key-required`` → 401).
+    """
+    aux_host = base_url_hostname(aux_base_url)
+    if not aux_host:
+        return ""
+    main_host = base_url_hostname(_read_main_base_url())
+    if not main_host or aux_host != main_host:
+        return ""
+    return _read_main_api_key()
+
+
 # Process-local override set by AIAgent at session/turn start. Single-threaded
 # per turn — no lock needed. Cleared by ``clear_runtime_main()``.
 _RUNTIME_MAIN_PROVIDER: str = ""
@@ -2392,11 +2462,19 @@ def _try_anthropic(explicit_api_key: str = None) -> Tuple[Optional[Any], Optiona
         return None, None
 
     pool_present, entry = _select_pool_entry("anthropic")
-    if pool_present:
-        if entry is None:
-            return None, None
+    if pool_present and entry is not None:
         token = explicit_api_key or _pool_runtime_api_key(entry)
     else:
+        # Pool absent, OR pool present but no usable entry (expired token +
+        # stale refresh_token, all entries exhausted, etc). Fall through to the
+        # legacy resolver instead of hard-failing: a temporarily dead pool
+        # entry must not wedge auxiliary tasks when a valid standalone
+        # credential (ANTHROPIC_TOKEN, credentials file, API key) exists. This
+        # matches the openrouter and codex paths, which already fall back to
+        # their env/auth-store credential on (True, None). Without this, the
+        # goal judge and every other Anthropic-routed side channel died with
+        # "no auxiliary client configured" while the main session stayed
+        # healthy (it resolves the env token directly).
         entry = None
         token = explicit_api_key or resolve_anthropic_token()
     if not token:
@@ -4234,6 +4312,7 @@ def _wrap_if_needed(client_obj, final_model_str: str, base_url_str: str = "",
             custom_key = (
                 (explicit_api_key or "").strip()
                 or os.getenv("OPENAI_API_KEY", "").strip()
+                or _read_main_api_key_if_same_host(custom_base)
                 or "no-key-required"  # local servers don't need auth
             )
             if not custom_base:
@@ -5524,6 +5603,18 @@ def _preserve_provider_with_base_url(prov: Optional[str]) -> bool:
     if cfg_provider:
         cfg_provider, cfg_base_url = _expand_direct_api_alias(cfg_provider, cfg_base_url)
 
+    # An explicit provider arg without an explicit base_url must not bypass
+    # the task's configured endpoint: adopt auxiliary.<task>.base_url/api_key
+    # when the config targets the same provider (or names none), so the
+    # early `if provider:` return below carries the configured endpoint
+    # instead of falling through to main-runtime resolution (#58515).
+    # An explicit "auto" is excluded — it means "inherit / auto-detect" and
+    # must keep flowing through the existing auto-resolution chain.
+    if provider and provider != "auto" and not base_url and cfg_base_url and cfg_provider in (None, provider):
+        base_url = cfg_base_url
+        if not api_key:
+            api_key = cfg_api_key
+
     if base_url and _preserve_provider_with_base_url(provider):
         return provider, resolved_model, base_url, api_key, resolved_api_mode
     if base_url:

agent/bounded_response.py

@@ -0,0 +1,148 @@
+"""Bounded reads of HTTP error response bodies.
+
+When a provider returns a non-OK status on a *streaming* request, Hermes reads
+the response body to build a useful diagnostic error. A bare ``response.read()``
+on a streaming httpx response is unbounded in two dangerous ways:
+
+1. A server can declare (or stream) an arbitrarily large body, so the read can
+   balloon memory.
+2. A server can open the body and then stall forever (no ``Content-Length``,
+   no further bytes), so the read hangs the agent indefinitely.
+
+Both are realistic against a misbehaving proxy, a hijacked endpoint, or a
+provider having a bad day. The diagnostic body is only ever shown to the user
+truncated to a few hundred characters, so reading megabytes — or blocking
+forever — buys nothing.
+
+``read_streaming_error_body`` bounds the read to a byte cap and enforces a
+hard wall-clock deadline, returning the decoded text snippet. Callers pass the
+returned text into their existing error builders instead of touching
+``response.text`` (which would be unbounded / would raise after a partial
+stream read).
+
+A subtlety the implementation must respect: ``httpx``'s ``iter_bytes()`` blocks
+*inside* the C/socket read while waiting for the next chunk. A wall-clock check
+placed only between yielded chunks cannot interrupt a server that opens the
+body and then stalls mid-chunk — control never returns to Python until httpx's
+own (often 30s+) read timeout fires. To guarantee a bounded stop regardless of
+socket behavior, the read runs on a daemon worker thread and the caller waits
+on it with a hard deadline; on timeout we close the response (which unblocks /
+cancels the read) and return whatever partial bytes were collected.
+
+Ported and adapted from openclaw/openclaw#95108 ("bound Anthropic error
+streams"), generalized to cover Hermes's three streaming error-body sites
+(native Gemini, Gemini Cloud Code, Antigravity Cloud Code).
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+from typing import List, Optional
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+# Defaults chosen to comfortably hold any real provider error envelope (Google
+# RPC error JSON, Anthropic error JSON) while rejecting pathological bodies.
+DEFAULT_ERROR_BODY_MAX_BYTES = 64 * 1024
+# Hard wall-clock deadline for the whole bounded read. A streaming error body
+# that does not finish within this window is abandoned and the connection is
+# closed; we keep whatever partial bytes arrived.
+DEFAULT_ERROR_BODY_TIMEOUT_S = 10.0
+
+
+def read_streaming_error_body(
+    response: httpx.Response,
+    *,
+    max_bytes: int = DEFAULT_ERROR_BODY_MAX_BYTES,
+    timeout_s: float = DEFAULT_ERROR_BODY_TIMEOUT_S,
+) -> str:
+    """Read a non-OK streaming response body with a byte cap and a hard deadline.
+
+    Returns the decoded body text (UTF-8, errors replaced), truncated to
+    ``max_bytes``. Never raises: any transport error, stall, or oversize
+    condition is swallowed and the best-effort partial text (or an empty
+    string) is returned, because this runs on the error path and must not
+    mask the original HTTP failure with a read error.
+
+    The byte cap protects against huge bodies; the wall-clock deadline (enforced
+    via a worker thread so it can interrupt a socket read that stalls mid-chunk)
+    protects against bodies that open and then hang.
+    """
+    chunks: List[bytes] = []
+    state = {"truncated": False}
+    done = threading.Event()
+
+    def _drain() -> None:
+        total = 0
+        try:
+            for chunk in response.iter_bytes():
+                if not chunk:
+                    continue
+                remaining = max_bytes - total
+                if remaining <= 0:
+                    state["truncated"] = True
+                    break
+                if len(chunk) > remaining:
+                    chunks.append(chunk[:remaining])
+                    total += remaining
+                    state["truncated"] = True
+                    break
+                chunks.append(chunk)
+                total += len(chunk)
+        except Exception as exc:  # noqa: BLE001 - error path must not raise
+            logger.debug("bounded error-body read failed: %s", exc)
+        finally:
+            done.set()
+
+    worker = threading.Thread(
+        target=_drain, name="bounded-error-body-read", daemon=True
+    )
+    worker.start()
+    finished = done.wait(timeout=timeout_s)
+
+    if not finished:
+        logger.debug(
+            "bounded error-body read: hard timeout after %.1fs (%d bytes so far)",
+            timeout_s,
+            sum(len(c) for c in chunks),
+        )
+        # Closing the response cancels the in-flight socket read, letting the
+        # worker thread unwind. We do not join (it is a daemon and may be
+        # blocked in C); the partial `chunks` collected so far are returned.
+        _safe_close(response)
+    else:
+        _safe_close(response)
+
+    if state["truncated"]:
+        logger.debug(
+            "bounded error-body read: capped at %d bytes (max=%d)",
+            sum(len(c) for c in chunks),
+            max_bytes,
+        )
+    return b"".join(chunks).decode("utf-8", errors="replace")
+
+
+def _safe_close(response: httpx.Response) -> None:
+    try:
+        response.close()
+    except Exception:  # noqa: BLE001
+        pass
+
+
+def read_error_body_or_default(
+    response: httpx.Response,
+    *,
+    max_bytes: int = DEFAULT_ERROR_BODY_MAX_BYTES,
+    timeout_s: float = DEFAULT_ERROR_BODY_TIMEOUT_S,
+) -> Optional[str]:
+    """Like ``read_streaming_error_body`` but returns ``None`` on empty body.
+
+    Convenience for callers that distinguish "no body" from "empty string".
+    """
+    text = read_streaming_error_body(
+        response, max_bytes=max_bytes, timeout_s=timeout_s
+    )
+    return text or None

agent/chat_completion_helpers.py

@@ -2194,15 +2194,23 @@ def _call_chat_completions():
                     idx = _active_slot_by_idx[raw_idx]
 
                     if idx not in tool_calls_acc:
+                        # Poolside may send integer id instead of string
+                        _tc_id = tc_delta.id
+                        if isinstance(_tc_id, int):
+                            _tc_id = str(_tc_id)
                         tool_calls_acc[idx] = {
-                            "id": tc_delta.id or "",
+                            "id": _tc_id or "",
                             "type": "function",
                             "function": {"name": "", "arguments": ""},
                             "extra_content": None,
                         }
                     entry = tool_calls_acc[idx]
-                    if tc_delta.id:
-                        entry["id"] = tc_delta.id
+                    if tc_delta.id is not None:
+                        _new_id = tc_delta.id
+                        if isinstance(_new_id, int):
+                            _new_id = str(_new_id)
+                        if _new_id:
+                            entry["id"] = _new_id
                     if tc_delta.function:
                         if tc_delta.function.name:
                             # Use assignment, not +=.  Function names are

agent/codex_responses_adapter.py

@@ -1166,15 +1166,28 @@ def _normalize_codex_response(
         if item_type == "message":
             item_phase = getattr(item, "phase", None)
             normalized_phase = None
+            is_commentary_phase = False
             if isinstance(item_phase, str):
                 normalized_phase = item_phase.strip().lower()
                 if normalized_phase in {"commentary", "analysis"}:
                     saw_commentary_phase = True
+                    is_commentary_phase = True
                 elif normalized_phase in {"final_answer", "final"}:
                     saw_final_answer_phase = True
             message_text = _extract_responses_message_text(item)
             if message_text:
-                content_parts.append(message_text)
+                # Responses ``commentary``/``analysis`` phase text is mid-turn
+                # preamble/progress narration, never the turn's final answer
+                # (Codex CLI excludes it from last-message extraction; issues
+                # #24933 / #41293).  Keep it out of assistant content so it
+                # can't be concatenated into — or leak as — the final response,
+                # but surface it through the reasoning channel so the CLI/
+                # gateway display it like thinking text.  The exact message
+                # item is still preserved below for replay/cache continuity.
+                if is_commentary_phase:
+                    reasoning_parts.append(message_text)
+                else:
+                    content_parts.append(message_text)
                 raw_message_item: Dict[str, Any] = {
                     "type": "message",
                     "role": "assistant",
@@ -1269,7 +1282,11 @@ def _normalize_codex_response(
             ))
 
     final_text = "\n".join([p for p in content_parts if p]).strip()
-    if not final_text and hasattr(response, "output_text"):
+    if (
+        not final_text
+        and hasattr(response, "output_text")
+        and not (saw_commentary_phase and not saw_final_answer_phase)
+    ):
         out_text = getattr(response, "output_text", "")
         if isinstance(out_text, str):
             final_text = out_text.strip()