feat(computer_use): follow cua-driver's verify → escalate ladder (#67052)

agent/prompt_builder.py

@@ -557,6 +557,29 @@ def computer_use_guidance(platform_name: Optional[str] = None) -> str:
         "4. After any state-changing action, re-capture to verify. You can "
         "pass `capture_after=true` to get the follow-up screenshot in one "
         "round-trip.\n\n"
+        "## Verify → escalate ladder (background-first, NOT background-only)\n"
+        "Background delivery is the DEFAULT and the co-work path, but it is "
+        "the first rung, not the only one. Read each action's structured "
+        "result and climb only when the driver tells you to:\n"
+        "- `effect: 'confirmed'` + `verified: true` — the driver read the "
+        "result back. Done.\n"
+        "- `effect: 'unverifiable'` — the input was delivered but the driver "
+        "can't confirm it. Re-capture and check the screenshot/tree yourself "
+        "before deciding it worked.\n"
+        "- `effect: 'suspected_noop'`, `code: 'background_unavailable'`, or an "
+        "`escalation.recommended` field — the action did NOT land. Follow "
+        "`escalation.recommended`:\n"
+        "  - `'px'` → re-issue addressing the target by `coordinate=[x,y]` "
+        "read off the screenshot instead of `element`.\n"
+        "  - `'foreground'` (or a pixel click still didn't land) → re-issue "
+        "the SAME action with `delivery_mode='foreground'`. This briefly "
+        "raises the window; it needs its own approval and is only appropriate "
+        "when the user isn't actively working. Common for Electron/Chromium "
+        "consent dialogs, DirectInput games, and raw-input canvases.\n"
+        "- Escalate to foreground as a REACTION to a returned signal, never "
+        "as a prediction from the app being Electron/Chromium/GTK. Do not "
+        "silently retry the same rung expecting a different result, and do "
+        "not conclude 'cua-driver can't drive this app' — climb the ladder.\n\n"
         "## Background mode rules\n"
         "- Do NOT use `raise_window=true` on `focus_app` unless the user "
         "explicitly asked you to bring a window to front. Input routing to "

skills/computer-use/SKILL.md

@@ -101,6 +101,56 @@ All actions accept optional `capture_after=True` to get a follow-up
 screenshot in the same tool call. All actions that target an element
 accept `modifiers=[…]` for held keys.
 
+The input actions (`click`, `double_click`, `right_click`, `middle_click`,
+`drag`, `scroll`, `type`, `key`) also accept `delivery_mode` and
+`bring_to_front` — see "The verify → escalate ladder" below.
+
+## The verify → escalate ladder (background-first)
+
+cua-driver delivers input in the **background** by default (no focus steal),
+but that is the first rung, not the only one. Every input action returns a
+structured verdict; read it and climb only when the driver tells you to.
+
+Returned fields (present when the driver supports them):
+- `effect`: `"confirmed"` (driver read the result back — done), `"unverifiable"`
+  (delivered, but confirm it yourself by re-capturing), or `"suspected_noop"`
+  (ran but almost certainly did nothing).
+- `escalation`: `{recommended: "px" | "foreground" | "page", reason}` — present
+  only when there's a next rung to try.
+- `code`: a structured refusal like `"background_unavailable"` or
+  `"foreground_unsupported"`.
+- `verified`: `true` only on AX read-back.
+
+Walk it in order:
+
+1. **Element, background (default).** `click(element=N)`. If `effect:"confirmed"`,
+   you're done.
+2. **Pixel, background.** On `escalation.recommended == "px"` (or a `degraded`
+   capture with an empty element list), click by `coordinate=[x,y]` read off the
+   screenshot instead of `element`.
+3. **Foreground.** On `escalation.recommended == "foreground"`,
+   `code:"background_unavailable"`, or a pixel click that still didn't land,
+   re-issue the SAME action with `delivery_mode="foreground"`. This briefly
+   raises the window and restores focus after; pair with `bring_to_front=True`
+   for a short sequence to avoid per-call flashes. It needs its own approval
+   (it's a visible focus change) and is only appropriate when the user isn't
+   actively working. Classic cases: Electron/Chromium consent dialogs (e.g.
+   tldraw offline's "Run Script"), DirectInput games, raw-input canvases.
+
+```
+computer_use(action="click", element=7)
+# → {effect: "suspected_noop", escalation: {recommended: "foreground", ...}}
+computer_use(action="click", element=7, delivery_mode="foreground")
+# → {effect: "unverifiable", path: "x11_pixel_fg"}   then re-capture to confirm
+```
+
+**Escalate to foreground as a REACTION to a returned signal, never as a
+prediction** from the app being Electron/Chromium/GTK. Different controls in
+the same app behave differently. Do NOT silently retry the same rung, and do
+NOT conclude "cua-driver can't drive this app" — climb the ladder. If
+`delivery_mode="foreground"` returns `code:"foreground_unsupported"`, the
+driver is too old; tell the user to update cua-driver.
+
 ### Key shortcuts vary per platform
 
 Use the host's idiomatic modifier:
@@ -205,7 +255,7 @@ in your conversation context.
 | `cua-driver not installed` | Run `hermes computer-use install`, or `hermes tools` and enable Computer Use |
 | Captures consistently return empty / "no on-screen window" | On Linux: DISPLAY may not be set (X11) or you're on pure Wayland — ask the user to run `hermes computer-use doctor`. On Windows: you may be in Session 0 (SSH session) instead of the interactive desktop — see the cua-driver `WINDOWS.md` deep-dive |
 | Element index stale ("Element N not in cache") | SOM indices are only valid until the next `capture`. Re-capture before clicking. The wrapper carries opaque `element_token`s for stale-detection; you'll see an explicit error rather than a wrong click |
-| Click had no effect | Re-capture and verify. A modal that wasn't visible before may be blocking input. Dismiss it (usually `escape` or click its close button) before retrying |
+| Click had no effect | Read the structured verdict, don't just recapture. `effect:"unverifiable"` → re-capture and confirm yourself. `effect:"suspected_noop"` / `code:"background_unavailable"` / `escalation.recommended` → climb the ladder: try `coordinate=[x,y]` (px), then `delivery_mode="foreground"`. A modal (e.g. an Electron consent dialog) may be blocking input — foreground delivery is how you dismiss it. Don't conclude the app is undrivable |
 | Type text disappears into a terminal emulator | cua-driver detects terminals (Ghostty, iTerm2, Terminal.app, Windows Terminal, mintty, etc.) and routes through key-event synthesis — should "just work" on a recent cua-driver. If it doesn't, ask the user to run `hermes computer-use doctor` |
 | `blocked pattern in type text` | You tried to `type` a shell command matching the dangerous-pattern block list (`curl ... \| bash`, `sudo rm -rf`, etc.). Break the command up or reconsider |
 | Anything else weird | **First action: ask the user to run `hermes computer-use doctor`.** It runs the cua-driver `health_report` MCP tool and prints a structured per-check matrix. Their output tells you (and them) exactly what's wrong |

tests/tools/test_computer_use_delivery_ladder.py

@@ -0,0 +1,291 @@
+"""Regression tests for the cua-driver verify → escalate ladder.
+
+Covers NousResearch/hermes-agent#67052:
+  - Phase A: cua-driver structured verdicts (verified/effect/escalation/code/
+    degraded/path) are preserved through ActionResult and surfaced in the
+    model-facing response, additively (old drivers omit them cleanly).
+  - Phase B: delivery_mode is model-reachable, capability-gated, and refuses
+    with foreground_unsupported on an old driver rather than silently
+    downgrading to background.
+  - Phase C: foreground approval is scoped by (action, delivery_mode) and by
+    session_id, so a background approval never silently authorizes foreground
+    and one run's unlock never leaks into another.
+
+Stdlib + pytest + unittest.mock only. No live cua-driver, no network.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from typing import Any, Dict, Optional
+from unittest.mock import patch
+
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def _reset():
+    from tools.computer_use.tool import reset_backend_for_tests
+    reset_backend_for_tests()
+    yield
+    reset_backend_for_tests()
+
+
+# ---------------------------------------------------------------------------
+# Phase A — structured verdict normalization (_action_result_from)
+# ---------------------------------------------------------------------------
+
+class _FakeSession:
+    """Minimal cua-driver session stub returning a canned tool result."""
+
+    def __init__(self, out: Dict[str, Any], capabilities: Optional[set] = None):
+        self._out = out
+        self._caps = capabilities or set()
+        self.last_args: Dict[str, Any] = {}
+
+    def call_tool(self, name: str, args: Dict[str, Any], timeout: float = 30.0):
+        self.last_args = args
+        return self._out
+
+    def supports_capability(self, capability: str, tool: Optional[str] = None) -> bool:
+        return capability in self._caps
+
+
+def _make_backend(session: _FakeSession):
+    from tools.computer_use.cua_backend import CuaDriverBackend
+    be = CuaDriverBackend.__new__(CuaDriverBackend)
+    be._session = session               # type: ignore[attr-defined]
+    be._session_id = "test-run"          # type: ignore[attr-defined]
+    be._snapshot_tokens = {}             # type: ignore[attr-defined]
+    be._active_pid = 4242                # type: ignore[attr-defined]
+    be._active_window_id = 7             # type: ignore[attr-defined]
+    return be
+
+
+def test_confirmed_verdict_is_preserved():
+    out = {
+        "isError": False, "data": {"message": "ok"},
+        "structuredContent": {"verified": True, "effect": "confirmed", "path": "ax"},
+    }
+    be = _make_backend(_FakeSession(out))
+    res = be.click(element=3)
+    assert res.ok is True
+    assert res.verified is True
+    assert res.effect == "confirmed"
+    assert res.path == "ax"
+    assert res.escalation is None
+
+
+def test_suspected_noop_carries_escalation():
+    out = {
+        "isError": False, "data": {},
+        "structuredContent": {
+            "effect": "suspected_noop",
+            "escalation": {"recommended": "foreground", "reason": "occluded renderer"},
+            "code": "background_unavailable",
+        },
+    }
+    be = _make_backend(_FakeSession(out))
+    res = be.click(element=3)
+    assert res.effect == "suspected_noop"
+    assert res.escalation == {"recommended": "foreground", "reason": "occluded renderer"}
+    assert res.code == "background_unavailable"
+    # transport ok, but semantically not confirmed
+    assert res.verified is None
+
+
+def test_unverifiable_distinct_from_success_and_failure():
+    out = {
+        "isError": False, "data": {},
+        "structuredContent": {"effect": "unverifiable", "verified": False, "path": "x11_pixel"},
+    }
+    be = _make_backend(_FakeSession(out))
+    res = be.click(x=10, y=20)
+    assert res.ok is True            # transport succeeded
+    assert res.verified is False     # ... but not confirmed
+    assert res.effect == "unverifiable"
+
+
+def test_degraded_capture_signal_preserved():
+    out = {
+        "isError": False, "data": {},
+        "structuredContent": {"effect": "suspected_noop", "degraded": True,
+                              "escalation": {"recommended": "px", "reason": "empty tree"}},
+    }
+    be = _make_backend(_FakeSession(out))
+    res = be.scroll(direction="down", element=1)
+    assert res.degraded is True
+    assert res.escalation["recommended"] == "px"
+
+
+def test_old_driver_without_structured_content_is_clean():
+    """A driver that returns no structuredContent leaves every verdict field
+    None — unchanged behavior, no crash."""
+    out = {"isError": False, "data": {"message": "done"}, "structuredContent": None}
+    be = _make_backend(_FakeSession(out))
+    res = be.click(element=3)
+    assert res.ok is True
+    assert res.message == "done"
+    assert res.verified is None
+    assert res.effect is None
+    assert res.escalation is None
+    assert res.code is None
+    assert res.path is None
+
+
+def test_text_response_surfaces_fields_additively():
+    from tools.computer_use.backend import ActionResult
+    from tools.computer_use.tool import _text_response
+
+    # Full verdict → all fields present.
+    r = ActionResult(ok=True, action="click", effect="suspected_noop",
+                     escalation={"recommended": "foreground"}, code="background_unavailable",
+                     path="ax", verified=False)
+    payload = json.loads(_text_response(r))
+    assert payload["effect"] == "suspected_noop"
+    assert payload["escalation"] == {"recommended": "foreground"}
+    assert payload["code"] == "background_unavailable"
+    assert payload["verified"] is False
+
+    # Bare result (old driver) → only ok/action, no None noise.
+    r2 = ActionResult(ok=True, action="click")
+    payload2 = json.loads(_text_response(r2))
+    assert payload2 == {"ok": True, "action": "click"}
+    for k in ("effect", "escalation", "code", "verified", "path", "degraded", "delivery_mode"):
+        assert k not in payload2
+
+
+# ---------------------------------------------------------------------------
+# Phase B — delivery_mode threading + capability gating
+# ---------------------------------------------------------------------------
+
+def test_background_is_default_no_flag_sent():
+    out = {"isError": False, "data": {}, "structuredContent": {"effect": "confirmed"}}
+    sess = _FakeSession(out)
+    be = _make_backend(sess)
+    be.click(element=1)  # no delivery_mode
+    assert "delivery_mode" not in sess.last_args
+
+
+def test_foreground_sent_when_capability_present():
+    out = {"isError": False, "data": {}, "structuredContent": {"effect": "unverifiable"}}
+    sess = _FakeSession(out, capabilities={"input.delivery_mode"})
+    be = _make_backend(sess)
+    res = be.click(element=1, delivery_mode="foreground", bring_to_front=True)
+    assert sess.last_args.get("delivery_mode") == "foreground"
+    assert sess.last_args.get("bring_to_front") is True
+    assert res.delivery_mode == "foreground"
+
+
+def test_foreground_refused_on_old_driver():
+    """Old driver lacking the capability must NOT silently downgrade — it
+    returns a structured foreground_unsupported result."""
+    out = {"isError": False, "data": {}, "structuredContent": {}}
+    sess = _FakeSession(out, capabilities=set())  # no input.delivery_mode
+    be = _make_backend(sess)
+    res = be.click(element=1, delivery_mode="foreground")
+    assert res.ok is False
+    assert res.code == "foreground_unsupported"
+    # crucially: no tool call was made with a silent background downgrade
+    assert sess.last_args == {}
+
+
+def test_bad_delivery_mode_rejected():
+    out = {"isError": False, "data": {}, "structuredContent": {}}
+    sess = _FakeSession(out, capabilities={"input.delivery_mode"})
+    be = _make_backend(sess)
+    res = be.type_text("hi", delivery_mode="sideways")
+    assert res.ok is False
+    assert res.code == "bad_delivery_mode"
+
+
+def test_dispatcher_threads_delivery_mode_to_backend():
+    """End-to-end through the tool dispatcher with the noop backend."""
+    from tools.computer_use import tool as cu
+    with patch.dict(os.environ, {"HERMES_COMPUTER_USE_BACKEND": "noop"}, clear=False):
+        cu.reset_backend_for_tests()
+        be = cu._get_backend()
+        cu.handle_computer_use({"action": "click", "element": 5,
+                                "delivery_mode": "foreground"})
+        # noop records kwargs; find the click call
+        clicks = [kw for (name, kw) in be.calls if name == "click"]  # type: ignore[attr-defined]
+        assert clicks and clicks[-1].get("delivery_mode") == "foreground"
+
+
+# ---------------------------------------------------------------------------
+# Phase C — foreground approval scoping (action + delivery_mode + session)
+# ---------------------------------------------------------------------------
+
+def test_background_approval_does_not_authorize_foreground():
+    from tools.computer_use import tool as cu
+
+    seen = []
+
+    def cb(action, args, summary):
+        seen.append((action, args.get("delivery_mode")))
+        return "approve_session"
+
+    cu.set_approval_callback(cb)
+    try:
+        # Background click, approve for session.
+        assert cu._request_approval("click", {}, "sess-A") is None
+        # A second background click needs no prompt (cached).
+        assert cu._request_approval("click", {}, "sess-A") is None
+        assert len(seen) == 1
+        # Foreground click on the SAME action must prompt again — the
+        # background approval does not cover it.
+        assert cu._request_approval("click", {"delivery_mode": "foreground"}, "sess-A") is None
+        assert len(seen) == 2
+        assert seen[-1] == ("click", "foreground")
+    finally:
+        cu.set_approval_callback(None)
+
+
+def test_approval_state_is_session_scoped():
+    from tools.computer_use import tool as cu
+
+    calls = []
+
+    def cb(action, args, summary):
+        calls.append((action, args.get("delivery_mode")))
+        return "approve_session"
+
+    cu.set_approval_callback(cb)
+    try:
+        # Run A approves foreground click.
+        cu._request_approval("click", {"delivery_mode": "foreground"}, "run-A")
+        # Run B has NOT — it must prompt independently.
+        n_before = len(calls)
+        cu._request_approval("click", {"delivery_mode": "foreground"}, "run-B")
+        assert len(calls) == n_before + 1
+    finally:
+        cu.set_approval_callback(None)
+
+
+def test_always_approve_covers_foreground():
+    from tools.computer_use import tool as cu
+
+    calls = []
+
+    def cb(action, args, summary):
+        calls.append(action)
+        return "always_approve"
+
+    cu.set_approval_callback(cb)
+    try:
+        # First call unlocks everything for this session.
+        cu._request_approval("click", {}, "run-C")
+        # Foreground now sails through without another prompt.
+        cu._request_approval("click", {"delivery_mode": "foreground"}, "run-C")
+        assert len(calls) == 1
+    finally:
+        cu.set_approval_callback(None)
+
+
+def test_foreground_summary_warns_about_focus_change():
+    from tools.computer_use.tool import _summarize_action
+    s = _summarize_action("click", {"element": 3, "delivery_mode": "foreground"})
+    assert "FOREGROUND" in s
+    bg = _summarize_action("click", {"element": 3})
+    assert "FOREGROUND" not in bg