diff --git a/docs/content/docs/reference/cua-driver/mcp-tools.mdx b/docs/content/docs/reference/cua-driver/mcp-tools.mdx index d3c49bc809..854476088e 100644 --- a/docs/content/docs/reference/cua-driver/mcp-tools.mdx +++ b/docs/content/docs/reference/cua-driver/mcp-tools.mdx @@ -220,7 +220,7 @@ from_zoom: set true after a zoom call to auto-translate zoom-image pixel coordin - `button` (string, optional): Mouse button. Default: "left" — omit for legacy left-click behaviour. Pixel path uses the matching CGEvent primitive; AX path maps "right" to AXShowMenu and falls back to a pixel middle-click at the element's center for "middle". - `count` (integer, optional): Click count (pixel path only). Default 1. - `debug_image_out` (string, optional): Optional file path. When set on a pixel-addressed click, captures a fresh screenshot, draws a red crosshair at (x, y), and writes the PNG. Use to verify coordinate spaces. Requires window_id; incompatible with from_zoom. -- `delivery_mode` (string, optional): Best-effort-background ladder rung (default "background"). "background": perform the AX action or post the CGEvent without fronting. "foreground": briefly front the window, act, let transient UI settle, then restore the prior frontmost app. Requires window_id. A click is never driver-verifiable (no read-back), so both report verified:false — confirm the effect via screenshot. Use the agent loop: background AX (element_index) → screenshot → background pixel (x/y) → screenshot → delivery_mode:"foreground". +- `delivery_mode` (string, optional): Best-effort-background ladder rung (default "background"). "background": perform the AX action or post the CGEvent without fronting. "foreground": briefly front the window, act, let transient UI settle, then restore the prior frontmost app. Requires window_id. A generic click has no independent postcondition read-back, so its action effect remains unverifiable — confirm the effect from a fresh state snapshot. Use the agent loop: background AX (element_index) → snapshot → background pixel (x/y) → snapshot → delivery_mode:"foreground". - `element_index` (integer, optional): Element index from last get_window_state. REQUIRES `pid` and `window_id` to be passed alongside it — element_index alone (no pid) fails fast with "Missing required integer field: pid"; it is not a silent no-op. - `element_token` (string, optional): Opaque per-snapshot element handle from `structuredContent.elements[].element_token` of the last get_window_state. Takes precedence over element_index when both supplied. Returns an explicit "stale" error if the snapshot has been superseded — re-snapshot in that case. - `from_zoom` (boolean, optional): When true, x and y are in the last zoom image for this pid; driver translates back to full-window coordinates. @@ -325,7 +325,7 @@ WEB CONTENT (Chromium/WebKit/Electron — browser tabs, Slack, VS Code, X's comp **Arguments:** - `delay_ms` (integer, optional): Milliseconds between characters in the CGEvent fallback path. Default 30. Ignored when the AX path succeeds. range: 0–200 -- `delivery_mode` (string, optional): Best-effort-background ladder rung (default "background"). "background": AX insert, then CGEvent keystrokes if needed — no focus steal; native controls can be verified via AXValue read-back, while web-content read-back remains unverified. "foreground": briefly front the window, type, restore the prior frontmost — the explicit last resort for focus-sensitive surfaces (e.g. WhatsApp/Catalyst) where background keystrokes don't land. Re-call with "foreground" when a background attempt returns `verified:false` and a screenshot shows the text didn't appear. +- `delivery_mode` (string, optional): Best-effort-background ladder rung (default "background"). "background": AX insert, then CGEvent keystrokes if needed — no focus steal; native controls can be confirmed via AXValue read-back, while web-content writes remain effect:"unverifiable". "foreground": briefly front the window, type, restore the prior frontmost — the explicit last resort for focus-sensitive surfaces (e.g. WhatsApp/Catalyst) where background keystrokes don't land. Re-call with "foreground" when a background attempt remains unverifiable and a fresh snapshot shows the text did not appear. - `element_index` (integer, optional): Element index from last get_window_state. Directs the write to a specific field. REQUIRES `pid` and `window_id` to be passed alongside it — element_index alone (no pid) fails fast with "Missing required integer field: pid"; it is not a silent no-op. - `element_token` (string, optional): Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit "stale" error if the snapshot has been superseded. - `pid` (integer, optional): Target process ID. diff --git a/libs/cua-driver/contract/README.md b/libs/cua-driver/contract/README.md index 69d5b7ebac..59721e67ce 100644 --- a/libs/cua-driver/contract/README.md +++ b/libs/cua-driver/contract/README.md @@ -73,14 +73,19 @@ from the contract rather than a second runtime map. Both SDKs retain a generic tool call so runtime-discovered and platform-specific tools remain usable. The generated manifest records tool platforms, capabilities, annotations, input schemas, and experimental success -schemas. Success schemas are not advertised as live MCP `outputSchema` values -until every transport path has passed parity tests. +schemas. The live MCP `tools/list` response advertises these successful-result +schemas as `outputSchema`; all action tools share the closed `ActionResult` +schema even when their richer runtime input is not part of the portable SDK +manifest. + +See [Action results and postcondition verification](../docs/action-result-contract.md) +for the wire shape and 0.14 migration guidance. Compatibility is tracked separately at each boundary: | Field | Current | Meaning | | --- | --- | --- | -| `contract_version` | `0.3.0` | Generated manifest and typed SDK shape | +| `contract_version` | `0.4.0` | Generated manifest and typed SDK shape | | `tools_list_schema_version` | `1` | cua-driver `tools/list` extension shape | | `capability_version` | `1` | Additive capability-token vocabulary | | `mcp_protocol_version` | `2025-06-18` | MCP initialization protocol served to agent runtimes | diff --git a/libs/cua-driver/contract/manifest.json b/libs/cua-driver/contract/manifest.json index 772cec97ac..720a580048 100644 --- a/libs/cua-driver/contract/manifest.json +++ b/libs/cua-driver/contract/manifest.json @@ -1,7 +1,7 @@ { "generated_notice": "Generated by cua-contract-gen; do not edit by hand.", "experimental": true, - "contract_version": "0.3.0", + "contract_version": "0.4.0", "tools_list_schema_version": "1", "capability_version": "1", "mcp_protocol_version": "2025-06-18", @@ -68,21 +68,116 @@ "type": "object" }, "success_output_schema": { - "additionalProperties": true, + "additionalProperties": false, "properties": { - "scope": { - "const": "desktop" + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] }, - "verified": { - "type": "boolean" + "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], + "type": "string" }, - "x": { - "type": "number" + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] }, - "y": { - "type": "number" + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "dom", + "trusted_input" + ], + "type": "string" } }, + "required": [ + "effect", + "route" + ], "type": "object" } }, @@ -164,18 +259,116 @@ "type": "object" }, "success_output_schema": { - "additionalProperties": true, + "additionalProperties": false, "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], "type": "string" }, - "scope": { - "const": "desktop" + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] }, - "verified": { - "type": "boolean" + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "dom", + "trusted_input" + ], + "type": "string" } }, + "required": [ + "effect", + "route" + ], "type": "object" } }, @@ -899,18 +1092,116 @@ "type": "object" }, "success_output_schema": { - "additionalProperties": true, + "additionalProperties": false, "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], "type": "string" }, - "scope": { - "const": "desktop" + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] }, - "verified": { - "type": "boolean" + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "dom", + "trusted_input" + ], + "type": "string" } }, + "required": [ + "effect", + "route" + ], "type": "object" } }, @@ -961,18 +1252,116 @@ "type": "object" }, "success_output_schema": { - "additionalProperties": true, + "additionalProperties": false, "properties": { - "scope": { - "const": "desktop" + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] }, - "x": { - "type": "number" + "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], + "type": "string" }, - "y": { - "type": "number" + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] + }, + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "dom", + "trusted_input" + ], + "type": "string" } }, + "required": [ + "effect", + "route" + ], "type": "object" } }, @@ -1025,18 +1414,116 @@ "type": "object" }, "success_output_schema": { - "additionalProperties": true, + "additionalProperties": false, "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], "type": "string" }, - "scope": { - "const": "desktop" + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] }, - "verified": { - "type": "boolean" + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "dom", + "trusted_input" + ], + "type": "string" } }, + "required": [ + "effect", + "route" + ], "type": "object" } }, @@ -1109,18 +1596,116 @@ "type": "object" }, "success_output_schema": { - "additionalProperties": true, + "additionalProperties": false, "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], "type": "string" }, - "scope": { - "const": "desktop" + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] }, - "verified": { - "type": "boolean" + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "dom", + "trusted_input" + ], + "type": "string" } }, + "required": [ + "effect", + "route" + ], "type": "object" } }, @@ -1616,18 +2201,116 @@ "type": "object" }, "success_output_schema": { - "additionalProperties": true, + "additionalProperties": false, "properties": { + "delivery": { + "additionalProperties": false, + "properties": { + "delivered_count": { + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "mode": { + "enum": [ + "background", + "foreground", + "not_applicable", + "unknown" + ], + "type": "string" + } + }, + "required": [ + "mode" + ], + "type": [ + "object", + "null" + ] + }, "effect": { + "enum": [ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ], "type": "string" }, - "scope": { - "const": "desktop" + "escalation": { + "additionalProperties": false, + "properties": { + "reason": { + "enum": [ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ], + "type": "string" + }, + "target": { + "enum": [ + "pixel", + "foreground", + "page", + "session" + ], + "type": "string" + } + }, + "required": [ + "target", + "reason" + ], + "type": [ + "object", + "null" + ] }, - "verified": { - "type": "boolean" + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "kind": { + "enum": [ + "value_readback", + "window_change" + ], + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "route": { + "enum": [ + "accessibility", + "synthetic_events", + "global_input", + "dom", + "trusted_input" + ], + "type": "string" } }, + "required": [ + "effect", + "route" + ], "type": "object" } }, diff --git a/libs/cua-driver/docs/action-result-contract.md b/libs/cua-driver/docs/action-result-contract.md new file mode 100644 index 0000000000..6b8889364a --- /dev/null +++ b/libs/cua-driver/docs/action-result-contract.md @@ -0,0 +1,129 @@ +# Action results and postcondition verification + +Cua Driver 0.15 separates two facts that earlier releases mixed together: + +- `ActionResult` says what route the driver used and how strongly it can + account for the action itself. +- `VerifyStateOutput` says whether a caller-defined postcondition is + `satisfied`, `unsatisfied`, or `unknown`. + +The driver reports these facts. The agent harness owns task meaning, visual +reading, stop/retry decisions, and movement through the action ladder. + +## MCP action result + +Every successful action returns a closed `structuredContent` object: + +```json +{ + "effect": "confirmed", + "route": "accessibility", + "delivery": {"mode": "background"}, + "evidence": [{"kind": "value_readback"}] +} +``` + +`effect` and `route` are required. + +| Field | Values | +| --- | --- | +| `effect` | `confirmed`, `partial`, `unverifiable`, `suspected_noop`, `refused` | +| `route` | `accessibility`, `synthetic_events`, `global_input`, `dom`, `trusted_input` | +| `delivery.mode` | `background`, `foreground`, `not_applicable`, `unknown` | +| `evidence[].kind` | `value_readback`, `window_change` | +| `escalation.target` | `pixel`, `foreground`, `page`, `session` | +| `escalation.reason` | `route_unavailable`, `delivery_failed`, `effect_unconfirmed`, `suspected_noop`, `permission_required` | + +The action-result tools are: + +`click`, `double_click`, `right_click`, `scroll`, `drag`, `mouse_drag`, +`parallel_mouse_drag`, `move_cursor`, `mouse_button_down`, `mouse_button_up`, +`type_text`, `type_text_chars`, `press_key`, `hotkey`, `set_value`, +`browser_click`, `browser_pointer`, and `browser_type`. + +Other mutating tools such as application launch, window activation, browser +navigation, dialogs, uploads, and downloads retain their own typed results. + +The contract is deliberately closed. It does not echo selectors, coordinates, +scope, targets, platform transport names, diagnostic pointers, or the old +`verified` boolean. + +The invariants are: + +- `confirmed` has publishable readback or window-change evidence; +- `partial` has `delivery.delivered_count`; +- `refused` has neither delivery nor evidence. + +An action that reached an actuator but lacks a trusted readback is +`unverifiable`, not `confirmed`. Screenshot change, native API acceptance, +event receipt, and operator observation may remain useful internal diagnostics, +but they do not independently justify `confirmed`. + +## Verification remains separate + +After an action, use `verify_state` for a bounded structured postcondition. +`satisfied` is the only successful terminal status. `unsatisfied` can justify a +retry or another ladder route. `unknown` means the available observation could +not prove either answer and must never be promoted to success. + +When `include_screenshot` is enabled, the screenshot is uninterpreted evidence. +A multimodal harness reads it and decides whether to stop, retry, or advance. + +## SDK access + +Rust, Python, and TypeScript keep the transport-neutral `ToolResult` envelope: +text, images, structured JSON, error state/code, degraded state, and raw JSON. +The ambiguous `verified` field is removed. + +Successful action calls expose the typed value at `result.action`; successful +`verify_state` calls expose it at `result.verification`. In Rust the equivalent +borrow accessors are `result.action()` and `result.verification()`. + +```python +result = await driver.click(click_input) +if result.action.effect is ActionEffect.CONFIRMED: + verification = await driver.verify_state(expectation) + if verification.verification.status is VerificationStatus.SATISFIED: + return "done" +``` + +```ts +const result = await driver.click(input) +if (result.action?.effect === ActionEffect.Confirmed) { + const checked = await driver.verifyState(expectation) + if (checked.verification?.status === VerificationStatus.Satisfied) { + return "done" + } +} +``` + +## Escalation belongs to the harness + +An optional escalation is advice, not an automatic retry: + +| Target | Harness action | +| --- | --- | +| `pixel` | refresh visual state and choose an exact pixel target | +| `foreground` | explicitly select foreground delivery when session policy permits | +| `page` | bind the native window to a supported browser page route | +| `session` | prepare or explicitly widen the session only when policy permits | + +SDK integrators, OpenClaw, Hermes, and other agent hosts can implement different +policies above this same narrow fact contract without duplicating platform +actuator details. + +## Migration from 0.14 + +- Replace `result.verified` checks with `result.action.effect` for action facts. +- Use `result.verification.status` only for `verify_state` postconditions. +- Replace imports of the removed `ClickOutput`, `DesktopActionOutput`, and + `MoveCursorOutput` types with `ActionResult`. +- Do not read coordinates, `scope`, `path`, `transport`, or request targets from + an action response; retain request context in the caller if it is needed. +- `move_cursor` no longer echoes `x`/`y`; call `get_cursor_position` when the + observed pointer location is needed. +- Treat `unverifiable` as unknown action effect, not failure and not success. +- Treat MCP `isError` as transport/tool failure; inspect a successful + `ActionResult` separately. +- Upgrade daemon and SDK together. A 0.15 SDK intentionally rejects legacy + 0.14 action payloads instead of guessing at their meaning. diff --git a/libs/cua-driver/docs/tool-output-format.md b/libs/cua-driver/docs/tool-output-format.md index a14c82e0a0..0ae00df560 100644 --- a/libs/cua-driver/docs/tool-output-format.md +++ b/libs/cua-driver/docs/tool-output-format.md @@ -1,39 +1,37 @@ -# cua-driver MCP Tool Output Format +# Cua Driver MCP tool output format -Every tool call returns a ✅ checkmark + concise summary. No structured JSON output. +Every call keeps the standard MCP `ToolResult` envelope: -## screenshot -``` -✅ Screenshot — 1920x1080 png +- `content` contains human-readable text and optional images; +- `structuredContent` contains the machine-readable successful result; +- `isError` distinguishes tool failure from a successful outcome. -On-screen windows: -- Terminal (pid 7476) "cua — Claude Code" [window_id: 2102] -- Blender (pid 6808) "* Untitled - Blender 5.1.1" [window_id: 2129] -- Google Chrome (pid 13313) "Download — Blender" [window_id: 1941] -→ Call get_window_state(pid, window_id) to inspect a window's UI. -``` +Text is diagnostic, not a stable parsing API. -## get_window_state -``` -✅ Blender — 11 elements, turn 3 + screenshot -⚠️ Small AX tree (11 elements) — this app likely uses custom rendering - (e.g. Blender, games, Electron). Use pixel clicks: click(pid, x, y) - with coordinates from the screenshot. - -- AXApplication "Blender" - - [0] AXWindow "* Untitled - Blender 5.1.1" actions=[AXRaise] - - [1] AXButton - ... -``` +## Action tools -## click -``` -✅ Posted click to pid 6808. -``` +Successful pointer, keyboard, value, and browser-input actions return the +closed shared `ActionResult` in `structuredContent`: -## zoom -``` -✅ Zoomed region captured at native resolution. To click a target in -this image, use `click(pid, x, y, from_zoom=true)` where x,y are pixel -coordinates in THIS zoomed image — the driver maps them back automatically. +```json +{ + "effect": "unverifiable", + "route": "global_input", + "delivery": {"mode": "foreground"}, + "escalation": {"target": "page", "reason": "effect_unconfirmed"} +} ``` + +Do not parse platform route names, coordinates, targets, or the removed +`verified` bit from text. Use `effect`, `route`, `delivery`, `evidence`, and +`escalation`, then call `verify_state` or take a fresh snapshot for the task +postcondition. See [Action results and postcondition +verification](action-result-contract.md). + +## Observation and state tools + +Observation tools retain their typed tool-specific structured payloads. +`get_window_state`, for example, returns the accessibility outline and element +records in `structuredContent` and can attach a PNG as image content. A +multimodal harness interprets the image; Cua Driver does not OCR it or assign +task meaning. diff --git a/libs/cua-driver/docs/why-cua-driver-uses-mcp-instead-of-uniffi.md b/libs/cua-driver/docs/why-cua-driver-uses-mcp-instead-of-uniffi.md index 5025bf8648..1ada52c262 100644 --- a/libs/cua-driver/docs/why-cua-driver-uses-mcp-instead-of-uniffi.md +++ b/libs/cua-driver/docs/why-cua-driver-uses-mcp-instead-of-uniffi.md @@ -35,10 +35,12 @@ imports a generated Cua client. by UniFFI. Remove their language-native MCP facades because those duplicate a runtime-neutral protocol client that agent runtimes already provide. This deliberately makes the package API breaking before publication. -3. Export the shared Rust request records for all 14 typed tools and the shared - typed session results. Desktop structured results retain platform extension - fields, so the native SDK preserves them in `ToolResult.structured_json` - while the live registry validates them with the canonical Rust result types. +3. Export the shared Rust request records for all typed tools and shared typed + results. Action tools use the closed, cross-platform `ActionResult`; richer + observation payloads remain available in `ToolResult.structured_json`. + The live registry validates both against the canonical Rust result types. + See [Action results and postcondition + verification](action-result-contract.md). 4. Use UniFFI as an SDK/server-composition architecture, not as a replacement for the agent MCP boundary. The first slice exports a shared Rust daemon client that application-owned servers can compose. diff --git a/libs/cua-driver/python/src/cua_driver/__init__.py b/libs/cua-driver/python/src/cua_driver/__init__.py index 9618fbb189..aa13f5b6af 100644 --- a/libs/cua-driver/python/src/cua_driver/__init__.py +++ b/libs/cua-driver/python/src/cua_driver/__init__.py @@ -48,6 +48,16 @@ request_mac_os_permissions, ) from ._native_contract import ( + ActionDelivery, + ActionDeliveryMode, + ActionEffect, + ActionEscalation, + ActionEscalationReason, + ActionEscalationTarget, + ActionEvidence, + ActionEvidenceKind, + ActionResult, + ActionRoute, BoundsExpectation, CaptureScope, ClickButton, @@ -158,6 +168,16 @@ def _create_private_worker_python_sdk(cls, options): __all__ = [ "ActionCompletion", + "ActionDelivery", + "ActionDeliveryMode", + "ActionEffect", + "ActionEscalation", + "ActionEscalationReason", + "ActionEscalationTarget", + "ActionEvidence", + "ActionEvidenceKind", + "ActionResult", + "ActionRoute", "BoundsExpectation", "CaptureScope", "ClickButton", diff --git a/libs/cua-driver/python/src/cua_driver/_native.py b/libs/cua-driver/python/src/cua_driver/_native.py index 4aa44fc6f3..478041bfa5 100644 --- a/libs/cua-driver/python/src/cua_driver/_native.py +++ b/libs/cua-driver/python/src/cua_driver/_native.py @@ -3115,11 +3115,40 @@ def read(cls, buf): _UniffiFfiConverterTypeImageContent.read(buf) for i in range(count) ] -class _UniffiFfiConverterOptionalBoolean(_UniffiConverterRustBuffer): + + +class _UniffiFfiConverterOptionalTypeActionResult(_UniffiConverterRustBuffer): + @classmethod + def check_lower(cls, value): + if value is not None: + cua_driver._native_contract._UniffiFfiConverterTypeActionResult.check_lower(value) + + @classmethod + def write(cls, value, buf): + if value is None: + buf.write_u8(0) + return + + buf.write_u8(1) + cua_driver._native_contract._UniffiFfiConverterTypeActionResult.write(value, buf) + + @classmethod + def read(cls, buf): + flag = buf.read_u8() + if flag == 0: + return None + elif flag == 1: + return cua_driver._native_contract._UniffiFfiConverterTypeActionResult.read(buf) + else: + raise InternalError("Unexpected flag byte for optional type") + + + +class _UniffiFfiConverterOptionalTypeVerifyStateOutput(_UniffiConverterRustBuffer): @classmethod def check_lower(cls, value): if value is not None: - _UniffiFfiConverterBoolean.check_lower(value) + cua_driver._native_contract._UniffiFfiConverterTypeVerifyStateOutput.check_lower(value) @classmethod def write(cls, value, buf): @@ -3128,7 +3157,7 @@ def write(cls, value, buf): return buf.write_u8(1) - _UniffiFfiConverterBoolean.write(value, buf) + cua_driver._native_contract._UniffiFfiConverterTypeVerifyStateOutput.write(value, buf) @classmethod def read(cls, buf): @@ -3136,7 +3165,7 @@ def read(cls, buf): if flag == 0: return None elif flag == 1: - return _UniffiFfiConverterBoolean.read(buf) + return cua_driver._native_contract._UniffiFfiConverterTypeVerifyStateOutput.read(buf) else: raise InternalError("Unexpected flag byte for optional type") @@ -3146,13 +3175,14 @@ class ToolResult: Transport-neutral result envelope used for open-ended tool calls and desktop tools whose platform extensions are intentionally preserved as JSON. """ - def __init__(self, *, text:str, images:typing.List[ImageContent], structured_json:typing.Optional[str], is_error:bool, error_code:typing.Optional[str], verified:typing.Optional[bool], degraded:bool, raw_json:str): + def __init__(self, *, text:str, images:typing.List[ImageContent], structured_json:typing.Optional[str], is_error:bool, error_code:typing.Optional[str], action:typing.Optional[cua_driver._native_contract.ActionResult], verification:typing.Optional[cua_driver._native_contract.VerifyStateOutput], degraded:bool, raw_json:str): self.text = text self.images = images self.structured_json = structured_json self.is_error = is_error self.error_code = error_code - self.verified = verified + self.action = action + self.verification = verification self.degraded = degraded self.raw_json = raw_json @@ -3160,7 +3190,7 @@ def __init__(self, *, text:str, images:typing.List[ImageContent], structured_jso def __str__(self): - return "ToolResult(text={}, images={}, structured_json={}, is_error={}, error_code={}, verified={}, degraded={}, raw_json={})".format(self.text, self.images, self.structured_json, self.is_error, self.error_code, self.verified, self.degraded, self.raw_json) + return "ToolResult(text={}, images={}, structured_json={}, is_error={}, error_code={}, action={}, verification={}, degraded={}, raw_json={})".format(self.text, self.images, self.structured_json, self.is_error, self.error_code, self.action, self.verification, self.degraded, self.raw_json) def __eq__(self, other): if self.text != other.text: return False @@ -3172,7 +3202,9 @@ def __eq__(self, other): return False if self.error_code != other.error_code: return False - if self.verified != other.verified: + if self.action != other.action: + return False + if self.verification != other.verification: return False if self.degraded != other.degraded: return False @@ -3189,7 +3221,8 @@ def read(buf): structured_json=_UniffiFfiConverterOptionalString.read(buf), is_error=_UniffiFfiConverterBoolean.read(buf), error_code=_UniffiFfiConverterOptionalString.read(buf), - verified=_UniffiFfiConverterOptionalBoolean.read(buf), + action=_UniffiFfiConverterOptionalTypeActionResult.read(buf), + verification=_UniffiFfiConverterOptionalTypeVerifyStateOutput.read(buf), degraded=_UniffiFfiConverterBoolean.read(buf), raw_json=_UniffiFfiConverterString.read(buf), ) @@ -3201,7 +3234,8 @@ def check_lower(value): _UniffiFfiConverterOptionalString.check_lower(value.structured_json) _UniffiFfiConverterBoolean.check_lower(value.is_error) _UniffiFfiConverterOptionalString.check_lower(value.error_code) - _UniffiFfiConverterOptionalBoolean.check_lower(value.verified) + _UniffiFfiConverterOptionalTypeActionResult.check_lower(value.action) + _UniffiFfiConverterOptionalTypeVerifyStateOutput.check_lower(value.verification) _UniffiFfiConverterBoolean.check_lower(value.degraded) _UniffiFfiConverterString.check_lower(value.raw_json) @@ -3212,7 +3246,8 @@ def write(value, buf): _UniffiFfiConverterOptionalString.write(value.structured_json, buf) _UniffiFfiConverterBoolean.write(value.is_error, buf) _UniffiFfiConverterOptionalString.write(value.error_code, buf) - _UniffiFfiConverterOptionalBoolean.write(value.verified, buf) + _UniffiFfiConverterOptionalTypeActionResult.write(value.action, buf) + _UniffiFfiConverterOptionalTypeVerifyStateOutput.write(value.verification, buf) _UniffiFfiConverterBoolean.write(value.degraded, buf) _UniffiFfiConverterString.write(value.raw_json, buf) diff --git a/libs/cua-driver/python/src/cua_driver/_native_contract.py b/libs/cua-driver/python/src/cua_driver/_native_contract.py index 52c47818f2..ff855b2838 100644 --- a/libs/cua-driver/python/src/cua_driver/_native_contract.py +++ b/libs/cua-driver/python/src/cua_driver/_native_contract.py @@ -754,6 +754,642 @@ class _UniffiForeignFutureDroppedCallbackStruct(ctypes.Structure): # Public interface members begin here. + + + + + +class ActionDeliveryMode(enum.Enum): + + BACKGROUND = 0 + + FOREGROUND = 1 + + NOT_APPLICABLE = 2 + + UNKNOWN = 3 + + + +class _UniffiFfiConverterTypeActionDeliveryMode(_UniffiConverterRustBuffer): + @staticmethod + def read(buf): + variant = buf.read_i32() + if variant == 1: + return ActionDeliveryMode.BACKGROUND + if variant == 2: + return ActionDeliveryMode.FOREGROUND + if variant == 3: + return ActionDeliveryMode.NOT_APPLICABLE + if variant == 4: + return ActionDeliveryMode.UNKNOWN + raise InternalError("Raw enum value doesn't match any cases") + + @staticmethod + def check_lower(value): + if value == ActionDeliveryMode.BACKGROUND: + return + if value == ActionDeliveryMode.FOREGROUND: + return + if value == ActionDeliveryMode.NOT_APPLICABLE: + return + if value == ActionDeliveryMode.UNKNOWN: + return + raise ValueError(value) + + @staticmethod + def write(value, buf): + if value == ActionDeliveryMode.BACKGROUND: + buf.write_i32(1) + if value == ActionDeliveryMode.FOREGROUND: + buf.write_i32(2) + if value == ActionDeliveryMode.NOT_APPLICABLE: + buf.write_i32(3) + if value == ActionDeliveryMode.UNKNOWN: + buf.write_i32(4) + + + +class _UniffiFfiConverterUInt32(_UniffiConverterPrimitiveInt): + CLASS_NAME = "u32" + VALUE_MIN = 0 + VALUE_MAX = 2**32 + + @staticmethod + def read(buf): + return buf.read_u32() + + @staticmethod + def write(value, buf): + buf.write_u32(value) + +class _UniffiFfiConverterOptionalUInt32(_UniffiConverterRustBuffer): + @classmethod + def check_lower(cls, value): + if value is not None: + _UniffiFfiConverterUInt32.check_lower(value) + + @classmethod + def write(cls, value, buf): + if value is None: + buf.write_u8(0) + return + + buf.write_u8(1) + _UniffiFfiConverterUInt32.write(value, buf) + + @classmethod + def read(cls, buf): + flag = buf.read_u8() + if flag == 0: + return None + elif flag == 1: + return _UniffiFfiConverterUInt32.read(buf) + else: + raise InternalError("Unexpected flag byte for optional type") + +@dataclass +class ActionDelivery: + def __init__(self, *, mode:ActionDeliveryMode, delivered_count:typing.Optional[int]): + self.mode = mode + self.delivered_count = delivered_count + + + + + def __str__(self): + return "ActionDelivery(mode={}, delivered_count={})".format(self.mode, self.delivered_count) + def __eq__(self, other): + if self.mode != other.mode: + return False + if self.delivered_count != other.delivered_count: + return False + return True + +class _UniffiFfiConverterTypeActionDelivery(_UniffiConverterRustBuffer): + @staticmethod + def read(buf): + return ActionDelivery( + mode=_UniffiFfiConverterTypeActionDeliveryMode.read(buf), + delivered_count=_UniffiFfiConverterOptionalUInt32.read(buf), + ) + + @staticmethod + def check_lower(value): + _UniffiFfiConverterTypeActionDeliveryMode.check_lower(value.mode) + _UniffiFfiConverterOptionalUInt32.check_lower(value.delivered_count) + + @staticmethod + def write(value, buf): + _UniffiFfiConverterTypeActionDeliveryMode.write(value.mode, buf) + _UniffiFfiConverterOptionalUInt32.write(value.delivered_count, buf) + + + + + + +class ActionEscalationTarget(enum.Enum): + + PIXEL = 0 + + FOREGROUND = 1 + + PAGE = 2 + + SESSION = 3 + + + +class _UniffiFfiConverterTypeActionEscalationTarget(_UniffiConverterRustBuffer): + @staticmethod + def read(buf): + variant = buf.read_i32() + if variant == 1: + return ActionEscalationTarget.PIXEL + if variant == 2: + return ActionEscalationTarget.FOREGROUND + if variant == 3: + return ActionEscalationTarget.PAGE + if variant == 4: + return ActionEscalationTarget.SESSION + raise InternalError("Raw enum value doesn't match any cases") + + @staticmethod + def check_lower(value): + if value == ActionEscalationTarget.PIXEL: + return + if value == ActionEscalationTarget.FOREGROUND: + return + if value == ActionEscalationTarget.PAGE: + return + if value == ActionEscalationTarget.SESSION: + return + raise ValueError(value) + + @staticmethod + def write(value, buf): + if value == ActionEscalationTarget.PIXEL: + buf.write_i32(1) + if value == ActionEscalationTarget.FOREGROUND: + buf.write_i32(2) + if value == ActionEscalationTarget.PAGE: + buf.write_i32(3) + if value == ActionEscalationTarget.SESSION: + buf.write_i32(4) + + + + + + + + +class ActionEscalationReason(enum.Enum): + + ROUTE_UNAVAILABLE = 0 + + DELIVERY_FAILED = 1 + + EFFECT_UNCONFIRMED = 2 + + SUSPECTED_NOOP = 3 + + PERMISSION_REQUIRED = 4 + + + +class _UniffiFfiConverterTypeActionEscalationReason(_UniffiConverterRustBuffer): + @staticmethod + def read(buf): + variant = buf.read_i32() + if variant == 1: + return ActionEscalationReason.ROUTE_UNAVAILABLE + if variant == 2: + return ActionEscalationReason.DELIVERY_FAILED + if variant == 3: + return ActionEscalationReason.EFFECT_UNCONFIRMED + if variant == 4: + return ActionEscalationReason.SUSPECTED_NOOP + if variant == 5: + return ActionEscalationReason.PERMISSION_REQUIRED + raise InternalError("Raw enum value doesn't match any cases") + + @staticmethod + def check_lower(value): + if value == ActionEscalationReason.ROUTE_UNAVAILABLE: + return + if value == ActionEscalationReason.DELIVERY_FAILED: + return + if value == ActionEscalationReason.EFFECT_UNCONFIRMED: + return + if value == ActionEscalationReason.SUSPECTED_NOOP: + return + if value == ActionEscalationReason.PERMISSION_REQUIRED: + return + raise ValueError(value) + + @staticmethod + def write(value, buf): + if value == ActionEscalationReason.ROUTE_UNAVAILABLE: + buf.write_i32(1) + if value == ActionEscalationReason.DELIVERY_FAILED: + buf.write_i32(2) + if value == ActionEscalationReason.EFFECT_UNCONFIRMED: + buf.write_i32(3) + if value == ActionEscalationReason.SUSPECTED_NOOP: + buf.write_i32(4) + if value == ActionEscalationReason.PERMISSION_REQUIRED: + buf.write_i32(5) + + + +@dataclass +class ActionEscalation: + def __init__(self, *, target:ActionEscalationTarget, reason:ActionEscalationReason): + self.target = target + self.reason = reason + + + + + def __str__(self): + return "ActionEscalation(target={}, reason={})".format(self.target, self.reason) + def __eq__(self, other): + if self.target != other.target: + return False + if self.reason != other.reason: + return False + return True + +class _UniffiFfiConverterTypeActionEscalation(_UniffiConverterRustBuffer): + @staticmethod + def read(buf): + return ActionEscalation( + target=_UniffiFfiConverterTypeActionEscalationTarget.read(buf), + reason=_UniffiFfiConverterTypeActionEscalationReason.read(buf), + ) + + @staticmethod + def check_lower(value): + _UniffiFfiConverterTypeActionEscalationTarget.check_lower(value.target) + _UniffiFfiConverterTypeActionEscalationReason.check_lower(value.reason) + + @staticmethod + def write(value, buf): + _UniffiFfiConverterTypeActionEscalationTarget.write(value.target, buf) + _UniffiFfiConverterTypeActionEscalationReason.write(value.reason, buf) + + + + + + +class ActionEvidenceKind(enum.Enum): + + VALUE_READBACK = 0 + + WINDOW_CHANGE = 1 + + + +class _UniffiFfiConverterTypeActionEvidenceKind(_UniffiConverterRustBuffer): + @staticmethod + def read(buf): + variant = buf.read_i32() + if variant == 1: + return ActionEvidenceKind.VALUE_READBACK + if variant == 2: + return ActionEvidenceKind.WINDOW_CHANGE + raise InternalError("Raw enum value doesn't match any cases") + + @staticmethod + def check_lower(value): + if value == ActionEvidenceKind.VALUE_READBACK: + return + if value == ActionEvidenceKind.WINDOW_CHANGE: + return + raise ValueError(value) + + @staticmethod + def write(value, buf): + if value == ActionEvidenceKind.VALUE_READBACK: + buf.write_i32(1) + if value == ActionEvidenceKind.WINDOW_CHANGE: + buf.write_i32(2) + + + +@dataclass +class ActionEvidence: + def __init__(self, *, kind:ActionEvidenceKind): + self.kind = kind + + + + + def __str__(self): + return "ActionEvidence(kind={})".format(self.kind) + def __eq__(self, other): + if self.kind != other.kind: + return False + return True + +class _UniffiFfiConverterTypeActionEvidence(_UniffiConverterRustBuffer): + @staticmethod + def read(buf): + return ActionEvidence( + kind=_UniffiFfiConverterTypeActionEvidenceKind.read(buf), + ) + + @staticmethod + def check_lower(value): + _UniffiFfiConverterTypeActionEvidenceKind.check_lower(value.kind) + + @staticmethod + def write(value, buf): + _UniffiFfiConverterTypeActionEvidenceKind.write(value.kind, buf) + + + + + + +class ActionEffect(enum.Enum): + + CONFIRMED = 0 + + PARTIAL = 1 + + UNVERIFIABLE = 2 + + SUSPECTED_NOOP = 3 + + REFUSED = 4 + + + +class _UniffiFfiConverterTypeActionEffect(_UniffiConverterRustBuffer): + @staticmethod + def read(buf): + variant = buf.read_i32() + if variant == 1: + return ActionEffect.CONFIRMED + if variant == 2: + return ActionEffect.PARTIAL + if variant == 3: + return ActionEffect.UNVERIFIABLE + if variant == 4: + return ActionEffect.SUSPECTED_NOOP + if variant == 5: + return ActionEffect.REFUSED + raise InternalError("Raw enum value doesn't match any cases") + + @staticmethod + def check_lower(value): + if value == ActionEffect.CONFIRMED: + return + if value == ActionEffect.PARTIAL: + return + if value == ActionEffect.UNVERIFIABLE: + return + if value == ActionEffect.SUSPECTED_NOOP: + return + if value == ActionEffect.REFUSED: + return + raise ValueError(value) + + @staticmethod + def write(value, buf): + if value == ActionEffect.CONFIRMED: + buf.write_i32(1) + if value == ActionEffect.PARTIAL: + buf.write_i32(2) + if value == ActionEffect.UNVERIFIABLE: + buf.write_i32(3) + if value == ActionEffect.SUSPECTED_NOOP: + buf.write_i32(4) + if value == ActionEffect.REFUSED: + buf.write_i32(5) + + + + + + + + +class ActionRoute(enum.Enum): + + ACCESSIBILITY = 0 + + SYNTHETIC_EVENTS = 1 + + GLOBAL_INPUT = 2 + + DOM = 3 + + TRUSTED_INPUT = 4 + + + +class _UniffiFfiConverterTypeActionRoute(_UniffiConverterRustBuffer): + @staticmethod + def read(buf): + variant = buf.read_i32() + if variant == 1: + return ActionRoute.ACCESSIBILITY + if variant == 2: + return ActionRoute.SYNTHETIC_EVENTS + if variant == 3: + return ActionRoute.GLOBAL_INPUT + if variant == 4: + return ActionRoute.DOM + if variant == 5: + return ActionRoute.TRUSTED_INPUT + raise InternalError("Raw enum value doesn't match any cases") + + @staticmethod + def check_lower(value): + if value == ActionRoute.ACCESSIBILITY: + return + if value == ActionRoute.SYNTHETIC_EVENTS: + return + if value == ActionRoute.GLOBAL_INPUT: + return + if value == ActionRoute.DOM: + return + if value == ActionRoute.TRUSTED_INPUT: + return + raise ValueError(value) + + @staticmethod + def write(value, buf): + if value == ActionRoute.ACCESSIBILITY: + buf.write_i32(1) + if value == ActionRoute.SYNTHETIC_EVENTS: + buf.write_i32(2) + if value == ActionRoute.GLOBAL_INPUT: + buf.write_i32(3) + if value == ActionRoute.DOM: + buf.write_i32(4) + if value == ActionRoute.TRUSTED_INPUT: + buf.write_i32(5) + + + +class _UniffiFfiConverterOptionalTypeActionDelivery(_UniffiConverterRustBuffer): + @classmethod + def check_lower(cls, value): + if value is not None: + _UniffiFfiConverterTypeActionDelivery.check_lower(value) + + @classmethod + def write(cls, value, buf): + if value is None: + buf.write_u8(0) + return + + buf.write_u8(1) + _UniffiFfiConverterTypeActionDelivery.write(value, buf) + + @classmethod + def read(cls, buf): + flag = buf.read_u8() + if flag == 0: + return None + elif flag == 1: + return _UniffiFfiConverterTypeActionDelivery.read(buf) + else: + raise InternalError("Unexpected flag byte for optional type") + +class _UniffiFfiConverterSequenceTypeActionEvidence(_UniffiConverterRustBuffer): + @classmethod + def check_lower(cls, value): + for item in value: + _UniffiFfiConverterTypeActionEvidence.check_lower(item) + + @classmethod + def write(cls, value, buf): + items = len(value) + buf.write_i32(items) + for item in value: + _UniffiFfiConverterTypeActionEvidence.write(item, buf) + + @classmethod + def read(cls, buf): + count = buf.read_i32() + if count < 0: + raise InternalError("Unexpected negative sequence length") + + return [ + _UniffiFfiConverterTypeActionEvidence.read(buf) for i in range(count) + ] + +class _UniffiFfiConverterOptionalSequenceTypeActionEvidence(_UniffiConverterRustBuffer): + @classmethod + def check_lower(cls, value): + if value is not None: + _UniffiFfiConverterSequenceTypeActionEvidence.check_lower(value) + + @classmethod + def write(cls, value, buf): + if value is None: + buf.write_u8(0) + return + + buf.write_u8(1) + _UniffiFfiConverterSequenceTypeActionEvidence.write(value, buf) + + @classmethod + def read(cls, buf): + flag = buf.read_u8() + if flag == 0: + return None + elif flag == 1: + return _UniffiFfiConverterSequenceTypeActionEvidence.read(buf) + else: + raise InternalError("Unexpected flag byte for optional type") + +class _UniffiFfiConverterOptionalTypeActionEscalation(_UniffiConverterRustBuffer): + @classmethod + def check_lower(cls, value): + if value is not None: + _UniffiFfiConverterTypeActionEscalation.check_lower(value) + + @classmethod + def write(cls, value, buf): + if value is None: + buf.write_u8(0) + return + + buf.write_u8(1) + _UniffiFfiConverterTypeActionEscalation.write(value, buf) + + @classmethod + def read(cls, buf): + flag = buf.read_u8() + if flag == 0: + return None + elif flag == 1: + return _UniffiFfiConverterTypeActionEscalation.read(buf) + else: + raise InternalError("Unexpected flag byte for optional type") + +@dataclass +class ActionResult: + def __init__(self, *, effect:ActionEffect, route:ActionRoute, delivery:typing.Optional[ActionDelivery], evidence:typing.Optional[typing.List[ActionEvidence]], escalation:typing.Optional[ActionEscalation]): + self.effect = effect + self.route = route + self.delivery = delivery + self.evidence = evidence + self.escalation = escalation + + + + + def __str__(self): + return "ActionResult(effect={}, route={}, delivery={}, evidence={}, escalation={})".format(self.effect, self.route, self.delivery, self.evidence, self.escalation) + def __eq__(self, other): + if self.effect != other.effect: + return False + if self.route != other.route: + return False + if self.delivery != other.delivery: + return False + if self.evidence != other.evidence: + return False + if self.escalation != other.escalation: + return False + return True + +class _UniffiFfiConverterTypeActionResult(_UniffiConverterRustBuffer): + @staticmethod + def read(buf): + return ActionResult( + effect=_UniffiFfiConverterTypeActionEffect.read(buf), + route=_UniffiFfiConverterTypeActionRoute.read(buf), + delivery=_UniffiFfiConverterOptionalTypeActionDelivery.read(buf), + evidence=_UniffiFfiConverterOptionalSequenceTypeActionEvidence.read(buf), + escalation=_UniffiFfiConverterOptionalTypeActionEscalation.read(buf), + ) + + @staticmethod + def check_lower(value): + _UniffiFfiConverterTypeActionEffect.check_lower(value.effect) + _UniffiFfiConverterTypeActionRoute.check_lower(value.route) + _UniffiFfiConverterOptionalTypeActionDelivery.check_lower(value.delivery) + _UniffiFfiConverterOptionalSequenceTypeActionEvidence.check_lower(value.evidence) + _UniffiFfiConverterOptionalTypeActionEscalation.check_lower(value.escalation) + + @staticmethod + def write(value, buf): + _UniffiFfiConverterTypeActionEffect.write(value.effect, buf) + _UniffiFfiConverterTypeActionRoute.write(value.route, buf) + _UniffiFfiConverterOptionalTypeActionDelivery.write(value.delivery, buf) + _UniffiFfiConverterOptionalSequenceTypeActionEvidence.write(value.evidence, buf) + _UniffiFfiConverterOptionalTypeActionEscalation.write(value.escalation, buf) + class _UniffiFfiConverterFloat64(_UniffiConverterPrimitiveFloat): @staticmethod def read(buf): @@ -1004,44 +1640,6 @@ def read(cls, buf): else: raise InternalError("Unexpected flag byte for optional type") -class _UniffiFfiConverterUInt32(_UniffiConverterPrimitiveInt): - CLASS_NAME = "u32" - VALUE_MIN = 0 - VALUE_MAX = 2**32 - - @staticmethod - def read(buf): - return buf.read_u32() - - @staticmethod - def write(value, buf): - buf.write_u32(value) - -class _UniffiFfiConverterOptionalUInt32(_UniffiConverterRustBuffer): - @classmethod - def check_lower(cls, value): - if value is not None: - _UniffiFfiConverterUInt32.check_lower(value) - - @classmethod - def write(cls, value, buf): - if value is None: - buf.write_u8(0) - return - - buf.write_u8(1) - _UniffiFfiConverterUInt32.write(value, buf) - - @classmethod - def read(cls, buf): - flag = buf.read_u8() - if flag == 0: - return None - elif flag == 1: - return _UniffiFfiConverterUInt32.read(buf) - else: - raise InternalError("Unexpected flag byte for optional type") - @dataclass class ClickInput: def __init__(self, *, x:float, y:float, scope:DesktopScope, session:typing.Optional[str], button:typing.Optional[ClickButton], count:typing.Optional[int]): @@ -3796,6 +4394,12 @@ def write(value, buf): __all__ = [ "InternalError", + "ActionDeliveryMode", + "ActionEscalationTarget", + "ActionEscalationReason", + "ActionEvidenceKind", + "ActionEffect", + "ActionRoute", "DesktopScope", "ClickButton", "CursorReducedMotion", @@ -3808,6 +4412,10 @@ def write(value, buf): "CaptureScope", "EffectiveScope", "Platform", + "ActionDelivery", + "ActionEscalation", + "ActionEvidence", + "ActionResult", "BoundsExpectation", "ClickInput", "CursorMotionOutput", diff --git a/libs/cua-driver/python/tests/test_uniffi_loader.py b/libs/cua-driver/python/tests/test_uniffi_loader.py index 1dd98bfa90..b470158243 100644 --- a/libs/cua-driver/python/tests/test_uniffi_loader.py +++ b/libs/cua-driver/python/tests/test_uniffi_loader.py @@ -58,7 +58,7 @@ def test_generated_python_embedded_host_owns_the_rust_lifecycle(self) -> None: if request["method"] == "metadata": result = { "driver_version": "0.10.0", - "contract_version": "0.3.0", + "contract_version": "0.4.0", "tools_list_schema_version": "1", "capability_version": "1", "mcp_protocol_version": "2025-06-18", @@ -104,10 +104,16 @@ async def scenario() -> str: def test_generated_python_sdk_calls_the_rust_daemon_interface(self) -> None: import cua_driver from cua_driver import ( + ActionEffect, + ActionRoute, + ClickButton, + ClickInput, CuaDriver, + DesktopScope, EffectiveScope, StatePredicate, StartSessionOutput, + VerificationStatus, VerifyStateInput, WindowPredicate, ) @@ -124,11 +130,11 @@ def test_generated_python_sdk_calls_the_rust_daemon_interface(self) -> None: socket_path = str(Path(directory) / "driver.sock") listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) listener.bind(socket_path) - listener.listen(2) + listener.listen(4) captured: list[dict[str, object]] = [] def serve() -> None: - for _ in range(2): + while len(captured) < 2: connection, _ = listener.accept() with connection: line = connection.makefile("r", encoding="utf-8").readline() @@ -136,7 +142,7 @@ def serve() -> None: if request["method"] == "metadata": result = { "driver_version": "0.12.6", - "contract_version": "0.3.0", + "contract_version": "0.4.0", "tools_list_schema_version": "1", "capability_version": "1", "mcp_protocol_version": "2025-06-18", @@ -145,6 +151,20 @@ def serve() -> None: } else: captured.append(request) + if request["name"] == "verify_state": + structured = { + "status": "satisfied", + "stable": True, + "elapsed_ms": 12, + "samples": 2, + "predicates": [], + } + else: + structured = { + "effect": "unverifiable", + "route": "global_input", + "delivery": {"mode": "not_applicable"}, + } result = { "content": [ {"type": "text", "text": "python ffi"}, @@ -154,7 +174,7 @@ def serve() -> None: "data": "cG5n", }, ], - "structuredContent": {"verified": True}, + "structuredContent": structured, "isError": False, } response = {"ok": True, "result": result} @@ -181,7 +201,7 @@ def serve() -> None: "verify_state", } self.assertTrue(all(hasattr(driver, name) for name in expected_methods)) - result = asyncio.run( + verification_result = asyncio.run( driver.verify_state( VerifyStateInput( pid=123, @@ -199,12 +219,31 @@ def serve() -> None: ) ) ) + action_result = asyncio.run( + driver.click( + ClickInput( + x=12.0, + y=34.0, + scope=DesktopScope.DESKTOP, + session="python-run", + button=ClickButton.LEFT, + count=1, + ) + ) + ) server.join(timeout=5) listener.close() - self.assertEqual(result.text, "python ffi") - self.assertEqual(result.images[0].mime_type, "image/png") - self.assertTrue(result.verified) + self.assertEqual(verification_result.text, "python ffi") + self.assertEqual(verification_result.images[0].mime_type, "image/png") + self.assertIsNone(verification_result.action) + self.assertEqual( + verification_result.verification.status, VerificationStatus.SATISFIED + ) + self.assertIsNone(action_result.verification) + self.assertEqual(action_result.action.effect, ActionEffect.UNVERIFIABLE) + self.assertEqual(action_result.action.route, ActionRoute.GLOBAL_INPUT) + self.assertFalse(hasattr(action_result, "verified")) self.assertEqual(captured[0]["name"], "verify_state") self.assertEqual( captured[0]["args"], @@ -219,6 +258,18 @@ def serve() -> None: }, ) self.assertEqual(captured[0]["client_kind"], "python_sdk") + self.assertEqual(captured[1]["name"], "click") + self.assertEqual( + captured[1]["args"], + { + "x": 12.0, + "y": 34.0, + "scope": "desktop", + "session": "python-run", + "button": "left", + "count": 1, + }, + ) def test_generated_python_sdk_can_own_the_runtime_in_process(self) -> None: from cua_driver import CuaDriver, DriverExecutionMode diff --git a/libs/cua-driver/rust/Skills/cua-driver/LINUX.md b/libs/cua-driver/rust/Skills/cua-driver/LINUX.md index 725b2ecee2..4034d41007 100644 --- a/libs/cua-driver/rust/Skills/cua-driver/LINUX.md +++ b/libs/cua-driver/rust/Skills/cua-driver/LINUX.md @@ -60,19 +60,19 @@ remote desktop session, or when repeated action-scoped activation prevents the remote surface from accepting input. On X11 it uses persistent `_NET_ACTIVE_WINDOW` activation (the `wmctrl -a` equivalent). -**Read-back / `verified`** — `type_text` reports `{verified}`: the AT-SPI -`EditableText.insertText` path (`path:"ax"`) is the **driver-verifiable** rung -(the a11y layer confirms the insert into the widget model) → `verified:true`; -keystroke / XSendEvent / XTest / foreground rungs are not read-back-confirmed -→ `verified:false` (confirm via screenshot). Mirrors the macOS/Windows verdict. - -**`effect` / `escalation`** — alongside `verified`, action responses carry the -cross-platform `effect` (`confirmed` / `unverifiable` / `suspected_noop`) and, -when you should change rung, `escalation:{recommended, reason}`. See `SKILL.md` -→ behavior matrix. On a standard Wayland compositor the Linux-specific value -of `recommended` is **`foreground`** (raw background pixels cannot target an -unfocused window); the opt-in nested compositor is a separate environment. Use -**`px` on X11** (an element px action — background pixel click — lands via +**Read-back / action effect** — AT-SPI `EditableText.insertText` can return +`effect:"confirmed"` with `evidence:[{"kind":"value_readback"}]` when the +accessibility layer reads the inserted value back from the widget model. +Keystroke / XSendEvent / XTest / foreground rungs return +`effect:"unverifiable"` unless another publishable readback exists. Confirm +those through `verify_state` or multimodal reading. + +**Escalation** — action responses use the cross-platform closed +`escalation:{target, reason}` shape. See `SKILL.md` → behavior matrix. On a +standard Wayland compositor the Linux-specific target is **`foreground`** +(raw background pixels cannot target an unfocused window); the opt-in nested +compositor is a separate environment. Use **`pixel` on X11** (an element px +action — background pixel click — lands via AT-SPI `do_action`-at-point off the screenshot already in the snapshot — the matrix below). @@ -162,17 +162,15 @@ bridge isn't up / the daemon isn't on the session bus". ## The validated modality matrix (X11 / XFCE) -Each input rung, and whether the **driver itself** can confirm it (vs. only -the caller agent confirming via screenshot — the same honesty line macOS and -Windows draw): +Each input rung and its stable public route: -| Modality | `delivery_mode` | Path reported | Driver-verifiable? | +| Modality | `delivery_mode` | `route` | Postcondition proof | |---|---|---|---| -| Element click (`element_index`) | `background` | `x11_atspi` (AT-SPI `do_action`) | ✅ a11y action | -| **element px action (x,y)** | `background` | `x11_atspi` (AT-SPI `do_action`-at-point) for AX apps; else MPX `x11_pixel` | ✅ when AT-SPI-at-point lands; else best-effort | -| Pixel (px) click, escalated | `foreground` | `x11_pixel_fg` (EWMH activate → inject → restore) | ❌ confirm via screenshot | -| `type_text` into editable | `background` | `ax` (AT-SPI `insertText`) | ✅ `verified:true` | -| `type_text`, non-editable focus | `background`/`foreground` | `key_events` / `key_events_fg` | ❌ confirm via screenshot | +| Element click (`element_index`) | `background` | `accessibility` | Use `verify_state`; invocation alone is not confirmation | +| **element px action (x,y)** | `background` | `accessibility` when AT-SPI-at-point lands, otherwise `global_input` | Use `verify_state` or multimodal reading | +| Pixel (px) click, escalated | `foreground` | `global_input` | Use `verify_state` or multimodal reading | +| `type_text` into editable | `background` | `accessibility` | `confirmed` only with `value_readback` evidence | +| `type_text`, non-editable focus | `background`/`foreground` | `synthetic_events` or `global_input` | Use `verify_state` or multimodal reading | **A background element px action does land on X11** — for an AX-exposing app it takes the focus-free AT-SPI `do_action`-at-point path (`x11_atspi`), exactly diff --git a/libs/cua-driver/rust/Skills/cua-driver/MACOS.md b/libs/cua-driver/rust/Skills/cua-driver/MACOS.md index 7917c97371..11320fce7b 100644 --- a/libs/cua-driver/rust/Skills/cua-driver/MACOS.md +++ b/libs/cua-driver/rust/Skills/cua-driver/MACOS.md @@ -376,7 +376,8 @@ so a naive read-back "confirms" a value that isn't really there. The driver **detects Electron and refuses to trust that echo**: an AX-path `type_text` on an Electron app returns `effect:"unverifiable"` + -`escalation:{recommended:"px"}`, **never** a false `verified:true`. +`escalation:{target:"pixel",reason:"effect_unconfirmed"}`, **never** a +false `effect:"confirmed"`. (On Catalyst the AX value reads back unreadable, so it reports unverified too.) Bottom line: on these surfaces **do not trust the AX confirm — the screenshot in the same response is the only truth.** diff --git a/libs/cua-driver/rust/Skills/cua-driver/SKILL.md b/libs/cua-driver/rust/Skills/cua-driver/SKILL.md index 73094232da..27dd800c56 100644 --- a/libs/cua-driver/rust/Skills/cua-driver/SKILL.md +++ b/libs/cua-driver/rust/Skills/cua-driver/SKILL.md @@ -249,6 +249,38 @@ For postconditions not expressible by `verify_state`, take a fresh state snapshot and let the agent judge the tree and/or image explicitly. This applies to pixel clicks and desktop actions too. +### Read action facts without confusing them with task success + +A successful action returns `effect` and `route`, with optional typed +`delivery`, `evidence`, and `escalation`. These fields describe the actuator; +they do not declare the user's task complete. + +- `confirmed` means the driver has publishable value readback or window-change + evidence for that action. +- `partial` means only `delivery.delivered_count` was delivered. +- `unverifiable` means the driver cannot prove the effect. +- `suspected_noop` means available evidence suggests no useful change. +- `refused` means the selected route deliberately did not deliver. + +The route vocabulary is intentionally cross-platform: +`accessibility`, `synthetic_events`, `global_input`, `dom`, and +`trusted_input`. Do not branch on private OS transport names. + +An optional escalation is a harness instruction, never an automatic retry: + +- `pixel`: refresh visual state and choose an exact pixel target; +- `foreground`: explicitly select foreground delivery if session policy allows; +- `page`: bind the native window to a supported browser page route; +- `session`: prepare or explicitly widen the session only when policy permits. + +Branch on the closed reason vocabulary: +`route_unavailable`, `delivery_failed`, `effect_unconfirmed`, +`suspected_noop`, and `permission_required`. + +After any action, keep using `verify_state` or a fresh state snapshot for the +actual task postcondition. The multimodal harness owns visual reading and the +decision to stop, retry, or advance the ladder. + ## Choose capture scope when the session starts `capture_scope` is a per-session policy, not persistent configuration. Declare @@ -362,8 +394,9 @@ for Chromium/Electron inputs the AX path can't reach, and **Typing default (the ladder).** Call `type_text` directly with `element_index` (ax) — it targets the field, no pre-click. On Electron/Catalyst the AX layer echoes the write without rendering it, -so the driver returns `effect:"unverifiable"` + `escalation:"px"` -there (never a false `verified:true`) — follow it, and cross-check the +so the driver returns `effect:"unverifiable"` with +`escalation.target:"pixel"` there (never a false `effect:"confirmed"`) — +follow it, and cross-check the screenshot in the response (the only ground truth). Escalate to the px form — `type_text({pid, window_id, x, y, text})` — which pixel-clicks to focus, then types. **If the target control is closed** (a search @@ -378,37 +411,25 @@ pixel counterpart is a `click`/`drag` on the control, not a "set value at a pixel." So: text → `type_text` (ax+px); non-text control values → `set_value`; pixel-manipulate a control → `click`/`drag`. -**Action responses carry an effect/escalation verdict** - -Every action response keeps `verified` (did the driver read back a -post-condition?) and adds two machine-readable fields so you know -whether — and where — to climb the ladder: - -- `effect`: one of - - `"confirmed"` — the driver read back the effect (`ax` rung only). - - `"unverifiable"` — dispatched, but the driver has no handle to - read back (every `px`/CGEvent path; foreground rung). **You** - confirm it off the screenshot — it is not a failure. - - `"suspected_noop"` — the `ax` action **likely did nothing** (the - element didn't actually advertise the action, or you hit a passive - label). This is the explicit **"cross to `px`"** trigger. -- `escalation`: `{recommended, reason}` when the driver thinks you - should change rung — - - `"px"` — the element isn't really actionable in `ax`; do an - **element px action** off the screenshot you already have. - - `"foreground"` — a background insert/click was _dropped_ on - delivery; re-call the same action with `delivery_mode:"foreground"`. +**Action responses carry closed action facts** + +Use the `effect`, `route`, optional `delivery`, `evidence`, and +`escalation` rules in “Read action facts without confusing them with task +success” above. The old `verified`, `path`, coordinates, scope, and +`escalation.recommended` response fields no longer exist. +The full wire contract and 0.14 migration notes are in +`../../../docs/action-result-contract.md`. `get_window_state` itself, when the AX tree comes back empty (a non-AX surface like Electron/Chromium/canvas), returns `degraded: true` -**plus the same `escalation` hint** — normally pointing at `px` (you +plus an observation-specific escalation hint — normally pointing at pixels (you still have the screenshot from the same call to click off). -**Platform nuance for `escalation`.** On **Wayland** an unfocused +**Platform nuance for action escalation.** On **Wayland** an unfocused window cannot be pixel-targeted in the background (libei → -`background_unavailable`), so there the recommendation is -**`foreground`, not `px`**. macOS, X11, and most Windows surfaces -_can_ pixel-target in the background, so they recommend `px`. See +`background_unavailable`), so the action target is +**`foreground`, not `pixel`**. macOS, X11, and most Windows surfaces +can pixel-target in the background, so they target `pixel`. See `LINUX.md` / `WINDOWS.md`. ## The verify-then-escalate ladder (algorithm) @@ -437,7 +458,7 @@ if check.status == "unknown" and check has an image: # escalate only on a real signal if resp.effect == "suspected_noop" - or resp.escalation.recommended == "px" + or resp.escalation.target == "pixel" or get_window_state.degraded # empty tree → non-AX surface or check.status != "satisfied" or the tree looks wrong vs the screenshot: # e.g. an h:1 / off-viewport row @@ -456,7 +477,7 @@ get_browser_state(session, pid, window_id) # verify with fresh refs if it landed: done # Rung 3 — background delivery was dropped (insert/click never arrived) -if resp.escalation.recommended == "foreground" +if resp.escalation.target == "foreground" or the px action still did nothing: re-call the same action with delivery_mode:"foreground" # on Wayland this is the ONLY escalation — px-bg can't target an @@ -712,8 +733,10 @@ Two consequences for callers: prior frontmost: the explicit last resort when a background attempt didn't land. **`foreground` is a reaction, never a prediction.** Always fire the `background` default first and let the driver tell you it - can't (a `background_unavailable` error or `escalation.recommended == -"foreground"`) — or observe a verified no-op — _before_ you escalate. + can't (a `background_unavailable` error with + `escalation.recommended == "foreground"`, or a successful action result + with `escalation.target == "foreground"`) — or observe a confirmed no-op — + _before_ you escalate. Do **not** reason "it's a GTK/Chromium/Electron app, so background will drop, so I'll front up-front": the toolkit lists in the tool schemas are the _driver's_ internal detectors, not a checklist for you to front @@ -841,12 +864,12 @@ Switch to an **element px action** only on a real signal: the action response carried `effect:"suspected_noop"`, verification returned `unsatisfied`/`unknown`, the snapshot came back `degraded` (empty tree → non-AX surface), the tree looks unchanged/unreadable or disagrees with the screenshot, or -`escalation.recommended` points you there (`px`). That's the +`escalation.target` points you there (`pixel`). That's the verify-then-escalate ladder in the behavior-matrix section. If the tree is unchanged AND the screenshot confirms nothing moved, the action likely failed silently — **tell the user what you attempted and what you observed**, don't paper over with "done" language (and consider -`delivery_mode:"foreground"` when `escalation.recommended == +`delivery_mode:"foreground"` when `escalation.target == "foreground"`). Agents that skip this step report success on silently-dropped actions — the single most common failure mode. diff --git a/libs/cua-driver/rust/Skills/cua-driver/WINDOWS.md b/libs/cua-driver/rust/Skills/cua-driver/WINDOWS.md index 1cf11a0e0b..a9a00050bc 100644 --- a/libs/cua-driver/rust/Skills/cua-driver/WINDOWS.md +++ b/libs/cua-driver/rust/Skills/cua-driver/WINDOWS.md @@ -89,11 +89,11 @@ costlier path; only use it for surfaces with no UIA peer. } ``` -The `escalation` field is the same machine-readable hint the action -responses carry (see `SKILL.md` → behavior matrix). On Windows the -recommendation is `"foreground"` because the dropped event needs the fronting -rung. (Contrast macOS / X11, where a background px click can still land -in the background, so there the hint is `px`.) +Errors retain their diagnostic `escalation.recommended` hint. Successful +action results use the narrower `escalation.target` contract instead (see +`SKILL.md` → behavior matrix). On Windows this error recommendation is +`"foreground"` because the dropped event needs the fronting rung. (Contrast +macOS / X11, where a background pixel click can still land in the background.) The normal flow when an agent gets that error: @@ -559,9 +559,9 @@ Canonical multi-step workflow: effect. The UIA tree change (new value, new window, disappeared menu, disabled button, etc.) is your evidence that the action registered. **Especially important on Windows** because the - layered click path can return "✅ Posted click to pid X" even when - the click did nothing (UWP target, PostMessage silently no-ops): - the success message reports the mechanism, not the outcome. Only + layered click path can return `effect:"unverifiable"` after + PostMessage even when the click did nothing (UWP silently no-ops): + the action result reports the route, not the task outcome. Only the re-snapshot tells you if the state changed. ## Click semantics on Windows @@ -594,9 +594,9 @@ Properties: child HWND** when the cached element doesn't expose `InvokePattern` (most edit fields, custom-drawn widgets, non-actionable elements). The fallback works for plain Win32 but - silently no-ops on UWP. The success message tells you which path - ran: `"✅ Performed UIA Invoke on [N] ..."` vs `"✅ Performed -PostMessage click on [N] ..."`. + silently no-ops on UWP. Read the closed action `route`: + `accessibility` means UIA/MSAA and `synthetic_events` means the + targeted event fallback. Do not parse the human-readable text. This is the right path for **any** "click button N" / "click menu item X" / "click checkbox Y" intent. @@ -657,9 +657,8 @@ steal focus from whatever the user is doing. `button: "right"` and `count > 1` **skip the UIA Invoke step** and go directly through the PostMessage path. Reason: UIA has no clean by-coord equivalent of `ShowContextMenu`, and `Invoke()` is single- -fire by definition. The success message will read -`"✅ Posted click/double-click/triple-click to pid X"` (PostMessage -path) regardless of the target's UWP-ness — this is expected and +fire by definition. The action result reports `route:"synthetic_events"` +regardless of the target's UWP-ness — this is expected and **will not work for UWP context menus**. To open a UWP context menu, prefer `hotkey({pid, keys: ["shift", "f10"]})` against the focused UWP element. @@ -758,8 +757,8 @@ typed browser tools yet. - **Calc display stuck at "0" after pixel clicks** — the (x,y) UIA hit-test missed and PostMessage fell through (PostMessage is a silent no-op on UWP). Switch to `element_index` mode. Symptom: - success messages say `Posted click to pid N` instead of - `Performed UIA Invoke at (sx,sy) ...`. + the action result reports `route:"synthetic_events"` instead of + `route:"accessibility"`. - **LibreOffice (VCL) `type_text` / `hotkey` reported success but nothing happened** — VCL/SAL apps route accelerators through `TranslateAccelerator` (reads `GetKeyState`, which PostMessage doesn't @@ -769,21 +768,22 @@ typed browser tools yet. blind success: - **`hotkey` / `press_key`** (keystroke + key-combo): `delivery_mode:"background"` surfaces a `background_unavailable` error for VCL. - - **`type_text`** does a **UIA read-back** and returns a three-way `verify` - in structured output: `confirmed` (✅, value reflects the text), - `unchanged` (📨, read OK but value didn't change → likely dropped, retry - foreground), or `unreadable` (✅ "delivered, not verified"). **Pass an + - **`type_text`** does a **UIA read-back** and returns the shared + `ActionResult`: `effect:"confirmed"` with `evidence:[{"kind": + "value_readback"}]` when the value reflects the text, and + `effect:"unverifiable"` when the value is unchanged or unreadable. Use the + optional `escalation` to choose the next rung. **Pass an `element_index`** for reliable verification: the read-back then reads _that specific element_ by handle (ValuePattern → TextPattern), which is - **focus-independent** — it reaches `confirmed`/`unchanged` whether or not + **focus-independent** — it can confirm or disprove a change whether or not the target is foreground. (Verified live against the WPF harness: typed - via element_index, read back `confirmed`, value independently present in + via element_index, read back confirmed, value independently present in the next snapshot — app never fronted.) **Without** an element_index it falls back to system-wide `GetFocusedElement`, which on Windows only resolves when the target is the **foreground** app (no per-app `AXFocusedUIElement` like macOS); a backgrounded target then reads - `unreadable` even when the text actually landed — so `unreadable` is NOT a - failure signal, verify via screenshot if it matters. + an unverifiable result even when the text actually landed — so it is NOT a + failure signal; call `verify_state` or inspect a fresh screenshot. Escalate to `delivery_mode:"foreground"` for both (SendInput Unicode / accelerator). **But** foreground needs the swap to actually land — if the daemon lacks UIAccess and `bring_to_front` returns `landed_on_target:false` diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/desktop.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/desktop.rs index 03c7c8a2ed..ecbcca7e5a 100644 --- a/libs/cua-driver/rust/crates/cua-driver-contract/src/desktop.rs +++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/desktop.rs @@ -8,11 +8,11 @@ //! do not replace the richer platform-owned runtime schemas. use crate::{ - ClickInput, ClickOutput, CursorAction, CursorPositionOutput, CursorSemantics, - DesktopActionOutput, DesktopStateOutput, DragInput, GetCursorPositionInput, - GetDesktopStateInput, GetScreenSizeInput, HotkeyInput, MoveCursorInput, MoveCursorOutput, - Platform, PressKeyInput, SchemaMode, ScreenSizeOutput, ScrollInput, ToolAnnotations, - ToolContract, ToolInput, ToolOutput, TypeTextInput, + ActionResult, ClickInput, CursorAction, CursorPositionOutput, CursorSemantics, + DesktopStateOutput, DragInput, GetCursorPositionInput, GetDesktopStateInput, + GetScreenSizeInput, HotkeyInput, MoveCursorInput, Platform, PressKeyInput, SchemaMode, + ScreenSizeOutput, ScrollInput, ToolAnnotations, ToolContract, ToolInput, ToolOutput, + TypeTextInput, }; const ALL_PLATFORMS: [Platform; 3] = [Platform::Macos, Platform::Windows, Platform::Linux]; @@ -101,7 +101,7 @@ fn get_cursor_position() -> ToolContract { } fn move_cursor() -> ToolContract { - contract::( + contract::( "move_cursor", "Move the real OS pointer in get_desktop_state coordinates.", &["agent_cursor.move", "input.pointer.move"], @@ -116,7 +116,7 @@ fn move_cursor() -> ToolContract { } fn click() -> ToolContract { - contract::( + contract::( "click", "Click an absolute point in get_desktop_state coordinates without targeting a window.", &[ @@ -135,7 +135,7 @@ fn click() -> ToolContract { } fn drag() -> ToolContract { - contract::( + contract::( "drag", "Drag between two absolute points in get_desktop_state coordinates.", &["input.pointer.drag"], @@ -150,7 +150,7 @@ fn drag() -> ToolContract { } fn scroll() -> ToolContract { - contract::( + contract::( "scroll", "Scroll at an absolute point in get_desktop_state coordinates.", &["input.pointer.scroll", "accessibility.element_tokens"], @@ -165,7 +165,7 @@ fn scroll() -> ToolContract { } fn type_text() -> ToolContract { - contract::( + contract::( "type_text", "Type text into the current foreground desktop application.", &[ @@ -184,7 +184,7 @@ fn type_text() -> ToolContract { } fn press_key() -> ToolContract { - contract::( + contract::( "press_key", "Press one key, with optional modifiers, in the foreground desktop application.", &["input.keyboard.press", "accessibility.element_tokens"], @@ -199,7 +199,7 @@ fn press_key() -> ToolContract { } fn hotkey() -> ToolContract { - contract::( + contract::( "hotkey", "Press a key chord in the foreground desktop application.", &["input.keyboard.hotkey"], diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs index 0538fa2517..8c34e1c01d 100644 --- a/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs +++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs @@ -35,11 +35,13 @@ pub use inputs::{ TypeTextInput, }; pub use outputs::{ - ClickOutput, CursorMotionOutput, CursorPointOutput, CursorPositionOutput, CursorThemeOutput, - CursorVisualOutput, DesktopActionOutput, DesktopStateOutput, EffectiveScope, EndSessionOutput, - GetAgentCursorStateOutput, MoveCursorOutput, ScreenSizeOutput, SessionStateOutput, - SetAgentCursorEnabledOutput, SetAgentCursorMotionOutput, SetAgentCursorThemeOutput, - StartSessionOutput, ToolOutput, + ActionDelivery, ActionDeliveryMode, ActionEffect, ActionEscalation, ActionEscalationReason, + ActionEscalationTarget, ActionEvidence, ActionEvidenceKind, ActionResult, + ActionResultValidationError, ActionRoute, CursorMotionOutput, CursorPointOutput, + CursorPositionOutput, CursorThemeOutput, CursorVisualOutput, DesktopStateOutput, + EffectiveScope, EndSessionOutput, GetAgentCursorStateOutput, ScreenSizeOutput, + SessionStateOutput, SetAgentCursorEnabledOutput, SetAgentCursorMotionOutput, + SetAgentCursorThemeOutput, StartSessionOutput, ToolOutput, }; pub use verification::{ BoundsExpectation, ElementPredicate, ElementSelector, PredicateOutcome, StatePredicate, @@ -54,11 +56,40 @@ pub const TOOLS_LIST_SCHEMA_VERSION: &str = "1"; pub const CAPABILITY_VERSION: &str = "1"; /// Shape version for the checked-in generated client contract. -pub const CONTRACT_VERSION: &str = "0.3.0"; +pub const CONTRACT_VERSION: &str = "0.4.0"; /// MCP protocol version used by current cua-driver clients. pub const MCP_PROTOCOL_VERSION: &str = "2025-06-18"; +/// Tools whose successful result is the shared closed [`ActionResult`]. +/// +/// Keep this vocabulary in the contract crate so MCP schema advertising, +/// runtime validation, SDK normalization, and the execution seam cannot drift. +pub const ACTION_RESULT_TOOLS: &[&str] = &[ + "click", + "double_click", + "right_click", + "scroll", + "drag", + "mouse_drag", + "parallel_mouse_drag", + "move_cursor", + "mouse_button_down", + "mouse_button_up", + "type_text", + "type_text_chars", + "press_key", + "hotkey", + "set_value", + "browser_click", + "browser_pointer", + "browser_type", +]; + +pub fn is_action_result_tool(name: &str) -> bool { + ACTION_RESULT_TOOLS.contains(&name) +} + #[derive( Debug, Clone, @@ -124,8 +155,8 @@ pub struct ToolContract { #[serde(skip_serializing_if = "Option::is_none")] pub cursor_semantics: Option, pub input_schema: Value, - /// Schema for successful `structuredContent` only. It is experimental and - /// is not advertised as MCP `outputSchema` until transport parity is proven. + /// Schema for successful `structuredContent`. The live MCP surface + /// advertises this as `outputSchema`. #[serde(skip_serializing_if = "Option::is_none")] pub success_output_schema: Option, /// Runtime-only validator bound to the same Rust output type that produced @@ -219,6 +250,10 @@ pub fn tool_input_fields(name: &str) -> Option<&'static BTreeSet> { /// Validate a successful structured payload against the Rust output type that /// also generates its SDK schema. Returns `Ok(false)` for non-SDK tools. pub fn validate_success_output(name: &str, value: Value) -> Result { + if is_action_result_tool(name) { + validate_typed_output::(value)?; + return Ok(true); + } if let Some(entry) = tool_index().get(name) { (entry.output_validator)(value)?; Ok(true) @@ -242,10 +277,34 @@ mod tests { let mut sorted = names.clone(); sorted.sort_unstable(); assert_eq!(names, sorted); - assert_eq!(manifest.contract_version, "0.3.0"); + assert_eq!(manifest.contract_version, "0.4.0"); assert!(manifest.experimental); } + #[test] + fn every_action_result_tool_uses_the_closed_runtime_validator() { + let valid = serde_json::json!({ + "effect": "unverifiable", + "route": "synthetic_events" + }); + let legacy = serde_json::json!({ + "path": "ax", + "verified": true + }); + + for name in ACTION_RESULT_TOOLS { + assert_eq!( + validate_success_output(name, valid.clone()), + Ok(true), + "{name} must accept the shared ActionResult" + ); + assert!( + validate_success_output(name, legacy.clone()).is_err(), + "{name} must reject a legacy action payload" + ); + } + } + #[test] fn verify_state_is_a_portable_read_only_contract() { let contract = tool_contract("verify_state").expect("verify_state contract"); @@ -301,6 +360,32 @@ mod tests { } } + #[test] + fn desktop_action_contracts_share_the_strict_action_result() { + let expected = ActionResult::output_schema(); + for name in [ + "click", + "drag", + "hotkey", + "move_cursor", + "press_key", + "scroll", + "type_text", + ] { + let contract = tool_contract(name).expect("desktop action contract"); + assert_eq!( + contract.success_output_schema, + Some(expected.clone()), + "{name}" + ); + assert_eq!( + contract.success_output_schema.as_ref().expect("schema")["additionalProperties"], + false, + "{name}" + ); + } + } + #[test] fn session_success_schemas_require_every_typed_field() { let start = tool_contract("start_session").expect("start_session contract"); diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs index 2cfba1bdb9..de90b7f2bc 100644 --- a/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs +++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2026 Cua AI, Inc. -use crate::{CaptureScope, DesktopScope, EscalationReason, Platform}; +use crate::{CaptureScope, EscalationReason, Platform}; use schemars::{generate::SchemaSettings, JsonSchema}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use serde_json::{Map, Value}; @@ -14,21 +14,27 @@ pub trait ToolOutput: Serialize + DeserializeOwned + JsonSchema { } fn output_schema() -> Value { - let mut settings = SchemaSettings::draft2020_12(); - settings.inline_subschemas = true; - settings.meta_schema = None; - let mut schema = - serde_json::to_value(settings.into_generator().into_root_schema_for::()) - .expect("JSON Schema serializes"); - strip_schema_titles(&mut schema); - if let Some(object) = schema.as_object_mut() { - object.insert("additionalProperties".into(), Value::Bool(true)); - object - .entry("properties") - .or_insert_with(|| Value::Object(Map::new())); - } - schema + output_schema_with_additional_properties::(true) + } +} + +fn output_schema_with_additional_properties(additional_properties: bool) -> Value { + let mut settings = SchemaSettings::draft2020_12(); + settings.inline_subschemas = true; + settings.meta_schema = None; + let mut schema = serde_json::to_value(settings.into_generator().into_root_schema_for::()) + .expect("JSON Schema serializes"); + strip_schema_titles(&mut schema); + if let Some(object) = schema.as_object_mut() { + object.insert( + "additionalProperties".into(), + Value::Bool(additional_properties), + ); + object + .entry("properties") + .or_insert_with(|| Value::Object(Map::new())); } + schema } fn strip_schema_titles(value: &mut Value) { @@ -257,59 +263,157 @@ pub struct CursorPositionOutput { impl ToolOutput for CursorPositionOutput {} -#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)] -pub struct MoveCursorOutput { - #[serde(default, skip_serializing_if = "Option::is_none")] - #[schemars(schema_with = "desktop_scope_schema")] - pub scope: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - #[schemars(schema_with = "number_schema")] - pub x: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - #[schemars(schema_with = "number_schema")] - pub y: Option, - #[serde(flatten)] - pub extensions: BTreeMap, +#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Enum)] +#[serde(rename_all = "snake_case")] +pub enum ActionEffect { + Confirmed, + Partial, + Unverifiable, + SuspectedNoop, + Refused, } -impl ToolOutput for MoveCursorOutput {} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Enum)] +#[serde(rename_all = "snake_case")] +pub enum ActionRoute { + Accessibility, + SyntheticEvents, + GlobalInput, + Dom, + TrustedInput, +} -#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)] -pub struct ClickOutput { - #[serde(default, skip_serializing_if = "Option::is_none")] - #[schemars(schema_with = "desktop_scope_schema")] - pub scope: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - #[schemars(schema_with = "number_schema")] - pub x: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - #[schemars(schema_with = "number_schema")] - pub y: Option, +#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Enum)] +#[serde(rename_all = "snake_case")] +pub enum ActionDeliveryMode { + Background, + Foreground, + NotApplicable, + Unknown, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Record)] +#[serde(deny_unknown_fields)] +pub struct ActionDelivery { + pub mode: ActionDeliveryMode, #[serde(default, skip_serializing_if = "Option::is_none")] - #[schemars(schema_with = "boolean_schema")] - pub verified: Option, - #[serde(flatten)] - pub extensions: BTreeMap, + pub delivered_count: Option, } -impl ToolOutput for ClickOutput {} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Enum)] +#[serde(rename_all = "snake_case")] +pub enum ActionEvidenceKind { + ValueReadback, + WindowChange, +} -#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)] -pub struct DesktopActionOutput { +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Record)] +#[serde(deny_unknown_fields)] +pub struct ActionEvidence { + pub kind: ActionEvidenceKind, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Enum)] +#[serde(rename_all = "snake_case")] +pub enum ActionEscalationTarget { + Pixel, + Foreground, + Page, + Session, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Enum)] +#[serde(rename_all = "snake_case")] +pub enum ActionEscalationReason { + RouteUnavailable, + DeliveryFailed, + EffectUnconfirmed, + SuspectedNoop, + PermissionRequired, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Record)] +#[serde(deny_unknown_fields)] +pub struct ActionEscalation { + pub target: ActionEscalationTarget, + pub reason: ActionEscalationReason, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Record)] +#[serde(deny_unknown_fields)] +pub struct ActionResult { + pub effect: ActionEffect, + pub route: ActionRoute, #[serde(default, skip_serializing_if = "Option::is_none")] - #[schemars(schema_with = "desktop_scope_schema")] - pub scope: Option, + pub delivery: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - #[schemars(schema_with = "string_schema")] - pub effect: Option, + pub evidence: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] - #[schemars(schema_with = "boolean_schema")] - pub verified: Option, - #[serde(flatten)] - pub extensions: BTreeMap, + pub escalation: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ActionResultValidationError { + ConfirmedRequiresEvidence, + PartialRequiresDeliveredCount, + RefusedCannotHaveDelivery, + RefusedCannotHaveEvidence, +} + +impl std::fmt::Display for ActionResultValidationError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(match self { + Self::ConfirmedRequiresEvidence => "confirmed effect requires evidence", + Self::PartialRequiresDeliveredCount => "partial effect requires delivered_count", + Self::RefusedCannotHaveDelivery => "refused effect cannot include delivery", + Self::RefusedCannotHaveEvidence => "refused effect cannot include evidence", + }) + } } -impl ToolOutput for DesktopActionOutput {} +impl std::error::Error for ActionResultValidationError {} + +impl ActionResult { + pub fn validate_invariants(&self) -> Result<(), ActionResultValidationError> { + match self.effect { + ActionEffect::Confirmed + if self + .evidence + .as_ref() + .is_none_or(|evidence| evidence.is_empty()) => + { + Err(ActionResultValidationError::ConfirmedRequiresEvidence) + } + ActionEffect::Partial + if self + .delivery + .as_ref() + .and_then(|delivery| delivery.delivered_count) + .is_none() => + { + Err(ActionResultValidationError::PartialRequiresDeliveredCount) + } + ActionEffect::Refused if self.delivery.is_some() => { + Err(ActionResultValidationError::RefusedCannotHaveDelivery) + } + ActionEffect::Refused if self.evidence.is_some() => { + Err(ActionResultValidationError::RefusedCannotHaveEvidence) + } + _ => Ok(()), + } + } +} + +impl ToolOutput for ActionResult { + fn validate(&self) -> Result<(), String> { + self.validate_invariants() + .map_err(|error| error.to_string()) + } + + fn output_schema() -> Value { + output_schema_with_additional_properties::(false) + } +} fn string_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { schemars::json_schema!({ "type": "string" }) @@ -327,10 +431,6 @@ fn integer_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { schemars::json_schema!({ "type": "integer" }) } -fn desktop_scope_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { - schemars::json_schema!({ "const": "desktop" }) -} - fn platform_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { schemars::json_schema!({ "type": "string", @@ -359,3 +459,216 @@ fn nullable_escalation_reason_schema(_: &mut schemars::SchemaGenerator) -> schem ] }) } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn object_variant(schema: &Value) -> &Value { + if schema.get("properties").is_some() { + return schema; + } + schema["anyOf"] + .as_array() + .expect("nullable anyOf") + .iter() + .find(|variant| variant.get("properties").is_some()) + .expect("object variant") + } + + fn confirmed_result() -> ActionResult { + ActionResult { + effect: ActionEffect::Confirmed, + route: ActionRoute::Accessibility, + delivery: Some(ActionDelivery { + mode: ActionDeliveryMode::Background, + delivered_count: None, + }), + evidence: Some(vec![ActionEvidence { + kind: ActionEvidenceKind::ValueReadback, + }]), + escalation: None, + } + } + + #[test] + fn action_result_schema_is_exact_and_closed() { + let schema = ActionResult::output_schema(); + assert_eq!(schema["additionalProperties"], false); + assert_eq!(schema["required"], json!(["effect", "route"])); + + let properties = schema["properties"].as_object().expect("properties"); + assert_eq!( + properties.keys().map(String::as_str).collect::>(), + ["delivery", "effect", "escalation", "evidence", "route"] + ); + assert_eq!( + properties["effect"]["enum"], + json!([ + "confirmed", + "partial", + "unverifiable", + "suspected_noop", + "refused" + ]) + ); + assert_eq!( + properties["route"]["enum"], + json!([ + "accessibility", + "synthetic_events", + "global_input", + "dom", + "trusted_input" + ]) + ); + + let delivery = object_variant(&properties["delivery"]); + assert_eq!(delivery["additionalProperties"], false); + assert_eq!(delivery["required"], json!(["mode"])); + assert_eq!( + delivery["properties"]["mode"]["enum"], + json!(["background", "foreground", "not_applicable", "unknown"]) + ); + + let evidence_schema = &properties["evidence"]; + let evidence_array = if evidence_schema.get("items").is_some() { + evidence_schema + } else { + evidence_schema["anyOf"] + .as_array() + .expect("nullable anyOf") + .iter() + .find(|variant| variant.get("items").is_some()) + .expect("array variant") + }; + let evidence = &evidence_array["items"]; + assert_eq!(evidence["additionalProperties"], false); + assert_eq!(evidence["required"], json!(["kind"])); + assert_eq!( + evidence["properties"]["kind"]["enum"], + json!(["value_readback", "window_change"]) + ); + + let escalation = object_variant(&properties["escalation"]); + assert_eq!(escalation["additionalProperties"], false); + assert_eq!(escalation["required"], json!(["target", "reason"])); + assert_eq!( + escalation["properties"]["target"]["enum"], + json!(["pixel", "foreground", "page", "session"]) + ); + assert_eq!( + escalation["properties"]["reason"]["enum"], + json!([ + "route_unavailable", + "delivery_failed", + "effect_unconfirmed", + "suspected_noop", + "permission_required" + ]) + ); + } + + #[test] + fn action_result_round_trips_without_legacy_or_request_fields() { + let result = confirmed_result(); + let value = serde_json::to_value(&result).expect("serialize"); + assert_eq!( + value, + json!({ + "effect": "confirmed", + "route": "accessibility", + "delivery": {"mode": "background"}, + "evidence": [{"kind": "value_readback"}] + }) + ); + assert_eq!( + serde_json::from_value::(value).expect("deserialize"), + result + ); + + for legacy in [ + ("scope", json!("desktop")), + ("target", json!("button")), + ("x", json!(10)), + ("y", json!(20)), + ("path", json!("cgevent")), + ("transport", json!("windows_send_input")), + ("verified", json!(true)), + ("extensions", json!({})), + ] { + let mut value = serde_json::to_value(&result).expect("serialize"); + value + .as_object_mut() + .expect("object") + .insert(legacy.0.into(), legacy.1); + assert!( + serde_json::from_value::(value).is_err(), + "accepted legacy field {}", + legacy.0 + ); + } + + let mut delivery_extension = serde_json::to_value(&result).expect("serialize"); + delivery_extension["delivery"]["requested"] = json!("background"); + assert!(serde_json::from_value::(delivery_extension).is_err()); + + let mut evidence_extension = serde_json::to_value(&result).expect("serialize"); + evidence_extension["evidence"][0]["detail"] = json!("private readback"); + assert!(serde_json::from_value::(evidence_extension).is_err()); + + let escalation_extension = json!({ + "effect": "unverifiable", + "route": "synthetic_events", + "escalation": { + "target": "foreground", + "reason": "delivery_failed", + "requires": ["window_id"] + } + }); + assert!(serde_json::from_value::(escalation_extension).is_err()); + } + + #[test] + fn action_result_enforces_effect_invariants() { + let mut result = confirmed_result(); + result.evidence = None; + assert_eq!( + result.validate_invariants(), + Err(ActionResultValidationError::ConfirmedRequiresEvidence) + ); + assert_eq!( + ToolOutput::validate(&result), + Err("confirmed effect requires evidence".into()) + ); + + result.effect = ActionEffect::Partial; + result.delivery = Some(ActionDelivery { + mode: ActionDeliveryMode::Foreground, + delivered_count: None, + }); + assert_eq!( + result.validate_invariants(), + Err(ActionResultValidationError::PartialRequiresDeliveredCount) + ); + result.delivery.as_mut().expect("delivery").delivered_count = Some(1); + assert_eq!(result.validate_invariants(), Ok(())); + + result.effect = ActionEffect::Refused; + assert_eq!( + result.validate_invariants(), + Err(ActionResultValidationError::RefusedCannotHaveDelivery) + ); + result.delivery = None; + result.evidence = Some(vec![ActionEvidence { + kind: ActionEvidenceKind::WindowChange, + }]); + assert_eq!( + result.validate_invariants(), + Err(ActionResultValidationError::RefusedCannotHaveEvidence) + ); + result.evidence = None; + assert_eq!(result.validate_invariants(), Ok(())); + } +} diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/action_record.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/action_record.rs new file mode 100644 index 0000000000..4fe9288242 --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/action_record.rs @@ -0,0 +1,1693 @@ +//! Internal, non-wire representation of what an action actually did. +//! +//! This deliberately has no serde derives: it is the driver's source of truth, +//! not a protocol contract. Callers that need to publish an outcome should use +//! [`ActionExecutionRecord::stable_projection`] after validation. + +/// The strongest truthful statement the driver can make about an action. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ActionEffect { + Confirmed, + Partial, + Unverifiable, + SuspectedNoop, + Refused, +} + +/// The delivery mode requested by the caller. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RequestedDelivery { + Background, + Foreground, + NotApplicable, +} + +/// The delivery mode actually used by the actuator. +/// +/// `Unknown` means an attempt was made but the actuator could not determine +/// whether it delivered in the requested mode. `None` means no delivery was +/// attempted (for example, a refusal). +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ActualDelivery { + Background, + Foreground, + NotApplicable, + Unknown, +} + +/// A concrete action transport known to the driver. +/// +/// Keep this exhaustive rather than accepting arbitrary strings so a new +/// actuator must choose both its internal identity and published route. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum ActionTransport { + AgentCursorOverlay, + MacosAxAction, + MacosAxValue, + MacosCgEventPid, + MacosCgEventHid, + WindowsUiaInvoke, + WindowsUiaToggle, + WindowsUiaSelection, + WindowsUiaExpandCollapse, + WindowsUiaValue, + WindowsUiaRangeValue, + WindowsUiaScroll, + WindowsMsaaAction, + WindowsPostMessage, + WindowsTargetedInjection, + WindowsSendInput, + WindowsSetCursorPos, + WindowsShellExecute, + LinuxAtSpiAction, + LinuxAtSpiValue, + LinuxPty, + LinuxXSendEvent, + LinuxXTest, + LinuxLibei, + LinuxWaylandVirtualPointer, + LinuxCuaCompositorInject, + BrowserCdpInputMouse, + BrowserCdpInputKey, + BrowserCdpRuntimeFunction, +} + +impl ActionTransport { + pub const ALL: &'static [Self] = &[ + Self::AgentCursorOverlay, + Self::MacosAxAction, + Self::MacosAxValue, + Self::MacosCgEventPid, + Self::MacosCgEventHid, + Self::WindowsUiaInvoke, + Self::WindowsUiaToggle, + Self::WindowsUiaSelection, + Self::WindowsUiaExpandCollapse, + Self::WindowsUiaValue, + Self::WindowsUiaRangeValue, + Self::WindowsUiaScroll, + Self::WindowsMsaaAction, + Self::WindowsPostMessage, + Self::WindowsTargetedInjection, + Self::WindowsSendInput, + Self::WindowsSetCursorPos, + Self::WindowsShellExecute, + Self::LinuxAtSpiAction, + Self::LinuxAtSpiValue, + Self::LinuxPty, + Self::LinuxXSendEvent, + Self::LinuxXTest, + Self::LinuxLibei, + Self::LinuxWaylandVirtualPointer, + Self::LinuxCuaCompositorInject, + Self::BrowserCdpInputMouse, + Self::BrowserCdpInputKey, + Self::BrowserCdpRuntimeFunction, + ]; + + pub const fn route(self) -> ActionRoute { + match self { + Self::AgentCursorOverlay => ActionRoute::SyntheticEvents, + Self::MacosAxAction + | Self::MacosAxValue + | Self::WindowsUiaInvoke + | Self::WindowsUiaToggle + | Self::WindowsUiaSelection + | Self::WindowsUiaExpandCollapse + | Self::WindowsUiaValue + | Self::WindowsUiaRangeValue + | Self::WindowsUiaScroll + | Self::WindowsMsaaAction + | Self::LinuxAtSpiAction + | Self::LinuxAtSpiValue => ActionRoute::Accessibility, + Self::MacosCgEventPid + | Self::WindowsPostMessage + | Self::LinuxPty + | Self::LinuxXSendEvent => ActionRoute::SyntheticEvents, + Self::MacosCgEventHid + | Self::WindowsTargetedInjection + | Self::WindowsSendInput + | Self::WindowsSetCursorPos + | Self::WindowsShellExecute + | Self::LinuxXTest + | Self::LinuxLibei + | Self::LinuxWaylandVirtualPointer + | Self::LinuxCuaCompositorInject => ActionRoute::GlobalInput, + Self::BrowserCdpRuntimeFunction => ActionRoute::Dom, + Self::BrowserCdpInputMouse | Self::BrowserCdpInputKey => ActionRoute::TrustedInput, + } + } +} + +/// A stable route label suitable for projecting internal action truth. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ActionRoute { + Accessibility, + SyntheticEvents, + GlobalInput, + Dom, + TrustedInput, +} + +/// Evidence supporting an action effect, intentionally without request data. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ActionEvidence { + pub kind: EvidenceKind, + pub detail: String, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum EvidenceKind { + AccessibilityReadback, + BrowserReadback, + WindowChange, + NativeApiResult, + ScreenshotComparison, + EventReceipt, + OperatorObservation, +} + +/// A failed or superseded transport attempt. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ActionAttempt { + pub transport: ActionTransport, + pub delivery: ActualDelivery, + pub detail: Option, +} + +/// A deliberate transition between transports. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ActionFallback { + pub from: ActionTransport, + pub to: ActionTransport, + pub reason: String, +} + +/// Escalation undertaken while trying to complete an action. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ActionEscalation { + pub kind: EscalationKind, + pub detail: Option, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum EscalationKind { + ActivateTarget, + RetryWithPixelTarget, + RetryWithPageAction, + RefreshPageState, + RequestPermission, + ElevateAccess, + ExpandCaptureScope, + PrepareSession, + RetryWithForegroundDelivery, +} + +/// Complete internal accounting for one action execution. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ActionExecutionRecord { + pub effect: ActionEffect, + pub transport: ActionTransport, + pub requested_delivery: RequestedDelivery, + pub actual_delivery: Option, + pub attempts: Vec, + pub fallbacks: Vec, + pub evidence: Vec, + pub escalation: Option, + pub delivered_count: Option, + pub detail: Option, +} + +impl ActionExecutionRecord { + pub fn new( + effect: ActionEffect, + transport: ActionTransport, + requested_delivery: RequestedDelivery, + ) -> Self { + Self { + effect, + transport, + requested_delivery, + actual_delivery: None, + attempts: Vec::new(), + fallbacks: Vec::new(), + evidence: Vec::new(), + escalation: None, + delivered_count: None, + detail: None, + } + } + + pub fn builder( + effect: ActionEffect, + transport: ActionTransport, + requested_delivery: RequestedDelivery, + ) -> ActionExecutionRecordBuilder { + ActionExecutionRecordBuilder::new(effect, transport, requested_delivery) + } + + pub fn validate(&self) -> Result<(), ActionRecordValidationError> { + match self.effect { + ActionEffect::Confirmed if projected_evidence(&self.evidence).is_none() => { + Err(ActionRecordValidationError::ConfirmedRequiresEvidence) + } + ActionEffect::Partial + if self.actual_delivery.is_none() || self.delivered_count.is_none() => + { + Err(ActionRecordValidationError::PartialRequiresDeliveredCount) + } + ActionEffect::Refused if self.actual_delivery.is_some() => { + Err(ActionRecordValidationError::RefusedCannotHaveDelivery) + } + ActionEffect::Refused if !self.evidence.is_empty() => { + Err(ActionRecordValidationError::RefusedCannotHaveEvidence) + } + _ => Ok(()), + } + } + + pub fn stable_projection( + &self, + ) -> Result { + self.validate()?; + Ok(ActionOutcomeProjection { + effect: self.effect, + route: self.transport.route(), + delivery: self.actual_delivery.map(|actual| ActionDeliveryProjection { + actual, + delivered_count: self.delivered_count, + }), + evidence: projected_evidence(&self.evidence), + escalation: self.escalation.clone(), + }) + } + + /// Build the closed public action contract from the validated internal + /// record. This is intentionally constructive: request data, raw + /// transports, diagnostic detail, and non-publishable evidence never + /// enter the value that is serialized for MCP or SDK clients. + pub fn public_result( + &self, + ) -> Result { + let projection = self.stable_projection()?; + Ok(cua_driver_contract::ActionResult { + effect: match projection.effect { + ActionEffect::Confirmed => cua_driver_contract::ActionEffect::Confirmed, + ActionEffect::Partial => cua_driver_contract::ActionEffect::Partial, + ActionEffect::Unverifiable => cua_driver_contract::ActionEffect::Unverifiable, + ActionEffect::SuspectedNoop => cua_driver_contract::ActionEffect::SuspectedNoop, + ActionEffect::Refused => cua_driver_contract::ActionEffect::Refused, + }, + route: match projection.route { + ActionRoute::Accessibility => cua_driver_contract::ActionRoute::Accessibility, + ActionRoute::SyntheticEvents => cua_driver_contract::ActionRoute::SyntheticEvents, + ActionRoute::GlobalInput => cua_driver_contract::ActionRoute::GlobalInput, + ActionRoute::Dom => cua_driver_contract::ActionRoute::Dom, + ActionRoute::TrustedInput => cua_driver_contract::ActionRoute::TrustedInput, + }, + delivery: projection + .delivery + .map(|delivery| cua_driver_contract::ActionDelivery { + mode: match delivery.actual { + ActualDelivery::Background => { + cua_driver_contract::ActionDeliveryMode::Background + } + ActualDelivery::Foreground => { + cua_driver_contract::ActionDeliveryMode::Foreground + } + ActualDelivery::NotApplicable => { + cua_driver_contract::ActionDeliveryMode::NotApplicable + } + ActualDelivery::Unknown => cua_driver_contract::ActionDeliveryMode::Unknown, + }, + delivered_count: delivery.delivered_count, + }), + evidence: projection.evidence.map(|evidence| { + evidence + .into_iter() + .map(|evidence| cua_driver_contract::ActionEvidence { + kind: match evidence.kind { + ProjectedEvidenceKind::AccessibilityReadback + | ProjectedEvidenceKind::BrowserReadback => { + cua_driver_contract::ActionEvidenceKind::ValueReadback + } + ProjectedEvidenceKind::WindowChange => { + cua_driver_contract::ActionEvidenceKind::WindowChange + } + }, + }) + .collect() + }), + escalation: projection.escalation.map(|escalation| { + use cua_driver_contract::{ + ActionEscalation, ActionEscalationReason, ActionEscalationTarget, + }; + + let (target, reason) = match escalation.kind { + EscalationKind::ActivateTarget + | EscalationKind::RetryWithForegroundDelivery => ( + ActionEscalationTarget::Foreground, + ActionEscalationReason::DeliveryFailed, + ), + EscalationKind::RetryWithPixelTarget => ( + ActionEscalationTarget::Pixel, + ActionEscalationReason::EffectUnconfirmed, + ), + EscalationKind::RetryWithPageAction => ( + ActionEscalationTarget::Page, + ActionEscalationReason::EffectUnconfirmed, + ), + EscalationKind::RefreshPageState => ( + ActionEscalationTarget::Page, + ActionEscalationReason::RouteUnavailable, + ), + EscalationKind::RequestPermission | EscalationKind::ElevateAccess => ( + ActionEscalationTarget::Session, + ActionEscalationReason::PermissionRequired, + ), + EscalationKind::ExpandCaptureScope => ( + ActionEscalationTarget::Session, + ActionEscalationReason::RouteUnavailable, + ), + EscalationKind::PrepareSession => ( + ActionEscalationTarget::Session, + ActionEscalationReason::RouteUnavailable, + ), + }; + ActionEscalation { + target, + reason: if projection.effect == ActionEffect::SuspectedNoop + && reason != ActionEscalationReason::PermissionRequired + { + ActionEscalationReason::SuspectedNoop + } else { + reason + }, + } + }), + }) + } + + /// Normalize the legacy hand-written payload at the canonical dispatch + /// seam before replacing it with the closed public action contract. + /// + /// This compatibility adapter is intentionally conservative: it reports + /// `Unknown` rather than copying a requested delivery mode when the old + /// payload did not prove what actually happened. Platform producers can + /// attach a richer record directly as they are migrated. + pub fn from_legacy( + tool_name: &str, + args: &serde_json::Value, + structured: &serde_json::Value, + ) -> Option { + if !is_action_tool(tool_name) { + return None; + } + let requested_delivery = requested_delivery(tool_name, args); + let raw_path = structured + .get("path") + .or_else(|| structured.get("route")) + .and_then(serde_json::Value::as_str); + let transport = transport_from_legacy(tool_name, args, raw_path)?; + let effect = legacy_effect(structured); + let actual_delivery = + actual_delivery_from_legacy(tool_name, args, structured, raw_path, transport, effect); + + let mut record = Self::new(effect, transport, requested_delivery); + record.actual_delivery = actual_delivery; + record.delivered_count = structured + .get("delivered_chars") + .or_else(|| structured.pointer("/refusal/detail/delivered_chars")) + .and_then(serde_json::Value::as_u64) + .and_then(|count| u32::try_from(count).ok()); + + if legacy_has_publishable_readback(tool_name, structured) { + record.evidence.push(ActionEvidence { + kind: if matches!( + transport, + ActionTransport::BrowserCdpInputMouse + | ActionTransport::BrowserCdpInputKey + | ActionTransport::BrowserCdpRuntimeFunction + ) { + EvidenceKind::BrowserReadback + } else { + EvidenceKind::AccessibilityReadback + }, + detail: structured + .get("verify") + .and_then(serde_json::Value::as_str) + .unwrap_or("confirmed") + .to_owned(), + }); + } + if let Some(escalation) = structured.get("escalation") { + let recommendation = escalation + .get("recommended") + .and_then(serde_json::Value::as_str); + let kind = match recommendation { + Some("foreground") => Some(EscalationKind::RetryWithForegroundDelivery), + Some("px" | "pixel") => Some(EscalationKind::RetryWithPixelTarget), + Some("page") => Some(EscalationKind::RetryWithPageAction), + Some("session") => Some(EscalationKind::ExpandCaptureScope), + _ => None, + }; + if let Some(kind) = kind { + record.escalation = Some(ActionEscalation { + kind, + detail: escalation + .get("reason") + .and_then(serde_json::Value::as_str) + .map(str::to_owned), + }); + } + } else if let Some(code) = structured + .pointer("/refusal/code") + .and_then(serde_json::Value::as_str) + { + record.escalation = browser_refusal_escalation(code); + } + if effect == ActionEffect::Partial && record.delivered_count.is_none() { + return None; + } + if effect == ActionEffect::Confirmed && projected_evidence(&record.evidence).is_none() { + // A legacy string claiming "confirmed" without a trusted readback + // is not enough to preserve that stronger statement. + record.effect = ActionEffect::Unverifiable; + } + record.validate().ok()?; + Some(record) + } + + /// Explicit internal-only representation for recording and diagnostics. + /// Keeping this constructor here prevents serde derives from accidentally + /// turning the rich record into a protocol surface. + pub fn debug_json(&self) -> serde_json::Value { + serde_json::json!({ + "effect": effect_name(self.effect), + "transport": transport_name(self.transport), + "route": route_name(self.transport.route()), + "requested_delivery": requested_delivery_name(self.requested_delivery), + "actual_delivery": self.actual_delivery.map(actual_delivery_name), + "delivered_count": self.delivered_count, + "attempts": self.attempts.iter().map(|attempt| serde_json::json!({ + "transport": transport_name(attempt.transport), + "delivery": actual_delivery_name(attempt.delivery), + "detail": attempt.detail, + })).collect::>(), + "fallbacks": self.fallbacks.iter().map(|fallback| serde_json::json!({ + "from": transport_name(fallback.from), + "to": transport_name(fallback.to), + "reason": fallback.reason, + })).collect::>(), + "evidence": self.evidence.iter().map(|evidence| serde_json::json!({ + "kind": evidence_kind_name(evidence.kind), + "detail": evidence.detail, + })).collect::>(), + "escalation": self.escalation.as_ref().map(|escalation| serde_json::json!({ + "kind": escalation_kind_name(escalation.kind), + "detail": escalation.detail, + })), + "detail": self.detail, + }) + } +} + +pub fn is_action_tool(tool_name: &str) -> bool { + cua_driver_contract::is_action_result_tool(tool_name) +} + +fn requested_delivery(tool_name: &str, args: &serde_json::Value) -> RequestedDelivery { + match args + .get("delivery_mode") + .and_then(serde_json::Value::as_str) + { + Some("foreground") => RequestedDelivery::Foreground, + Some("background") => RequestedDelivery::Background, + _ if args.get("scope").and_then(serde_json::Value::as_str) == Some("desktop") => { + RequestedDelivery::NotApplicable + } + _ if matches!( + tool_name, + "browser_click" | "browser_pointer" | "browser_type" + ) => + { + RequestedDelivery::NotApplicable + } + _ => RequestedDelivery::Background, + } +} + +fn legacy_effect(structured: &serde_json::Value) -> ActionEffect { + if structured.get("status").and_then(serde_json::Value::as_str) == Some("refused") { + let is_partial = structured + .pointer("/refusal/code") + .and_then(serde_json::Value::as_str) + == Some("browser_input_incomplete") + && structured + .pointer("/refusal/detail/delivered_chars") + .and_then(serde_json::Value::as_u64) + .is_some_and(|count| count > 0); + return if is_partial { + ActionEffect::Partial + } else { + ActionEffect::Refused + }; + } + match structured.get("effect").and_then(serde_json::Value::as_str) { + Some("confirmed") => ActionEffect::Confirmed, + Some("partial") => ActionEffect::Partial, + Some("suspected_noop") => ActionEffect::SuspectedNoop, + Some("refused") => ActionEffect::Refused, + _ => ActionEffect::Unverifiable, + } +} + +fn browser_refusal_escalation(code: &str) -> Option { + let kind = match code { + "browser_consent_required" | "browser_consent_revoked" | "browser_origin_outside_scope" => { + EscalationKind::RequestPermission + } + "browser_requires_setup" + | "browser_route_unavailable" + | "browser_endpoint_owner_mismatch" + | "browser_reconnect_exhausted" => EscalationKind::PrepareSession, + "browser_binding_ambiguous" + | "browser_binding_stale" + | "browser_wrong_target_refused" + | "browser_tab_required" + | "browser_tab_not_found" + | "browser_ref_stale" + | "browser_input_trust_unavailable" + | "browser_action_unavailable" => EscalationKind::RefreshPageState, + // A delivered prefix is already represented by `effect: partial` and + // `delivery.delivered_count`; verification decides whether to stop. + "browser_input_incomplete" => return None, + _ => return None, + }; + Some(ActionEscalation { kind, detail: None }) +} + +/// Legacy `verified` was not a sufficient proof by itself: some action +/// producers used it as a delivery acknowledgement, and clicks have no +/// independent postcondition read-back. Only value-changing tools whose +/// platform implementations compare a fresh value against the request may +/// promote that legacy bit to publishable evidence. +fn legacy_has_publishable_readback(tool_name: &str, structured: &serde_json::Value) -> bool { + matches!(tool_name, "type_text" | "type_text_chars" | "set_value") + && structured + .get("verified") + .and_then(serde_json::Value::as_bool) + == Some(true) + && structured.get("effect").and_then(serde_json::Value::as_str) == Some("confirmed") +} + +fn transport_from_legacy( + tool_name: &str, + args: &serde_json::Value, + raw_path: Option<&str>, +) -> Option { + let path = raw_path.unwrap_or(""); + let transport = match path { + "ax" | "ax_fg" => { + if cfg!(target_os = "macos") { + if matches!(tool_name, "type_text" | "type_text_chars" | "set_value") { + ActionTransport::MacosAxValue + } else { + ActionTransport::MacosAxAction + } + } else if cfg!(target_os = "windows") { + if matches!(tool_name, "type_text" | "type_text_chars" | "set_value") { + ActionTransport::WindowsUiaValue + } else { + ActionTransport::WindowsUiaInvoke + } + } else { + if matches!(tool_name, "type_text" | "type_text_chars" | "set_value") { + ActionTransport::LinuxAtSpiValue + } else { + ActionTransport::LinuxAtSpiAction + } + } + } + "uia" if tool_name == "scroll" => ActionTransport::WindowsUiaScroll, + "uia" => ActionTransport::WindowsUiaInvoke, + "uia_expand_collapse" => ActionTransport::WindowsUiaExpandCollapse, + "msaa" => ActionTransport::WindowsMsaaAction, + "post_message" | "PostMessage" => ActionTransport::WindowsPostMessage, + "SendInput" => ActionTransport::WindowsSendInput, + "SetCursorPos" => ActionTransport::WindowsSetCursorPos, + "atspi" | "wayland_atspi" | "x11_atspi" => ActionTransport::LinuxAtSpiAction, + "pty" => ActionTransport::LinuxPty, + "x11_pixel" | "x11_pixel_fg" | "x11_xtest_fg" | "xtest" | "xtest_desktop" => { + ActionTransport::LinuxXTest + } + "wayland_activate" | "wayland_focused" => ActionTransport::LinuxLibei, + "wayland_desktop" => ActionTransport::LinuxWaylandVirtualPointer, + "cua_compositor_inject" | "wayland_cua_compositor" => { + ActionTransport::LinuxCuaCompositorInject + } + "hid" | "cgevent_hid" | "cgevent_fg" => ActionTransport::MacosCgEventHid, + "cgevent" => { + if args + .get("delivery_mode") + .and_then(serde_json::Value::as_str) + == Some("foreground") + { + ActionTransport::MacosCgEventHid + } else { + ActionTransport::MacosCgEventPid + } + } + "dom_event" => ActionTransport::BrowserCdpRuntimeFunction, + "trusted" => ActionTransport::BrowserCdpInputMouse, + "key_events" | "key_events_fg" => { + let foreground = path.ends_with("_fg") + || args + .get("delivery_mode") + .and_then(serde_json::Value::as_str) + == Some("foreground"); + if cfg!(target_os = "macos") { + if foreground { + ActionTransport::MacosCgEventHid + } else { + ActionTransport::MacosCgEventPid + } + } else if cfg!(target_os = "windows") { + if foreground { + ActionTransport::WindowsSendInput + } else { + ActionTransport::WindowsPostMessage + } + } else { + if foreground { + ActionTransport::LinuxXTest + } else { + ActionTransport::LinuxXSendEvent + } + } + } + "pixel" => { + if cfg!(target_os = "macos") { + if args + .get("delivery_mode") + .and_then(serde_json::Value::as_str) + == Some("foreground") + { + ActionTransport::MacosCgEventHid + } else { + ActionTransport::MacosCgEventPid + } + } else if cfg!(target_os = "windows") { + if args + .get("delivery_mode") + .and_then(serde_json::Value::as_str) + == Some("foreground") + { + ActionTransport::WindowsSendInput + } else { + ActionTransport::WindowsTargetedInjection + } + } else { + ActionTransport::LinuxXTest + } + } + "" if matches!(tool_name, "browser_click" | "browser_pointer") => { + if args.get("input_route").and_then(serde_json::Value::as_str) == Some("dom_event") { + ActionTransport::BrowserCdpRuntimeFunction + } else { + ActionTransport::BrowserCdpInputMouse + } + } + "" if tool_name == "browser_type" => ActionTransport::BrowserCdpInputKey, + "" if tool_name == "move_cursor" + && args.get("scope").and_then(serde_json::Value::as_str) != Some("desktop") => + { + ActionTransport::AgentCursorOverlay + } + "" if args.get("scope").and_then(serde_json::Value::as_str) == Some("desktop") => { + if cfg!(target_os = "macos") { + ActionTransport::MacosCgEventHid + } else if cfg!(target_os = "windows") { + ActionTransport::WindowsSendInput + } else { + ActionTransport::LinuxXTest + } + } + "" if matches!( + tool_name, + "type_text" | "type_text_chars" | "press_key" | "hotkey" + ) => + { + if cfg!(target_os = "macos") { + ActionTransport::MacosCgEventPid + } else if cfg!(target_os = "windows") { + if args + .get("delivery_mode") + .and_then(serde_json::Value::as_str) + == Some("foreground") + { + ActionTransport::WindowsSendInput + } else { + ActionTransport::WindowsPostMessage + } + } else { + ActionTransport::LinuxXSendEvent + } + } + "" if tool_name == "set_value" => { + if cfg!(target_os = "macos") { + ActionTransport::MacosAxValue + } else if cfg!(target_os = "windows") { + ActionTransport::WindowsUiaValue + } else { + ActionTransport::LinuxAtSpiValue + } + } + "" if matches!( + tool_name, + "click" + | "double_click" + | "right_click" + | "scroll" + | "drag" + | "mouse_drag" + | "parallel_mouse_drag" + | "mouse_button_down" + | "mouse_button_up" + ) => + { + if cfg!(target_os = "macos") { + if args + .get("delivery_mode") + .and_then(serde_json::Value::as_str) + == Some("foreground") + { + ActionTransport::MacosCgEventHid + } else { + ActionTransport::MacosCgEventPid + } + } else if cfg!(target_os = "windows") { + if args + .get("delivery_mode") + .and_then(serde_json::Value::as_str) + == Some("foreground") + { + ActionTransport::WindowsSendInput + } else { + ActionTransport::WindowsTargetedInjection + } + } else { + ActionTransport::LinuxXTest + } + } + _ => return None, + }; + Some(transport) +} + +fn actual_delivery_from_legacy( + tool_name: &str, + args: &serde_json::Value, + structured: &serde_json::Value, + raw_path: Option<&str>, + transport: ActionTransport, + effect: ActionEffect, +) -> Option { + if effect == ActionEffect::Refused { + return None; + } + if matches!( + tool_name, + "browser_click" | "browser_pointer" | "browser_type" + ) { + return Some(ActualDelivery::Background); + } + if transport == ActionTransport::AgentCursorOverlay + || args.get("scope").and_then(serde_json::Value::as_str) == Some("desktop") + { + return Some(ActualDelivery::NotApplicable); + } + match structured_delivery_mode(args, structured) { + Some(delivery) => return Some(delivery), + None => {} + } + match raw_path { + Some(path) if path.ends_with("_fg") => Some(ActualDelivery::Foreground), + Some("hid" | "cgevent_hid" | "SendInput" | "wayland_activate" | "wayland_focused") => { + Some(ActualDelivery::Foreground) + } + Some(_) => Some(ActualDelivery::Background), + None => Some(ActualDelivery::Unknown), + } +} + +/// Successful legacy foreground branches either complete their activation and +/// dispatch or return an error. The request therefore identifies the executed +/// branch even when the old payload omitted a path or reused a background path +/// label such as Windows `pixel`. Prefer an explicit producer-emitted mode when +/// one exists. +fn structured_delivery_mode( + args: &serde_json::Value, + structured: &serde_json::Value, +) -> Option { + let mode = structured + .get("delivery_mode") + .or_else(|| args.get("delivery_mode")) + .and_then(serde_json::Value::as_str); + match mode { + Some("foreground") => Some(ActualDelivery::Foreground), + Some("background") if structured.get("delivery_mode").is_some() => { + Some(ActualDelivery::Background) + } + // A requested background mode alone is not proof when the legacy + // producer returned no route. Keep it unknown until the path branch + // below establishes background delivery. + _ => None, + } +} + +fn projected_evidence(evidence: &[ActionEvidence]) -> Option> { + let evidence: Vec<_> = evidence + .iter() + .filter_map(|evidence| { + let kind = match evidence.kind { + EvidenceKind::AccessibilityReadback => ProjectedEvidenceKind::AccessibilityReadback, + EvidenceKind::BrowserReadback => ProjectedEvidenceKind::BrowserReadback, + EvidenceKind::WindowChange => ProjectedEvidenceKind::WindowChange, + EvidenceKind::NativeApiResult + | EvidenceKind::ScreenshotComparison + | EvidenceKind::EventReceipt + | EvidenceKind::OperatorObservation => return None, + }; + Some(ActionEvidenceProjection { + kind, + detail: evidence.detail.clone(), + }) + }) + .collect(); + (!evidence.is_empty()).then_some(evidence) +} + +fn effect_name(effect: ActionEffect) -> &'static str { + match effect { + ActionEffect::Confirmed => "confirmed", + ActionEffect::Partial => "partial", + ActionEffect::Unverifiable => "unverifiable", + ActionEffect::SuspectedNoop => "suspected_noop", + ActionEffect::Refused => "refused", + } +} + +fn route_name(route: ActionRoute) -> &'static str { + match route { + ActionRoute::Accessibility => "accessibility", + ActionRoute::SyntheticEvents => "synthetic_events", + ActionRoute::GlobalInput => "global_input", + ActionRoute::Dom => "dom", + ActionRoute::TrustedInput => "trusted_input", + } +} + +fn requested_delivery_name(delivery: RequestedDelivery) -> &'static str { + match delivery { + RequestedDelivery::Background => "background", + RequestedDelivery::Foreground => "foreground", + RequestedDelivery::NotApplicable => "not_applicable", + } +} + +fn actual_delivery_name(delivery: ActualDelivery) -> &'static str { + match delivery { + ActualDelivery::Background => "background", + ActualDelivery::Foreground => "foreground", + ActualDelivery::NotApplicable => "not_applicable", + ActualDelivery::Unknown => "unknown", + } +} + +fn evidence_kind_name(kind: EvidenceKind) -> &'static str { + match kind { + EvidenceKind::AccessibilityReadback => "accessibility_readback", + EvidenceKind::BrowserReadback => "browser_readback", + EvidenceKind::WindowChange => "window_change", + EvidenceKind::NativeApiResult => "native_api_result", + EvidenceKind::ScreenshotComparison => "screenshot_comparison", + EvidenceKind::EventReceipt => "event_receipt", + EvidenceKind::OperatorObservation => "operator_observation", + } +} + +fn escalation_kind_name(kind: EscalationKind) -> &'static str { + match kind { + EscalationKind::ActivateTarget => "activate_target", + EscalationKind::RetryWithPixelTarget => "retry_with_pixel_target", + EscalationKind::RetryWithPageAction => "retry_with_page_action", + EscalationKind::RefreshPageState => "refresh_page_state", + EscalationKind::RequestPermission => "request_permission", + EscalationKind::ElevateAccess => "elevate_access", + EscalationKind::ExpandCaptureScope => "expand_capture_scope", + EscalationKind::PrepareSession => "prepare_session", + EscalationKind::RetryWithForegroundDelivery => "retry_with_foreground_delivery", + } +} + +fn transport_name(transport: ActionTransport) -> &'static str { + match transport { + ActionTransport::AgentCursorOverlay => "agent_cursor_overlay", + ActionTransport::MacosAxAction => "macos_ax_action", + ActionTransport::MacosAxValue => "macos_ax_value", + ActionTransport::MacosCgEventPid => "macos_cg_event_pid", + ActionTransport::MacosCgEventHid => "macos_cg_event_hid", + ActionTransport::WindowsUiaInvoke => "windows_uia_invoke", + ActionTransport::WindowsUiaToggle => "windows_uia_toggle", + ActionTransport::WindowsUiaSelection => "windows_uia_selection", + ActionTransport::WindowsUiaExpandCollapse => "windows_uia_expand_collapse", + ActionTransport::WindowsUiaValue => "windows_uia_value", + ActionTransport::WindowsUiaRangeValue => "windows_uia_range_value", + ActionTransport::WindowsUiaScroll => "windows_uia_scroll", + ActionTransport::WindowsMsaaAction => "windows_msaa_action", + ActionTransport::WindowsPostMessage => "windows_post_message", + ActionTransport::WindowsTargetedInjection => "windows_targeted_injection", + ActionTransport::WindowsSendInput => "windows_send_input", + ActionTransport::WindowsSetCursorPos => "windows_set_cursor_pos", + ActionTransport::WindowsShellExecute => "windows_shell_execute", + ActionTransport::LinuxAtSpiAction => "linux_at_spi_action", + ActionTransport::LinuxAtSpiValue => "linux_at_spi_value", + ActionTransport::LinuxPty => "linux_pty", + ActionTransport::LinuxXSendEvent => "linux_x_send_event", + ActionTransport::LinuxXTest => "linux_x_test", + ActionTransport::LinuxLibei => "linux_libei", + ActionTransport::LinuxWaylandVirtualPointer => "linux_wayland_virtual_pointer", + ActionTransport::LinuxCuaCompositorInject => "linux_cua_compositor_inject", + ActionTransport::BrowserCdpInputMouse => "browser_cdp_input_mouse", + ActionTransport::BrowserCdpInputKey => "browser_cdp_input_key", + ActionTransport::BrowserCdpRuntimeFunction => "browser_cdp_runtime_function", + } +} + +/// Builder that requires callers to state both effect and selected transport. +#[derive(Clone, Debug)] +pub struct ActionExecutionRecordBuilder(ActionExecutionRecord); + +impl ActionExecutionRecordBuilder { + pub fn new( + effect: ActionEffect, + transport: ActionTransport, + requested_delivery: RequestedDelivery, + ) -> Self { + Self(ActionExecutionRecord::new( + effect, + transport, + requested_delivery, + )) + } + + pub fn actual_delivery(mut self, delivery: ActualDelivery) -> Self { + self.0.actual_delivery = Some(delivery); + self + } + + pub fn attempt(mut self, attempt: ActionAttempt) -> Self { + self.0.attempts.push(attempt); + self + } + + pub fn fallback(mut self, fallback: ActionFallback) -> Self { + self.0.fallbacks.push(fallback); + self + } + + pub fn evidence(mut self, evidence: ActionEvidence) -> Self { + self.0.evidence.push(evidence); + self + } + + pub fn escalation(mut self, escalation: ActionEscalation) -> Self { + self.0.escalation = Some(escalation); + self + } + + pub fn delivered_count(mut self, delivered_count: u32) -> Self { + self.0.delivered_count = Some(delivered_count); + self + } + + pub fn detail(mut self, detail: impl Into) -> Self { + self.0.detail = Some(detail.into()); + self + } + + pub fn build(self) -> Result { + self.0.validate()?; + Ok(self.0) + } +} + +/// Projection deliberately excludes request target, coordinates, and scope. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ActionOutcomeProjection { + pub effect: ActionEffect, + pub route: ActionRoute, + pub delivery: Option, + pub evidence: Option>, + pub escalation: Option, +} + +/// Published delivery accounting; the original request is intentionally absent. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ActionDeliveryProjection { + pub actual: ActualDelivery, + pub delivered_count: Option, +} + +/// Evidence families that may appear in the stable projection. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ActionEvidenceProjection { + pub kind: ProjectedEvidenceKind, + pub detail: String, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ProjectedEvidenceKind { + AccessibilityReadback, + BrowserReadback, + WindowChange, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ActionRecordValidationError { + ConfirmedRequiresEvidence, + PartialRequiresDeliveredCount, + RefusedCannotHaveDelivery, + RefusedCannotHaveEvidence, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn evidence() -> ActionEvidence { + ActionEvidence { + kind: EvidenceKind::AccessibilityReadback, + detail: "value changed".to_owned(), + } + } + + #[test] + fn every_transport_maps_to_one_of_the_five_stable_routes() { + let mut routes = Vec::new(); + for transport in ActionTransport::ALL { + let record = ActionExecutionRecord::builder( + ActionEffect::Unverifiable, + *transport, + RequestedDelivery::Foreground, + ) + .actual_delivery(ActualDelivery::Unknown) + .build() + .unwrap(); + let route = record.stable_projection().unwrap().route; + assert_eq!(route, transport.route()); + routes.push(route); + } + assert!(routes.contains(&ActionRoute::Accessibility)); + assert!(routes.contains(&ActionRoute::SyntheticEvents)); + assert!(routes.contains(&ActionRoute::GlobalInput)); + assert!(routes.contains(&ActionRoute::Dom)); + assert!(routes.contains(&ActionRoute::TrustedInput)); + } + + #[test] + fn confirmed_requires_evidence() { + assert_eq!( + ActionExecutionRecord::new( + ActionEffect::Confirmed, + ActionTransport::MacosAxAction, + RequestedDelivery::Background, + ) + .validate(), + Err(ActionRecordValidationError::ConfirmedRequiresEvidence) + ); + + assert!(ActionExecutionRecord::builder( + ActionEffect::Confirmed, + ActionTransport::MacosAxAction, + RequestedDelivery::Background, + ) + .evidence(evidence()) + .build() + .is_ok()); + } + + #[test] + fn partial_requires_delivered_count() { + assert_eq!( + ActionExecutionRecord::new( + ActionEffect::Partial, + ActionTransport::WindowsSendInput, + RequestedDelivery::Foreground, + ) + .validate(), + Err(ActionRecordValidationError::PartialRequiresDeliveredCount) + ); + + let mut record = ActionExecutionRecord::new( + ActionEffect::Partial, + ActionTransport::WindowsSendInput, + RequestedDelivery::Foreground, + ); + record.delivered_count = Some(1); + assert_eq!( + record.validate(), + Err(ActionRecordValidationError::PartialRequiresDeliveredCount), + "a count without an actual delivery cannot form a valid public partial result" + ); + } + + #[test] + fn refused_cannot_claim_delivery_or_evidence() { + let delivery = ActionExecutionRecord::builder( + ActionEffect::Refused, + ActionTransport::BrowserCdpInputMouse, + RequestedDelivery::NotApplicable, + ) + .actual_delivery(ActualDelivery::Unknown) + .build(); + assert_eq!( + delivery, + Err(ActionRecordValidationError::RefusedCannotHaveDelivery) + ); + + let evidence = ActionExecutionRecord::builder( + ActionEffect::Refused, + ActionTransport::BrowserCdpInputMouse, + RequestedDelivery::NotApplicable, + ) + .evidence(evidence()) + .build(); + assert_eq!( + evidence, + Err(ActionRecordValidationError::RefusedCannotHaveEvidence) + ); + } + + #[test] + fn projection_contains_only_outcome_information() { + let projection = ActionExecutionRecord::builder( + ActionEffect::Confirmed, + ActionTransport::LinuxLibei, + RequestedDelivery::Foreground, + ) + .actual_delivery(ActualDelivery::Foreground) + .evidence(evidence()) + .delivered_count(1) + .detail("portal session accepted input") + .build() + .unwrap() + .stable_projection() + .unwrap(); + + assert_eq!(projection.effect, ActionEffect::Confirmed); + assert_eq!(projection.route, ActionRoute::GlobalInput); + assert_eq!( + projection.delivery, + Some(ActionDeliveryProjection { + actual: ActualDelivery::Foreground, + delivered_count: Some(1), + }) + ); + assert!(projection.evidence.is_some()); + } + + #[test] + fn projection_filters_non_publishable_evidence() { + let record = ActionExecutionRecord::builder( + ActionEffect::Unverifiable, + ActionTransport::MacosCgEventHid, + RequestedDelivery::Foreground, + ) + .evidence(ActionEvidence { + kind: EvidenceKind::ScreenshotComparison, + detail: "pixels changed".to_owned(), + }) + .build() + .unwrap(); + + assert_eq!(record.stable_projection().unwrap().evidence, None); + } + + #[test] + fn escalation_projection_preserves_decision_critical_reasons() { + use cua_driver_contract::{ActionEscalationReason, ActionEscalationTarget}; + + let cases = [ + ( + EscalationKind::RetryWithPixelTarget, + ActionEscalationTarget::Pixel, + ActionEscalationReason::EffectUnconfirmed, + ), + ( + EscalationKind::RetryWithPageAction, + ActionEscalationTarget::Page, + ActionEscalationReason::EffectUnconfirmed, + ), + ( + EscalationKind::RefreshPageState, + ActionEscalationTarget::Page, + ActionEscalationReason::RouteUnavailable, + ), + ( + EscalationKind::RetryWithForegroundDelivery, + ActionEscalationTarget::Foreground, + ActionEscalationReason::DeliveryFailed, + ), + ( + EscalationKind::RequestPermission, + ActionEscalationTarget::Session, + ActionEscalationReason::PermissionRequired, + ), + ( + EscalationKind::ExpandCaptureScope, + ActionEscalationTarget::Session, + ActionEscalationReason::RouteUnavailable, + ), + ( + EscalationKind::PrepareSession, + ActionEscalationTarget::Session, + ActionEscalationReason::RouteUnavailable, + ), + ]; + + for (kind, target, reason) in cases { + let result = ActionExecutionRecord::builder( + ActionEffect::Unverifiable, + ActionTransport::MacosCgEventPid, + RequestedDelivery::Background, + ) + .escalation(ActionEscalation { kind, detail: None }) + .build() + .unwrap() + .public_result() + .unwrap(); + assert_eq!( + result.escalation, + Some(cua_driver_contract::ActionEscalation { target, reason }) + ); + } + + let permission = ActionExecutionRecord::builder( + ActionEffect::SuspectedNoop, + ActionTransport::MacosCgEventPid, + RequestedDelivery::Background, + ) + .escalation(ActionEscalation { + kind: EscalationKind::RequestPermission, + detail: None, + }) + .build() + .unwrap() + .public_result() + .unwrap(); + assert_eq!( + permission.escalation.unwrap().reason, + ActionEscalationReason::PermissionRequired, + "suspected-noop classification must not hide a permission blocker" + ); + } + + #[test] + fn public_result_is_the_closed_contract_not_a_serialized_debug_record() { + let result = ActionExecutionRecord::builder( + ActionEffect::Confirmed, + ActionTransport::MacosAxValue, + RequestedDelivery::Background, + ) + .actual_delivery(ActualDelivery::Background) + .evidence(ActionEvidence { + kind: EvidenceKind::AccessibilityReadback, + detail: "secret request-adjacent diagnostic".to_owned(), + }) + .detail("pid=42 window_id=7 x=10 y=20") + .build() + .unwrap() + .public_result() + .unwrap(); + + let value = serde_json::to_value(result).unwrap(); + assert_eq!( + value, + serde_json::json!({ + "effect": "confirmed", + "route": "accessibility", + "delivery": {"mode": "background"}, + "evidence": [{"kind": "value_readback"}] + }) + ); + let rendered = value.to_string(); + for forbidden in [ + "secret", + "pid", + "window_id", + "\"x\"", + "\"y\"", + "macos_ax_value", + "requested_delivery", + ] { + assert!( + !rendered.contains(forbidden), + "public ActionResult leaked {forbidden}: {rendered}" + ); + } + } + + #[test] + fn confirmed_rejects_debug_only_evidence() { + let record = ActionExecutionRecord::builder( + ActionEffect::Confirmed, + ActionTransport::MacosCgEventHid, + RequestedDelivery::Foreground, + ) + .evidence(ActionEvidence { + kind: EvidenceKind::ScreenshotComparison, + detail: "pixels changed".to_owned(), + }) + .build(); + assert_eq!( + record, + Err(ActionRecordValidationError::ConfirmedRequiresEvidence) + ); + } + + #[test] + fn legacy_value_readback_preserves_truth_without_echoing_target() { + let args = serde_json::json!({ + "pid": 42, + "window_id": 77, + "x": 12, + "y": 18, + "delivery_mode": "background", + }); + let structured = serde_json::json!({ + "path": "ax", + "verified": true, + "verify": "confirmed", + "effect": "confirmed", + }); + let record = ActionExecutionRecord::from_legacy("type_text", &args, &structured) + .expect("legacy action should normalize"); + assert_eq!(record.effect, ActionEffect::Confirmed); + assert_eq!(record.requested_delivery, RequestedDelivery::Background); + assert_eq!(record.actual_delivery, Some(ActualDelivery::Background)); + let debug = record.debug_json(); + let rendered = debug.to_string(); + assert!(!rendered.contains("\"pid\"")); + assert!(!rendered.contains("\"window_id\"")); + assert!(!rendered.contains("\"x\"")); + assert!(!rendered.contains("\"y\"")); + } + + #[test] + fn legacy_pointer_and_key_acknowledgements_cannot_become_confirmed() { + for tool in [ + "click", + "double_click", + "right_click", + "scroll", + "drag", + "mouse_drag", + "parallel_mouse_drag", + "mouse_button_down", + "mouse_button_up", + "press_key", + "hotkey", + ] { + let record = ActionExecutionRecord::from_legacy( + tool, + &serde_json::json!({"delivery_mode": "background"}), + &serde_json::json!({ + "path": "ax", + "verified": true, + "verify": "confirmed", + "effect": "confirmed", + }), + ) + .unwrap_or_else(|| panic!("{tool} legacy action should normalize")); + assert_eq!( + record.effect, + ActionEffect::Unverifiable, + "{tool} cannot confirm from a delivery acknowledgement" + ); + assert!(record.evidence.is_empty()); + } + } + + #[test] + fn browser_pointer_routes_share_the_closed_action_contract() { + for (input_route, expected_route) in [ + ("trusted", cua_driver_contract::ActionRoute::TrustedInput), + ("dom_event", cua_driver_contract::ActionRoute::Dom), + ] { + let args = serde_json::json!({ + "session": "browser-test", + "target_id": "secret-target", + "tab_id": "secret-tab", + "action": "drag", + "ref": "secret-ref", + "destination_ref": "secret-destination", + "input_route": input_route, + }); + let structured = serde_json::json!({ + "status": "ok", + "route": input_route, + "target_id": "secret-target", + "tab_id": "secret-tab", + "ref": "secret-ref", + "destination_ref": "secret-destination", + "x": 10, + "y": 20, + }); + let record = ActionExecutionRecord::from_legacy("browser_pointer", &args, &structured) + .expect("browser pointer action should normalize"); + let public = record + .public_result() + .expect("browser pointer action should publish"); + assert_eq!( + public.effect, + cua_driver_contract::ActionEffect::Unverifiable + ); + assert_eq!(public.route, expected_route); + assert_eq!( + public.delivery.as_ref().map(|delivery| delivery.mode), + Some(cua_driver_contract::ActionDeliveryMode::Background) + ); + let rendered = serde_json::to_string(&public).expect("serialize ActionResult"); + for forbidden in [ + "secret-target", + "secret-tab", + "secret-ref", + "secret-destination", + "\"x\"", + "\"y\"", + ] { + assert!( + !rendered.contains(forbidden), + "browser ActionResult leaked {forbidden}: {rendered}" + ); + } + } + } + + #[test] + fn browser_refusals_and_partial_delivery_survive_projection() { + let refused = ActionExecutionRecord::from_legacy( + "browser_click", + &serde_json::json!({"input_route": "trusted"}), + &serde_json::json!({ + "status": "refused", + "refusal": { + "code": "browser_ref_stale", + "message": "refresh browser state" + } + }), + ) + .expect("browser refusal should normalize"); + let refused = refused.public_result().expect("refusal should project"); + assert_eq!(refused.effect, cua_driver_contract::ActionEffect::Refused); + assert!(refused.delivery.is_none()); + assert_eq!( + refused.escalation, + Some(cua_driver_contract::ActionEscalation { + target: cua_driver_contract::ActionEscalationTarget::Page, + reason: cua_driver_contract::ActionEscalationReason::RouteUnavailable, + }) + ); + + let partial = ActionExecutionRecord::from_legacy( + "browser_type", + &serde_json::json!({}), + &serde_json::json!({ + "status": "refused", + "refusal": { + "code": "browser_input_incomplete", + "message": "typing stopped", + "detail": { + "requested_chars": 4, + "delivered_chars": 2 + } + } + }), + ) + .expect("partial browser input should normalize"); + let partial = partial.public_result().expect("partial should project"); + assert_eq!(partial.effect, cua_driver_contract::ActionEffect::Partial); + assert_eq!( + partial + .delivery + .as_ref() + .and_then(|delivery| delivery.delivered_count), + Some(2) + ); + assert!(partial.escalation.is_none()); + } + + #[test] + fn successful_foreground_pixel_branch_reports_actual_global_delivery() { + let record = ActionExecutionRecord::from_legacy( + "click", + &serde_json::json!({"delivery_mode": "foreground"}), + &serde_json::json!({ + "path": "pixel", + "verified": false, + "effect": "unverifiable", + }), + ) + .expect("foreground pixel action should normalize"); + assert_eq!(record.actual_delivery, Some(ActualDelivery::Foreground)); + assert_eq!(record.transport.route(), ActionRoute::GlobalInput); + let public = record.public_result().expect("public ActionResult"); + assert_eq!( + public.delivery.map(|delivery| delivery.mode), + Some(cua_driver_contract::ActionDeliveryMode::Foreground) + ); + assert_eq!(public.route, cua_driver_contract::ActionRoute::GlobalInput); + } + + #[test] + fn producer_emitted_delivery_wins_over_requested_delivery() { + let record = ActionExecutionRecord::from_legacy( + "scroll", + &serde_json::json!({"delivery_mode": "foreground"}), + &serde_json::json!({ + "path": "uia", + "delivery_mode": "background", + "verified": false, + "effect": "unverifiable", + }), + ) + .expect("producer delivery fact should normalize"); + assert_eq!(record.actual_delivery, Some(ActualDelivery::Background)); + assert_eq!(record.transport.route(), ActionRoute::Accessibility); + } + + #[test] + fn legacy_confirmation_without_readback_is_downgraded() { + let record = ActionExecutionRecord::from_legacy( + "type_text", + &serde_json::json!({"delivery_mode": "background"}), + &serde_json::json!({ + "path": "key_events", + "effect": "confirmed", + "characters": 3, + }), + ) + .expect("legacy action should normalize"); + assert_eq!(record.effect, ActionEffect::Unverifiable); + assert!(record.evidence.is_empty()); + assert_eq!( + record.delivered_count, None, + "legacy request-count echoes are not delivery evidence" + ); + } + + #[test] + fn legacy_partial_requires_an_explicit_delivered_count() { + assert!( + ActionExecutionRecord::from_legacy( + "browser_type", + &serde_json::json!({}), + &serde_json::json!({ + "route": "trusted", + "effect": "partial", + "chars": 3, + "requested_chars": 3, + }), + ) + .is_none(), + "requested character counts must not be published as delivered counts" + ); + + let record = ActionExecutionRecord::from_legacy( + "browser_type", + &serde_json::json!({}), + &serde_json::json!({ + "route": "trusted", + "effect": "partial", + "requested_chars": 3, + "delivered_chars": 2, + }), + ) + .expect("an explicit delivered count should normalize"); + assert_eq!(record.delivered_count, Some(2)); + } + + #[test] + fn every_known_legacy_action_path_normalizes() { + let paths = [ + "ax", + "ax_fg", + "uia", + "uia_expand_collapse", + "msaa", + "post_message", + "PostMessage", + "SendInput", + "SetCursorPos", + "atspi", + "wayland_atspi", + "x11_atspi", + "pty", + "x11_pixel", + "x11_pixel_fg", + "x11_xtest_fg", + "xtest", + "xtest_desktop", + "wayland_activate", + "wayland_focused", + "wayland_desktop", + "cua_compositor_inject", + "wayland_cua_compositor", + "hid", + "cgevent_hid", + "cgevent", + "cgevent_fg", + "dom_event", + "trusted", + "key_events", + "key_events_fg", + "pixel", + ]; + for path in paths { + assert!( + ActionExecutionRecord::from_legacy( + "click", + &serde_json::json!({"delivery_mode": "background"}), + &serde_json::json!({ + "path": path, + "effect": "unverifiable", + }), + ) + .is_some(), + "legacy path {path} must normalize before the breaking cutover" + ); + } + } +} diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs index 1e77be9656..872cc106fc 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs @@ -45,6 +45,7 @@ pub fn parent_liveness_stdin_enabled() -> bool { embedded_mode() && std::env::var_os(PARENT_LIVENESS_STDIN_ENV).is_some_and(|value| value == "1") } +pub mod action_record; pub mod authorization; pub mod browser; pub mod capture_mode; diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/protocol.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/protocol.rs index 94a1211cfb..70780ead1c 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/protocol.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/protocol.rs @@ -289,6 +289,11 @@ pub struct ToolResult { pub is_error: Option, #[serde(rename = "structuredContent", skip_serializing_if = "Option::is_none")] pub structured_content: Option, + /// Rich actuator facts retained inside the daemon. This is deliberately + /// skipped by serde so the nonbreaking truth-layer migration cannot alter + /// the MCP result envelope or its legacy structured payload. + #[serde(skip)] + pub action_record: Option, } impl ToolResult { @@ -311,6 +316,14 @@ impl ToolResult { self.structured_content = Some(v); self } + + pub fn with_action_record( + mut self, + record: crate::action_record::ActionExecutionRecord, + ) -> Self { + self.action_record = Some(record); + self + } } // ── Initialize result ───────────────────────────────────────────────────────── @@ -417,6 +430,40 @@ mod image_mime_type_tests { } } +#[cfg(test)] +mod action_record_wire_tests { + use super::ToolResult; + use crate::action_record::{ + ActionEffect, ActionExecutionRecord, ActionTransport, ActualDelivery, RequestedDelivery, + }; + + #[test] + fn internal_action_record_never_changes_mcp_serialization() { + let legacy = serde_json::json!({ + "path": "cgevent", + "verified": false, + "effect": "unverifiable", + }); + let plain = ToolResult::text("clicked").with_structured(legacy.clone()); + let with_truth = ToolResult::text("clicked") + .with_structured(legacy) + .with_action_record( + ActionExecutionRecord::builder( + ActionEffect::Unverifiable, + ActionTransport::MacosCgEventPid, + RequestedDelivery::Background, + ) + .actual_delivery(ActualDelivery::Background) + .build() + .expect("valid action record"), + ); + assert_eq!( + serde_json::to_value(plain).expect("serialize plain result"), + serde_json::to_value(with_truth).expect("serialize result with internal truth"), + ); + } +} + #[cfg(test)] mod agent_instruction_tests { use super::agent_instructions; diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs index 82b45129a6..07f676128c 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs @@ -573,6 +573,18 @@ impl RecordingSession { /// Finalize a previously reserved turn after tool dispatch. pub fn finish_turn(&self, pending: PendingTurn, result_text: &str) { + self.finish_turn_with_action(pending, result_text, None); + } + + /// Finalize a turn while retaining the daemon's rich, non-wire action + /// truth in the recording artifact. Existing trajectory readers can ignore + /// the additive `action_truth` key. + pub fn finish_turn_with_action( + &self, + pending: PendingTurn, + result_text: &str, + action_record: Option<&crate::action_record::ActionExecutionRecord>, + ) { let mut inner = self.inner.lock().unwrap(); if !inner.enabled || inner.generation != pending.generation { tracing::warn!( @@ -581,7 +593,7 @@ impl RecordingSession { ); return; } - if let Err(error) = write_turn(pending, result_text) { + if let Err(error) = write_turn(pending, result_text, action_record) { inner.last_error = Some(error.to_string()); } } @@ -721,7 +733,11 @@ fn strip_internal_keys(args: &Value) -> std::borrow::Cow<'_, Value> { } } -fn write_turn(pending: PendingTurn, result_text: &str) -> anyhow::Result<()> { +fn write_turn( + pending: PendingTurn, + result_text: &str, + action_record: Option<&crate::action_record::ActionExecutionRecord>, +) -> anyhow::Result<()> { let PendingTurn { generation: _, turn_dir, @@ -759,6 +775,9 @@ fn write_turn(pending: PendingTurn, result_text: &str) -> anyhow::Result<()> { if let Some((cx, cy)) = click_point { payload["click_point"] = serde_json::json!({"x": cx, "y": cy}); } + if let Some(action_record) = action_record { + payload["action_truth"] = action_record.debug_json(); + } write_json_atomic(&turn_dir.join("action.json"), &payload)?; write_phase_artifacts(&turn_dir, "after", &after)?; @@ -938,7 +957,15 @@ mod tests { assert_eq!(std::fs::read(turn.join("before.png")).unwrap(), b"before"); assert!(!turn.join("after.png").exists()); - session.finish_turn(pending, "clicked"); + let action_record = crate::action_record::ActionExecutionRecord::builder( + crate::action_record::ActionEffect::Unverifiable, + crate::action_record::ActionTransport::MacosCgEventPid, + crate::action_record::RequestedDelivery::Background, + ) + .actual_delivery(crate::action_record::ActualDelivery::Background) + .build() + .expect("valid action record"); + session.finish_turn_with_action(pending, "clicked", Some(&action_record)); assert_eq!(std::fs::read(turn.join("after.png")).unwrap(), b"after"); assert_eq!( std::fs::read(turn.join("screenshot.png")).unwrap(), @@ -949,6 +976,13 @@ mod tests { std::fs::read(turn.join("after_state.json")).unwrap() ); assert_eq!(std::fs::read(turn.join("click.png")).unwrap(), b"click"); + let action: Value = serde_json::from_slice( + &std::fs::read(turn.join("action.json")).expect("read action truth"), + ) + .expect("parse action truth"); + assert_eq!(action["action_truth"]["effect"], "unverifiable"); + assert_eq!(action["action_truth"]["route"], "synthetic_events"); + assert_eq!(action["action_truth"]["requested_delivery"], "background"); let snapshot_id = crate::element_token::global().register_snapshot(1, 77, 1); let token = crate::element_token::token_for(snapshot_id, 0); diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/server.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/server.rs index f093d6e298..3f11d19c63 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/server.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/server.rs @@ -730,15 +730,39 @@ fn structured_refusal_code(tool_name: &str, result: Option<&serde_json::Value>) if structured .and_then(|value| value.get("status")) .and_then(serde_json::Value::as_str) + == Some("refused") + { + return ToolRefusalCode::from_wire_code( + structured + .and_then(|value| value.pointer("/refusal/code")) + .and_then(serde_json::Value::as_str), + ); + } + if structured + .and_then(|value| value.get("effect")) + .and_then(serde_json::Value::as_str) != Some("refused") { return ToolRefusalCode::None; } - ToolRefusalCode::from_wire_code( - structured - .and_then(|value| value.pointer("/refusal/code")) - .and_then(serde_json::Value::as_str), - ) + + // ActionResult deliberately keeps refusal details out of the narrow + // structured contract. The original bounded text is preserved at the MCP + // boundary, so recover only the closed wire code from its stable prefix; + // refusal prose and details never cross the observer seam. + let wire_code = result + .and_then(|value| value.get("content")) + .and_then(serde_json::Value::as_array) + .and_then(|items| { + items.iter().find_map(|item| { + item.get("text") + .and_then(serde_json::Value::as_str) + .and_then(|text| text.strip_prefix("refused (")) + .and_then(|tail| tail.split_once("):")) + .map(|(code, _)| code) + }) + }); + ToolRefusalCode::from_wire_code(wire_code) } fn serialized_size_without_retaining(value: &serde_json::Value) -> Option { @@ -1192,6 +1216,64 @@ mod observation_tests { } } + #[test] + fn action_result_browser_refusals_preserve_closed_telemetry_code() { + let response = Response::ok( + serde_json::json!(1), + serde_json::json!({ + "content": [{ + "type": "text", + "text": "refused (browser_ref_stale): private refusal prose" + }], + "structuredContent": { + "effect": "refused", + "route": "dom", + "actual_delivery": {"mode": "not_applicable"} + } + }), + ); + let observation = ToolObservationTimer::start_with_operation( + "browser_click".to_owned(), + ToolOperation::BrowserClickTrusted, + true, + true, + StdioExecutionPath::DirectDaemon, + ) + .finish(&response); + assert!(observation.success); + assert_eq!(observation.refusal_code, ToolRefusalCode::BrowserRefStale); + assert!( + !format!("{observation:?}").contains("private refusal prose"), + "observer retained refusal prose" + ); + } + + #[test] + fn action_result_refusal_without_a_known_text_code_is_other() { + let response = Response::ok( + serde_json::json!(1), + serde_json::json!({ + "content": [{"type": "text", "text": "refused without a stable prefix"}], + "structuredContent": { + "effect": "refused", + "route": "dom", + "actual_delivery": {"mode": "not_applicable"} + } + }), + ); + assert_eq!( + ToolObservationTimer::start( + "browser_click".to_owned(), + true, + true, + StdioExecutionPath::DirectDaemon, + ) + .finish(&response) + .refusal_code, + ToolRefusalCode::Other + ); + } + #[test] fn browser_refusal_allowlist_matches_the_public_contract() { use crate::browser::refusal::BrowserRefusalCode; diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs index ea34bb684c..21a281518d 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs @@ -120,7 +120,7 @@ impl ToolDef { // contract. The legacy map remains only for runtime-only tools. let caps = advertised_capabilities_for(&self.name, &self.input_schema); let risk = crate::authorization::risk_metadata_json(&self.name); - serde_json::json!({ + let mut entry = serde_json::json!({ "name": self.name, "description": self.description, "inputSchema": self.input_schema, @@ -132,7 +132,23 @@ impl ToolDef { }, "capabilities": caps, "risk": risk, - }) + }); + let output_schema = if crate::action_record::is_action_tool(&self.name) { + Some( + ::output_schema( + ), + ) + } else { + cua_driver_contract::tool_contract(&self.name) + .and_then(|contract| contract.success_output_schema) + }; + if let Some(output_schema) = output_schema { + entry + .as_object_mut() + .expect("tool list entry is an object") + .insert("outputSchema".into(), output_schema); + } + entry } } @@ -1239,6 +1255,24 @@ impl ToolRegistry { .flatten(); let mut result = tool.invoke(args.clone()).await; + if result.action_record.is_none() { + if let Some(structured) = result.structured_content.as_ref() { + result.action_record = crate::action_record::ActionExecutionRecord::from_legacy( + resolved_name, + &public_args, + structured, + ); + } else if result.is_error != Some(true) { + // Some legacy successful actions return text only. Normalize + // them through an empty payload so internal accounting exists + // before the public contract cutover. + result.action_record = crate::action_record::ActionExecutionRecord::from_legacy( + resolved_name, + &public_args, + &Value::Null, + ); + } + } if resolved_name == "launch_app" && result.is_error != Some(true) { if let (Some(session), Some(before), Some(pid)) = ( runtime_session.as_deref(), @@ -1270,14 +1304,30 @@ impl ToolRegistry { crate::session::touch_session(session); } restore_public_runtime_result(&mut result, &runtime_prefix); - let validate_portable_output = match resolved_name { - "get_desktop_state" | "get_screen_size" | "get_cursor_position" => true, - "move_cursor" | "click" | "drag" | "scroll" | "type_text" | "press_key" | "hotkey" => { - args.get("scope").and_then(Value::as_str) == Some("desktop") + // Preserve the producer's private summary for recording/replay before + // the public ActionResult projection deliberately replaces legacy + // prose. Coordinate recovery and other internal diagnostics must not + // depend on the narrowed MCP text surface. + let recording_result_text = result.content.iter().find_map(|content| { + if let Content::Text { text, .. } = content { + Some(text.clone()) + } else { + None } - _ => true, - }; - if result.is_error != Some(true) && validate_portable_output { + }); + if result.is_error != Some(true) && crate::action_record::is_action_tool(resolved_name) { + if let Err(error) = publish_action_result(&mut result) { + result = ToolResult::error(format!( + "internal action outcome mismatch for {resolved_name}: {error}" + )) + .with_structured(serde_json::json!({ + "code": "action_outcome_mismatch", + "tool": resolved_name, + "detail": error, + })); + } + } + if result.is_error != Some(true) { if let Some(structured) = result.structured_content.clone() { if let Err(error) = cua_driver_contract::validate_success_output(resolved_name, structured) @@ -1303,18 +1353,11 @@ impl ToolRegistry { // stream stays the actual user-action sequence (not the meta // start/stop frames). if let Some(pending_turn) = pending_turn { - let result_text = result - .content - .iter() - .find_map(|c| { - if let Content::Text { text, .. } = c { - Some(text.as_str()) - } else { - None - } - }) - .unwrap_or(""); - self.recording.finish_turn(pending_turn, result_text); + self.recording.finish_turn_with_action( + pending_turn, + recording_result_text.as_deref().unwrap_or(""), + result.action_record.as_ref(), + ); } // Experimental PiP push — only when --experimental-pip is on argv @@ -2154,6 +2197,27 @@ fn namespace_runtime_args( runtime_prefix } +fn publish_action_result(result: &mut ToolResult) -> Result<(), String> { + let action = result + .action_record + .as_ref() + .ok_or_else(|| "successful action omitted its internal execution record".to_owned())?; + let public = action + .public_result() + .map_err(|error| format!("invalid internal execution record: {error:?}"))?; + public + .validate_invariants() + .map_err(|error| format!("invalid public projection: {error}"))?; + let structured = + serde_json::to_value(public).map_err(|error| format!("projection failed: {error}"))?; + // The breaking contract narrows machine-readable structured content. Keep + // the outer ToolResult text/images intact for human diagnostics, refusal + // messages, recording/replay, and clients that intentionally degrade to + // raw content. + result.structured_content = Some(structured); + Ok(()) +} + fn restore_public_runtime_result(result: &mut ToolResult, runtime_prefix: &str) { for content in &mut result.content { if let Content::Text { text, .. } = content { @@ -2206,7 +2270,8 @@ fn restore_public_runtime_value(value: &mut Value, runtime_prefix: &str) -> bool mod runtime_isolation_tests { use super::{ canonical_proposed_path, desktop_action_coordinator, namespace_runtime_args, - restore_public_runtime_result, TrustedInvocationEvidence, DISPATCH_RUNTIME_SCOPE, + publish_action_result, restore_public_runtime_result, TrustedInvocationEvidence, + DISPATCH_RUNTIME_SCOPE, }; use crate::{ authorization::PermissionMode, @@ -3290,6 +3355,18 @@ resources: .invoke_with_context("click", args.clone(), context.clone()) .await; assert_ne!(first.is_error, Some(true)); + let action = first + .action_record + .as_ref() + .expect("canonical dispatch should normalize legacy action truth"); + assert_eq!( + action.effect, + crate::action_record::ActionEffect::Unverifiable + ); + assert!(action + .debug_json() + .get("requested_delivery") + .is_some_and(|value| value == "background")); assert_eq!(hits.load(Ordering::SeqCst), 1); crate::session::fire_session_end(&runtime_session); @@ -3298,6 +3375,89 @@ resources: assert_eq!(hits.load(Ordering::SeqCst), 1); } + #[tokio::test] + async fn canonical_dispatch_replaces_legacy_action_payload_with_the_closed_outcome() { + let hits = Arc::new(AtomicUsize::new(0)); + let registry = input_registry(None, hits.clone()); + let context = standard_context(); + let runtime_session = context.runtime_session_key("projection"); + registry + .protected_resource_ownership() + .mark_driver_owned_pid(&runtime_session, 42); + + let result = registry + .invoke_with_context( + "click", + serde_json::json!({ + "pid": 42, + "window_id": 7, + "session": "projection", + "x": 10, + "y": 20 + }), + context, + ) + .await; + + assert_ne!(result.is_error, Some(true)); + let structured = result + .structured_content + .expect("successful actions publish an ActionResult"); + let object = structured.as_object().expect("ActionResult is an object"); + assert_eq!( + object.keys().map(String::as_str).collect::>(), + ["delivery", "effect", "route"] + ); + assert_eq!(structured["effect"], "unverifiable"); + assert_eq!(structured["delivery"]["mode"], "unknown"); + assert!(matches!( + structured["route"].as_str(), + Some("accessibility" | "synthetic_events" | "global_input" | "dom" | "trusted_input") + )); + assert!(structured.get("snapshot_id").is_none()); + assert!(structured.get("x").is_none()); + assert!(structured.get("y").is_none()); + assert!(structured.get("scope").is_none()); + assert!(structured.get("verified").is_none()); + cua_driver_contract::validate_success_output("click", structured) + .expect("the projected action must satisfy the public contract"); + assert_eq!(hits.load(Ordering::SeqCst), 1); + } + + #[test] + fn action_projection_keeps_browser_refusal_diagnostics_and_closes_structured_content() { + let legacy = serde_json::json!({ + "status": "refused", + "refusal": { + "code": "browser_ref_stale", + "message": "take a fresh browser snapshot" + } + }); + let mut result = crate::protocol::ToolResult::text( + "refused (browser_ref_stale): take a fresh browser snapshot", + ) + .with_structured(legacy.clone()); + result.action_record = crate::action_record::ActionExecutionRecord::from_legacy( + "browser_click", + &serde_json::json!({"input_route": "trusted"}), + &legacy, + ); + + publish_action_result(&mut result).expect("browser refusal should project"); + let structured = result.structured_content.as_ref().unwrap(); + assert_eq!(structured["effect"], "refused"); + assert_eq!(structured["route"], "trusted_input"); + assert_eq!(structured["escalation"]["target"], "page"); + assert!(structured.get("refusal").is_none()); + assert!(result.content.iter().any(|content| { + matches!( + content, + crate::protocol::Content::Text { text, .. } + if text.contains("browser_ref_stale") + ) + })); + } + #[tokio::test] async fn element_tokens_are_bound_to_the_dispatch_runtime_generation() { let pid = 8_675_309; @@ -4081,6 +4241,56 @@ mod capability_tests { assert!(entry["capabilities"].is_array()); } + #[test] + fn action_tools_advertise_the_same_closed_output_schema() { + let expected = + ::output_schema(); + for name in ["click", "browser_click", "browser_pointer", "browser_type"] { + let def = ToolDef { + name: name.into(), + description: "Action.".into(), + input_schema: serde_json::json!({"type":"object","properties":{}}), + read_only: false, + destructive: false, + idempotent: false, + open_world: true, + }; + let entry = def.to_list_entry(); + assert_eq!(entry["outputSchema"], expected, "{name}"); + assert_eq!( + entry["outputSchema"]["additionalProperties"], false, + "{name}" + ); + } + } + + #[test] + fn typed_non_action_tools_advertise_outputs_without_inventing_runtime_schemas() { + let verify = ToolDef { + name: "verify_state".into(), + description: "Verify.".into(), + input_schema: serde_json::json!({"type":"object","properties":{}}), + read_only: true, + destructive: false, + idempotent: true, + open_world: false, + } + .to_list_entry(); + assert!(verify["outputSchema"].is_object()); + + let runtime_only = ToolDef { + name: "runtime_only_probe".into(), + description: "Probe.".into(), + input_schema: serde_json::json!({"type":"object","properties":{}}), + read_only: true, + destructive: false, + idempotent: true, + open_world: false, + } + .to_list_entry(); + assert!(runtime_only.get("outputSchema").is_none()); + } + #[test] fn type_text_claims_terminal_safe_capability() { // The terminal-emulator fallback shipped per platform must be diff --git a/libs/cua-driver/rust/crates/cua-driver-sdk/src/lib.rs b/libs/cua-driver/rust/crates/cua-driver-sdk/src/lib.rs index a95719e408..542ec3f2e2 100644 --- a/libs/cua-driver/rust/crates/cua-driver-sdk/src/lib.rs +++ b/libs/cua-driver/rust/crates/cua-driver-sdk/src/lib.rs @@ -6,12 +6,12 @@ //! MCP and daemon transports are downstream adapters rather than peer contracts. use cua_driver_contract::{ - ClickInput, DragInput, EndSessionInput, EndSessionOutput, EscalateSessionInput, + ActionResult, ClickInput, DragInput, EndSessionInput, EndSessionOutput, EscalateSessionInput, GetAgentCursorStateInput, GetCursorPositionInput, GetDesktopStateInput, GetScreenSizeInput, GetSessionStateInput, HotkeyInput, MoveCursorInput, PressKeyInput, ScrollInput, SessionStateOutput, SetAgentCursorEnabledInput, SetAgentCursorMotionInput, SetAgentCursorThemeInput, StartSessionInput, StartSessionOutput, ToolInput, TypeTextInput, - VerifyStateInput, + VerifyStateInput, VerifyStateOutput, }; use cua_driver_core::daemon::{ is_daemon_listening, request_daemon_metadata, send_request, socket_path_for_namespace, @@ -58,7 +58,8 @@ pub struct ToolResult { pub structured_json: Option, pub is_error: bool, pub error_code: Option, - pub verified: Option, + pub action: Option, + pub verification: Option, pub degraded: bool, pub raw_json: String, } @@ -1388,7 +1389,7 @@ impl CuaDriverSession { } SessionBackend::Remote(session) => session.invoke(name, arguments).await?, }; - normalize_result(raw) + normalize_result(name, raw) } } @@ -1410,7 +1411,7 @@ impl CuaDriver { } if let DriverBackend::Embedded(runtime) = &self.backend { let raw = runtime.invoke_from_trusted_adapter(name, arguments).await?; - return normalize_result(raw); + return normalize_result(name, raw); } // Trust evidence is local to the adapter/runtime boundary. A private @@ -1486,11 +1487,22 @@ impl CuaDriver { })? } }; - normalize_result(raw) + normalize_result(name, raw) } } impl ToolResult { + /// Typed action facts when this result came from an action tool. + pub fn action(&self) -> Option<&ActionResult> { + self.action.as_ref() + } + + /// Typed tri-state postcondition result when this result came from + /// `verify_state`. + pub fn verification(&self) -> Option<&VerifyStateOutput> { + self.verification.as_ref() + } + fn typed_success(self, tool: &str) -> Result { if self.is_error { return Err(DriverError::Tool { @@ -1523,7 +1535,7 @@ fn parse_arguments(tool: &str, arguments_json: &str) -> Result Result { +fn normalize_result(tool: &str, raw: Value) -> Result { let object = raw.as_object().ok_or_else(|| DriverError::Protocol { reason: "tool result must be a JSON object".into(), })?; @@ -1563,9 +1575,43 @@ fn normalize_result(raw: Value) -> Result { .or_else(|| value.get("refusal")?.get("code")?.as_str()) .map(str::to_owned) }); - let verified = structured - .and_then(|value| value.get("verified")) - .and_then(Value::as_bool); + let is_error = object + .get("isError") + .and_then(Value::as_bool) + .unwrap_or(false); + let action = if !is_error && cua_driver_core::action_record::is_action_tool(tool) { + let structured = structured.ok_or_else(|| DriverError::Protocol { + reason: format!("{tool} response omitted ActionResult"), + })?; + let action = + serde_json::from_value::(structured.clone()).map_err(|error| { + DriverError::Protocol { + reason: format!("{tool} returned an invalid ActionResult: {error}"), + } + })?; + action + .validate_invariants() + .map_err(|error| DriverError::Protocol { + reason: format!("{tool} returned an invalid ActionResult: {error}"), + })?; + Some(action) + } else { + None + }; + let verification = if !is_error && tool == VerifyStateInput::TOOL_NAME { + let structured = structured.ok_or_else(|| DriverError::Protocol { + reason: "verify_state response omitted VerifyStateOutput".into(), + })?; + Some( + serde_json::from_value::(structured.clone()).map_err(|error| { + DriverError::Protocol { + reason: format!("verify_state returned an invalid VerifyStateOutput: {error}"), + } + })?, + ) + } else { + None + }; let degraded = structured .and_then(|value| value.get("degraded")) .and_then(Value::as_bool) @@ -1575,12 +1621,10 @@ fn normalize_result(raw: Value) -> Result { text: text_parts.join("\n"), images, structured_json: structured.map(Value::to_string), - is_error: object - .get("isError") - .and_then(Value::as_bool) - .unwrap_or(false), + is_error, error_code, - verified, + action, + verification, degraded, raw_json: raw.to_string(), }) @@ -2273,7 +2317,7 @@ mod tests { {"type": "text", "text": "captured"}, {"type": "image", "mimeType": "image/png", "data": "cG5n"} ], - "structuredContent": {"screenshot_width": 2, "verified": true}, + "structuredContent": {"screenshot_width": 2}, "isError": false } }); @@ -2289,7 +2333,8 @@ mod tests { .unwrap(); assert_eq!(result.text, "captured"); assert_eq!(result.images[0].mime_type, "image/png"); - assert_eq!(result.verified, Some(true)); + assert!(result.action().is_none()); + assert!(result.verification().is_none()); let request = server.join().unwrap(); assert_eq!(request["method"], "call"); @@ -2299,6 +2344,106 @@ mod tests { assert_eq!(request["client_kind"], "python_sdk"); } + #[test] + fn result_normalization_exposes_typed_action_and_verification_views() { + let action = normalize_result( + "type_text", + serde_json::json!({ + "content": [{"type": "text", "text": "Action outcome"}], + "structuredContent": { + "effect": "confirmed", + "route": "accessibility", + "delivery": {"mode": "background"}, + "evidence": [{"kind": "value_readback"}] + }, + "isError": false + }), + ) + .unwrap(); + assert_eq!( + action.action().map(|value| value.effect), + Some(cua_driver_contract::ActionEffect::Confirmed) + ); + assert!(action.verification().is_none()); + + let verification = normalize_result( + "verify_state", + serde_json::json!({ + "content": [{"type": "text", "text": "satisfied"}], + "structuredContent": { + "status": "satisfied", + "stable": true, + "elapsed_ms": 42, + "samples": 2, + "predicates": [] + }, + "isError": false + }), + ) + .unwrap(); + assert_eq!( + verification.verification().map(|value| value.status), + Some(cua_driver_contract::VerificationStatus::Satisfied) + ); + assert!(verification.action().is_none()); + } + + #[test] + fn result_normalization_rejects_an_unsubstantiated_confirmation() { + let error = normalize_result( + "browser_click", + serde_json::json!({ + "content": [{"type": "text", "text": "claimed success"}], + "structuredContent": { + "effect": "confirmed", + "route": "trusted_input" + }, + "isError": false + }), + ) + .unwrap_err(); + assert!(matches!( + error, + DriverError::Protocol { reason } + if reason.contains("confirmed effect requires evidence") + )); + } + + #[test] + fn result_normalization_fails_closed_on_legacy_or_missing_action_payloads() { + let legacy = normalize_result( + "click", + serde_json::json!({ + "content": [{"type": "text", "text": "legacy click"}], + "structuredContent": { + "path": "ax", + "verified": true + }, + "isError": false + }), + ) + .unwrap_err(); + assert!(matches!( + legacy, + DriverError::Protocol { reason } + if reason.contains("invalid ActionResult") + )); + + let missing = normalize_result( + "click", + serde_json::json!({ + "content": [{"type": "text", "text": "missing payload"}], + "isError": false + }), + ) + .unwrap_err(); + assert!(matches!( + missing, + DriverError::Protocol { reason } + if reason.contains("omitted ActionResult") + )); + } + #[tokio::test] #[cfg(unix)] async fn tool_discovery_uses_the_shared_direct_daemon_protocol() { diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/response.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/response.rs index 41418727c2..38f8255ecb 100644 --- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/response.rs +++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/response.rs @@ -72,24 +72,27 @@ impl ToolResponse { self.is_error } - // ── Best-effort-background ladder accessors ────────────────────────────── - // Action tools (type_text / click / drag / scroll) report which rung - // delivered (`path`) and whether the driver could confirm the effect - // (`verified`). These let the modality matrix assert per (surface, rung) - // outcomes instead of scraping result text. - - /// The delivery rung that ran: `"ax" | "cgevent" | "cgevent_fg" | - /// "key_events" | "key_events_fg"`. `None` when the tool reports no `path`. - pub fn path(&self) -> Option<&str> { - self.structured.get("path").and_then(Value::as_str) + // ── Action-result accessors ─────────────────────────────────────────────── + // Action tools expose only the narrow public truth contract. Tests assert + // the stable semantic route/effect instead of depending on platform + // transport names or a lossy verification boolean. + + /// The strongest effect the driver can substantiate for an action. + pub fn action_effect(&self) -> Option<&str> { + self.structured.get("effect").and_then(Value::as_str) + } + + /// The stable semantic route used by the action. + pub fn action_route(&self) -> Option<&str> { + self.structured.get("route").and_then(Value::as_str) } - /// Whether the driver confirmed the effect via read-back. `Some(true)` only - /// when verified; `Some(false)` = dispatched-but-unconfirmed (the caller - /// must confirm via screenshot — e.g. a click, or a Catalyst type); `None` - /// when the tool doesn't carry the field. - pub fn verified(&self) -> Option { - self.structured.get("verified").and_then(Value::as_bool) + /// The delivery mode the driver actually used, when applicable. + pub fn action_delivery_mode(&self) -> Option<&str> { + self.structured + .get("delivery") + .and_then(|delivery| delivery.get("mode")) + .and_then(Value::as_str) } /// `get_window_state` degraded flag: an AX walk that ran but returned zero diff --git a/libs/cua-driver/rust/crates/cua-driver/src/proxy.rs b/libs/cua-driver/rust/crates/cua-driver/src/proxy.rs index 093ab4d1f5..4b93b61bd0 100644 --- a/libs/cua-driver/rust/crates/cua-driver/src/proxy.rs +++ b/libs/cua-driver/rust/crates/cua-driver/src/proxy.rs @@ -498,10 +498,10 @@ fn fetch_tools_list_from_daemon( .and_then(|v| v.as_array()) .ok_or_else(|| anyhow::anyhow!("daemon list response missing `tools` array"))?; - // Reshape the daemon's `{name, description, input_schema, read_only, - // ..., capabilities}` envelope into MCP's `{name, description, - // inputSchema, annotations: {...}, capabilities}` shape. Same - // translation `ToolDef::to_list_entry` defines for the core protocol. + // Reshape the daemon's `{name, description, input_schema, output_schema, + // read_only, ..., capabilities}` envelope into MCP's `{name, description, + // inputSchema, outputSchema, annotations: {...}, capabilities}` shape. + // Same translation `ToolDef::to_list_entry` defines for the core protocol. // // `capabilities` is passed through verbatim when the daemon // provides it; older daemons that don't emit the field fall back @@ -564,7 +564,7 @@ fn fetch_tools_list_from_daemon( }) }) }); - serde_json::json!({ + let mut tool = serde_json::json!({ "name": name, "description": description, "inputSchema": input_schema, @@ -576,7 +576,16 @@ fn fetch_tools_list_from_daemon( }, "capabilities": capabilities, "risk": risk, - }) + }); + // Do not derive a new schema when an older daemon omitted it: + // mixed-version proxies must advertise only the result contract + // that the executing daemon actually owns. + if let Some(output_schema) = t.get("output_schema") { + tool.as_object_mut() + .expect("MCP tool entry is an object") + .insert("outputSchema".into(), output_schema.clone()); + } + tool }) .collect(); diff --git a/libs/cua-driver/rust/crates/cua-driver/src/sdk_adapter.rs b/libs/cua-driver/rust/crates/cua-driver/src/sdk_adapter.rs index 9cbc37fa43..9cf90faab9 100644 --- a/libs/cua-driver/rust/crates/cua-driver/src/sdk_adapter.rs +++ b/libs/cua-driver/rust/crates/cua-driver/src/sdk_adapter.rs @@ -304,7 +304,7 @@ fn daemon_tools_list_from(tools_list: &Value) -> Value { .flatten() .map(|tool| { let annotations = tool.get("annotations").unwrap_or(&Value::Null); - json!({ + let mut daemon_tool = json!({ "name": tool.get("name").cloned().unwrap_or(Value::Null), "description": tool.get("description").cloned().unwrap_or(Value::String(String::new())), "input_schema": tool.get("inputSchema").cloned().unwrap_or_else(|| json!({"type": "object"})), @@ -314,7 +314,14 @@ fn daemon_tools_list_from(tools_list: &Value) -> Value { "open_world": annotations.get("openWorldHint").cloned().unwrap_or(Value::Bool(false)), "capabilities": tool.get("capabilities").cloned().unwrap_or_else(|| json!([])), "risk": tool.get("risk").cloned().unwrap_or(Value::Null), - }) + }); + if let Some(output_schema) = tool.get("outputSchema") { + daemon_tool + .as_object_mut() + .expect("daemon tool entry is an object") + .insert("output_schema".into(), output_schema.clone()); + } + daemon_tool }) .collect::>(); json!({ @@ -371,7 +378,12 @@ mod tests { "openWorldHint": false }, "capabilities": ["probe.read"], - "risk": {"level": "low"} + "risk": {"level": "low"}, + "outputSchema": { + "type": "object", + "required": ["value"], + "properties": {"value": {"type": "string"}} + } }], "capability_version": "1", "schema_version": "1", @@ -382,6 +394,10 @@ mod tests { }); let daemon = daemon_tools_list_from(&tools_list); assert_eq!(daemon["tools"][0]["input_schema"]["type"], "object"); + assert_eq!( + daemon["tools"][0]["output_schema"]["required"], + json!(["value"]) + ); assert_eq!(daemon["tools"][0]["read_only"], true); assert_eq!( daemon["enforcement_adapters"][0]["id"], diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs index 28a20a4e1c..fd01636f02 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs @@ -750,15 +750,9 @@ fn unverified_background_protocol_oracle( return Vec::new(); } assert_eq!( - response.verified(), - Some(false), - "background dispatch without independent read-back must remain unverified: {}", - response.text() - ); - assert_ne!( - response.structured()["verify"].as_str(), - Some("confirmed"), - "background dispatch overclaimed a confirmed read-back: {}", + response.action_effect(), + Some("unverifiable"), + "background dispatch without independent read-back must remain unverifiable: {}", response.text() ); vec![OracleKind::Protocol] @@ -967,10 +961,9 @@ fn run_macos_selection_hotkeys(fixture: &mut Fixture) { args["keys"] = serde_json::json!(["cmd", "a"]); let response = fixture.driver.call("hotkey", args); assert!(!response.is_error(), "Cmd+A failed: {}", response.raw); - assert_eq!(response.verified(), Some(false), "{}", response.raw); assert_eq!( - response.structured()["effect"], - "unverifiable", + response.action_effect(), + Some("unverifiable"), "generic hotkeys must retain the honest unverifiable contract: {}", response.raw ); @@ -992,8 +985,12 @@ fn run_macos_selection_hotkeys(fixture: &mut Fixture) { paste_args["keys"] = serde_json::json!(["cmd", "v"]); let pasted = fixture.driver.call("hotkey", paste_args); assert!(!pasted.is_error(), "Cmd+V failed: {}", pasted.raw); - assert_eq!(pasted.verified(), Some(false), "{}", pasted.raw); - assert_eq!(pasted.structured()["effect"], "unverifiable"); + assert_eq!( + pasted.action_effect(), + Some("unverifiable"), + "{}", + pasted.raw + ); assert_fixture_value(fixture, "keyboard-input", PASTED); } @@ -1400,12 +1397,12 @@ fn run_browser_tool_roundtrip(fixture: &mut Fixture) -> Observation { }), ); assert_eq!( - click.structured()["status"].as_str(), - Some("ok"), + click.action_effect(), + Some("unverifiable"), "trusted browser click failed: {}", click.raw ); - assert_eq!(click.structured()["route"].as_str(), Some("trusted")); + assert_eq!(click.action_route(), Some("trusted_input")); assert_fixture_text(fixture, "lbl-counter", "counter=1"); let second_snapshot = fixture.driver.call( @@ -1428,8 +1425,8 @@ fn run_browser_tool_roundtrip(fixture: &mut Fixture) -> Observation { }), ); assert_eq!( - typed.structured()["status"].as_str(), - Some("ok"), + typed.action_effect(), + Some("unverifiable"), "browser type failed: {}", typed.raw ); @@ -1445,10 +1442,10 @@ fn run_browser_tool_roundtrip(fixture: &mut Fixture) -> Observation { "session": session, }), ); - assert_eq!( - stale.structured()["refusal"]["code"].as_str(), - Some("browser_ref_stale"), - "older snapshot ref should fail closed: {}", + assert_eq!(stale.action_effect(), Some("refused"), "{}", stale.raw); + assert!( + stale.text().contains("browser_ref_stale"), + "older snapshot ref should retain its diagnostic code: {}", stale.raw ); assert_fixture_text(fixture, "lbl-counter", "counter=1"); @@ -1498,9 +1495,14 @@ fn run_browser_tool_roundtrip(fixture: &mut Fixture) -> Observation { }), ); assert_eq!( - stale_after_navigation.structured()["refusal"]["code"].as_str(), - Some("browser_ref_stale"), - "navigation-invalidated ref should fail closed: {}", + stale_after_navigation.action_effect(), + Some("refused"), + "{}", + stale_after_navigation.raw + ); + assert!( + stale_after_navigation.text().contains("browser_ref_stale"), + "navigation-invalidated ref should retain its diagnostic code: {}", stale_after_navigation.raw ); let ended = fixture diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs index d82ae939f3..1be4e7870e 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs @@ -377,8 +377,8 @@ fn harness_webview_left_click_px_background() { click.text() ); assert_eq!( - click.structured()["path"].as_str(), - Some("ax"), + click.action_route(), + Some("accessibility"), "WebView2 PX background click used an unexpected driver route: {}", click.text() ); diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs index 61f48b5f71..56f3e54c2d 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs @@ -508,8 +508,8 @@ fn harness_wpf_left_click_px_background() { click.text() ); assert_eq!( - click.structured()["path"].as_str(), - Some("ax"), + click.action_route(), + Some("accessibility"), "WPF PX background click used an unexpected driver route: {}", click.text() ); diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/installed_app_textedit_macos_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/installed_app_textedit_macos_test.rs index 62efeaefde..8d6cffc3b9 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/installed_app_textedit_macos_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/installed_app_textedit_macos_test.rs @@ -10,7 +10,7 @@ // ── End-to-end ladder behavior (interactive; needs a GUI session) ──────────── /// On a NATIVE Cocoa field (TextEdit), `delivery_mode:"background"` lands via the -/// AX value-write and the driver confirms it: `path:"ax", verified:true`. This is +/// AX value-write and the driver confirms it with accessibility read-back. This is /// the driver-verifiable happy path — no foreground needed, no screenshot needed. #[test] #[ignore] @@ -111,15 +111,21 @@ fn background_type_on_native_cocoa_is_ax_verified() { .unwrap_or_else(|error| panic!("background TextEdit contract failed: {error}")); assert!(!typed.is_error(), "type_text errored: {}", typed.text()); assert_eq!( - typed.path(), - Some("ax"), + typed.action_route(), + Some("accessibility"), "native Cocoa field should land via AX: {}", typed.text() ); assert_eq!( - typed.verified(), - Some(true), - "AX write should read back as verified: {}", + typed.action_effect(), + Some("confirmed"), + "AX write should be confirmed by read-back: {}", + typed.text() + ); + assert_eq!( + typed.structured()["evidence"][0]["kind"], + "value_readback", + "confirmed actions must expose publishable evidence: {}", typed.text() ); passed.push(OracleKind::AxState); diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/schema_consistency_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/schema_consistency_test.rs index 35fc8612ff..41ea9e7f10 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/schema_consistency_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/schema_consistency_test.rs @@ -14,7 +14,8 @@ //! permissions, or display, so it runs in normal CI. use cua_driver_contract::{ - compatibility::schema_subset_violations, manifest, Platform, SchemaMode, + compatibility::schema_subset_violations, is_action_result_tool, manifest, ActionResult, + Platform, SchemaMode, ToolOutput, }; use cua_driver_core::tool::advertised_capabilities_for; use cua_driver_core::tool_schema::shared_schema_violations; @@ -81,6 +82,18 @@ fn registered_tool_contracts_match_on_active_backend() { input.delivery_mode capability={advertises_delivery_mode}" )); } + + if is_action_result_tool(name) { + let expected = ActionResult::output_schema(); + let actual = tool.get("outputSchema"); + if actual != Some(&expected) { + violations.push(format!( + "{name}: live outputSchema does not equal the shared ActionResult schema; \ + actual={} expected={expected}", + actual.unwrap_or(&Value::Null) + )); + } + } } assert!( diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs index 614972eea4..f921f7e6aa 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs @@ -1549,7 +1549,7 @@ fn run_roundtrip(spec: &BrowserSpec) { "session": session, }), ); - assert_eq!(click.structured()["status"], "ok", "{}", click.raw); + assert_eq!(click.action_effect(), Some("unverifiable"), "{}", click.raw); wait_for_text(&fixture.server, "lbl-counter", "counter=1"); let snapshot = fixture.driver.call( @@ -1571,7 +1571,7 @@ fn run_roundtrip(spec: &BrowserSpec) { "session": session, }), ); - assert_eq!(typed.structured()["status"], "ok", "{}", typed.raw); + assert_eq!(typed.action_effect(), Some("unverifiable"), "{}", typed.raw); wait_for_value(&fixture.server, "txt-input", "standalone-browser"); Observation::delivered(vec![OracleKind::FixtureState], Evidence::default()) @@ -1651,7 +1651,12 @@ fn run_semantic_state(spec: &BrowserSpec) { "session": session, }), ); - assert_eq!(clicked.structured()["status"], "ok", "{}", clicked.raw); + assert_eq!( + clicked.action_effect(), + Some("unverifiable"), + "{}", + clicked.raw + ); wait_for_text(&fixture.server, "lbl-counter", "counter=1"); let refreshed = fixture.driver.call( @@ -1675,7 +1680,7 @@ fn run_semantic_state(spec: &BrowserSpec) { "session": session, }), ); - assert_eq!(typed.structured()["status"], "ok", "{}", typed.raw); + assert_eq!(typed.action_effect(), Some("unverifiable"), "{}", typed.raw); wait_for_value(&fixture.server, "txt-input", "semantic-browser"); Observation::delivered(vec![OracleKind::FixtureState], Evidence::default()) @@ -1706,7 +1711,7 @@ fn run_background_type(spec: &BrowserSpec) { "session": session, }), ); - assert_eq!(typed.structured()["status"], "ok", "{}", typed.raw); + assert_eq!(typed.action_effect(), Some("unverifiable"), "{}", typed.raw); wait_for_value(&fixture.server, "txt-input", "standalone-browser"); Observation::delivered(vec![OracleKind::FixtureState], Evidence::default()) }) @@ -1831,8 +1836,18 @@ fn run_native_omnibox_select_all(spec: &BrowserSpec) { "native omnibox Cmd+A: {}", selected.raw ); - assert_eq!(selected.verified(), Some(false), "{}", selected.raw); - assert_eq!(selected.structured()["effect"], "unverifiable"); + assert_eq!( + selected.action_effect(), + Some("unverifiable"), + "{}", + selected.raw + ); + assert_eq!( + selected.action_route(), + Some("global_input"), + "{}", + selected.raw + ); let replaced = fixture.driver.call( "type_text", @@ -1931,16 +1946,11 @@ fn run_generic_type_text_completion(spec: &BrowserSpec) { "generic long type_text failed: {}", typed.raw ); - assert_eq!( - typed.structured()["requested_chars"].as_u64(), - Some(requested_chars as u64), - "{}", - typed.raw - ); - assert_eq!( - typed.structured()["delivered_chars"].as_u64(), - Some(requested_chars as u64), - "{}", + assert_eq!(typed.action_effect(), Some("unverifiable"), "{}", typed.raw); + assert!( + typed.structured().get("requested_chars").is_none() + && typed.structured().get("delivered_chars").is_none(), + "the narrow action result must not echo request accounting: {}", typed.raw ); @@ -1994,16 +2004,16 @@ fn run_web_type_text_verification(spec: &BrowserSpec) { }), ); assert!(!typed.is_error(), "web type_text failed: {}", typed.raw); - assert_eq!(typed.path(), Some("key_events_fg"), "{}", typed.raw); - assert_eq!(typed.verified(), Some(false), "{}", typed.raw); + assert_eq!(typed.action_route(), Some("global_input"), "{}", typed.raw); assert_eq!( - typed.structured()["effect"], - "unverifiable", + typed.action_delivery_mode(), + Some("foreground"), "{}", typed.raw ); + assert_eq!(typed.action_effect(), Some("unverifiable"), "{}", typed.raw); assert_eq!( - typed.structured()["escalation"]["recommended"], + typed.structured()["escalation"]["target"], "page", "{}", typed.raw @@ -2066,22 +2076,10 @@ fn run_trusted_click(spec: &BrowserSpec) { }), ); if cfg!(any(target_os = "linux", target_os = "macos")) { - assert_eq!( - click.structured()["refusal"]["code"], - "browser_input_trust_unavailable", - "{}", - click.raw - ); - assert_eq!( - click.structured()["refusal"]["detail"]["alternative_route"], - "dom_event", - "{}", - click.raw - ); - assert_eq!( - click.structured()["refusal"]["detail"]["trusted_delivery_attempted"], - false, - "{}", + assert_eq!(click.action_effect(), Some("refused"), "{}", click.raw); + assert!( + click.text().contains("browser_input_trust_unavailable"), + "refusal diagnostics must retain the precise code: {}", click.raw ); wait_for_text(&fixture.server, "lbl-counter", "counter=0"); @@ -2092,7 +2090,7 @@ fn run_trusted_click(spec: &BrowserSpec) { Evidence::default(), ) } else { - assert_eq!(click.structured()["status"], "ok", "{}", click.raw); + assert_eq!(click.action_effect(), Some("unverifiable"), "{}", click.raw); wait_for_text(&fixture.server, "lbl-counter", "counter=1"); Observation::delivered(vec![OracleKind::FixtureState], Evidence::default()) } @@ -2241,7 +2239,12 @@ fn run_prepare_isolated_launch(spec: &BrowserSpec) { "session": session, }), ); - assert_eq!(clicked.structured()["status"], "ok", "{}", clicked.raw); + assert_eq!( + clicked.action_effect(), + Some("unverifiable"), + "{}", + clicked.raw + ); wait_for_text(&target_server, "lbl-counter", "counter=1"); let source_windows = driver.call("list_windows", serde_json::json!({"pid": source_pid})); @@ -2465,7 +2468,12 @@ fn run_existing_profile_attach(spec: &BrowserSpec) { "session": session, }), ); - assert_eq!(clicked.structured()["status"], "ok", "{}", clicked.raw); + assert_eq!( + clicked.action_effect(), + Some("unverifiable"), + "{}", + clicked.raw + ); wait_for_text(&fixture.server, "lbl-counter", "counter=1"); let ended = fixture @@ -2625,7 +2633,12 @@ fn run_existing_profile_setup(spec: &BrowserSpec) { "session": session, }), ); - assert_eq!(clicked.structured()["status"], "ok", "{}", clicked.raw); + assert_eq!( + clicked.action_effect(), + Some("unverifiable"), + "{}", + clicked.raw + ); wait_for_text(&fixture.server, "lbl-counter", "counter=1"); let ended = fixture @@ -2787,10 +2800,10 @@ fn run_stale_ref(spec: &BrowserSpec) { "session": session, }), ); - assert_eq!( - refused.structured()["refusal"]["code"], - "browser_ref_stale", - "{}", + assert_eq!(refused.action_effect(), Some("refused"), "{}", refused.raw); + assert!( + refused.text().contains("browser_ref_stale"), + "stale-ref refusal diagnostics must retain the precise code: {}", refused.raw ); wait_for_text(&fixture.server, "lbl-counter", "counter=0"); @@ -2843,7 +2856,12 @@ fn run_frame_roundtrip(spec: &BrowserSpec) { "session": session, }), ); - assert_eq!(clicked.structured()["status"], "ok", "{}", clicked.raw); + assert_eq!( + clicked.action_effect(), + Some("unverifiable"), + "{}", + clicked.raw + ); } wait_for_text(&fixture.server, "standalone-shadow-state", "shadow=clicked"); wait_for_text(&fixture.server, "standalone-frame-state", "iframe=clicked"); @@ -2864,7 +2882,7 @@ fn run_frame_roundtrip(spec: &BrowserSpec) { "session": session, }), ); - assert_eq!(typed.structured()["status"], "ok", "{}", typed.raw); + assert_eq!(typed.action_effect(), Some("unverifiable"), "{}", typed.raw); } wait_for_text( &fixture.server, @@ -3138,12 +3156,17 @@ fn run_multi_tab(spec: &BrowserSpec) { }), ); let trusted_clicks = if cfg!(target_os = "windows") { - assert_eq!(trusted.structured()["status"], "ok", "{}", trusted.raw); + assert_eq!( + trusted.action_effect(), + Some("unverifiable"), + "{}", + trusted.raw + ); 1 } else { - assert_eq!( - trusted.structured()["refusal"]["code"], - "browser_input_trust_unavailable", + assert_eq!(trusted.action_effect(), Some("refused"), "{}", trusted.raw); + assert!( + trusted.text().contains("browser_input_trust_unavailable"), "{}", trusted.raw ); @@ -3176,7 +3199,12 @@ fn run_multi_tab(spec: &BrowserSpec) { "session": session, }), ); - assert_eq!(clicked.structured()["status"], "ok", "{}", clicked.raw); + assert_eq!( + clicked.action_effect(), + Some("unverifiable"), + "{}", + clicked.raw + ); let snapshot = fixture.driver.call( "get_browser_state", @@ -3199,7 +3227,7 @@ fn run_multi_tab(spec: &BrowserSpec) { "session": session, }), ); - assert_eq!(typed.structured()["status"], "ok", "{}", typed.raw); + assert_eq!(typed.action_effect(), Some("unverifiable"), "{}", typed.raw); let snapshot = fixture.driver.call( "get_browser_state", @@ -3223,7 +3251,7 @@ fn run_multi_tab(spec: &BrowserSpec) { "session": session, }), ); - assert_eq!(keyed.structured()["status"], "ok", "{}", keyed.raw); + assert_eq!(keyed.action_effect(), Some("unverifiable"), "{}", keyed.raw); wait_for_text( &second_server, @@ -3893,7 +3921,12 @@ fn run_dialogs(spec: &BrowserSpec) { "session": session, }), ); - assert_eq!(clicked.structured()["status"], "ok", "{}", clicked.raw); + assert_eq!( + clicked.action_effect(), + Some("unverifiable"), + "{}", + clicked.raw + ); thread::sleep(Duration::from_millis(100)); if cfg!(target_os = "linux") { @@ -4137,9 +4170,9 @@ fn run_pointer_actions(spec: &BrowserSpec) { "session": session, }), ); - assert_eq!( - refused.structured()["refusal"]["code"], - "browser_action_unavailable", + assert_eq!(refused.action_effect(), Some("refused"), "{}", refused.raw); + assert!( + refused.text().contains("browser_action_unavailable"), "{}", refused.raw ); @@ -4158,7 +4191,12 @@ fn run_pointer_actions(spec: &BrowserSpec) { "session": session, }), ); - assert_eq!(response.structured()["status"], "ok", "{}", response.raw); + assert_eq!( + response.action_effect(), + Some("unverifiable"), + "{}", + response.raw + ); } wait_for_text(&fixture.server, "standalone-hover-state", "hover=true"); wait_for_text( @@ -4179,7 +4217,12 @@ fn run_pointer_actions(spec: &BrowserSpec) { "session": session, }), ); - assert_eq!(scrolled.structured()["status"], "ok", "{}", scrolled.raw); + assert_eq!( + scrolled.action_effect(), + Some("unverifiable"), + "{}", + scrolled.raw + ); let deadline = Instant::now() + Duration::from_secs(5); loop { if fixture @@ -4205,7 +4248,12 @@ fn run_pointer_actions(spec: &BrowserSpec) { "session": session, }), ); - assert_eq!(dragged.structured()["status"], "ok", "{}", dragged.raw); + assert_eq!( + dragged.action_effect(), + Some("unverifiable"), + "{}", + dragged.raw + ); wait_for_text(&fixture.server, "drag-status", "drag_status=dropped"); Observation::delivered(vec![OracleKind::FixtureState], Evidence::default()) }) @@ -4370,7 +4418,12 @@ fn run_type_replace(spec: &BrowserSpec) { args["mode"] = serde_json::json!(mode); } let response = fixture.driver.call("browser_type", args); - assert_eq!(response.structured()["status"], "ok", "{}", response.raw); + assert_eq!( + response.action_effect(), + Some("unverifiable"), + "{}", + response.raw + ); response }; @@ -4381,25 +4434,26 @@ fn run_type_replace(spec: &BrowserSpec) { type_text("second", None, None); wait_for_value(&fixture.server, "txt-input", "firstsecond"); - // replace=true sets the field instead of extending it, and reports - // how much it displaced so the caller need not re-read the page. + // replace=true sets the field instead of extending it. The narrow + // action result does not duplicate page state; the fixture is the + // independent postcondition oracle. let replaced = type_text("third🙂", Some(true), None); wait_for_value(&fixture.server, "txt-input", "third🙂"); - assert_eq!(replaced.structured()["replace"], true, "{}", replaced.raw); assert_eq!( - replaced.structured()["replaced_chars"], - 11, - "replaced_chars must count the displaced text: {}", + replaced.action_effect(), + Some("unverifiable"), + "{}", replaced.raw ); + assert!(replaced.structured().get("replaced_chars").is_none()); // The trusted keystroke path replaces through the same selection. let replaced_unicode = type_text("fourth", Some(true), Some("keystrokes")); wait_for_value(&fixture.server, "txt-input", "fourth"); assert_eq!( - replaced_unicode.structured()["replaced_chars"], - 6, - "replaced_chars must count Unicode scalar values like requested_chars: {}", + replaced_unicode.action_effect(), + Some("unverifiable"), + "{}", replaced_unicode.raw ); @@ -4407,9 +4461,9 @@ fn run_type_replace(spec: &BrowserSpec) { let cleared = type_text("", Some(true), None); wait_for_value(&fixture.server, "txt-input", ""); assert_eq!( - cleared.structured()["replaced_chars"], - 6, - "clearing must report what it removed: {}", + cleared.action_effect(), + Some("unverifiable"), + "{}", cleared.raw ); @@ -4428,8 +4482,13 @@ fn run_type_replace(spec: &BrowserSpec) { }), ); assert_eq!( - unsupported.structured()["refusal"]["code"], - "browser_action_unavailable", + unsupported.action_effect(), + Some("refused"), + "{}", + unsupported.raw + ); + assert!( + unsupported.text().contains("browser_action_unavailable"), "{}", unsupported.raw ); diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs index 84db5fe35e..cc408ff563 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs @@ -844,6 +844,7 @@ impl Tool for GetWindowStateTool { content, is_error: None, structured_content: Some(structured), + action_record: None, } } Ok(Err(e)) => ToolResult::error(format!("Capture error: {e}")), @@ -1141,9 +1142,10 @@ fn non_ax_escalation() -> Value { /// Structured payload for a `type_text` response. Keeps the legacy /// `path`/`characters`/`verified` fields for back-compat and adds the cross-tool -/// `effect` tri-state: a read-back-confirmed insert (the AT-SPI `insertText` -/// rung) is `"confirmed"`; every keystroke / XSendEvent / XTest / Wayland rung -/// is `"unverifiable"` (no read-back — the caller confirms via screenshot) and +/// `effect` tri-state. Linux's AT-SPI `insertText` return value acknowledges +/// the method call but does not read the widget value back, so it and every +/// keystroke / XSendEvent / XTest / Wayland rung are `"unverifiable"` (the +/// caller confirms through a separate observation) and /// carries a `foreground` escalation, because the field IS in the AT-SPI tree — /// it's a delivery/focus problem, not a missing element. The foreground rung /// itself (`key_events_fg`) is already the last resort, so it emits no @@ -1189,16 +1191,12 @@ fn type_text_structured_electron(text_len: usize) -> Value { }) } -/// Build the success `ToolResult` for an AT-SPI insert that the driver would -/// otherwise mark `verified:true` (path=="ax"). Applies the Electron/Chromium -/// AX-echo suppression: when `pid` is a Chromium embedder, the a11y layer can -/// echo the `insertText` write back while the renderer ignores it, so we refuse -/// to claim a confirmed insert — downgrade to effect:"unverifiable" + -/// escalation:{recommended:"px"} and tell the agent to confirm via screenshot. -/// Probe ONLY here, on the rung that would otherwise confirm, so native AT-SPI -/// types pay nothing. `route` is the human route phrase, e.g. "via AT-SPI". -/// Mirrors macOS `type_text`'s `ax_echo_surface` gate. -fn type_text_ax_confirm_result(pid: u32, text_len: usize, route: &str) -> ToolResult { +/// Build the success `ToolResult` for an AT-SPI insert. The EditableText +/// method's boolean is a delivery acknowledgement, not a fresh value readback, +/// so native widgets remain `unverifiable`. Chromium embedders additionally +/// recommend the pixel rung because their accessibility bridge can acknowledge +/// a write the renderer never observes. +fn type_text_ax_result(pid: u32, text_len: usize, route: &str) -> ToolResult { if is_chromium_embedder(pid) { return ToolResult::text(format!( "📨 Sent (unverified) {text_len} character(s) ({route}). — Electron/web \ @@ -1210,14 +1208,14 @@ fn type_text_ax_confirm_result(pid: u32, text_len: usize, route: &str) -> ToolRe .with_structured(type_text_structured_electron(text_len)); } ToolResult::text(format!("Typed {text_len} character(s) ({route}).")) - .with_structured(type_text_structured("ax", text_len, true)) + .with_structured(type_text_structured("ax", text_len, false)) } /// True when `pid` is a Chromium-based embedder — a Chrome/Chromium browser or /// any Electron/CEF app. On these surfaces an AT-SPI `EditableText.insertText` /// can succeed at the bridge while the Chromium *renderer* never observes it, /// so the AT-SPI "ax" rung must not be trusted as a confirmed insert (see -/// [`type_text_ax_confirm_result`]). +/// [`type_text_ax_result`]). /// /// This is the Linux analogue of macOS `ElectronJs::is_electron` (which checks /// for a bundled Electron Framework). Linux has no bundle, so the signal is @@ -2421,7 +2419,7 @@ impl Tool for TypeTextTool { }) .await; if let Ok(Ok(())) = targeted { - return type_text_ax_confirm_result(pid, text_len, "via targeted AT-SPI"); + return type_text_ax_result(pid, text_len, "via targeted AT-SPI"); } } // The private nested compositor can target the owning Wayland client @@ -2540,7 +2538,7 @@ impl Tool for TypeTextTool { .await; match targeted { Ok(Ok(())) => { - return type_text_ax_confirm_result(pid, text_len, "via targeted AT-SPI"); + return type_text_ax_result(pid, text_len, "via targeted AT-SPI"); } Ok(Err(_)) | Err(_) if !delivery.is_foreground() && crate::wayland::wayland_input_enabled() => @@ -2727,7 +2725,7 @@ impl Tool for TypeTextTool { // AT-SPI succeeded — focus-free typing worked (Qt6, GTK4, etc.)! // Electron/Chromium can echo this write without the renderer // observing it, so the confirm is suppressed there (mirrors macOS). - return type_text_ax_confirm_result(pid, text_len, "via AT-SPI"); + return type_text_ax_result(pid, text_len, "via AT-SPI"); } _ => { // AT-SPI failed (no editable exposed). Qt5 doesn't expose widgets @@ -2756,11 +2754,7 @@ impl Tool for TypeTextTool { match qt5_result { Ok(Ok(())) => { - return type_text_ax_confirm_result( - pid, - text_len, - "via AT-SPI with focus workaround", - ); + return type_text_ax_result(pid, text_len, "via AT-SPI with focus workaround"); } _ => { // AT-SPI still didn't work. Fall back to X11 XSendEvent. @@ -2808,15 +2802,10 @@ impl Tool for TypeTextTool { "background" }; match result { - // Read-back verdict: the AT-SPI EditableText.insertText path ("ax") is - // the driver-verifiable rung on Linux — the a11y layer confirms the - // insert into the widget model (truthful on GTK/Qt). The keystroke / - // XSendEvent / XTest rungs aren't read-back-confirmed (verified:false; - // caller confirms via screenshot). - // Only the AT-SPI ("ax") rung is read-back-confirmable; route it - // through the Electron/Chromium AX-echo suppression (mirrors macOS). - // Every other rung is already verified:false. - Ok(Ok("ax")) => type_text_ax_confirm_result( + // AT-SPI's boolean acknowledges the EditableText call; it is not a + // fresh value readback. Keep the result unverifiable and apply the + // stricter Chromium escalation where appropriate. + Ok(Ok("ax")) => type_text_ax_result( pid, text_len, &format!("via X11, delivery_mode={mode_label}"), @@ -5850,6 +5839,7 @@ impl Tool for GetDesktopStateTool { content, is_error: None, structured_content: Some(structured), + action_record: None, } } Ok(Err(e)) => ToolResult::error(format!("Capture error: {e}")), @@ -5951,7 +5941,13 @@ impl Tool for MoveCursorTool { let (x, y) = (input.x, input.y); let xi = x.round() as i32; let yi = y.round() as i32; - let result = if crate::wayland::wayland_input_enabled() { + let wayland = crate::wayland::wayland_input_enabled(); + let path = if wayland { + "wayland_desktop" + } else { + "xtest_desktop" + }; + let result = if wayland { tokio::task::spawn_blocking(move || { crate::wayland::move_cursor_absolute(None, xi, yi) }) @@ -5961,12 +5957,12 @@ impl Tool for MoveCursorTool { .await }; return match result { - Ok(Ok(())) => { - ToolResult::text(format!("Moved the real desktop pointer to ({xi}, {yi}).")) - .with_structured( - json!({"scope":"desktop","x":xi,"y":yi,"effect":"unverifiable"}), - ) - } + Ok(Ok(())) => ToolResult::text(format!( + "Moved the real desktop pointer to ({xi}, {yi})." + )) + .with_structured( + json!({"scope":"desktop","path":path,"x":xi,"y":yi,"effect":"unverifiable"}), + ), Ok(Err(error)) => ToolResult::error(error.to_string()), Err(error) => ToolResult::error(format!("Task error: {error}")), }; @@ -6677,6 +6673,7 @@ impl Tool for ZoomTool { "width": w, "height": h, "format": "jpeg", "mime_type": "image/jpeg" })), + action_record: None, } } Ok(Err(e)) => ToolResult::error(format!("Zoom failed: {e}")), diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs index b40532d629..662faf3f91 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs @@ -163,7 +163,7 @@ fn def() -> &'static ToolDef { "delivery_mode": { "type": "string", "enum": ["background", "foreground"], - "description": "Best-effort-background ladder rung (default \"background\"). \"background\": perform the AX action or post the CGEvent without fronting. \"foreground\": briefly front the window, act, let transient UI settle, then restore the prior frontmost app. Requires window_id. A click is never driver-verifiable (no read-back), so both report verified:false — confirm the effect via screenshot. Use the agent loop: background AX (element_index) → screenshot → background pixel (x/y) → screenshot → delivery_mode:\"foreground\"." + "description": "Best-effort-background ladder rung (default \"background\"). \"background\": perform the AX action or post the CGEvent without fronting. \"foreground\": briefly front the window, act, let transient UI settle, then restore the prior frontmost app. Requires window_id. A generic click has no independent postcondition read-back, so its action effect remains unverifiable — confirm the effect from a fresh state snapshot. Use the agent loop: background AX (element_index) → snapshot → background pixel (x/y) → snapshot → delivery_mode:\"foreground\"." }, "scope": { "type": "string", diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs index bfd57baf30..1a6775ca16 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs @@ -129,6 +129,7 @@ impl Tool for GetDesktopStateTool { content, is_error: None, structured_content: Some(structured), + action_record: None, } } } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs index 8036a271c7..c4c5e218b0 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs @@ -614,6 +614,7 @@ impl Tool for GetWindowStateTool { content, is_error: None, structured_content: Some(structured), + action_record: None, } } } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs index b659d622af..3cd684cb42 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs @@ -113,7 +113,7 @@ fn def() -> &'static ToolDef { "delivery_mode": { "type": "string", "enum": ["background", "foreground"], - "description": "Best-effort-background ladder rung (default \"background\"). \"background\": AX insert, then CGEvent keystrokes if needed — no focus steal; native controls can be verified via AXValue read-back, while web-content read-back remains unverified. \"foreground\": briefly front the window, type, restore the prior frontmost — the explicit last resort for focus-sensitive surfaces (e.g. WhatsApp/Catalyst) where background keystrokes don't land. Re-call with \"foreground\" when a background attempt returns `verified:false` and a screenshot shows the text didn't appear." + "description": "Best-effort-background ladder rung (default \"background\"). \"background\": AX insert, then CGEvent keystrokes if needed — no focus steal; native controls can be confirmed via AXValue read-back, while web-content writes remain effect:\"unverifiable\". \"foreground\": briefly front the window, type, restore the prior frontmost — the explicit last resort for focus-sensitive surfaces (e.g. WhatsApp/Catalyst) where background keystrokes don't land. Re-call with \"foreground\" when a background attempt remains unverifiable and a fresh snapshot shows the text did not appear." } }, "additionalProperties": false diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/zoom.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/zoom.rs index 3dfe2fb1a8..f389448f9a 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/zoom.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/zoom.rs @@ -116,6 +116,7 @@ impl Tool for ZoomTool { "width": w, "height": h, "format": "jpeg", "mime_type": "image/jpeg" })), + action_record: None, } } Ok(Err(e)) => ToolResult::error(format!("Zoom failed: {e}")), diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs index 6902e5d912..2c207ee5f1 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs @@ -1249,6 +1249,7 @@ impl Tool for GetWindowStateTool { content, is_error: None, structured_content: Some(structured), + action_record: None, } } Err(e) => ToolResult::error(format!("Error: {e}")), @@ -3548,7 +3549,7 @@ impl Tool for TypeTextTool { "pid":{"type":"integer","description":"Target process ID."}, "text":{"type":"string","description":"Text to insert at the focused element's cursor."}, "window_id":{"type":"integer","description":"HWND of the target window. Required when element_index is used. Optional when element_token is supplied (the token carries it)."}, - "element_index":{"type":"integer","description":"Element index from the last get_window_state for the same (pid, window_id). When supplied, type_text (1) writes via UIA ValuePattern.SetValue on that element (works for WPF/WinForms/UWP/XAML without focus steal) and (2) verifies by reading that element's value back by handle — focus-independent, so the structured `verify` reaches `confirmed`/`unchanged` even when the target isn't foreground. Strongly preferred over typing into 'whatever is focused' (no element_index), which falls back to PostMessage WM_CHAR + a foreground-only focused-element read-back. Requires window_id."}, + "element_index":{"type":"integer","description":"Element index from the last get_window_state for the same (pid, window_id). When supplied, type_text (1) writes via UIA ValuePattern.SetValue on that element (works for WPF/WinForms/UWP/XAML without focus steal) and (2) confirms by reading that element's value back by handle — focus-independent. A matching read-back produces effect:\"confirmed\" with value_readback evidence; an unchanged or unreadable value remains effect:\"unverifiable\". Strongly preferred over typing into 'whatever is focused' (no element_index), which falls back to PostMessage WM_CHAR plus a foreground-only focused-element read-back. Requires window_id."}, "element_token": cua_driver_core::tool_schema::element_token_schema(), "x":{"type":"number","description":"Window-local screenshot-pixel X of the field to type into — the element px action form. Pass x,y (no element_index) and the tool pixel-clicks there to establish real renderer focus, then types. Use for Chromium/Electron inputs the UIA/WM_CHAR path can't reach. Read straight off the get_window_state PNG, same convention as click."}, "y":{"type":"number","description":"Window-local screenshot-pixel Y of the field (see x)."}, @@ -4054,11 +4055,13 @@ impl Tool for TypeTextTool { let post_res = crate::input::post_type_text(hwnd, &text_for_post); std::thread::sleep(std::time::Duration::from_millis(40)); let after = read(verify_idx); - (post_res, before, after) + let confirmed = + post_message_readback_confirms(before.as_deref(), after.as_deref(), &text_for_post); + (post_res, before, after, confirmed) }) .await; match result { - Ok((Ok(()), before, after)) => { + Ok((Ok(()), before, after, confirmed)) => { // Three-way verdict. Crucially, "couldn't read the field" is NOT // the same as "the text didn't land": on Windows, UIA's // GetFocusedElement is system-wide (unlike macOS's per-app @@ -4066,10 +4069,11 @@ impl Tool for TypeTextTool { // app we cannot read it back even though a PostMessage WM_CHAR // may have landed fine. Treating unreadable as failure would // spuriously push agents to foreground and break the - // background-first contract. So only DOWNGRADE on positive - // evidence (read OK both times, value unchanged). + // background-first contract. Confirm only when the value + // changed and the resulting value contains the requested + // text; every other readable result remains unverified. match (&before, &after) { - (Some(b), Some(a)) if a != b => ToolResult::text(format!( + (Some(_), Some(_)) if confirmed => ToolResult::text(format!( "✅ Typed {text_len} char(s) on pid {raw_pid} via PostMessage \ ({_delay_ms}ms delay; verified via UIA read-back)." )) @@ -4080,8 +4084,9 @@ impl Tool for TypeTextTool { })), (Some(_), Some(_)) => ToolResult::text(format!( "📨 Sent {text_len} char(s) to pid {raw_pid} via PostMessage, but \ - the focused field's value did not change — the text was likely \ - dropped (e.g. a VCL/LibreOffice grid not in cell-edit mode). \ + the focused field's value did not contain the requested text \ + after dispatch — delivery was incomplete or dropped (e.g. a \ + VCL/LibreOffice grid not in cell-edit mode). \ Retry with delivery_mode:\"foreground\"." )) .with_structured(serde_json::json!({ @@ -4115,12 +4120,54 @@ impl Tool for TypeTextTool { })), } } - Ok((Err(e), _, _)) => ToolResult::error(e.to_string()), + Ok((Err(e), _, _, _)) => ToolResult::error(e.to_string()), Err(e) => ToolResult::error(format!("Task error: {e}")), } } } +fn post_message_readback_confirms( + before: Option<&str>, + after: Option<&str>, + requested_text: &str, +) -> bool { + matches!( + (before, after), + (Some(before), Some(after)) + if before != after && after.contains(requested_text) + ) +} + +#[cfg(test)] +mod post_message_readback_tests { + use super::post_message_readback_confirms; + + #[test] + fn confirms_only_a_changed_value_containing_the_requested_text() { + assert!(post_message_readback_confirms( + Some("prefix"), + Some("prefixhello"), + "hello" + )); + assert!(!post_message_readback_confirms(None, None, "hello")); + assert!(!post_message_readback_confirms( + Some("hello"), + Some("hello"), + "hello" + )); + assert!(!post_message_readback_confirms( + Some(""), + Some("h"), + "hello" + )); + assert!(!post_message_readback_confirms( + Some("before"), + Some("different"), + "hello" + )); + } +} + /// Read a **specific cached element's** text value by its `element_index`, /// for type_text read-back verification. Tries `ValuePattern.CurrentValue` /// then falls back to `TextPattern` document text. Unlike @@ -6841,6 +6888,7 @@ impl Tool for GetDesktopStateTool { content, is_error: None, structured_content: Some(structured), + action_record: None, } } } @@ -7797,6 +7845,7 @@ impl Tool for ZoomTool { "width": w, "height": h, "format": "jpeg", "mime_type": "image/jpeg" })), + action_record: None, } } Ok(Err(e)) => ToolResult::error(format!("Zoom failed: {e}")), diff --git a/libs/cua-driver/typescript/src/native/cua_driver_contract.ts b/libs/cua-driver/typescript/src/native/cua_driver_contract.ts index 370ab35361..6831aff82b 100644 --- a/libs/cua-driver/typescript/src/native/cua_driver_contract.ts +++ b/libs/cua-driver/typescript/src/native/cua_driver_contract.ts @@ -19,6 +19,399 @@ const uniffiIsDebug = // Public interface members begin here. +export enum ActionDeliveryMode { + Background, + Foreground, + NotApplicable, + Unknown +} + +const FfiConverterTypeActionDeliveryMode = (() => { + const ordinalConverter = FfiConverterInt32; + type TypeName = ActionDeliveryMode; + class FFIConverter extends AbstractFfiConverterByteArray { + read(from: RustBuffer): TypeName { + switch (ordinalConverter.read(from)) { + case 1: return ActionDeliveryMode.Background; + case 2: return ActionDeliveryMode.Foreground; + case 3: return ActionDeliveryMode.NotApplicable; + case 4: return ActionDeliveryMode.Unknown; + default: throw new UniffiInternalError.UnexpectedEnumCase(); + } + } + write(value: TypeName, into: RustBuffer): void { + switch (value) { + case ActionDeliveryMode.Background: return ordinalConverter.write(1, into); + case ActionDeliveryMode.Foreground: return ordinalConverter.write(2, into); + case ActionDeliveryMode.NotApplicable: return ordinalConverter.write(3, into); + case ActionDeliveryMode.Unknown: return ordinalConverter.write(4, into); + } + } + allocationSize(value: TypeName): number { + return ordinalConverter.allocationSize(0); + } + } + return new FFIConverter(); +})(); + +export type ActionDelivery = { + mode: ActionDeliveryMode, + deliveredCount?: number +} + +/** + * Generated factory for {@link ActionDelivery} record objects. + */ +export const ActionDelivery = (() => { + const defaults = () => ({ + }); + const create = (() => { + return uniffiCreateRecord>(defaults); + })(); + return Object.freeze({ + create, + new: create, + defaults: () => Object.freeze(defaults()) as Partial, + }); +})(); + +const FfiConverterTypeActionDelivery = (() => { + type TypeName = ActionDelivery; + class FFIConverter extends AbstractFfiConverterByteArray { + read(from: RustBuffer): TypeName { + return { + mode: FfiConverterTypeActionDeliveryMode.read(from), + deliveredCount: FfiConverterOptionalUInt32.read(from) + }; + } + write(value: TypeName, into: RustBuffer): void { + FfiConverterTypeActionDeliveryMode.write(value.mode, into); + FfiConverterOptionalUInt32.write(value.deliveredCount, into); + } + allocationSize(value: TypeName): number { + return FfiConverterTypeActionDeliveryMode.allocationSize(value.mode) + + FfiConverterOptionalUInt32.allocationSize(value.deliveredCount); + + } + }; + return new FFIConverter(); +})(); + +export enum ActionEscalationTarget { + Pixel, + Foreground, + Page, + Session +} + +const FfiConverterTypeActionEscalationTarget = (() => { + const ordinalConverter = FfiConverterInt32; + type TypeName = ActionEscalationTarget; + class FFIConverter extends AbstractFfiConverterByteArray { + read(from: RustBuffer): TypeName { + switch (ordinalConverter.read(from)) { + case 1: return ActionEscalationTarget.Pixel; + case 2: return ActionEscalationTarget.Foreground; + case 3: return ActionEscalationTarget.Page; + case 4: return ActionEscalationTarget.Session; + default: throw new UniffiInternalError.UnexpectedEnumCase(); + } + } + write(value: TypeName, into: RustBuffer): void { + switch (value) { + case ActionEscalationTarget.Pixel: return ordinalConverter.write(1, into); + case ActionEscalationTarget.Foreground: return ordinalConverter.write(2, into); + case ActionEscalationTarget.Page: return ordinalConverter.write(3, into); + case ActionEscalationTarget.Session: return ordinalConverter.write(4, into); + } + } + allocationSize(value: TypeName): number { + return ordinalConverter.allocationSize(0); + } + } + return new FFIConverter(); +})(); + +export enum ActionEscalationReason { + RouteUnavailable, + DeliveryFailed, + EffectUnconfirmed, + SuspectedNoop, + PermissionRequired +} + +const FfiConverterTypeActionEscalationReason = (() => { + const ordinalConverter = FfiConverterInt32; + type TypeName = ActionEscalationReason; + class FFIConverter extends AbstractFfiConverterByteArray { + read(from: RustBuffer): TypeName { + switch (ordinalConverter.read(from)) { + case 1: return ActionEscalationReason.RouteUnavailable; + case 2: return ActionEscalationReason.DeliveryFailed; + case 3: return ActionEscalationReason.EffectUnconfirmed; + case 4: return ActionEscalationReason.SuspectedNoop; + case 5: return ActionEscalationReason.PermissionRequired; + default: throw new UniffiInternalError.UnexpectedEnumCase(); + } + } + write(value: TypeName, into: RustBuffer): void { + switch (value) { + case ActionEscalationReason.RouteUnavailable: return ordinalConverter.write(1, into); + case ActionEscalationReason.DeliveryFailed: return ordinalConverter.write(2, into); + case ActionEscalationReason.EffectUnconfirmed: return ordinalConverter.write(3, into); + case ActionEscalationReason.SuspectedNoop: return ordinalConverter.write(4, into); + case ActionEscalationReason.PermissionRequired: return ordinalConverter.write(5, into); + } + } + allocationSize(value: TypeName): number { + return ordinalConverter.allocationSize(0); + } + } + return new FFIConverter(); +})(); + +export type ActionEscalation = { + target: ActionEscalationTarget, + reason: ActionEscalationReason +} + +/** + * Generated factory for {@link ActionEscalation} record objects. + */ +export const ActionEscalation = (() => { + const defaults = () => ({ + }); + const create = (() => { + return uniffiCreateRecord>(defaults); + })(); + return Object.freeze({ + create, + new: create, + defaults: () => Object.freeze(defaults()) as Partial, + }); +})(); + +const FfiConverterTypeActionEscalation = (() => { + type TypeName = ActionEscalation; + class FFIConverter extends AbstractFfiConverterByteArray { + read(from: RustBuffer): TypeName { + return { + target: FfiConverterTypeActionEscalationTarget.read(from), + reason: FfiConverterTypeActionEscalationReason.read(from) + }; + } + write(value: TypeName, into: RustBuffer): void { + FfiConverterTypeActionEscalationTarget.write(value.target, into); + FfiConverterTypeActionEscalationReason.write(value.reason, into); + } + allocationSize(value: TypeName): number { + return FfiConverterTypeActionEscalationTarget.allocationSize(value.target) + + FfiConverterTypeActionEscalationReason.allocationSize(value.reason); + + } + }; + return new FFIConverter(); +})(); + +export enum ActionEvidenceKind { + ValueReadback, + WindowChange +} + +const FfiConverterTypeActionEvidenceKind = (() => { + const ordinalConverter = FfiConverterInt32; + type TypeName = ActionEvidenceKind; + class FFIConverter extends AbstractFfiConverterByteArray { + read(from: RustBuffer): TypeName { + switch (ordinalConverter.read(from)) { + case 1: return ActionEvidenceKind.ValueReadback; + case 2: return ActionEvidenceKind.WindowChange; + default: throw new UniffiInternalError.UnexpectedEnumCase(); + } + } + write(value: TypeName, into: RustBuffer): void { + switch (value) { + case ActionEvidenceKind.ValueReadback: return ordinalConverter.write(1, into); + case ActionEvidenceKind.WindowChange: return ordinalConverter.write(2, into); + } + } + allocationSize(value: TypeName): number { + return ordinalConverter.allocationSize(0); + } + } + return new FFIConverter(); +})(); + +export type ActionEvidence = { + kind: ActionEvidenceKind +} + +/** + * Generated factory for {@link ActionEvidence} record objects. + */ +export const ActionEvidence = (() => { + const defaults = () => ({ + }); + const create = (() => { + return uniffiCreateRecord>(defaults); + })(); + return Object.freeze({ + create, + new: create, + defaults: () => Object.freeze(defaults()) as Partial, + }); +})(); + +const FfiConverterTypeActionEvidence = (() => { + type TypeName = ActionEvidence; + class FFIConverter extends AbstractFfiConverterByteArray { + read(from: RustBuffer): TypeName { + return { + kind: FfiConverterTypeActionEvidenceKind.read(from) + }; + } + write(value: TypeName, into: RustBuffer): void { + FfiConverterTypeActionEvidenceKind.write(value.kind, into); + } + allocationSize(value: TypeName): number { + return FfiConverterTypeActionEvidenceKind.allocationSize(value.kind); + + } + }; + return new FFIConverter(); +})(); + +export enum ActionEffect { + Confirmed, + Partial, + Unverifiable, + SuspectedNoop, + Refused +} + +const FfiConverterTypeActionEffect = (() => { + const ordinalConverter = FfiConverterInt32; + type TypeName = ActionEffect; + class FFIConverter extends AbstractFfiConverterByteArray { + read(from: RustBuffer): TypeName { + switch (ordinalConverter.read(from)) { + case 1: return ActionEffect.Confirmed; + case 2: return ActionEffect.Partial; + case 3: return ActionEffect.Unverifiable; + case 4: return ActionEffect.SuspectedNoop; + case 5: return ActionEffect.Refused; + default: throw new UniffiInternalError.UnexpectedEnumCase(); + } + } + write(value: TypeName, into: RustBuffer): void { + switch (value) { + case ActionEffect.Confirmed: return ordinalConverter.write(1, into); + case ActionEffect.Partial: return ordinalConverter.write(2, into); + case ActionEffect.Unverifiable: return ordinalConverter.write(3, into); + case ActionEffect.SuspectedNoop: return ordinalConverter.write(4, into); + case ActionEffect.Refused: return ordinalConverter.write(5, into); + } + } + allocationSize(value: TypeName): number { + return ordinalConverter.allocationSize(0); + } + } + return new FFIConverter(); +})(); + +export enum ActionRoute { + Accessibility, + SyntheticEvents, + GlobalInput, + Dom, + TrustedInput +} + +const FfiConverterTypeActionRoute = (() => { + const ordinalConverter = FfiConverterInt32; + type TypeName = ActionRoute; + class FFIConverter extends AbstractFfiConverterByteArray { + read(from: RustBuffer): TypeName { + switch (ordinalConverter.read(from)) { + case 1: return ActionRoute.Accessibility; + case 2: return ActionRoute.SyntheticEvents; + case 3: return ActionRoute.GlobalInput; + case 4: return ActionRoute.Dom; + case 5: return ActionRoute.TrustedInput; + default: throw new UniffiInternalError.UnexpectedEnumCase(); + } + } + write(value: TypeName, into: RustBuffer): void { + switch (value) { + case ActionRoute.Accessibility: return ordinalConverter.write(1, into); + case ActionRoute.SyntheticEvents: return ordinalConverter.write(2, into); + case ActionRoute.GlobalInput: return ordinalConverter.write(3, into); + case ActionRoute.Dom: return ordinalConverter.write(4, into); + case ActionRoute.TrustedInput: return ordinalConverter.write(5, into); + } + } + allocationSize(value: TypeName): number { + return ordinalConverter.allocationSize(0); + } + } + return new FFIConverter(); +})(); + +export type ActionResult = { + effect: ActionEffect, + route: ActionRoute, + delivery?: ActionDelivery, + evidence?: Array, + escalation?: ActionEscalation +} + +/** + * Generated factory for {@link ActionResult} record objects. + */ +export const ActionResult = (() => { + const defaults = () => ({ + }); + const create = (() => { + return uniffiCreateRecord>(defaults); + })(); + return Object.freeze({ + create, + new: create, + defaults: () => Object.freeze(defaults()) as Partial, + }); +})(); + +const FfiConverterTypeActionResult = (() => { + type TypeName = ActionResult; + class FFIConverter extends AbstractFfiConverterByteArray { + read(from: RustBuffer): TypeName { + return { + effect: FfiConverterTypeActionEffect.read(from), + route: FfiConverterTypeActionRoute.read(from), + delivery: FfiConverterOptionalTypeActionDelivery.read(from), + evidence: FfiConverterOptionalSequenceTypeActionEvidence.read(from), + escalation: FfiConverterOptionalTypeActionEscalation.read(from) + }; + } + write(value: TypeName, into: RustBuffer): void { + FfiConverterTypeActionEffect.write(value.effect, into); + FfiConverterTypeActionRoute.write(value.route, into); + FfiConverterOptionalTypeActionDelivery.write(value.delivery, into); + FfiConverterOptionalSequenceTypeActionEvidence.write(value.evidence, into); + FfiConverterOptionalTypeActionEscalation.write(value.escalation, into); + } + allocationSize(value: TypeName): number { + return FfiConverterTypeActionEffect.allocationSize(value.effect) + + FfiConverterTypeActionRoute.allocationSize(value.route) + + FfiConverterOptionalTypeActionDelivery.allocationSize(value.delivery) + + FfiConverterOptionalSequenceTypeActionEvidence.allocationSize(value.evidence) + + FfiConverterOptionalTypeActionEscalation.allocationSize(value.escalation); + + } + }; + return new FFIConverter(); +})(); + export type BoundsExpectation = { x: number, y: number, @@ -2456,6 +2849,21 @@ const FfiConverterTypePlatform = (() => { return new FFIConverter(); })(); +// FfiConverter for number | undefined +const FfiConverterOptionalUInt32 = new FfiConverterOptional(FfiConverterUInt32); + +// FfiConverter for ActionDelivery | undefined +const FfiConverterOptionalTypeActionDelivery = new FfiConverterOptional(FfiConverterTypeActionDelivery); + +// FfiConverter for Array +const FfiConverterSequenceTypeActionEvidence = new FfiConverterArray(FfiConverterTypeActionEvidence); + +// FfiConverter for Array | undefined +const FfiConverterOptionalSequenceTypeActionEvidence = new FfiConverterOptional(FfiConverterSequenceTypeActionEvidence); + +// FfiConverter for ActionEscalation | undefined +const FfiConverterOptionalTypeActionEscalation = new FfiConverterOptional(FfiConverterTypeActionEscalation); + // FfiConverter for number | undefined const FfiConverterOptionalFloat64 = new FfiConverterOptional(FfiConverterFloat64); @@ -2465,9 +2873,6 @@ const FfiConverterOptionalString = new FfiConverterOptional(FfiConverterString); // FfiConverter for ClickButton | undefined const FfiConverterOptionalTypeClickButton = new FfiConverterOptional(FfiConverterTypeClickButton); -// FfiConverter for number | undefined -const FfiConverterOptionalUInt32 = new FfiConverterOptional(FfiConverterUInt32); - // FfiConverter for Array const FfiConverterSequenceString = new FfiConverterArray(FfiConverterString); @@ -2538,6 +2943,16 @@ function uniffiEnsureInitialized() { export default Object.freeze({ initialize: uniffiEnsureInitialized, converters: { + FfiConverterTypeActionDelivery, + FfiConverterTypeActionDeliveryMode, + FfiConverterTypeActionEffect, + FfiConverterTypeActionEscalation, + FfiConverterTypeActionEscalationReason, + FfiConverterTypeActionEscalationTarget, + FfiConverterTypeActionEvidence, + FfiConverterTypeActionEvidenceKind, + FfiConverterTypeActionResult, + FfiConverterTypeActionRoute, FfiConverterTypeBoundsExpectation, FfiConverterTypeCaptureScope, FfiConverterTypeClickButton, diff --git a/libs/cua-driver/typescript/src/native/cua_driver_sdk.ts b/libs/cua-driver/typescript/src/native/cua_driver_sdk.ts index 77ca84c133..2c2ea5aad3 100644 --- a/libs/cua-driver/typescript/src/native/cua_driver_sdk.ts +++ b/libs/cua-driver/typescript/src/native/cua_driver_sdk.ts @@ -6,12 +6,12 @@ import nativeModule from "./cua_driver_sdk-ffi.js"; import { type UniffiRustFutureContinuationCallback, type UniffiForeignFutureDroppedCallback, type UniffiForeignFutureDroppedCallbackStruct, type UniffiVTableCallbackInterfaceCuaDriverSdkDriverActivityObserver, type UniffiForeignFutureResultRustBuffer, type UniffiForeignFutureCompleterustBuffer, type UniffiVTableCallbackInterfaceCuaDriverSdkDriverAuthorizationHost, } from "./cua_driver_sdk-ffi.js"; -import { type ClickInput, type DragInput, type EndSessionInput, type EndSessionOutput, type EscalateSessionInput, type GetAgentCursorStateInput, type GetCursorPositionInput, type GetDesktopStateInput, type GetScreenSizeInput, type GetSessionStateInput, type HotkeyInput, type MoveCursorInput, type PressKeyInput, type ScrollInput, type SessionStateOutput, type SetAgentCursorEnabledInput, type SetAgentCursorMotionInput, type SetAgentCursorThemeInput, type StartSessionInput, type StartSessionOutput, type TypeTextInput, type VerifyStateInput, +import { type ActionResult, type ClickInput, type DragInput, type EndSessionInput, type EndSessionOutput, type EscalateSessionInput, type GetAgentCursorStateInput, type GetCursorPositionInput, type GetDesktopStateInput, type GetScreenSizeInput, type GetSessionStateInput, type HotkeyInput, type MoveCursorInput, type PressKeyInput, type ScrollInput, type SessionStateOutput, type SetAgentCursorEnabledInput, type SetAgentCursorMotionInput, type SetAgentCursorThemeInput, type StartSessionInput, type StartSessionOutput, type TypeTextInput, type VerifyStateInput, type VerifyStateOutput, } from "./cua_driver_contract.js"; import { type FfiConverter, type UniffiByteArray, type UniffiGcObject, type UniffiHandle, type UniffiObjectFactory, type UniffiReferenceHolder, type UniffiRustCallStatus, AbstractFfiConverterByteArray, FfiConverterArray, FfiConverterBool, FfiConverterInt32, FfiConverterObject, FfiConverterObjectWithCallbacks, FfiConverterOptional, FfiConverterUInt32, FfiConverterUInt64, FfiConverterUInt8, RustBuffer, UniffiAbstractObject, UniffiEnum, UniffiError, UniffiInternalError, UniffiResult, UniffiRustCaller, destructorGuardSymbol, pointerLiteralSymbol, uniffiCreateFfiConverterString, uniffiCreateRecord, uniffiRustCallAsync, uniffiTraitInterfaceCall, uniffiTraitInterfaceCallAsyncWithError, uniffiTypeNameSymbol, variantOrdinalSymbol, } from "@ubjs/core"; import uniffiCuaDriverContractModule from "./cua_driver_contract.js"; -const { FfiConverterTypeClickInput, FfiConverterTypeDragInput, FfiConverterTypeEndSessionInput, FfiConverterTypeEndSessionOutput, FfiConverterTypeEscalateSessionInput, FfiConverterTypeGetAgentCursorStateInput, FfiConverterTypeGetCursorPositionInput, FfiConverterTypeGetDesktopStateInput, FfiConverterTypeGetScreenSizeInput, FfiConverterTypeGetSessionStateInput, FfiConverterTypeHotkeyInput, FfiConverterTypeMoveCursorInput, FfiConverterTypePressKeyInput, FfiConverterTypeScrollInput, FfiConverterTypeSessionStateOutput, FfiConverterTypeSetAgentCursorEnabledInput, FfiConverterTypeSetAgentCursorMotionInput, FfiConverterTypeSetAgentCursorThemeInput, FfiConverterTypeStartSessionInput, FfiConverterTypeStartSessionOutput, FfiConverterTypeTypeTextInput, FfiConverterTypeVerifyStateInput } = uniffiCuaDriverContractModule.converters; +const { FfiConverterTypeActionResult, FfiConverterTypeClickInput, FfiConverterTypeDragInput, FfiConverterTypeEndSessionInput, FfiConverterTypeEndSessionOutput, FfiConverterTypeEscalateSessionInput, FfiConverterTypeGetAgentCursorStateInput, FfiConverterTypeGetCursorPositionInput, FfiConverterTypeGetDesktopStateInput, FfiConverterTypeGetScreenSizeInput, FfiConverterTypeGetSessionStateInput, FfiConverterTypeHotkeyInput, FfiConverterTypeMoveCursorInput, FfiConverterTypePressKeyInput, FfiConverterTypeScrollInput, FfiConverterTypeSessionStateOutput, FfiConverterTypeSetAgentCursorEnabledInput, FfiConverterTypeSetAgentCursorMotionInput, FfiConverterTypeSetAgentCursorThemeInput, FfiConverterTypeStartSessionInput, FfiConverterTypeStartSessionOutput, FfiConverterTypeTypeTextInput, FfiConverterTypeVerifyStateInput, FfiConverterTypeVerifyStateOutput } = uniffiCuaDriverContractModule.converters; const uniffiCaller = new UniffiRustCaller(() => ({ code: 0 })); const uniffiIsDebug = @@ -1142,7 +1142,8 @@ export type ToolResult = { structuredJson?: string, isError: boolean, errorCode?: string, - verified?: boolean, + action?: ActionResult, + verification?: VerifyStateOutput, degraded: boolean, rawJson: string } @@ -1173,7 +1174,8 @@ const FfiConverterTypeToolResult = (() => { structuredJson: FfiConverterOptionalString.read(from), isError: FfiConverterBool.read(from), errorCode: FfiConverterOptionalString.read(from), - verified: FfiConverterOptionalBoolean.read(from), + action: FfiConverterOptionalTypeActionResult.read(from), + verification: FfiConverterOptionalTypeVerifyStateOutput.read(from), degraded: FfiConverterBool.read(from), rawJson: FfiConverterString.read(from) }; @@ -1184,7 +1186,8 @@ const FfiConverterTypeToolResult = (() => { FfiConverterOptionalString.write(value.structuredJson, into); FfiConverterBool.write(value.isError, into); FfiConverterOptionalString.write(value.errorCode, into); - FfiConverterOptionalBoolean.write(value.verified, into); + FfiConverterOptionalTypeActionResult.write(value.action, into); + FfiConverterOptionalTypeVerifyStateOutput.write(value.verification, into); FfiConverterBool.write(value.degraded, into); FfiConverterString.write(value.rawJson, into); } @@ -1194,7 +1197,8 @@ const FfiConverterTypeToolResult = (() => { FfiConverterOptionalString.allocationSize(value.structuredJson) + FfiConverterBool.allocationSize(value.isError) + FfiConverterOptionalString.allocationSize(value.errorCode) + - FfiConverterOptionalBoolean.allocationSize(value.verified) + + FfiConverterOptionalTypeActionResult.allocationSize(value.action) + + FfiConverterOptionalTypeVerifyStateOutput.allocationSize(value.verification) + FfiConverterBool.allocationSize(value.degraded) + FfiConverterString.allocationSize(value.rawJson); @@ -5204,8 +5208,11 @@ const FfiConverterOptionalTypeEmbeddedPermissionMode = new FfiConverterOptional( // FfiConverter for Array const FfiConverterSequenceTypeImageContent = new FfiConverterArray(FfiConverterTypeImageContent); -// FfiConverter for boolean | undefined -const FfiConverterOptionalBoolean = new FfiConverterOptional(FfiConverterBool); +// FfiConverter for ActionResult | undefined +const FfiConverterOptionalTypeActionResult = new FfiConverterOptional(FfiConverterTypeActionResult); + +// FfiConverter for VerifyStateOutput | undefined +const FfiConverterOptionalTypeVerifyStateOutput = new FfiConverterOptional(FfiConverterTypeVerifyStateOutput); // FfiConverter for EmbeddedDriverConnection | undefined const FfiConverterOptionalTypeEmbeddedDriverConnection = new FfiConverterOptional(FfiConverterTypeEmbeddedDriverConnection); diff --git a/libs/cua-driver/typescript/test/electron-main-fixture.mjs b/libs/cua-driver/typescript/test/electron-main-fixture.mjs index 577daa01c4..0361e41db1 100644 --- a/libs/cua-driver/typescript/test/electron-main-fixture.mjs +++ b/libs/cua-driver/typescript/test/electron-main-fixture.mjs @@ -27,7 +27,7 @@ const server = net.createServer(socket => { const request = JSON.parse(buffer.split("\\n", 1)[0]); const result = request.method === "metadata" ? { driver_version: ${JSON.stringify(packageVersion)}, - contract_version: hostBundleId.endsWith(".failure") ? "incompatible" : "0.3.0", + contract_version: hostBundleId.endsWith(".failure") ? "incompatible" : "0.4.0", tools_list_schema_version: "1", capability_version: "1", mcp_protocol_version: "2025-06-18", diff --git a/libs/cua-driver/typescript/test/native-daemon-fixture.mjs b/libs/cua-driver/typescript/test/native-daemon-fixture.mjs index bd95385e28..95e078e553 100644 --- a/libs/cua-driver/typescript/test/native-daemon-fixture.mjs +++ b/libs/cua-driver/typescript/test/native-daemon-fixture.mjs @@ -3,6 +3,7 @@ import net from "node:net" const socketPath = process.argv[2] if (!socketPath) throw new Error("missing socket path") +let completedCalls = 0 const server = net.createServer((connection) => { let buffer = "" connection.setEncoding("utf8") @@ -17,7 +18,7 @@ const server = net.createServer((connection) => { ok: true, result: { driver_version: "0.12.6", - contract_version: "0.3.0", + contract_version: "0.4.0", tools_list_schema_version: "1", capability_version: "1", mcp_protocol_version: "2025-06-18", @@ -30,6 +31,20 @@ const server = net.createServer((connection) => { return } process.send?.({ request }) + const structuredContent = + request.name === "verify_state" + ? { + status: "satisfied", + stable: true, + elapsed_ms: 12, + samples: 2, + predicates: [], + } + : { + effect: "unverifiable", + route: "global_input", + delivery: { mode: "not_applicable" }, + } connection.end( `${JSON.stringify({ ok: true, @@ -38,12 +53,13 @@ const server = net.createServer((connection) => { { type: "text", text: "node ffi" }, { type: "image", mimeType: "image/png", data: "cG5n" }, ], - structuredContent: { verified: true }, + structuredContent, isError: false, }, })}\n`, ) - server.close() + completedCalls += 1 + if (completedCalls === 2) server.close() }) }) diff --git a/libs/cua-driver/typescript/test/native-loader.test.mjs b/libs/cua-driver/typescript/test/native-loader.test.mjs index 80326e6913..b110b6f39b 100644 --- a/libs/cua-driver/typescript/test/native-loader.test.mjs +++ b/libs/cua-driver/typescript/test/native-loader.test.mjs @@ -53,7 +53,7 @@ const server = net.createServer(socket => { const request = JSON.parse(buffer.split("\\n", 1)[0]); const result = request.method === "metadata" ? { driver_version: "0.10.0", - contract_version: "0.3.0", + contract_version: "0.4.0", tools_list_schema_version: "1", capability_version: "1", mcp_protocol_version: "2025-06-18", @@ -107,12 +107,14 @@ test( if (message.ready) resolve(null) }) }) - const requestPromise = readyPromise.then( + const requests = [] + const requestsPromise = readyPromise.then( () => new Promise((resolve, reject) => { fixture.on("error", reject) fixture.on("message", (message) => { - if (message.request) resolve(message.request) + if (message.request) requests.push(message.request) + if (requests.length === 2) resolve(requests) }) }), ) @@ -121,7 +123,18 @@ test( await readyPromise assert.equal(existsSync(socketPath), true) const sdk = await import("@trycua/cua-driver") - const { CuaDriver, StatePredicate, VerifyStateInput, WindowPredicate } = sdk + const { + ActionEffect, + ActionRoute, + ClickButton, + ClickInput, + CuaDriver, + DesktopScope, + StatePredicate, + VerificationStatus, + VerifyStateInput, + WindowPredicate, + } = sdk assert.equal("StdioMcpTransport" in sdk, false) await assert.rejects( import("@trycua/cua-driver/sdk"), @@ -153,7 +166,7 @@ test( expectedMethods.every((name) => typeof driver[name] === "function"), true, ) - const result = await driver.verifyState( + const verificationResult = await driver.verifyState( VerifyStateInput.new({ pid: 123n, windowId: 456n, @@ -168,14 +181,29 @@ test( includeScreenshot: true, }), ) - const request = await requestPromise + const actionResult = await driver.click( + ClickInput.new({ + x: 12, + y: 34, + scope: DesktopScope.Desktop, + session: "node-run", + button: ClickButton.Left, + count: 1, + }), + ) + await requestsPromise driver.uniffiDestroy() - assert.equal(result.text, "node ffi") - assert.equal(result.images[0].mimeType, "image/png") - assert.equal(result.verified, true) - assert.equal(request.name, "verify_state") - assert.deepEqual(request.args, { + assert.equal(verificationResult.text, "node ffi") + assert.equal(verificationResult.images[0].mimeType, "image/png") + assert.equal(verificationResult.action, undefined) + assert.equal(verificationResult.verification.status, VerificationStatus.Satisfied) + assert.equal(actionResult.verification, undefined) + assert.equal(actionResult.action.effect, ActionEffect.Unverifiable) + assert.equal(actionResult.action.route, ActionRoute.GlobalInput) + assert.equal("verified" in actionResult, false) + assert.equal(requests[0].name, "verify_state") + assert.deepEqual(requests[0].args, { pid: 123, window_id: 456, expect: [{ window: { exists: true } }], @@ -184,7 +212,16 @@ test( stable_samples: 1, include_screenshot: true, }) - assert.equal(request.client_kind, "typescript_sdk") + assert.equal(requests[0].client_kind, "typescript_sdk") + assert.equal(requests[1].name, "click") + assert.deepEqual(requests[1].args, { + x: 12, + y: 34, + scope: "desktop", + session: "node-run", + button: "left", + count: 1, + }) } finally { fixture.kill() rmSync(directory, { recursive: true, force: true }) diff --git a/libs/python/computer-server/computer_server/handlers/cua_driver.py b/libs/python/computer-server/computer_server/handlers/cua_driver.py index f6983d1703..97df86de60 100644 --- a/libs/python/computer-server/computer_server/handlers/cua_driver.py +++ b/libs/python/computer-server/computer_server/handlers/cua_driver.py @@ -35,7 +35,6 @@ class DriverResult(Protocol): is_error: bool text: str error_code: Optional[str] - verified: Optional[bool] degraded: bool @@ -185,8 +184,19 @@ def _raise_for_error(result: DriverResult) -> None: def _result_data(cls, result: DriverResult) -> Dict[str, Any]: cls._raise_for_error(result) data = cls._structured(result) - if result.verified is not None: - data.setdefault("verified", result.verified) + # computer-server's legacy envelope predates ActionResult. Preserve a + # compatibility boolean as a deliberately lossy "confirmed" bit: + # only a confirmed effect maps to True; every other action effect maps + # to False. New integrations must use `effect` instead. + if data.get("effect") == "confirmed": + data.setdefault("verified", True) + elif data.get("effect") in { + "unverifiable", + "partial", + "suspected_noop", + "refused", + }: + data.setdefault("verified", False) if result.degraded: data.setdefault("degraded", True) return data diff --git a/libs/python/computer-server/tests/test_cua_driver_handler.py b/libs/python/computer-server/tests/test_cua_driver_handler.py index 638bbfeedb..e16b2942af 100644 --- a/libs/python/computer-server/tests/test_cua_driver_handler.py +++ b/libs/python/computer-server/tests/test_cua_driver_handler.py @@ -59,7 +59,6 @@ def __init__(self, structured=None, *, images=None, error=None): self.is_error = error is not None self.text = error or "ok" self.error_code = "test_error" if error else None - self.verified = True self.degraded = False @@ -120,33 +119,43 @@ async def get_cursor_position(self, input): self.calls.append(("get_cursor_position", input)) return _Result({"x": 12, "y": 34, "available": True}) + @staticmethod + def _action_result(): + return _Result( + { + "effect": "unverifiable", + "route": "global_input", + "delivery": {"mode": "not_applicable"}, + } + ) + async def move_cursor(self, input): self.calls.append(("move_cursor", input)) - return _Result({}) + return self._action_result() async def click(self, input): self.calls.append(("click", input)) - return _Result({}) + return self._action_result() async def drag(self, input): self.calls.append(("drag", input)) - return _Result({}) + return self._action_result() async def scroll(self, input): self.calls.append(("scroll", input)) - return _Result({}) + return self._action_result() async def type_text(self, input): self.calls.append(("type_text", input)) - return _Result({}) + return self._action_result() async def press_key(self, input): self.calls.append(("press_key", input)) - return _Result({}) + return self._action_result() async def hotkey(self, input): self.calls.append(("hotkey", input)) - return _Result({}) + return self._action_result() async def shutdown(self): self.shutdown_count += 1 @@ -211,6 +220,8 @@ async def test_desktop_actions_share_one_typed_session(sdk, fallback): assert desktop["image_data"] == "cG5n" assert desktop["screen_width"] == 1280 assert click["success"] is True + assert click["effect"] == "unverifiable" + assert click["verified"] is False assert scroll["success"] is True assert [name for name, _ in driver.calls].count("start_session") == 1 start = next(value for name, value in driver.calls if name == "start_session")