diff --git a/libs/cua-driver/contract/manifest.json b/libs/cua-driver/contract/manifest.json index 9965f5f455..c12f26f12e 100644 --- a/libs/cua-driver/contract/manifest.json +++ b/libs/cua-driver/contract/manifest.json @@ -172,7 +172,8 @@ "delivery_failed", "effect_unconfirmed", "suspected_noop", - "permission_required" + "permission_required", + "surface_changed" ], "type": "string" }, @@ -181,9 +182,37 @@ "pixel", "foreground", "page", - "session" + "session", + "rebind" ], "type": "string" + }, + "window": { + "additionalProperties": false, + "properties": { + "app_name": { + "type": "string" + }, + "pid": { + "format": "int64", + "type": "integer" + }, + "window_id": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "app_name", + "title" + ], + "type": [ + "object", + "null" + ] } }, "required": [ @@ -201,8 +230,7 @@ "properties": { "kind": { "enum": [ - "value_readback", - "window_change" + "value_readback" ], "type": "string" } @@ -227,6 +255,49 @@ "trusted_input" ], "type": "string" + }, + "window_change": { + "additionalProperties": false, + "properties": { + "foreground_changed": { + "type": "boolean" + }, + "new_windows": { + "items": { + "additionalProperties": false, + "properties": { + "app_name": { + "type": "string" + }, + "pid": { + "format": "int64", + "type": "integer" + }, + "window_id": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "app_name", + "title" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "new_windows", + "foreground_changed" + ], + "type": [ + "object", + "null" + ] } }, "required": [ @@ -575,7 +646,8 @@ "delivery_failed", "effect_unconfirmed", "suspected_noop", - "permission_required" + "permission_required", + "surface_changed" ], "type": "string" }, @@ -584,9 +656,37 @@ "pixel", "foreground", "page", - "session" + "session", + "rebind" ], "type": "string" + }, + "window": { + "additionalProperties": false, + "properties": { + "app_name": { + "type": "string" + }, + "pid": { + "format": "int64", + "type": "integer" + }, + "window_id": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "app_name", + "title" + ], + "type": [ + "object", + "null" + ] } }, "required": [ @@ -604,8 +704,7 @@ "properties": { "kind": { "enum": [ - "value_readback", - "window_change" + "value_readback" ], "type": "string" } @@ -630,6 +729,49 @@ "trusted_input" ], "type": "string" + }, + "window_change": { + "additionalProperties": false, + "properties": { + "foreground_changed": { + "type": "boolean" + }, + "new_windows": { + "items": { + "additionalProperties": false, + "properties": { + "app_name": { + "type": "string" + }, + "pid": { + "format": "int64", + "type": "integer" + }, + "window_id": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "app_name", + "title" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "new_windows", + "foreground_changed" + ], + "type": [ + "object", + "null" + ] } }, "required": [ @@ -1565,7 +1707,8 @@ "delivery_failed", "effect_unconfirmed", "suspected_noop", - "permission_required" + "permission_required", + "surface_changed" ], "type": "string" }, @@ -1574,9 +1717,37 @@ "pixel", "foreground", "page", - "session" + "session", + "rebind" ], "type": "string" + }, + "window": { + "additionalProperties": false, + "properties": { + "app_name": { + "type": "string" + }, + "pid": { + "format": "int64", + "type": "integer" + }, + "window_id": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "app_name", + "title" + ], + "type": [ + "object", + "null" + ] } }, "required": [ @@ -1594,8 +1765,7 @@ "properties": { "kind": { "enum": [ - "value_readback", - "window_change" + "value_readback" ], "type": "string" } @@ -1620,6 +1790,49 @@ "trusted_input" ], "type": "string" + }, + "window_change": { + "additionalProperties": false, + "properties": { + "foreground_changed": { + "type": "boolean" + }, + "new_windows": { + "items": { + "additionalProperties": false, + "properties": { + "app_name": { + "type": "string" + }, + "pid": { + "format": "int64", + "type": "integer" + }, + "window_id": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "app_name", + "title" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "new_windows", + "foreground_changed" + ], + "type": [ + "object", + "null" + ] } }, "required": [ @@ -1736,7 +1949,8 @@ "delivery_failed", "effect_unconfirmed", "suspected_noop", - "permission_required" + "permission_required", + "surface_changed" ], "type": "string" }, @@ -1745,9 +1959,37 @@ "pixel", "foreground", "page", - "session" + "session", + "rebind" ], "type": "string" + }, + "window": { + "additionalProperties": false, + "properties": { + "app_name": { + "type": "string" + }, + "pid": { + "format": "int64", + "type": "integer" + }, + "window_id": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "app_name", + "title" + ], + "type": [ + "object", + "null" + ] } }, "required": [ @@ -1765,8 +2007,7 @@ "properties": { "kind": { "enum": [ - "value_readback", - "window_change" + "value_readback" ], "type": "string" } @@ -1791,6 +2032,49 @@ "trusted_input" ], "type": "string" + }, + "window_change": { + "additionalProperties": false, + "properties": { + "foreground_changed": { + "type": "boolean" + }, + "new_windows": { + "items": { + "additionalProperties": false, + "properties": { + "app_name": { + "type": "string" + }, + "pid": { + "format": "int64", + "type": "integer" + }, + "window_id": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "app_name", + "title" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "new_windows", + "foreground_changed" + ], + "type": [ + "object", + "null" + ] } }, "required": [ @@ -2088,7 +2372,8 @@ "delivery_failed", "effect_unconfirmed", "suspected_noop", - "permission_required" + "permission_required", + "surface_changed" ], "type": "string" }, @@ -2097,9 +2382,37 @@ "pixel", "foreground", "page", - "session" + "session", + "rebind" ], "type": "string" + }, + "window": { + "additionalProperties": false, + "properties": { + "app_name": { + "type": "string" + }, + "pid": { + "format": "int64", + "type": "integer" + }, + "window_id": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "app_name", + "title" + ], + "type": [ + "object", + "null" + ] } }, "required": [ @@ -2117,8 +2430,7 @@ "properties": { "kind": { "enum": [ - "value_readback", - "window_change" + "value_readback" ], "type": "string" } @@ -2143,6 +2455,49 @@ "trusted_input" ], "type": "string" + }, + "window_change": { + "additionalProperties": false, + "properties": { + "foreground_changed": { + "type": "boolean" + }, + "new_windows": { + "items": { + "additionalProperties": false, + "properties": { + "app_name": { + "type": "string" + }, + "pid": { + "format": "int64", + "type": "integer" + }, + "window_id": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "app_name", + "title" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "new_windows", + "foreground_changed" + ], + "type": [ + "object", + "null" + ] } }, "required": [ @@ -2305,7 +2660,8 @@ "delivery_failed", "effect_unconfirmed", "suspected_noop", - "permission_required" + "permission_required", + "surface_changed" ], "type": "string" }, @@ -2314,9 +2670,37 @@ "pixel", "foreground", "page", - "session" + "session", + "rebind" ], "type": "string" + }, + "window": { + "additionalProperties": false, + "properties": { + "app_name": { + "type": "string" + }, + "pid": { + "format": "int64", + "type": "integer" + }, + "window_id": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "app_name", + "title" + ], + "type": [ + "object", + "null" + ] } }, "required": [ @@ -2334,8 +2718,7 @@ "properties": { "kind": { "enum": [ - "value_readback", - "window_change" + "value_readback" ], "type": "string" } @@ -2360,6 +2743,49 @@ "trusted_input" ], "type": "string" + }, + "window_change": { + "additionalProperties": false, + "properties": { + "foreground_changed": { + "type": "boolean" + }, + "new_windows": { + "items": { + "additionalProperties": false, + "properties": { + "app_name": { + "type": "string" + }, + "pid": { + "format": "int64", + "type": "integer" + }, + "window_id": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "app_name", + "title" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "new_windows", + "foreground_changed" + ], + "type": [ + "object", + "null" + ] } }, "required": [ @@ -2542,7 +2968,8 @@ "delivery_failed", "effect_unconfirmed", "suspected_noop", - "permission_required" + "permission_required", + "surface_changed" ], "type": "string" }, @@ -2551,9 +2978,37 @@ "pixel", "foreground", "page", - "session" + "session", + "rebind" ], "type": "string" + }, + "window": { + "additionalProperties": false, + "properties": { + "app_name": { + "type": "string" + }, + "pid": { + "format": "int64", + "type": "integer" + }, + "window_id": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "app_name", + "title" + ], + "type": [ + "object", + "null" + ] } }, "required": [ @@ -2571,8 +3026,7 @@ "properties": { "kind": { "enum": [ - "value_readback", - "window_change" + "value_readback" ], "type": "string" } @@ -2597,6 +3051,49 @@ "trusted_input" ], "type": "string" + }, + "window_change": { + "additionalProperties": false, + "properties": { + "foreground_changed": { + "type": "boolean" + }, + "new_windows": { + "items": { + "additionalProperties": false, + "properties": { + "app_name": { + "type": "string" + }, + "pid": { + "format": "int64", + "type": "integer" + }, + "window_id": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "app_name", + "title" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "new_windows", + "foreground_changed" + ], + "type": [ + "object", + "null" + ] } }, "required": [ @@ -3026,7 +3523,8 @@ "delivery_failed", "effect_unconfirmed", "suspected_noop", - "permission_required" + "permission_required", + "surface_changed" ], "type": "string" }, @@ -3035,9 +3533,37 @@ "pixel", "foreground", "page", - "session" + "session", + "rebind" ], "type": "string" + }, + "window": { + "additionalProperties": false, + "properties": { + "app_name": { + "type": "string" + }, + "pid": { + "format": "int64", + "type": "integer" + }, + "window_id": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "app_name", + "title" + ], + "type": [ + "object", + "null" + ] } }, "required": [ @@ -3055,8 +3581,7 @@ "properties": { "kind": { "enum": [ - "value_readback", - "window_change" + "value_readback" ], "type": "string" } @@ -3081,6 +3606,49 @@ "trusted_input" ], "type": "string" + }, + "window_change": { + "additionalProperties": false, + "properties": { + "foreground_changed": { + "type": "boolean" + }, + "new_windows": { + "items": { + "additionalProperties": false, + "properties": { + "app_name": { + "type": "string" + }, + "pid": { + "format": "int64", + "type": "integer" + }, + "window_id": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "app_name", + "title" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "new_windows", + "foreground_changed" + ], + "type": [ + "object", + "null" + ] } }, "required": [ @@ -3375,7 +3943,8 @@ "delivery_failed", "effect_unconfirmed", "suspected_noop", - "permission_required" + "permission_required", + "surface_changed" ], "type": "string" }, @@ -3384,9 +3953,37 @@ "pixel", "foreground", "page", - "session" + "session", + "rebind" ], "type": "string" + }, + "window": { + "additionalProperties": false, + "properties": { + "app_name": { + "type": "string" + }, + "pid": { + "format": "int64", + "type": "integer" + }, + "window_id": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "app_name", + "title" + ], + "type": [ + "object", + "null" + ] } }, "required": [ @@ -3404,8 +4001,7 @@ "properties": { "kind": { "enum": [ - "value_readback", - "window_change" + "value_readback" ], "type": "string" } @@ -3430,6 +4026,49 @@ "trusted_input" ], "type": "string" + }, + "window_change": { + "additionalProperties": false, + "properties": { + "foreground_changed": { + "type": "boolean" + }, + "new_windows": { + "items": { + "additionalProperties": false, + "properties": { + "app_name": { + "type": "string" + }, + "pid": { + "format": "int64", + "type": "integer" + }, + "window_id": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid", + "window_id", + "app_name", + "title" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "new_windows", + "foreground_changed" + ], + "type": [ + "object", + "null" + ] } }, "required": [ diff --git a/libs/cua-driver/docs/action-result-contract.md b/libs/cua-driver/docs/action-result-contract.md index 409dbfcc7e..2084d0a13e 100644 --- a/libs/cua-driver/docs/action-result-contract.md +++ b/libs/cua-driver/docs/action-result-contract.md @@ -30,9 +30,10 @@ Every successful action returns a closed `structuredContent` object: | `effect` | `confirmed`, `partial`, `unverifiable`, `suspected_noop`, `refused` | | `route` | `accessibility`, `synthetic_events`, `global_input`, `dom`, `trusted_input` | | `delivery.mode` | `background`, `foreground`, `not_applicable`, `unknown` | -| `evidence[].kind` | `value_readback`, `window_change` | -| `escalation.target` | `pixel`, `foreground`, `page`, `session` | -| `escalation.reason` | `route_unavailable`, `delivery_failed`, `effect_unconfirmed`, `suspected_noop`, `permission_required` | +| `evidence[].kind` | `value_readback` | +| `window_change.new_windows[]` | target-scoped, owner-verified `pid`, `window_id`, application, and title | +| `escalation.target` | `pixel`, `foreground`, `page`, `session`, `rebind` | +| `escalation.reason` | `route_unavailable`, `delivery_failed`, `effect_unconfirmed`, `suspected_noop`, `permission_required`, `surface_changed` | The action-result tools are: @@ -51,7 +52,7 @@ scope, targets, platform transport names, diagnostic pointers, or the old The invariants are: -- `confirmed` has publishable readback or window-change evidence; +- `confirmed` has publishable value readback; topology alone cannot confirm an action; - `partial` has `delivery.delivered_count`; - `refused` has neither delivery nor evidence. @@ -72,8 +73,15 @@ resolution semantics; the guard does not replace or reinterpret them. An action that reached an actuator but lacks a trusted readback is `unverifiable`, not `confirmed`. Screenshot change, native API acceptance, -event receipt, and operator observation may remain useful internal diagnostics, -but they do not independently justify `confirmed`. +event receipt, window topology, and operator observation may remain useful +facts, but they do not independently justify `confirmed`. + +On macOS, one action decorator snapshots accessibility roots for the target PID +before and after dispatch. It reports new windows, dialogs, sheets, and popovers +without activating them. An exact `rebind` target is present only when one new +owner-verified root is modal or focused; ambiguous changes require +`list_windows`. Windows and Linux currently emit no topology record rather than +substituting a global desktop heuristic. ## Verification remains separate @@ -123,6 +131,7 @@ An optional escalation is advice, not an automatic retry: | `foreground` | explicitly select foreground delivery when session policy permits | | `page` | bind the native window to a supported browser page route | | `session` | prepare or explicitly widen the session only when policy permits | +| `rebind` | refresh the validated window target without activation; use `list_windows` when no exact target is present | SDK integrators, OpenClaw, Hermes, and other agent hosts can implement different policies above this same narrow fact contract without duplicating platform diff --git a/libs/cua-driver/python/src/cua_driver/__init__.py b/libs/cua-driver/python/src/cua_driver/__init__.py index 5a83149c70..65477f52e6 100644 --- a/libs/cua-driver/python/src/cua_driver/__init__.py +++ b/libs/cua-driver/python/src/cua_driver/__init__.py @@ -58,6 +58,8 @@ ActionEvidenceKind, ActionResult, ActionRoute, + ActionWindowChange, + ActionWindowTarget, BoundsExpectation, CaptureScope, ClickButton, @@ -180,6 +182,8 @@ def _create_private_worker_python_sdk(cls, options): "ActionEvidenceKind", "ActionResult", "ActionRoute", + "ActionWindowChange", + "ActionWindowTarget", "BoundsExpectation", "CaptureScope", "ClickButton", diff --git a/libs/cua-driver/python/src/cua_driver/_native_contract.py b/libs/cua-driver/python/src/cua_driver/_native_contract.py index 61a69eb96c..6ee4e2093f 100644 --- a/libs/cua-driver/python/src/cua_driver/_native_contract.py +++ b/libs/cua-driver/python/src/cua_driver/_native_contract.py @@ -899,6 +899,8 @@ class ActionEscalationTarget(enum.Enum): SESSION = 3 + REBIND = 4 + class _UniffiFfiConverterTypeActionEscalationTarget(_UniffiConverterRustBuffer): @@ -913,6 +915,8 @@ def read(buf): return ActionEscalationTarget.PAGE if variant == 4: return ActionEscalationTarget.SESSION + if variant == 5: + return ActionEscalationTarget.REBIND raise InternalError("Raw enum value doesn't match any cases") @staticmethod @@ -925,6 +929,8 @@ def check_lower(value): return if value == ActionEscalationTarget.SESSION: return + if value == ActionEscalationTarget.REBIND: + return raise ValueError(value) @staticmethod @@ -937,6 +943,8 @@ def write(value, buf): buf.write_i32(3) if value == ActionEscalationTarget.SESSION: buf.write_i32(4) + if value == ActionEscalationTarget.REBIND: + buf.write_i32(5) @@ -957,6 +965,8 @@ class ActionEscalationReason(enum.Enum): PERMISSION_REQUIRED = 4 + SURFACE_CHANGED = 5 + class _UniffiFfiConverterTypeActionEscalationReason(_UniffiConverterRustBuffer): @@ -973,6 +983,8 @@ def read(buf): return ActionEscalationReason.SUSPECTED_NOOP if variant == 5: return ActionEscalationReason.PERMISSION_REQUIRED + if variant == 6: + return ActionEscalationReason.SURFACE_CHANGED raise InternalError("Raw enum value doesn't match any cases") @staticmethod @@ -987,6 +999,8 @@ def check_lower(value): return if value == ActionEscalationReason.PERMISSION_REQUIRED: return + if value == ActionEscalationReason.SURFACE_CHANGED: + return raise ValueError(value) @staticmethod @@ -1001,25 +1015,164 @@ def write(value, buf): buf.write_i32(4) if value == ActionEscalationReason.PERMISSION_REQUIRED: buf.write_i32(5) + if value == ActionEscalationReason.SURFACE_CHANGED: + buf.write_i32(6) + + + +class _UniffiFfiConverterInt64(_UniffiConverterPrimitiveInt): + CLASS_NAME = "i64" + VALUE_MIN = -2**63 + VALUE_MAX = 2**63 + + @staticmethod + def read(buf): + return buf.read_i64() + + @staticmethod + def write(value, buf): + buf.write_i64(value) + +class _UniffiFfiConverterUInt64(_UniffiConverterPrimitiveInt): + CLASS_NAME = "u64" + VALUE_MIN = 0 + VALUE_MAX = 2**64 + + @staticmethod + def read(buf): + return buf.read_u64() + + @staticmethod + def write(value, buf): + buf.write_u64(value) + +class _UniffiFfiConverterString: + @staticmethod + def check_lower(value): + if not isinstance(value, str): + raise TypeError("argument must be str, not {}".format(type(value).__name__)) + return value + @staticmethod + def read(buf): + size = buf.read_i32() + if size < 0: + raise InternalError("Unexpected negative string length") + utf8_bytes = buf.read(size) + return utf8_bytes.decode("utf-8") + @staticmethod + def write(value, buf): + utf8_bytes = value.encode("utf-8") + buf.write_i32(len(utf8_bytes)) + buf.write(utf8_bytes) + + @staticmethod + def lift(buf): + with buf.consume_with_stream() as stream: + return stream.read(stream.remaining()).decode("utf-8") + + @staticmethod + def lower(value): + with _UniffiRustBuffer.alloc_with_builder() as builder: + builder.write(value.encode("utf-8")) + return builder.finalize() + +@dataclass +class ActionWindowTarget: + """ + A target-owned native interaction root discovered after an action. +""" + def __init__(self, *, pid:int, window_id:int, app_name:str, title:str): + self.pid = pid + self.window_id = window_id + self.app_name = app_name + self.title = title + + + + + def __str__(self): + return "ActionWindowTarget(pid={}, window_id={}, app_name={}, title={})".format(self.pid, self.window_id, self.app_name, self.title) + def __eq__(self, other): + if self.pid != other.pid: + return False + if self.window_id != other.window_id: + return False + if self.app_name != other.app_name: + return False + if self.title != other.title: + return False + return True + +class _UniffiFfiConverterTypeActionWindowTarget(_UniffiConverterRustBuffer): + @staticmethod + def read(buf): + return ActionWindowTarget( + pid=_UniffiFfiConverterInt64.read(buf), + window_id=_UniffiFfiConverterUInt64.read(buf), + app_name=_UniffiFfiConverterString.read(buf), + title=_UniffiFfiConverterString.read(buf), + ) + + @staticmethod + def check_lower(value): + _UniffiFfiConverterInt64.check_lower(value.pid) + _UniffiFfiConverterUInt64.check_lower(value.window_id) + _UniffiFfiConverterString.check_lower(value.app_name) + _UniffiFfiConverterString.check_lower(value.title) + + @staticmethod + def write(value, buf): + _UniffiFfiConverterInt64.write(value.pid, buf) + _UniffiFfiConverterUInt64.write(value.window_id, buf) + _UniffiFfiConverterString.write(value.app_name, buf) + _UniffiFfiConverterString.write(value.title, buf) + +class _UniffiFfiConverterOptionalTypeActionWindowTarget(_UniffiConverterRustBuffer): + @classmethod + def check_lower(cls, value): + if value is not None: + _UniffiFfiConverterTypeActionWindowTarget.check_lower(value) + + @classmethod + def write(cls, value, buf): + if value is None: + buf.write_u8(0) + return + + buf.write_u8(1) + _UniffiFfiConverterTypeActionWindowTarget.write(value, buf) + + @classmethod + def read(cls, buf): + flag = buf.read_u8() + if flag == 0: + return None + elif flag == 1: + return _UniffiFfiConverterTypeActionWindowTarget.read(buf) + else: + raise InternalError("Unexpected flag byte for optional type") @dataclass class ActionEscalation: - def __init__(self, *, target:ActionEscalationTarget, reason:ActionEscalationReason): + def __init__(self, *, target:ActionEscalationTarget, reason:ActionEscalationReason, window:typing.Optional[ActionWindowTarget]): self.target = target self.reason = reason + self.window = window def __str__(self): - return "ActionEscalation(target={}, reason={})".format(self.target, self.reason) + return "ActionEscalation(target={}, reason={}, window={})".format(self.target, self.reason, self.window) def __eq__(self, other): if self.target != other.target: return False if self.reason != other.reason: return False + if self.window != other.window: + return False return True class _UniffiFfiConverterTypeActionEscalation(_UniffiConverterRustBuffer): @@ -1028,17 +1181,20 @@ def read(buf): return ActionEscalation( target=_UniffiFfiConverterTypeActionEscalationTarget.read(buf), reason=_UniffiFfiConverterTypeActionEscalationReason.read(buf), + window=_UniffiFfiConverterOptionalTypeActionWindowTarget.read(buf), ) @staticmethod def check_lower(value): _UniffiFfiConverterTypeActionEscalationTarget.check_lower(value.target) _UniffiFfiConverterTypeActionEscalationReason.check_lower(value.reason) + _UniffiFfiConverterOptionalTypeActionWindowTarget.check_lower(value.window) @staticmethod def write(value, buf): _UniffiFfiConverterTypeActionEscalationTarget.write(value.target, buf) _UniffiFfiConverterTypeActionEscalationReason.write(value.reason, buf) + _UniffiFfiConverterOptionalTypeActionWindowTarget.write(value.window, buf) @@ -1049,8 +1205,6 @@ class ActionEvidenceKind(enum.Enum): VALUE_READBACK = 0 - WINDOW_CHANGE = 1 - class _UniffiFfiConverterTypeActionEvidenceKind(_UniffiConverterRustBuffer): @@ -1059,24 +1213,18 @@ def read(buf): variant = buf.read_i32() if variant == 1: return ActionEvidenceKind.VALUE_READBACK - if variant == 2: - return ActionEvidenceKind.WINDOW_CHANGE raise InternalError("Raw enum value doesn't match any cases") @staticmethod def check_lower(value): if value == ActionEvidenceKind.VALUE_READBACK: return - if value == ActionEvidenceKind.WINDOW_CHANGE: - return raise ValueError(value) @staticmethod def write(value, buf): if value == ActionEvidenceKind.VALUE_READBACK: buf.write_i32(1) - if value == ActionEvidenceKind.WINDOW_CHANGE: - buf.write_i32(2) @@ -1319,6 +1467,114 @@ def read(cls, buf): else: raise InternalError("Unexpected flag byte for optional type") +class _UniffiFfiConverterSequenceTypeActionWindowTarget(_UniffiConverterRustBuffer): + @classmethod + def check_lower(cls, value): + for item in value: + _UniffiFfiConverterTypeActionWindowTarget.check_lower(item) + + @classmethod + def write(cls, value, buf): + items = len(value) + buf.write_i32(items) + for item in value: + _UniffiFfiConverterTypeActionWindowTarget.write(item, buf) + + @classmethod + def read(cls, buf): + count = buf.read_i32() + if count < 0: + raise InternalError("Unexpected negative sequence length") + + return [ + _UniffiFfiConverterTypeActionWindowTarget.read(buf) for i in range(count) + ] + +class _UniffiFfiConverterBoolean: + @classmethod + def check_lower(cls, value): + return not not value + + @classmethod + def lower(cls, value): + return 1 if value else 0 + + @staticmethod + def lift(value): + return value != 0 + + @classmethod + def read(cls, buf): + return cls.lift(buf.read_u8()) + + @classmethod + def write(cls, value, buf): + buf.write_u8(value) + +@dataclass +class ActionWindowChange: + """ + Read-only topology observed for the action's target process. +""" + def __init__(self, *, new_windows:typing.List[ActionWindowTarget], foreground_changed:bool): + self.new_windows = new_windows + self.foreground_changed = foreground_changed + + + + + def __str__(self): + return "ActionWindowChange(new_windows={}, foreground_changed={})".format(self.new_windows, self.foreground_changed) + def __eq__(self, other): + if self.new_windows != other.new_windows: + return False + if self.foreground_changed != other.foreground_changed: + return False + return True + +class _UniffiFfiConverterTypeActionWindowChange(_UniffiConverterRustBuffer): + @staticmethod + def read(buf): + return ActionWindowChange( + new_windows=_UniffiFfiConverterSequenceTypeActionWindowTarget.read(buf), + foreground_changed=_UniffiFfiConverterBoolean.read(buf), + ) + + @staticmethod + def check_lower(value): + _UniffiFfiConverterSequenceTypeActionWindowTarget.check_lower(value.new_windows) + _UniffiFfiConverterBoolean.check_lower(value.foreground_changed) + + @staticmethod + def write(value, buf): + _UniffiFfiConverterSequenceTypeActionWindowTarget.write(value.new_windows, buf) + _UniffiFfiConverterBoolean.write(value.foreground_changed, buf) + +class _UniffiFfiConverterOptionalTypeActionWindowChange(_UniffiConverterRustBuffer): + @classmethod + def check_lower(cls, value): + if value is not None: + _UniffiFfiConverterTypeActionWindowChange.check_lower(value) + + @classmethod + def write(cls, value, buf): + if value is None: + buf.write_u8(0) + return + + buf.write_u8(1) + _UniffiFfiConverterTypeActionWindowChange.write(value, buf) + + @classmethod + def read(cls, buf): + flag = buf.read_u8() + if flag == 0: + return None + elif flag == 1: + return _UniffiFfiConverterTypeActionWindowChange.read(buf) + else: + raise InternalError("Unexpected flag byte for optional type") + class _UniffiFfiConverterOptionalTypeActionEscalation(_UniffiConverterRustBuffer): @classmethod def check_lower(cls, value): @@ -1346,18 +1602,19 @@ def read(cls, buf): @dataclass class ActionResult: - def __init__(self, *, effect:ActionEffect, route:ActionRoute, delivery:typing.Optional[ActionDelivery], evidence:typing.Optional[typing.List[ActionEvidence]], escalation:typing.Optional[ActionEscalation]): + def __init__(self, *, effect:ActionEffect, route:ActionRoute, delivery:typing.Optional[ActionDelivery], evidence:typing.Optional[typing.List[ActionEvidence]], window_change:typing.Optional[ActionWindowChange], escalation:typing.Optional[ActionEscalation]): self.effect = effect self.route = route self.delivery = delivery self.evidence = evidence + self.window_change = window_change self.escalation = escalation def __str__(self): - return "ActionResult(effect={}, route={}, delivery={}, evidence={}, escalation={})".format(self.effect, self.route, self.delivery, self.evidence, self.escalation) + return "ActionResult(effect={}, route={}, delivery={}, evidence={}, window_change={}, escalation={})".format(self.effect, self.route, self.delivery, self.evidence, self.window_change, self.escalation) def __eq__(self, other): if self.effect != other.effect: return False @@ -1367,6 +1624,8 @@ def __eq__(self, other): return False if self.evidence != other.evidence: return False + if self.window_change != other.window_change: + return False if self.escalation != other.escalation: return False return True @@ -1379,6 +1638,7 @@ def read(buf): route=_UniffiFfiConverterTypeActionRoute.read(buf), delivery=_UniffiFfiConverterOptionalTypeActionDelivery.read(buf), evidence=_UniffiFfiConverterOptionalSequenceTypeActionEvidence.read(buf), + window_change=_UniffiFfiConverterOptionalTypeActionWindowChange.read(buf), escalation=_UniffiFfiConverterOptionalTypeActionEscalation.read(buf), ) @@ -1388,6 +1648,7 @@ def check_lower(value): _UniffiFfiConverterTypeActionRoute.check_lower(value.route) _UniffiFfiConverterOptionalTypeActionDelivery.check_lower(value.delivery) _UniffiFfiConverterOptionalSequenceTypeActionEvidence.check_lower(value.evidence) + _UniffiFfiConverterOptionalTypeActionWindowChange.check_lower(value.window_change) _UniffiFfiConverterOptionalTypeActionEscalation.check_lower(value.escalation) @staticmethod @@ -1396,6 +1657,7 @@ def write(value, buf): _UniffiFfiConverterTypeActionRoute.write(value.route, buf) _UniffiFfiConverterOptionalTypeActionDelivery.write(value.delivery, buf) _UniffiFfiConverterOptionalSequenceTypeActionEvidence.write(value.evidence, buf) + _UniffiFfiConverterOptionalTypeActionWindowChange.write(value.window_change, buf) _UniffiFfiConverterOptionalTypeActionEscalation.write(value.escalation, buf) class _UniffiFfiConverterFloat64(_UniffiConverterPrimitiveFloat): @@ -1486,51 +1748,6 @@ def write(value, buf): _UniffiFfiConverterFloat64.write(value.height, buf) _UniffiFfiConverterOptionalFloat64.write(value.tolerance_px, buf) -class _UniffiFfiConverterUInt64(_UniffiConverterPrimitiveInt): - CLASS_NAME = "u64" - VALUE_MIN = 0 - VALUE_MAX = 2**64 - - @staticmethod - def read(buf): - return buf.read_u64() - - @staticmethod - def write(value, buf): - buf.write_u64(value) - -class _UniffiFfiConverterString: - @staticmethod - def check_lower(value): - if not isinstance(value, str): - raise TypeError("argument must be str, not {}".format(type(value).__name__)) - return value - - @staticmethod - def read(buf): - size = buf.read_i32() - if size < 0: - raise InternalError("Unexpected negative string length") - utf8_bytes = buf.read(size) - return utf8_bytes.decode("utf-8") - - @staticmethod - def write(value, buf): - utf8_bytes = value.encode("utf-8") - buf.write_i32(len(utf8_bytes)) - buf.write(utf8_bytes) - - @staticmethod - def lift(buf): - with buf.consume_with_stream() as stream: - return stream.read(stream.remaining()).decode("utf-8") - - @staticmethod - def lower(value): - with _UniffiRustBuffer.alloc_with_builder() as builder: - builder.write(value.encode("utf-8")) - return builder.finalize() - @@ -1908,27 +2125,6 @@ def write(value, buf): _UniffiFfiConverterOptionalTypeClickButton.write(value.button, buf) _UniffiFfiConverterOptionalUInt32.write(value.count, buf) -class _UniffiFfiConverterBoolean: - @classmethod - def check_lower(cls, value): - return not not value - - @classmethod - def lower(cls, value): - return 1 if value else 0 - - @staticmethod - def lift(value): - return value != 0 - - @classmethod - def read(cls, buf): - return cls.lift(buf.read_u8()) - - @classmethod - def write(cls, value, buf): - buf.write_u8(value) - @dataclass class ClipboardReadInput: def __init__(self, *, include_text:bool, session:typing.Optional[str]): @@ -5060,19 +5256,6 @@ def write(value, buf): _UniffiFfiConverterOptionalTypeDesktopScope.write(value.scope, buf) _UniffiFfiConverterOptionalString.write(value.session, buf) -class _UniffiFfiConverterInt64(_UniffiConverterPrimitiveInt): - CLASS_NAME = "i64" - VALUE_MIN = -2**63 - VALUE_MAX = 2**63 - - @staticmethod - def read(buf): - return buf.read_i64() - - @staticmethod - def write(value, buf): - buf.write_i64(value) - class _UniffiFfiConverterSequenceTypeStatePredicate(_UniffiConverterRustBuffer): @classmethod def check_lower(cls, value): @@ -5325,8 +5508,10 @@ def write(value, buf): "EffectiveScope", "Platform", "ActionDelivery", + "ActionWindowTarget", "ActionEscalation", "ActionEvidence", + "ActionWindowChange", "ActionResult", "BoundsExpectation", "ClickInput", diff --git a/libs/cua-driver/rust/Skills/cua-driver/SKILL.md b/libs/cua-driver/rust/Skills/cua-driver/SKILL.md index 2ad9f79edf..0aa9ccd82c 100644 --- a/libs/cua-driver/rust/Skills/cua-driver/SKILL.md +++ b/libs/cua-driver/rust/Skills/cua-driver/SKILL.md @@ -354,11 +354,11 @@ to pixel clicks and desktop actions too. ### Read action facts without confusing them with task success A successful action returns `effect` and `route`, with optional typed -`delivery`, `evidence`, and `escalation`. These fields describe the actuator; -they do not declare the user's task complete. +`delivery`, `evidence`, `window_change`, and `escalation`. Action effect +accounts for the actuator; topology does not declare the action or task +successful. -- `confirmed` means the driver has publishable value readback or window-change - evidence for that action. +- `confirmed` means the driver has publishable value readback for that action. - `partial` means only `delivery.delivered_count` was delivered. - `unverifiable` means the driver cannot prove the effect. - `suspected_noop` means available evidence suggests no useful change. @@ -375,11 +375,14 @@ An optional escalation is a harness instruction, never an automatic retry: stack admits the tool and exact target; - `page`: bind the native window to a supported browser page route; - `session`: a legacy compatibility signal from an older capture-scope daemon; - current callers choose a desktop target on the specific action instead. + current callers choose a desktop target on the specific action instead; +- `rebind`: refresh the exact `escalation.window` target without activating it. + If the target is absent, correlate `window_change.new_windows` with one fresh + `list_windows` call before choosing a window. Branch on the closed reason vocabulary: `route_unavailable`, `delivery_failed`, `effect_unconfirmed`, -`suspected_noop`, and `permission_required`. +`suspected_noop`, `permission_required`, and `surface_changed`. After any action, keep using `verify_state` or a fresh state snapshot for the actual task postcondition. The multimodal harness owns visual reading and the diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs index a5a72e84dc..58dfd250bd 100644 --- a/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs +++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs @@ -39,12 +39,12 @@ pub use inputs::{ pub use outputs::{ advertised_output_schema, refusal_envelope_schema, ActionDelivery, ActionDeliveryMode, ActionEffect, ActionEscalation, ActionEscalationReason, ActionEscalationTarget, ActionEvidence, - ActionEvidenceKind, ActionResult, ActionResultValidationError, ActionRoute, - ClipboardReadOutput, ClipboardWriteOutput, CursorMotionOutput, CursorPointOutput, - CursorPositionOutput, CursorThemeOutput, CursorVisualOutput, DesktopStateOutput, - EffectiveScope, EndSessionOutput, GetAgentCursorStateOutput, ListSessionsOutput, - ScreenSizeOutput, SessionClientKindOutput, SessionLifecycleState, SessionOutput, - SessionStateOutput, SessionTransportOutput, SetAgentCursorEnabledOutput, + ActionEvidenceKind, ActionResult, ActionResultValidationError, ActionRoute, ActionWindowChange, + ActionWindowTarget, ClipboardReadOutput, ClipboardWriteOutput, CursorMotionOutput, + CursorPointOutput, CursorPositionOutput, CursorThemeOutput, CursorVisualOutput, + DesktopStateOutput, EffectiveScope, EndSessionOutput, GetAgentCursorStateOutput, + ListSessionsOutput, ScreenSizeOutput, SessionClientKindOutput, SessionLifecycleState, + SessionOutput, SessionStateOutput, SessionTransportOutput, SetAgentCursorEnabledOutput, SetAgentCursorMotionOutput, SetAgentCursorThemeOutput, StartSessionOutput, ToolOutput, }; pub use verification::{ @@ -96,6 +96,14 @@ pub fn is_action_result_tool(name: &str) -> bool { ACTION_RESULT_TOOLS.contains(&name) } +/// Whether an action uses the native desktop interaction lane. +/// +/// Browser actions own their scheduling inside the browser engine; every other +/// action-result tool runs against native desktop state. +pub fn is_desktop_action_result_tool(name: &str) -> bool { + is_action_result_tool(name) && !name.starts_with("browser_") +} + #[derive( Debug, Clone, diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs index 849ed0b0b6..a00fd5baf6 100644 --- a/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs +++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs @@ -441,7 +441,6 @@ pub struct ActionDelivery { #[serde(rename_all = "snake_case")] pub enum ActionEvidenceKind { ValueReadback, - WindowChange, } #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Record)] @@ -450,6 +449,24 @@ pub struct ActionEvidence { pub kind: ActionEvidenceKind, } +/// A target-owned native interaction root discovered after an action. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Record)] +#[serde(deny_unknown_fields)] +pub struct ActionWindowTarget { + pub pid: i64, + pub window_id: u64, + pub app_name: String, + pub title: String, +} + +/// Read-only topology observed for the action's target process. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Record)] +#[serde(deny_unknown_fields)] +pub struct ActionWindowChange { + pub new_windows: Vec, + pub foreground_changed: bool, +} + #[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Enum)] #[serde(rename_all = "snake_case")] pub enum ActionEscalationTarget { @@ -457,6 +474,7 @@ pub enum ActionEscalationTarget { Foreground, Page, Session, + Rebind, } #[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Enum)] @@ -467,6 +485,7 @@ pub enum ActionEscalationReason { EffectUnconfirmed, SuspectedNoop, PermissionRequired, + SurfaceChanged, } #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Record)] @@ -474,6 +493,8 @@ pub enum ActionEscalationReason { pub struct ActionEscalation { pub target: ActionEscalationTarget, pub reason: ActionEscalationReason, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub window: Option, } #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Record)] @@ -486,6 +507,8 @@ pub struct ActionResult { #[serde(default, skip_serializing_if = "Option::is_none")] pub evidence: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] + pub window_change: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub escalation: Option, } @@ -495,6 +518,8 @@ pub enum ActionResultValidationError { PartialRequiresDeliveredCount, RefusedCannotHaveDelivery, RefusedCannotHaveEvidence, + RebindRequiresWindowChange, + RebindWindowMustBeCandidate, } impl std::fmt::Display for ActionResultValidationError { @@ -504,6 +529,8 @@ impl std::fmt::Display for ActionResultValidationError { Self::PartialRequiresDeliveredCount => "partial effect requires delivered_count", Self::RefusedCannotHaveDelivery => "refused effect cannot include delivery", Self::RefusedCannotHaveEvidence => "refused effect cannot include evidence", + Self::RebindRequiresWindowChange => "rebind escalation requires a window change", + Self::RebindWindowMustBeCandidate => "rebind window must be an observed candidate", }) } } @@ -514,10 +541,11 @@ impl ActionResult { pub fn validate_invariants(&self) -> Result<(), ActionResultValidationError> { match self.effect { ActionEffect::Confirmed - if self - .evidence - .as_ref() - .is_none_or(|evidence| evidence.is_empty()) => + if self.evidence.as_ref().is_none_or(|evidence| { + !evidence + .iter() + .any(|item| item.kind == ActionEvidenceKind::ValueReadback) + }) => { Err(ActionResultValidationError::ConfirmedRequiresEvidence) } @@ -536,6 +564,29 @@ impl ActionResult { ActionEffect::Refused if self.evidence.is_some() => { Err(ActionResultValidationError::RefusedCannotHaveEvidence) } + _ if self + .escalation + .as_ref() + .is_some_and(|escalation| escalation.target == ActionEscalationTarget::Rebind) + && self.window_change.is_none() => + { + Err(ActionResultValidationError::RebindRequiresWindowChange) + } + _ if self + .escalation + .as_ref() + .and_then(|escalation| escalation.window.as_ref()) + .is_some_and(|window| { + self.window_change.as_ref().is_none_or(|change| { + !change + .new_windows + .iter() + .any(|candidate| candidate == window) + }) + }) => + { + Err(ActionResultValidationError::RebindWindowMustBeCandidate) + } _ => Ok(()), } } @@ -625,6 +676,7 @@ mod tests { evidence: Some(vec![ActionEvidence { kind: ActionEvidenceKind::ValueReadback, }]), + window_change: None, escalation: None, } } @@ -638,7 +690,14 @@ mod tests { let properties = schema["properties"].as_object().expect("properties"); assert_eq!( properties.keys().map(String::as_str).collect::>(), - ["delivery", "effect", "escalation", "evidence", "route"] + [ + "delivery", + "effect", + "escalation", + "evidence", + "route", + "window_change" + ] ); assert_eq!( properties["effect"]["enum"], @@ -686,7 +745,7 @@ mod tests { assert_eq!(evidence["required"], json!(["kind"])); assert_eq!( evidence["properties"]["kind"]["enum"], - json!(["value_readback", "window_change"]) + json!(["value_readback"]) ); let escalation = object_variant(&properties["escalation"]); @@ -694,7 +753,7 @@ mod tests { assert_eq!(escalation["required"], json!(["target", "reason"])); assert_eq!( escalation["properties"]["target"]["enum"], - json!(["pixel", "foreground", "page", "session"]) + json!(["pixel", "foreground", "page", "session", "rebind"]) ); assert_eq!( escalation["properties"]["reason"]["enum"], @@ -703,9 +762,24 @@ mod tests { "delivery_failed", "effect_unconfirmed", "suspected_noop", - "permission_required" + "permission_required", + "surface_changed" ]) ); + assert!(escalation["properties"].get("window").is_some()); + + let window_change = object_variant(&properties["window_change"]); + assert_eq!(window_change["additionalProperties"], false); + assert_eq!( + window_change["required"], + json!(["new_windows", "foreground_changed"]) + ); + let target = &window_change["properties"]["new_windows"]["items"]; + assert_eq!(target["additionalProperties"], false); + assert_eq!( + target["required"], + json!(["pid", "window_id", "app_name", "title"]) + ); } #[test] @@ -800,7 +874,7 @@ mod tests { ); result.delivery = None; result.evidence = Some(vec![ActionEvidence { - kind: ActionEvidenceKind::WindowChange, + kind: ActionEvidenceKind::ValueReadback, }]); assert_eq!( result.validate_invariants(), @@ -809,4 +883,38 @@ mod tests { result.evidence = None; assert_eq!(result.validate_invariants(), Ok(())); } + + #[test] + fn topology_cannot_confirm_effect_or_invent_an_exact_rebind() { + let target = ActionWindowTarget { + pid: 42, + window_id: 7, + app_name: "Editor".into(), + title: "Open".into(), + }; + let mut result = confirmed_result(); + result.evidence = None; + result.window_change = Some(ActionWindowChange { + new_windows: vec![target.clone()], + foreground_changed: false, + }); + assert_eq!( + result.validate_invariants(), + Err(ActionResultValidationError::ConfirmedRequiresEvidence) + ); + + result.effect = ActionEffect::Unverifiable; + result.escalation = Some(ActionEscalation { + target: ActionEscalationTarget::Rebind, + reason: ActionEscalationReason::SurfaceChanged, + window: Some(ActionWindowTarget { + window_id: 8, + ..target + }), + }); + assert_eq!( + result.validate_invariants(), + Err(ActionResultValidationError::RebindWindowMustBeCandidate) + ); + } } diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/action_record.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/action_record.rs index 4b21971a6a..5feaff6371 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/action_record.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/action_record.rs @@ -187,13 +187,77 @@ pub enum EvidenceKind { AccessibilityReadback, BrowserReadback, ValueReadback, - WindowChange, NativeApiResult, ScreenshotComparison, EventReceipt, OperatorObservation, } +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ActionSurfaceTarget { + pub pid: i64, + pub window_id: u64, + pub app_name: String, + pub title: String, + pub modal: bool, +} + +/// A target-owned candidate passed from a platform observer to shared policy. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ActionSurfaceCandidate { + pub target: ActionSurfaceTarget, + pub focused: bool, +} + +/// Typed platform observation with an optional validated exact rebind. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ActionSurfaceDelta { + pub new_windows: Vec, + pub foreground_changed: bool, + pub rebind: Option, +} + +/// Apply the same fail-closed exact-target rule to every platform observer. +/// Ownership must already be proven by target-scoped native enumeration. +pub fn resolve_surface_delta( + candidates: Vec, + foreground_changed: bool, +) -> Option { + let mut unique: Vec = Vec::new(); + for candidate in candidates { + if let Some(index) = unique.iter().position(|existing| { + existing.target.pid == candidate.target.pid + && existing.target.window_id == candidate.target.window_id + }) { + if (candidate.target.modal && !unique[index].target.modal) + || (candidate.focused && !unique[index].focused) + { + unique[index] = candidate; + } + } else { + unique.push(candidate); + } + } + let new_windows: Vec = unique + .iter() + .map(|candidate| candidate.target.clone()) + .collect(); + if new_windows.is_empty() { + return None; + } + let eligible: Vec = unique + .iter() + .filter(|candidate| candidate.focused || candidate.target.modal) + .map(|candidate| candidate.target.clone()) + .collect(); + let rebind = (eligible.len() == 1).then(|| eligible[0].clone()); + Some(ActionSurfaceDelta { + new_windows, + foreground_changed, + rebind, + }) +} + /// A failed or superseded transport attempt. #[derive(Clone, Debug, Eq, PartialEq)] pub struct ActionAttempt { @@ -241,6 +305,7 @@ pub struct ActionExecutionRecord { pub fallbacks: Vec, pub evidence: Vec, pub escalation: Option, + pub surface_delta: Option, pub delivered_count: Option, pub detail: Option, } @@ -260,6 +325,7 @@ impl ActionExecutionRecord { fallbacks: Vec::new(), evidence: Vec::new(), escalation: None, + surface_delta: None, delivered_count: None, detail: None, } @@ -275,7 +341,16 @@ impl ActionExecutionRecord { pub fn validate(&self) -> Result<(), ActionRecordValidationError> { match self.effect { - ActionEffect::Confirmed if projected_evidence(&self.evidence).is_none() => { + ActionEffect::Confirmed + if !self.evidence.iter().any(|evidence| { + matches!( + evidence.kind, + EvidenceKind::AccessibilityReadback + | EvidenceKind::BrowserReadback + | EvidenceKind::ValueReadback + ) + }) => + { Err(ActionRecordValidationError::ConfirmedRequiresEvidence) } ActionEffect::Partial @@ -293,6 +368,12 @@ impl ActionExecutionRecord { } } + /// Attach observer facts without changing the actuator's effect or + /// overwriting its internal escalation account. + pub fn observe_surface_delta(&mut self, delta: ActionSurfaceDelta) { + self.surface_delta = Some(delta); + } + pub fn stable_projection( &self, ) -> Result { @@ -306,6 +387,7 @@ impl ActionExecutionRecord { }), evidence: projected_evidence(&self.evidence), escalation: self.escalation.clone(), + surface_delta: self.surface_delta.clone(), }) } @@ -360,60 +442,78 @@ impl ActionExecutionRecord { | ProjectedEvidenceKind::ValueReadback => { cua_driver_contract::ActionEvidenceKind::ValueReadback } - ProjectedEvidenceKind::WindowChange => { - cua_driver_contract::ActionEvidenceKind::WindowChange - } }, }) .collect() }), - escalation: projection.escalation.map(|escalation| { - use cua_driver_contract::{ - ActionEscalation, ActionEscalationReason, ActionEscalationTarget, - }; - - let (target, reason) = match escalation.kind { - EscalationKind::ActivateTarget - | EscalationKind::RetryWithForegroundDelivery => ( - ActionEscalationTarget::Foreground, - ActionEscalationReason::DeliveryFailed, - ), - EscalationKind::RetryWithPixelTarget => ( - ActionEscalationTarget::Pixel, - ActionEscalationReason::EffectUnconfirmed, - ), - EscalationKind::RetryWithPageAction => ( - ActionEscalationTarget::Page, - ActionEscalationReason::EffectUnconfirmed, - ), - EscalationKind::RefreshPageState => ( - ActionEscalationTarget::Page, - ActionEscalationReason::RouteUnavailable, - ), - EscalationKind::RequestPermission | EscalationKind::ElevateAccess => ( - ActionEscalationTarget::Session, - ActionEscalationReason::PermissionRequired, - ), - EscalationKind::ExpandCaptureScope => ( - ActionEscalationTarget::Session, - ActionEscalationReason::RouteUnavailable, - ), - EscalationKind::PrepareSession => ( - ActionEscalationTarget::Session, - ActionEscalationReason::RouteUnavailable, - ), - }; - ActionEscalation { - target, - reason: if projection.effect == ActionEffect::SuspectedNoop - && reason != ActionEscalationReason::PermissionRequired - { - ActionEscalationReason::SuspectedNoop - } else { - reason - }, + window_change: projection.surface_delta.as_ref().map(|delta| { + cua_driver_contract::ActionWindowChange { + new_windows: delta + .new_windows + .iter() + .map(public_surface_target) + .collect(), + foreground_changed: delta.foreground_changed, } }), + escalation: projection + .surface_delta + .as_ref() + .map(|delta| cua_driver_contract::ActionEscalation { + target: cua_driver_contract::ActionEscalationTarget::Rebind, + reason: cua_driver_contract::ActionEscalationReason::SurfaceChanged, + window: delta.rebind.as_ref().map(public_surface_target), + }) + .or_else(|| { + projection.escalation.map(|escalation| { + use cua_driver_contract::{ + ActionEscalation, ActionEscalationReason, ActionEscalationTarget, + }; + + let (target, reason) = match escalation.kind { + EscalationKind::ActivateTarget + | EscalationKind::RetryWithForegroundDelivery => ( + ActionEscalationTarget::Foreground, + ActionEscalationReason::DeliveryFailed, + ), + EscalationKind::RetryWithPixelTarget => ( + ActionEscalationTarget::Pixel, + ActionEscalationReason::EffectUnconfirmed, + ), + EscalationKind::RetryWithPageAction => ( + ActionEscalationTarget::Page, + ActionEscalationReason::EffectUnconfirmed, + ), + EscalationKind::RefreshPageState => ( + ActionEscalationTarget::Page, + ActionEscalationReason::RouteUnavailable, + ), + EscalationKind::RequestPermission | EscalationKind::ElevateAccess => ( + ActionEscalationTarget::Session, + ActionEscalationReason::PermissionRequired, + ), + EscalationKind::ExpandCaptureScope => ( + ActionEscalationTarget::Session, + ActionEscalationReason::RouteUnavailable, + ), + EscalationKind::PrepareSession => ( + ActionEscalationTarget::Session, + ActionEscalationReason::RouteUnavailable, + ), + }; + ActionEscalation { + target, + reason: if projection.effect == ActionEffect::SuspectedNoop + && reason != ActionEscalationReason::PermissionRequired + { + ActionEscalationReason::SuspectedNoop + } else { + reason + }, + window: None, + } + }) + }), }) } @@ -536,11 +636,25 @@ impl ActionExecutionRecord { "kind": escalation_kind_name(escalation.kind), "detail": escalation.detail, })), + "surface_delta": self.surface_delta.as_ref().map(|delta| serde_json::json!({ + "foreground_changed": delta.foreground_changed, + "new_window_ids": delta.new_windows.iter().map(|window| window.window_id).collect::>(), + "rebind_window_id": delta.rebind.as_ref().map(|window| window.window_id), + })), "detail": self.detail, }) } } +fn public_surface_target(target: &ActionSurfaceTarget) -> cua_driver_contract::ActionWindowTarget { + cua_driver_contract::ActionWindowTarget { + pid: target.pid, + window_id: target.window_id, + app_name: target.app_name.clone(), + title: target.title.clone(), + } +} + pub fn is_action_tool(tool_name: &str) -> bool { cua_driver_contract::is_action_result_tool(tool_name) } @@ -916,7 +1030,6 @@ fn projected_evidence(evidence: &[ActionEvidence]) -> Option ProjectedEvidenceKind::AccessibilityReadback, EvidenceKind::BrowserReadback => ProjectedEvidenceKind::BrowserReadback, EvidenceKind::ValueReadback => ProjectedEvidenceKind::ValueReadback, - EvidenceKind::WindowChange => ProjectedEvidenceKind::WindowChange, EvidenceKind::NativeApiResult | EvidenceKind::ScreenshotComparison | EvidenceKind::EventReceipt @@ -974,7 +1087,6 @@ fn evidence_kind_name(kind: EvidenceKind) -> &'static str { EvidenceKind::AccessibilityReadback => "accessibility_readback", EvidenceKind::BrowserReadback => "browser_readback", EvidenceKind::ValueReadback => "value_readback", - EvidenceKind::WindowChange => "window_change", EvidenceKind::NativeApiResult => "native_api_result", EvidenceKind::ScreenshotComparison => "screenshot_comparison", EvidenceKind::EventReceipt => "event_receipt", @@ -1099,6 +1211,7 @@ pub struct ActionOutcomeProjection { pub delivery: Option, pub evidence: Option>, pub escalation: Option, + pub surface_delta: Option, } /// Published delivery accounting; the original request is intentionally absent. @@ -1120,7 +1233,6 @@ pub enum ProjectedEvidenceKind { AccessibilityReadback, BrowserReadback, ValueReadback, - WindowChange, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -1212,6 +1324,84 @@ mod tests { .is_ok()); } + #[test] + fn surface_observation_preserves_actuator_effect() { + let mut record = ActionExecutionRecord::new( + ActionEffect::Unverifiable, + ActionTransport::MacosAxAction, + RequestedDelivery::Background, + ); + let target = ActionSurfaceTarget { + pid: 42, + window_id: 7, + app_name: "Editor".into(), + title: "Open".into(), + modal: true, + }; + let parent = ActionSurfaceTarget { + modal: false, + ..target.clone() + }; + record.observe_surface_delta( + resolve_surface_delta( + vec![ + ActionSurfaceCandidate { + target: parent, + focused: false, + }, + ActionSurfaceCandidate { + target: target.clone(), + focused: false, + }, + ], + false, + ) + .expect("surface delta"), + ); + + assert_eq!(record.effect, ActionEffect::Unverifiable); + let public = record.public_result().expect("public result"); + assert_eq!( + public.effect, + cua_driver_contract::ActionEffect::Unverifiable + ); + assert_eq!( + public + .window_change + .as_ref() + .map(|change| change.new_windows.len()), + Some(1), + "several AX roots for one native window are one rebind address" + ); + assert_eq!( + public.escalation.and_then(|escalation| escalation.window), + public + .window_change + .and_then(|change| change.new_windows.into_iter().next()) + ); + + let second = ActionSurfaceTarget { + window_id: 8, + ..target.clone() + }; + assert!(resolve_surface_delta( + vec![ + ActionSurfaceCandidate { + target, + focused: false, + }, + ActionSurfaceCandidate { + target: second, + focused: true, + }, + ], + false, + ) + .expect("ambiguous surface delta") + .rebind + .is_none()); + } + #[test] fn partial_requires_delivered_count() { assert_eq!( @@ -1364,7 +1554,11 @@ mod tests { .unwrap(); assert_eq!( result.escalation, - Some(cua_driver_contract::ActionEscalation { target, reason }) + Some(cua_driver_contract::ActionEscalation { + target, + reason, + window: None, + }) ); } @@ -1648,6 +1842,7 @@ mod tests { Some(cua_driver_contract::ActionEscalation { target: cua_driver_contract::ActionEscalationTarget::Page, reason: cua_driver_contract::ActionEscalationReason::RouteUnavailable, + window: None, }) ); diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/history.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/history.rs index a4e686afb8..3153f60ac6 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/history.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/history.rs @@ -2182,7 +2182,6 @@ fn evidence_name(value: ProjectedEvidenceKind) -> &'static str { ProjectedEvidenceKind::AccessibilityReadback => "accessibility_readback", ProjectedEvidenceKind::BrowserReadback => "browser_readback", ProjectedEvidenceKind::ValueReadback => "value_readback", - ProjectedEvidenceKind::WindowChange => "window_change", } } diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs index 4638b9c264..e984b3ceaf 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs @@ -66,6 +66,7 @@ pub mod ffmpeg_install; pub mod health_report; pub mod history; pub mod image_utils; +pub(crate) mod native_action_scheduler; pub mod page; pub mod pip_hook; pub mod policy; diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/native_action_scheduler.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/native_action_scheduler.rs new file mode 100644 index 0000000000..da06c13dbd --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/native_action_scheduler.rs @@ -0,0 +1,125 @@ +//! Keyed scheduling for native action resources. +//! +//! The scheduler owns one bounded lease lifecycle for physical desktop input +//! and fail-fast process-scoped text admission. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock, Weak}; + +use tokio::sync::{Mutex as AsyncMutex, OwnedMutexGuard}; + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub(crate) enum NativeActionResource { + PhysicalDesktop, + TextInputProcess(i64), +} + +pub(crate) struct NativeActionScheduler { + gates: Mutex>>>, +} + +impl Default for NativeActionScheduler { + fn default() -> Self { + Self { + gates: Mutex::new(HashMap::new()), + } + } +} + +impl NativeActionScheduler { + fn new() -> Self { + Self::default() + } + + fn gate(&self, resource: NativeActionResource) -> Arc> { + let mut gates = self + .gates + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if let Some(gate) = gates.get(&resource).and_then(Weak::upgrade) { + return gate; + } + + // Weak entries do not keep completed process resources alive. Prune + // them when admitting a new resource so transient pids stay bounded. + gates.retain(|_, gate| gate.strong_count() > 0); + let gate = Arc::new(AsyncMutex::new(())); + gates.insert(resource, Arc::downgrade(&gate)); + gate + } + + pub(crate) async fn lock(&self, resource: NativeActionResource) -> OwnedMutexGuard<()> { + self.gate(resource).lock_owned().await + } + + pub(crate) fn try_lock(&self, resource: NativeActionResource) -> Option> { + self.gate(resource).try_lock_owned().ok() + } + + #[cfg(test)] + fn retained_gate_count(&self) -> usize { + self.gates + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .len() + } +} + +pub(crate) fn global() -> &'static NativeActionScheduler { + static SCHEDULER: OnceLock = OnceLock::new(); + SCHEDULER.get_or_init(NativeActionScheduler::new) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + fn text(pid: i64) -> NativeActionResource { + NativeActionResource::TextInputProcess(pid) + } + + #[tokio::test] + async fn same_resource_serializes_while_independent_resources_proceed() { + let scheduler = NativeActionScheduler::new(); + let first = scheduler.lock(text(7)).await; + + assert!(scheduler.try_lock(text(7)).is_none()); + tokio::time::timeout(Duration::from_millis(20), scheduler.lock(text(8))) + .await + .expect("independent resource should not block"); + + drop(first); + tokio::time::timeout(Duration::from_millis(20), scheduler.lock(text(7))) + .await + .expect("completed resource should release its lane"); + } + + #[tokio::test] + async fn cancelled_waiter_does_not_keep_or_poison_a_resource() { + let scheduler = NativeActionScheduler::new(); + let first = scheduler.lock(text(7)).await; + + assert!( + tokio::time::timeout(Duration::from_millis(20), scheduler.lock(text(7))) + .await + .is_err() + ); + drop(first); + + tokio::time::timeout(Duration::from_millis(20), scheduler.lock(text(7))) + .await + .expect("cancelling a waiter must leave the resource usable"); + } + + #[tokio::test] + async fn completed_process_gates_are_pruned() { + let scheduler = NativeActionScheduler::new(); + for pid in 1..65 { + drop(scheduler.lock(text(pid)).await); + } + + let _live = scheduler.lock(text(65)).await; + assert_eq!(scheduler.retained_gate_count(), 1); + } +} diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/protocol.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/protocol.rs index e572f48542..e36d57fb91 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/protocol.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/protocol.rs @@ -294,6 +294,9 @@ pub struct ToolResult { /// the MCP result envelope or its legacy structured payload. #[serde(skip)] pub action_record: Option, + /// Typed platform observation consumed by the shared dispatch seam. + #[serde(skip)] + pub surface_delta: Option, } impl ToolResult { diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs index c9d82805f0..9ad2c9f4c2 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs @@ -1,8 +1,8 @@ //! Tool trait and registry. -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::path::{Component, Path, PathBuf}; -use std::sync::{Arc, Mutex, OnceLock}; +use std::sync::{Arc, Mutex}; use std::time::Duration; use async_trait::async_trait; @@ -14,6 +14,7 @@ thread_local! { } use crate::{ + native_action_scheduler::{global as native_action_scheduler, NativeActionResource}, pip_hook, protocol::{Content, ToolResult}, recording::{now_ms, screenshot_for, RecordingSession}, @@ -71,54 +72,57 @@ pub fn with_runtime_scope(scope: String, action: impl FnOnce() -> T) -> T { action() } -fn desktop_action_coordinator() -> &'static tokio::sync::Mutex<()> { - static COORDINATOR: OnceLock> = OnceLock::new(); - COORDINATOR.get_or_init(|| tokio::sync::Mutex::new(())) -} - -fn active_text_input_pids() -> &'static Mutex> { - static ACTIVE: OnceLock>> = OnceLock::new(); - ACTIVE.get_or_init(|| Mutex::new(HashSet::new())) -} - #[derive(Debug)] -struct TextInputAdmission { - pid: i64, +struct ActionLeasePlan { + serialize_desktop: bool, + text_input_pid: Option, } -impl Drop for TextInputAdmission { - fn drop(&mut self) { - active_text_input_pids() - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .remove(&self.pid); +impl ActionLeasePlan { + fn for_call(tool_name: &str, args: &Value) -> Self { + let pid = args + .get("pid") + .and_then(Value::as_i64) + .filter(|pid| *pid > 0); + Self { + serialize_desktop: cua_driver_contract::is_desktop_action_result_tool(tool_name) + || tool_name == "bring_to_front", + text_input_pid: if matches!(tool_name, "type_text" | "type_text_chars") { + pid + } else { + None + }, + } } -} -fn try_admit_text_input( - tool_name: &str, - args: &Value, -) -> Result, ToolResult> { - if tool_name != "type_text" { - return Ok(None); - } - let Some(pid) = args - .get("pid") - .and_then(Value::as_i64) - .filter(|pid| *pid > 0) - else { - // Desktop-scoped input has no stable process identity. It continues to - // use the process-wide physical action coordinator below. - return Ok(None); - }; - let mut active = active_text_input_pids() - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - if !active.insert(pid) { - let message = format!("text input is already active for pid {pid}"); - return Err(protected_refusal("input_busy", &message)); - } - Ok(Some(TextInputAdmission { pid })) + fn try_admit_text(&self) -> Result>, ToolResult> { + let Some(pid) = self.text_input_pid else { + return Ok(None); + }; + native_action_scheduler() + .try_lock(NativeActionResource::TextInputProcess(pid)) + .map(Some) + .ok_or_else(|| { + protected_refusal( + "input_busy", + &format!("text input is already active for pid {pid}"), + ) + }) + } + + async fn acquire_desktop(&self) -> Option> { + if !self.serialize_desktop { + return None; + } + let scheduler = native_action_scheduler(); + let resource = NativeActionResource::PhysicalDesktop; + // Avoid a dispatch yield between fresh target proof and native input + // when the process-wide resource is uncontended. + match scheduler.try_lock(resource) { + Some(admission) => Some(admission), + None => Some(scheduler.lock(resource).await), + } + } } pub use cua_driver_contract::{CAPABILITY_VERSION, TOOLS_LIST_SCHEMA_VERSION}; @@ -1136,14 +1140,13 @@ impl ToolRegistry { return result; } - // A queued text mutation can become stale while another agent is - // typing into the same process. Refuse that overlap before consent, - // cursor, recording, or platform focus behavior can begin. The guard - // is process-global so independent registries cannot interleave text - // through separate platform workers, and RAII releases it on task - // cancellation as well as normal completion. - let _text_input_admission = match try_admit_text_input(resolved_name, &public_args) { - Ok(admission) => admission, + // Build one immutable lease plan from normalized public arguments. + // Fail-fast text admission happens before consent or platform behavior; + // the desktop lease starts immediately before evidence capture and is + // held through the complete decorated invocation. + let action_lease_plan = ActionLeasePlan::for_call(resolved_name, &public_args); + let _text_input_lease = match action_lease_plan.try_admit_text() { + Ok(admissions) => admissions, Err(mut refusal) => { restore_public_runtime_result(&mut refusal, &runtime_prefix); return refusal; @@ -1496,20 +1499,7 @@ impl ToolRegistry { "start_recording" | "stop_recording" | "get_recording_state" | "replay_trajectory" ); let private_consent_turn = is_existing_profile_prepare(resolved_name, &args); - let _desktop_action = if is_physical_desktop_action(resolved_name) { - let coordinator = desktop_action_coordinator(); - // Avoid yielding the dispatch task when the process-wide input - // lane is uncontended. On Windows, that yield creates a window in - // which the foreground target can lose keyboard eligibility - // between the fixture's focus proof and SendInput. Contended - // runtimes still wait and serialize through the same mutex. - Some(match coordinator.try_lock() { - Ok(guard) => guard, - Err(_) => coordinator.lock().await, - }) - } else { - None - }; + let _desktop_action_lease = action_lease_plan.acquire_desktop().await; let pending_turn = should_record .then(|| { if private_consent_turn { @@ -1525,9 +1515,8 @@ impl ToolRegistry { let mut result = tool.invoke(args.clone()).await; drop(lifecycle_dispatch); // The platform worker has exited, so another text operation for this - // pid may now start even while result projection and evidence capture - // finish for the completed call. - drop(_text_input_admission); + // pid may start while result projection finishes. + drop(_text_input_lease); if result.action_record.is_none() { if let Some(structured) = result.structured_content.as_ref() { result.action_record = crate::action_record::ActionExecutionRecord::from_legacy( @@ -1546,6 +1535,11 @@ impl ToolRegistry { ); } } + if let Some(record) = result.action_record.as_mut() { + if let Some(delta) = result.surface_delta.take() { + record.observe_surface_delta(delta); + } + } if resolved_name == "launch_app" && result.is_error != Some(true) { if let (Some(before), Some(pid)) = ( launch_snapshot.as_ref(), @@ -1568,7 +1562,7 @@ impl ToolRegistry { // capture or result shaping. Keeping the global desktop lock through // recording/PiP screenshots would unnecessarily block an unrelated // runtime after the input side effect has already completed. - drop(_desktop_action); + drop(_desktop_action_lease); restore_public_runtime_result(&mut result, &runtime_prefix); // Preserve the producer's private summary for recording/replay before // the public ActionResult projection deliberately replaces legacy @@ -2539,28 +2533,6 @@ fn canonical_proposed_path(raw: &str) -> Result { Ok(canonical.to_string_lossy().into_owned()) } -fn is_physical_desktop_action(tool: &str) -> bool { - matches!( - tool, - "click" - | "double_click" - | "right_click" - | "scroll" - | "drag" - | "mouse_drag" - | "parallel_mouse_drag" - | "move_cursor" - | "mouse_button_down" - | "mouse_button_up" - | "type_text" - | "press_key" - | "hotkey" - | "set_value" - | "bring_to_front" - | "set_window_frame" - ) -} - /// Bucket that owns the processes a call is allowed to terminate. /// /// A call that declares a session keys its launches to that session, so @@ -2755,9 +2727,9 @@ fn restore_public_runtime_value(value: &mut Value, runtime_prefix: &str) -> bool #[cfg(test)] mod runtime_isolation_tests { use super::{ - canonical_proposed_path, desktop_action_coordinator, namespace_runtime_args, - publish_action_result, restore_public_runtime_result, try_admit_text_input, - TrustedInvocationEvidence, DISPATCH_RUNTIME_SCOPE, + canonical_proposed_path, namespace_runtime_args, native_action_scheduler, + publish_action_result, restore_public_runtime_result, ActionLeasePlan, + NativeActionResource, TrustedInvocationEvidence, DISPATCH_RUNTIME_SCOPE, }; use crate::{ authorization::PermissionMode, @@ -4472,8 +4444,31 @@ resources: assert_eq!(owner.unwrap(), (44, 0)); } + #[test] + fn one_lease_plan_covers_native_actions_text_and_browser_exclusion() { + let click = ActionLeasePlan::for_call("click", &serde_json::json!({"pid": 42})); + assert!(click.serialize_desktop); + assert_eq!(click.text_input_pid, None); + + let menu = ActionLeasePlan::for_call("invoke_menu", &serde_json::json!({"pid": 42})); + assert!(menu.serialize_desktop); + assert_eq!(menu.text_input_pid, None); + + let text = ActionLeasePlan::for_call("type_text_chars", &serde_json::json!({"pid": 42})); + assert!(text.serialize_desktop); + assert_eq!(text.text_input_pid, Some(42)); + + let browser = ActionLeasePlan::for_call("browser_click", &serde_json::json!({})); + assert!(!browser.serialize_desktop); + assert_eq!(browser.text_input_pid, None); + + let read = ActionLeasePlan::for_call("get_window_state", &serde_json::json!({"pid": 42})); + assert!(!read.serialize_desktop); + assert_eq!(read.text_input_pid, None); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn physical_desktop_actions_are_admitted_one_at_a_time() { + async fn desktop_actions_are_admitted_one_at_a_time() { let active = Arc::new(AtomicUsize::new(0)); let max_active = Arc::new(AtomicUsize::new(0)); let mut tasks = Vec::new(); @@ -4481,7 +4476,9 @@ resources: let active = active.clone(); let max_active = max_active.clone(); tasks.push(tokio::spawn(async move { - let _admission = desktop_action_coordinator().lock().await; + let _admission = native_action_scheduler() + .lock(NativeActionResource::PhysicalDesktop) + .await; let now = active.fetch_add(1, Ordering::SeqCst) + 1; max_active.fetch_max(now, Ordering::SeqCst); tokio::task::yield_now().await; @@ -4497,11 +4494,13 @@ resources: #[test] fn overlapping_text_input_for_one_pid_fails_fast_and_releases_after_completion() { let pid = 8_675_410; - let first = try_admit_text_input("type_text", &serde_json::json!({"pid": pid})) - .expect("first text operation should be admitted") - .expect("pid-scoped text operation should receive a guard"); + let first_plan = ActionLeasePlan::for_call("type_text", &serde_json::json!({"pid": pid})); + let first = first_plan + .try_admit_text() + .expect("first text operation should be admitted"); - let refusal = try_admit_text_input("type_text", &serde_json::json!({"pid": pid})) + let refusal = first_plan + .try_admit_text() .expect_err("overlapping text operation should fail fast"); assert_eq!(refusal.is_error, Some(true)); assert_eq!( @@ -4513,17 +4512,18 @@ resources: Some("input_busy") ); - let other_pid = try_admit_text_input("type_text", &serde_json::json!({"pid": pid + 1})) - .expect("a different pid should have an independent input lane") - .expect("pid-scoped text operation should receive a guard"); + let other_plan = + ActionLeasePlan::for_call("type_text", &serde_json::json!({"pid": pid + 1})); + let other_pid = other_plan + .try_admit_text() + .expect("a different pid should have an independent input lane"); drop(other_pid); drop(first); - assert!( - try_admit_text_input("type_text", &serde_json::json!({"pid": pid})) - .expect("completed text operation should release its pid") - .is_some() - ); + assert!(first_plan + .try_admit_text() + .expect("completed text operation should release its pid") + .is_some()); } #[test] diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/installed_app_textedit_macos_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/installed_app_textedit_macos_test.rs index edf11916f6..4384497707 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/installed_app_textedit_macos_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/installed_app_textedit_macos_test.rs @@ -133,3 +133,148 @@ fn background_type_on_native_cocoa_is_ax_verified() { Observation::delivered(passed, Default::default()) }); } + +/// A background shortcut that opens TextEdit's AppKit file panel reports one +/// owner-verified rebind target while preserving the actuator's effect and the +/// user's foreground application. +#[test] +#[ignore] +fn background_open_panel_returns_a_typed_rebind() { + use cua_driver_testkit::e2e::{ + execute_case, recording_evidence, CaseSpec, Delivery, DriverRoute, Observation, OracleKind, + Scope, Targeting, + }; + use cua_driver_testkit::observer::TargetWindow; + use cua_driver_testkit::sentinel::run_with_background_oracles; + use cua_driver_testkit::{Driver, McpDriver}; + + let cell_id = "macos-textedit-open-panel-background-rebind"; + let case = CaseSpec::delivered( + cell_id, + "textedit", + "appkit", + "hotkey", + Targeting::Ax, + Delivery::Background, + Scope::Window, + DriverRoute::MacosCgEventPid, + vec![ + OracleKind::AxState, + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::Cursor, + OracleKind::NoLeakedInput, + ], + ); + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_macos_daemon_proxy_named(cell_id) + .expect("start installed macOS daemon proxy"); + *evidence = recording_evidence(driver.recording_dir()); + + let launch = driver.call( + "launch_app", + serde_json::json!({ "bundle_id": "com.apple.TextEdit" }), + ); + assert!( + !launch.is_error(), + "could not launch TextEdit: {}", + launch.text() + ); + let pid = launch.structured()["pid"].as_i64().expect("TextEdit pid"); + let window_id = launch.structured()["windows"] + .as_array() + .into_iter() + .flatten() + .filter_map(|window| window["window_id"].as_u64()) + .find(|window_id| { + let state = driver.call( + "get_window_state", + serde_json::json!({ + "pid": pid, + "window_id": window_id, + "include_screenshot": false + }), + ); + state.structured()["elements"] + .as_array() + .is_some_and(|elements| { + elements + .iter() + .any(|element| element["role"] == "AXTextArea") + }) + }) + .expect("TextEdit document window"); + + let (opened, mut passed) = run_with_background_oracles( + &mut driver, + TargetWindow { + pid: pid as u32, + native_id: window_id, + }, + |driver| { + driver.call( + "hotkey", + serde_json::json!({ + "pid": pid, + "window_id": window_id, + "keys": ["cmd", "o"], + "delivery_mode": "background" + }), + ) + }, + ) + .unwrap_or_else(|error| panic!("background Open-panel contract failed: {error}")); + assert!(!opened.is_error(), "hotkey errored: {}", opened.text()); + assert_eq!( + opened.action_effect(), + Some("unverifiable"), + "topology must not promote the hotkey effect: {}", + opened.text() + ); + + let change = &opened.structured()["window_change"]; + let candidates = change["new_windows"] + .as_array() + .expect("typed window_change candidates"); + assert_eq!(candidates.len(), 1, "expected one Open-panel root"); + let candidate = &candidates[0]; + let panel_pid = candidate["pid"].as_i64().expect("panel owner pid"); + let panel_window_id = candidate["window_id"].as_u64().expect("panel window id"); + assert_ne!( + panel_pid, pid, + "AppKit panel service ownership was not resolved" + ); + assert_eq!(opened.structured()["escalation"]["target"], "rebind"); + assert_eq!( + opened.structured()["escalation"]["window"], + *candidate, + "exact rebind must be one observed owner-verified candidate" + ); + + let panel = driver.call( + "get_window_state", + serde_json::json!({ + "pid": panel_pid, + "window_id": panel_window_id, + "include_screenshot": false + }), + ); + assert!( + !panel.is_error(), + "rebound panel was not addressable: {}", + panel.text() + ); + passed.push(OracleKind::AxState); + + let _ = driver.call( + "press_key", + serde_json::json!({ + "pid": panel_pid, + "window_id": panel_window_id, + "key": "escape", + "delivery_mode": "background" + }), + ); + Observation::delivered(passed, Default::default()) + }); +} diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs index e71a9c8f34..b9cde3a419 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs @@ -943,9 +943,8 @@ impl Tool for GetWindowStateTool { ToolResult { content, - is_error: None, structured_content: Some(structured), - action_record: None, + ..Default::default() } } Ok(Err(e)) => ToolResult::error(format!("Capture error: {e}")), @@ -6408,9 +6407,8 @@ impl Tool for GetDesktopStateTool { } ToolResult { content, - is_error: None, structured_content: Some(structured), - action_record: None, + ..Default::default() } } Ok(Err(e)) => ToolResult::error(format!("Capture error: {e}")), @@ -7239,7 +7237,7 @@ impl Tool for ZoomTool { "width": w, "height": h, "format": "jpeg", "mime_type": "image/jpeg" })), - action_record: None, + ..Default::default() } } Ok(Err(e)) => ToolResult::error(format!("Zoom failed: {e}")), diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs index f50f240905..7d8f976069 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs @@ -693,7 +693,16 @@ pub unsafe fn ax_get_window_id(element: AXUIElementRef) -> Option { /// /// `element` must be valid, and the caller must release every returned element. pub unsafe fn copy_ax_windows(element: AXUIElementRef) -> Vec { - let attr = CFStr::new("AXWindows"); + copy_element_array_attr(element, "AXWindows") +} + +/// Read an AX attribute containing an array of accessibility elements. +/// Every returned element is retained and must be released by the caller. +pub unsafe fn copy_element_array_attr( + element: AXUIElementRef, + attribute: &str, +) -> Vec { + let attr = CFStr::new(attribute); let mut value: CFTypeRef = std::ptr::null(); let err = AXUIElementCopyAttributeValue(element, attr.as_concrete_TypeRef(), &mut value); if err != kAXErrorSuccess || value.is_null() { diff --git a/libs/cua-driver/rust/crates/platform-macos/src/focus_guard.rs b/libs/cua-driver/rust/crates/platform-macos/src/focus_guard.rs index b94288ace6..468903facf 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/focus_guard.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/focus_guard.rs @@ -29,9 +29,9 @@ //! This Rust port only ships **layer 3** (the reactive suppressor). //! Layers 1+2 require AX assertion + AX attribute write/restore machinery //! that isn't yet ported — and empirically the layer-3 reactive guard -//! catches the majority of side-effects when combined with -//! `WindowChangeDetector`'s wildcard lease at the snapshot→detect -//! boundary. This gap is a known, intentional limitation. +//! catches the majority of side-effects when combined with the action +//! decorator's cross-application suppression lease. This gap is a known, +//! intentional limitation. //! //! ## Why this is a separate module //! @@ -56,12 +56,11 @@ use crate::focus_steal; /// Wrap an async closure `f` with a targeted focus-steal suppressor. /// /// - `target_pid` — the pid the action is dispatched to. `Some(pid)` is -/// the standard case; `None` skips the targeted entry entirely -/// (caller relies on the surrounding `WindowChangeDetector` wildcard -/// lease). +/// the standard case; `None` skips the targeted entry entirely because the +/// surrounding action decorator owns cross-application suppression. /// - `prior_frontmost` — the pid to restore focus to if the target /// activates. Typically captured from `apps::frontmost_pid()` before -/// the snapshot. +/// dispatch. /// - `origin` — short static label for tracing, e.g. `"click.AXPress"`. /// - `f` — the action to run with suppression armed. /// diff --git a/libs/cua-driver/rust/crates/platform-macos/src/lib.rs b/libs/cua-driver/rust/crates/platform-macos/src/lib.rs index c1a426c2a7..3049d1e624 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/lib.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/lib.rs @@ -31,6 +31,8 @@ pub mod permissions; #[cfg(target_os = "macos")] pub mod pip; #[cfg(target_os = "macos")] +mod post_action_observer; +#[cfg(target_os = "macos")] pub mod recording_hooks; #[cfg(target_os = "macos")] pub mod session; @@ -41,8 +43,6 @@ pub mod tools; #[cfg(target_os = "macos")] pub mod video_sckit; #[cfg(target_os = "macos")] -pub mod window_change_detector; -#[cfg(target_os = "macos")] pub mod windows; use cua_driver_core::tool::ToolRegistry; diff --git a/libs/cua-driver/rust/crates/platform-macos/src/post_action_observer.rs b/libs/cua-driver/rust/crates/platform-macos/src/post_action_observer.rs new file mode 100644 index 0000000000..3dfef3af00 --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-macos/src/post_action_observer.rs @@ -0,0 +1,411 @@ +//! Target-scoped post-action accessibility-root observation. +//! +//! The decorator in this module is the only action topology producer on +//! macOS. It snapshots the addressed process, invokes the actuator, then +//! attaches one typed delta to `ToolResult`. Native notifications and global +//! window lists are not used as causality evidence. + +use std::collections::{HashMap, HashSet}; +use std::time::{Duration, Instant}; + +use async_trait::async_trait; +use core_foundation::base::{CFRelease, CFTypeRef}; +use cua_driver_core::action_record::{ + resolve_surface_delta, ActionSurfaceCandidate, ActionSurfaceDelta, ActionSurfaceTarget, +}; +use cua_driver_core::protocol::ToolResult; +use cua_driver_core::tool::{ProtectedResourceOwnership, Tool, ToolDef}; +use serde_json::Value; + +use crate::ax::bindings::{ + ax_get_window_id, copy_ax_windows, copy_bool_attr, copy_element_array_attr, copy_string_attr, + element_screen_rect, AXUIElementCreateApplication, AXUIElementRef, + AXUIElementSetMessagingTimeout, +}; + +const OBSERVATION_TIMEOUT: Duration = Duration::from_millis(400); +const POLL_INTERVAL: Duration = Duration::from_millis(30); +const CATCH_UP_INTERVAL: Duration = Duration::from_millis(80); +const CATCH_UP_ATTEMPTS: usize = 3; + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +enum RootKey { + Native { + window_id: u32, + role: String, + subrole: String, + }, + Transient { + parent_window_id: Option, + role: String, + subrole: String, + title: String, + frame: Option<[i64; 4]>, + }, +} + +#[derive(Clone, Debug, PartialEq)] +struct Root { + window_id: Option, + title: String, + modal: bool, + focused: bool, +} + +type RootSnapshot = HashMap; + +#[derive(Default)] +struct RootObservation { + roots: RootSnapshot, + window_signature: HashSet, +} + +pub struct ObservedActionTool { + inner: Box, +} + +impl ObservedActionTool { + pub fn new(inner: Box) -> Self { + Self { inner } + } +} + +#[async_trait] +impl Tool for ObservedActionTool { + fn def(&self) -> &ToolDef { + self.inner.def() + } + + async fn protected_resource_ownership( + &self, + adapter_id: &str, + args: &Value, + ) -> ProtectedResourceOwnership { + self.inner + .protected_resource_ownership(adapter_id, args) + .await + } + + async fn protected_resource_scope( + &self, + adapter_id: &str, + args: &Value, + ) -> Result, String> { + self.inner.protected_resource_scope(adapter_id, args).await + } + + async fn validate_protected_resource_scope( + &self, + adapter_id: &str, + args: &Value, + approved_scope: &Value, + ) -> Result<(), String> { + self.inner + .validate_protected_resource_scope(adapter_id, args, approved_scope) + .await + } + + async fn invoke(&self, args: Value) -> ToolResult { + let Some(pid) = args + .get("pid") + .and_then(Value::as_i64) + .and_then(|pid| i32::try_from(pid).ok()) + .filter(|pid| *pid > 0) + else { + return self.inner.invoke(args).await; + }; + let prior_front = crate::apps::frontmost_pid(); + let suppress_cross_app = args + .get("delivery_mode") + .and_then(Value::as_str) + .is_none_or(|mode| !mode.eq_ignore_ascii_case("foreground")); + let _suppression = prior_front + .filter(|_| suppress_cross_app) + .map(|restore_to| { + crate::focus_steal::begin_suppression_allowing( + pid, + restore_to, + "ObservedActionTool", + ) + }); + + let before = tokio::task::spawn_blocking(move || begin_observation(pid)) + .await + .unwrap_or_default(); + let mut result = self.inner.invoke(args).await; + let delta = tokio::task::spawn_blocking(move || observe_delta(pid, prior_front, before)) + .await + .ok() + .flatten(); + if let Some(delta) = delta { + result.surface_delta = Some(delta); + } + result + } +} + +fn observe_delta( + pid: i32, + prior_front: Option, + before: RootObservation, +) -> Option { + let deadline = Instant::now() + OBSERVATION_TIMEOUT; + let signaled = loop { + if foreground_changed(prior_front) + || target_window_signature(pid) != before.window_signature + { + break true; + } + if Instant::now() >= deadline { + break false; + } + std::thread::sleep(POLL_INTERVAL); + }; + + let mut appeared = appeared_roots(&before.roots, &snapshot_roots(pid)); + if appeared.is_empty() && signaled { + for _ in 0..CATCH_UP_ATTEMPTS { + std::thread::sleep(CATCH_UP_INTERVAL); + appeared = appeared_roots(&before.roots, &snapshot_roots(pid)); + if !appeared.is_empty() { + break; + } + } + } + resolve_appeared_roots(pid, &appeared, foreground_changed(prior_front)) +} + +fn foreground_changed(prior_front: Option) -> bool { + matches!( + (prior_front, crate::apps::frontmost_pid()), + (Some(before), Some(after)) if before != after + ) +} + +fn appeared_roots(before: &RootSnapshot, after: &RootSnapshot) -> Vec { + after + .iter() + .filter(|(key, _)| !before.contains_key(*key)) + .map(|(_, root)| root.clone()) + .collect() +} + +/// Resolve ownership after AX has established which roots appeared. WindowServer +/// may lag AX, so unresolved roots retry ownership only and block an exact target. +fn resolve_appeared_roots( + pid: i32, + roots: &[Root], + foreground_changed: bool, +) -> Option { + if roots.is_empty() { + return None; + } + let app_name = crate::apps::get_app_name_for_pid(pid).unwrap_or_default(); + let mut resolved = resolve_candidates(pid, &app_name, roots, &crate::windows::all_windows()); + for _ in 0..CATCH_UP_ATTEMPTS { + if resolved.len() == roots.len() { + break; + } + std::thread::sleep(CATCH_UP_INTERVAL); + resolved = resolve_candidates(pid, &app_name, roots, &crate::windows::all_windows()); + } + let incomplete = resolved.len() != roots.len(); + let mut delta = resolve_surface_delta(resolved, foreground_changed)?; + if incomplete { + delta.rebind = None; + } + Some(delta) +} + +fn resolve_candidates( + pid: i32, + app_name: &str, + roots: &[Root], + windows: &[crate::windows::WindowInfo], +) -> Vec { + roots + .iter() + .filter_map(|root| { + let window_id = root.window_id?; + let (owner_pid, owner_app_name) = surface_owner(windows, pid, window_id, app_name)?; + Some(ActionSurfaceCandidate { + target: ActionSurfaceTarget { + pid: i64::from(owner_pid), + window_id: u64::from(window_id), + app_name: owner_app_name, + title: root.title.clone(), + modal: root.modal, + }, + focused: root.focused, + }) + }) + .collect() +} + +fn begin_observation(pid: i32) -> RootObservation { + RootObservation { + roots: snapshot_roots(pid), + window_signature: target_window_signature(pid), + } +} + +fn target_window_signature(pid: i32) -> HashSet { + crate::windows::visible_windows() + .into_iter() + .filter(|window| window.pid == pid) + .map(|window| window.window_id) + .collect() +} + +fn snapshot_roots(pid: i32) -> RootSnapshot { + unsafe { + let app = AXUIElementCreateApplication(pid); + if app.is_null() { + return RootSnapshot::default(); + } + AXUIElementSetMessagingTimeout(app, 0.25); + let windows = copy_ax_windows(app); + let mut roots = HashMap::new(); + for window in windows { + let parent_window_id = ax_get_window_id(window); + insert_root(&mut roots, window, parent_window_id); + for attribute in ["AXSheets", "AXChildren"] { + for child in copy_element_array_attr(window, attribute) { + let role = copy_string_attr(child, "AXRole").unwrap_or_default(); + if matches!(role.as_str(), "AXSheet" | "AXDialog" | "AXPopover") { + insert_root(&mut roots, child, parent_window_id); + } + CFRelease(child as CFTypeRef); + } + } + CFRelease(window as CFTypeRef); + } + CFRelease(app as CFTypeRef); + roots + } +} + +unsafe fn insert_root( + roots: &mut RootSnapshot, + element: AXUIElementRef, + parent_window_id: Option, +) { + let role = copy_string_attr(element, "AXRole").unwrap_or_default(); + let subrole = copy_string_attr(element, "AXSubrole").unwrap_or_default(); + let title = copy_string_attr(element, "AXTitle").unwrap_or_default(); + let own_window_id = ax_get_window_id(element); + let effective_window_id = own_window_id.or(parent_window_id); + let modal = copy_bool_attr(element, "AXModal").unwrap_or(false) + || role == "AXSheet" + || role == "AXDialog" + || subrole.to_ascii_lowercase().contains("dialog") + || subrole.to_ascii_lowercase().contains("modal"); + let key = match own_window_id { + Some(window_id) => RootKey::Native { + window_id, + role, + subrole, + }, + None => RootKey::Transient { + parent_window_id, + role, + subrole, + title: title.clone(), + frame: element_screen_rect(element) + .map(|frame| frame.map(|value| value.round() as i64)), + }, + }; + roots.insert( + key, + Root { + window_id: effective_window_id, + title, + modal, + focused: copy_bool_attr(element, "AXFocused").unwrap_or(false), + }, + ); +} + +fn surface_owner( + windows: &[crate::windows::WindowInfo], + target_pid: i32, + window_id: u32, + target_app_name: &str, +) -> Option<(i32, String)> { + match crate::windows::resolve_window_owner_in(windows, target_pid, window_id) { + crate::windows::WindowOwner::SamePid => Some((target_pid, target_app_name.to_owned())), + crate::windows::WindowOwner::ForeignPid { + owner_pid, + owner_app_name, + } => Some((owner_pid, owner_app_name)), + crate::windows::WindowOwner::Unknown => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn root(window_id: u32, title: &str, modal: bool) -> Root { + Root { + window_id: Some(window_id), + title: title.into(), + modal, + focused: false, + } + } + + #[test] + fn root_diff_ignores_metadata_changes_and_reports_an_appeared_modal() { + let parent = RootKey::Native { + window_id: 7, + role: "AXWindow".into(), + subrole: "AXStandardWindow".into(), + }; + let before = RootSnapshot::from([(parent.clone(), root(7, "Draft", false))]); + let mut after = RootSnapshot::from([(parent, root(7, "Draft — Edited", false))]); + assert!(appeared_roots(&before, &after).is_empty()); + + let sheet = RootKey::Native { + window_id: 8, + role: "AXSheet".into(), + subrole: String::new(), + }; + after.insert(sheet, root(8, "Open", true)); + assert_eq!(appeared_roots(&before, &after), vec![root(8, "Open", true)]); + } + + #[test] + fn owner_resolution_can_follow_an_ax_root_without_guessing() { + let roots = vec![root(8, "Open", true)]; + assert!(resolve_candidates(42, "TextEdit", &roots, &[]).is_empty()); + + let windows = vec![crate::windows::WindowInfo { + window_id: 8, + pid: 99, + app_name: "Open and Save Panel Service".into(), + title: "Open".into(), + bounds: crate::windows::WindowBounds { + x: 0.0, + y: 0.0, + width: 640.0, + height: 480.0, + }, + layer: 0, + z_index: 1, + is_on_screen: true, + current_space_id: None, + on_current_space: None, + space_ids: None, + }]; + let candidates = resolve_candidates(42, "TextEdit", &roots, &windows); + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].target.pid, 99); + assert_eq!(candidates[0].target.window_id, 8); + } + + #[test] + fn unrelated_foreground_change_is_not_a_surface_delta() { + assert_eq!(resolve_surface_delta(Vec::new(), true), None); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs index 26b45ef591..ba1ac1806b 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs @@ -29,7 +29,6 @@ use crate::ax::bindings::{ element_screen_rect, kAXErrorSuccess, AXUIElementPerformAction, AXUIElementRef, }; use crate::focus_guard; -use crate::window_change_detector::WindowChangeDetector; use core_foundation::base::{CFRelease, TCFType}; use super::ToolState; @@ -601,19 +600,8 @@ impl Tool for ClickTool { selection_pixel = None; } - // ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ── - // Capture prior frontmost, arm the wildcard suppressor in the - // snapshot, then arm a targeted suppressor across the AX action - // itself via FocusGuard. After the action returns, detect any - // new-window / foreground side-effects and append a one-liner - // suffix matching Swift's wording. let prior_front = apps::frontmost_pid(); let foreground = delivery_mode.is_foreground(); - let snapshot = if foreground { - WindowChangeDetector::snapshot_without_suppression(prior_front) - } else { - WindowChangeDetector::snapshot(prior_front) - }; // Run AX work on a blocking thread (can't block async executor). // Use `effective_action` so button=right rewrites press → show_menu. @@ -686,13 +674,10 @@ impl Tool for ClickTool { ) .await; - // Drop the wildcard lease + detect window/foreground side-effects. - let changes = super::finish_window_observation(snapshot, &args).await; - match result { Ok(Ok(( ( - mut msg, + msg, needs_webkit_delay, suspected_noop, selection_verified, @@ -705,7 +690,6 @@ impl Tool for ClickTool { if needs_webkit_delay { tokio::time::sleep(std::time::Duration::from_millis(800)).await; } - msg.push_str(&changes.result_suffix()); // AX dispatch went through, but AXPerformAction returning // success does not confirm the on-screen effect (many elements // no-op silently). A click is never driver-verifiable (no @@ -975,22 +959,7 @@ impl Tool for ClickTool { .cursor_registry .update_position(&cursor_key, screen_x, screen_y); - // ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ── - // A pixel click can land on a "Sign In" button that opens a sheet - // or a Safari link that activates a new tab — same side-effect - // shape as the AX path, so we wrap identically. let prior_front = apps::frontmost_pid(); - let snapshot = match activation_policy { - PixelActivationPolicy::SuppressTarget => { - WindowChangeDetector::snapshot(prior_front) - } - PixelActivationPolicy::AllowTargetWithoutRaise => { - WindowChangeDetector::snapshot_allowing_activation(prior_front, pid) - } - PixelActivationPolicy::ForegroundAssist => { - WindowChangeDetector::snapshot_without_suppression(prior_front) - } - }; // Restore the Swift background-click prologue that was left // disconnected in the original Rust port. It makes an opaque @@ -1151,8 +1120,6 @@ impl Tool for ClickTool { } } - let changes = super::finish_window_observation(snapshot, &args).await; - let button_label = match button_str.as_str() { "right" => "right-click", "middle" => "middle-click", @@ -1170,8 +1137,7 @@ impl Tool for ClickTool { }; ToolResult::text(format!( "✅ Posted {button_label} to pid {pid} ({mode_label}; \ - not driver-verified — confirm via screenshot).{}", - changes.result_suffix() + not driver-verified — confirm via screenshot)." )) .with_structured(serde_json::json!({ "path": path, diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs index f5eb9d0395..1d7d0db405 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs @@ -21,7 +21,6 @@ use super::ToolState; use crate::apps; use crate::focus_guard; use crate::input::mouse::DragButton; -use crate::window_change_detector::WindowChangeDetector; pub struct DragTool { pub state: Arc, @@ -287,13 +286,7 @@ impl Tool for DragTool { } crate::cursor::overlay::animate_cursor_to(cursor_key.clone(), from_sx, from_sy).await; - // ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ── - // Drags can trigger drag-and-drop side-effects that spawn helper - // windows (drop on Dock, drop on background app icon) and the - // mouseDown half-event alone can activate the target app on some - // Chromium builds. Wrap to catch + report both. let prior_front = apps::frontmost_pid(); - let snapshot = WindowChangeDetector::snapshot(prior_front); // Dispatch blocking drag synthesis. let mods_owned = modifiers.clone(); @@ -395,8 +388,6 @@ impl Tool for DragTool { .update_position(&cursor_key, to_sx, to_sy); } - let changes = super::finish_window_observation(snapshot, &args).await; - if let Some(wid) = window_id { crate::cursor::overlay::send_command( cursor_key.clone(), @@ -426,12 +417,11 @@ impl Tool for DragTool { from window-pixel ({}, {}) → ({}, {}), \ screen ({}, {}) → ({}, {}) \ in {duration_ms}ms / {steps} steps{mode_label} \ - (background CGEvent; not driver-verified — confirm via screenshot).{}", + (background CGEvent; not driver-verified — confirm via screenshot).", from_x as i64, from_y as i64, to_x as i64, to_y as i64, from_sx as i64, from_sy as i64, to_sx as i64, to_sy as i64, - changes.result_suffix(), )) .with_structured(serde_json::json!({ "path": if fg { "cgevent_fg" } else { "cgevent" }, "verified": false, "effect": "unverifiable" diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs index 652a372ade..1f324581e7 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs @@ -125,9 +125,8 @@ impl Tool for GetDesktopStateTool { ToolResult { content, - is_error: None, structured_content: Some(structured), - action_record: None, + ..Default::default() } } } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs index 378023c169..a900804ce3 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs @@ -649,9 +649,8 @@ impl Tool for GetWindowStateTool { ); ToolResult { content, - is_error: None, structured_content: Some(structured), - action_record: None, + ..Default::default() } } } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs index 699a8e8f12..2a7a7da1a3 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs @@ -11,7 +11,6 @@ use std::sync::Arc; use crate::apps; use crate::focus_guard; -use crate::window_change_detector::WindowChangeDetector; use super::ToolState; @@ -391,14 +390,7 @@ impl Tool for HotkeyTool { } }; - // ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ── - // Hotkeys like Cmd+N, Cmd+W, Cmd+T explicitly open/close - // windows. The NSMenu path also briefly activates the target via - // SLPSSetFrontProcessWithOptions which can race the wildcard - // suppressor — wrapping ensures both side-effects are observed - // and the prior frontmost is restored if the activation lingers. let prior_front = apps::frontmost_pid(); - let snapshot = WindowChangeDetector::snapshot(prior_front); let result = focus_guard::with_focus_suppressed( Some(pid), @@ -479,8 +471,6 @@ impl Tool for HotkeyTool { ) .await; - let changes = super::finish_window_observation(snapshot, &args).await; - match result { Ok(Ok(())) => { let label = if fg { @@ -505,11 +495,8 @@ impl Tool for HotkeyTool { pixel-click to focus then type_text instead.)" }); } - ToolResult::text(format!( - "Pressed {key_display} on pid {pid}{label}.{}", - changes.result_suffix() - )) - .with_structured(structured) + ToolResult::text(format!("Pressed {key_display} on pid {pid}{label}.")) + .with_structured(structured) } Ok(Err(e)) => ToolResult::error(format!("hotkey failed: {e}")), Err(e) => ToolResult::error(format!("Task error: {e}")), diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs index 68287e0c93..df457b524d 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs @@ -111,6 +111,12 @@ mod pid_window_target_tests { #[cfg(test)] mod background_input_regression_tests; +fn observed_action(tool: T) -> Box { + Box::new(crate::post_action_observer::ObservedActionTool::new( + Box::new(tool), + )) +} + fn pid_window_guarded( tool: T, candidates: &WindowTargetCandidates, @@ -121,6 +127,16 @@ fn pid_window_guarded( )) } +fn observed_pid_window_action( + tool: T, + candidates: &WindowTargetCandidates, +) -> Box { + Box::new(PidOnlyWindowTargetGuard::new( + observed_action(tool), + candidates.clone(), + )) +} + pub use check_permissions::{ request_from_launchservices_host as request_permissions_from_launchservices_host, PERMISSIONS_HOST_REQUEST_ARG, @@ -281,42 +297,6 @@ pub(crate) async fn acquire_background_mutation(pid: i32) -> BackgroundMutationL } } -/// Finish the post-action observation window. Embedded interactive clients -/// that already observe the target continuously may opt out through the -/// private registry argument to avoid adding a one-second acknowledgement -/// delay to every input event. Regular MCP callers retain the full observer. -pub(crate) async fn finish_window_observation( - snapshot: crate::window_change_detector::Snapshot, - args: &serde_json::Value, -) -> crate::window_change_detector::Changes { - if args - .get("_skip_window_change_detection") - .and_then(serde_json::Value::as_bool) - .unwrap_or(false) - { - drop(snapshot); - crate::window_change_detector::Changes::no_change() - } else { - snapshot.detect_async().await - } -} - -#[cfg(test)] -mod interactive_observation_tests { - use super::*; - - #[tokio::test] - async fn embedded_interactive_input_can_finish_without_polling() { - let snapshot = crate::window_change_detector::WindowChangeDetector::snapshot(None); - let changes = finish_window_observation( - snapshot, - &serde_json::json!({"_skip_window_change_detection": true}), - ) - .await; - assert!(!changes.needs_restore()); - } -} - /// px-focus for the keyboard family (type_text / press_key / hotkey): focus the /// element at (x,y) before a keystroke — the *element px action* form of a /// keyboard tool. Prefer non-destructive AX focus so an existing selection is @@ -907,40 +887,40 @@ pub fn register_all( &pid_window_candidates, )); registry.register(Box::new(set_window_frame::SetWindowFrameTool)); - registry.register(Box::new(invoke_menu::InvokeMenuTool)); - registry.register(pid_window_guarded( + registry.register(observed_action(invoke_menu::InvokeMenuTool)); + registry.register(observed_pid_window_action( click::ClickTool::new(state.clone()), &pid_window_candidates, )); - registry.register(pid_window_guarded( + registry.register(observed_pid_window_action( double_click::DoubleClickTool::new(state.clone()), &pid_window_candidates, )); - registry.register(pid_window_guarded( + registry.register(observed_pid_window_action( right_click::RightClickTool::new(state.clone()), &pid_window_candidates, )); - registry.register(pid_window_guarded( + registry.register(observed_pid_window_action( drag::DragTool::new(state.clone()), &pid_window_candidates, )); - registry.register(pid_window_guarded( + registry.register(observed_pid_window_action( type_text::TypeTextTool::new(state.clone()), &pid_window_candidates, )); - registry.register(pid_window_guarded( + registry.register(observed_pid_window_action( press_key::PressKeyTool::new(state.clone()), &pid_window_candidates, )); - registry.register(pid_window_guarded( + registry.register(observed_pid_window_action( hotkey::HotkeyTool::new(state.clone()), &pid_window_candidates, )); - registry.register(pid_window_guarded( + registry.register(observed_pid_window_action( set_value::SetValueTool::new(state.clone()), &pid_window_candidates, )); - registry.register(pid_window_guarded( + registry.register(observed_pid_window_action( scroll::ScrollTool::new(state.clone()), &pid_window_candidates, )); diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs index 8378796e41..fcf0199a13 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs @@ -19,7 +19,6 @@ use crate::ax::bindings::{ copy_bool_attr, copy_string_attr, focused_element_of_pid, AXUIElementRef, }; use crate::focus_guard; -use crate::window_change_detector::WindowChangeDetector; use super::ToolState; @@ -403,16 +402,7 @@ impl Tool for PressKeyTool { } }; - // ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ── - // Single-key presses can fire autocomplete (Return on a search - // box opens a results popover) or trigger menu shortcuts that - // open windows. Wrapping mirrors the hotkey path. - // - // The AX focus_element() pre-write also runs inside the closure - // so any reflex activations it triggers are caught by both the - // wildcard snapshot suppressor and the targeted FocusGuard lease. let prior_front = apps::frontmost_pid(); - let snapshot = WindowChangeDetector::snapshot(prior_front); let result = focus_guard::with_focus_suppressed( Some(pid), @@ -471,8 +461,6 @@ impl Tool for PressKeyTool { ) .await; - let changes = super::finish_window_observation(snapshot, &args).await; - let delivery_outcome = match result { Ok(result) => map_delivery_outcome(result), Err(error) => { @@ -494,12 +482,9 @@ impl Tool for PressKeyTool { "verified": confirmed, "effect": if confirmed { "confirmed" } else { "unverifiable" }, }); - ToolResult::text(format!( - "✅ Pressed {display_key} on pid {pid}{label}.{}", - changes.result_suffix() - )) - .with_structured(structured) - .with_action_record(action_record(confirmed, fg)) + ToolResult::text(format!("✅ Pressed {display_key} on pid {pid}{label}.")) + .with_structured(structured) + .with_action_record(action_record(confirmed, fg)) } PressKeyDeliveryOutcome::Failed(error) => delivery_failed(error), } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs index 3ac5b1477c..290dd71534 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs @@ -15,7 +15,6 @@ use crate::ax::bindings::{ perform_action, AXUIElementRef, }; use crate::focus_guard; -use crate::window_change_detector::WindowChangeDetector; use super::ToolState; @@ -544,7 +543,6 @@ impl Tool for ScrollTool { ); let prior_front = apps::frontmost_pid(); - let snapshot = WindowChangeDetector::snapshot(prior_front); let WheelTarget { screen_x, @@ -590,7 +588,6 @@ impl Tool for ScrollTool { ) .await; - let changes = super::finish_window_observation(snapshot, &args).await; let mode_label = if fg { " (delivery_mode:foreground)" } else { @@ -600,8 +597,7 @@ impl Tool for ScrollTool { Ok(Ok(())) => ToolResult::text(format!( "✅ Sent {direction} scroll by {by} × {amount} via pixel wheel at \ ({screen_x:.0}, {screen_y:.0}){mode_label} (background CGEvent; not \ - driver-verified — confirm via screenshot).{}", - changes.result_suffix() + driver-verified — confirm via screenshot)." )) .with_structured(serde_json::json!({ "path": if fg { "cgevent_fg" } else { "cgevent" }, "verified": false, "effect": "unverifiable" @@ -651,16 +647,7 @@ impl Tool for ScrollTool { } } - // ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ── - // Scroll keystrokes (PageDown / arrow) into search-box autocomplete - // can spawn floating helper windows; rare but real. Wrap for parity - // with the other action tools. - // - // The AX focus_element() pre-write also runs inside the closure so - // any reflex activations it triggers are caught by both the wildcard - // snapshot suppressor and the targeted FocusGuard lease. let prior_front = apps::frontmost_pid(); - let snapshot = WindowChangeDetector::snapshot(prior_front); let result = focus_guard::with_focus_suppressed( Some(pid), @@ -689,13 +676,10 @@ impl Tool for ScrollTool { ) .await; - let changes = super::finish_window_observation(snapshot, &args).await; - match result { Ok(Ok(())) => ToolResult::text(format!( "✅ Sent {direction} scroll by {by} × {amount} via keystroke \ - (background; not driver-verified — confirm via screenshot).{}", - changes.result_suffix() + (background; not driver-verified — confirm via screenshot)." )) .with_structured(serde_json::json!({ "path": "key_events", "verified": false })), Ok(Err(e)) => ToolResult::error(format!("Scroll failed: {e}")), diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs index 449f6306fe..8413063f82 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs @@ -25,7 +25,6 @@ use crate::ax::bindings::{ set_number_attr, set_string_attr, AXUIElementRef, }; use crate::focus_guard; -use crate::window_change_detector::WindowChangeDetector; use core_foundation::base::CFRelease; use super::ToolState; @@ -204,12 +203,7 @@ impl Tool for SetValueTool { Some(window_id), ); - // ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ── - // AXValue writes on popups / sliders can cause reflex activations - // in Chromium-based apps; the AXPopUpButton path also AXPresses a - // child option which can trigger app activation in some setups. let prior_front = apps::frontmost_pid(); - let snapshot = WindowChangeDetector::snapshot(prior_front); let result = focus_guard::with_focus_suppressed( Some(pid), @@ -224,14 +218,11 @@ impl Tool for SetValueTool { ) .await; - let changes = snapshot.detect_async().await; - match result { Ok(Ok(mut outcome)) => { apply_surface_trust(&mut outcome, ax_echo_surface); apply_verification_label(&mut outcome); - let mut msg = outcome.detail; - msg.push_str(&changes.result_suffix()); + let msg = outcome.detail; let verified = outcome.verified.unwrap_or(false); let mut structured = serde_json::json!({ "path": "ax", diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs index 49f8eca442..21b81c5c19 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs @@ -36,7 +36,6 @@ use crate::ax::bindings::{ copy_string_attr, focused_element_of_pid, kAXErrorSuccess, set_string_attr, AXUIElementRef, }; use crate::focus_guard; -use crate::window_change_detector::WindowChangeDetector; use core_foundation::base::CFRelease; use cua_driver_core::background_input::BackgroundRefusal; @@ -356,13 +355,7 @@ impl Tool for TypeTextTool { let text_clone = text.clone(); let char_count = text.chars().count(); - // ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ── - // Typing into a field can trigger autocomplete popovers or - // Chrome/Safari's "Save Password?" prompt, both of which open - // helper windows. Wrap so callers see them in the result suffix - // and the wildcard suppressor catches reflex activations. let prior_front = apps::frontmost_pid(); - let snapshot = WindowChangeDetector::snapshot(prior_front); // Terminal-emulator short-circuit: when the target pid belongs // to a known terminal (Ghostty / Terminal.app / iTerm2 / …), the @@ -394,8 +387,6 @@ impl Tool for TypeTextTool { ) .await; - let changes = super::finish_window_observation(snapshot, &args).await; - // Unwrap the delivery envelope: a structured refusal means no // actuator ran and the caller gets the exact reason. let result = match result { @@ -492,33 +483,32 @@ impl Tool for TypeTextTool { .to_string(), ) }; - ToolResult::text(format!( - "{mark} {char_count} char(s){detail}.{note}{}", - changes.result_suffix() - )) - .with_structured({ - // `effect` mirrors `verified`'s read-back tri-state: a TRUSTED - // positive read-back is "confirmed"; an unreadable/unchanged - // AXValue, a dropped CGEvent rung, or an Electron AX echo we - // refuse to trust is "unverifiable". - let mut s = serde_json::json!({ - "path": path, - "characters": char_count, - "requested_chars": char_count, - "verified": verified, - "effect": if verified { "confirmed" } else { "unverifiable" }, - }); - if let Some(delivered_chars) = delivered_chars { - s["delivered_chars"] = serde_json::json!(delivered_chars); - } - if untrusted_web_readback { - // Web-content AXValue read-back. A real browser TAB → the - // `page` tool (drives the DOM via CDP) is the reliable rung; - // an embedded web view (Electron, no CDP) → the element px - // action. It's a renderer/DOM-focus problem, never a - // foreground one. - let escalation = - match web_readback_next_rung(electron_web_content, used_pixel_focus) { + ToolResult::text(format!("{mark} {char_count} char(s){detail}.{note}")) + .with_structured({ + // `effect` mirrors `verified`'s read-back tri-state: a TRUSTED + // positive read-back is "confirmed"; an unreadable/unchanged + // AXValue, a dropped CGEvent rung, or an Electron AX echo we + // refuse to trust is "unverifiable". + let mut s = serde_json::json!({ + "path": path, + "characters": char_count, + "requested_chars": char_count, + "verified": verified, + "effect": if verified { "confirmed" } else { "unverifiable" }, + }); + if let Some(delivered_chars) = delivered_chars { + s["delivered_chars"] = serde_json::json!(delivered_chars); + } + if untrusted_web_readback { + // Web-content AXValue read-back. A real browser TAB → the + // `page` tool (drives the DOM via CDP) is the reliable rung; + // an embedded web view (Electron, no CDP) → the element px + // action. It's a renderer/DOM-focus problem, never a + // foreground one. + let escalation = match web_readback_next_rung( + electron_web_content, + used_pixel_focus, + ) { Some("px") => Some(( "px", "Electron web view — AXValue read-back cannot prove \ @@ -542,22 +532,22 @@ impl Tool for TypeTextTool { )), _ => None, }; - if let Some((recommended, reason)) = escalation { + if let Some((recommended, reason)) = escalation { + s["escalation"] = serde_json::json!({ + "recommended": recommended, + "reason": reason, + }); + } + } else if !verified && path != PATH_KEY_EVENTS_FG { s["escalation"] = serde_json::json!({ - "recommended": recommended, - "reason": reason, + "recommended": "foreground", + "reason": "background insert could not be confirmed — \ + re-call with delivery_mode:\"foreground\" if a \ + screenshot shows the text didn't land." }); } - } else if !verified && path != PATH_KEY_EVENTS_FG { - s["escalation"] = serde_json::json!({ - "recommended": "foreground", - "reason": "background insert could not be confirmed — \ - re-call with delivery_mode:\"foreground\" if a \ - screenshot shows the text didn't land." - }); - } - s - }) + s + }) } Ok(Err(e)) => ToolResult::error(format!("type_text failed: {e}")), Err(e) => ToolResult::error(format!("Task error: {e}")), diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/zoom.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/zoom.rs index f389448f9a..67c3a19c12 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/zoom.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/zoom.rs @@ -116,7 +116,7 @@ impl Tool for ZoomTool { "width": w, "height": h, "format": "jpeg", "mime_type": "image/jpeg" })), - action_record: None, + ..Default::default() } } Ok(Err(e)) => ToolResult::error(format!("Zoom failed: {e}")), diff --git a/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs b/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs deleted file mode 100644 index 3286ac6230..0000000000 --- a/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs +++ /dev/null @@ -1,530 +0,0 @@ -//! Window-change detector — Rust port of Swift's -//! `WindowChangeDetector` (`libs/cua-driver/Sources/CuaDriverServer/Tools/WindowChangeDetector.swift`). -//! -//! ## What this does -//! -//! Action tools (click, type_text, hotkey, …) on a backgrounded app can -//! trigger window/foreground side-effects: a "Sign In" button opens a -//! modal sheet, a Safari link spawns a new tab, an autocomplete dropdown -//! pops a helper window. The Rust port mirrors Swift's -//! snapshot → action → detect cycle so tool results can: -//! -//! 1. Surface the side-effect to the agent (one-line suffix on the -//! tool result, matching Swift verbatim). -//! 2. Arm a **wildcard** focus-steal suppression entry that covers the -//! full snapshot→detect window. Wildcards (`target_pid = None`) -//! catch any activation other than the prior frontmost — so even an -//! app we didn't know about (Safari activating because a UTM Gallery -//! link routed to it) is suppressed before the first compositor -//! frame. -//! -//! ## Usage -//! -//! ```ignore -//! // Callers capture frontmost BEFORE the snapshot so the wildcard -//! // suppressor and the snapshot's recorded frontmost agree on the -//! // pid to restore to — avoids a race where another app activates -//! // between the caller's `frontmost_pid()` and the detector's own. -//! let prior_front = apps::frontmost_pid(); -//! let snapshot = WindowChangeDetector::snapshot(prior_front); -//! // … perform action … -//! let changes = snapshot.detect(); -//! // changes.result_suffix() — append to ToolResult text. -//! ``` -//! -//! Dropping the `Snapshot` ends the suppression lease (RAII). `detect()` -//! also drops the lease before returning — the lease's `Drop` is -//! idempotent so explicit-detect + later-drop is safe. - -use std::collections::HashSet; -use std::time::{Duration, Instant}; - -use crate::apps; -use crate::focus_steal::{self, SuppressionLease}; -use crate::windows::{self, WindowInfo}; - -/// One window that appeared between `snapshot()` and `detect()`. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct WindowEvent { - pub window_id: u32, - pub pid: i32, - pub app_name: String, - pub title: String, -} - -/// Categorical diff entry. We mirror Swift which only emits -/// `WindowEvent` rows for *new* windows — closed/changed never appear -/// in the result suffix — but keep them as enum variants for future -/// extensibility and so unit tests can pin down the diff semantics. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum WindowChange { - Opened(WindowEvent), - Closed { window_id: u32 }, -} - -/// State captured immediately before the action fires. -/// -/// Holds: -/// - `window_ids` — the set of visible layer-0 window IDs at snapshot -/// time. `detect()` diffs against this. -/// - `front_pid` — the OS frontmost pid at snapshot time. `detect()` -/// reports whether a *different* pid became frontmost. The wildcard -/// suppressor in `focus_steal` will normally restore the original -/// front before `detect()`'s poll loop observes the change, so this -/// field is best-effort. -/// - `_lease` — the wildcard suppression lease. Dropping the snapshot -/// ends suppression. Held inside `Option` so `detect()` can take it -/// and drop early. -pub struct Snapshot { - window_ids: HashSet, - front_pid: Option, - _lease: Option, -} - -/// Result of `detect()` — what changed during the action window. -#[derive(Debug, Clone)] -pub struct Changes { - pub new_windows: Vec, - pub foreground_changed: bool, -} - -impl Changes { - pub fn no_change() -> Self { - Self { - new_windows: Vec::new(), - foreground_changed: false, - } - } - - /// True when we found evidence that the action triggered a cross-app - /// side-effect that required (or would have required) a foreground - /// restore. Matches Swift's `Changes.needsRestore`. - pub fn needs_restore(&self) -> bool { - self.foreground_changed || !self.new_windows.is_empty() - } - - /// One-liner summary to append to a tool result, or empty string - /// when nothing interesting happened. - /// - /// Format mirrors Swift `WindowChangeDetector.Changes.resultSuffix` - /// **verbatim** so MCP callers that key off the suffix wording - /// don't need a per-binary special case. - pub fn result_suffix(&self) -> String { - if !self.needs_restore() { - return String::new(); - } - - if !self.new_windows.is_empty() { - // Group by app name (stable order), join titles per app. - let mut by_app: std::collections::BTreeMap<&str, Vec<&str>> = - std::collections::BTreeMap::new(); - for w in &self.new_windows { - by_app.entry(&w.app_name).or_default().push(&w.title); - } - let summaries: Vec = by_app - .into_iter() - .map(|(app, titles)| { - let titles: Vec = titles - .into_iter() - .filter(|t| !t.is_empty()) - .map(|t| format!("\"{t}\"")) - .collect(); - if titles.is_empty() { - app.to_string() - } else { - format!("{app} ({})", titles.join(", ")) - } - }) - .collect(); - format!( - "\n\n🪟 Action opened new window(s): {}.", - summaries.join("; ") - ) - } else { - "\n\n🔀 Action caused a different app to become frontmost.".to_string() - } - } -} - -/// Default poll deadline — new windows triggered by a click typically -/// appear within ~200ms on macOS; 1.0s gives the wildcard suppressor -/// time to fire and settle. -const DEFAULT_TIMEOUT: Duration = Duration::from_millis(1000); - -/// Default inter-poll interval. Matches Swift's 50ms. -const DEFAULT_POLL_INTERVAL: Duration = Duration::from_millis(50); - -/// Public API. Mirrors Swift `enum WindowChangeDetector` — no state of -/// its own; all state lives inside the returned `Snapshot`. -pub struct WindowChangeDetector; - -impl WindowChangeDetector { - /// Capture the current window set + frontmost pid and arm the - /// wildcard focus-steal suppressor. Call immediately before - /// dispatching the action. - /// - /// `prior_front` is the frontmost pid the **caller** already - /// observed — typically captured one line earlier via - /// `apps::frontmost_pid()` for the surrounding `focus_guard` - /// lease. We use the caller's value (not a fresh re-read) so the - /// wildcard suppressor's `restore_to` matches what the focus-guard - /// lease saw; a race where another app became frontmost between - /// the caller's read and this method would otherwise leave the - /// two leases targeting different pids. - /// - /// Returns `Snapshot`. Drop ends suppression (via the held - /// `SuppressionLease`); call `Snapshot::detect()` to consume the - /// snapshot and get a `Changes` summary. - /// - /// Safe to call from any thread — `CGWindowListCopyWindowInfo` is - /// documented as thread-safe. - pub fn snapshot(prior_front: Option) -> Snapshot { - Self::capture(prior_front, true, None) - } - - /// Capture the same before-state without arming reactive focus suppression. - /// Foreground delivery owns its temporary activation and restoration, so a - /// wildcard lease would race the target while the action is settling. - pub fn snapshot_without_suppression(prior_front: Option) -> Snapshot { - Self::capture(prior_front, false, None) - } - - /// Capture the before-state and suppress cross-app activations while - /// allowing one intentional target activation. - /// - /// The raw background pixel-click path needs this middle ground: - /// focus-without-raise makes `allowed_pid` AppKit-active so its event queue - /// accepts the click, but a link or hand-off that activates a different app - /// must still restore the user's original foreground. - pub fn snapshot_allowing_activation(prior_front: Option, allowed_pid: i32) -> Snapshot { - Self::capture(prior_front, true, Some(allowed_pid)) - } - - fn capture( - prior_front: Option, - suppress_focus: bool, - allowed_pid: Option, - ) -> Snapshot { - let window_ids: HashSet = windows::visible_windows() - .into_iter() - .filter(|w| w.layer == 0) - .map(|w| w.window_id) - .collect(); - - // Arm wildcard suppression — covers snapshot → detect window. - // restore_to = caller-captured frontmost; target = wildcard - // (any other pid). If there's no frontmost (rare — screensaver, - // login window), we skip the lease; foreground-change tracking - // still runs. - let lease = prior_front - .filter(|_| suppress_focus) - .map(|restore_to| match allowed_pid { - Some(pid) => focus_steal::begin_suppression_allowing( - pid, - restore_to, - "WindowChangeDetector.snapshot_allowing_activation", - ), - None => focus_steal::begin_suppression( - None, // wildcard - restore_to, - "WindowChangeDetector.snapshot", - ), - }); - - Snapshot { - window_ids, - front_pid: prior_front, - _lease: lease, - } - } -} - -impl Snapshot { - /// Frontmost pid at snapshot time, if any. - pub fn front_pid(&self) -> Option { - self.front_pid - } - - /// Poll for up to `DEFAULT_TIMEOUT` for new windows or a - /// foreground-app change. Returns as soon as a change is detected - /// or the timeout elapses. - /// - /// Consumes the snapshot — the wildcard suppression lease is - /// dropped when this returns (covers the full action + detection - /// window). - pub fn detect(self) -> Changes { - self.detect_with(DEFAULT_TIMEOUT, DEFAULT_POLL_INTERVAL) - } - - /// Async wrapper around `detect()` — runs the synchronous poll - /// loop on a `spawn_blocking` thread so it doesn't stall the - /// tokio runtime. Most action-tool call sites should prefer this - /// over the blocking `detect()`. - pub async fn detect_async(self) -> Changes { - // Move the Snapshot (and its embedded lease) onto the blocking - // thread; the lease's Drop runs there when detect_with returns. - tokio::task::spawn_blocking(move || self.detect()) - .await - .unwrap_or_else(|_| Changes::no_change()) - } - - /// Same as `detect()` but with configurable timing — exposed for - /// tests / callers that want a tighter or looser poll window. - pub fn detect_with(self, timeout: Duration, poll_interval: Duration) -> Changes { - let deadline = Instant::now() + timeout; - loop { - std::thread::sleep(poll_interval); - - let current: Vec = windows::visible_windows() - .into_iter() - .filter(|w| w.layer == 0) - .collect(); - let current_ids: HashSet = current.iter().map(|w| w.window_id).collect(); - - let new_windows: Vec = current - .iter() - .filter(|w| !self.window_ids.contains(&w.window_id)) - .map(|w| WindowEvent { - window_id: w.window_id, - pid: w.pid, - app_name: w.app_name.clone(), - title: w.title.clone(), - }) - .collect(); - // Diff the other direction too — keeps unit tests honest - // even though Swift's result_suffix only uses opened windows. - let _closed: Vec = self - .window_ids - .iter() - .copied() - .filter(|id| !current_ids.contains(id)) - .collect(); - - let current_front = apps::frontmost_pid(); - let foreground_changed = match (self.front_pid, current_front) { - (Some(orig), Some(cur)) => orig != cur, - _ => false, - }; - - if !new_windows.is_empty() || foreground_changed { - return Changes { - new_windows, - foreground_changed, - }; - } - if Instant::now() >= deadline { - return Changes::no_change(); - } - } - } - - // ── Internal helpers — also used by unit tests via the `pub(super)` - // path so the diff logic can be exercised without driving the live - // window enumerator. ──────────────────────────────────────────── - - /// Pure-function diff: given the snapshot's window-id set + a - /// list of currently-visible windows, return the (opened, closed) - /// classification. - /// - /// `#[allow(dead_code)]`: today only the `#[cfg(test)]` block below - /// constructs this — production callers `wait_for_window_change` / - /// `wait_for_window_close` keep the (opened, closed) split inline. - /// Kept `pub(crate)` because the doc comment near the top of this - /// `impl` block calls it out as the entry point for unit-testing the - /// diff logic without driving the live window enumerator. - #[allow(dead_code)] - pub(crate) fn diff( - snapshot_ids: &HashSet, - current: &[WindowInfo], - ) -> (Vec, Vec) { - let current_ids: HashSet = current.iter().map(|w| w.window_id).collect(); - let opened: Vec = current - .iter() - .filter(|w| !snapshot_ids.contains(&w.window_id)) - .map(|w| WindowEvent { - window_id: w.window_id, - pid: w.pid, - app_name: w.app_name.clone(), - title: w.title.clone(), - }) - .collect(); - let closed: Vec = snapshot_ids - .iter() - .copied() - .filter(|id| !current_ids.contains(id)) - .collect(); - (opened, closed) - } -} - -// ── Tests ──────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::windows::WindowBounds; - - fn win(id: u32, pid: i32, app: &str, title: &str) -> WindowInfo { - WindowInfo { - window_id: id, - pid, - app_name: app.into(), - title: title.into(), - bounds: WindowBounds { - x: 0., - y: 0., - width: 100., - height: 100., - }, - layer: 0, - z_index: 0, - is_on_screen: true, - current_space_id: None, - on_current_space: None, - space_ids: None, - } - } - - #[test] - fn diff_finds_opened_window() { - let snap: HashSet = [1, 2].into_iter().collect(); - let cur = vec![ - win(1, 100, "Safari", "Home"), - win(2, 100, "Safari", "Tab2"), - win(3, 101, "Mail", "Inbox"), - ]; - let (opened, closed) = Snapshot::diff(&snap, &cur); - assert_eq!(opened.len(), 1); - assert_eq!(opened[0].window_id, 3); - assert_eq!(opened[0].app_name, "Mail"); - assert_eq!(opened[0].title, "Inbox"); - assert!(closed.is_empty()); - } - - #[test] - fn diff_finds_closed_window() { - let snap: HashSet = [1, 2, 3].into_iter().collect(); - let cur = vec![win(1, 100, "Safari", "Home")]; - let (opened, closed) = Snapshot::diff(&snap, &cur); - assert!(opened.is_empty()); - assert_eq!(closed.len(), 2); - let closed_set: HashSet = closed.into_iter().collect(); - assert!(closed_set.contains(&2)); - assert!(closed_set.contains(&3)); - } - - #[test] - fn diff_no_change() { - let snap: HashSet = [1, 2].into_iter().collect(); - let cur = vec![win(1, 100, "Safari", "A"), win(2, 100, "Safari", "B")]; - let (opened, closed) = Snapshot::diff(&snap, &cur); - assert!(opened.is_empty()); - assert!(closed.is_empty()); - } - - #[test] - fn changes_result_suffix_no_change_is_empty() { - let c = Changes::no_change(); - assert_eq!(c.result_suffix(), ""); - assert!(!c.needs_restore()); - } - - #[test] - fn changes_result_suffix_single_new_window_with_title() { - let c = Changes { - new_windows: vec![WindowEvent { - window_id: 99, - pid: 100, - app_name: "Chrome".into(), - title: "New Tab".into(), - }], - foreground_changed: false, - }; - assert!(c.needs_restore()); - assert_eq!( - c.result_suffix(), - "\n\n🪟 Action opened new window(s): Chrome (\"New Tab\")." - ); - } - - #[test] - fn changes_result_suffix_groups_windows_by_app() { - let c = Changes { - new_windows: vec![ - WindowEvent { - window_id: 1, - pid: 100, - app_name: "Chrome".into(), - title: "Tab A".into(), - }, - WindowEvent { - window_id: 2, - pid: 100, - app_name: "Chrome".into(), - title: "Tab B".into(), - }, - WindowEvent { - window_id: 3, - pid: 101, - app_name: "Mail".into(), - title: "".into(), - }, - ], - foreground_changed: true, - }; - let suffix = c.result_suffix(); - // BTreeMap sort order is alphabetical by app name → Chrome before Mail. - assert_eq!( - suffix, - "\n\n🪟 Action opened new window(s): Chrome (\"Tab A\", \"Tab B\"); Mail." - ); - } - - #[test] - fn changes_result_suffix_foreground_change_only() { - let c = Changes { - new_windows: vec![], - foreground_changed: true, - }; - assert!(c.needs_restore()); - assert_eq!( - c.result_suffix(), - "\n\n🔀 Action caused a different app to become frontmost." - ); - } - - #[test] - fn changes_result_suffix_empty_title_is_dropped() { - let c = Changes { - new_windows: vec![WindowEvent { - window_id: 1, - pid: 100, - app_name: "Finder".into(), - title: "".into(), - }], - foreground_changed: false, - }; - // No title → just the app name, no parentheses. - assert_eq!( - c.result_suffix(), - "\n\n🪟 Action opened new window(s): Finder." - ); - } - - /// Regression: `snapshot(prior_front)` must store the caller's - /// captured front pid verbatim (rather than re-reading it inside - /// the function and racing with concurrent activations). - #[test] - fn snapshot_stores_caller_prior_front() { - // Use an obviously bogus pid so we'd notice if the impl silently - // fell back to the live frontmost on this test runner. - let bogus_prior = Some(424242_i32); - let snap = WindowChangeDetector::snapshot(bogus_prior); - assert_eq!(snap.front_pid(), bogus_prior); - - // None must round-trip too — and must skip the lease without - // panicking (no frontmost to restore to). - let snap_none = WindowChangeDetector::snapshot(None); - assert_eq!(snap_none.front_pid(), None); - } -} diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs index ef1477e1d6..0743f1aac8 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs @@ -1506,9 +1506,8 @@ impl Tool for GetWindowStateTool { ToolResult { content, - is_error: None, structured_content: Some(structured), - action_record: None, + ..Default::default() } } Err(e) => ToolResult::error(format!("Error: {e}")), @@ -7470,9 +7469,8 @@ impl Tool for GetDesktopStateTool { ToolResult { content, - is_error: None, structured_content: Some(structured), - action_record: None, + ..Default::default() } } } @@ -8424,7 +8422,7 @@ impl Tool for ZoomTool { "width": w, "height": h, "format": "jpeg", "mime_type": "image/jpeg" })), - action_record: None, + ..Default::default() } } Ok(Err(e)) => ToolResult::error(format!("Zoom failed: {e}")), diff --git a/libs/cua-driver/typescript/src/native/cua_driver_contract.ts b/libs/cua-driver/typescript/src/native/cua_driver_contract.ts index 48aaa34a5b..0a26b4e13e 100644 --- a/libs/cua-driver/typescript/src/native/cua_driver_contract.ts +++ b/libs/cua-driver/typescript/src/native/cua_driver_contract.ts @@ -101,7 +101,8 @@ export enum ActionEscalationTarget { Pixel, Foreground, Page, - Session + Session, + Rebind } const FfiConverterTypeActionEscalationTarget = (() => { @@ -114,6 +115,7 @@ const FfiConverterTypeActionEscalationTarget = (() => { case 2: return ActionEscalationTarget.Foreground; case 3: return ActionEscalationTarget.Page; case 4: return ActionEscalationTarget.Session; + case 5: return ActionEscalationTarget.Rebind; default: throw new UniffiInternalError.UnexpectedEnumCase(); } } @@ -123,6 +125,7 @@ const FfiConverterTypeActionEscalationTarget = (() => { case ActionEscalationTarget.Foreground: return ordinalConverter.write(2, into); case ActionEscalationTarget.Page: return ordinalConverter.write(3, into); case ActionEscalationTarget.Session: return ordinalConverter.write(4, into); + case ActionEscalationTarget.Rebind: return ordinalConverter.write(5, into); } } allocationSize(value: TypeName): number { @@ -137,7 +140,8 @@ export enum ActionEscalationReason { DeliveryFailed, EffectUnconfirmed, SuspectedNoop, - PermissionRequired + PermissionRequired, + SurfaceChanged } const FfiConverterTypeActionEscalationReason = (() => { @@ -151,6 +155,7 @@ const FfiConverterTypeActionEscalationReason = (() => { case 3: return ActionEscalationReason.EffectUnconfirmed; case 4: return ActionEscalationReason.SuspectedNoop; case 5: return ActionEscalationReason.PermissionRequired; + case 6: return ActionEscalationReason.SurfaceChanged; default: throw new UniffiInternalError.UnexpectedEnumCase(); } } @@ -161,6 +166,7 @@ const FfiConverterTypeActionEscalationReason = (() => { case ActionEscalationReason.EffectUnconfirmed: return ordinalConverter.write(3, into); case ActionEscalationReason.SuspectedNoop: return ordinalConverter.write(4, into); case ActionEscalationReason.PermissionRequired: return ordinalConverter.write(5, into); + case ActionEscalationReason.SurfaceChanged: return ordinalConverter.write(6, into); } } allocationSize(value: TypeName): number { @@ -170,9 +176,85 @@ const FfiConverterTypeActionEscalationReason = (() => { return new FFIConverter(); })(); +const stringConverter = (() => { + const encoder = new TextEncoder(); + const decoder = new TextDecoder(); + return { + stringToBytes: (s: string) => encoder.encode(s), + bytesToString: (ab: UniffiByteArray) => decoder.decode(ab), + stringByteLength: (s: string) => encoder.encode(s).byteLength, + writeStringIntoBuffer: (s: string, buf: any, offset: number): number => { + const view = new Uint8Array( + buf.arrayBuffer, + offset, + buf.arrayBuffer.byteLength - offset, + ); + return encoder.encodeInto(s, view).written; + }, + readStringFromBuffer: (buf: any, offset: number, length: number): string => + decoder.decode(new Uint8Array(buf.arrayBuffer, offset, length)), + }; +})(); +const FfiConverterString = uniffiCreateFfiConverterString(stringConverter); + +/** + * A target-owned native interaction root discovered after an action. + */ +export type ActionWindowTarget = { + pid: bigint, + windowId: bigint, + appName: string, + title: string +} + +/** + * Generated factory for {@link ActionWindowTarget} record objects. + */ +export const ActionWindowTarget = (() => { + const defaults = () => ({ + }); + const create = (() => { + return uniffiCreateRecord>(defaults); + })(); + return Object.freeze({ + create, + new: create, + defaults: () => Object.freeze(defaults()) as Partial, + }); +})(); + +const FfiConverterTypeActionWindowTarget = (() => { + type TypeName = ActionWindowTarget; + class FFIConverter extends AbstractFfiConverterByteArray { + read(from: RustBuffer): TypeName { + return { + pid: FfiConverterInt64.read(from), + windowId: FfiConverterUInt64.read(from), + appName: FfiConverterString.read(from), + title: FfiConverterString.read(from) + }; + } + write(value: TypeName, into: RustBuffer): void { + FfiConverterInt64.write(value.pid, into); + FfiConverterUInt64.write(value.windowId, into); + FfiConverterString.write(value.appName, into); + FfiConverterString.write(value.title, into); + } + allocationSize(value: TypeName): number { + return FfiConverterInt64.allocationSize(value.pid) + + FfiConverterUInt64.allocationSize(value.windowId) + + FfiConverterString.allocationSize(value.appName) + + FfiConverterString.allocationSize(value.title); + + } + }; + return new FFIConverter(); +})(); + export type ActionEscalation = { target: ActionEscalationTarget, - reason: ActionEscalationReason + reason: ActionEscalationReason, + window?: ActionWindowTarget } /** @@ -197,16 +279,19 @@ const FfiConverterTypeActionEscalation = (() => { read(from: RustBuffer): TypeName { return { target: FfiConverterTypeActionEscalationTarget.read(from), - reason: FfiConverterTypeActionEscalationReason.read(from) + reason: FfiConverterTypeActionEscalationReason.read(from), + window: FfiConverterOptionalTypeActionWindowTarget.read(from) }; } write(value: TypeName, into: RustBuffer): void { FfiConverterTypeActionEscalationTarget.write(value.target, into); FfiConverterTypeActionEscalationReason.write(value.reason, into); + FfiConverterOptionalTypeActionWindowTarget.write(value.window, into); } allocationSize(value: TypeName): number { return FfiConverterTypeActionEscalationTarget.allocationSize(value.target) + - FfiConverterTypeActionEscalationReason.allocationSize(value.reason); + FfiConverterTypeActionEscalationReason.allocationSize(value.reason) + + FfiConverterOptionalTypeActionWindowTarget.allocationSize(value.window); } }; @@ -214,8 +299,7 @@ const FfiConverterTypeActionEscalation = (() => { })(); export enum ActionEvidenceKind { - ValueReadback, - WindowChange + ValueReadback } const FfiConverterTypeActionEvidenceKind = (() => { @@ -225,14 +309,12 @@ const FfiConverterTypeActionEvidenceKind = (() => { read(from: RustBuffer): TypeName { switch (ordinalConverter.read(from)) { case 1: return ActionEvidenceKind.ValueReadback; - case 2: return ActionEvidenceKind.WindowChange; default: throw new UniffiInternalError.UnexpectedEnumCase(); } } write(value: TypeName, into: RustBuffer): void { switch (value) { case ActionEvidenceKind.ValueReadback: return ordinalConverter.write(1, into); - case ActionEvidenceKind.WindowChange: return ordinalConverter.write(2, into); } } allocationSize(value: TypeName): number { @@ -360,11 +442,58 @@ const FfiConverterTypeActionRoute = (() => { return new FFIConverter(); })(); +/** + * Read-only topology observed for the action's target process. + */ +export type ActionWindowChange = { + newWindows: Array, + foregroundChanged: boolean +} + +/** + * Generated factory for {@link ActionWindowChange} record objects. + */ +export const ActionWindowChange = (() => { + const defaults = () => ({ + }); + const create = (() => { + return uniffiCreateRecord>(defaults); + })(); + return Object.freeze({ + create, + new: create, + defaults: () => Object.freeze(defaults()) as Partial, + }); +})(); + +const FfiConverterTypeActionWindowChange = (() => { + type TypeName = ActionWindowChange; + class FFIConverter extends AbstractFfiConverterByteArray { + read(from: RustBuffer): TypeName { + return { + newWindows: FfiConverterSequenceTypeActionWindowTarget.read(from), + foregroundChanged: FfiConverterBool.read(from) + }; + } + write(value: TypeName, into: RustBuffer): void { + FfiConverterSequenceTypeActionWindowTarget.write(value.newWindows, into); + FfiConverterBool.write(value.foregroundChanged, into); + } + allocationSize(value: TypeName): number { + return FfiConverterSequenceTypeActionWindowTarget.allocationSize(value.newWindows) + + FfiConverterBool.allocationSize(value.foregroundChanged); + + } + }; + return new FFIConverter(); +})(); + export type ActionResult = { effect: ActionEffect, route: ActionRoute, delivery?: ActionDelivery, evidence?: Array, + windowChange?: ActionWindowChange, escalation?: ActionEscalation } @@ -393,6 +522,7 @@ const FfiConverterTypeActionResult = (() => { route: FfiConverterTypeActionRoute.read(from), delivery: FfiConverterOptionalTypeActionDelivery.read(from), evidence: FfiConverterOptionalSequenceTypeActionEvidence.read(from), + windowChange: FfiConverterOptionalTypeActionWindowChange.read(from), escalation: FfiConverterOptionalTypeActionEscalation.read(from) }; } @@ -401,6 +531,7 @@ const FfiConverterTypeActionResult = (() => { FfiConverterTypeActionRoute.write(value.route, into); FfiConverterOptionalTypeActionDelivery.write(value.delivery, into); FfiConverterOptionalSequenceTypeActionEvidence.write(value.evidence, into); + FfiConverterOptionalTypeActionWindowChange.write(value.windowChange, into); FfiConverterOptionalTypeActionEscalation.write(value.escalation, into); } allocationSize(value: TypeName): number { @@ -408,6 +539,7 @@ const FfiConverterTypeActionResult = (() => { FfiConverterTypeActionRoute.allocationSize(value.route) + FfiConverterOptionalTypeActionDelivery.allocationSize(value.delivery) + FfiConverterOptionalSequenceTypeActionEvidence.allocationSize(value.evidence) + + FfiConverterOptionalTypeActionWindowChange.allocationSize(value.windowChange) + FfiConverterOptionalTypeActionEscalation.allocationSize(value.escalation); } @@ -470,27 +602,6 @@ const FfiConverterTypeBoundsExpectation = (() => { return new FFIConverter(); })(); -const stringConverter = (() => { - const encoder = new TextEncoder(); - const decoder = new TextDecoder(); - return { - stringToBytes: (s: string) => encoder.encode(s), - bytesToString: (ab: UniffiByteArray) => decoder.decode(ab), - stringByteLength: (s: string) => encoder.encode(s).byteLength, - writeStringIntoBuffer: (s: string, buf: any, offset: number): number => { - const view = new Uint8Array( - buf.arrayBuffer, - offset, - buf.arrayBuffer.byteLength - offset, - ); - return encoder.encodeInto(s, view).written; - }, - readStringFromBuffer: (buf: any, offset: number, length: number): string => - decoder.decode(new Uint8Array(buf.arrayBuffer, offset, length)), - }; -})(); -const FfiConverterString = uniffiCreateFfiConverterString(stringConverter); - // Enum: ActionTarget export enum ActionTarget_Tags { @@ -3747,6 +3858,9 @@ const FfiConverterTypePlatform = (() => { // FfiConverter for number | undefined const FfiConverterOptionalUInt32 = new FfiConverterOptional(FfiConverterUInt32); +// FfiConverter for ActionWindowTarget | undefined +const FfiConverterOptionalTypeActionWindowTarget = new FfiConverterOptional(FfiConverterTypeActionWindowTarget); + // FfiConverter for ActionDelivery | undefined const FfiConverterOptionalTypeActionDelivery = new FfiConverterOptional(FfiConverterTypeActionDelivery); @@ -3756,6 +3870,12 @@ const FfiConverterSequenceTypeActionEvidence = new FfiConverterArray(FfiConverte // FfiConverter for Array | undefined const FfiConverterOptionalSequenceTypeActionEvidence = new FfiConverterOptional(FfiConverterSequenceTypeActionEvidence); +// FfiConverter for Array +const FfiConverterSequenceTypeActionWindowTarget = new FfiConverterArray(FfiConverterTypeActionWindowTarget); + +// FfiConverter for ActionWindowChange | undefined +const FfiConverterOptionalTypeActionWindowChange = new FfiConverterOptional(FfiConverterTypeActionWindowChange); + // FfiConverter for ActionEscalation | undefined const FfiConverterOptionalTypeActionEscalation = new FfiConverterOptional(FfiConverterTypeActionEscalation); @@ -3858,6 +3978,8 @@ export default Object.freeze({ FfiConverterTypeActionResult, FfiConverterTypeActionRoute, FfiConverterTypeActionTarget, + FfiConverterTypeActionWindowChange, + FfiConverterTypeActionWindowTarget, FfiConverterTypeBoundsExpectation, FfiConverterTypeCaptureScope, FfiConverterTypeClickButton, diff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.sh index d631746aed..b938450f83 100755 --- a/scripts/ci/macos/run-rust-e2e.sh +++ b/scripts/ci/macos/run-rust-e2e.sh @@ -390,6 +390,9 @@ EOF run_test installed-app-textedit cargo test -p cua-driver --test installed_app_textedit_macos_test -- \ --ignored --exact background_type_on_native_cocoa_is_ax_verified \ --nocapture --test-threads=1 + run_test installed-app-textedit-rebind cargo test -p cua-driver --test installed_app_textedit_macos_test -- \ + --ignored --exact background_open_panel_returns_a_typed_rebind \ + --nocapture --test-threads=1 fi fi if [[ "${SUITE}" == capture || "${SUITE}" == all ]]; then