Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
47 commits
Select commit Hold shift + click to select a range
b9809aa
docs: propose Cua Driver environment convergence RFC
Jul 23, 2026
6d3bec1
docs: link Cua Driver convergence RFC review
Jul 23, 2026
7b9251f
docs: keep RFC tracking public
Jul 23, 2026
e307d02
docs(rfc): scope the typed Driver Python Fleet first slice
f-trycua Sep 8, 2026
b630fbf
Merge main into Driver convergence RFC review branch
f-trycua Sep 8, 2026
3b6128d
Merge refreshed RFC parent into typed Driver Phase A
f-trycua Sep 8, 2026
5afd726
docs(rfc): clarify baseline after parent branch refresh
f-trycua Sep 8, 2026
56fdcde
docs(rfc): reuse Fleet authorization for typed Driver slice
f-trycua Sep 8, 2026
54d8ebe
feat(cua-driver): add bounded typed guest envelope receiver
f-trycua Sep 8, 2026
a989072
feat(cua-driver): expose foreign remote envelope channels
f-trycua Sep 8, 2026
0ad4455
Merge commit '54d8ebe75' into feat/driver-python-channel-20260907
f-trycua Sep 8, 2026
704add5
feat(cua-driver): add private loopback envelope HTTP carrier
f-trycua Sep 8, 2026
9f5e0f9
Merge commit '0ad4455e5' into feat/driver-private-http-20260907
f-trycua Sep 8, 2026
44f6ea5
docs(cua-driver): specify private typed HTTP carrier limits
f-trycua Sep 8, 2026
eef8602
fix(cua-driver): own remote callback loops and rejected bindings
f-trycua Sep 8, 2026
5e704fb
Merge commit 'eef8602d7393e28a6386fd08c38bfbe6261bdd6b' into feat/dri…
f-trycua Sep 8, 2026
d7478c4
feat(sandbox): add typed Driver sessions over Fleet services
f-trycua Sep 8, 2026
ee8ac8c
Merge commit '5e704fb39' into feat/sandbox-typed-driver-20260907
f-trycua Sep 8, 2026
ed7c1e6
test(sandbox): require native typed Driver integration in CI
f-trycua Sep 8, 2026
9867816
style(sandbox): match repository import and formatting checks
f-trycua Sep 8, 2026
3ddc1ee
fix(cua-driver): export canonical Python action targets
f-trycua Sep 8, 2026
122ef30
Merge commit '3ddc1eed7' into feat/driver-private-http-20260907
f-trycua Sep 8, 2026
73f76e6
test(sandbox): preserve typed targets and structured Driver refusals
f-trycua Sep 8, 2026
2d00eae
Merge branch 'feat/driver-private-http-20260907' into feat/sandbox-ty…
f-trycua Sep 8, 2026
421610f
fix(sandbox): bound typed Driver lifecycle and request deadlines
f-trycua Sep 8, 2026
89ea6e5
fix(cua-driver): allow explicit unrestricted envelope sessions
f-trycua Sep 8, 2026
e4d58d3
docs(sandbox): clarify launcher-selected Driver permissions
f-trycua Sep 8, 2026
85c10c8
fix(cua-driver): preserve unavailable Windows UIA clicks
f-trycua Sep 9, 2026
5bc11d3
ci(cua-driver): quote Windows test module selectors
f-trycua Sep 9, 2026
c1dc529
Merge main into accepted typed Driver slice
f-trycua Sep 9, 2026
ac75db6
Merge branch 'docs/typed-driver-phase-a-20260907' into feat/driver-en…
f-trycua Sep 9, 2026
48ee2b0
Merge branch 'main' into feat/driver-envelope-receiver-20260907
f-trycua Sep 9, 2026
9e35646
Merge branch 'feat/driver-envelope-receiver-20260907' into feat/drive…
f-trycua Sep 9, 2026
5b88d2d
Merge branch 'feat/driver-python-channel-20260907' into feat/driver-p…
f-trycua Sep 9, 2026
292622e
fix(sandbox): bound typed Driver lifecycle and request deadlines (#3655)
f-trycua Sep 9, 2026
46c5eeb
Merge branch 'feat/sandbox-typed-driver-20260907' into fix/driver-env…
f-trycua Sep 9, 2026
9857e2c
Merge branch 'feat/driver-private-http-20260907' into feat/sandbox-ty…
f-trycua Sep 9, 2026
e94f145
Merge branch 'feat/sandbox-typed-driver-20260907' into fix/driver-env…
f-trycua Sep 9, 2026
95fe75b
Merge branch 'fix/driver-envelope-mode-20260908' into fix/windows-uia…
f-trycua Sep 9, 2026
8b4fe60
Merge receiver landing into generated Driver bridge
f-trycua Sep 9, 2026
2ae53d3
Merge branch 'feat/driver-python-channel-20260907' into feat/driver-p…
f-trycua Sep 9, 2026
ff0007a
fix(cua-driver): allow explicit unrestricted envelope sessions (#3664)
f-trycua Sep 9, 2026
347d2c0
Merge branch 'main' into feat/driver-python-channel-20260907
f-trycua Sep 9, 2026
e9999f2
Merge branch 'feat/driver-python-channel-20260907' into feat/driver-p…
f-trycua Sep 9, 2026
1d7a867
Merge branch 'feat/driver-private-http-20260907' into feat/sandbox-ty…
f-trycua Sep 9, 2026
f8c89aa
Merge branch 'feat/sandbox-typed-driver-20260907' into fix/windows-ui…
f-trycua Sep 9, 2026
5e6a090
Merge branch 'main' into fix/windows-uia-unavailability-20260908
f-trycua Sep 9, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions .github/workflows/ci-rust-windows.yml
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,12 @@ jobs:
if ($LASTEXITCODE -ne 0) { throw "PostMessage read-back tests failed with exit $LASTEXITCODE" }
cargo test -p platform-windows value_write_readback_tests:: --lib --locked
if ($LASTEXITCODE -ne 0) { throw "ValuePattern read-back tests failed with exit $LASTEXITCODE" }
- name: Run pixel click transport regression
- name: Run pixel click transport regressions
working-directory: libs/cua-driver/rust
run: cargo test -p platform-windows post_message_pixel_click_reports_synthetic_transport --lib --locked
run: "cargo test -p platform-windows pixel_click_transport_tests:: --lib --locked"
- name: Run bounded UIA worker regressions
working-directory: libs/cua-driver/rust
run: "cargo test -p platform-windows windows_enum::tests:: --lib --locked"
- name: Run named-pipe readiness regression
working-directory: libs/cua-driver/rust
run: cargo test -p cua-driver-testkit stalled_pipe_attempt_is_bounded_and_does_not_block_the_next_probe --lib --locked
Expand Down
108 changes: 94 additions & 14 deletions libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2956,6 +2956,46 @@ fn posted_pixel_click_result(pid: u32, click_word: &str) -> ToolResult {
}))
}

/// Only a completed miss permits another transport. A provider may finish an
/// activation after the deadline, so errors must not replay the click.
fn finish_pixel_uia_attempt(
outcome: crate::uia::windows_enum::PointInvokeOutcome,
pid: u32,
sx: i32,
sy: i32,
) -> Option<ToolResult> {
use crate::uia::windows_enum::PointInvokeOutcome;
let status = match outcome {
PointInvokeOutcome::Miss => return None,
PointInvokeOutcome::Invoked => {
return Some(
ToolResult::text(format!(
"✅ Performed UIA Invoke at ({sx},{sy}) for pid {pid}."
))
.with_structured(json!({
"path": "ax", "verified": false, "effect": "unverifiable"
})),
);
}
PointInvokeOutcome::Busy => "busy",
PointInvokeOutcome::Timeout => "timeout",
PointInvokeOutcome::Unavailable => "unavailable",
};
Some(
ToolResult::error(format!(
"UIA pixel click {status} for pid {pid}. No fallback input was sent. \
The click effect is unknown; inspect the target state before another action."
))
.with_structured(json!({
"code": "background_unavailable",
"uia_status": status,
"path": "ax",
"verified": false,
"effect": "unverifiable"
})),
)
}

enum BackgroundElementClick {
Semantic {
message: String,
Expand Down Expand Up @@ -3850,7 +3890,7 @@ impl Tool for ClickTool {
// WebView2 / DirectComposition surfaces — those route
// input through `Windows.UI.Input`, not WM_LBUTTONDOWN.
//
// 2. If UIA returns false (no Invokable element under the
// 2. If UIA completes with a miss (no Invokable element under the
// pixel inside `hwnd`, or `hwnd` has no useful UIA
// tree at all), fall through to
// `PostMessage(WM_LBUTTONDOWN/UP)` against the deepest
Expand Down Expand Up @@ -3968,26 +4008,21 @@ impl Tool for ClickTool {
}
let use_uia = (btn == "left" || btn == "middle") && count == 1;
if use_uia {
let invoked = tokio::task::spawn_blocking(move || {
let outcome = tokio::task::spawn_blocking(move || {
crate::uia::windows_enum::try_invoke_in_window_at_point(
hwnd as isize,
sx as i32,
sy as i32,
)
})
.await
.unwrap_or(false);
if invoked {
return ToolResult::text(format!(
"✅ Performed UIA Invoke at ({sx},{sy}) for pid {pid}."
))
.with_structured(
json!({ "path": "ax", "verified": false, "effect": "unverifiable" }),
);
.unwrap_or(crate::uia::windows_enum::PointInvokeOutcome::Unavailable);
if let Some(result) = finish_pixel_uia_attempt(outcome, pid, sx_i, sy_i) {
return result;
}
}

// UIA did not land. Known dropped surfaces other than direct
// UIA completed with a miss or was not applicable. Known dropped surfaces other than direct
// Chromium (handled above) get one targeted injection attempt.
if delivery == DeliveryMode::Background
&& crate::input::delivery::would_be_silently_dropped(hwnd, EventKind::MouseClick)
Expand All @@ -4012,8 +4047,8 @@ impl Tool for ClickTool {
}

// delivery_mode:"background", non-dropped + non-UIA click. This
// includes plain Win32 targets and embedded Chromium when its UIA
// provider is temporarily unavailable. Attempt PostMessage without a
// includes plain Win32 targets with no invokable UIA element.
// Attempt PostMessage without a
// foreground swap; the caller must verify the resulting fixture state.
// Foreground was handled at the top, and known dropped surfaces use
// the targeted-injection path above.
Expand Down Expand Up @@ -4046,11 +4081,56 @@ impl Tool for ClickTool {

#[cfg(test)]
mod pixel_click_transport_tests {
use super::posted_pixel_click_result;
use super::{finish_pixel_uia_attempt, posted_pixel_click_result};
use crate::uia::windows_enum::PointInvokeOutcome;
use cua_driver_core::action_record::{
ActionExecutionRecord, ActionTransport, ActualDelivery, RequestedDelivery,
};

#[test]
fn pixel_uia_only_completed_miss_reaches_fallback_transport() {
for (outcome, status) in [
(PointInvokeOutcome::Invoked, None),
(PointInvokeOutcome::Busy, Some("busy")),
(PointInvokeOutcome::Timeout, Some("timeout")),
(PointInvokeOutcome::Unavailable, Some("unavailable")),
] {
// Exercise the same early-return boundary used before either
// targeted injection or PostMessage in the pixel click route.
let mut fallback_calls = 0;
let result = finish_pixel_uia_attempt(outcome, 42, 10, 20).unwrap_or_else(|| {
fallback_calls += 1;
posted_pixel_click_result(42, "click")
});
assert_eq!(fallback_calls, 0, "{outcome:?}");
assert_eq!(result.is_error.unwrap_or(false), status.is_some());
let data = result.structured_content.unwrap();
assert_eq!(data["path"], "ax");
assert_eq!(data["effect"], "unverifiable");
if let Some(status) = status {
assert_eq!(data["uia_status"], status);
assert_eq!(data["code"], "background_unavailable");
}
let record = ActionExecutionRecord::from_legacy(
"click",
&serde_json::json!({ "delivery_mode": "background" }),
&data,
)
.expect("UIA outcome should normalize into the public action contract");
let public = serde_json::to_value(record.public_result().expect("valid ActionResult"))
.expect("serialize ActionResult");
assert_eq!(public["effect"], "unverifiable");
}
let mut fallback_calls = 0;
let result =
finish_pixel_uia_attempt(PointInvokeOutcome::Miss, 42, 10, 20).unwrap_or_else(|| {
fallback_calls += 1;
posted_pixel_click_result(42, "click")
});
assert_eq!(fallback_calls, 1);
assert_eq!(result.structured_content.unwrap()["path"], "post_message");
}

#[test]
fn post_message_pixel_click_reports_synthetic_transport() {
let result = posted_pixel_click_result(42, "click");
Expand Down
Loading
Loading