diff --git a/libs/cua-driver-rs/Cargo.lock b/libs/cua-driver-rs/Cargo.lock index bd21ee23c8..8a6a7c6c8b 100644 --- a/libs/cua-driver-rs/Cargo.lock +++ b/libs/cua-driver-rs/Cargo.lock @@ -869,6 +869,7 @@ version = "0.2.18" dependencies = [ "anyhow", "async-trait", + "image", "serde", "serde_json", "thiserror", diff --git a/libs/cua-driver-rs/crates/cursor-overlay/src/lib.rs b/libs/cua-driver-rs/crates/cursor-overlay/src/lib.rs index d4349115a5..2669ef248d 100644 --- a/libs/cua-driver-rs/crates/cursor-overlay/src/lib.rs +++ b/libs/cua-driver-rs/crates/cursor-overlay/src/lib.rs @@ -15,12 +15,14 @@ pub mod path_planner; pub mod shape; pub mod capture_utils; pub mod util; +pub mod render_state; pub use palette::Palette; -pub use motion::MotionConfig; +pub use motion::{MotionConfig, Spring}; pub use bezier::CubicBezier; pub use path_planner::{PathPlanner, PlannedPath, PathState}; pub use shape::CursorShape; +pub use render_state::{RenderStateCore, FocusRect, render_frame, draw_default_arrow}; /// Configuration assembled from CLI arguments and passed to every /// platform backend when it initialises the overlay window. diff --git a/libs/cua-driver-rs/crates/cursor-overlay/src/motion.rs b/libs/cua-driver-rs/crates/cursor-overlay/src/motion.rs index 990d0b0ae3..59d9586730 100644 --- a/libs/cua-driver-rs/crates/cursor-overlay/src/motion.rs +++ b/libs/cua-driver-rs/crates/cursor-overlay/src/motion.rs @@ -82,3 +82,20 @@ impl MotionConfig { } } } + +/// Post-arrival spring physics state. +/// +/// When the cursor reaches the end of a planned path the engine +/// hands control to a spring-damper that overshoots a touch and +/// settles to the target. This struct holds the spring's mutable +/// state across ticks. Identical across all platform crates — was +/// duplicated 3× before the 2026-05 dedup audit. +/// +/// `(ox, oy)` = offset from the spring target; `(vx, vy)` = velocity. +#[derive(Clone, Copy, Default)] +pub struct Spring { + pub ox: f64, + pub oy: f64, + pub vx: f64, + pub vy: f64, +} diff --git a/libs/cua-driver-rs/crates/cursor-overlay/src/render_state.rs b/libs/cua-driver-rs/crates/cursor-overlay/src/render_state.rs new file mode 100644 index 0000000000..3011569eee --- /dev/null +++ b/libs/cua-driver-rs/crates/cursor-overlay/src/render_state.rs @@ -0,0 +1,734 @@ +//! Shared cursor-overlay render state, animation tick, and pixel pipeline. +//! +//! Lifts the platform-agnostic render state out of the three per-OS +//! `overlay.rs` files (macOS / Windows / Linux). Before the 2026-05 dedup +//! audit each platform owned a ~600-line copy of the same animation logic +//! that differed only in a few constants and feature flags. +//! +//! ## What lives here +//! +//! - [`RenderStateCore`] — the platform-agnostic animation fields +//! (`cfg`, `palette`, `motion`, `pos`, `heading`, `path`, `dist`, `spring`, +//! `spring_tgt`, `click_t`, `shape`, `visible`, `idle_secs`, `idle_alpha`, +//! `pinned_wid`, `gradient_colors`, `bloom_override`). +//! - [`RenderStateCore::tick_motion`] — speed-profile + spring physics + +//! click-pulse + idle-fade using runtime [`MotionConfig`] (Windows + Linux). +//! - [`RenderStateCore::tick_swift_constants`] — same physics but with the +//! hardcoded Swift reference constants used by macOS; returns whether the +//! path just ended (so the caller can fire arrival signals). +//! - [`RenderStateCore::apply_command_base`] — the OverlayCommand match arms +//! that all three platforms implement identically (MoveTo / ClickPulse / +//! SetEnabled / SetMotion / SetPalette / PinAbove / SetShape / SetGradient). +//! Returns `false` for variants the core doesn't handle so platforms can +//! layer their own behaviour on top (e.g. macOS ShowFocusRect). +//! - [`render_frame`] — the tiny-skia paint of bloom + click-pulse + arrow. +//! Parametrised by pixmap dimensions and an origin offset so Windows can +//! pass `(virt_x, virt_y)` while macOS / Linux pass `(0, 0)`. +//! - [`draw_default_arrow`] — gradient-arrow rasteriser. +//! +//! ## What stays per-platform +//! +//! - The OS window / surface (NSWindow / HWND / X11 Window) and its message +//! loop or run-loop. +//! - The paint dispatch: `dispatch_set_layer_contents` (CGImage), +//! `UpdateLayeredWindow` (BGRA DIB), `XPutImage` (BGRA ZPixmap). +//! - Origin/coordinate translation (Windows uses virtual-screen offset; +//! macOS uses NSScreen coordinates; Linux uses display coordinates). +//! - Platform-specific extras like macOS's `focus_rect` (post-arrival +//! element highlight — drawn inside [`render_frame`] when the caller +//! supplies one via the optional argument). + +use crate::{ + CursorConfig, CursorShape, MotionConfig, OverlayCommand, Palette, PathPlanner, PathState, + PlannedPath, Spring, +}; + +/// Platform-agnostic render state shared by macOS / Windows / Linux overlays. +/// +/// Each platform wraps this in its own struct that adds OS-specific fields +/// (e.g. `virt_x/y/w/h` on Windows, `focus_rect` on macOS). +pub struct RenderStateCore { + /// Frozen copy of the launch-time CursorConfig. + pub cfg: CursorConfig, + /// Current colour palette (mutable via [`OverlayCommand::SetPalette`]). + pub palette: Palette, + /// Current motion / timing config (mutable via [`OverlayCommand::SetMotion`]). + pub motion: MotionConfig, + /// Current rendered position in screen / overlay-window coordinates. + pub pos: (f64, f64), + /// Visual heading in radians (tip direction = motion_dir + π). + pub heading: f64, + /// In-flight planned path; `None` = at rest. + pub path: Option, + /// Arc-distance travelled along the current path so far. + pub dist: f64, + /// Post-arrival spring-settle state. + pub spring: Option, + /// Target the spring is settling toward: `(x, y, heading)`. + pub spring_tgt: Option<(f64, f64, f64)>, + /// Click-pulse phase 0..1; `None` = no pulse in flight. + pub click_t: Option, + /// Custom cursor shape; `None` = built-in gradient arrow. + pub shape: Option, + /// User-controlled visibility. + pub visible: bool, + /// Idle-hide: elapsed seconds since last activity. + pub idle_secs: f64, + /// Idle-hide fade: 1.0 = fully visible, 0.0 = fully hidden. + pub idle_alpha: f64, + /// Window id the overlay should be pinned above (for z-ordering). + pub pinned_wid: Option, + /// Runtime-overridden gradient colours (from `set_agent_cursor_style`). + /// Empty = use palette defaults. + pub gradient_colors: Vec<[u8; 4]>, + /// Runtime-overridden bloom colour. `None` = palette default. + pub bloom_override: Option<[u8; 4]>, +} + +impl RenderStateCore { + /// Build the core from a launch-time CursorConfig. + /// `pos` starts at the off-screen sentinel `(-200, -200)` to indicate + /// "never placed on screen yet" — the click path uses this to detect + /// first-placement and snap rather than animate. + pub fn new(cfg: CursorConfig) -> Self { + let palette = cfg.palette(); + let motion = cfg.motion.clone(); + let shape = cfg.shape.clone(); + Self { + cfg, + palette, + motion, + shape, + gradient_colors: vec![], + bloom_override: None, + pos: (-200.0, -200.0), + heading: std::f64::consts::FRAC_PI_4, + path: None, + dist: 0.0, + spring: None, + spring_tgt: None, + click_t: None, + visible: true, + idle_secs: 0.0, + idle_alpha: 1.0, + pinned_wid: None, + } + } + + /// Advance the animation by `dt` seconds using runtime [`MotionConfig`] + /// for peak / floor / spring constants. Used by Windows + Linux. + /// + /// The speed profile is `16·u²·(1-u)²` (peaks at 1.0 at u=0.5) — the + /// 1:1 port of `AgentCursorRenderer`'s smootherstep envelope. Floor + /// speed switches from `min_start_speed` to `min_end_speed` at the + /// midpoint so the cursor decelerates as it approaches the target. + /// Spring overshoot is `0.5` (Windows/Linux convention). + pub fn tick_motion(&mut self, dt: f64) { + let spring_k = self.motion.spring * 400.0; + let spring_c = self.motion.spring * 20.0; + + if let Some(ref p) = self.path { + let path_frac = (self.dist / p.length.max(1.0)).clamp(0.0, 1.0); + let profile = + 16.0 * path_frac * path_frac * (1.0 - path_frac) * (1.0 - path_frac); + let floor = if path_frac < 0.5 { + self.motion.min_start_speed + } else { + self.motion.min_end_speed + }; + let speed = (floor + (self.motion.peak_speed - floor) * profile).max(floor); + self.dist += speed * dt; + + let path_len = p.length.max(1.0); + if self.dist >= path_len { + let end = p.sample(path_len); + let end_heading = p.end_visual_heading; + let vh = end.heading; + self.spring = Some(Spring { + ox: 0.0, + oy: 0.0, + vx: speed * 0.5 * vh.cos(), + vy: speed * 0.5 * vh.sin(), + }); + self.spring_tgt = Some((end.x, end.y, end_heading)); + self.pos = (end.x, end.y); + self.heading = end_heading; + self.path = None; + self.dist = 0.0; + } else { + let s: PathState = p.sample(self.dist); + self.pos = (s.x, s.y); + let desired = s.heading + std::f64::consts::PI; + let max_step = 14.0 * dt; + self.heading = crate::util::rotate_toward(self.heading, desired, max_step); + } + } else if let Some(mut s) = self.spring { + if let Some((tx, ty, th)) = self.spring_tgt { + let substeps = 4; + let sdt = dt / substeps as f64; + for _ in 0..substeps { + s.vx += (-spring_k * s.ox - spring_c * s.vx) * sdt; + s.vy += (-spring_k * s.oy - spring_c * s.vy) * sdt; + s.ox += s.vx * sdt; + s.oy += s.vy * sdt; + } + self.pos = (tx + s.ox, ty + s.oy); + self.heading = th; + if s.ox.hypot(s.oy) < 0.3 && s.vx.hypot(s.vy) < 2.0 { + self.pos = (tx, ty); + self.spring = None; + } else { + self.spring = Some(s); + } + } + } + + if let Some(t) = self.click_t { + let next = t + dt * 4.0; + self.click_t = if next >= 1.0 { None } else { Some(next) }; + } + + self.tick_idle(dt); + } + + /// Advance the animation by `dt` seconds using the hardcoded Swift + /// reference constants (`peakSpeed=900`, `minStart=300`, `minEnd=200`, + /// `springK=400`, `springC=17`, `springOvershoot=0.8`). Used by macOS, + /// which mirrors `AgentCursorRenderer.swift` 1:1. + /// + /// Returns `true` when the path just ended (so the caller can fire its + /// arrival oneshot to unblock `animate_cursor_to`). + /// + /// The speed profile is `(30·u²·(1-u)²) / 1.875` which is algebraically + /// equivalent to the `16·u²·(1-u)²` form used by [`tick_motion`]; both + /// peak at 1.0 at u=0.5. The original Swift code uses the 30/1.875 + /// form so we preserve it here for parity. + pub fn tick_swift_constants(&mut self, dt: f64) -> bool { + const PEAK_SPEED: f64 = 900.0; + const MIN_START_SPEED: f64 = 300.0; + const MIN_END_SPEED: f64 = 200.0; + const SPRING_K: f64 = 400.0; + const SPRING_C: f64 = 17.0; + const SPRING_OVERSHOOT: f64 = 0.8; + + let mut fire_arrival = false; + + if let Some(ref p) = self.path { + let path_len = p.length.max(1.0); + let u = (self.dist / path_len).min(1.0); + + // Smootherstep speed profile (normalised: peak = 1.0). + let profile = (30.0 * u * u * (1.0 - u) * (1.0 - u)) / 1.875; + let floor_speed = if u < 0.5 { MIN_START_SPEED } else { MIN_END_SPEED }; + let current_speed = floor_speed + (PEAK_SPEED - floor_speed) * profile; + self.dist += current_speed * dt; + + if self.dist >= path_len { + // Transition to spring settle. + let end = p.sample(path_len); + let end_heading = p.end_visual_heading; + let vh = end.heading; + self.spring = Some(Spring { + ox: 0.0, + oy: 0.0, + vx: current_speed * SPRING_OVERSHOOT * vh.cos(), + vy: current_speed * SPRING_OVERSHOOT * vh.sin(), + }); + self.spring_tgt = Some((end.x, end.y, end_heading)); + self.pos = (end.x, end.y); + self.heading = end_heading; + self.path = None; + self.dist = 0.0; + fire_arrival = true; + } else { + let s: PathState = p.sample(self.dist); + self.pos = (s.x, s.y); + // Smooth heading rotation toward motion heading. + let desired = s.heading + std::f64::consts::PI; + let max_step = 14.0 * dt; + self.heading = crate::util::rotate_toward(self.heading, desired, max_step); + } + } else if let Some(mut s) = self.spring { + if let Some((tx, ty, th)) = self.spring_tgt { + let substeps = 4; + let sdt = dt / substeps as f64; + for _ in 0..substeps { + s.vx += (-SPRING_K * s.ox - SPRING_C * s.vx) * sdt; + s.vy += (-SPRING_K * s.oy - SPRING_C * s.vy) * sdt; + s.ox += s.vx * sdt; + s.oy += s.vy * sdt; + } + self.pos = (tx + s.ox, ty + s.oy); + self.heading = th; + if s.ox.hypot(s.oy) < 0.3 && s.vx.hypot(s.vy) < 2.0 { + self.pos = (tx, ty); + self.spring = None; + } else { + self.spring = Some(s); + } + } + } + + // Advance click pulse. + if let Some(t) = self.click_t { + let next = t + dt * 4.0; // full pulse over 0.25s + self.click_t = if next >= 1.0 { None } else { Some(next) }; + } + + self.tick_idle(dt); + + fire_arrival + } + + /// Shared idle-hide / fade logic — accumulate idle time when nothing is + /// moving, then fade `idle_alpha` from 1→0 over 180ms once + /// `motion.idle_hide_ms` has elapsed. Identical across all platforms. + fn tick_idle(&mut self, dt: f64) { + let idle_hide_ms = self.motion.idle_hide_ms; + if idle_hide_ms > 0.0 { + let moving = + self.path.is_some() || self.spring.is_some() || self.click_t.is_some(); + if moving { + self.idle_secs = 0.0; + self.idle_alpha = 1.0; + } else { + self.idle_secs += dt; + let fade_start = idle_hide_ms / 1000.0; + let fade_end = fade_start + 0.18; // 180ms fade like Windows ref + if self.idle_secs > fade_end { + self.idle_alpha = 0.0; + } else if self.idle_secs > fade_start { + let t = (self.idle_secs - fade_start) / 0.18; + self.idle_alpha = 1.0 - t.clamp(0.0, 1.0); + } + } + } else { + self.idle_alpha = 1.0; + } + } + + /// Handle the OverlayCommand variants that are identical across all + /// three platforms. Returns `true` if the command was consumed; `false` + /// for variants the platform must handle itself (e.g. macOS's + /// `ShowFocusRect`). + /// + /// `move_to_snap_sentinel` controls macOS-only behaviour: when `true`, + /// `MoveTo` snaps `self.pos` to the offset target if the cursor is + /// still at the off-screen sentinel (`pos.0 < -50.0`). Windows/Linux + /// pass `false` here. + /// + /// `click_pulse_sentinel_only` likewise controls macOS-only behaviour: + /// when `true`, `ClickPulse` only updates `self.pos` if the cursor is + /// still at the sentinel (the animation already landed it there + /// otherwise). Windows/Linux pass `false`, which always snaps + /// `self.pos` to the click point. + pub fn apply_command_base( + &mut self, + cmd: OverlayCommand, + move_to_snap_sentinel: bool, + click_pulse_sentinel_only: bool, + ) -> bool { + match cmd { + OverlayCommand::MoveTo { + x, + y, + end_heading_radians, + } => { + // Apply click offset (16 pt along end_heading) before planning, + // matching Swift `moveTo(point:endAngleRadians:)`: + // tx = clickPoint.x + cos(endAngle) * clickOffset + // ty = clickPoint.y + sin(endAngle) * clickOffset + const CLICK_OFFSET: f64 = 16.0; + const TURN_RADIUS: f64 = 80.0; + let tx = x + end_heading_radians.cos() * CLICK_OFFSET; + let ty = y + end_heading_radians.sin() * CLICK_OFFSET; + + // macOS-only: if the cursor is still at the initial off-screen + // sentinel, snap it to the offset target so the path starts on-screen. + if move_to_snap_sentinel && self.pos.0 < -50.0 { + self.pos = (tx, ty); + } + let (x0, y0) = self.pos; + let th0 = self.heading + std::f64::consts::PI; + let th1 = end_heading_radians + std::f64::consts::PI; + let plan = PathPlanner::plan( + x0, + y0, + th0, + tx, + ty, + th1, + end_heading_radians, + TURN_RADIUS, + ); + self.path = Some(plan); + self.dist = 0.0; + self.spring = None; + self.spring_tgt = None; + self.idle_secs = 0.0; + self.idle_alpha = 1.0; + true + } + OverlayCommand::ClickPulse { x, y } => { + if click_pulse_sentinel_only { + // macOS: only snap position on first placement (sentinel state). + // After that the cursor stays where the animation landed. + if self.pos.0 < -50.0 { + // Apply same click offset so tip lands at click point. + const CLICK_OFFSET: f64 = 16.0; + let angle = std::f64::consts::FRAC_PI_4; + self.pos = ( + x + angle.cos() * CLICK_OFFSET, + y + angle.sin() * CLICK_OFFSET, + ); + } + } else { + self.pos = (x, y); + } + self.click_t = Some(0.0); + self.idle_secs = 0.0; + self.idle_alpha = 1.0; + true + } + OverlayCommand::SetEnabled(v) => { + self.visible = v; + true + } + OverlayCommand::SetMotion(m) => { + self.motion = m; + true + } + OverlayCommand::SetPalette(p) => { + self.palette = p; + true + } + OverlayCommand::PinAbove(wid) => { + self.pinned_wid = Some(wid); + true + } + OverlayCommand::SetShape(shape) => { + self.shape = shape; + true + } + OverlayCommand::SetGradient { + gradient_colors, + bloom_color, + } => { + self.gradient_colors = gradient_colors; + self.bloom_override = bloom_color; + true + } + OverlayCommand::ShowFocusRect(_) => false, // caller-specific + } + } +} + +// ── tiny-skia rendering ────────────────────────────────────────────────── + +/// Optional focus-rect overlay drawn on top of the cursor (macOS only at +/// the moment — the other platforms always pass `None`). +#[derive(Clone, Copy)] +pub struct FocusRect { + /// Rectangle `[x, y, w, h]` in screen coordinates (top-left origin), + /// relative to the same origin the cursor `pos` uses. + pub rect: [f64; 4], + /// Fade progress 0.0 = fully visible, 1.0 = gone. + pub t: f64, +} + +/// Render the cursor + bloom + click-pulse + (optional) focus-rect into a +/// fresh tiny-skia [`tiny_skia::Pixmap`] of `(width, height)`. +/// +/// `origin_x`, `origin_y` are subtracted from the cursor `core.pos` before +/// drawing — Windows passes the virtual-screen `(virt_x, virt_y)` so the +/// pixmap is laid out in window-local coordinates. macOS / Linux pass +/// `(0.0, 0.0)`. +pub fn render_frame( + core: &RenderStateCore, + width: u32, + height: u32, + origin_x: f64, + origin_y: f64, + focus_rect: Option, +) -> tiny_skia::Pixmap { + let w = width.max(1); + let h = height.max(1); + let mut pm = tiny_skia::Pixmap::new(w, h) + .unwrap_or_else(|| tiny_skia::Pixmap::new(1, 1).unwrap()); + + if !core.visible || core.pos.0 < -100.0 || core.idle_alpha < 0.004 { + return pm; + } + + let (px, py) = (core.pos.0 - origin_x, core.pos.1 - origin_y); + let heading = core.heading; + let alpha_scale = core.idle_alpha as f32; + + // --- Bloom (radial gradient behind the arrow) --- + let bloom_r: f32 = 22.0; + // Use runtime bloom_override if set, otherwise fall back to palette. + let (br, bg, bb) = if let Some([r, g, b, _]) = core.bloom_override { + (r, g, b) + } else { + let [r, g, b, _] = core.palette.bloom_inner; + (r, g, b) + }; + let bloom_inner = tiny_skia::Color::from_rgba8(br, bg, bb, (115.0 * alpha_scale) as u8); + let (or_, og, ob) = if let Some([r, g, b, _]) = core.bloom_override { + (r, g, b) + } else { + let [r, g, b, _] = core.palette.bloom_outer; + (r, g, b) + }; + let bloom_outer = tiny_skia::Color::from_rgba8(or_, og, ob, (26.0 * alpha_scale) as u8); + let bloom_zero = tiny_skia::Color::from_rgba8(or_, og, ob, 0); + + let bloom_paint = { + let mut p = tiny_skia::Paint::default(); + p.shader = tiny_skia::RadialGradient::new( + tiny_skia::Point::from_xy(px as f32, py as f32), + tiny_skia::Point::from_xy(px as f32, py as f32), // focal = center + bloom_r, + vec![ + tiny_skia::GradientStop::new(0.0, bloom_inner), + tiny_skia::GradientStop::new(0.5, bloom_outer), + tiny_skia::GradientStop::new(1.0, bloom_zero), + ], + tiny_skia::SpreadMode::Pad, + tiny_skia::Transform::identity(), + ) + .unwrap_or(tiny_skia::Shader::SolidColor(bloom_inner)); + p.anti_alias = true; + p + }; + + if let Some(r) = tiny_skia::Rect::from_xywh( + (px - bloom_r as f64) as f32, + (py - bloom_r as f64) as f32, + bloom_r * 2.0, + bloom_r * 2.0, + ) { + pm.fill_rect(r, &bloom_paint, tiny_skia::Transform::identity(), None); + } + + // --- Focus rect highlight (macOS only — others pass None) --- + // Cyan glow border + faint fill, matching Swift AgentCursor.showFocusRect. + if let Some(fr) = focus_rect { + let [fx, fy, fw, fh] = fr.rect; + let t = fr.t as f32; + let fade = (1.0 - t) * (1.0 - t); // quadratic ease-out + let border_a = (230.0 * fade * alpha_scale) as u8; + let fill_a = (20.0 * fade * alpha_scale) as u8; + // Cyan: #5EC0E8 + let (cr, cg, cb) = (0x5Eu8, 0xC0u8, 0xE8u8); + + if let Some(rect) = + tiny_skia::Rect::from_xywh(fx as f32, fy as f32, fw as f32, fh as f32) + { + // Faint fill + let mut fill_paint = tiny_skia::Paint::default(); + fill_paint.shader = tiny_skia::Shader::SolidColor( + tiny_skia::Color::from_rgba8(cr, cg, cb, fill_a), + ); + pm.fill_rect(rect, &fill_paint, tiny_skia::Transform::identity(), None); + + // Border stroke (2px glow) + let mut border_paint = tiny_skia::Paint::default(); + border_paint.shader = tiny_skia::Shader::SolidColor( + tiny_skia::Color::from_rgba8(cr, cg, cb, border_a), + ); + border_paint.anti_alias = true; + let stroke = tiny_skia::Stroke { + width: 2.5, + ..Default::default() + }; + let mut pb = tiny_skia::PathBuilder::new(); + pb.push_rect(rect); + if let Some(path) = pb.finish() { + pm.stroke_path( + &path, + &border_paint, + &stroke, + tiny_skia::Transform::identity(), + None, + ); + } + } + } + + // --- Click pulse ring --- + if let Some(t) = core.click_t { + let ring_r = (bloom_r + 20.0 * t as f32) * (1.0 - t as f32 * 0.5); + let alpha = ((1.0 - t) * 180.0 * alpha_scale as f64) as u8; + let [cr, cg, cb, _] = core.palette.cursor_mid; + let ring_color = tiny_skia::Color::from_rgba8(cr, cg, cb, alpha); + let mut ring_paint = tiny_skia::Paint::default(); + ring_paint.shader = tiny_skia::Shader::SolidColor(ring_color); + ring_paint.anti_alias = true; + let stroke = tiny_skia::Stroke { + width: 2.0, + ..Default::default() + }; + let mut pb = tiny_skia::PathBuilder::new(); + pb.push_circle(px as f32, py as f32, ring_r); + if let Some(path) = pb.finish() { + pm.stroke_path( + &path, + &ring_paint, + &stroke, + tiny_skia::Transform::identity(), + None, + ); + } + } + + // --- Arrow (custom shape or default gradient arrow) --- + if let Some(ref shape) = core.shape { + // Custom icon: draw as a 32×32 image centered at (px, py), opacity-faded. + let sz = 32.0_f32; + if let Some(pix) = + tiny_skia::PixmapRef::from_bytes(&shape.pixels, shape.width, shape.height) + { + let transform = tiny_skia::Transform::from_rotate_at( + heading.to_degrees() as f32 + 180.0, + px as f32, + py as f32, + ) + .pre_translate(px as f32 - sz / 2.0, py as f32 - sz / 2.0); + let mut paint = tiny_skia::PixmapPaint::default(); + paint.opacity = alpha_scale; + pm.draw_pixmap(0, 0, pix, &paint, transform, None); + } + } else { + let grad_override = if core.gradient_colors.is_empty() { + None + } else { + Some(&core.gradient_colors) + }; + draw_default_arrow( + &mut pm, + &core.palette, + grad_override, + px as f32, + py as f32, + heading as f32, + alpha_scale, + ); + } + + pm +} + +/// Rasterise the built-in gradient arrow at `(px, py)` rotated by +/// `heading` radians. `alpha_scale` is the idle-fade multiplier +/// (1.0 = fully opaque, 0.0 = fully faded out). +/// +/// `gradient_override` lets `set_agent_cursor_style` substitute custom +/// gradient stops at runtime. When `None` the palette's +/// `cursor_start/cursor_mid/cursor_end` are used. +pub fn draw_default_arrow( + pm: &mut tiny_skia::Pixmap, + palette: &Palette, + gradient_override: Option<&Vec<[u8; 4]>>, + px: f32, + py: f32, + heading: f32, + alpha_scale: f32, +) { + // Arrow vertices (tip at +x). + let verts: [(f32, f32); 4] = [(14.0, 0.0), (-8.0, -9.0), (-3.0, 0.0), (-8.0, 9.0)]; + + // Rotate by (heading + π) so tip points in the motion direction. + let angle = heading + std::f64::consts::PI as f32; + let (sa, ca) = (angle.sin(), angle.cos()); + let transform_pt = |(vx, vy): (f32, f32)| -> (f32, f32) { + (px + ca * vx - sa * vy, py + sa * vx + ca * vy) + }; + + let pts: Vec<(f32, f32)> = verts.iter().map(|&v| transform_pt(v)).collect(); + + let mut pb = tiny_skia::PathBuilder::new(); + pb.move_to(pts[0].0, pts[0].1); + for p in &pts[1..] { + pb.line_to(p.0, p.1); + } + pb.close(); + let arrow_path = match pb.finish() { + Some(p) => p, + None => return, + }; + + // Gradient fill: start color at tip, end color at tail. + // Use runtime overrides when available, otherwise fall back to palette. + let tip = pts[0]; + let tail = ( + (pts[1].0 + pts[3].0) / 2.0, + (pts[1].1 + pts[3].1) / 2.0, + ); + let (r0, g0, b0) = if let Some(g) = gradient_override.and_then(|g| g.first()) { + (g[0], g[1], g[2]) + } else { + let [r, g, b, _] = palette.cursor_start; + (r, g, b) + }; + let (r1, g1, b1) = if let Some(g) = + gradient_override.and_then(|g| g.get(1).or_else(|| g.first())) + { + (g[0], g[1], g[2]) + } else { + let [r, g, b, _] = palette.cursor_mid; + (r, g, b) + }; + let (r2, g2, b2) = if let Some(g) = gradient_override.and_then(|g| g.last()) { + (g[0], g[1], g[2]) + } else { + let [r, g, b, _] = palette.cursor_end; + (r, g, b) + }; + + let a = (255.0 * alpha_scale) as u8; + let fill_paint = { + let mut p = tiny_skia::Paint::default(); + p.shader = tiny_skia::LinearGradient::new( + tiny_skia::Point::from_xy(tip.0, tip.1), + tiny_skia::Point::from_xy(tail.0, tail.1), + vec![ + tiny_skia::GradientStop::new(0.00, tiny_skia::Color::from_rgba8(r0, g0, b0, a)), + tiny_skia::GradientStop::new(0.53, tiny_skia::Color::from_rgba8(r1, g1, b1, a)), + tiny_skia::GradientStop::new(1.00, tiny_skia::Color::from_rgba8(r2, g2, b2, a)), + ], + tiny_skia::SpreadMode::Pad, + tiny_skia::Transform::identity(), + ) + .unwrap_or(tiny_skia::Shader::SolidColor( + tiny_skia::Color::from_rgba8(r1, g1, b1, a), + )); + p.anti_alias = true; + p + }; + + pm.fill_path( + &arrow_path, + &fill_paint, + tiny_skia::FillRule::Winding, + tiny_skia::Transform::identity(), + None, + ); + + // White outline (faded with alpha_scale). + let mut stroke_paint = tiny_skia::Paint::default(); + stroke_paint.shader = + tiny_skia::Shader::SolidColor(tiny_skia::Color::from_rgba8(255, 255, 255, a)); + stroke_paint.anti_alias = true; + let stroke = tiny_skia::Stroke { + width: 1.5, + ..Default::default() + }; + pm.stroke_path( + &arrow_path, + &stroke_paint, + &stroke, + tiny_skia::Transform::identity(), + None, + ); +} diff --git a/libs/cua-driver-rs/crates/mcp-server/Cargo.toml b/libs/cua-driver-rs/crates/mcp-server/Cargo.toml index 3594790e24..d3b541aa58 100644 --- a/libs/cua-driver-rs/crates/mcp-server/Cargo.toml +++ b/libs/cua-driver-rs/crates/mcp-server/Cargo.toml @@ -11,3 +11,7 @@ anyhow = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } async-trait = "0.1" +# Used by the shared `image_utils` module — pure PNG/JPEG/resize/crosshair +# helpers extracted from `platform-{macos,windows,linux}/src/capture.rs` so +# all three platforms call the same code path. +image = { workspace = true } diff --git a/libs/cua-driver-rs/crates/mcp-server/src/element_cache.rs b/libs/cua-driver-rs/crates/mcp-server/src/element_cache.rs new file mode 100644 index 0000000000..5c3e9cf22b --- /dev/null +++ b/libs/cua-driver-rs/crates/mcp-server/src/element_cache.rs @@ -0,0 +1,159 @@ +//! Generic locked-HashMap plumbing for per-platform element caches. +//! +//! Each platform crate stores a per-(pid, window) snapshot of the +//! actionable accessibility elements it last walked, so subsequent +//! tool calls (`click`, `type_text`, etc.) can resolve an +//! `element_index` back to a native handle without re-walking the +//! tree. Before this module the three crates each owned a +//! near-identical `Mutex>` plus +//! the same insert / lookup / count methods — see +//! `docs/dedup-audit.md` item #3. +//! +//! What lives here: the lock + HashMap, generic over the caller's +//! key type `K` and snapshot type `S`. What stays per-platform: +//! +//! - the concrete `CacheKey` (different pid/window-id widths) +//! - the concrete `CachedSnapshot` and its `Drop` impl, which on +//! macOS calls `CFRelease` on every cached AXUIElementRef and on +//! Windows calls COM `Release` on every IUIAutomationElement. +//! Those releases are load-bearing — dropping the wrong way leaks +//! the AX / UIA handles. The generic core just stores `S` by +//! value; when the entry is removed or replaced (via +//! `HashMap::insert`'s replace-return semantics) Rust runs `S`'s +//! destructor, so the platform `Drop` impl still fires exactly as +//! it did pre-refactor. +//! +//! Each platform's `ElementCache` is now a thin wrapper around an +//! `ElementCacheCore`. Specialised +//! accessors (`get_element_ptr`, `get_element_center` on Windows, +//! `get_element_key` on Linux) call `with_snapshot` and project the +//! field they care about. + +use std::collections::HashMap; +use std::hash::Hash; +use std::sync::Mutex; + +/// Generic locked-HashMap holding one snapshot of type `S` per key. +/// +/// The `K: Clone` bound lets the wrapper build a key value on the +/// stack and pass `&K` in for lookups without forcing callers to +/// hand out owned keys on every read. +pub struct ElementCacheCore { + inner: Mutex>, +} + +impl ElementCacheCore { + pub fn new() -> Self { + Self { inner: Mutex::new(HashMap::new()) } + } + + /// Replace the snapshot for `key`. If an entry already existed + /// its destructor runs here — for macOS that fires `CFRelease` + /// on every retained AXUIElementRef, for Windows it fires COM + /// `Release`, for Linux it's a no-op (just frees the `Vec`). + pub fn insert(&self, key: K, snapshot: S) { + let mut inner = self.inner.lock().unwrap(); + inner.insert(key, snapshot); + } + + /// Run `f` against the snapshot for `key` while the lock is + /// held. Returns `None` if there is no entry. + pub fn with_snapshot(&self, key: &K, f: impl FnOnce(&S) -> R) -> Option { + let inner = self.inner.lock().unwrap(); + inner.get(key).map(f) + } + + /// Drop the snapshot for `key` if present. + #[allow(dead_code)] + pub fn remove(&self, key: &K) { + let mut inner = self.inner.lock().unwrap(); + inner.remove(key); + } +} + +impl Default for ElementCacheCore { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + struct TestKey { + pid: u32, + window_id: u64, + } + + struct TestSnapshot { + elements: Vec, + } + + #[test] + fn insert_then_get_returns_projection() { + let cache: ElementCacheCore = ElementCacheCore::new(); + let key = TestKey { pid: 42, window_id: 7 }; + cache.insert(key, TestSnapshot { elements: vec![10, 20, 30] }); + + let third = cache.with_snapshot(&key, |s| s.elements.get(2).copied()); + assert_eq!(third, Some(Some(30))); + } + + #[test] + fn miss_returns_none() { + let cache: ElementCacheCore = ElementCacheCore::new(); + let key = TestKey { pid: 1, window_id: 1 }; + let v = cache.with_snapshot(&key, |s| s.elements.len()); + assert_eq!(v, None); + } + + #[test] + fn count_via_with_snapshot() { + let cache: ElementCacheCore = ElementCacheCore::new(); + let key = TestKey { pid: 9, window_id: 99 }; + + // Before insert: None. + assert_eq!(cache.with_snapshot(&key, |s| s.elements.len()), None); + + cache.insert(key, TestSnapshot { elements: vec![1, 2, 3, 4, 5] }); + assert_eq!(cache.with_snapshot(&key, |s| s.elements.len()), Some(5)); + } + + #[test] + fn insert_replaces_existing_and_runs_drop() { + // Smoke test that re-inserting under the same key replaces + // the prior snapshot — the platform Drop fires here in real + // code (CFRelease/COM Release). We can't exercise FFI in a + // unit test, but we can confirm the value was replaced. + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + + struct DropCounter { + counter: Arc, + } + impl Drop for DropCounter { + fn drop(&mut self) { + self.counter.fetch_add(1, Ordering::SeqCst); + } + } + + let drops = Arc::new(AtomicUsize::new(0)); + let cache: ElementCacheCore = ElementCacheCore::new(); + + cache.insert(1, DropCounter { counter: drops.clone() }); + assert_eq!(drops.load(Ordering::SeqCst), 0); + + cache.insert(1, DropCounter { counter: drops.clone() }); + assert_eq!(drops.load(Ordering::SeqCst), 1, "replacement should drop the prior snapshot"); + + cache.remove(&1); + assert_eq!(drops.load(Ordering::SeqCst), 2, "remove should drop the snapshot"); + } + + #[test] + fn default_impl_matches_new() { + let _cache: ElementCacheCore = ElementCacheCore::default(); + } +} diff --git a/libs/cua-driver-rs/crates/mcp-server/src/image_utils.rs b/libs/cua-driver-rs/crates/mcp-server/src/image_utils.rs new file mode 100644 index 0000000000..3d141ae90f --- /dev/null +++ b/libs/cua-driver-rs/crates/mcp-server/src/image_utils.rs @@ -0,0 +1,348 @@ +//! Cross-platform PNG / JPEG / crosshair / resize helpers. +//! +//! These functions are pure consumers of the [`image`] crate with no +//! platform-specific dependencies, so they were perfect candidates for +//! deduplication. Until 2026-05 they lived as near-identical copies in +//! `platform-{macos,windows,linux}/src/capture.rs` — see +//! [`CUA_DRIVER_RS_DEDUP_AUDIT.md`] for the full audit trail. +//! +//! Each platform's `capture.rs` still owns its own +//! `screenshot_window_bytes` / `screenshot_display_bytes` (CGImage on +//! macOS, BitBlt+PrintWindow on Windows, XGetImage / ImageMagick +//! `import` on Linux). Everything downstream of "I have RGBA pixels" — +//! PNG encoding, JPEG encoding, downscaling to a max long edge, drawing +//! a crosshair, reading width/height from an IHDR — is here. +//! +//! No public API removed: the platform `capture.rs` modules re-export +//! the same function names so existing callers keep compiling. + +use anyhow::{anyhow, bail, Result}; +use image::{ColorType, DynamicImage, ImageBuffer, ImageDecoder, ImageFormat}; + +// ── PNG → JPEG ──────────────────────────────────────────────────────────── + +/// Convert raw PNG bytes to JPEG at the given quality (1-95). +/// +/// Strips alpha (RGBA → RGB) since JPEG doesn't carry alpha. Quality is +/// clamped to the encoder's accepted range by the underlying `image` +/// crate. +pub fn png_bytes_to_jpeg(png_bytes: &[u8], quality: u8) -> Result> { + let cursor = std::io::Cursor::new(png_bytes); + let decoder = image::codecs::png::PngDecoder::new(cursor)?; + let (w, h) = decoder.dimensions(); + let color = decoder.color_type(); + let mut buf = vec![0u8; decoder.total_bytes() as usize]; + decoder.read_image(&mut buf)?; + + // RGBA → RGB if needed (drop the alpha channel — JPEG can't store it). + let rgb_buf: Vec = if color == ColorType::Rgba8 { + buf.chunks_exact(4) + .flat_map(|px| [px[0], px[1], px[2]]) + .collect() + } else { + buf + }; + + let mut jpeg_bytes = Vec::new(); + let mut enc = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg_bytes, quality); + enc.encode(&rgb_buf, w, h, ColorType::Rgb8.into())?; + Ok(jpeg_bytes) +} + +// ── Downscale ───────────────────────────────────────────────────────────── + +/// Downscale `png_bytes` so neither dimension exceeds `max_dim`. +/// +/// `max_dim == 0` is treated as "no cap"; the original bytes are +/// returned unchanged. Aspect ratio is preserved. The resampler is +/// Lanczos3 — same choice the platform crates made before extraction. +pub fn resize_png_if_needed(png_bytes: &[u8], max_dim: u32) -> Result> { + if max_dim == 0 { + return Ok(png_bytes.to_vec()); + } + let (w, h) = png_dimensions(png_bytes)?; + if w <= max_dim && h <= max_dim { + return Ok(png_bytes.to_vec()); + } + let scale = (max_dim as f64) / (w.max(h) as f64); + let new_w = (w as f64 * scale).round() as u32; + let new_h = (h as f64 * scale).round() as u32; + + let cursor = std::io::Cursor::new(png_bytes); + let decoder = image::codecs::png::PngDecoder::new(cursor)?; + let color = decoder.color_type(); + let mut buf = vec![0u8; decoder.total_bytes() as usize]; + decoder.read_image(&mut buf)?; + + let img = match color { + ColorType::Rgba8 => DynamicImage::ImageRgba8( + ImageBuffer::from_raw(w, h, buf).ok_or_else(|| anyhow!("invalid RGBA buffer"))?, + ), + ColorType::Rgb8 => DynamicImage::ImageRgb8( + ImageBuffer::from_raw(w, h, buf).ok_or_else(|| anyhow!("invalid RGB buffer"))?, + ), + _ => bail!("unsupported color type for resize: {color:?}"), + }; + + let resized = img.resize(new_w, new_h, image::imageops::FilterType::Lanczos3); + let mut out = Vec::new(); + resized.write_to(&mut std::io::Cursor::new(&mut out), ImageFormat::Png)?; + Ok(out) +} + +// ── Crosshair ───────────────────────────────────────────────────────────── + +/// Draw a red crosshair at pixel (cx, cy) on a PNG and write it to `path`. +/// Used by `click`'s `debug_image_out` param to verify coordinate spaces. +/// The crosshair uses top-left-origin coords matching the click tool's +/// convention. +/// +/// Tilde-expands `~` in `path` to `$HOME` (or `%USERPROFILE%` via the same +/// env-var name on Windows). +pub fn write_crosshair_png(png_bytes: &[u8], cx: f64, cy: f64, path: &str) -> Result<()> { + let mut img = decode_png_to_rgba8(png_bytes)?; + draw_crosshair(&mut img, cx, cy); + + let path = if let Some(rest) = path.strip_prefix('~') { + let home = std::env::var("HOME") + .or_else(|_| std::env::var("USERPROFILE")) + .map_err(|_| anyhow::anyhow!( + "Cannot expand `~` in path {path:?}: neither HOME nor USERPROFILE is set"))?; + if home.is_empty() { + anyhow::bail!("Cannot expand `~` in path {path:?}: HOME/USERPROFILE is empty"); + } + format!("{home}{rest}") + } else { + path.to_owned() + }; + img.save_with_format(&path, ImageFormat::Png)?; + Ok(()) +} + +/// Draw a red crosshair at pixel (cx, cy) and return the modified PNG bytes. +/// Used by the recording subsystem's click-marker callback to produce +/// `click.png` without a temp file. +pub fn crosshair_png_bytes(png_bytes: &[u8], cx: f64, cy: f64) -> Result> { + let mut img = decode_png_to_rgba8(png_bytes)?; + draw_crosshair(&mut img, cx, cy); + + let mut out = Vec::new(); + DynamicImage::ImageRgba8(img) + .write_to(&mut std::io::Cursor::new(&mut out), ImageFormat::Png)?; + Ok(out) +} + +/// Internal: decode PNG to a mutable RGBA8 image buffer. +fn decode_png_to_rgba8(png_bytes: &[u8]) -> Result, Vec>> { + let cursor = std::io::Cursor::new(png_bytes); + let decoder = image::codecs::png::PngDecoder::new(cursor)?; + let (w, h) = decoder.dimensions(); + let color = decoder.color_type(); + let mut buf = vec![0u8; decoder.total_bytes() as usize]; + decoder.read_image(&mut buf)?; + + let img: DynamicImage = match color { + ColorType::Rgba8 => DynamicImage::ImageRgba8( + ImageBuffer::from_raw(w, h, buf).ok_or_else(|| anyhow!("invalid RGBA buffer"))?, + ), + ColorType::Rgb8 => DynamicImage::ImageRgb8( + ImageBuffer::from_raw(w, h, buf).ok_or_else(|| anyhow!("invalid RGB buffer"))?, + ), + _ => bail!("unsupported color type for crosshair: {color:?}"), + }; + Ok(img.to_rgba8()) +} + +/// Internal: draw the red ring + horizontal + vertical arms in place. +fn draw_crosshair(img: &mut ImageBuffer, Vec>, cx: f64, cy: f64) { + let w = img.width(); + let h = img.height(); + + let ring_r = (w as f64 / 80.0).max(6.0) as i32; + let arm_len = (w as f64 / 40.0).max(12.0) as i32; + let line_w = ((w as f64 / 400.0).max(1.5)) as i32; + let red = image::Rgba([255u8, 26, 26, 242]); + let cx = cx as i32; + let cy = cy as i32; + + // Horizontal + vertical arms. + for lw in 0..=line_w { + let off = lw - line_w / 2; + for dx in -arm_len..=arm_len { + if let Some(p) = img.get_pixel_mut_checked( + (cx + dx).clamp(0, w as i32 - 1) as u32, + (cy + off).clamp(0, h as i32 - 1) as u32, + ) { + *p = red; + } + } + for dy in -arm_len..=arm_len { + if let Some(p) = img.get_pixel_mut_checked( + (cx + off).clamp(0, w as i32 - 1) as u32, + (cy + dy).clamp(0, h as i32 - 1) as u32, + ) { + *p = red; + } + } + } + + // Stroked ring. + let steps = (ring_r * 12).max(48) as usize; + for i in 0..steps { + let theta = 2.0 * std::f64::consts::PI * i as f64 / steps as f64; + let rx = (cx as f64 + ring_r as f64 * theta.cos()) as i32; + let ry = (cy as f64 + ring_r as f64 * theta.sin()) as i32; + for lw in 0..=line_w { + let off = lw - line_w / 2; + for dx in 0..=1 { + let fx = (rx + off + dx).clamp(0, w as i32 - 1) as u32; + let fy = (ry + off).clamp(0, h as i32 - 1) as u32; + *img.get_pixel_mut(fx, fy) = red; + } + } + } +} + +// ── PNG dimensions ──────────────────────────────────────────────────────── + +/// Parse width and height from a PNG file's IHDR chunk. +/// +/// Pure byte-slice parsing — does not depend on the `image` crate's +/// decoder. Used by `resize_png_if_needed`'s fast-path size check. +pub fn png_dimensions(data: &[u8]) -> Result<(u32, u32)> { + // PNG signature: 8 bytes. + // Then IHDR chunk: 4-byte length, 4-byte "IHDR", 4-byte width, 4-byte height. + if data.len() < 24 { + bail!("PNG data too small"); + } + const PNG_SIG: [u8; 8] = [137, 80, 78, 71, 13, 10, 26, 10]; + if data[..8] != PNG_SIG { + bail!("not a PNG: signature mismatch"); + } + if &data[12..16] != b"IHDR" { + bail!("not a PNG: missing IHDR chunk"); + } + let w = u32::from_be_bytes([data[16], data[17], data[18], data[19]]); + let h = u32::from_be_bytes([data[20], data[21], data[22], data[23]]); + Ok((w, h)) +} + +// ── Raw RGBA → PNG (used by Windows + Linux capture paths) ──────────────── + +/// Encode raw RGBA bytes (top-down, row-major) as a PNG. +/// +/// Uses the `image` crate's encoder rather than the hand-rolled +/// uncompressed-PNG path that Windows + Linux previously carried — that +/// path was a workspace-local optimization that traded ~5x larger output +/// for less code, but the `image` crate's encoder is fast enough (it +/// already ships in every consumer of `image`, no new code paths) and +/// the smaller output saves more bytes downstream than the encode cost. +/// +/// Caller guarantees `rgba.len() == w * h * 4`. +pub fn encode_rgba_to_png(rgba: &[u8], w: u32, h: u32) -> Result> { + if rgba.len() as u64 != (w as u64) * (h as u64) * 4 { + bail!( + "encode_rgba_to_png: buffer size {} != w({w}) * h({h}) * 4", + rgba.len() + ); + } + let buf: ImageBuffer, Vec> = + ImageBuffer::from_raw(w, h, rgba.to_vec()) + .ok_or_else(|| anyhow!("invalid RGBA buffer for w={w} h={h}"))?; + let mut out = Vec::new(); + DynamicImage::ImageRgba8(buf).write_to(&mut std::io::Cursor::new(&mut out), ImageFormat::Png)?; + Ok(out) +} + +/// Encode raw BGRA bytes (top-down, row-major) as a PNG. +/// +/// Windows GDI gives us BGRA; we swap channels in-place then defer to +/// [`encode_rgba_to_png`]. Caller guarantees the buffer's size invariant. +pub fn encode_bgra_to_png(bgra: &[u8], w: u32, h: u32) -> Result> { + let mut rgba = bgra.to_vec(); + for px in rgba.chunks_exact_mut(4) { + px.swap(0, 2); // B ↔ R + } + encode_rgba_to_png(&rgba, w, h) +} + +// ── Tests ───────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn png_dimensions_round_trip() { + // Build a tiny 3x2 RGBA image then read its dimensions back. + let rgba = vec![0xFFu8; 3 * 2 * 4]; + let png = encode_rgba_to_png(&rgba, 3, 2).expect("encode"); + let (w, h) = png_dimensions(&png).expect("dimensions"); + assert_eq!((w, h), (3, 2)); + } + + #[test] + fn png_dimensions_rejects_non_png() { + assert!(png_dimensions(&[0; 100]).is_err()); + assert!(png_dimensions(b"not a PNG file at all").is_err()); + assert!(png_dimensions(&[]).is_err()); + } + + #[test] + fn resize_no_op_when_within_bound() { + let rgba = vec![0u8; 10 * 10 * 4]; + let png = encode_rgba_to_png(&rgba, 10, 10).unwrap(); + let resized = resize_png_if_needed(&png, 100).unwrap(); + // Returned bytes should match the input exactly (no re-encode). + assert_eq!(resized, png); + } + + #[test] + fn resize_no_op_when_max_dim_zero() { + let rgba = vec![0u8; 100 * 100 * 4]; + let png = encode_rgba_to_png(&rgba, 100, 100).unwrap(); + let resized = resize_png_if_needed(&png, 0).unwrap(); + assert_eq!(resized, png); + } + + #[test] + fn resize_downscales_to_long_edge() { + // 200x100 → max_dim=50 → 50x25 (long edge dictates). + let rgba = vec![0u8; 200 * 100 * 4]; + let png = encode_rgba_to_png(&rgba, 200, 100).unwrap(); + let resized = resize_png_if_needed(&png, 50).unwrap(); + let (w, h) = png_dimensions(&resized).unwrap(); + assert_eq!(w, 50); + assert_eq!(h, 25); + } + + #[test] + fn png_to_jpeg_strips_alpha() { + let rgba = vec![0x80u8; 4 * 4 * 4]; + let png = encode_rgba_to_png(&rgba, 4, 4).unwrap(); + let jpeg = png_bytes_to_jpeg(&png, 80).unwrap(); + // JPEG file signature: FF D8 FF. + assert_eq!(&jpeg[..3], &[0xFF, 0xD8, 0xFF]); + } + + #[test] + fn crosshair_returns_valid_png() { + let rgba = vec![0u8; 40 * 40 * 4]; + let png = encode_rgba_to_png(&rgba, 40, 40).unwrap(); + let marked = crosshair_png_bytes(&png, 20.0, 20.0).unwrap(); + let (w, h) = png_dimensions(&marked).unwrap(); + assert_eq!((w, h), (40, 40)); + } + + #[test] + fn bgra_to_png_swaps_channels() { + // 1x1 BGRA pixel: B=10, G=20, R=30, A=40. + let bgra = vec![10u8, 20, 30, 40]; + let png = encode_bgra_to_png(&bgra, 1, 1).unwrap(); + // Decode and check the RGBA bytes have R↔B swapped. + let decoder = image::codecs::png::PngDecoder::new(std::io::Cursor::new(&png)).unwrap(); + let mut decoded = vec![0u8; decoder.total_bytes() as usize]; + decoder.read_image(&mut decoded).unwrap(); + assert_eq!(decoded, vec![30u8, 20, 10, 40]); // RGBA: R=30, G=20, B=10, A=40 + } +} diff --git a/libs/cua-driver-rs/crates/mcp-server/src/lib.rs b/libs/cua-driver-rs/crates/mcp-server/src/lib.rs index acf3d94279..93ca07fc36 100644 --- a/libs/cua-driver-rs/crates/mcp-server/src/lib.rs +++ b/libs/cua-driver-rs/crates/mcp-server/src/lib.rs @@ -11,11 +11,14 @@ //! - Notifications (no `id`) are silently ignored pub mod cdp; +pub mod element_cache; +pub mod image_utils; pub mod page; pub mod protocol; pub mod recording; pub mod recording_tools; pub mod server; pub mod tool; +pub mod tool_args; pub use recording::RecordingSession; diff --git a/libs/cua-driver-rs/crates/mcp-server/src/recording.rs b/libs/cua-driver-rs/crates/mcp-server/src/recording.rs index 10fdd51034..a71d2ac037 100644 --- a/libs/cua-driver-rs/crates/mcp-server/src/recording.rs +++ b/libs/cua-driver-rs/crates/mcp-server/src/recording.rs @@ -181,15 +181,16 @@ fn write_turn( std::fs::create_dir_all(turn_dir)?; let now = now_ms(); + use crate::tool_args::ArgsExt; // Extract window_id and pid from args for screenshot capture. - let window_id = args.get("window_id").and_then(|v| v.as_u64()); - let pid = args.get("pid").and_then(|v| v.as_i64()); + let window_id = args.opt_u64("window_id"); + let pid = args.opt_i64("pid"); // Extract click point for click-family tools. let click_point: Option<(f64, f64)> = if matches!( tool_name, "click" | "double_click" | "right_click" ) { - match (args.get("x").and_then(|v| v.as_f64()), args.get("y").and_then(|v| v.as_f64())) { + match (args.opt_f64("x"), args.opt_f64("y")) { (Some(x), Some(y)) => Some((x, y)), _ => None, } diff --git a/libs/cua-driver-rs/crates/mcp-server/src/recording_tools.rs b/libs/cua-driver-rs/crates/mcp-server/src/recording_tools.rs index b607e92c99..9c616ebcf6 100644 --- a/libs/cua-driver-rs/crates/mcp-server/src/recording_tools.rs +++ b/libs/cua-driver-rs/crates/mcp-server/src/recording_tools.rs @@ -93,13 +93,14 @@ impl Tool for SetRecordingTool { Some(v) => v, None => return ToolResult::error("Missing required boolean field `enabled`."), }; - let output_dir = args.get("output_dir").and_then(|v| v.as_str()); - if enabled && output_dir.map(|s| s.is_empty()).unwrap_or(true) { + use crate::tool_args::ArgsExt; + let output_dir = args.opt_str("output_dir"); + if enabled && output_dir.as_deref().map(str::is_empty).unwrap_or(true) { return ToolResult::error("`output_dir` is required when enabling recording."); } - let video_experimental = args.get("video_experimental").and_then(|v| v.as_bool()).unwrap_or(false); + let video_experimental = args.bool_or("video_experimental", false); - match self.session.configure(enabled, output_dir) { + match self.session.configure(enabled, output_dir.as_deref()) { Ok(()) => { let state = self.session.current_state(); let msg = if state.enabled { @@ -223,8 +224,9 @@ impl Tool for ReplayTrajectoryTool { Some(v) if !v.is_empty() => v.to_owned(), _ => return ToolResult::error("Missing required string field `dir`."), }; - let delay_ms = args.get("delay_ms").and_then(|v| v.as_u64()).unwrap_or(500).min(10_000); - let stop_on_error = args.get("stop_on_error").and_then(|v| v.as_bool()).unwrap_or(true); + use crate::tool_args::ArgsExt; + let delay_ms = args.u64_or("delay_ms", 500).min(10_000); + let stop_on_error = args.bool_or("stop_on_error", true); // Expand ~/ let dir = { diff --git a/libs/cua-driver-rs/crates/mcp-server/src/tool_args.rs b/libs/cua-driver-rs/crates/mcp-server/src/tool_args.rs new file mode 100644 index 0000000000..6e12ca45d2 --- /dev/null +++ b/libs/cua-driver-rs/crates/mcp-server/src/tool_args.rs @@ -0,0 +1,336 @@ +//! Shared tool-argument extraction helpers. +//! +//! Every tool in every platform crate does the same dance to pull +//! pid / window_id / element_index / text / etc. out of the inbound +//! JSON `Value`. Before this module they were 4-line `match` blocks +//! repeated 200+ times across the codebase with subtly different +//! error wording. The trait below consolidates those into one +//! consistent shape. +//! +//! Two flavours of accessor: +//! +//! - `require_*` — bails with a `ToolResult::error` if the field is +//! missing or the wrong type. Narrowing casts (i64 → i32, u64 → u32) +//! go through `try_from` and surface an actionable range error +//! instead of silently truncating — the same CodeRabbit fix landed +//! on PR #1666's page tool now applies uniformly. +//! - `opt_*` — returns `Option` / `Result>` for callers +//! that already have a sensible default. Range-checked variants +//! return `Result>` because "the user passed a value but +//! it's out of range" is still an error, just one we don't enforce +//! when the field is absent. +//! +//! Error message format: `"Missing required {kind} field: {name}"`. +//! Used uniformly so MCP clients can pattern-match the wording. +//! +//! See `libs/cua-driver-rs/docs/dedup-audit.md` for the audit trail +//! that motivated this extraction. + +use serde_json::Value; + +use crate::protocol::ToolResult; + +/// Format the canonical "missing required field" error. +#[inline] +fn missing(kind: &str, name: &str) -> ToolResult { + ToolResult::error(format!("Missing required {kind} field: {name}")) +} + +/// Format the canonical "wrong type" error. Used when the field IS +/// present but a different JSON type than the caller asked for. +#[inline] +fn wrong_type(kind: &str, name: &str) -> ToolResult { + ToolResult::error(format!( + "Field {name} has wrong type — expected {kind}" + )) +} + +/// Format the canonical out-of-range error for narrowing casts. +#[inline] +fn out_of_range(kind: &str, name: &str, raw: i128) -> ToolResult { + ToolResult::error(format!( + "Field {name} is out of range for {kind}: {raw}" + )) +} + +/// Extension trait on `&serde_json::Value` (the MCP `arguments` blob) +/// that provides typed, error-formatted accessors for every field +/// shape the tool surface uses. +pub trait ArgsExt { + // ── Required scalars ────────────────────────────────────────────────── + fn require_i32(&self, name: &str) -> Result; + fn require_i64(&self, name: &str) -> Result; + fn require_u32(&self, name: &str) -> Result; + fn require_u64(&self, name: &str) -> Result; + fn require_f64(&self, name: &str) -> Result; + fn require_str(&self, name: &str) -> Result; + fn require_bool(&self, name: &str) -> Result; + + // ── Optional scalars ────────────────────────────────────────────────── + /// Returns `None` if the field is absent. Returns `Err` if the + /// field is present but doesn't fit in i32 — silent truncation + /// is the bug class this trait exists to prevent. + fn opt_i32(&self, name: &str) -> Result, ToolResult>; + fn opt_u32(&self, name: &str) -> Result, ToolResult>; + fn opt_u64(&self, name: &str) -> Option; + fn opt_i64(&self, name: &str) -> Option; + fn opt_f64(&self, name: &str) -> Option; + fn opt_str(&self, name: &str) -> Option; + fn opt_bool(&self, name: &str) -> Option; + + // ── Default-fallback scalars (the most common pattern) ──────────────── + fn u64_or(&self, name: &str, default: u64) -> u64; + fn i64_or(&self, name: &str, default: i64) -> i64; + fn f64_or(&self, name: &str, default: f64) -> f64; + fn str_or<'a>(&'a self, name: &str, default: &'a str) -> String; + fn bool_or(&self, name: &str, default: bool) -> bool; + + // ── Arrays ──────────────────────────────────────────────────────────── + /// Extract an array of strings (for `modifiers`, `keys`, `urls`, + /// `attributes`, etc.). Returns an empty vec if the field is + /// absent or not an array. Non-string elements are skipped. + fn str_array(&self, name: &str) -> Vec; +} + +impl ArgsExt for Value { + fn require_i32(&self, name: &str) -> Result { + let raw = self + .get(name) + .and_then(|v| v.as_i64()) + .ok_or_else(|| missing("integer", name))?; + i32::try_from(raw).map_err(|_| out_of_range("i32", name, raw as i128)) + } + + fn require_i64(&self, name: &str) -> Result { + self.get(name) + .and_then(|v| v.as_i64()) + .ok_or_else(|| missing("integer", name)) + } + + fn require_u32(&self, name: &str) -> Result { + let raw = self + .get(name) + .and_then(|v| v.as_u64()) + .ok_or_else(|| missing("integer", name))?; + u32::try_from(raw).map_err(|_| out_of_range("u32", name, raw as i128)) + } + + fn require_u64(&self, name: &str) -> Result { + self.get(name) + .and_then(|v| v.as_u64()) + .ok_or_else(|| missing("integer", name)) + } + + fn require_f64(&self, name: &str) -> Result { + self.get(name) + .and_then(|v| v.as_f64()) + .ok_or_else(|| missing("number", name)) + } + + fn require_str(&self, name: &str) -> Result { + self.get(name) + .and_then(|v| v.as_str()) + .map(str::to_owned) + .ok_or_else(|| missing("string", name)) + } + + fn require_bool(&self, name: &str) -> Result { + self.get(name) + .and_then(|v| v.as_bool()) + .ok_or_else(|| missing("boolean", name)) + } + + fn opt_i32(&self, name: &str) -> Result, ToolResult> { + match self.get(name) { + None | Some(Value::Null) => Ok(None), + Some(v) => { + let raw = v.as_i64().ok_or_else(|| wrong_type("integer", name))?; + i32::try_from(raw) + .map(Some) + .map_err(|_| out_of_range("i32", name, raw as i128)) + } + } + } + + fn opt_u32(&self, name: &str) -> Result, ToolResult> { + match self.get(name) { + None | Some(Value::Null) => Ok(None), + Some(v) => { + let raw = v.as_u64().ok_or_else(|| wrong_type("integer", name))?; + u32::try_from(raw) + .map(Some) + .map_err(|_| out_of_range("u32", name, raw as i128)) + } + } + } + + fn opt_u64(&self, name: &str) -> Option { + self.get(name).and_then(|v| v.as_u64()) + } + + fn opt_i64(&self, name: &str) -> Option { + self.get(name).and_then(|v| v.as_i64()) + } + + fn opt_f64(&self, name: &str) -> Option { + self.get(name).and_then(|v| v.as_f64()) + } + + fn opt_str(&self, name: &str) -> Option { + self.get(name) + .and_then(|v| v.as_str()) + .map(str::to_owned) + } + + fn opt_bool(&self, name: &str) -> Option { + self.get(name).and_then(|v| v.as_bool()) + } + + fn u64_or(&self, name: &str, default: u64) -> u64 { + self.opt_u64(name).unwrap_or(default) + } + + fn i64_or(&self, name: &str, default: i64) -> i64 { + self.opt_i64(name).unwrap_or(default) + } + + fn f64_or(&self, name: &str, default: f64) -> f64 { + self.opt_f64(name).unwrap_or(default) + } + + fn str_or<'a>(&'a self, name: &str, default: &'a str) -> String { + self.opt_str(name).unwrap_or_else(|| default.to_owned()) + } + + fn bool_or(&self, name: &str, default: bool) -> bool { + self.opt_bool(name).unwrap_or(default) + } + + fn str_array(&self, name: &str) -> Vec { + self.get(name) + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|v| v.as_str().map(str::to_owned)) + .collect() + }) + .unwrap_or_default() + } +} + +// ── Tests ───────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn require_i32_happy_path() { + let args = json!({ "pid": 1234 }); + assert_eq!(args.require_i32("pid").unwrap(), 1234); + } + + #[test] + fn require_i32_missing_returns_actionable_error() { + let args = json!({}); + let err = args.require_i32("pid").unwrap_err(); + let body = format!("{:?}", err.content[0]); + assert!(body.contains("Missing required integer field: pid")); + } + + #[test] + fn require_i32_out_of_range_rejected() { + // i32::MAX + 1 doesn't fit in i32. + let args = json!({ "pid": (i32::MAX as i64) + 1 }); + let err = args.require_i32("pid").unwrap_err(); + let body = format!("{:?}", err.content[0]); + assert!(body.contains("out of range for i32")); + } + + #[test] + fn require_u32_out_of_range_rejected() { + let args = json!({ "window_id": (u32::MAX as u64) + 1 }); + let err = args.require_u32("window_id").unwrap_err(); + let body = format!("{:?}", err.content[0]); + assert!(body.contains("out of range for u32")); + } + + #[test] + fn require_str_missing_says_string() { + let args = json!({}); + let err = args.require_str("text").unwrap_err(); + let body = format!("{:?}", err.content[0]); + assert!(body.contains("Missing required string field: text")); + } + + #[test] + fn opt_u32_returns_none_when_absent() { + let args = json!({}); + assert_eq!(args.opt_u32("window_id").unwrap(), None); + } + + #[test] + fn opt_u32_returns_none_when_null() { + let args = json!({ "window_id": null }); + assert_eq!(args.opt_u32("window_id").unwrap(), None); + } + + #[test] + fn opt_u32_present_and_in_range() { + let args = json!({ "window_id": 42 }); + assert_eq!(args.opt_u32("window_id").unwrap(), Some(42)); + } + + #[test] + fn opt_u32_present_but_wrong_type_errors() { + let args = json!({ "window_id": "not a number" }); + assert!(args.opt_u32("window_id").is_err()); + } + + #[test] + fn opt_u32_present_but_out_of_range_errors() { + let args = json!({ "window_id": (u32::MAX as u64) + 1 }); + let err = args.opt_u32("window_id").unwrap_err(); + let body = format!("{:?}", err.content[0]); + assert!(body.contains("out of range for u32")); + } + + #[test] + fn u64_or_returns_default_when_absent() { + let args = json!({}); + assert_eq!(args.u64_or("delay_ms", 30), 30); + } + + #[test] + fn u64_or_returns_value_when_present() { + let args = json!({ "delay_ms": 100 }); + assert_eq!(args.u64_or("delay_ms", 30), 100); + } + + #[test] + fn str_or_uses_default_when_absent() { + let args = json!({}); + assert_eq!(args.str_or("button", "left"), "left"); + } + + #[test] + fn str_array_handles_absent_and_non_string_entries() { + let args = json!({}); + assert_eq!(args.str_array("modifiers"), Vec::::new()); + + let args = json!({ "modifiers": ["ctrl", "shift", 42, "alt"] }); + assert_eq!( + args.str_array("modifiers"), + vec!["ctrl".to_owned(), "shift".to_owned(), "alt".to_owned()] + ); + } + + #[test] + fn bool_or_handles_typical_cases() { + let args = json!({}); + assert!(!args.bool_or("from_zoom", false)); + let args = json!({ "from_zoom": true }); + assert!(args.bool_or("from_zoom", false)); + } +} diff --git a/libs/cua-driver-rs/crates/platform-linux/src/atspi/cache.rs b/libs/cua-driver-rs/crates/platform-linux/src/atspi/cache.rs index 16df860ed1..3b031f1d15 100644 --- a/libs/cua-driver-rs/crates/platform-linux/src/atspi/cache.rs +++ b/libs/cua-driver-rs/crates/platform-linux/src/atspi/cache.rs @@ -1,9 +1,13 @@ //! AT-SPI element cache for Linux. //! Stores element keys (u64 hash) indexed by (pid, xid) → element_index. +//! +//! The locked-HashMap plumbing lives in `mcp_server::element_cache` — see +//! `docs/dedup-audit.md` item #3. This module owns the Linux-specific +//! `CacheKey` and `CachedSnapshot` (no Drop needed — `Vec` frees +//! itself). use super::AtspiNode; -use std::collections::HashMap; -use std::sync::Mutex; +use mcp_server::element_cache::ElementCacheCore; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct CacheKey { pub pid: u32, pub xid: u64 } @@ -14,30 +18,29 @@ pub struct CachedSnapshot { } pub struct ElementCache { - inner: Mutex>, + core: ElementCacheCore, } impl ElementCache { - pub fn new() -> Self { Self { inner: Mutex::new(HashMap::new()) } } + pub fn new() -> Self { Self { core: ElementCacheCore::new() } } pub fn update(&self, pid: u32, xid: u64, nodes: &[AtspiNode]) { let elements: Vec = nodes.iter() .filter(|n| n.element_index.is_some()) .map(|n| n.element_key) .collect(); - self.inner.lock().unwrap().insert(CacheKey { pid, xid }, CachedSnapshot { elements }); + self.core.insert(CacheKey { pid, xid }, CachedSnapshot { elements }); } pub fn get_element_key(&self, pid: u32, xid: u64, idx: usize) -> Option { - self.inner.lock().unwrap() - .get(&CacheKey { pid, xid })? - .elements.get(idx).copied() + self.core + .with_snapshot(&CacheKey { pid, xid }, |s| s.elements.get(idx).copied()) + .flatten() } pub fn element_count(&self, pid: u32, xid: u64) -> usize { - self.inner.lock().unwrap() - .get(&CacheKey { pid, xid }) - .map(|s| s.elements.len()) + self.core + .with_snapshot(&CacheKey { pid, xid }, |s| s.elements.len()) .unwrap_or(0) } } diff --git a/libs/cua-driver-rs/crates/platform-linux/src/capture.rs b/libs/cua-driver-rs/crates/platform-linux/src/capture.rs index 279eaab2ef..b3beffb7fa 100644 --- a/libs/cua-driver-rs/crates/platform-linux/src/capture.rs +++ b/libs/cua-driver-rs/crates/platform-linux/src/capture.rs @@ -27,7 +27,7 @@ pub fn screenshot_window_bytes(xid: u64) -> Result> { pub fn screenshot_window(xid: u64) -> Result<(String, u32, u32)> { // Try `import -window png:-` (ImageMagick). if let Ok(bytes) = capture_via_import(xid) { - let (w, h) = png_dimensions(&bytes)?; + let (w, h) = mcp_server::image_utils::png_dimensions(&bytes)?; return Ok((BASE64.encode(&bytes), w, h)); } @@ -82,76 +82,21 @@ fn capture_via_xgetimage(xid: u64) -> Result<(String, u32, u32)> { rgba.extend_from_slice(&[r, g, b, a]); } - let png = write_uncompressed_png(&rgba, w, h)?; + let png = mcp_server::image_utils::encode_rgba_to_png(&rgba, w, h)?; Ok((BASE64.encode(&png), w, h)) } /// Public version of png_dimensions for use in tool code. pub fn png_dimensions_pub(data: &[u8]) -> Result<(u32, u32)> { - png_dimensions(data) + mcp_server::image_utils::png_dimensions(data) } -fn png_dimensions(data: &[u8]) -> Result<(u32, u32)> { - if data.len() < 24 { bail!("PNG too small"); } - // Signature (8) + IHDR length (4) + "IHDR" (4) + width (4) + height (4) - let w = u32::from_be_bytes([data[16], data[17], data[18], data[19]]); - let h = u32::from_be_bytes([data[20], data[21], data[22], data[23]]); - Ok((w, h)) -} - -// Inline minimal uncompressed PNG writer (same as platform-windows/capture.rs). -fn write_uncompressed_png(rgba: &[u8], w: u32, h: u32) -> Result> { - let mut out = Vec::with_capacity(rgba.len() + 4096); - out.extend_from_slice(b"\x89PNG\r\n\x1a\n"); - let ihdr: [u8; 13] = [ - (w >> 24) as u8, (w >> 16) as u8, (w >> 8) as u8, w as u8, - (h >> 24) as u8, (h >> 16) as u8, (h >> 8) as u8, h as u8, - 8, 6, 0, 0, 0, - ]; - write_png_chunk(&mut out, b"IHDR", &ihdr); - let row_bytes = (w * 4) as usize; - let mut raw = Vec::with_capacity((row_bytes + 1) * h as usize); - for row in 0..h as usize { - raw.push(0u8); - raw.extend_from_slice(&rgba[row * row_bytes..(row + 1) * row_bytes]); - } - let zlib_data = zlib_store(&raw); - write_png_chunk(&mut out, b"IDAT", &zlib_data); - write_png_chunk(&mut out, b"IEND", &[]); - Ok(out) -} - -fn write_png_chunk(out: &mut Vec, name: &[u8; 4], data: &[u8]) { - out.extend_from_slice(&(data.len() as u32).to_be_bytes()); - out.extend_from_slice(name); - out.extend_from_slice(data); - out.extend_from_slice(&crc32_ieee(name, data).to_be_bytes()); -} - -fn zlib_store(data: &[u8]) -> Vec { - let adler = adler32(data); - let mut out = vec![0x78, 0x01]; - let mut pos = 0; - loop { - let end = (pos + 65535).min(data.len()); - let is_last = end == data.len(); - let blen = (end - pos) as u16; - out.push(if is_last { 1 } else { 0 }); - out.extend_from_slice(&blen.to_le_bytes()); - out.extend_from_slice(&(!blen).to_le_bytes()); - out.extend_from_slice(&data[pos..end]); - pos = end; - if pos >= data.len() { break; } - } - out.extend_from_slice(&adler.to_be_bytes()); - out -} - -fn adler32(data: &[u8]) -> u32 { - let (mut s1, mut s2) = (1u32, 0u32); - for &b in data { s1 = (s1 + b as u32) % 65521; s2 = (s2 + s1) % 65521; } - (s2 << 16) | s1 -} +// NOTE: the previously-inline `png_dimensions`, `write_uncompressed_png`, +// `write_png_chunk`, `zlib_store`, `adler32` (and `crc32_ieee` below) +// were extracted to `mcp_server::image_utils` in the 2026-05 dedup +// audit so all three platforms call the same code. See +// `CUA_DRIVER_RS_DEDUP_AUDIT.md`. RGBA-encoding callers below now go +// through `mcp_server::image_utils::encode_rgba_to_png`. /// Capture the primary display (root window) as raw PNG bytes. pub fn screenshot_display_bytes() -> Result> { @@ -182,96 +127,37 @@ pub fn screenshot_display_bytes() -> Result> { let (b, g, r) = (chunk[0], chunk[1], chunk[2]); rgba.extend_from_slice(&[r, g, b, 255]); } - write_uncompressed_png(&rgba, w, h) + mcp_server::image_utils::encode_rgba_to_png(&rgba, w, h) } /// Capture the primary display, returning (base64_png, width, height). pub fn screenshot_display() -> Result<(String, u32, u32)> { let png_bytes = screenshot_display_bytes()?; - let (w, h) = png_dimensions(&png_bytes)?; + let (w, h) = mcp_server::image_utils::png_dimensions(&png_bytes)?; Ok((BASE64.encode(&png_bytes), w, h)) } +// PNG/JPEG/resize/crosshair helpers — re-exports of the shared +// `mcp_server::image_utils` module. The previous file-local copies were +// near-identical to the macOS and Windows versions; the dedup-audit +// (2026-05) moved them all to one place. + /// Convert PNG bytes to JPEG at the given quality (1–95). pub fn png_bytes_to_jpeg(png_bytes: &[u8], quality: u8) -> Result> { - let img = image::load_from_memory_with_format(png_bytes, image::ImageFormat::Png)?; - let mut buf = Vec::new(); - { - let mut cursor = std::io::Cursor::new(&mut buf); - let encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut cursor, quality); - img.write_with_encoder(encoder)?; - } - Ok(buf) + mcp_server::image_utils::png_bytes_to_jpeg(png_bytes, quality) } /// Downscale `png_bytes` so neither dimension exceeds `max_dim`. -/// If `max_dim == 0` or the image already fits, returns a copy of the original bytes unchanged. +/// If `max_dim == 0` or the image already fits, returns a copy of the +/// original bytes unchanged. pub fn resize_png_if_needed(png_bytes: &[u8], max_dim: u32) -> Result> { - if max_dim == 0 { - return Ok(png_bytes.to_vec()); - } - let (w, h) = png_dimensions_pub(png_bytes)?; - if w <= max_dim && h <= max_dim { - return Ok(png_bytes.to_vec()); - } - let scale = max_dim as f64 / w.max(h) as f64; - let new_w = (w as f64 * scale).round() as u32; - let new_h = (h as f64 * scale).round() as u32; - let img = image::load_from_memory_with_format(png_bytes, image::ImageFormat::Png)?; - let resized = img.resize(new_w, new_h, image::imageops::FilterType::Lanczos3); - let mut out = Vec::new(); - resized.write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png)?; - Ok(out) + mcp_server::image_utils::resize_png_if_needed(png_bytes, max_dim) } -/// Draw a red crosshair at pixel (cx, cy) on a PNG image and return modified PNG bytes. -/// Used by recording's click-marker callback to produce click.png. +/// Draw a red crosshair at pixel (cx, cy) on a PNG image and return +/// modified PNG bytes. Used by recording's click-marker callback to +/// produce click.png. pub fn crosshair_png_bytes(png_bytes: &[u8], cx: f64, cy: f64) -> Result> { - let img = image::load_from_memory_with_format(png_bytes, image::ImageFormat::Png)?; - let (w, h) = (img.width(), img.height()); - let mut img = img.to_rgba8(); - - let arm_len = (w as f64 / 40.0).max(12.0) as i32; - let line_w = ((w as f64 / 400.0).max(1.5)) as i32; - let red = image::Rgba([255u8, 26, 26, 242]); - let cx = cx as i32; - let cy = cy as i32; - - for lw in 0..=line_w { - let off = lw - line_w / 2; - for dx in -arm_len..=arm_len { - if let Some(p) = img.get_pixel_mut_checked( - (cx + dx).clamp(0, w as i32 - 1) as u32, - (cy + off).clamp(0, h as i32 - 1) as u32, - ) { *p = red; } - } - for dy in -arm_len..=arm_len { - if let Some(p) = img.get_pixel_mut_checked( - (cx + off).clamp(0, w as i32 - 1) as u32, - (cy + dy).clamp(0, h as i32 - 1) as u32, - ) { *p = red; } - } - } - - let mut out = Vec::new(); - image::DynamicImage::ImageRgba8(img) - .write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png)?; - Ok(out) + mcp_server::image_utils::crosshair_png_bytes(png_bytes, cx, cy) } -fn crc32_ieee(name: &[u8], data: &[u8]) -> u32 { - const T: [u32; 256] = { - let mut t = [0u32; 256]; - let mut i = 0usize; - while i < 256 { - let mut c = i as u32; - let mut j = 0; - while j < 8 { c = if c & 1 != 0 { 0xEDB88320 ^ (c >> 1) } else { c >> 1 }; j += 1; } - t[i] = c; i += 1; - } - t - }; - let mut crc = !0u32; - for &b in name.iter().chain(data.iter()) { crc = T[((crc ^ b as u32) & 0xFF) as usize] ^ (crc >> 8); } - !crc -} diff --git a/libs/cua-driver-rs/crates/platform-linux/src/overlay.rs b/libs/cua-driver-rs/crates/platform-linux/src/overlay.rs index 07a4971e72..2d5ab55d40 100644 --- a/libs/cua-driver-rs/crates/platform-linux/src/overlay.rs +++ b/libs/cua-driver-rs/crates/platform-linux/src/overlay.rs @@ -9,14 +9,18 @@ //! - Z-ordering: `XRaiseWindow` every 80ms to stay above normal windows. //! - Wayland: when WAYLAND_DISPLAY is set but DISPLAY is also available (XWayland), //! the X11 path is used. Pure Wayland support is a TODO. +//! +//! ## Cross-platform note (2026-05 dedup audit) +//! +//! Animation state + render pipeline live in `cursor_overlay::render_state` +//! (`RenderStateCore`, `tick_motion`, `apply_command_base`, `render_frame`). +//! What stays here is the X11 window plumbing: connection setup, +//! override-redirect visual, ShapeInput passthrough, and the XPutImage paint. use std::sync::{Mutex, OnceLock}; use std::time::{Duration, Instant}; -use cursor_overlay::{ - CursorConfig, CursorShape, MotionConfig, OverlayCommand, Palette, PathPlanner, PathState, - PlannedPath, -}; +use cursor_overlay::{CursorConfig, OverlayCommand, RenderStateCore}; // ── Global channel ──────────────────────────────────────────────────────── @@ -47,7 +51,7 @@ pub fn run_on_thread() { let cfg = { let guard = RENDER.lock().unwrap(); match &*guard { - Some(rs) => rs.cfg.clone(), + Some(rs) => rs.core.cfg.clone(), None => return, } }; @@ -65,307 +69,40 @@ pub fn run_on_thread() { } // ── Animation state ─────────────────────────────────────────────────────── +// +// The platform-agnostic fields + tick + apply_command + render pipeline live +// in `cursor_overlay::render_state` (2026-05 dedup audit). What stays here +// is the X11-specific screen dimensions. struct RenderState { - cfg: CursorConfig, - palette: Palette, - motion: MotionConfig, - pos: (f64, f64), - heading: f64, - path: Option, - dist: f64, - start_t: Instant, - spring: Option, - spring_tgt: Option<(f64, f64, f64)>, - click_t: Option, - shape: Option, - visible: bool, - idle_secs: f64, - idle_alpha: f64, - pinned_wid: Option, + core: RenderStateCore, + /// X11 screen dimensions in pixels (populated after XOpenDisplay). scr_w: u32, scr_h: u32, } -#[derive(Clone, Copy)] -struct Spring { ox: f64, oy: f64, vx: f64, vy: f64 } - impl RenderState { fn new(cfg: CursorConfig) -> Self { - let palette = cfg.palette(); - let motion = cfg.motion.clone(); - let shape = cfg.shape.clone(); RenderState { - cfg, palette, motion, shape, - pos: (-200.0, -200.0), - heading: std::f64::consts::FRAC_PI_4, - path: None, - dist: 0.0, - start_t: Instant::now(), - spring: None, - spring_tgt: None, - click_t: None, - visible: true, - idle_secs: 0.0, - idle_alpha: 1.0, - pinned_wid: None, + core: RenderStateCore::new(cfg), scr_w: 1920, scr_h: 1080, } } fn tick(&mut self, dt: f64) { - let spring_k = self.motion.spring * 400.0; - let spring_c = self.motion.spring * 20.0; - - if let Some(ref p) = self.path { - let path_frac = (self.dist / p.length.max(1.0)).clamp(0.0, 1.0); - let profile = 16.0 * path_frac * path_frac * (1.0 - path_frac) * (1.0 - path_frac); - let floor = if path_frac < 0.5 { self.motion.min_start_speed } else { self.motion.min_end_speed }; - let speed = (floor + (self.motion.peak_speed - floor) * profile).max(floor); - self.dist += speed * dt; - - let path_len = p.length.max(1.0); - if self.dist >= path_len { - let end = p.sample(path_len); - let end_heading = p.end_visual_heading; - let vh = end.heading; - self.spring = Some(Spring { - ox: 0.0, oy: 0.0, - vx: speed * 0.5 * vh.cos(), - vy: speed * 0.5 * vh.sin(), - }); - self.spring_tgt = Some((end.x, end.y, end_heading)); - self.pos = (end.x, end.y); - self.heading = end_heading; - self.path = None; - self.dist = 0.0; - } else { - let s: PathState = p.sample(self.dist); - self.pos = (s.x, s.y); - let desired = s.heading + std::f64::consts::PI; - let max_step = 14.0 * dt; - self.heading = rotate_toward(self.heading, desired, max_step); - } - } else if let Some(mut s) = self.spring { - if let Some((tx, ty, th)) = self.spring_tgt { - let sdt = dt / 4.0; - for _ in 0..4 { - s.vx += (-spring_k * s.ox - spring_c * s.vx) * sdt; - s.vy += (-spring_k * s.oy - spring_c * s.vy) * sdt; - s.ox += s.vx * sdt; - s.oy += s.vy * sdt; - } - self.pos = (tx + s.ox, ty + s.oy); - self.heading = th; - if s.ox.hypot(s.oy) < 0.3 && s.vx.hypot(s.vy) < 2.0 { - self.pos = (tx, ty); - self.spring = None; - } else { - self.spring = Some(s); - } - } - } - - if let Some(t) = self.click_t { - let next = t + dt * 4.0; - self.click_t = if next >= 1.0 { None } else { Some(next) }; - } - - let idle_hide_ms = self.motion.idle_hide_ms; - if idle_hide_ms > 0.0 { - let moving = self.path.is_some() || self.spring.is_some() || self.click_t.is_some(); - if moving { - self.idle_secs = 0.0; - self.idle_alpha = 1.0; - } else { - self.idle_secs += dt; - let fade_start = idle_hide_ms / 1000.0; - let fade_end = fade_start + 0.18; - if self.idle_secs > fade_end { - self.idle_alpha = 0.0; - } else if self.idle_secs > fade_start { - let t = (self.idle_secs - fade_start) / 0.18; - self.idle_alpha = 1.0 - t.clamp(0.0, 1.0); - } - } - } else { - self.idle_alpha = 1.0; - } + self.core.tick_motion(dt); } fn apply_command(&mut self, cmd: OverlayCommand) { - match cmd { - OverlayCommand::MoveTo { x, y, end_heading_radians } => { - let (x0, y0) = self.pos; - let th0 = self.heading + std::f64::consts::PI; - let th1 = end_heading_radians + std::f64::consts::PI; - const CLICK_OFFSET: f64 = 16.0; - const TURN_RADIUS: f64 = 80.0; - let tx = x + end_heading_radians.cos() * CLICK_OFFSET; - let ty = y + end_heading_radians.sin() * CLICK_OFFSET; - let plan = PathPlanner::plan( - x0, y0, th0, - tx, ty, th1, - end_heading_radians, - TURN_RADIUS, - ); - self.path = Some(plan); - self.dist = 0.0; - self.start_t = Instant::now(); - self.spring = None; - self.spring_tgt = None; - self.idle_secs = 0.0; - self.idle_alpha = 1.0; - } - OverlayCommand::ClickPulse { x, y } => { - self.pos = (x, y); - self.click_t = Some(0.0); - self.idle_secs = 0.0; - self.idle_alpha = 1.0; - } - OverlayCommand::SetEnabled(v) => { self.visible = v; } - OverlayCommand::SetMotion(m) => { self.motion = m; } - OverlayCommand::SetPalette(p) => { self.palette = p; } - OverlayCommand::PinAbove(wid) => { self.pinned_wid = Some(wid); } - // Custom cursor shape/gradient/focus-rect — not yet rendered on Linux; - // accepted silently so the tool doesn't return an error. - OverlayCommand::SetShape(_) | OverlayCommand::SetGradient { .. } - | OverlayCommand::ShowFocusRect(_) => {} - } - } -} - -// Shared with platform-windows — pulled into `cursor_overlay::util` so both -// per-OS render loops use the exact same easing primitive. -use cursor_overlay::util::rotate_toward; - -// ── Renderer (shared tiny-skia logic) ───────────────────────────────────── - -fn render_frame(rs: &RenderState) -> tiny_skia::Pixmap { - let w = rs.scr_w.max(1); - let h = rs.scr_h.max(1); - let mut pm = tiny_skia::Pixmap::new(w, h) - .unwrap_or_else(|| tiny_skia::Pixmap::new(1, 1).unwrap()); - - if !rs.visible || rs.pos.0 < -100.0 || rs.idle_alpha < 0.004 { - return pm; - } - - let (px, py) = rs.pos; - let heading = rs.heading; - let alpha_scale = rs.idle_alpha as f32; - - let bloom_r: f32 = 22.0; - let [br, bg, bb, _] = rs.palette.bloom_inner; - let bloom_inner = tiny_skia::Color::from_rgba8(br, bg, bb, (115.0 * alpha_scale) as u8); - let [or_, og, ob, _] = rs.palette.bloom_outer; - let bloom_outer = tiny_skia::Color::from_rgba8(or_, og, ob, (26.0 * alpha_scale) as u8); - let bloom_zero = tiny_skia::Color::from_rgba8(or_, og, ob, 0); - - let bloom_paint = { - let mut p = tiny_skia::Paint::default(); - p.shader = tiny_skia::RadialGradient::new( - tiny_skia::Point::from_xy(px as f32, py as f32), - tiny_skia::Point::from_xy(px as f32, py as f32), - bloom_r, - vec![ - tiny_skia::GradientStop::new(0.0, bloom_inner), - tiny_skia::GradientStop::new(0.5, bloom_outer), - tiny_skia::GradientStop::new(1.0, bloom_zero), - ], - tiny_skia::SpreadMode::Pad, - tiny_skia::Transform::identity(), - ).unwrap_or(tiny_skia::Shader::SolidColor(bloom_inner)); - p.anti_alias = true; - p - }; - if let Some(r) = tiny_skia::Rect::from_xywh( - (px - bloom_r as f64) as f32, (py - bloom_r as f64) as f32, - bloom_r * 2.0, bloom_r * 2.0, - ) { - pm.fill_rect(r, &bloom_paint, tiny_skia::Transform::identity(), None); - } - - if let Some(t) = rs.click_t { - let ring_r = (bloom_r + 20.0 * t as f32) * (1.0 - t as f32 * 0.5); - let alpha = ((1.0 - t) * 180.0 * alpha_scale as f64) as u8; - let [cr, cg, cb, _] = rs.palette.cursor_mid; - let ring_color = tiny_skia::Color::from_rgba8(cr, cg, cb, alpha); - let mut ring_paint = tiny_skia::Paint::default(); - ring_paint.shader = tiny_skia::Shader::SolidColor(ring_color); - ring_paint.anti_alias = true; - let stroke = tiny_skia::Stroke { width: 2.0, ..Default::default() }; - let mut pb = tiny_skia::PathBuilder::new(); - pb.push_circle(px as f32, py as f32, ring_r); - if let Some(path) = pb.finish() { - pm.stroke_path(&path, &ring_paint, &stroke, tiny_skia::Transform::identity(), None); - } - } - - if let Some(ref shape) = rs.shape { - let sz = 32.0_f32; - if let Some(pix) = tiny_skia::PixmapRef::from_bytes(&shape.pixels, shape.width, shape.height) { - let transform = tiny_skia::Transform::from_rotate_at( - heading.to_degrees() as f32 + 180.0, px as f32, py as f32, - ).pre_translate(px as f32 - sz / 2.0, py as f32 - sz / 2.0); - let mut paint = tiny_skia::PixmapPaint::default(); - paint.opacity = alpha_scale; - pm.draw_pixmap(0, 0, pix, &paint, transform, None); - } - } else { - draw_default_arrow(&mut pm, &rs.palette, px as f32, py as f32, heading as f32, alpha_scale); + // Linux uses the non-sentinel-snap behaviour for both MoveTo and + // ClickPulse: every command updates `self.pos` unconditionally. + // Custom-shape / gradient / focus-rect commands are not rendered on + // Linux at present; `apply_command_base` consumes SetShape + + // SetGradient and returns false for ShowFocusRect — both cases drop + // the visual update silently so callers don't see an error. + let _ = self.core.apply_command_base(cmd, false, false); } - - pm -} - -fn draw_default_arrow( - pm: &mut tiny_skia::Pixmap, - palette: &Palette, - px: f32, py: f32, - heading: f32, - alpha_scale: f32, -) { - let verts: [(f32, f32); 4] = [(14.0, 0.0), (-8.0, -9.0), (-3.0, 0.0), (-8.0, 9.0)]; - let angle = heading + std::f64::consts::PI as f32; - let (sa, ca) = (angle.sin(), angle.cos()); - let xform = |(vx, vy): (f32, f32)| (px + ca * vx - sa * vy, py + sa * vx + ca * vy); - let pts: Vec<(f32, f32)> = verts.iter().map(|&v| xform(v)).collect(); - let mut pb = tiny_skia::PathBuilder::new(); - pb.move_to(pts[0].0, pts[0].1); - for p in &pts[1..] { pb.line_to(p.0, p.1); } - pb.close(); - let arrow_path = match pb.finish() { Some(p) => p, None => return }; - let tip = pts[0]; - let tail = ((pts[1].0 + pts[3].0) / 2.0, (pts[1].1 + pts[3].1) / 2.0); - let [r0, g0, b0, _] = palette.cursor_start; - let [r1, g1, b1, _] = palette.cursor_mid; - let [r2, g2, b2, _] = palette.cursor_end; - let a = (255.0 * alpha_scale) as u8; - let fill_paint = { - let mut p = tiny_skia::Paint::default(); - p.shader = tiny_skia::LinearGradient::new( - tiny_skia::Point::from_xy(tip.0, tip.1), - tiny_skia::Point::from_xy(tail.0, tail.1), - vec![ - tiny_skia::GradientStop::new(0.00, tiny_skia::Color::from_rgba8(r0, g0, b0, a)), - tiny_skia::GradientStop::new(0.53, tiny_skia::Color::from_rgba8(r1, g1, b1, a)), - tiny_skia::GradientStop::new(1.00, tiny_skia::Color::from_rgba8(r2, g2, b2, a)), - ], - tiny_skia::SpreadMode::Pad, - tiny_skia::Transform::identity(), - ).unwrap_or(tiny_skia::Shader::SolidColor(tiny_skia::Color::from_rgba8(r1, g1, b1, a))); - p.anti_alias = true; - p - }; - pm.fill_path(&arrow_path, &fill_paint, tiny_skia::FillRule::Winding, - tiny_skia::Transform::identity(), None); - let mut sp = tiny_skia::Paint::default(); - sp.shader = tiny_skia::Shader::SolidColor(tiny_skia::Color::from_rgba8(255, 255, 255, a)); - sp.anti_alias = true; - let stroke = tiny_skia::Stroke { width: 1.5, ..Default::default() }; - pm.stroke_path(&arrow_path, &sp, &stroke, tiny_skia::Transform::identity(), None); } // ── X11 thread ──────────────────────────────────────────────────────────── @@ -488,7 +225,15 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver ToolResult { - let filter_pid = args.get("pid").and_then(|v| v.as_u64()).map(|v| v as u32); + use mcp_server::tool_args::ArgsExt; + let filter_pid = args.opt_u64("pid").map(|v| v as u32); let windows = tokio::task::spawn_blocking(move || crate::x11::list_windows(filter_pid)).await.unwrap_or_default(); let mut lines = vec![format!("Found {} windows:", windows.len())]; for w in &windows { @@ -338,21 +339,15 @@ impl Tool for GetWindowStateTool { } async fn invoke(&self, args: Value) -> ToolResult { - let pid = match args.get("pid").and_then(|v| v.as_u64()) { - Some(v) => v as u32, - None => return ToolResult::error("Missing required parameter: pid"), - }; - let xid = match args.get("window_id").and_then(|v| v.as_u64()) { - Some(v) => v, - None => return ToolResult::error("Missing required parameter: window_id"), - }; + use mcp_server::tool_args::ArgsExt; + let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e }; + let xid = match args.require_u64("window_id") { Ok(v) => v, Err(e) => return e }; let (default_mode, max_dim) = { let cfg = self.state.config.read().unwrap(); (cfg.capture_mode.clone(), cfg.max_image_dimension) }; - let capture_mode = args.get("capture_mode").and_then(|v| v.as_str()) - .unwrap_or(&default_mode).to_owned(); - let query = args.get("query").and_then(|v| v.as_str()).map(str::to_owned); + let capture_mode = args.str_or("capture_mode", &default_mode); + let query = args.opt_str("query"); // "ax" = tree only; "vision" = screenshot only; "som" (default) = both. let do_tree = capture_mode != "vision"; @@ -441,11 +436,10 @@ impl Tool for LaunchAppTool { } async fn invoke(&self, args: Value) -> ToolResult { - let launch_path_opt = args.get("launch_path").and_then(|v| v.as_str()).map(str::to_owned); - let name_opt = args.get("name").and_then(|v| v.as_str()).map(str::to_owned); - let urls: Vec = args.get("urls").and_then(|v| v.as_array()) - .map(|a| a.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect()) - .unwrap_or_default(); + use mcp_server::tool_args::ArgsExt; + let launch_path_opt = args.opt_str("launch_path"); + let name_opt = args.opt_str("name"); + let urls: Vec = args.str_array("urls"); if launch_path_opt.is_none() && name_opt.is_none() && urls.is_empty() { return ToolResult::error("Provide at least one of: launch_path, name, or urls."); @@ -561,20 +555,18 @@ impl Tool for ClickTool { } async fn invoke(&self, args: Value) -> ToolResult { - let pid = match args.get("pid").and_then(|v| v.as_u64()) { - Some(v) => v as u32, - None => return ToolResult::error("Missing required parameter: pid"), - }; - let count = args.get("count").and_then(|v| v.as_u64()).unwrap_or(1) as usize; - let button: u8 = match args.get("button").and_then(|v| v.as_str()).unwrap_or("left") { + use mcp_server::tool_args::ArgsExt; + let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e }; + let count = args.u64_or("count", 1) as usize; + let button: u8 = match args.str_or("button", "left").as_str() { "right" => 3, "middle" => 2, _ => 1, }; - if let Some(idx) = args.get("element_index").and_then(|v| v.as_u64()) { + if let Some(idx) = args.opt_u64("element_index") { let idx = idx as usize; - let xid_hint = args.get("window_id").and_then(|v| v.as_u64()); + let xid_hint = args.opt_u64("window_id"); // For element_index: try AT-SPI perform_action first (background-safe). // Always get bounds to send the overlay ClickPulse at the element center. let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(f64, f64)> { @@ -610,13 +602,13 @@ impl Tool for ClickTool { } // Coordinate-based path. - let xid = match args.get("window_id").and_then(|v| v.as_u64()) { + let xid = match args.opt_u64("window_id") { Some(v) => v, None => return ToolResult::error("Provide either element_index or window_id + x/y."), }; - let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false); - let mut x = args.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0); - let mut y = args.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0); + let from_zoom = args.bool_or("from_zoom", false); + let mut x = args.f64_or("x", 0.0); + let mut y = args.f64_or("y", 0.0); if from_zoom { match self.state.zoom_registry.get(pid) { Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(x, y); x = wx; y = wy; } @@ -669,12 +661,10 @@ impl Tool for TypeTextTool { } async fn invoke(&self, args: Value) -> ToolResult { - let pid = args.get("pid").and_then(|v| v.as_u64()).unwrap_or(0) as u32; - let text = match args.get("text").and_then(|v| v.as_str()) { - Some(t) => t.to_owned(), - None => return ToolResult::error("Missing required parameter: text"), - }; - let xid_opt = args.get("window_id").and_then(|v| v.as_u64()); + use mcp_server::tool_args::ArgsExt; + let pid = args.u64_or("pid", 0) as u32; + let text = match args.require_str("text") { Ok(v) => v, Err(e) => return e }; + let xid_opt = args.opt_u64("window_id"); // Resolve XID: use window_id if given, else first window for pid. let xid = match xid_opt { @@ -723,16 +713,11 @@ impl Tool for PressKeyTool { } async fn invoke(&self, args: Value) -> ToolResult { - let pid = args.get("pid").and_then(|v| v.as_u64()).unwrap_or(0) as u32; - let key = match args.get("key").and_then(|v| v.as_str()) { - Some(k) => k.to_owned(), - None => return ToolResult::error("Missing required parameter: key"), - }; - let mods: Vec = args.get("modifiers") - .and_then(|v| v.as_array()) - .map(|a| a.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect()) - .unwrap_or_default(); - let xid_opt = args.get("window_id").and_then(|v| v.as_u64()); + use mcp_server::tool_args::ArgsExt; + let pid = args.u64_or("pid", 0) as u32; + let key = match args.require_str("key") { Ok(v) => v, Err(e) => return e }; + let mods: Vec = args.str_array("modifiers"); + let xid_opt = args.opt_u64("window_id"); let xid = match xid_opt { Some(x) => x, None => { @@ -786,8 +771,9 @@ impl Tool for HotkeyTool { } async fn invoke(&self, args: Value) -> ToolResult { - let pid = args.get("pid").and_then(|v| v.as_u64()).unwrap_or(0) as u32; - let xid_opt = args.get("window_id").and_then(|v| v.as_u64()); + use mcp_server::tool_args::ArgsExt; + let pid = args.u64_or("pid", 0) as u32; + let xid_opt = args.opt_u64("window_id"); // Resolve XID: use window_id if given, else first window for pid. let xid = match xid_opt { @@ -810,11 +796,9 @@ impl Tool for HotkeyTool { return ToolResult::error("keys must include at least one non-modifier key."); } (non_mods.last().unwrap().clone(), modifiers) - } else if let Some(k) = args.get("key").and_then(|v| v.as_str()) { - let mods: Vec = args.get("modifiers").and_then(|v| v.as_array()) - .map(|a| a.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect()) - .unwrap_or_default(); - (k.to_owned(), mods) + } else if let Some(k) = args.opt_str("key") { + let mods: Vec = args.str_array("modifiers"); + (k, mods) } else { return ToolResult::error("Provide 'keys' array (e.g. [\"ctrl\",\"c\"]) or 'key'+'modifiers' parameters."); }; @@ -856,18 +840,10 @@ impl Tool for SetValueTool { } async fn invoke(&self, args: Value) -> ToolResult { - let pid = match args.get("pid").and_then(|v| v.as_u64()) { - Some(v) => v as u32, - None => return ToolResult::error("Missing required parameter: pid"), - }; - let idx = match args.get("element_index").and_then(|v| v.as_u64()) { - Some(v) => v as usize, - None => return ToolResult::error("Missing required parameter: element_index"), - }; - let value = match args.get("value").and_then(|v| v.as_str()) { - Some(v) => v.to_owned(), - None => return ToolResult::error("Missing required parameter: value"), - }; + use mcp_server::tool_args::ArgsExt; + let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e }; + let idx = match args.require_u64("element_index") { Ok(v) => v as usize, Err(e) => return e }; + let value = match args.require_str("value") { Ok(v) => v, Err(e) => return e }; let value_for_task = value.clone(); let result = tokio::task::spawn_blocking(move || { crate::atspi::set_value(pid, idx, &value_for_task) @@ -907,17 +883,11 @@ impl Tool for ScrollTool { } async fn invoke(&self, args: Value) -> ToolResult { - let pid = match args.get("pid").and_then(|v| v.as_u64()) { - Some(v) => v as u32, - None => return ToolResult::error("Missing required parameter: pid"), - }; - let direction = match args.get("direction").and_then(|v| v.as_str()) { - Some(d) => d.to_owned(), - None => return ToolResult::error("Missing required parameter: direction"), - }; - let amount = args.get("amount").and_then(|v| v.as_u64()) - .unwrap_or(3).clamp(1, 50) as usize; - let xid_opt = args.get("window_id").and_then(|v| v.as_u64()); + use mcp_server::tool_args::ArgsExt; + let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e }; + let direction = match args.require_str("direction") { Ok(v) => v, Err(e) => return e }; + let amount = args.u64_or("amount", 3).clamp(1, 50) as usize; + let xid_opt = args.opt_u64("window_id"); // Resolve XID: use window_id if given, else first window for pid. let xid = match xid_opt { @@ -983,9 +953,10 @@ impl Tool for ScreenshotTool { } async fn invoke(&self, args: Value) -> ToolResult { - let xid_opt = args.get("window_id").and_then(|v| v.as_u64()); - let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("jpeg").to_owned(); - let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(85) as u8; + use mcp_server::tool_args::ArgsExt; + let xid_opt = args.opt_u64("window_id"); + let format = args.str_or("format", "jpeg"); + let quality = args.u64_or("quality", 85) as u8; let is_jpeg = format == "jpeg"; let max_dim = self.state.config.read().unwrap().max_image_dimension; @@ -1054,13 +1025,11 @@ impl Tool for DoubleClickTool { }) } async fn invoke(&self, args: Value) -> ToolResult { - let pid = match args.get("pid").and_then(|v| v.as_u64()) { - Some(v) => v as u32, - None => return ToolResult::error("Missing required parameter: pid"), - }; - if let Some(idx) = args.get("element_index").and_then(|v| v.as_u64()) { + use mcp_server::tool_args::ArgsExt; + let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e }; + if let Some(idx) = args.opt_u64("element_index") { let idx = idx as usize; - let xid_hint = args.get("window_id").and_then(|v| v.as_u64()); + let xid_hint = args.opt_u64("window_id"); let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(u64, f64, f64)> { resolve_element_local_coords(pid, idx, xid_hint) }).await; @@ -1077,12 +1046,12 @@ impl Tool for DoubleClickTool { Err(e) => ToolResult::error(format!("Task error: {e}")), }; } - let xid = match args.get("window_id").and_then(|v| v.as_u64()) { + let xid = match args.opt_u64("window_id") { Some(v) => v, None => return ToolResult::error("Provide either element_index or window_id + x/y."), }; - let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false); - let mut x = args.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0); - let mut y = args.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0); + let from_zoom = args.bool_or("from_zoom", false); + let mut x = args.f64_or("x", 0.0); + let mut y = args.f64_or("y", 0.0); if from_zoom { match self.state.zoom_registry.get(pid) { Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(x, y); x = wx; y = wy; } @@ -1133,13 +1102,11 @@ impl Tool for RightClickTool { }) } async fn invoke(&self, args: Value) -> ToolResult { - let pid = match args.get("pid").and_then(|v| v.as_u64()) { - Some(v) => v as u32, - None => return ToolResult::error("Missing required parameter: pid"), - }; - if let Some(idx) = args.get("element_index").and_then(|v| v.as_u64()) { + use mcp_server::tool_args::ArgsExt; + let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e }; + if let Some(idx) = args.opt_u64("element_index") { let idx = idx as usize; - let xid_hint = args.get("window_id").and_then(|v| v.as_u64()); + let xid_hint = args.opt_u64("window_id"); let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(u64, f64, f64)> { resolve_element_local_coords(pid, idx, xid_hint) }).await; @@ -1156,12 +1123,12 @@ impl Tool for RightClickTool { Err(e) => ToolResult::error(format!("Task error: {e}")), }; } - let xid = match args.get("window_id").and_then(|v| v.as_u64()) { + let xid = match args.opt_u64("window_id") { Some(v) => v, None => return ToolResult::error("Provide either element_index or window_id + x/y."), }; - let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false); - let mut x = args.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0); - let mut y = args.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0); + let from_zoom = args.bool_or("from_zoom", false); + let mut x = args.f64_or("x", 0.0); + let mut y = args.f64_or("y", 0.0); if from_zoom { match self.state.zoom_registry.get(pid) { Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(x, y); x = wx; y = wy; } @@ -1216,27 +1183,25 @@ impl Tool for DragTool { }) } async fn invoke(&self, args: Value) -> ToolResult { - let pid = match args.get("pid").and_then(|v| v.as_u64()) { - Some(v) => v as u32, None => return ToolResult::error("Missing required parameter: pid"), - }; - let xid = match args.get("window_id").and_then(|v| v.as_u64()) { + use mcp_server::tool_args::ArgsExt; + let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e }; + let xid = match args.opt_u64("window_id") { Some(v) => v, None => return ToolResult::error("window_id is required on Linux."), }; let coerce = |key: &str| -> Option { - args.get(key).and_then(|v| v.as_f64()) - .or_else(|| args.get(key).and_then(|v| v.as_i64()).map(|i| i as f64)) + args.opt_f64(key).or_else(|| args.opt_i64(key).map(|i| i as f64)) }; let mut from_x = match coerce("from_x") { Some(v) => v, None => return ToolResult::error("Missing: from_x") }; let mut from_y = match coerce("from_y") { Some(v) => v, None => return ToolResult::error("Missing: from_y") }; let mut to_x = match coerce("to_x") { Some(v) => v, None => return ToolResult::error("Missing: to_x") }; let mut to_y = match coerce("to_y") { Some(v) => v, None => return ToolResult::error("Missing: to_y") }; - let duration_ms = args.get("duration_ms").and_then(|v| v.as_u64()).unwrap_or(500); - let steps = args.get("steps").and_then(|v| v.as_u64()).unwrap_or(20) as usize; - let button_str = args.get("button").and_then(|v| v.as_str()).unwrap_or("left"); - let button: u8 = match button_str { "right" => 3, "middle" => 2, _ => 1 }; - let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false); + let duration_ms = args.u64_or("duration_ms", 500); + let steps = args.u64_or("steps", 20) as usize; + let button_str = args.str_or("button", "left"); + let button: u8 = match button_str.as_str() { "right" => 3, "middle" => 2, _ => 1 }; + let from_zoom = args.bool_or("from_zoom", false); if from_zoom { match self.state.zoom_registry.get(pid) { @@ -1373,10 +1338,11 @@ impl Tool for MoveCursorTool { }) } async fn invoke(&self, args: Value) -> ToolResult { - let x = args.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0); - let y = args.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0); - let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default"); - self.state.cursor_registry.update_position(cursor_id, x, y); + use mcp_server::tool_args::ArgsExt; + let x = args.f64_or("x", 0.0); + let y = args.f64_or("y", 0.0); + let cursor_id = args.str_or("cursor_id", "default"); + self.state.cursor_registry.update_position(&cursor_id, x, y); // End pointing upper-left (45°) — matches Swift's // `AgentCursor.animateAndWait(endAngleDegrees: 45)` convention so the // overlay arrow settles to the natural macOS-style pose. @@ -1408,11 +1374,10 @@ impl Tool for SetAgentCursorEnabledTool { }) } async fn invoke(&self, args: Value) -> ToolResult { - let enabled = match args.get("enabled").and_then(|v| v.as_bool()) { - Some(v) => v, None => return ToolResult::error("Missing required parameter: enabled"), - }; - let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default"); - self.state.cursor_registry.set_enabled(cursor_id, enabled); + use mcp_server::tool_args::ArgsExt; + let enabled = match args.require_bool("enabled") { Ok(v) => v, Err(e) => return e }; + let cursor_id = args.str_or("cursor_id", "default"); + self.state.cursor_registry.set_enabled(&cursor_id, enabled); crate::overlay::send_command(cursor_overlay::OverlayCommand::SetEnabled(enabled)); ToolResult::text(format!("Agent cursor '{cursor_id}' {}.", if enabled { "enabled" } else { "disabled" })) } @@ -1452,13 +1417,14 @@ impl Tool for SetAgentCursorMotionTool { }) } async fn invoke(&self, args: Value) -> ToolResult { - let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default").to_owned(); + use mcp_server::tool_args::ArgsExt; + let cursor_id = args.str_or("cursor_id", "default"); self.state.cursor_registry.update_config(&cursor_id, |cfg| { - if let Some(v) = args.get("cursor_icon").and_then(|v| v.as_str()) { cfg.cursor_icon = Some(v.to_owned()); } - if let Some(v) = args.get("cursor_color").and_then(|v| v.as_str()) { cfg.cursor_color = Some(v.to_owned()); } - if let Some(v) = args.get("cursor_label").and_then(|v| v.as_str()) { cfg.cursor_label = Some(v.to_owned()); } - if let Some(v) = args.get("cursor_size").and_then(|v| v.as_f64()) { cfg.cursor_size = Some(v); } - if let Some(v) = args.get("cursor_opacity").and_then(|v| v.as_f64()) { cfg.cursor_opacity = Some(v.clamp(0.0, 1.0)); } + if let Some(v) = args.opt_str("cursor_icon") { cfg.cursor_icon = Some(v); } + if let Some(v) = args.opt_str("cursor_color") { cfg.cursor_color = Some(v); } + if let Some(v) = args.opt_str("cursor_label") { cfg.cursor_label = Some(v); } + if let Some(v) = args.opt_f64("cursor_size") { cfg.cursor_size = Some(v); } + if let Some(v) = args.opt_f64("cursor_opacity") { cfg.cursor_opacity = Some(v.clamp(0.0, 1.0)); } }); ToolResult::text(format!("Cursor '{cursor_id}' config updated.")).with_structured(args) } @@ -1543,7 +1509,8 @@ impl Tool for SetAgentCursorStyleTool { } async fn invoke(&self, args: Value) -> ToolResult { - let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default").to_owned(); + use mcp_server::tool_args::ArgsExt; + let cursor_id = args.str_or("cursor_id", "default"); // image_path let image_path = args.get("image_path").and_then(|v| v.as_str()); @@ -1741,13 +1708,14 @@ impl Tool for SetConfigTool { }) } async fn invoke(&self, args: Value) -> ToolResult { + use mcp_server::tool_args::ArgsExt; let mut cfg = self.state.config.write().unwrap(); let mut parts = Vec::new(); - if let Some(mode) = args.get("capture_mode").and_then(|v| v.as_str()) { - cfg.capture_mode = mode.to_owned(); + if let Some(mode) = args.opt_str("capture_mode") { parts.push(format!("capture_mode={mode}")); + cfg.capture_mode = mode; } - if let Some(dim) = args.get("max_image_dimension").and_then(|v| v.as_u64()) { + if let Some(dim) = args.opt_u64("max_image_dimension") { cfg.max_image_dimension = dim as u32; parts.push(format!("max_image_dimension={dim}")); } @@ -1850,14 +1818,13 @@ impl Tool for ZoomTool { } async fn invoke(&self, args: Value) -> ToolResult { - let xid = match args.get("window_id").and_then(|v| v.as_u64()) { - Some(v) => v, None => return ToolResult::error("Missing required parameter: window_id"), - }; - let pid = args.get("pid").and_then(|v| v.as_u64()).map(|v| v as u32); - let x1 = match args.get("x1").and_then(|v| v.as_f64()) { Some(v) => v, None => return ToolResult::error("Missing x1") }; - let y1 = match args.get("y1").and_then(|v| v.as_f64()) { Some(v) => v, None => return ToolResult::error("Missing y1") }; - let x2 = match args.get("x2").and_then(|v| v.as_f64()) { Some(v) => v, None => return ToolResult::error("Missing x2") }; - let y2 = match args.get("y2").and_then(|v| v.as_f64()) { Some(v) => v, None => return ToolResult::error("Missing y2") }; + use mcp_server::tool_args::ArgsExt; + let xid = match args.require_u64("window_id") { Ok(v) => v, Err(e) => return e }; + let pid = args.opt_u64("pid").map(|v| v as u32); + let x1 = match args.opt_f64("x1") { Some(v) => v, None => return ToolResult::error("Missing x1") }; + let y1 = match args.opt_f64("y1") { Some(v) => v, None => return ToolResult::error("Missing y1") }; + let x2 = match args.opt_f64("x2") { Some(v) => v, None => return ToolResult::error("Missing x2") }; + let y2 = match args.opt_f64("y2") { Some(v) => v, None => return ToolResult::error("Missing y2") }; if x2 <= x1 || y2 <= y1 { return ToolResult::error("x2 must be > x1 and y2 must be > y1"); } let state = self.state.clone(); @@ -1922,12 +1889,11 @@ impl Tool for TypeTextCharsTool { } async fn invoke(&self, args: Value) -> ToolResult { - let pid = args.get("pid").and_then(|v| v.as_u64()).unwrap_or(0) as u32; - let text = match args.get("text").and_then(|v| v.as_str()) { - Some(t) => t.to_owned(), None => return ToolResult::error("Missing required parameter: text"), - }; - let delay_ms = args.get("delay_ms").and_then(|v| v.as_u64()).unwrap_or(30); - let xid_opt = args.get("window_id").and_then(|v| v.as_u64()); + use mcp_server::tool_args::ArgsExt; + let pid = args.u64_or("pid", 0) as u32; + let text = match args.require_str("text") { Ok(v) => v, Err(e) => return e }; + let delay_ms = args.u64_or("delay_ms", 30); + let xid_opt = args.opt_u64("window_id"); let xid = match xid_opt { Some(x) => x, None => { diff --git a/libs/cua-driver-rs/crates/platform-macos/src/ax/cache.rs b/libs/cua-driver-rs/crates/platform-macos/src/ax/cache.rs index f0a1c1180f..b962bef829 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/ax/cache.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/ax/cache.rs @@ -10,12 +10,16 @@ //! Memory contract: //! tree::walk_element retains each actionable element before storing its ptr. //! CachedSnapshot::drop releases those retains so we have no AX leaks. +//! +//! The locked-HashMap plumbing lives in `mcp_server::element_cache` — see +//! `docs/dedup-audit.md` item #3. This module owns the macOS-specific +//! `CacheKey`, `CachedSnapshot`, and the `Drop` impl that fires `CFRelease` +//! when an entry is replaced or removed. use super::bindings::AXUIElementRef; use super::tree::AXNode; use core_foundation::base::{CFRelease, CFTypeRef}; -use std::collections::HashMap; -use std::sync::Mutex; +use mcp_server::element_cache::ElementCacheCore; /// Key for the element cache. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -43,12 +47,12 @@ impl Drop for CachedSnapshot { /// Global element cache. pub struct ElementCache { - inner: Mutex>, + core: ElementCacheCore, } impl ElementCache { pub fn new() -> Self { - Self { inner: Mutex::new(HashMap::new()) } + Self { core: ElementCacheCore::new() } } /// Replace the snapshot for (pid, window_id) with the nodes from a fresh walk. @@ -58,20 +62,21 @@ impl ElementCache { .filter(|n| n.element_index.is_some()) .map(|n| n.element_ptr) .collect(); - let mut inner = self.inner.lock().unwrap(); - inner.insert(CacheKey { pid, window_id }, CachedSnapshot { elements }); + self.core.insert(CacheKey { pid, window_id }, CachedSnapshot { elements }); } /// Look up the raw AXUIElementRef pointer for `element_index` in (pid, window_id). pub fn get_element_ptr(&self, pid: i32, window_id: u32, element_index: usize) -> Option { - let inner = self.inner.lock().unwrap(); - inner.get(&CacheKey { pid, window_id })?.elements.get(element_index).copied() + self.core + .with_snapshot(&CacheKey { pid, window_id }, |s| s.elements.get(element_index).copied()) + .flatten() } /// Number of indexed elements for (pid, window_id), or 0 if not cached. pub fn element_count(&self, pid: i32, window_id: u32) -> usize { - let inner = self.inner.lock().unwrap(); - inner.get(&CacheKey { pid, window_id }).map(|s| s.elements.len()).unwrap_or(0) + self.core + .with_snapshot(&CacheKey { pid, window_id }, |s| s.elements.len()) + .unwrap_or(0) } } diff --git a/libs/cua-driver-rs/crates/platform-macos/src/capture.rs b/libs/cua-driver-rs/crates/platform-macos/src/capture.rs index 66c58eecb3..2a20d4e28c 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/capture.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/capture.rs @@ -79,179 +79,45 @@ pub fn screenshot_display() -> anyhow::Result<(String, u32, u32)> { Ok((b64, w, h)) } +// PNG/JPEG/resize/crosshair helpers — re-exports of the shared +// `mcp_server::image_utils` module. The previous file-local copies were +// near-identical to the Windows and Linux versions; the dedup-audit +// (2026-05) moved them all to one place. See +// `CUA_DRIVER_RS_DEDUP_AUDIT.md` for the audit trail. + /// Convert raw PNG bytes to JPEG at the given quality (1-95). -/// Uses the `cursor_overlay::capture_utils` JPEG encoder. pub fn png_bytes_to_jpeg(png_bytes: &[u8], quality: u8) -> anyhow::Result> { - use image::ImageDecoder; - let cursor = std::io::Cursor::new(png_bytes); - let decoder = image::codecs::png::PngDecoder::new(cursor)?; - let (w, h) = decoder.dimensions(); - let color = decoder.color_type(); - let mut buf = vec![0u8; decoder.total_bytes() as usize]; - decoder.read_image(&mut buf)?; - - // Ensure RGBA → RGB for JPEG encoding. - let rgb_buf: Vec = if color == image::ColorType::Rgba8 { - buf.chunks_exact(4).flat_map(|px| [px[0], px[1], px[2]]).collect() - } else { - buf - }; - - let mut jpeg_bytes = Vec::new(); - let mut enc = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg_bytes, quality); - enc.encode(&rgb_buf, w, h, image::ColorType::Rgb8.into())?; - Ok(jpeg_bytes) + mcp_server::image_utils::png_bytes_to_jpeg(png_bytes, quality) } /// Downscale `png_bytes` so neither dimension exceeds `max_dim`. -/// If `max_dim == 0` or the image already fits, returns the original bytes unchanged. +/// If `max_dim == 0` or the image already fits, returns the original +/// bytes unchanged. pub fn resize_png_if_needed(png_bytes: &[u8], max_dim: u32) -> anyhow::Result> { - if max_dim == 0 { - return Ok(png_bytes.to_vec()); - } - let (w, h) = png_dimensions(png_bytes)?; - if w <= max_dim && h <= max_dim { - return Ok(png_bytes.to_vec()); - } - // Determine scale factor to fit within max_dim x max_dim. - let scale = (max_dim as f64) / (w.max(h) as f64); - let new_w = (w as f64 * scale).round() as u32; - let new_h = (h as f64 * scale).round() as u32; - - use image::ImageDecoder; - let cursor = std::io::Cursor::new(png_bytes); - let decoder = image::codecs::png::PngDecoder::new(cursor)?; - let color = decoder.color_type(); - let mut buf = vec![0u8; decoder.total_bytes() as usize]; - decoder.read_image(&mut buf)?; - - let img = match color { - image::ColorType::Rgba8 => { - image::DynamicImage::ImageRgba8( - image::ImageBuffer::from_raw(w, h, buf) - .ok_or_else(|| anyhow::anyhow!("invalid RGBA buffer"))?, - ) - } - image::ColorType::Rgb8 => { - image::DynamicImage::ImageRgb8( - image::ImageBuffer::from_raw(w, h, buf) - .ok_or_else(|| anyhow::anyhow!("invalid RGB buffer"))?, - ) - } - _ => anyhow::bail!("unsupported color type for resize: {color:?}"), - }; - - let resized = img.resize(new_w, new_h, image::imageops::FilterType::Lanczos3); - let mut out = Vec::new(); - resized.write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png)?; - Ok(out) + mcp_server::image_utils::resize_png_if_needed(png_bytes, max_dim) } -/// Draw a red crosshair at pixel (cx, cy) on a PNG image and write to `path`. -/// Used by `click`'s `debug_image_out` param to verify coordinate spaces. -/// The crosshair uses top-left-origin coords matching the click tool's convention. +/// Draw a red crosshair at pixel (cx, cy) on a PNG image and write to +/// `path`. Used by `click`'s `debug_image_out` param to verify +/// coordinate spaces. The crosshair uses top-left-origin coords +/// matching the click tool's convention. pub fn write_crosshair_png( png_bytes: &[u8], cx: f64, cy: f64, path: &str, ) -> anyhow::Result<()> { - use image::{ImageDecoder, DynamicImage}; - - let cursor = std::io::Cursor::new(png_bytes); - let decoder = image::codecs::png::PngDecoder::new(cursor)?; - let (w, h) = decoder.dimensions(); - let color = decoder.color_type(); - let mut buf = vec![0u8; decoder.total_bytes() as usize]; - decoder.read_image(&mut buf)?; - - let mut img: DynamicImage = match color { - image::ColorType::Rgba8 => DynamicImage::ImageRgba8( - image::ImageBuffer::from_raw(w, h, buf) - .ok_or_else(|| anyhow::anyhow!("invalid RGBA buffer"))?, - ), - image::ColorType::Rgb8 => DynamicImage::ImageRgb8( - image::ImageBuffer::from_raw(w, h, buf) - .ok_or_else(|| anyhow::anyhow!("invalid RGB buffer"))?, - ).into(), - _ => anyhow::bail!("unsupported color type for crosshair: {color:?}"), - }; - // Ensure RGBA for drawing. - let mut img = img.to_rgba8(); - - // Crosshair geometry. - let ring_r = (w as f64 / 80.0).max(6.0) as i32; - let arm_len = (w as f64 / 40.0).max(12.0) as i32; - let line_w = ((w as f64 / 400.0).max(1.5)) as i32; - let red = image::Rgba([255u8, 26, 26, 242]); - let cx = cx as i32; - let cy = cy as i32; - - // Draw horizontal + vertical arms. - for lw in 0..=line_w { - let off = lw - line_w / 2; - for dx in -arm_len..=arm_len { - if let Some(p) = img.get_pixel_mut_checked( - (cx + dx).clamp(0, w as i32 - 1) as u32, - (cy + off).clamp(0, h as i32 - 1) as u32, - ) { *p = red; } - } - for dy in -arm_len..=arm_len { - if let Some(p) = img.get_pixel_mut_checked( - (cx + off).clamp(0, w as i32 - 1) as u32, - (cy + dy).clamp(0, h as i32 - 1) as u32, - ) { *p = red; } - } - } - - // Draw ring (stroke circle). - let steps = (ring_r * 12).max(48) as usize; - for i in 0..steps { - let theta = 2.0 * std::f64::consts::PI * i as f64 / steps as f64; - let rx = (cx as f64 + ring_r as f64 * theta.cos()) as i32; - let ry = (cy as f64 + ring_r as f64 * theta.sin()) as i32; - for lw in 0..=line_w { - let off = lw - line_w / 2; - for dx in 0..=1 { - let fx = (rx + off + dx).clamp(0, w as i32 - 1) as u32; - let fy = (ry + off).clamp(0, h as i32 - 1) as u32; - *img.get_pixel_mut(fx, fy) = red; - } - } - } - - // Write PNG to path. - let path = if let Some(rest) = path.strip_prefix('~') { - format!("{}{}", std::env::var("HOME").unwrap_or_default(), rest) - } else { - path.to_owned() - }; - img.save_with_format(&path, image::ImageFormat::Png)?; - Ok(()) + mcp_server::image_utils::write_crosshair_png(png_bytes, cx, cy, path) } -/// Draw a red crosshair at pixel (cx, cy) on a PNG image and return the modified PNG bytes. -/// Used by recording's click-marker callback to produce click.png. +/// Draw a red crosshair at pixel (cx, cy) on a PNG image and return the +/// modified PNG bytes. Used by recording's click-marker callback to +/// produce click.png. pub fn crosshair_png_bytes(png_bytes: &[u8], cx: f64, cy: f64) -> anyhow::Result> { - use std::io::Cursor; - let tmp_path = format!("/tmp/cua-driver-rs-clickmarker-{}.png", std::process::id()); - write_crosshair_png(png_bytes, cx, cy, &tmp_path)?; - let out = std::fs::read(&tmp_path)?; - let _ = std::fs::remove_file(&tmp_path); - Ok(out) + mcp_server::image_utils::crosshair_png_bytes(png_bytes, cx, cy) } /// Parse width and height from a PNG file's IHDR chunk. pub fn png_dimensions(data: &[u8]) -> anyhow::Result<(u32, u32)> { - // PNG signature: 8 bytes, IHDR chunk: 4-byte length, "IHDR", 4-byte width, 4-byte height - if data.len() < 24 { - anyhow::bail!("PNG data too small"); - } - // Signature check - if &data[0..8] != b"\x89PNG\r\n\x1a\n" { - anyhow::bail!("Not a PNG file"); - } - let w = u32::from_be_bytes([data[16], data[17], data[18], data[19]]); - let h = u32::from_be_bytes([data[20], data[21], data[22], data[23]]); - Ok((w, h)) + mcp_server::image_utils::png_dimensions(data) } diff --git a/libs/cua-driver-rs/crates/platform-macos/src/cursor/overlay.rs b/libs/cua-driver-rs/crates/platform-macos/src/cursor/overlay.rs index 1091d8a8ba..febfc2d6aa 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/cursor/overlay.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/cursor/overlay.rs @@ -19,14 +19,22 @@ //! (matching `OverlayCommand::MoveTo` and AX element coordinates). The //! NSWindow covers `NSScreen.mainScreen.frame` which AppKit places with //! a bottom-left origin, so we flip Y when drawing into the Pixmap. +//! +//! ## Cross-platform note (2026-05 dedup audit) +//! +//! Animation state + render pipeline live in `cursor_overlay::render_state` +//! (`RenderStateCore`, `tick_swift_constants`, `apply_command_base`, +//! `render_frame`). macOS uses the hardcoded Swift reference constants +//! (peakSpeed=900, springK=400, overshoot=0.8) and the sentinel-snap +//! variants of MoveTo / ClickPulse — see the wrapper around +//! `apply_command_base` below. use std::ffi::c_void; use std::sync::{Mutex, OnceLock}; use std::time::{Duration, Instant}; use cursor_overlay::{ - CursorConfig, CursorShape, MotionConfig, OverlayCommand, Palette, PathPlanner, PathState, - PlannedPath, + CursorConfig, FocusRect, MotionConfig, OverlayCommand, RenderStateCore, }; // ── Arrival-signal channel ──────────────────────────────────────────────── @@ -61,7 +69,7 @@ pub fn send_command(cmd: OverlayCommand) { pub fn current_motion() -> MotionConfig { RENDER.lock().unwrap() .as_ref() - .map(|rs| rs.motion.clone()) + .map(|rs| rs.core.motion.clone()) .unwrap_or_default() } @@ -78,7 +86,7 @@ pub async fn animate_cursor_to(x: f64, y: f64) { let should_animate = { let guard = RENDER.lock().unwrap(); match guard.as_ref() { - Some(rs) if rs.cfg.enabled && rs.pos.0 > -50.0 => true, + Some(rs) if rs.core.cfg.enabled && rs.core.pos.0 > -50.0 => true, _ => false, } }; @@ -128,7 +136,7 @@ pub fn run_on_main_thread() { let cfg = { let guard = RENDER.lock().unwrap(); match &*guard { - Some(rs) => rs.cfg.clone(), + Some(rs) => rs.core.cfg.clone(), None => return, } }; @@ -144,155 +152,43 @@ pub fn run_on_main_thread() { } // ── Animation / render state ────────────────────────────────────────────── +// +// The platform-agnostic fields + tick + apply_command + render pipeline live +// in `cursor_overlay::render_state` (2026-05 dedup audit). What stays here +// is the macOS-specific NSScreen window dimensions and the focus-rect +// overlay (a macOS-only post-arrival element highlight). struct RenderState { - cfg: CursorConfig, - palette: Palette, - motion: MotionConfig, - /// Current rendered position (screen top-left coords). - pos: (f64, f64), - /// Visual heading in radians (tip direction = motion_dir + π). - heading: f64, - /// In-flight path (None = at rest). - path: Option, - dist: f64, // arc-distance travelled so far - /// Spring settle after path arrival. - spring: Option, - spring_tgt: Option<(f64, f64, f64)>, // (x, y, heading) - /// Flash phase for ClickPulse (0..1 or None). - click_t: Option, - /// Window (frame) size. - win_w: f64, - win_h: f64, - /// Custom cursor shape (if any); None = gradient arrow. - shape: Option, - /// Runtime-overridden gradient colours (from set_agent_cursor_style). - /// Each entry is [R, G, B, A]. Empty = use palette defaults. - gradient_colors: Vec<[u8; 4]>, - /// Runtime-overridden bloom colour (from set_agent_cursor_style). - /// None = use palette default. - bloom_override: Option<[u8; 4]>, - /// Whether the overlay is visible (user-controlled). - visible: bool, - /// Idle-hide: elapsed seconds since last activity. - idle_secs: f64, - /// Idle-hide fade: 1.0 = fully visible, 0.0 = fully hidden. - idle_alpha: f64, - /// Currently pinned window id for z-ordering. - pinned_wid: Option, + core: RenderStateCore, + /// Window (frame) size in NSScreen points (top-left origin after Y-flip). + win_w: f64, + win_h: f64, /// Focus-highlight rectangle `[x, y, w, h]` in screen coords; None = not shown. focus_rect: Option<[f64; 4]>, /// Fade progress for the focus rect: 0.0 = fully visible, 1.0 = gone. focus_rect_t: f64, } -#[derive(Clone, Copy)] -struct Spring { - ox: f64, oy: f64, - vx: f64, vy: f64, -} - impl RenderState { fn new(cfg: CursorConfig) -> Self { - let palette = cfg.palette(); - let motion = cfg.motion.clone(); - let shape = cfg.shape.clone(); RenderState { - cfg, palette, motion, shape, - gradient_colors: vec![], - bloom_override: None, - pos: (-200.0, -200.0), - heading: std::f64::consts::FRAC_PI_4, - path: None, - dist: 0.0, - spring: None, - spring_tgt: None, - click_t: None, - win_w: 0.0, - win_h: 0.0, - visible: true, - idle_secs: 0.0, - idle_alpha: 1.0, - pinned_wid: None, + core: RenderStateCore::new(cfg), + win_w: 0.0, + win_h: 0.0, focus_rect: None, focus_rect_t: 1.0, } } + /// Advance the animation by `dt`. Uses the Swift reference constants + /// (peakSpeed=900, springK=400, overshoot=0.8) — see + /// [`RenderStateCore::tick_swift_constants`]. Returns true if an + /// arrival signal should be fired (the path just ended). fn tick(&mut self, dt: f64) -> bool { - // Returns true if an arrival signal should be fired (path just ended). - // Swift constants. - const PEAK_SPEED: f64 = 900.0; - const MIN_START_SPEED: f64 = 300.0; - const MIN_END_SPEED: f64 = 200.0; - const SPRING_K: f64 = 400.0; - const SPRING_C: f64 = 17.0; - const SPRING_OVERSHOOT: f64 = 0.8; - - let mut fire_arrival = false; - - if let Some(ref p) = self.path { - let path_len = p.length.max(1.0); - let u = (self.dist / path_len).min(1.0); - - // Smootherstep speed profile (normalised: peak = 1.0). - let profile = (30.0 * u * u * (1.0 - u) * (1.0 - u)) / 1.875; - let floor_speed = if u < 0.5 { MIN_START_SPEED } else { MIN_END_SPEED }; - let current_speed = floor_speed + (PEAK_SPEED - floor_speed) * profile; - self.dist += current_speed * dt; - - if self.dist >= path_len { - // Transition to spring settle. - let end = p.sample(path_len); - let end_heading = p.end_visual_heading; - let vh = end.heading; - self.spring = Some(Spring { - ox: 0.0, oy: 0.0, - vx: current_speed * SPRING_OVERSHOOT * vh.cos(), - vy: current_speed * SPRING_OVERSHOOT * vh.sin(), - }); - self.spring_tgt = Some((end.x, end.y, end_heading)); - self.pos = (end.x, end.y); - self.heading = end_heading; - self.path = None; - self.dist = 0.0; - fire_arrival = true; - } else { - let s: PathState = p.sample(self.dist); - self.pos = (s.x, s.y); - // Smooth heading rotation toward motion heading. - let desired = s.heading + std::f64::consts::PI; - let max_step = 14.0 * dt; - self.heading = rotate_toward(self.heading, desired, max_step); - } - } else if let Some(mut s) = self.spring { - if let Some((tx, ty, th)) = self.spring_tgt { - let substeps = 4; - let sdt = dt / substeps as f64; - for _ in 0..substeps { - s.vx += (-SPRING_K * s.ox - SPRING_C * s.vx) * sdt; - s.vy += (-SPRING_K * s.oy - SPRING_C * s.vy) * sdt; - s.ox += s.vx * sdt; - s.oy += s.vy * sdt; - } - self.pos = (tx + s.ox, ty + s.oy); - self.heading = th; - if s.ox.hypot(s.oy) < 0.3 && s.vx.hypot(s.vy) < 2.0 { - self.pos = (tx, ty); - self.spring = None; - } else { - self.spring = Some(s); - } - } - } - - // Advance click pulse. - if let Some(t) = self.click_t { - let next = t + dt * 4.0; // full pulse over 0.25s - self.click_t = if next >= 1.0 { None } else { Some(next) }; - } + let fire_arrival = self.core.tick_swift_constants(dt); - // Advance focus-rect fade (fades out over ~600ms). + // Advance focus-rect fade (fades out over ~600ms). macOS-only — + // the shared core has no focus_rect concept. if self.focus_rect.is_some() { self.focus_rect_t = (self.focus_rect_t + dt / 0.6).min(1.0); if self.focus_rect_t >= 1.0 { @@ -301,324 +197,26 @@ impl RenderState { } } - // Idle-hide: accumulate idle time; fade out over 180ms once threshold reached. - let idle_hide_ms = self.motion.idle_hide_ms; - if idle_hide_ms > 0.0 { - let moving = self.path.is_some() - || self.spring.is_some() - || self.click_t.is_some(); - if moving { - self.idle_secs = 0.0; - self.idle_alpha = 1.0; - } else { - self.idle_secs += dt; - let fade_start = idle_hide_ms / 1000.0; - let fade_end = fade_start + 0.18; // 180ms fade like Windows ref - if self.idle_secs > fade_end { - self.idle_alpha = 0.0; - } else if self.idle_secs > fade_start { - let t = (self.idle_secs - fade_start) / 0.18; - self.idle_alpha = 1.0 - t.clamp(0.0, 1.0); - } - } - } else { - self.idle_alpha = 1.0; - } - fire_arrival } fn apply_command(&mut self, cmd: OverlayCommand) { + // macOS uses the sentinel-snap variants of MoveTo / ClickPulse: + // - MoveTo only snaps `self.pos` if the cursor is still at the + // off-screen sentinel `(-200, -200)` (otherwise the path starts + // from the current position so the animation is continuous). + // - ClickPulse only updates `self.pos` if the cursor is still at + // the sentinel (otherwise the animation already landed it there). match cmd { - OverlayCommand::MoveTo { x, y, end_heading_radians } => { - // Apply click offset (16 pt along end_heading) before planning, - // matching Swift `moveTo(point:endAngleRadians:)`: - // tx = clickPoint.x + cos(endAngle) * clickOffset - // ty = clickPoint.y + sin(endAngle) * clickOffset - const CLICK_OFFSET: f64 = 16.0; - const TURN_RADIUS: f64 = 80.0; - let tx = x + end_heading_radians.cos() * CLICK_OFFSET; - let ty = y + end_heading_radians.sin() * CLICK_OFFSET; - - // If the cursor is still at the initial off-screen sentinel, - // snap it to the offset target so the path starts on-screen. - if self.pos.0 < -50.0 { - self.pos = (tx, ty); - } - let (x0, y0) = self.pos; - let th0 = self.heading + std::f64::consts::PI; - let th1 = end_heading_radians + std::f64::consts::PI; - let plan = PathPlanner::plan( - x0, y0, th0, - tx, ty, th1, - end_heading_radians, - TURN_RADIUS, - ); - self.path = Some(plan); - self.dist = 0.0; - self.spring = None; - self.spring_tgt = None; - self.idle_secs = 0.0; - self.idle_alpha = 1.0; - } - OverlayCommand::ClickPulse { x, y } => { - // Only snap position on first placement (sentinel state). - // After that the cursor stays where the animation landed. - if self.pos.0 < -50.0 { - // Apply same click offset so tip lands at click point. - const CLICK_OFFSET: f64 = 16.0; - let angle = std::f64::consts::FRAC_PI_4; - self.pos = (x + angle.cos() * CLICK_OFFSET, y + angle.sin() * CLICK_OFFSET); - } - self.click_t = Some(0.0); - self.idle_secs = 0.0; - self.idle_alpha = 1.0; - } - OverlayCommand::SetEnabled(v) => { - self.visible = v; - } - OverlayCommand::SetMotion(m) => { - self.motion = m; - } - OverlayCommand::SetPalette(p) => { - self.palette = p; - } - OverlayCommand::PinAbove(wid) => { - self.pinned_wid = Some(wid); - } - OverlayCommand::SetShape(shape) => { - self.shape = shape; - } - OverlayCommand::SetGradient { gradient_colors, bloom_color } => { - self.gradient_colors = gradient_colors; - self.bloom_override = bloom_color; - } OverlayCommand::ShowFocusRect(rect) => { self.focus_rect = rect; self.focus_rect_t = 0.0; // reset fade to fully visible } - } - } -} - -fn rotate_toward(current: f64, desired: f64, max_step: f64) -> f64 { - let mut diff = desired - current; - while diff > std::f64::consts::PI { diff -= 2.0 * std::f64::consts::PI; } - while diff < -std::f64::consts::PI { diff += 2.0 * std::f64::consts::PI; } - current + diff.clamp(-max_step, max_step) -} - -// ── tiny-skia rendering ─────────────────────────────────────────────────── - -fn render_frame(rs: &RenderState) -> tiny_skia::Pixmap { - let w = rs.win_w.max(1.0) as u32; - let h = rs.win_h.max(1.0) as u32; - let mut pm = tiny_skia::Pixmap::new(w, h).unwrap_or_else(|| tiny_skia::Pixmap::new(1, 1).unwrap()); - - if !rs.visible || rs.pos.0 < -100.0 || rs.idle_alpha < 0.004 { - return pm; - } - - let (px, py) = rs.pos; - let heading = rs.heading; - let alpha_scale = rs.idle_alpha as f32; - - // --- Bloom (radial gradient behind the arrow) --- - let bloom_r: f32 = 22.0; - // Use runtime bloom_override if set, otherwise fall back to palette. - let (br, bg, bb) = if let Some([r, g, b, _]) = rs.bloom_override { - (r, g, b) - } else { - let [r, g, b, _] = rs.palette.bloom_inner; - (r, g, b) - }; - let bloom_inner = tiny_skia::Color::from_rgba8(br, bg, bb, (115.0 * alpha_scale) as u8); - let (or_, og, ob) = if let Some([r, g, b, _]) = rs.bloom_override { - (r, g, b) - } else { - let [r, g, b, _] = rs.palette.bloom_outer; - (r, g, b) - }; - let bloom_outer = tiny_skia::Color::from_rgba8(or_, og, ob, (26.0 * alpha_scale) as u8); - let bloom_zero = tiny_skia::Color::from_rgba8(or_, og, ob, 0); - - let bloom_paint = { - let mut p = tiny_skia::Paint::default(); - p.shader = tiny_skia::RadialGradient::new( - tiny_skia::Point::from_xy(px as f32, py as f32), - tiny_skia::Point::from_xy(px as f32, py as f32), // focal = center - bloom_r, - vec![ - tiny_skia::GradientStop::new(0.0, bloom_inner), - tiny_skia::GradientStop::new(0.5, bloom_outer), - tiny_skia::GradientStop::new(1.0, bloom_zero), - ], - tiny_skia::SpreadMode::Pad, - tiny_skia::Transform::identity(), - ).unwrap_or(tiny_skia::Shader::SolidColor(bloom_inner)); - p.anti_alias = true; - p - }; - - let bloom_rect = tiny_skia::Rect::from_xywh( - (px - bloom_r as f64) as f32, (py - bloom_r as f64) as f32, - bloom_r * 2.0, bloom_r * 2.0, - ); - if let Some(r) = bloom_rect { - pm.fill_rect(r, &bloom_paint, tiny_skia::Transform::identity(), None); - } - - // --- Focus rect highlight --- - // Cyan glow border + faint fill, matching Swift AgentCursor.showFocusRect. - if let Some([fx, fy, fw, fh]) = rs.focus_rect { - let t = rs.focus_rect_t as f32; - let fade = (1.0 - t) * (1.0 - t); // quadratic ease-out - let border_a = (230.0 * fade * alpha_scale) as u8; - let fill_a = (20.0 * fade * alpha_scale) as u8; - // Cyan: #5EC0E8 - let (cr, cg, cb) = (0x5Eu8, 0xC0u8, 0xE8u8); - - let rect = tiny_skia::Rect::from_xywh(fx as f32, fy as f32, fw as f32, fh as f32); - if let Some(rect) = rect { - // Faint fill - let mut fill_paint = tiny_skia::Paint::default(); - fill_paint.shader = tiny_skia::Shader::SolidColor( - tiny_skia::Color::from_rgba8(cr, cg, cb, fill_a) - ); - pm.fill_rect(rect, &fill_paint, tiny_skia::Transform::identity(), None); - - // Border stroke (2px glow) - let mut border_paint = tiny_skia::Paint::default(); - border_paint.shader = tiny_skia::Shader::SolidColor( - tiny_skia::Color::from_rgba8(cr, cg, cb, border_a) - ); - border_paint.anti_alias = true; - let stroke = tiny_skia::Stroke { width: 2.5, ..Default::default() }; - let mut pb = tiny_skia::PathBuilder::new(); - pb.push_rect(rect); - if let Some(path) = pb.finish() { - pm.stroke_path(&path, &border_paint, &stroke, - tiny_skia::Transform::identity(), None); + other => { + let _ = self.core.apply_command_base(other, true, true); } } } - - // --- Click pulse ring --- - if let Some(t) = rs.click_t { - let ring_r = (bloom_r + 20.0 * t as f32) * (1.0 - t as f32 * 0.5); - let alpha = ((1.0 - t) * 180.0 * alpha_scale as f64) as u8; - let [cr, cg, cb, _] = rs.palette.cursor_mid; - let ring_color = tiny_skia::Color::from_rgba8(cr, cg, cb, alpha); - let mut ring_paint = tiny_skia::Paint::default(); - ring_paint.shader = tiny_skia::Shader::SolidColor(ring_color); - ring_paint.anti_alias = true; - let stroke = tiny_skia::Stroke { width: 2.0, ..Default::default() }; - let mut pb = tiny_skia::PathBuilder::new(); - pb.push_circle(px as f32, py as f32, ring_r); - if let Some(path) = pb.finish() { - pm.stroke_path(&path, &ring_paint, &stroke, tiny_skia::Transform::identity(), None); - } - } - - // --- Arrow (custom shape or default gradient arrow) --- - if let Some(ref shape) = rs.shape { - // Custom icon: draw as a 32×32 image centered at (px, py), opacity-faded. - let sz = 32.0_f32; - if let Some(pix) = tiny_skia::PixmapRef::from_bytes(&shape.pixels, shape.width, shape.height) { - let transform = tiny_skia::Transform::from_rotate_at( - heading.to_degrees() as f32 + 180.0, - px as f32, py as f32, - ).pre_translate(px as f32 - sz / 2.0, py as f32 - sz / 2.0); - let mut paint = tiny_skia::PixmapPaint::default(); - paint.opacity = alpha_scale; - pm.draw_pixmap(0, 0, pix, &paint, transform, None); - } - } else { - draw_default_arrow( - &mut pm, &rs.palette, - if rs.gradient_colors.is_empty() { None } else { Some(&rs.gradient_colors) }, - px as f32, py as f32, heading as f32, alpha_scale, - ); - } - - pm -} - -fn draw_default_arrow( - pm: &mut tiny_skia::Pixmap, - palette: &Palette, - gradient_override: Option<&Vec<[u8; 4]>>, - px: f32, py: f32, - heading: f32, - alpha_scale: f32, -) { - // Arrow vertices (tip at +x). - let verts: [(f32, f32); 4] = [(14.0, 0.0), (-8.0, -9.0), (-3.0, 0.0), (-8.0, 9.0)]; - - // Rotate by (heading + π) so tip points in the motion direction. - let angle = heading + std::f64::consts::PI as f32; - let (sa, ca) = (angle.sin(), angle.cos()); - let transform_pt = |(vx, vy): (f32, f32)| -> (f32, f32) { - (px + ca * vx - sa * vy, py + sa * vx + ca * vy) - }; - - let pts: Vec<(f32, f32)> = verts.iter().map(|&v| transform_pt(v)).collect(); - - let mut pb = tiny_skia::PathBuilder::new(); - pb.move_to(pts[0].0, pts[0].1); - for p in &pts[1..] { pb.line_to(p.0, p.1); } - pb.close(); - let arrow_path = match pb.finish() { Some(p) => p, None => return }; - - // Gradient fill: start color at tip, end color at tail. - // Use runtime overrides when available, otherwise fall back to palette. - let tip = pts[0]; - let tail = ((pts[1].0 + pts[3].0) / 2.0, (pts[1].1 + pts[3].1) / 2.0); - let (r0, g0, b0) = if let Some(g) = gradient_override.and_then(|g| g.first()) { - (g[0], g[1], g[2]) - } else { - let [r, g, b, _] = palette.cursor_start; (r, g, b) - }; - let (r1, g1, b1) = if let Some(g) = gradient_override.and_then(|g| g.get(1).or_else(|| g.first())) { - (g[0], g[1], g[2]) - } else { - let [r, g, b, _] = palette.cursor_mid; (r, g, b) - }; - let (r2, g2, b2) = if let Some(g) = gradient_override.and_then(|g| g.last()) { - (g[0], g[1], g[2]) - } else { - let [r, g, b, _] = palette.cursor_end; (r, g, b) - }; - - let a = (255.0 * alpha_scale) as u8; - let fill_paint = { - let mut p = tiny_skia::Paint::default(); - p.shader = tiny_skia::LinearGradient::new( - tiny_skia::Point::from_xy(tip.0, tip.1), - tiny_skia::Point::from_xy(tail.0, tail.1), - vec![ - tiny_skia::GradientStop::new(0.00, tiny_skia::Color::from_rgba8(r0, g0, b0, a)), - tiny_skia::GradientStop::new(0.53, tiny_skia::Color::from_rgba8(r1, g1, b1, a)), - tiny_skia::GradientStop::new(1.00, tiny_skia::Color::from_rgba8(r2, g2, b2, a)), - ], - tiny_skia::SpreadMode::Pad, - tiny_skia::Transform::identity(), - ).unwrap_or(tiny_skia::Shader::SolidColor( - tiny_skia::Color::from_rgba8(r1, g1, b1, a) - )); - p.anti_alias = true; - p - }; - - pm.fill_path(&arrow_path, &fill_paint, tiny_skia::FillRule::Winding, - tiny_skia::Transform::identity(), None); - - // White outline (faded with alpha_scale). - let mut stroke_paint = tiny_skia::Paint::default(); - stroke_paint.shader = tiny_skia::Shader::SolidColor(tiny_skia::Color::from_rgba8(255, 255, 255, a)); - stroke_paint.anti_alias = true; - let stroke = tiny_skia::Stroke { width: 1.5, ..Default::default() }; - pm.stroke_path(&arrow_path, &stroke_paint, &stroke, tiny_skia::Transform::identity(), None); } // ── AppKit / CGImage plumbing ───────────────────────────────────────────── @@ -744,7 +342,7 @@ fn render_loop( rs.apply_command(cmd); } let arrived = rs.tick(dt); - (rs.pinned_wid, arrived) + (rs.core.pinned_wid, arrived) } None => break, } @@ -776,7 +374,17 @@ fn render_loop( let pixmap = { let guard = RENDER.lock().unwrap(); if let Some(rs) = guard.as_ref() { - render_frame(rs) + let focus = rs.focus_rect.map(|rect| FocusRect { + rect, + t: rs.focus_rect_t, + }); + cursor_overlay::render_frame( + &rs.core, + rs.win_w.max(1.0) as u32, + rs.win_h.max(1.0) as u32, + 0.0, 0.0, // macOS uses screen-local coords (no origin offset) + focus, + ) } else { break; } diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/check_permissions.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/check_permissions.rs index 7d87269f06..1b40ba35b1 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/tools/check_permissions.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/check_permissions.rs @@ -44,8 +44,9 @@ impl Tool for CheckPermissionsTool { fn def(&self) -> &ToolDef { def() } async fn invoke(&self, args: Value) -> ToolResult { + use mcp_server::tool_args::ArgsExt; // Default to prompting — same default + rationale as Swift. - let should_prompt = args.get("prompt").and_then(|v| v.as_bool()).unwrap_or(true); + let should_prompt = args.bool_or("prompt", true); if should_prompt { let _ = request_accessibility(); let _ = request_screen_recording(); diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/click.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/click.rs index dca1d580b0..322a683fab 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/tools/click.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/click.rs @@ -101,25 +101,18 @@ impl Tool for ClickTool { fn def(&self) -> &ToolDef { def() } async fn invoke(&self, args: Value) -> ToolResult { - let pid = match args.get("pid").and_then(|v| v.as_i64()) { - Some(v) => v as i32, - None => return ToolResult::error("Missing required parameter: pid"), - }; - - let element_index = args.get("element_index").and_then(|v| v.as_u64()).map(|v| v as usize); - let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32); - let x = args.get("x").and_then(|v| v.as_f64()) - .or_else(|| args.get("x").and_then(|v| v.as_i64()).map(|i| i as f64)); - let y = args.get("y").and_then(|v| v.as_f64()) - .or_else(|| args.get("y").and_then(|v| v.as_i64()).map(|i| i as f64)); - let action = args.get("action").and_then(|v| v.as_str()).unwrap_or("press").to_owned(); - let count = args.get("count").and_then(|v| v.as_u64()).unwrap_or(1) as usize; - let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false); - let debug_image_out = args.get("debug_image_out").and_then(|v| v.as_str()).map(str::to_owned); - let modifiers: Vec = args.get("modifier") - .and_then(|v| v.as_array()) - .map(|arr| arr.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect()) - .unwrap_or_default(); + use mcp_server::tool_args::ArgsExt; + let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e }; + + let element_index = args.opt_u64("element_index").map(|v| v as usize); + let window_id = args.opt_u64("window_id").map(|v| v as u32); + let x = args.opt_f64("x").or_else(|| args.opt_i64("x").map(|i| i as f64)); + let y = args.opt_f64("y").or_else(|| args.opt_i64("y").map(|i| i as f64)); + let action = args.str_or("action", "press"); + let count = args.u64_or("count", 1) as usize; + let from_zoom = args.bool_or("from_zoom", false); + let debug_image_out = args.opt_str("debug_image_out"); + let modifiers: Vec = args.str_array("modifier"); if let (Some(idx), Some(wid)) = (element_index, window_id) { // ── AX element path ──────────────────────────────────────────── diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/cursor_tools.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/cursor_tools.rs index 2b1f817b6c..df7140fab7 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/tools/cursor_tools.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/cursor_tools.rs @@ -49,11 +49,10 @@ impl Tool for SetAgentCursorEnabledTool { fn def(&self) -> &ToolDef { enabled_def() } async fn invoke(&self, args: Value) -> ToolResult { - let enabled = match args.get("enabled").and_then(|v| v.as_bool()) { - Some(v) => v, - None => return ToolResult::error("Missing required parameter: enabled"), - }; - let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default"); + use mcp_server::tool_args::ArgsExt; + let enabled = match args.require_bool("enabled") { Ok(v) => v, Err(e) => return e }; + let cursor_id_owned = args.str_or("cursor_id", "default"); + let cursor_id = cursor_id_owned.as_str(); self.state.cursor_registry.set_enabled(cursor_id, enabled); // Drive the visual overlay. crate::cursor::overlay::send_command(cursor_overlay::OverlayCommand::SetEnabled(enabled)); @@ -155,26 +154,27 @@ impl Tool for SetAgentCursorMotionTool { fn def(&self) -> &ToolDef { motion_def() } async fn invoke(&self, args: Value) -> ToolResult { - let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default").to_owned(); + use mcp_server::tool_args::ArgsExt; + let cursor_id = args.str_or("cursor_id", "default"); // Start from the current state or defaults. let mut current = self.state.cursor_registry.get_or_create(&cursor_id); let config = &mut current.config; // ── Appearance fields ──────────────────────────────────────────────── - if let Some(icon) = args.get("cursor_icon").and_then(|v| v.as_str()) { - config.cursor_icon = Some(icon.to_owned()); + if let Some(icon) = args.opt_str("cursor_icon") { + config.cursor_icon = Some(icon); } - if let Some(color) = args.get("cursor_color").and_then(|v| v.as_str()) { - config.cursor_color = Some(color.to_owned()); + if let Some(color) = args.opt_str("cursor_color") { + config.cursor_color = Some(color); } - if let Some(label) = args.get("cursor_label").and_then(|v| v.as_str()) { - config.cursor_label = Some(label.to_owned()); + if let Some(label) = args.opt_str("cursor_label") { + config.cursor_label = Some(label); } - if let Some(size) = args.get("cursor_size").and_then(|v| v.as_f64()) { + if let Some(size) = args.opt_f64("cursor_size") { config.cursor_size = Some(size); } - if let Some(opacity) = args.get("cursor_opacity").and_then(|v| v.as_f64()) { + if let Some(opacity) = args.opt_f64("cursor_opacity") { config.cursor_opacity = Some(opacity.clamp(0.0, 1.0)); } @@ -290,7 +290,8 @@ impl Tool for SetAgentCursorStyleTool { fn def(&self) -> &ToolDef { style_def() } async fn invoke(&self, args: Value) -> ToolResult { - let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default").to_owned(); + use mcp_server::tool_args::ArgsExt; + let cursor_id = args.str_or("cursor_id", "default"); // ── image_path ──────────────────────────────────────────────────────── let image_path = args.get("image_path").and_then(|v| v.as_str()); diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/double_click.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/double_click.rs index c5126b0146..a56b4a68f2 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/tools/double_click.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/double_click.rs @@ -54,12 +54,10 @@ impl Tool for DoubleClickTool { fn def(&self) -> &ToolDef { def() } async fn invoke(&self, args: Value) -> ToolResult { - let pid = match args.get("pid").and_then(|v| v.as_i64()) { - Some(v) => v as i32, - None => return ToolResult::error("Missing required parameter: pid"), - }; - let element_index = args.get("element_index").and_then(|v| v.as_u64()).map(|v| v as usize); - let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32); + use mcp_server::tool_args::ArgsExt; + let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e }; + let element_index = args.opt_u64("element_index").map(|v| v as usize); + let window_id = args.opt_u64("window_id").map(|v| v as u32); // ── AX element path ────────────────────────────────────────────────── if let (Some(idx), Some(wid)) = (element_index, window_id) { diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/drag.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/drag.rs index 6e50724f80..006f4962c8 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/tools/drag.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/drag.rs @@ -99,15 +99,12 @@ impl Tool for DragTool { fn def(&self) -> &ToolDef { def() } async fn invoke(&self, args: Value) -> ToolResult { - let pid = match args.get("pid").and_then(|v| v.as_i64()) { - Some(v) => v as i32, - None => return ToolResult::error("Missing required parameter: pid"), - }; + use mcp_server::tool_args::ArgsExt; + let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e }; // Coerce integer or float from JSON for coordinate fields. let coerce = |key: &str| -> Option { - args.get(key).and_then(|v| v.as_f64()) - .or_else(|| args.get(key).and_then(|v| v.as_i64()).map(|i| i as f64)) + args.opt_f64(key).or_else(|| args.opt_i64(key).map(|i| i as f64)) }; let mut from_x = match coerce("from_x") { @@ -127,15 +124,12 @@ impl Tool for DragTool { None => return ToolResult::error("Missing required parameter: to_y"), }; - let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32); - let duration_ms = args.get("duration_ms").and_then(|v| v.as_u64()).unwrap_or(500); - let steps = args.get("steps").and_then(|v| v.as_u64()).unwrap_or(20) as usize; - let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false); - let button_str = args.get("button").and_then(|v| v.as_str()).unwrap_or("left"); - let modifiers: Vec = args.get("modifier") - .and_then(|v| v.as_array()) - .map(|arr| arr.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect()) - .unwrap_or_default(); + let window_id = args.opt_u64("window_id").map(|v| v as u32); + let duration_ms = args.u64_or("duration_ms", 500); + let steps = args.u64_or("steps", 20) as usize; + let from_zoom = args.bool_or("from_zoom", false); + let button_str = args.str_or("button", "left"); + let modifiers: Vec = args.str_array("modifier"); let button = match button_str.to_lowercase().as_str() { "left" => DragButton::Left, diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/get_window_state.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/get_window_state.rs index f455c5ca22..67bec3be48 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/tools/get_window_state.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/get_window_state.rs @@ -58,29 +58,24 @@ impl Tool for GetWindowStateTool { fn def(&self) -> &ToolDef { def() } async fn invoke(&self, args: Value) -> ToolResult { - let pid = match args.get("pid").and_then(|v| v.as_i64()) { - Some(v) => v as i32, - None => return ToolResult::error("Missing required parameter: pid"), - }; - let window_id = match args.get("window_id").and_then(|v| v.as_u64()) { - Some(v) => v as u32, - None => return ToolResult::error("Missing required parameter: window_id"), - }; - let query = args.get("query").and_then(|v| v.as_str()).map(str::to_owned); - let screenshot_out_file = args.get("screenshot_out_file").and_then(|v| v.as_str()).map(|s| { + use mcp_server::tool_args::ArgsExt; + let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e }; + let window_id = match args.require_u32("window_id") { Ok(v) => v, Err(e) => return e }; + let query = args.opt_str("query"); + let screenshot_out_file = args.opt_str("screenshot_out_file").map(|s| { // Expand ~ prefix. if s.starts_with("~/") { let home = std::env::var("HOME").unwrap_or_default(); format!("{home}/{}", &s[2..]) } else { - s.to_owned() + s } }); let default_mode = self.state.config.read().unwrap().capture_mode.clone(); - let capture_mode = args.get("capture_mode").and_then(|v| v.as_str()).unwrap_or(&default_mode); + let capture_mode = args.opt_str("capture_mode").unwrap_or(default_mode); // Walk AX tree (unless vision-only mode). Accept "tree" as deprecated alias for "ax". - let capture_mode = if capture_mode == "tree" { "ax" } else { capture_mode }; + let capture_mode = if capture_mode == "tree" { "ax".to_owned() } else { capture_mode }; let tree_result = if capture_mode != "vision" { let q = query.clone(); let result = tokio::task::spawn_blocking(move || { diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/hotkey.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/hotkey.rs index cde28faa09..d8ae5da351 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/tools/hotkey.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/hotkey.rs @@ -78,17 +78,15 @@ impl Tool for HotkeyTool { fn def(&self) -> &ToolDef { def() } async fn invoke(&self, args: Value) -> ToolResult { + use mcp_server::tool_args::ArgsExt; let _ = &self.state; - let pid = match args.get("pid").and_then(|v| v.as_i64()) { - Some(v) => v as i32, - None => return ToolResult::error("Missing required parameter: pid"), - }; + let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e }; - let raw_keys = match args.get("keys").and_then(|v| v.as_array()) { - Some(arr) => arr.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect::>(), - None => return ToolResult::error("Missing required parameter: keys"), - }; + if args.get("keys").and_then(|v| v.as_array()).is_none() { + return ToolResult::error("Missing required parameter: keys"); + } + let raw_keys = args.str_array("keys"); if raw_keys.is_empty() { return ToolResult::error("keys must be a non-empty array of strings."); @@ -114,7 +112,7 @@ impl Tool for HotkeyTool { // Use the last non-modifier key; if there are multiple, treat earlier ones as extra keys. let key = non_modifiers.last().unwrap().clone(); let key_display = raw_keys.join("+"); - let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32); + let window_id = args.opt_u64("window_id").map(|v| v as u32); // ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ── // Hotkeys like Cmd+N, Cmd+W, Cmd+T explicitly open/close diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/launch_app.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/launch_app.rs index 8f3ee54087..0bf003e390 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/tools/launch_app.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/launch_app.rs @@ -79,19 +79,14 @@ impl Tool for LaunchAppTool { fn def(&self) -> &ToolDef { def() } async fn invoke(&self, args: Value) -> ToolResult { - let bundle_id = args.get("bundle_id").and_then(|v| v.as_str()).map(str::to_owned); - let name = args.get("name").and_then(|v| v.as_str()).map(str::to_owned); - let urls: Vec = args.get("urls") - .and_then(|v| v.as_array()) - .map(|arr| arr.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect()) - .unwrap_or_default(); - let electron_debugging_port = args.get("electron_debugging_port").and_then(|v| v.as_u64()).map(|v| v as u16); - let webkit_inspector_port = args.get("webkit_inspector_port").and_then(|v| v.as_u64()).map(|v| v as u16); - let creates_new_instance = args.get("creates_new_application_instance").and_then(|v| v.as_bool()).unwrap_or(false); - let mut additional_arguments: Vec = args.get("additional_arguments") - .and_then(|v| v.as_array()) - .map(|arr| arr.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect()) - .unwrap_or_default(); + use mcp_server::tool_args::ArgsExt; + let bundle_id = args.opt_str("bundle_id"); + let name = args.opt_str("name"); + let urls: Vec = args.str_array("urls"); + let electron_debugging_port = args.opt_u64("electron_debugging_port").map(|v| v as u16); + let webkit_inspector_port = args.opt_u64("webkit_inspector_port").map(|v| v as u16); + let creates_new_instance = args.bool_or("creates_new_application_instance", false); + let mut additional_arguments: Vec = args.str_array("additional_arguments"); if bundle_id.is_none() && name.is_none() { return ToolResult::error( diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/list_windows.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/list_windows.rs index 933942ab73..b2010154b0 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/tools/list_windows.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/list_windows.rs @@ -41,8 +41,9 @@ impl Tool for ListWindowsTool { fn def(&self) -> &ToolDef { def() } async fn invoke(&self, args: Value) -> ToolResult { - let pid_filter: Option = args.get("pid").and_then(|v| v.as_i64()).map(|v| v as i32); - let on_screen_only = args.get("on_screen_only").and_then(|v| v.as_bool()).unwrap_or(false); + use mcp_server::tool_args::ArgsExt; + let pid_filter: Option = args.opt_i64("pid").map(|v| v as i32); + let on_screen_only = args.bool_or("on_screen_only", false); let mut windows = if on_screen_only { crate::windows::visible_windows() diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/move_cursor.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/move_cursor.rs index a3138edba8..433332b0a7 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/tools/move_cursor.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/move_cursor.rs @@ -43,15 +43,11 @@ impl Tool for MoveCursorTool { fn def(&self) -> &ToolDef { def() } async fn invoke(&self, args: Value) -> ToolResult { - let x = match args.get("x").and_then(|v| v.as_f64()) { - Some(v) => v, - None => return ToolResult::error("Missing required parameter: x"), - }; - let y = match args.get("y").and_then(|v| v.as_f64()) { - Some(v) => v, - None => return ToolResult::error("Missing required parameter: y"), - }; - let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default"); + use mcp_server::tool_args::ArgsExt; + let x = match args.require_f64("x") { Ok(v) => v, Err(e) => return e }; + let y = match args.require_f64("y") { Ok(v) => v, Err(e) => return e }; + let cursor_id_owned = args.str_or("cursor_id", "default"); + let cursor_id = cursor_id_owned.as_str(); self.state.cursor_registry.update_position(cursor_id, x, y); // Drive the visual overlay (no-op when overlay is disabled). diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/press_key.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/press_key.rs index fd8066ba48..a1daefca76 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/tools/press_key.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/press_key.rs @@ -64,20 +64,12 @@ impl Tool for PressKeyTool { fn def(&self) -> &ToolDef { def() } async fn invoke(&self, args: Value) -> ToolResult { - let pid = match args.get("pid").and_then(|v| v.as_i64()) { - Some(v) => v as i32, - None => return ToolResult::error("Missing required parameter: pid"), - }; - let key_raw = match args.get("key").and_then(|v| v.as_str()) { - Some(v) => v.to_owned(), - None => return ToolResult::error("Missing required parameter: key"), - }; - let mut modifiers: Vec = args.get("modifiers") - .and_then(|v| v.as_array()) - .map(|arr| arr.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect()) - .unwrap_or_default(); - let element_index = args.get("element_index").and_then(|v| v.as_u64()).map(|v| v as usize); - let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32); + use mcp_server::tool_args::ArgsExt; + let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e }; + let key_raw = match args.require_str("key") { Ok(v) => v, Err(e) => return e }; + let mut modifiers: Vec = args.str_array("modifiers"); + let element_index = args.opt_u64("element_index").map(|v| v as usize); + let window_id = args.opt_u64("window_id").map(|v| v as u32); // Remap "+" / "plus" → "=" + Shift (same physical key on US layout). let key = if key_raw == "+" || key_raw == "plus" { diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/right_click.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/right_click.rs index 4b515ad754..01c57976b3 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/tools/right_click.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/right_click.rs @@ -76,21 +76,16 @@ impl Tool for RightClickTool { fn def(&self) -> &ToolDef { def() } async fn invoke(&self, args: Value) -> ToolResult { - let pid = match args.get("pid").and_then(|v| v.as_i64()) { - Some(v) => v as i32, - None => return ToolResult::error("Missing required parameter: pid"), - }; + use mcp_server::tool_args::ArgsExt; + let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e }; - let element_index = args.get("element_index").and_then(|v| v.as_u64()).map(|v| v as usize); - let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32); - let x = args.get("x").and_then(|v| v.as_f64()); - let y = args.get("y").and_then(|v| v.as_f64()); + let element_index = args.opt_u64("element_index").map(|v| v as usize); + let window_id = args.opt_u64("window_id").map(|v| v as u32); + let x = args.opt_f64("x"); + let y = args.opt_f64("y"); let has_xy = x.is_some() && y.is_some(); let partial_xy = x.is_some() != y.is_some(); - let modifiers: Vec = args.get("modifier") - .and_then(|v| v.as_array()) - .map(|arr| arr.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect()) - .unwrap_or_default(); + let modifiers: Vec = args.str_array("modifier"); if partial_xy { return ToolResult::error("Provide both x and y together, not just one."); diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot.rs index 1ec1dbed92..9a855f043f 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot.rs @@ -63,9 +63,10 @@ impl Tool for ScreenshotTool { fn def(&self) -> &ToolDef { def() } async fn invoke(&self, args: Value) -> ToolResult { - let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32); - let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("jpeg").to_owned(); - let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(85) as u8; + use mcp_server::tool_args::ArgsExt; + let window_id = args.opt_u64("window_id").map(|v| v as u32); + let format = args.str_or("format", "jpeg"); + let quality = args.u64_or("quality", 85) as u8; let use_jpeg = format == "jpeg"; let max_dim = self.state.config.read().unwrap().max_image_dimension; diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot_compat.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot_compat.rs index 4468996a38..338818917a 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot_compat.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot_compat.rs @@ -60,14 +60,9 @@ impl Tool for ClaudeCodeCompatScreenshotTool { fn def(&self) -> &ToolDef { def() } async fn invoke(&self, args: Value) -> ToolResult { - let pid = match args.get("pid").and_then(|v| v.as_i64()) { - Some(v) => v as i32, - None => return ToolResult::error("Missing required parameter: pid"), - }; - let window_id = match args.get("window_id").and_then(|v| v.as_u64()) { - Some(v) => v as u32, - None => return ToolResult::error("Missing required parameter: window_id"), - }; + use mcp_server::tool_args::ArgsExt; + let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e }; + let window_id = match args.require_u32("window_id") { Ok(v) => v, Err(e) => return e }; // Validate: window must be visible and layer-0. let window = { diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/scroll.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/scroll.rs index cb07976470..2f6d05df57 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/tools/scroll.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/scroll.rs @@ -64,18 +64,13 @@ impl Tool for ScrollTool { fn def(&self) -> &ToolDef { def() } async fn invoke(&self, args: Value) -> ToolResult { - let pid = match args.get("pid").and_then(|v| v.as_i64()) { - Some(v) => v as i32, - None => return ToolResult::error("Missing required parameter: pid"), - }; - let direction = match args.get("direction").and_then(|v| v.as_str()) { - Some(d) => d.to_owned(), - None => return ToolResult::error("Missing required parameter: direction"), - }; - let by = args.get("by").and_then(|v| v.as_str()).unwrap_or("line").to_owned(); - let amount = args.get("amount").and_then(|v| v.as_u64()).unwrap_or(3) as usize; - let element_index = args.get("element_index").and_then(|v| v.as_u64()).map(|v| v as usize); - let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32); + use mcp_server::tool_args::ArgsExt; + let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e }; + let direction = match args.require_str("direction") { Ok(v) => v, Err(e) => return e }; + let by = args.str_or("by", "line"); + let amount = args.u64_or("amount", 3) as usize; + let element_index = args.opt_u64("element_index").map(|v| v as usize); + let window_id = args.opt_u64("window_id").map(|v| v as u32); // Resolve the pre-focus element pointer (if requested) outside // the suppression closure — only the focus_element() write itself diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/set_config.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/set_config.rs index 6dd3a8e562..bfe4d5efdf 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/tools/set_config.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/set_config.rs @@ -46,14 +46,15 @@ impl Tool for SetConfigTool { fn def(&self) -> &ToolDef { def() } async fn invoke(&self, args: Value) -> ToolResult { + use mcp_server::tool_args::ArgsExt; let mut cfg = self.state.config.write().unwrap(); - if let Some(mode) = args.get("capture_mode").and_then(|v| v.as_str()) { - cfg.capture_mode = mode.to_owned(); - if let Err(e) = write_driver_config_key("capture_mode", &Value::String(mode.to_owned())) { + if let Some(mode) = args.opt_str("capture_mode") { + cfg.capture_mode = mode.clone(); + if let Err(e) = write_driver_config_key("capture_mode", &Value::String(mode)) { tracing::warn!("set_config: failed to persist capture_mode: {e}"); } } - if let Some(dim) = args.get("max_image_dimension").and_then(|v| v.as_u64()) { + if let Some(dim) = args.opt_u64("max_image_dimension") { if let Ok(dim32) = u32::try_from(dim) { cfg.max_image_dimension = dim32; if let Err(e) = write_driver_config_key("max_image_dimension", &Value::Number(dim.into())) { diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/set_value.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/set_value.rs index 2380be7779..1f229babfc 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/tools/set_value.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/set_value.rs @@ -84,22 +84,11 @@ impl Tool for SetValueTool { fn def(&self) -> &ToolDef { def() } async fn invoke(&self, args: Value) -> ToolResult { - let pid = match args.get("pid").and_then(|v| v.as_i64()) { - Some(v) => v as i32, - None => return ToolResult::error("Missing required parameter: pid"), - }; - let window_id = match args.get("window_id").and_then(|v| v.as_u64()) { - Some(v) => v as u32, - None => return ToolResult::error("Missing required parameter: window_id"), - }; - let element_index = match args.get("element_index").and_then(|v| v.as_u64()) { - Some(v) => v as usize, - None => return ToolResult::error("Missing required parameter: element_index"), - }; - let value = match args.get("value").and_then(|v| v.as_str()) { - Some(v) => v.to_owned(), - None => return ToolResult::error("Missing required parameter: value"), - }; + use mcp_server::tool_args::ArgsExt; + let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e }; + let window_id = match args.require_u32("window_id") { Ok(v) => v, Err(e) => return e }; + let element_index = match args.require_u64("element_index") { Ok(v) => v as usize, Err(e) => return e }; + let value = match args.require_str("value") { Ok(v) => v, Err(e) => return e }; let element_ptr = match self.state.element_cache.get_element_ptr(pid, window_id, element_index) { Some(p) => p, diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/type_text.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/type_text.rs index 9c5e4d63d2..f835415e09 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/tools/type_text.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/type_text.rs @@ -89,17 +89,12 @@ impl Tool for TypeTextTool { fn def(&self) -> &ToolDef { def() } async fn invoke(&self, args: Value) -> ToolResult { - let pid = match args.get("pid").and_then(|v| v.as_i64()) { - Some(v) => v as i32, - None => return ToolResult::error("Missing required parameter: pid"), - }; - let text = match args.get("text").and_then(|v| v.as_str()) { - Some(v) => v.to_owned(), - None => return ToolResult::error("Missing required parameter: text"), - }; - let element_index = args.get("element_index").and_then(|v| v.as_u64()).map(|v| v as usize); - let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32); - let delay_ms = args.get("delay_ms").and_then(|v| v.as_u64()).unwrap_or(30); + use mcp_server::tool_args::ArgsExt; + let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e }; + let text = match args.require_str("text") { Ok(v) => v, Err(e) => return e }; + let element_index = args.opt_u64("element_index").map(|v| v as usize); + let window_id = args.opt_u64("window_id").map(|v| v as u32); + let delay_ms = args.u64_or("delay_ms", 30); // Validate element_index requires window_id. if element_index.is_some() && window_id.is_none() { diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/type_text_chars.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/type_text_chars.rs index fb2628c8c5..b1f6809b81 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/tools/type_text_chars.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/type_text_chars.rs @@ -46,18 +46,13 @@ impl Tool for TypeTextCharsTool { fn def(&self) -> &ToolDef { def() } async fn invoke(&self, args: Value) -> ToolResult { - let pid = match args.get("pid").and_then(|v| v.as_i64()) { - Some(v) => v as i32, - None => return ToolResult::error("Missing required parameter: pid"), - }; - let text = match args.get("text").and_then(|v| v.as_str()) { - Some(v) => v.to_owned(), - None => return ToolResult::error("Missing required parameter: text"), - }; - let delay_ms = args.get("delay_ms").and_then(|v| v.as_u64()).unwrap_or(30); - let element_index = args.get("element_index").and_then(|v| v.as_u64()).map(|v| v as usize); - let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32); - let type_chars_only = args.get("type_chars_only").and_then(|v| v.as_bool()).unwrap_or(false); + use mcp_server::tool_args::ArgsExt; + let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e }; + let text = match args.require_str("text") { Ok(v) => v, Err(e) => return e }; + let delay_ms = args.u64_or("delay_ms", 30); + let element_index = args.opt_u64("element_index").map(|v| v as usize); + let window_id = args.opt_u64("window_id").map(|v| v as u32); + let type_chars_only = args.bool_or("type_chars_only", false); // Pre-focus element if requested. if !type_chars_only { diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/zoom.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/zoom.rs index 0ff2ab6c6b..290738dd6f 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/tools/zoom.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/zoom.rs @@ -45,27 +45,13 @@ impl Tool for ZoomTool { fn def(&self) -> &ToolDef { def() } async fn invoke(&self, args: Value) -> ToolResult { - let window_id = match args.get("window_id").and_then(|v| v.as_u64()) { - Some(v) => v as u32, - None => return ToolResult::error("Missing required parameter: window_id"), - }; - let pid = args.get("pid").and_then(|v| v.as_i64()).map(|v| v as i32); - let x1 = match args.get("x1").and_then(|v| v.as_f64()) { - Some(v) => v, - None => return ToolResult::error("Missing required parameter: x1"), - }; - let y1 = match args.get("y1").and_then(|v| v.as_f64()) { - Some(v) => v, - None => return ToolResult::error("Missing required parameter: y1"), - }; - let x2 = match args.get("x2").and_then(|v| v.as_f64()) { - Some(v) => v, - None => return ToolResult::error("Missing required parameter: x2"), - }; - let y2 = match args.get("y2").and_then(|v| v.as_f64()) { - Some(v) => v, - None => return ToolResult::error("Missing required parameter: y2"), - }; + use mcp_server::tool_args::ArgsExt; + let window_id = match args.require_u32("window_id") { Ok(v) => v, Err(e) => return e }; + let pid = args.opt_i64("pid").map(|v| v as i32); + let x1 = match args.require_f64("x1") { Ok(v) => v, Err(e) => return e }; + let y1 = match args.require_f64("y1") { Ok(v) => v, Err(e) => return e }; + let x2 = match args.require_f64("x2") { Ok(v) => v, Err(e) => return e }; + let y2 = match args.require_f64("y2") { Ok(v) => v, Err(e) => return e }; if x2 <= x1 || y2 <= y1 { return ToolResult::error("x2 must be > x1 and y2 must be > y1"); diff --git a/libs/cua-driver-rs/crates/platform-windows/src/capture.rs b/libs/cua-driver-rs/crates/platform-windows/src/capture.rs index dcd8de39f3..cc3ffffa67 100644 --- a/libs/cua-driver-rs/crates/platform-windows/src/capture.rs +++ b/libs/cua-driver-rs/crates/platform-windows/src/capture.rs @@ -160,7 +160,7 @@ unsafe fn screenshot_window_bytes_unsafe(hwnd: u64) -> Result> { if crate::input::is_xaml_host_hwnd(hwnd_raw) { match screenshot_via_screen_region(hwnd) { Ok((pixels, w, h)) => { - return encode_bgra_to_png(&pixels, w as u32, h as u32); + return mcp_server::image_utils::encode_bgra_to_png(&pixels, w as u32, h as u32); } Err(e) => { // Screen-region failed — fall through and try PrintWindow as a @@ -223,7 +223,7 @@ unsafe fn screenshot_window_bytes_unsafe(hwnd: u64) -> Result> { if is_mostly_black_bgra(&pixels) { match screenshot_via_screen_region(hwnd) { Ok((alt_pixels, alt_w, alt_h)) => { - return encode_bgra_to_png(&alt_pixels, alt_w as u32, alt_h as u32); + return mcp_server::image_utils::encode_bgra_to_png(&alt_pixels, alt_w as u32, alt_h as u32); } Err(e) => { // Screen-region path failed too — return the (black) PrintWindow @@ -239,114 +239,27 @@ unsafe fn screenshot_window_bytes_unsafe(hwnd: u64) -> Result> { } } - encode_bgra_to_png(&pixels, w as u32, h as u32) + // BGRA → PNG via the shared `image_utils::encode_bgra_to_png` + // helper (extracted from this file 2026-05; was a hand-rolled + // uncompressed-PNG path that produced ~5x larger output. The + // `image` crate's encoder is already a workspace dep so the + // smaller output is free). + mcp_server::image_utils::encode_bgra_to_png(&pixels, w as u32, h as u32) } - -/// Minimal BGRA→PNG encoder (no external dependency on image/lodepng). -fn encode_bgra_to_png(bgra: &[u8], w: u32, h: u32) -> Result> { - // Convert BGRA to RGBA. - let mut rgba = bgra.to_vec(); - for px in rgba.chunks_exact_mut(4) { - px.swap(0, 2); // B↔R - } - - // Build PNG in-memory using the flate2 + adler32 approach manually, - // or simply call out to a temp file via GDI+ / stb_image_write. - // For simplicity, use the `image` crate if available, otherwise write raw - // uncompressed PNG (which most tools accept). - write_uncompressed_png(&rgba, w, h) -} - -/// Write a minimal uncompressed PNG (IDAT with zlib level 0 = store). -fn write_uncompressed_png(rgba: &[u8], w: u32, h: u32) -> Result> { - let mut out = Vec::with_capacity(rgba.len() + 4096); - - // PNG signature. - out.extend_from_slice(b"\x89PNG\r\n\x1a\n"); - - // IHDR. - let mut ihdr = Vec::with_capacity(13); - ihdr.extend_from_slice(&w.to_be_bytes()); - ihdr.extend_from_slice(&h.to_be_bytes()); - ihdr.push(8); // bit depth - ihdr.push(2); // color type: RGB (we'll drop alpha for simplicity) — actually use 6 for RGBA - ihdr[9] = 6; // RGBA - ihdr.push(0); // compression - ihdr.push(0); // filter - ihdr.push(0); // interlace - // Rewrite properly. - let ihdr: [u8; 13] = [ - (w >> 24) as u8, (w >> 16) as u8, (w >> 8) as u8, w as u8, - (h >> 24) as u8, (h >> 16) as u8, (h >> 8) as u8, h as u8, - 8, // bit depth - 6, // RGBA - 0, // deflate - 0, // adaptive filter - 0, // no interlace - ]; - write_png_chunk(&mut out, b"IHDR", &ihdr); - - // IDAT: zlib-wrap with store (DEFLATE BTYPE=00). - // Build raw scanlines: [filter_byte(0), row_pixels...] - let row_bytes = (w * 4) as usize; - let mut raw = Vec::with_capacity((row_bytes + 1) * h as usize); - for row in 0..h as usize { - raw.push(0u8); // filter = None - raw.extend_from_slice(&rgba[row * row_bytes..(row + 1) * row_bytes]); - } - let zlib_data = zlib_store(&raw); - write_png_chunk(&mut out, b"IDAT", &zlib_data); - - // IEND. - write_png_chunk(&mut out, b"IEND", &[]); - - Ok(out) -} - -fn write_png_chunk(out: &mut Vec, name: &[u8; 4], data: &[u8]) { - let len = data.len() as u32; - out.extend_from_slice(&len.to_be_bytes()); - out.extend_from_slice(name); - out.extend_from_slice(data); - let crc = crc32_ieee(name, data); - out.extend_from_slice(&crc.to_be_bytes()); -} - -/// zlib store wrapper (BTYPE=00 non-compressed blocks, max 65535 bytes/block). -fn zlib_store(data: &[u8]) -> Vec { - let adler = adler32(data); - let mut out = Vec::new(); - // zlib header: CMF=0x78, FLG=0x01 (no dict, check bits). - out.push(0x78); - out.push(0x01); - // DEFLATE non-compressed blocks. - let mut pos = 0; - while pos < data.len() || data.is_empty() { - let end = (pos + 65535).min(data.len()); - let is_last = end == data.len(); - let blen = (end - pos) as u16; - out.push(if is_last { 1 } else { 0 }); // BFINAL | BTYPE=00 - out.extend_from_slice(&blen.to_le_bytes()); - out.extend_from_slice(&(!blen).to_le_bytes()); - out.extend_from_slice(&data[pos..end]); - pos = end; - if data.is_empty() { break; } - } - // Adler-32 checksum (big-endian). - out.extend_from_slice(&adler.to_be_bytes()); - out -} - -fn adler32(data: &[u8]) -> u32 { - let mut s1: u32 = 1; - let mut s2: u32 = 0; - for &b in data { - s1 = (s1 + b as u32) % 65521; - s2 = (s2 + s1) % 65521; - } - (s2 << 16) | s1 -} +// NOTE: previously this module carried a hand-rolled +// `write_uncompressed_png` + `write_png_chunk` + `zlib_store` + +// `adler32` + `crc32_ieee` (~110 lines) plus a local +// `encode_bgra_to_png` that used them. All of that is replaced by +// `mcp_server::image_utils::encode_bgra_to_png` which goes through +// the `image` crate's PNG encoder — already a workspace dep, +// produces ~5x smaller files than the uncompressed-store path. +// +// Same extraction applies to the four pub helpers below +// (`png_bytes_to_jpeg`, `resize_png_if_needed`, `crosshair_png_bytes`, +// `png_dimensions_pub`). They're now thin re-exports of the shared +// `mcp_server::image_utils::*` so all three platform crates call the +// same code. See `CUA_DRIVER_RS_DEDUP_AUDIT.md` for the full audit. /// Capture the primary display (full screen), returning raw PNG bytes. pub fn screenshot_display_bytes() -> Result> { @@ -376,116 +289,46 @@ pub fn screenshot_display_bytes() -> Result> { let _ = DeleteDC(mem_dc); ReleaseDC(HWND::default(), screen_dc); if ok == 0 { bail!("GetDIBits returned 0"); } - encode_bgra_to_png(&pixels, w as u32, h as u32) + mcp_server::image_utils::encode_bgra_to_png(&pixels, w as u32, h as u32) } } /// Capture primary display, returning (base64_png, width, height). pub fn screenshot_display() -> Result<(String, u32, u32)> { let png_bytes = screenshot_display_bytes()?; - if png_bytes.len() < 24 { bail!("PNG too small"); } - let w = u32::from_be_bytes([png_bytes[16], png_bytes[17], png_bytes[18], png_bytes[19]]); - let h = u32::from_be_bytes([png_bytes[20], png_bytes[21], png_bytes[22], png_bytes[23]]); + let (w, h) = mcp_server::image_utils::png_dimensions(&png_bytes)?; Ok((BASE64.encode(&png_bytes), w, h)) } +// PNG/JPEG/resize/crosshair helpers — re-exports of the shared +// `mcp_server::image_utils` module. The previous file-local copies were +// near-identical to the macOS and Linux versions; the dedup-audit +// (2026-05) moved them all to one place. + /// Convert PNG bytes to JPEG at the given quality (1–95). pub fn png_bytes_to_jpeg(png_bytes: &[u8], quality: u8) -> Result> { - let img = image::load_from_memory_with_format(png_bytes, image::ImageFormat::Png)?; - let mut buf = Vec::new(); - { - let mut cursor = std::io::Cursor::new(&mut buf); - let encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut cursor, quality); - img.write_with_encoder(encoder)?; - } - Ok(buf) + mcp_server::image_utils::png_bytes_to_jpeg(png_bytes, quality) } /// Downscale `png_bytes` so neither dimension exceeds `max_dim`. -/// If `max_dim == 0` or the image already fits, returns a copy of the original bytes unchanged. +/// If `max_dim == 0` or the image already fits, returns a copy of the +/// original bytes unchanged. pub fn resize_png_if_needed(png_bytes: &[u8], max_dim: u32) -> Result> { - if max_dim == 0 { - return Ok(png_bytes.to_vec()); - } - if png_bytes.len() < 24 { bail!("PNG too small"); } - let w = u32::from_be_bytes([png_bytes[16], png_bytes[17], png_bytes[18], png_bytes[19]]); - let h = u32::from_be_bytes([png_bytes[20], png_bytes[21], png_bytes[22], png_bytes[23]]); - if w <= max_dim && h <= max_dim { - return Ok(png_bytes.to_vec()); - } - let scale = max_dim as f64 / w.max(h) as f64; - let new_w = (w as f64 * scale).round() as u32; - let new_h = (h as f64 * scale).round() as u32; - let img = image::load_from_memory_with_format(png_bytes, image::ImageFormat::Png)?; - let resized = img.resize(new_w, new_h, image::imageops::FilterType::Lanczos3); - let mut out = Vec::new(); - resized.write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png)?; - Ok(out) + mcp_server::image_utils::resize_png_if_needed(png_bytes, max_dim) } -/// Draw a red crosshair at pixel (cx, cy) on a PNG image and return modified PNG bytes. -/// Used by recording's click-marker callback to produce click.png. +/// Draw a red crosshair at pixel (cx, cy) on a PNG image and return +/// modified PNG bytes. Used by recording's click-marker callback to +/// produce click.png. pub fn crosshair_png_bytes(png_bytes: &[u8], cx: f64, cy: f64) -> Result> { - let img = image::load_from_memory_with_format(png_bytes, image::ImageFormat::Png)?; - let (w, h) = (img.width(), img.height()); - let mut img = img.to_rgba8(); - - let arm_len = (w as f64 / 40.0).max(12.0) as i32; - let line_w = ((w as f64 / 400.0).max(1.5)) as i32; - let red = image::Rgba([255u8, 26, 26, 242]); - let cx = cx as i32; - let cy = cy as i32; - - for lw in 0..=line_w { - let off = lw - line_w / 2; - for dx in -arm_len..=arm_len { - if let Some(p) = img.get_pixel_mut_checked( - (cx + dx).clamp(0, w as i32 - 1) as u32, - (cy + off).clamp(0, h as i32 - 1) as u32, - ) { *p = red; } - } - for dy in -arm_len..=arm_len { - if let Some(p) = img.get_pixel_mut_checked( - (cx + off).clamp(0, w as i32 - 1) as u32, - (cy + dy).clamp(0, h as i32 - 1) as u32, - ) { *p = red; } - } - } - - let mut out = Vec::new(); - image::DynamicImage::ImageRgba8(img) - .write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png)?; - Ok(out) + mcp_server::image_utils::crosshair_png_bytes(png_bytes, cx, cy) } /// Parse width and height from a PNG IHDR chunk. +/// +/// Suffixed `_pub` because an older private `png_dimensions` predated +/// the `_pub` export; the public alias is what callers use today. pub fn png_dimensions_pub(data: &[u8]) -> Result<(u32, u32)> { - if data.len() < 24 { bail!("PNG data too small"); } - if &data[0..8] != b"\x89PNG\r\n\x1a\n" { bail!("Not a PNG"); } - let w = u32::from_be_bytes([data[16], data[17], data[18], data[19]]); - let h = u32::from_be_bytes([data[20], data[21], data[22], data[23]]); - Ok((w, h)) + mcp_server::image_utils::png_dimensions(data) } -fn crc32_ieee(name: &[u8], data: &[u8]) -> u32 { - const TABLE: [u32; 256] = { - let mut t = [0u32; 256]; - let mut i = 0usize; - while i < 256 { - let mut c = i as u32; - let mut j = 0; - while j < 8 { - c = if c & 1 != 0 { 0xEDB88320 ^ (c >> 1) } else { c >> 1 }; - j += 1; - } - t[i] = c; - i += 1; - } - t - }; - let mut crc: u32 = !0u32; - for &b in name.iter().chain(data.iter()) { - crc = TABLE[((crc ^ b as u32) & 0xFF) as usize] ^ (crc >> 8); - } - !crc -} diff --git a/libs/cua-driver-rs/crates/platform-windows/src/overlay.rs b/libs/cua-driver-rs/crates/platform-windows/src/overlay.rs index 4b9b515dcf..0b96e4692f 100644 --- a/libs/cua-driver-rs/crates/platform-windows/src/overlay.rs +++ b/libs/cua-driver-rs/crates/platform-windows/src/overlay.rs @@ -8,6 +8,13 @@ //! - Pixel pipeline: `tiny-skia` → BGRA DIB → `UpdateLayeredWindow` per-pixel alpha. //! - Z-ordering: every 80ms call `SetWindowPos` to stay just above the pinned target. //! - Idle-hide: fade out over 180ms once `idle_hide_ms` has elapsed with no activity. +//! +//! ## Cross-platform note (2026-05 dedup audit) +//! +//! Animation state + render pipeline live in `cursor_overlay::render_state` +//! (`RenderStateCore`, `tick_motion`, `apply_command_base`, `render_frame`). +//! What stays here is purely the Win32 window plumbing: message loop, +//! UpdateLayeredWindow paint, virtual-screen offset, z-order maintenance. #![allow(non_snake_case, non_upper_case_globals)] @@ -15,8 +22,7 @@ use std::sync::{Mutex, OnceLock}; use std::time::Instant; use cursor_overlay::{ - CursorConfig, CursorShape, MotionConfig, OverlayCommand, Palette, PathPlanner, PathState, - PlannedPath, + CursorConfig, MotionConfig, OverlayCommand, RenderStateCore, }; // ── Global channel ──────────────────────────────────────────────────────── @@ -42,14 +48,14 @@ pub fn send_command(cmd: OverlayCommand) { /// Used by the click path to wait for the animation before firing ClickPulse. pub fn glide_duration_ms() -> f64 { RENDER.lock().ok() - .and_then(|g| g.as_ref().map(|rs| rs.motion.glide_duration_ms)) + .and_then(|g| g.as_ref().map(|rs| rs.core.motion.glide_duration_ms)) .unwrap_or(750.0) } /// Returns true if the cursor overlay is currently enabled/visible. pub fn is_enabled() -> bool { RENDER.lock().ok() - .and_then(|g| g.as_ref().map(|rs| rs.visible)) + .and_then(|g| g.as_ref().map(|rs| rs.core.visible)) .unwrap_or(false) } @@ -59,14 +65,14 @@ pub fn is_enabled() -> bool { /// `get_agent_cursor_state` can report the live values. pub fn current_motion() -> MotionConfig { RENDER.lock().ok() - .and_then(|g| g.as_ref().map(|rs| rs.motion.clone())) + .and_then(|g| g.as_ref().map(|rs| rs.core.motion.clone())) .unwrap_or_default() } /// Returns the current cursor position in screen coordinates. pub fn current_position() -> (f64, f64) { RENDER.lock().ok() - .and_then(|g| g.as_ref().map(|rs| rs.pos)) + .and_then(|g| g.as_ref().map(|rs| rs.core.pos)) .unwrap_or((-200.0, -200.0)) } @@ -74,7 +80,7 @@ pub fn current_position() -> (f64, f64) { /// (-200, -200), meaning it has never been positioned on screen yet. pub fn is_at_initial_position() -> bool { RENDER.lock().ok() - .and_then(|g| g.as_ref().map(|rs| rs.pos.0 < 0.0 && rs.pos.1 < 0.0)) + .and_then(|g| g.as_ref().map(|rs| rs.core.pos.0 < 0.0 && rs.core.pos.1 < 0.0)) .unwrap_or(true) } @@ -89,7 +95,7 @@ pub fn run_on_thread() { let cfg = { let guard = RENDER.lock().unwrap(); match &*guard { - Some(rs) => rs.cfg.clone(), + Some(rs) => rs.core.cfg.clone(), None => return, } }; @@ -107,361 +113,48 @@ pub fn run_on_thread() { .expect("spawn overlay thread"); } -// ── Animation state (same structure as macOS) ───────────────────────────── +// ── Animation state ─────────────────────────────────────────────────────── +// +// The platform-agnostic fields + tick + apply_command + render pipeline live +// in `cursor_overlay::render_state` (2026-05 dedup audit). What stays here +// is the Windows-specific virtual-screen geometry + last_tick stamp for the +// WM_TIMER dt calculation. struct RenderState { - cfg: CursorConfig, - palette: Palette, - motion: MotionConfig, - pos: (f64, f64), - heading: f64, - path: Option, - dist: f64, - start_t: Instant, - spring: Option, - spring_tgt: Option<(f64, f64, f64)>, - click_t: Option, - shape: Option, - visible: bool, - idle_secs: f64, - idle_alpha: f64, - pinned_wid: Option, - gradient_colors: Vec<[u8; 4]>, - bloom_override: Option<[u8; 4]>, - last_tick: Instant, - // Virtual screen dimensions set after window creation. + core: RenderStateCore, + /// Virtual screen dimensions set after window creation (Win32 DIPs). + /// `virt_x/y` are subtracted from `core.pos` when rendering so the + /// pixmap is laid out in window-local coordinates. virt_x: i32, virt_y: i32, virt_w: i32, virt_h: i32, + /// Last WM_TIMER wall-clock stamp; used to compute real `dt` (Windows + /// timer resolution defaults to 15ms so a hardcoded 8ms would run the + /// animation at half speed). + last_tick: Instant, } -#[derive(Clone, Copy)] -struct Spring { ox: f64, oy: f64, vx: f64, vy: f64 } - impl RenderState { fn new(cfg: CursorConfig) -> Self { - let palette = cfg.palette(); - let motion = cfg.motion.clone(); - let shape = cfg.shape.clone(); RenderState { - cfg, palette, motion, shape, - pos: (-200.0, -200.0), - heading: std::f64::consts::FRAC_PI_4, - path: None, - dist: 0.0, - start_t: Instant::now(), - spring: None, - spring_tgt: None, - click_t: None, - visible: true, - idle_secs: 0.0, - idle_alpha: 1.0, - pinned_wid: None, - gradient_colors: vec![], - bloom_override: None, + core: RenderStateCore::new(cfg), last_tick: Instant::now(), virt_x: 0, virt_y: 0, virt_w: 1920, virt_h: 1080, } } fn tick(&mut self, dt: f64) { - let spring_k = self.motion.spring * 400.0; - let spring_c = self.motion.spring * 20.0; - - if let Some(ref p) = self.path { - // Speed-based motion: 16*u²*(1-u)² peaks at exactly 1.0 at u=0.5, - // matching Swift AgentCursorRenderer (peakSpeed=900, min=300/200). - let path_frac = (self.dist / p.length.max(1.0)).clamp(0.0, 1.0); - let profile = 16.0 * path_frac * path_frac * (1.0 - path_frac) * (1.0 - path_frac); - let floor = if path_frac < 0.5 { self.motion.min_start_speed } else { self.motion.min_end_speed }; - let speed = (floor + (self.motion.peak_speed - floor) * profile).max(floor); - self.dist += speed * dt; - - let path_len = p.length.max(1.0); - if self.dist >= path_len { - let end = p.sample(path_len); - let end_heading = p.end_visual_heading; - let vh = end.heading; - self.spring = Some(Spring { - ox: 0.0, oy: 0.0, - vx: speed * 0.5 * vh.cos(), - vy: speed * 0.5 * vh.sin(), - }); - self.spring_tgt = Some((end.x, end.y, end_heading)); - self.pos = (end.x, end.y); - self.heading = end_heading; - self.path = None; - self.dist = 0.0; - } else { - let s: PathState = p.sample(self.dist); - self.pos = (s.x, s.y); - let desired = s.heading + std::f64::consts::PI; - let max_step = 14.0 * dt; - self.heading = rotate_toward(self.heading, desired, max_step); - } - } else if let Some(mut s) = self.spring { - if let Some((tx, ty, th)) = self.spring_tgt { - let substeps = 4; - let sdt = dt / substeps as f64; - for _ in 0..substeps { - s.vx += (-spring_k * s.ox - spring_c * s.vx) * sdt; - s.vy += (-spring_k * s.oy - spring_c * s.vy) * sdt; - s.ox += s.vx * sdt; - s.oy += s.vy * sdt; - } - self.pos = (tx + s.ox, ty + s.oy); - self.heading = th; - if s.ox.hypot(s.oy) < 0.3 && s.vx.hypot(s.vy) < 2.0 { - self.pos = (tx, ty); - self.spring = None; - } else { - self.spring = Some(s); - } - } - } - - if let Some(t) = self.click_t { - let next = t + dt * 4.0; - self.click_t = if next >= 1.0 { None } else { Some(next) }; - } - - // Idle-hide with 180ms fade matching Windows reference. - let idle_hide_ms = self.motion.idle_hide_ms; - if idle_hide_ms > 0.0 { - let moving = self.path.is_some() || self.spring.is_some() || self.click_t.is_some(); - if moving { - self.idle_secs = 0.0; - self.idle_alpha = 1.0; - } else { - self.idle_secs += dt; - let fade_start = idle_hide_ms / 1000.0; - let fade_end = fade_start + 0.18; - if self.idle_secs > fade_end { - self.idle_alpha = 0.0; - } else if self.idle_secs > fade_start { - let t = (self.idle_secs - fade_start) / 0.18; - self.idle_alpha = 1.0 - t.clamp(0.0, 1.0); - } - } - } else { - self.idle_alpha = 1.0; - } + self.core.tick_motion(dt); } fn apply_command(&mut self, cmd: OverlayCommand) { - match cmd { - OverlayCommand::MoveTo { x, y, end_heading_radians } => { - let (x0, y0) = self.pos; - let th0 = self.heading + std::f64::consts::PI; - let th1 = end_heading_radians + std::f64::consts::PI; - const CLICK_OFFSET: f64 = 16.0; - const TURN_RADIUS: f64 = 80.0; - let tx = x + end_heading_radians.cos() * CLICK_OFFSET; - let ty = y + end_heading_radians.sin() * CLICK_OFFSET; - let plan = PathPlanner::plan( - x0, y0, th0, - tx, ty, th1, - end_heading_radians, - TURN_RADIUS, - ); - self.path = Some(plan); - self.dist = 0.0; - self.start_t = Instant::now(); - self.spring = None; - self.spring_tgt = None; - self.idle_secs = 0.0; - self.idle_alpha = 1.0; - } - OverlayCommand::ClickPulse { x, y } => { - self.pos = (x, y); - self.click_t = Some(0.0); - self.idle_secs = 0.0; - self.idle_alpha = 1.0; - } - OverlayCommand::SetEnabled(v) => { - self.visible = v; - } - OverlayCommand::SetMotion(m) => { - self.motion = m; - } - OverlayCommand::SetPalette(p) => { - self.palette = p; - } - OverlayCommand::PinAbove(wid) => { - self.pinned_wid = Some(wid); - } - OverlayCommand::SetGradient { gradient_colors, bloom_color } => { - self.gradient_colors = gradient_colors; - self.bloom_override = bloom_color; - } - OverlayCommand::SetShape(shape) => { - self.shape = shape; - } - OverlayCommand::ShowFocusRect(_) => {} - } - } -} - -// Shared with platform-linux — pulled into `cursor_overlay::util` so both -// per-OS render loops use the exact same easing primitive. -use cursor_overlay::util::rotate_toward; - -// ── tiny-skia render (shared logic) ────────────────────────────────────── - -fn render_frame(rs: &RenderState) -> tiny_skia::Pixmap { - let w = rs.virt_w.max(1) as u32; - let h = rs.virt_h.max(1) as u32; - let mut pm = tiny_skia::Pixmap::new(w, h) - .unwrap_or_else(|| tiny_skia::Pixmap::new(1, 1).unwrap()); - - if !rs.visible || rs.pos.0 < -100.0 || rs.idle_alpha < 0.004 { - return pm; - } - - let (px, py) = ( - rs.pos.0 - rs.virt_x as f64, - rs.pos.1 - rs.virt_y as f64, - ); - let heading = rs.heading; - let alpha_scale = rs.idle_alpha as f32; - - // Bloom — use runtime bloom_override if set. - let bloom_r: f32 = 22.0; - let (br, bg, bb) = if let Some([r, g, b, _]) = rs.bloom_override { - (r, g, b) - } else { - let [r, g, b, _] = rs.palette.bloom_inner; (r, g, b) - }; - let (or_, og, ob) = if let Some([r, g, b, _]) = rs.bloom_override { - (r, g, b) - } else { - let [r, g, b, _] = rs.palette.bloom_outer; (r, g, b) - }; - let bloom_inner = tiny_skia::Color::from_rgba8(br, bg, bb, (115.0 * alpha_scale) as u8); - let bloom_outer = tiny_skia::Color::from_rgba8(or_, og, ob, (26.0 * alpha_scale) as u8); - let bloom_zero = tiny_skia::Color::from_rgba8(or_, og, ob, 0); - - let bloom_paint = { - let mut p = tiny_skia::Paint::default(); - p.shader = tiny_skia::RadialGradient::new( - tiny_skia::Point::from_xy(px as f32, py as f32), - tiny_skia::Point::from_xy(px as f32, py as f32), - bloom_r, - vec![ - tiny_skia::GradientStop::new(0.0, bloom_inner), - tiny_skia::GradientStop::new(0.5, bloom_outer), - tiny_skia::GradientStop::new(1.0, bloom_zero), - ], - tiny_skia::SpreadMode::Pad, - tiny_skia::Transform::identity(), - ).unwrap_or(tiny_skia::Shader::SolidColor(bloom_inner)); - p.anti_alias = true; - p - }; - - if let Some(r) = tiny_skia::Rect::from_xywh( - (px - bloom_r as f64) as f32, (py - bloom_r as f64) as f32, - bloom_r * 2.0, bloom_r * 2.0, - ) { - pm.fill_rect(r, &bloom_paint, tiny_skia::Transform::identity(), None); - } - - // Click pulse. - if let Some(t) = rs.click_t { - let ring_r = (bloom_r + 20.0 * t as f32) * (1.0 - t as f32 * 0.5); - let alpha = ((1.0 - t) * 180.0 * alpha_scale as f64) as u8; - let [cr, cg, cb, _] = rs.palette.cursor_mid; - let ring_color = tiny_skia::Color::from_rgba8(cr, cg, cb, alpha); - let mut ring_paint = tiny_skia::Paint::default(); - ring_paint.shader = tiny_skia::Shader::SolidColor(ring_color); - ring_paint.anti_alias = true; - let stroke = tiny_skia::Stroke { width: 2.0, ..Default::default() }; - let mut pb = tiny_skia::PathBuilder::new(); - pb.push_circle(px as f32, py as f32, ring_r); - if let Some(path) = pb.finish() { - pm.stroke_path(&path, &ring_paint, &stroke, tiny_skia::Transform::identity(), None); - } - } - - // Arrow / custom shape. - if let Some(ref shape) = rs.shape { - let sz = 32.0_f32; - if let Some(pix) = tiny_skia::PixmapRef::from_bytes(&shape.pixels, shape.width, shape.height) { - let transform = tiny_skia::Transform::from_rotate_at( - heading.to_degrees() as f32 + 180.0, - px as f32, py as f32, - ).pre_translate(px as f32 - sz / 2.0, py as f32 - sz / 2.0); - let mut paint = tiny_skia::PixmapPaint::default(); - paint.opacity = alpha_scale; - pm.draw_pixmap(0, 0, pix, &paint, transform, None); - } - } else { - let grad_override = if rs.gradient_colors.is_empty() { None } else { Some(&rs.gradient_colors) }; - draw_default_arrow(&mut pm, &rs.palette, grad_override, px as f32, py as f32, heading as f32, alpha_scale); + // Windows uses the non-sentinel-snap behaviour for both MoveTo and + // ClickPulse: every command updates `self.pos` unconditionally. + // `ShowFocusRect` is not rendered on Windows — `apply_command_base` + // returns `false` for it and we silently drop it here. + let _ = self.core.apply_command_base(cmd, false, false); } - - pm -} - -fn draw_default_arrow( - pm: &mut tiny_skia::Pixmap, - palette: &Palette, - gradient_override: Option<&Vec<[u8; 4]>>, - px: f32, py: f32, - heading: f32, - alpha_scale: f32, -) { - let verts: [(f32, f32); 4] = [(14.0, 0.0), (-8.0, -9.0), (-3.0, 0.0), (-8.0, 9.0)]; - let angle = heading + std::f64::consts::PI as f32; - let (sa, ca) = (angle.sin(), angle.cos()); - let transform_pt = |(vx, vy): (f32, f32)| -> (f32, f32) { - (px + ca * vx - sa * vy, py + sa * vx + ca * vy) - }; - let pts: Vec<(f32, f32)> = verts.iter().map(|&v| transform_pt(v)).collect(); - let mut pb = tiny_skia::PathBuilder::new(); - pb.move_to(pts[0].0, pts[0].1); - for p in &pts[1..] { pb.line_to(p.0, p.1); } - pb.close(); - let arrow_path = match pb.finish() { Some(p) => p, None => return }; - - let tip = pts[0]; - let tail = ((pts[1].0 + pts[3].0) / 2.0, (pts[1].1 + pts[3].1) / 2.0); - let (r0, g0, b0) = if let Some(g) = gradient_override.and_then(|g| g.first()) { - (g[0], g[1], g[2]) - } else { let [r, g, b, _] = palette.cursor_start; (r, g, b) }; - let (r1, g1, b1) = if let Some(g) = gradient_override.and_then(|g| g.get(1).or_else(|| g.first())) { - (g[0], g[1], g[2]) - } else { let [r, g, b, _] = palette.cursor_mid; (r, g, b) }; - let (r2, g2, b2) = if let Some(g) = gradient_override.and_then(|g| g.last()) { - (g[0], g[1], g[2]) - } else { let [r, g, b, _] = palette.cursor_end; (r, g, b) }; - let a = (255.0 * alpha_scale) as u8; - - let fill_paint = { - let mut p = tiny_skia::Paint::default(); - p.shader = tiny_skia::LinearGradient::new( - tiny_skia::Point::from_xy(tip.0, tip.1), - tiny_skia::Point::from_xy(tail.0, tail.1), - vec![ - tiny_skia::GradientStop::new(0.00, tiny_skia::Color::from_rgba8(r0, g0, b0, a)), - tiny_skia::GradientStop::new(0.53, tiny_skia::Color::from_rgba8(r1, g1, b1, a)), - tiny_skia::GradientStop::new(1.00, tiny_skia::Color::from_rgba8(r2, g2, b2, a)), - ], - tiny_skia::SpreadMode::Pad, - tiny_skia::Transform::identity(), - ).unwrap_or(tiny_skia::Shader::SolidColor(tiny_skia::Color::from_rgba8(r1, g1, b1, a))); - p.anti_alias = true; - p - }; - pm.fill_path(&arrow_path, &fill_paint, tiny_skia::FillRule::Winding, - tiny_skia::Transform::identity(), None); - - let mut stroke_paint = tiny_skia::Paint::default(); - stroke_paint.shader = tiny_skia::Shader::SolidColor(tiny_skia::Color::from_rgba8(255, 255, 255, a)); - stroke_paint.anti_alias = true; - let stroke = tiny_skia::Stroke { width: 1.5, ..Default::default() }; - pm.stroke_path(&arrow_path, &stroke_paint, &stroke, tiny_skia::Transform::identity(), None); } // ── Win32 message-loop thread ───────────────────────────────────────────── @@ -612,7 +305,14 @@ unsafe extern "system" fn wnd_proc( let dt = now.duration_since(rs.last_tick).as_secs_f64().clamp(0.0, 0.05); rs.last_tick = now; rs.tick(dt); - Some(render_frame(rs)) + Some(cursor_overlay::render_frame( + &rs.core, + rs.virt_w.max(1) as u32, + rs.virt_h.max(1) as u32, + rs.virt_x as f64, + rs.virt_y as f64, + None, // focus-rect is macOS-only + )) } else { None } @@ -742,7 +442,7 @@ unsafe fn reapply_z_order(hwnd: windows::Win32::Foundation::HWND) { // Read pinned_wid from render state (RENDER lock released before this is called). let pinned_wid = { let guard = RENDER.lock().unwrap(); - guard.as_ref().and_then(|rs| rs.pinned_wid) + guard.as_ref().and_then(|rs| rs.core.pinned_wid) }; // If a target window is pinned, place the overlay just above it. diff --git a/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs index 36c382e125..4a7e54d532 100644 --- a/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs +++ b/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs @@ -356,8 +356,9 @@ impl Tool for ListWindowsTool { } async fn invoke(&self, args: Value) -> ToolResult { - let filter_pid = args.get("pid").and_then(|v| v.as_u64()).map(|v| v as u32); - let _on_screen_only = args.get("on_screen_only").and_then(|v| v.as_bool()).unwrap_or(false); + use mcp_server::tool_args::ArgsExt; + let filter_pid = args.opt_u64("pid").map(|v| v as u32); + let _on_screen_only = args.bool_or("on_screen_only", false); let (windows, pid_to_name) = tokio::task::spawn_blocking(move || { let wins = crate::win32::list_windows(filter_pid); let procs = crate::win32::list_processes(); @@ -532,9 +533,9 @@ impl Tool for GetWindowStateTool { let cfg = self.state.config.read().unwrap(); (cfg.capture_mode.clone(), cfg.max_image_dimension) }; - let capture_mode = args.get("capture_mode").and_then(|v| v.as_str()) - .unwrap_or(&default_mode).to_owned(); - let query = args.get("query").and_then(|v| v.as_str()).map(str::to_owned); + use mcp_server::tool_args::ArgsExt; + let capture_mode = args.str_or("capture_mode", &default_mode); + let query = args.opt_str("query"); // "ax" = tree only; "vision" = screenshot only; "som" (default) = both. let do_tree = capture_mode != "vision"; @@ -1506,16 +1507,14 @@ impl Tool for ClickTool { } async fn invoke(&self, args: Value) -> ToolResult { - let pid = match args.get("pid").and_then(|v| v.as_u64()) { - Some(v) => v as u32, - None => return ToolResult::error("Missing required parameter: pid"), - }; - let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64()); - let elem_idx = args.get("element_index").and_then(|v| v.as_u64()).map(|v| v as usize); - let x = args.get("x").and_then(|v| v.as_f64()); - let y = args.get("y").and_then(|v| v.as_f64()); - let button = args.get("button").and_then(|v| v.as_str()).unwrap_or("left").to_owned(); - let count = args.get("count").and_then(|v| v.as_u64()).unwrap_or(1) as usize; + use mcp_server::tool_args::ArgsExt; + let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e }; + let hwnd_opt = args.opt_u64("window_id"); + let elem_idx = args.opt_u64("element_index").map(|v| v as usize); + let x = args.opt_f64("x"); + let y = args.opt_f64("y"); + let button = args.str_or("button", "left"); + let count = args.u64_or("count", 1) as usize; // Resolve HWND: explicit, or auto from pid. let hwnd = match hwnd_opt { @@ -1792,25 +1791,19 @@ impl Tool for TypeTextTool { } async fn invoke(&self, args: Value) -> ToolResult { - // Swift error wording 1:1. - let raw_pid = match args.get("pid").and_then(|v| v.as_i64()) { - Some(p) => p, - None => return ToolResult::error("Missing required integer field pid."), - }; + use mcp_server::tool_args::ArgsExt; + let raw_pid = match args.require_i64("pid") { Ok(v) => v, Err(e) => return e }; let pid = raw_pid as u32; - let text = match args.get("text").and_then(|v| v.as_str()) { - Some(t) => t.to_owned(), - None => return ToolResult::error("Missing required string field text."), - }; - let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64()); - let elem_idx = args.get("element_index").and_then(|v| v.as_u64()); + let text = match args.require_str("text") { Ok(v) => v, Err(e) => return e }; + let hwnd_opt = args.opt_u64("window_id"); + let elem_idx = args.opt_u64("element_index"); if elem_idx.is_some() && hwnd_opt.is_none() { return ToolResult::error( "window_id is required when element_index is used — the element_index cache \ is scoped per (pid, window_id). Pass the same window_id you used in \ `get_window_state`."); } - let _delay_ms = args.get("delay_ms").and_then(|v| v.as_u64()).unwrap_or(30); + let _delay_ms = args.u64_or("delay_ms", 30); let hwnd = match hwnd_opt { Some(h) => h, None => { @@ -1927,24 +1920,15 @@ impl Tool for PressKeyTool { } async fn invoke(&self, args: Value) -> ToolResult { - // Swift error wording 1:1. - let raw_pid = match args.get("pid").and_then(|v| v.as_i64()) { - Some(p) => p, - None => return ToolResult::error("Missing required integer field pid."), - }; + use mcp_server::tool_args::ArgsExt; + let raw_pid = match args.require_i64("pid") { Ok(v) => v, Err(e) => return e }; let pid = raw_pid as u32; - let key = match args.get("key").and_then(|v| v.as_str()) { - Some(k) => k.to_owned(), - None => return ToolResult::error("Missing required string field key."), - }; - let mods: Vec = args.get("modifiers") - .and_then(|v| v.as_array()) - .map(|a| a.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect()) - .unwrap_or_default(); - let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64()); + let key = match args.require_str("key") { Ok(v) => v, Err(e) => return e }; + let mods: Vec = args.str_array("modifiers"); + let hwnd_opt = args.opt_u64("window_id"); // Swift requires window_id when element_index is supplied — ports the // same validation. - let elem_idx = args.get("element_index").and_then(|v| v.as_u64()); + let elem_idx = args.opt_u64("element_index"); if elem_idx.is_some() && hwnd_opt.is_none() { return ToolResult::error( "window_id is required when element_index is used — the element_index cache \ @@ -2034,12 +2018,9 @@ impl Tool for HotkeyTool { } async fn invoke(&self, args: Value) -> ToolResult { - let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64()); - // Swift error wording 1:1. - let raw_pid = match args.get("pid").and_then(|v| v.as_i64()) { - Some(p) => p, - None => return ToolResult::error("Missing required integer field pid."), - }; + use mcp_server::tool_args::ArgsExt; + let hwnd_opt = args.opt_u64("window_id"); + let raw_pid = match args.require_i64("pid") { Ok(v) => v, Err(e) => return e }; let pid = raw_pid as u32; // Parse keys array (Swift's only path). Legacy key+modifiers shape @@ -2303,13 +2284,13 @@ impl Tool for ScrollTool { Some(d) => d.to_owned(), None => return ToolResult::error("Missing required string field direction."), }; - let by = args.get("by").and_then(|v| v.as_str()).unwrap_or("line").to_owned(); + use mcp_server::tool_args::ArgsExt; + let by = args.str_or("by", "line"); let direction_display = direction.clone(); let by_display = by.clone(); - let amount = args.get("amount").and_then(|v| v.as_u64()) - .unwrap_or(3).clamp(1, 50) as u32; - let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64()); - let elem_idx = args.get("element_index").and_then(|v| v.as_u64()); + let amount = args.u64_or("amount", 3).clamp(1, 50) as u32; + let hwnd_opt = args.opt_u64("window_id"); + let elem_idx = args.opt_u64("element_index"); if elem_idx.is_some() && hwnd_opt.is_none() { return ToolResult::error( "window_id is required when element_index is used — the element_index cache \ @@ -2427,9 +2408,10 @@ impl Tool for ScreenshotTool { } async fn invoke(&self, args: Value) -> ToolResult { - let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64()); - let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("jpeg").to_owned(); - let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(85) as u8; + use mcp_server::tool_args::ArgsExt; + let hwnd_opt = args.opt_u64("window_id"); + let format = args.str_or("format", "jpeg"); + let quality = args.u64_or("quality", 85) as u8; let is_jpeg = format == "jpeg"; let max_dim = self.state.config.read().unwrap().max_image_dimension; @@ -2525,10 +2507,11 @@ impl Tool for DoubleClickTool { None => return ToolResult::error("Missing required integer field pid."), }; let pid = raw_pid as u32; - let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64()); - let elem_idx = args.get("element_index").and_then(|v| v.as_u64()).map(|v| v as usize); - let x = args.get("x").and_then(|v| v.as_f64()); - let y = args.get("y").and_then(|v| v.as_f64()); + use mcp_server::tool_args::ArgsExt; + let hwnd_opt = args.opt_u64("window_id"); + let elem_idx = args.opt_u64("element_index").map(|v| v as usize); + let x = args.opt_f64("x"); + let y = args.opt_f64("y"); // Swift validates "both x and y or neither" and "no element_index without window_id". let has_xy = x.is_some() && y.is_some(); let partial_xy = x.is_some() != y.is_some(); @@ -2668,10 +2651,11 @@ impl Tool for RightClickTool { None => return ToolResult::error("Missing required integer field pid."), }; let pid = raw_pid as u32; - let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64()); - let elem_idx = args.get("element_index").and_then(|v| v.as_u64()).map(|v| v as usize); - let x = args.get("x").and_then(|v| v.as_f64()); - let y = args.get("y").and_then(|v| v.as_f64()); + use mcp_server::tool_args::ArgsExt; + let hwnd_opt = args.opt_u64("window_id"); + let elem_idx = args.opt_u64("element_index").map(|v| v as usize); + let x = args.opt_f64("x"); + let y = args.opt_f64("y"); // Port Swift's full validation set. let has_xy = x.is_some() && y.is_some(); let partial_xy = x.is_some() != y.is_some(); @@ -2803,9 +2787,10 @@ impl Tool for DragTool { }; let pid = raw_pid as u32; + use mcp_server::tool_args::ArgsExt; + // Accepts numeric JSON as either float or integer — coerce both to f64. let coerce = |key: &str| -> Option { - args.get(key).and_then(|v| v.as_f64()) - .or_else(|| args.get(key).and_then(|v| v.as_i64()).map(|i| i as f64)) + args.opt_f64(key).or_else(|| args.opt_i64(key).map(|i| i as f64)) }; let from_x_opt = coerce("from_x"); let from_y_opt = coerce("from_y"); @@ -2818,11 +2803,11 @@ impl Tool for DragTool { "from_x, from_y, to_x, and to_y are all required (window-local pixels)."), }; - let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64()); - let duration_ms = args.get("duration_ms").and_then(|v| v.as_u64()).unwrap_or(500); - let steps = args.get("steps").and_then(|v| v.as_u64()).unwrap_or(20) as usize; - let button = args.get("button").and_then(|v| v.as_str()).unwrap_or("left").to_owned(); - let from_zoom = args.get("from_zoom").and_then(|v| v.as_bool()).unwrap_or(false); + let hwnd_opt = args.opt_u64("window_id"); + let duration_ms = args.u64_or("duration_ms", 500); + let steps = args.u64_or("steps", 20) as usize; + let button = args.str_or("button", "left"); + let from_zoom = args.bool_or("from_zoom", false); if from_zoom { match self.state.zoom_registry.get(pid) { @@ -2967,9 +2952,11 @@ impl Tool for MoveCursorTool { }) } async fn invoke(&self, args: Value) -> ToolResult { - let x = args.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0); - let y = args.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0); - let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default"); + use mcp_server::tool_args::ArgsExt; + let x = args.f64_or("x", 0.0); + let y = args.f64_or("y", 0.0); + let cursor_id_owned = args.str_or("cursor_id", "default"); + let cursor_id = cursor_id_owned.as_str(); self.state.cursor_registry.update_position(cursor_id, x, y); // End pointing upper-left (45°) — matches Swift's // `AgentCursor.animateAndWait(endAngleDegrees: 45)` convention so @@ -3017,7 +3004,9 @@ impl Tool for SetAgentCursorEnabledTool { Some(v) => v, None => return ToolResult::error("Missing required boolean field `enabled`."), }; - let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default"); + use mcp_server::tool_args::ArgsExt; + let cursor_id_owned = args.str_or("cursor_id", "default"); + let cursor_id = cursor_id_owned.as_str(); self.state.cursor_registry.set_enabled(cursor_id, enabled); crate::overlay::send_command(cursor_overlay::OverlayCommand::SetEnabled(enabled)); // Match Swift text format 1:1: `"✅ Agent cursor enabled."` @@ -3756,7 +3745,8 @@ impl Tool for ZoomTool { Some(v) => v, None => return ToolResult::error("Missing required integer field window_id."), }; - let coerce = |k: &str| args.get(k).and_then(|v| v.as_f64().or_else(|| v.as_i64().map(|i| i as f64))); + use mcp_server::tool_args::ArgsExt; + let coerce = |k: &str| args.opt_f64(k).or_else(|| args.opt_i64(k).map(|i| i as f64)); let (x1, y1, x2, y2) = match (coerce("x1"), coerce("y1"), coerce("x2"), coerce("y2")) { (Some(a), Some(b), Some(c), Some(d)) => (a, b, c, d), _ => return ToolResult::error("Missing required region coordinates (x1, y1, x2, y2)."), @@ -3839,12 +3829,11 @@ impl Tool for TypeTextCharsTool { } async fn invoke(&self, args: Value) -> ToolResult { - let pid = args.get("pid").and_then(|v| v.as_u64()).unwrap_or(0) as u32; - let text = match args.get("text").and_then(|v| v.as_str()) { - Some(t) => t.to_owned(), None => return ToolResult::error("Missing required parameter: text"), - }; - let delay_ms = args.get("delay_ms").and_then(|v| v.as_u64()).unwrap_or(30); - let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64()); + use mcp_server::tool_args::ArgsExt; + let pid = args.u64_or("pid", 0) as u32; + let text = match args.require_str("text") { Ok(v) => v, Err(e) => return e }; + let delay_ms = args.u64_or("delay_ms", 30); + let hwnd_opt = args.opt_u64("window_id"); let hwnd = match hwnd_opt { Some(h) => h, None => { diff --git a/libs/cua-driver-rs/crates/platform-windows/src/uia/cache.rs b/libs/cua-driver-rs/crates/platform-windows/src/uia/cache.rs index 50056d1a42..fbf2dfc509 100644 --- a/libs/cua-driver-rs/crates/platform-windows/src/uia/cache.rs +++ b/libs/cua-driver-rs/crates/platform-windows/src/uia/cache.rs @@ -6,10 +6,14 @@ //! Memory contract: UiaNode::element_ptr is a raw IUIAutomationElement vtable //! pointer with an extra AddRef from clone()+forget() in the walker. Drop here //! calls Release() to balance. +//! +//! The locked-HashMap plumbing lives in `mcp_server::element_cache` — see +//! `docs/dedup-audit.md` item #3. This module owns the Windows-specific +//! `CacheKey`, `CachedSnapshot`, and the `Drop` impl that fires COM `Release` +//! when an entry is replaced or removed. use super::UiaNode; -use std::collections::HashMap; -use std::sync::Mutex; +use mcp_server::element_cache::ElementCacheCore; use windows::Win32::UI::Accessibility::IUIAutomationElement; use windows::core::Interface; @@ -41,35 +45,37 @@ impl Drop for CachedSnapshot { } pub struct ElementCache { - inner: Mutex>, + core: ElementCacheCore, } impl ElementCache { pub fn new() -> Self { - Self { inner: Mutex::new(HashMap::new()) } + Self { core: ElementCacheCore::new() } } pub fn update(&self, pid: u32, hwnd: u64, nodes: &[UiaNode]) { let actionable: Vec<&UiaNode> = nodes.iter().filter(|n| n.element_index.is_some()).collect(); let elements: Vec = actionable.iter().map(|n| n.element_ptr).collect(); let centers: Vec<(i32, i32)> = actionable.iter().map(|n| (n.center_x, n.center_y)).collect(); - let mut inner = self.inner.lock().unwrap(); - inner.insert(CacheKey { pid, hwnd }, CachedSnapshot { elements, centers }); + self.core.insert(CacheKey { pid, hwnd }, CachedSnapshot { elements, centers }); } pub fn get_element_ptr(&self, pid: u32, hwnd: u64, element_index: usize) -> Option { - let inner = self.inner.lock().unwrap(); - inner.get(&CacheKey { pid, hwnd })?.elements.get(element_index).copied() + self.core + .with_snapshot(&CacheKey { pid, hwnd }, |s| s.elements.get(element_index).copied()) + .flatten() } pub fn get_element_center(&self, pid: u32, hwnd: u64, element_index: usize) -> Option<(i32, i32)> { - let inner = self.inner.lock().unwrap(); - inner.get(&CacheKey { pid, hwnd })?.centers.get(element_index).copied() + self.core + .with_snapshot(&CacheKey { pid, hwnd }, |s| s.centers.get(element_index).copied()) + .flatten() } pub fn element_count(&self, pid: u32, hwnd: u64) -> usize { - let inner = self.inner.lock().unwrap(); - inner.get(&CacheKey { pid, hwnd }).map(|s| s.elements.len()).unwrap_or(0) + self.core + .with_snapshot(&CacheKey { pid, hwnd }, |s| s.elements.len()) + .unwrap_or(0) } } diff --git a/libs/cua-driver-rs/docs/dedup-audit.md b/libs/cua-driver-rs/docs/dedup-audit.md new file mode 100644 index 0000000000..23c95fce9f --- /dev/null +++ b/libs/cua-driver-rs/docs/dedup-audit.md @@ -0,0 +1,163 @@ +# cua-driver-rs cross-platform code-duplication audit + +**Date:** 2026-05-23 evening (overnight session) +**Branch:** `cross-platform-dedup-audit` (off `main @ 7436ba89`) +**Approach:** scan `crates/platform-{macos,windows,linux}` for function signatures + struct definitions that appear in 2+ platforms with the same shape, rank by ROI (lines saved vs refactoring risk). + +## Existing shared crates + +Two cross-platform crates already in the workspace: + +| Crate | What it owns | Notes | +|---|---|---| +| `mcp-server` | Tool trait, ToolDef, page tool + PageBackend trait, shared CDP helper, MCP protocol structs | PR #1666 added the cross-platform `page` tool. Right home for tool-side shared logic. | +| `cursor-overlay` | CursorConfig / CursorInstanceConfig / CursorRegistry / Palette / MotionConfig / Shape / PathPlanner / Bezier / `rotate_toward` / `crop_png_to_jpeg` | PR #1662 already extracted `rotate_toward`. Right home for cursor-overlay shared primitives. | + +## Duplication candidates ranked by ROI + +### 🥇 #1 — capture.rs image-processing helpers (high ROI, low risk) + +**The duplication:** + +| Function | macOS | Windows | Linux | +|---|---|---|---| +| `png_bytes_to_jpeg(png_bytes, quality)` | ✓ 24 lines | ✓ 13 lines | ✓ 13 lines | +| `resize_png_if_needed(png_bytes, max_dim)` | ✓ 45 lines | ✓ 22 lines | ✓ 20 lines | +| `crosshair_png_bytes(png_bytes, cx, cy)` | ✓ 10 lines | ✓ 34 lines | ✓ varies | +| `write_crosshair_png(...)` (macOS only — wraps `crosshair_png_bytes` + writes to disk) | ✓ | — | — | +| `png_dimensions(data)` | ✓ 12 lines | ✓ 8 lines | ✓ 9 lines | +| `write_uncompressed_png(rgba, w, h)` | (uses CGImage) | ✓ 44 lines | ✓ 20 lines | +| `write_png_chunk(out, name, data)` | (CGImage) | ✓ 8 lines | ✓ 6 lines | +| `zlib_store(data)` | (CGImage) | ✓ 23 lines | ✓ 18 lines | +| `adler32(data)` | (CGImage) | ✓ 9 lines | ✓ 5 lines | +| `crc32_ieee(name, data)` | (CGImage) | ✓ ~25 lines | ? | + +**What stays per-platform:** +- `screenshot_window_bytes(window_id_or_hwnd_or_xid) -> Vec` — calls CGImage on macOS / BitBlt+PrintWindow on Windows / XGetImage on Linux +- `screenshot_display_bytes() -> Vec` — same + +**Estimated savings:** ~400 lines across 3 platforms. Pure refactor with zero behavioural change. The `image` crate is already a workspace dependency (used by `resize_png_if_needed`'s lanczos resampler). + +**Extraction target:** new module `mcp_server::image_utils` (in `crates/mcp-server/src/image_utils.rs`). All callers already depend on mcp-server. + +**Risk:** very low — pure functions, no platform-specific deps, no FFI. + +### 🥈 #2 — overlay.rs RenderState + render pipeline (highest absolute ROI, medium risk) + +**The duplication:** + +| Item | macOS overlay.rs | Windows overlay.rs | Linux overlay.rs | +|---|---|---|---| +| `struct RenderState { ... }` | 42 fields | 24 fields | 22 fields | +| `struct Spring { ox, oy, vx, vy }` | identical | identical | identical | +| `impl RenderState::new(cfg)` | ~25 lines | ~25 lines | ~25 lines | +| `impl RenderState::tick(dt)` | ~50 lines (path planner + spring physics) | ~50 lines (same) | ~50 lines (same) | +| `impl RenderState::apply_command(cmd)` | ~80 lines (OverlayCommand match arms) | ~80 lines (same) | ~80 lines (same) | +| `fn render_frame(rs) -> Pixmap` | ~135 lines tiny_skia | ~95 lines (same logic, fewer features) | ~78 lines (same) | +| `fn draw_default_arrow(...)` | ~175 lines tiny_skia path | ~63 lines (same path, simpler) | ~52 lines (same) | +| `fn rotate_toward` | ✓ already extracted to cursor-overlay | ✓ already extracted | ✓ already extracted | + +**What stays per-platform:** +- The window-creation + message-loop / runloop: AppKit on macOS, Win32 message loop on Windows, X11 / Wayland on Linux +- `dispatch_set_layer_contents` (macOS) / `update_layered_window` (Windows) / `paint_x11` (Linux) — platform-specific paint plumbing + +**Estimated savings:** ~1800 lines across 3 platforms (~600 per platform). This is the **biggest absolute win** in the codebase. + +**Extraction target:** `cursor-overlay` crate. Add `RenderState` + `Spring` + `tick` + `apply_command` + `render_frame` + `draw_default_arrow` to `cursor-overlay::lib`. Each platform's `overlay.rs` becomes a thin "platform paint loop" wrapper that: +1. Owns the platform Window / Layer / X11Window resource +2. Owns the per-tick paint dispatch (CGImage / DIB / XImage) +3. Calls `cursor_overlay::RenderState::tick(dt)` → `cursor_overlay::render_frame(rs)` → platform paint + +**Risk:** medium. The `RenderState` field sets differ slightly per platform (virt_x/y/w/h on Windows are Win32-DIP-specific; macOS uses CGFloat in NSScreen coords). Need to either: +- Generalise `RenderState` with platform-agnostic `virt_bounds: (f64, f64, f64, f64)` and let each platform convert +- OR split into `core: RenderStateCore` (shared) + `platform: WindowsRenderState { core, virt_w_dip }` (specific) + +**Suggested approach:** the split — `RenderStateCore` shared, per-platform wrapper structs that hold the platform-specific extras. + +### 🥉 #3 — element_index caches (medium ROI, medium risk) + +**The duplication:** + +| Module | Size | Pattern | +|---|---|---| +| `platform-macos/src/ax/cache.rs` | 82 lines | Per-pid LRU keyed on `(pid, window_id)` holding `Vec` | +| `platform-windows/src/uia/cache.rs` | 78 lines | Same pattern, holds `Vec` (refcounted) | +| `platform-linux/src/atspi/cache.rs` | 45 lines | Same pattern, holds AT-SPI accessibles | + +**Estimated savings:** ~150 lines across 3 platforms. + +**Extraction target:** `mcp-server::element_cache` — generic `ElementCache` over the platform-specific element type. Each platform inserts its own T. + +**Risk:** medium — the cache's lifetime semantics differ (macOS AXUIElementRef is CFType-refcounted; Windows IUIAutomationElement is COM-refcounted; Linux uses raw pointers). Generic-T cache handles this cleanly with `Drop` on T. + +### #4 — Tool argument-parsing boilerplate (low ROI per tool, but many tools) + +Every tool in every platform repeats: + +```rust +let pid = match args.get("pid").and_then(|v| v.as_i64()) { + Some(p) => p as i32, + None => return ToolResult::error("Missing required integer field pid."), +}; +let window_id = ...; +let element_index = ...; +``` + +**The duplication:** ~30 tools × 3 platforms × ~5 fields per tool = ~450 small repetitions, often differing by 1-2 chars in the error message. + +**Estimated savings:** ~600 lines, but each one is 2-4 lines so individually small. + +**Extraction target:** `mcp-server::tool_args` helper: +```rust +pub trait ArgsExt { + fn pid(&self) -> Result; + fn window_id(&self) -> Result; + fn element_index(&self) -> Result, ToolResult>; + fn javascript(&self) -> Result; + // etc. +} +``` + +**Risk:** very low — pure refactor of argument parsing. Per-platform tools opt into the helper. The CodeRabbit fix on PR #1666 (`i32::try_from` instead of `as i32`) becomes free for every caller. + +### #5 — list_apps / list_windows shape (low ROI, high risk) + +Each platform implements its own `list_apps` / `list_windows`. The RESPONSE shape (the JSON fields cua-driver returns) is normalized via the `WindowInfo` / `AppInfo` structs — but those structs themselves are duplicated per platform crate. + +**Estimated savings:** ~50 lines for the struct definitions; doesn't dedupe the actual `list_*` functions. + +**Risk:** high — `WindowInfo` on Windows carries an `hwnd: u64`, on macOS a `window_id: u32`, on Linux an `xid: u64`. The field semantics overlap but the types differ; a unified struct needs a typed wrapper. + +**Recommendation:** **skip** this one unless the response shape changes warrant it. The boilerplate is small relative to the win. + +### #6 — Tool description strings (zero ROI) + +Each platform's tool has its own description string (already verified in PR #1666). These are intentionally per-platform — they describe per-platform behaviour (Windows uses UIA, macOS uses AX, Linux uses AT-SPI). Don't try to dedupe. + +## Recommended PR sequence + +All five candidates landed on `cross-platform-dedup-audit` (PR #1670). + +| PR | What | Estimated | Actual | Status | +|---|---|---|---|---| +| #A | `mcp-server::image_utils` — capture.rs PNG/JPEG/crosshair/resize helpers | ~400 | **−399** | ✓ shipped (`9c998916` + `921dcdc9`) | +| #1b | `cursor-overlay::Spring` — physics struct | small | small | ✓ shipped (`f10a3258`) | +| #B | `cursor-overlay::render_state` — `RenderStateCore` + `tick` + `apply_command` + `render_frame` + `draw_default_arrow` | ~1800 | **−947 across overlays / −211 net** | ✓ shipped (`e0336893`) | +| #C | `mcp-server::tool_args` — `ArgsExt` helper trait | ~600 | **−125 consumers + trait/tests** | ✓ shipped (`f1c0cca1` + `3824b0d7`) | +| #2 | `mcp-server::element_cache` — generic `ElementCacheCore` | ~150 | **+14 net** | ✓ shipped (`f798e3ad`) | + +## What the audit got wrong + +Worth recording for future audits — three places the original estimates were off: + +- **#B "byte-for-byte identical tick/apply_command" was wrong.** macOS uses Swift-reference hardcoded constants (peakSpeed=900, springK=400, springC=17, overshoot=0.8) while Windows/Linux use runtime `MotionConfig`. macOS's `MoveTo` and `ClickPulse` have sentinel-snap behaviour the others don't. The shared core resolves this with two tick variants (`tick_motion` vs `tick_swift_constants`) and a parameterised `apply_command_base(snap_mt, click_only)`. So `RenderStateCore` is real, but it's a *family* of behaviours, not one. +- **#2 line-count estimate was optimistic.** The three caches were already terse (Linux at 45 lines); the wrapper indirection costs almost as much as the shared plumbing saves. Net is +14 lines on the cache files. The win is structural (one place to add metrics / eviction / instrumentation), not raw lines. +- **#B `start_t: Instant`** was dead code on Windows + Linux (assigned, never read) — not a real shared field, dropped during extraction. + +## What this audit did NOT cover + +- **Cross-os FFI bindings** — windows-rs vs core-foundation-rs vs x11rb — each platform crate's transitive deps are different; deduping is mostly about depending on the right shared crate, not refactoring code. +- **MCP protocol layer** — already lives in `mcp-server`, no duplication. +- **`installed_apps.rs` (Linux) / `apps/` (macOS) / `win32/apps.rs` (Windows)** — same shape but completely different impls (XDG desktop files / NSWorkspace / Start Menu shortcuts). Sharing the response struct only. +- **#5 `list_apps` / `list_windows` response shape** — recommended skip (high risk for ~50 lines). +- **#6 Tool description strings** — recommended skip (intentionally per-platform).