diff --git a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursor.swift b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursor.swift index 35137cd476..e9398f9106 100644 --- a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursor.swift +++ b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursor.swift @@ -1,5 +1,6 @@ import AppKit import QuartzCore +import SwiftUI /// Named color stop for the agent-cursor's axial stroke gradient. /// Pairing color + location keeps the spec's lavender stops grep-able @@ -182,10 +183,9 @@ public final class AgentCursor { /// follow-up action to arrive without the overlay popping in and /// out, while still being short enough that the cursor is gone /// before the user starts wondering if the agent is still running. - public var idleHideDelay: TimeInterval = 3.0 + public var idleHideDelay: TimeInterval = 8.0 private var overlay: AgentCursorOverlayWindow? - private var view: AgentCursorView? private var idleHideTask: Task? /// CGWindowID of the target window the overlay is currently @@ -244,14 +244,12 @@ public final class AgentCursor { if !enabled { cancelIdleHide() hide() - view?.cursorLayer.removeAllAnimations() tearDownActivationObserver() pinnedPid = nil - // Drop the NSWindow + content view so a later enable + - // show() rebuilds from scratch. See docstring above. + // Drop the NSWindow so a later enable + show() rebuilds from + // scratch. See docstring above. overlay?.close() overlay = nil - view = nil } } @@ -461,7 +459,7 @@ public final class AgentCursor { /// Coordinates are screen points (top-left origin), matching what /// `AXUIElement`'s `AXPosition` attribute returns. public func setPosition(_ point: CGPoint) { - ensureView().setPosition(point) + AgentCursorRenderer.shared.setInitialPosition(point) } /// Animate the cursor to `point`, then suspend until the glide is @@ -476,25 +474,20 @@ public final class AgentCursor { options: CursorMotionPath.Options? = nil ) async { guard isEnabled else { return } - let effectiveOptions = options ?? defaultMotionOptions let duration = duration ?? glideDurationSeconds cancelIdleHide() // incoming activity — defer auto-hide show() // ensure the overlay is visible; no-op if already shown - animate(to: point, duration: duration, options: effectiveOptions) - // Sleep the caller for the glide duration. Rotation settle - // continues after `duration` but the cursor has already - // arrived at the target by then; the caller is free to - // dispatch the AX action. - try? await Task.sleep(nanoseconds: UInt64(duration * 1_000_000_000)) + animate(to: point) + // Block until the cursor reaches the endpoint (spring begins). + // The actual click fires immediately after this returns, so the + // user sees the cursor land before the AX action dispatches. + await AgentCursorRenderer.shared.waitForArrival() } - /// Animate the cursor to `point` over `duration` seconds along a - /// cubic-Bezier arc. Pass `options: .default` for tuned defaults - /// or override the 5 knobs for custom motion. - /// - /// On arrival the rotation is settled via a spring animation - /// driven by the `spring` knob, so the cursor lands at its - /// resting tilt (-35°) with a damped overshoot. + /// Animate the cursor to `point` along a Dubins arc path. The + /// renderer computes the minimum-turning-radius arc from the current + /// position to `point` and integrates it forward with a speed + /// profile and spring settle. /// /// No-op when disabled. public func animate( @@ -503,151 +496,23 @@ public final class AgentCursor { options: CursorMotionPath.Options? = nil ) { guard isEnabled else { return } - let options = options ?? defaultMotionOptions - let duration = duration ?? glideDurationSeconds - let view = ensureView() - let from = view.cursorLayer.position - let path = CursorMotionPath(from: from, to: point, options: options) - - // Main glide along the Bezier. - let glide = path.positionAnimation(duration: duration) - view.cursorLayer.add(glide, forKey: "glide") - view.setPosition(point) - - // Bloom breath — while the cursor is gliding, pulse the - // radial bloom's center alpha up and back so the halo feels - // "alive" during agent action and subtle when at rest. The - // envelope matches the glide duration exactly so the halo - // resolves to its resting alpha as the cursor lands. - playBloomBreath(bloomLayer: view.bloomLayer, duration: duration) - - // Note: no rotation. Previous builds rotated the cursor to - // match the motion tangent on arrival; the SVG pointer path - // already has the tip at upper-left and the ring-ripple on - // click-landing carries the arrival-beat visual. - } - - /// Pulse the bloom's center-stop alpha up to - /// `AgentCursorStyle.bloomBreathPeak` and back over `duration`, - /// giving the halo a "breath" during a glide. Animates the - /// gradient layer's `colors` array (three NSColor stops with - /// matching alphas) so only the center stop's alpha varies — the - /// falloff shape stays constant. `isRemovedOnCompletion = true` - /// restores the static colors at the end. - private func playBloomBreath(bloomLayer: CAGradientLayer, duration: CFTimeInterval) { - let style = AgentCursorStyle.default - let bloom = style.bloomColor - let mid = bloom.withAlphaComponent(style.bloomMidAlpha).cgColor - let edge = bloom.withAlphaComponent(0.0).cgColor - let rest: [CGColor] = [ - bloom.withAlphaComponent(style.bloomCenterAlpha).cgColor, mid, edge, - ] - let peak: [CGColor] = [ - bloom.withAlphaComponent(style.bloomBreathPeak).cgColor, mid, edge, - ] - let anim = CAKeyframeAnimation(keyPath: "colors") - anim.values = [rest, peak, rest] - anim.keyTimes = [0.0, 0.5, 1.0] - anim.duration = duration - anim.timingFunctions = [ - CAMediaTimingFunction(name: .easeInEaseOut), - CAMediaTimingFunction(name: .easeInEaseOut), - ] - anim.isRemovedOnCompletion = true - bloomLayer.add(anim, forKey: "bloomBreath") + _ = ensureWindow() + // Always arrive pointing upper-left (45°), approaching from the + // lower-right — matches the macOS system-cursor convention and + // gives every click a consistent visual signature regardless of + // where the cursor started. + AgentCursorRenderer.shared.moveTo(point: point, endAngleDegrees: 45.0) } - /// Emit a ring-ripple centered on the cursor's tip — a circle - /// that starts small and opaque, expands outward while fading, - /// and removes itself when done. Fires right after the AX action - /// so the viewer sees "something landed here" without the cursor - /// itself moving or rotating. + /// Post-click visual beat. Suspends the caller for `duration` so the + /// cursor visibly rests on the target before the next action fires. + /// A future iteration can add a SwiftUI ripple drawn in + /// `AgentCursorView`; for now the dwell time alone is sufficient. /// - /// Implemented as a short-lived `CAShapeLayer` added as a child - /// of the cursor container. Because the container's anchor is at - /// the tip, adding the ripple centered on the anchor auto-aligns - /// it with the click point. Layer is removed after the animation - /// so repeated clicks don't accumulate dead sublayers. - /// - /// No-op when disabled. Caller awaits the full duration so the - /// ripple visibly completes before the dwell starts. + /// No-op when disabled. public func playClickPress(duration: CFTimeInterval = 0.65) async { - guard isEnabled, let cursorLayer = view?.cursorLayer else { return } - - // Center all rings on the cursor's tip. The container's - // anchor encodes the tip in normalized (0–1) coords; scale - // to container-local to place the rings. - let anchor = cursorLayer.anchorPoint - let size = cursorLayer.bounds.size - let tip = CGPoint(x: anchor.x * size.width, y: anchor.y * size.height) - - // Three concentric rings expanding outward together. Innermost - // is the brightest + smallest; each successive ring is larger - // and more transparent — a ripple "stack" that reads as - // radiating presence rather than a single flashbulb. Each - // ring still uses the same scale curve so they stay - // proportional as they grow. - struct Ring { - let startDiameter: CGFloat - let peakAlpha: Double - } - let rings: [Ring] = [ - Ring(startDiameter: 6, peakAlpha: 0.30), // inner - Ring(startDiameter: 10, peakAlpha: 0.12), // outer — ghost - ] - let endScale: CGFloat = 1.8 - - // Collect the layers so we can tear them down cleanly below. - var rippleLayers: [CAShapeLayer] = [] - - for ring in rings { - let layer = CAShapeLayer() - let rect = CGRect( - origin: .zero, - size: CGSize(width: ring.startDiameter, height: ring.startDiameter) - ) - layer.path = CGPath(ellipseIn: rect, transform: nil) - layer.fillColor = NSColor.clear.cgColor - layer.strokeColor = NSColor.white.withAlphaComponent(ring.peakAlpha).cgColor - layer.lineWidth = 0.8 - layer.frame = CGRect( - x: tip.x - ring.startDiameter / 2, - y: tip.y - ring.startDiameter / 2, - width: ring.startDiameter, - height: ring.startDiameter - ) - layer.opacity = 0 // starts invisible; opacity keyframe ramps in - cursorLayer.addSublayer(layer) - rippleLayers.append(layer) - - // Scale is identical across rings so they stay concentric as - // they grow. Ease-out curve punches outward and settles. - let scaleAnim = CABasicAnimation(keyPath: "transform.scale") - scaleAnim.fromValue = 0.7 - scaleAnim.toValue = endScale - scaleAnim.timingFunction = CAMediaTimingFunction( - controlPoints: 0.16, 1, 0.3, 1) - scaleAnim.duration = duration - - // Peak is 1.0 here — the real peak alpha is baked into - // `strokeColor`'s alpha. This keyframe just ramps the ring - // in briefly then fades over a long tail. - let opacityAnim = CAKeyframeAnimation(keyPath: "opacity") - opacityAnim.values = [0.0, 1.0, 0.0] - opacityAnim.keyTimes = [0.0, 0.1, 1.0] - opacityAnim.duration = duration - - let group = CAAnimationGroup() - group.animations = [scaleAnim, opacityAnim] - group.duration = duration - group.isRemovedOnCompletion = true - layer.add(group, forKey: "ripple") - } - + guard isEnabled else { return } try? await Task.sleep(nanoseconds: UInt64(duration * 1_000_000_000)) - for layer in rippleLayers { - layer.removeFromSuperlayer() - } } /// Mark the "click landed" moment — pauses the caller for the @@ -687,7 +552,6 @@ public final class AgentCursor { cancelIdleHide() overlay?.orderOut(nil) overlay = nil - view = nil } // MARK: - Private @@ -718,15 +582,9 @@ public final class AgentCursor { private func ensureWindow() -> AgentCursorOverlayWindow { if let overlay { return overlay } let win = AgentCursorOverlayWindow() - let view = AgentCursorView(frame: win.frame) - win.contentView = view + let hostView = NSHostingView(rootView: AgentCursorView()) + win.contentView = hostView self.overlay = win - self.view = view return win } - - private func ensureView() -> AgentCursorView { - _ = ensureWindow() - return view! // set by ensureWindow's side effect - } } diff --git a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorRenderer.swift b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorRenderer.swift new file mode 100644 index 0000000000..dba27b7a19 --- /dev/null +++ b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorRenderer.swift @@ -0,0 +1,361 @@ +import CoreGraphics +import Foundation +import Observation + +// MARK: - Public API -------------------------------------------------------- + +/// Dubins-path motion engine for the agent cursor overlay. +/// +/// Plans a minimum-turning-radius path (arc → straight → arc) from the +/// cursor's current position to each new target, then integrates it +/// forward at a speed-profiled rate and settles with a damped spring. +/// +/// `AgentCursor` is the public facade; this type owns the math and +/// per-frame state. Call `AgentCursor.shared.animate(to:)` from tool +/// invocation sites — do not call `AgentCursorRenderer.shared` directly. +@Observable +@MainActor +public final class AgentCursorRenderer { + public static let shared = AgentCursorRenderer() + + // -------- Tuning knobs (defaults are demo-quality) -------------------- + + /// Minimum turning radius in points. Smaller = tighter curves. + public var turnRadius: Double = 80 + + /// Peak speed in points/second. + public var peakSpeed: Double = 900 + + /// Speed floor during the first half of the trip. + public var minStartSpeed: Double = 300 + + /// Speed floor during the second half (deceleration phase). + public var minEndSpeed: Double = 200 + + /// Offset in points applied to the click target along the end-angle + /// vector before planning. The spring overshoots past this point and + /// settles back, giving the cursor a small "click-through" feel. + public var clickOffset: Double = 16 + + public var easing: Easing = .smootherstep + + /// Spring constant k in `a = -kx - cv`. + public var springStiffness: Double = 400 + /// Damping coefficient c. + public var springDamping: Double = 17 + /// Fraction of the arrival speed that seeds the spring. + public var springOvershoot: Double = 0.8 + + // -------- Observable state read by the overlay view ------------------ + + private(set) public var position: CGPoint = .init(x: -200, y: -200) + /// Visual heading in radians (tip points along this vector). + private(set) public var heading: Double = .pi / 4 // ~NW, like the OS cursor + + // -------- Internal state --------------------------------------------- + + private var path: PlannedPath? + private var trip: Trip? + private var spring: Spring? + private var distanceSoFar: Double = 0 + private var lastFrameTime: CFTimeInterval? + private var springTarget: (point: CGPoint, heading: Double)? + /// Resolved when the cursor finishes the Dubins path and the spring starts. + private var arrivalContinuation: CheckedContinuation? + + public init() {} + + // MARK: Public operations + + /// Animate the cursor to `point`, arriving with heading + /// `endAngleDegrees` (clockwise from +x in screen-top-left space). + public func moveTo(point: CGPoint, endAngleDegrees: Double) { + moveTo(point: point, endAngleRadians: endAngleDegrees * .pi / 180) + } + + public func moveTo(point clickPoint: CGPoint, endAngleRadians endAngle: Double) { + // If a caller is waiting on the previous arrival, unblock it now + // so it doesn't hang when a new animation supersedes the old one. + let prev = arrivalContinuation + arrivalContinuation = nil + prev?.resume() + let R = max(1, turnRadius) + let tx = clickPoint.x + CGFloat(cos(endAngle)) * CGFloat(clickOffset) + let ty = clickPoint.y + CGFloat(sin(endAngle)) * CGFloat(clickOffset) + let targetPoint = CGPoint(x: tx, y: ty) + let sMotion = heading + .pi + let tMotion = endAngle + .pi + path = planPath( + x0: Double(position.x), y0: Double(position.y), th0: sMotion, + x1: Double(targetPoint.x), y1: Double(targetPoint.y), th1: tMotion, + R: R, endVisualHeading: endAngle, targetPoint: targetPoint) + trip = Trip(peak: peakSpeed, + minStart: min(minStartSpeed, peakSpeed), + minEnd: min(minEndSpeed, peakSpeed), + easing: easing) + spring = nil; springTarget = nil; distanceSoFar = 0 + } + + /// Teleport without animation — use once to seed the initial position. + public func setInitialPosition(_ point: CGPoint, heading h: Double? = nil) { + position = point + heading = h ?? self.heading + path = nil; trip = nil; spring = nil; springTarget = nil + distanceSoFar = 0; lastFrameTime = nil + } + + /// Suspend until the cursor finishes its Dubins path and the spring + /// overshoot begins. Returns immediately if no path is in flight. + /// Used by `AgentCursor.animateAndWait` to time the actual click. + public func waitForArrival() async { + guard path != nil else { return } + await withCheckedContinuation { continuation in + // Only one waiter at a time. If a second call races in (shouldn't + // happen in normal tool flow), resolve the old one immediately. + let prev = arrivalContinuation + arrivalContinuation = continuation + prev?.resume() + } + } + + /// Estimated travel time in seconds for the current planned path. + /// Returns 0 if no motion is in flight. Used by `animateAndWait` to + /// know how long to suspend. + public var estimatedTravelSeconds: Double { + guard let p = path else { return 0 } + let remaining = max(0, p.length - distanceSoFar) + let avgSpeed = (minStartSpeed + peakSpeed + minEndSpeed) / 3 + return remaining / max(1, avgSpeed) + } + + // MARK: Per-frame tick (called by AgentCursorView's TimelineView) + + public func tick(now: CFTimeInterval) { + let prev = lastFrameTime ?? now + let dt = min(0.05, now - prev) + lastFrameTime = now + + if let p = path, let t = trip { + let u = min(1.0, distanceSoFar / max(p.length, 1)) + let profileValue = t.easing.profile(at: u) + let floorSpeed = (u < 0.5) ? t.minStart : t.minEnd + let currentSpeed = floorSpeed + (t.peak - floorSpeed) * profileValue + distanceSoFar += currentSpeed * dt + + if distanceSoFar >= p.length { + let endState = p.sample(at: p.length) + let vx = cos(endState.heading) * currentSpeed * springOvershoot + let vy = sin(endState.heading) * currentSpeed * springOvershoot + spring = Spring(ox: 0, oy: 0, vx: vx, vy: vy) + springTarget = (p.targetPoint, p.endVisualHeading) + position = p.targetPoint; heading = p.endVisualHeading + path = nil; trip = nil; distanceSoFar = 0 + // Signal arrival so waitForArrival() callers unblock here, + // right as the spring overshoot starts — before the settle. + let cont = arrivalContinuation + arrivalContinuation = nil + cont?.resume() + } else { + let st = p.sample(at: distanceSoFar) + position = CGPoint(x: st.x, y: st.y) + heading = rotateToward(current: heading, + desired: st.heading + .pi, + maxStep: 14 * dt) + } + } else if var s = spring, let tgt = springTarget { + let k = springStiffness, c = springDamping + let substeps = 4; let sdt = dt / Double(substeps) + for _ in 0.. Double { + var diff = desired - current + while diff > .pi { diff -= 2 * .pi } + while diff < -.pi { diff += 2 * .pi } + return current + max(-maxStep, min(maxStep, diff)) + } +} + +// MARK: - Easing ------------------------------------------------------------ + +public extension AgentCursorRenderer { + enum Easing: String, CaseIterable, Sendable { + case linear, smoothstep, smootherstep, cubic, quint + + func profile(at u: Double) -> Double { + switch self { + case .linear: return 1 + case .smoothstep: return (6 * u * (1 - u)) / 1.5 + case .smootherstep: return (30 * u * u * (1 - u) * (1 - u)) / 1.875 + case .cubic: return ((u < 0.5) ? 12 * u * u : 12 * (1 - u) * (1 - u)) / 6 + case .quint: return ((u < 0.5) ? 80 * pow(u, 4) : 80 * pow(1 - u, 4)) / 5 + } + } + } +} + +// MARK: - Internal types ---------------------------------------------------- + +private struct Trip { + let peak, minStart, minEnd: Double + let easing: AgentCursorRenderer.Easing +} + +private struct Spring { var ox, oy, vx, vy: Double } + +// MARK: - Dubins path planner ----------------------------------------------- + +struct DubinsPlannedPath { + enum Kind { case dubins, linear } + let kind: Kind + let length: Double + let endVisualHeading: Double + let targetPoint: CGPoint + // Dubins state + let x0, y0, th0, R, seg1, seg2, seg3: Double + let types: [Character] + // Linear fallback + let x1, y1, th1: Double + + struct State { let x, y, heading: Double } + + func sample(at s: Double) -> State { + switch kind { case .linear: return sampleLinear(s); case .dubins: return sampleDubins(s) } + } + + private func sampleLinear(_ s: Double) -> State { + let u = max(0, min(1, s / length)) + var diff = th1 - th0 + while diff > .pi { diff -= 2 * .pi } + while diff < -.pi { diff += 2 * .pi } + return State(x: x0 + (x1 - x0) * u, y: y0 + (y1 - y0) * u, heading: th0 + diff * u) + } + + private func sampleDubins(_ sIn: Double) -> State { + guard sIn > 0 else { return State(x: x0, y: y0, heading: th0) } + let L1 = seg1 * R, L2 = seg2 * R, L3 = seg3 * R + let s = min(sIn, L1 + L2 + L3) + var x = x0, y = y0, th = th0 + + func advance(length L: Double, type: Character) { + if type == "S" { x += cos(th) * L; y += sin(th) * L } + else { + let dth = L / R * (type == "L" ? 1.0 : -1.0) + let perp: Double = (type == "L") ? .pi / 2 : -.pi / 2 + let cx = x + cos(th + perp) * R, cy = y + sin(th + perp) * R + let ang = atan2(y - cy, x - cx) + x = cx + cos(ang + dth) * R; y = cy + sin(ang + dth) * R; th += dth + } + } + if s <= L1 { advance(length: s, type: types[0]); return State(x: x, y: y, heading: th) } + advance(length: L1, type: types[0]) + if s <= L1 + L2 { advance(length: s - L1, type: types[1]); return State(x: x, y: y, heading: th) } + advance(length: L2, type: types[1]) + advance(length: s - L1 - L2, type: types[2]) + return State(x: x, y: y, heading: th) + } +} + +// Type alias used in AgentCursorRenderer +private typealias PlannedPath = DubinsPlannedPath + +private func mod2pi(_ x: Double) -> Double { + let tau = 2 * Double.pi; let r = x - tau * floor(x / tau); return r < 0 ? r + tau : r +} + +private struct DubinsSolution { let t, p, q: Double; let types: [Character]; var length: Double { t + p + q } } + +private func dubinsLSL(_ d: Double, _ a: Double, _ b: Double) -> DubinsSolution? { + let tmp0 = d + sin(a) - sin(b) + let p2 = 2 + d * d - 2 * cos(a - b) + 2 * d * (sin(a) - sin(b)) + guard p2 >= 0 else { return nil } + let tmp1 = atan2(cos(b) - cos(a), tmp0) + return DubinsSolution(t: mod2pi(-a + tmp1), p: sqrt(p2), q: mod2pi(b - tmp1), types: ["L","S","L"]) +} +private func dubinsRSR(_ d: Double, _ a: Double, _ b: Double) -> DubinsSolution? { + let tmp0 = d - sin(a) + sin(b) + let p2 = 2 + d * d - 2 * cos(a - b) + 2 * d * (sin(b) - sin(a)) + guard p2 >= 0 else { return nil } + let tmp1 = atan2(cos(a) - cos(b), tmp0) + return DubinsSolution(t: mod2pi(a - tmp1), p: sqrt(p2), q: mod2pi(-b + tmp1), types: ["R","S","R"]) +} +private func dubinsLSR(_ d: Double, _ a: Double, _ b: Double) -> DubinsSolution? { + let p2 = -2 + d * d + 2 * cos(a - b) + 2 * d * (sin(a) + sin(b)) + guard p2 >= 0 else { return nil } + let p = sqrt(p2) + let tmp1 = atan2(-cos(a) - cos(b), d + sin(a) + sin(b)) - atan2(-2, p) + return DubinsSolution(t: mod2pi(-a + tmp1), p: p, q: mod2pi(-mod2pi(b) + tmp1), types: ["L","S","R"]) +} +private func dubinsRSL(_ d: Double, _ a: Double, _ b: Double) -> DubinsSolution? { + let p2 = d * d - 2 + 2 * cos(a - b) - 2 * d * (sin(a) + sin(b)) + guard p2 >= 0 else { return nil } + let p = sqrt(p2) + let tmp1 = atan2(cos(a) + cos(b), d - sin(a) - sin(b)) - atan2(2, p) + return DubinsSolution(t: mod2pi(a - tmp1), p: p, q: mod2pi(b - tmp1), types: ["R","S","L"]) +} +private func dubinsRLR(_ d: Double, _ a: Double, _ b: Double) -> DubinsSolution? { + let tmp = (6 - d * d + 2 * cos(a - b) + 2 * d * (sin(a) - sin(b))) / 8 + guard abs(tmp) <= 1 else { return nil } + let p = mod2pi(2 * .pi - acos(tmp)) + let t = mod2pi(a - atan2(cos(a) - cos(b), d - sin(a) + sin(b)) + p / 2) + return DubinsSolution(t: t, p: p, q: mod2pi(a - b - t + p), types: ["R","L","R"]) +} +private func dubinsLRL(_ d: Double, _ a: Double, _ b: Double) -> DubinsSolution? { + let tmp = (6 - d * d + 2 * cos(a - b) + 2 * d * (sin(b) - sin(a))) / 8 + guard abs(tmp) <= 1 else { return nil } + let p = mod2pi(2 * .pi - acos(tmp)) + let t = mod2pi(-a + atan2(-cos(a) + cos(b), d + sin(a) - sin(b)) + p / 2) + return DubinsSolution(t: t, p: p, q: mod2pi(mod2pi(b) - a - t + p), types: ["L","R","L"]) +} + +private func planPath(x0: Double, y0: Double, th0: Double, + x1: Double, y1: Double, th1: Double, + R: Double, endVisualHeading: Double, + targetPoint: CGPoint) -> PlannedPath { + if let p = planDubins(x0: x0, y0: y0, th0: th0, x1: x1, y1: y1, th1: th1, + R: R, endVisualHeading: endVisualHeading, targetPoint: targetPoint) { + return p + } + let D = max(1, hypot(x1 - x0, y1 - y0)) + return PlannedPath(kind: .linear, length: D, endVisualHeading: endVisualHeading, + targetPoint: targetPoint, x0: x0, y0: y0, th0: th0, R: R, + seg1: 0, seg2: 0, seg3: 0, types: [], x1: x1, y1: y1, th1: th1) +} + +private func planDubins(x0: Double, y0: Double, th0: Double, + x1: Double, y1: Double, th1: Double, + R: Double, endVisualHeading: Double, + targetPoint: CGPoint) -> PlannedPath? { + let dx = x1 - x0, dy = y1 - y0, D = hypot(dx, dy) + guard D > 0.5 else { return nil } + let d = D / R, theta = mod2pi(atan2(dy, dx)) + let a = mod2pi(th0 - theta), b = mod2pi(th1 - theta) + let solvers: [(Double, Double, Double) -> DubinsSolution?] = [ + dubinsLSL, dubinsRSR, dubinsLSR, dubinsRSL, dubinsRLR, dubinsLRL, + ] + var best: DubinsSolution?; var bestLen = Double.infinity + for s in solvers { + if let sol = s(d, a, b), sol.length.isFinite, sol.length >= 0, sol.length < bestLen { + bestLen = sol.length; best = sol + } + } + guard let b = best else { return nil } + return PlannedPath(kind: .dubins, length: (b.t + b.p + b.q) * R, + endVisualHeading: endVisualHeading, targetPoint: targetPoint, + x0: x0, y0: y0, th0: th0, R: R, + seg1: b.t, seg2: b.p, seg3: b.q, types: b.types, + x1: x1, y1: y1, th1: th1) +} diff --git a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorView.swift b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorView.swift index 81445e5b62..797b5f7c8a 100644 --- a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorView.swift +++ b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorView.swift @@ -1,250 +1,89 @@ import AppKit -import QuartzCore +import SwiftUI -/// The layer-backed content view for the overlay window. Hosts a single -/// container `CALayer` — the "agent cursor composite" — made of a wide -/// radial cyan bloom, a gradient-filled classic cursor arrow, and a -/// white outline stroke. Tinted distinctly from the system cursor so the -/// user can tell them apart at a glance. +/// SwiftUI overlay view that drives `AgentCursorRenderer.shared` every +/// display frame and draws the cursor arrow via `Canvas`. Hosted inside +/// `AgentCursorOverlayWindow` via an `NSHostingView`. /// -/// Uses `isFlipped = true` so the view's coordinate system matches -/// global screen points (origin top-left, y increases downward). This -/// lets callers set `cursorLayer.position` directly from screen-point -/// coordinates without a per-update flip. -public final class AgentCursorView: NSView { - public override var isFlipped: Bool { true } - - /// Animators drive this layer's `position` and `transform`. It's a - /// container `CALayer` holding the bloom + stroke sublayers — - /// setting `.position` here moves the whole composite as one unit. - /// The layer's `anchorPoint` is (0.5, 0.5) so `position` is the - /// cursor's visual center. - public let cursorLayer: CALayer - - /// The radial bloom layer. Exposed so `AgentCursor` can animate its - /// alpha envelope ("breath") during a glide without poking into the - /// sublayer tree by index. - public let bloomLayer: CAGradientLayer - - public override init(frame: NSRect) { - let built = AgentCursorView.makeCursorLayer() - self.cursorLayer = built.container - self.bloomLayer = built.bloom - super.init(frame: frame) - wantsLayer = true - let root = CALayer() - root.backgroundColor = NSColor.clear.cgColor - layer = root - root.addSublayer(cursorLayer) - } - - public required init?(coder: NSCoder) { - fatalError("init(coder:) is not supported for AgentCursorView") +/// The cursor tip points in the direction of `renderer.heading`. Shape +/// matches the existing gradient-arrow design: a classic pointer with +/// the tip at upper-left, scaled for legibility at any display density. +public struct AgentCursorView: View { + @Bindable var renderer: AgentCursorRenderer + + public init(renderer: AgentCursorRenderer = .shared) { + self.renderer = renderer } - /// Position the cursor at a screen-point coordinate. Call from the - /// main actor. Disables the implicit CA animation on `position` - /// changes so instantaneous moves stay snappy; explicit animations - /// are driven by the caller wrapping a `CATransaction` of their - /// own around this call. - public func setPosition(_ point: CGPoint) { - CATransaction.begin() - CATransaction.setDisableActions(true) - cursorLayer.position = point - CATransaction.commit() + public var body: some View { + TimelineView(.animation(minimumInterval: 1.0 / 120.0)) { ctx in + Canvas { gctx, _ in + renderer.tick(now: ctx.date.timeIntervalSinceReferenceDate) + drawCursor(in: gctx) + } + .ignoresSafeArea() + .allowsHitTesting(false) + } } - /// A classic cursor-arrow pointer (SVG-derived path, tip at upper- - /// left) sitting inside a 60pt container. The container holds three - /// layers, back → front: - /// - /// 1. Radial `CAGradientLayer` — 60×60pt cyan bloom, alpha falloff - /// center `0.55` → mid `0.15` → edge `0.0`. Wide, perfectly- - /// circular glow that reads as "agent presence" without tracking - /// the arrow silhouette. - /// 2. Gradient-filled arrow — `CAGradientLayer` (axial 135°) masked - /// by a FILLED `CAShapeLayer` of the cursor path. Gradient - /// colors fill the entire arrow interior. - /// 3. White outline stroke — `CAShapeLayer` with `strokeColor = - /// white`, 2pt line along the same path, centered. Half the - /// stroke extends outside the gradient silhouette, creating - /// the white border that defines the shape against any - /// background. - /// - /// The container's `anchorPoint` is `(0.5, 0.5)` so `layer.position` - /// is the cursor's visual center. No container-level rotation — the - /// SVG path already has the tip at upper-left. - private static func makeCursorLayer() -> (container: CALayer, bloom: CAGradientLayer) { - let style = AgentCursorStyle.default - let containerSize = style.containerSize - let shapeSize = style.shapeSize - - // Arrow path, scaled from the SVG 24-unit box to `shapeSize` - // centered in the container. The SVG content only occupies ~18 - // of the 24 units (padding on each side); the effective drawn - // size is therefore `shapeSize * 18/24` ≈ 75% of the frame. - let shapeFrame = CGRect( - x: (containerSize - shapeSize) / 2, - y: (containerSize - shapeSize) / 2, - width: shapeSize, - height: shapeSize - ) - let arrowPath = makeCursorArrowPath(in: shapeFrame) - - // --- 1. Radial bloom (back) --- - let bloom = CAGradientLayer() - bloom.type = .radial - let bloomCenter = style.bloomColor - bloom.colors = [ - bloomCenter.withAlphaComponent(style.bloomCenterAlpha).cgColor, - bloomCenter.withAlphaComponent(style.bloomMidAlpha).cgColor, - bloomCenter.withAlphaComponent(0.0).cgColor, - ] - bloom.locations = [0.0, 0.5, 1.0] - bloom.startPoint = CGPoint(x: 0.5, y: 0.5) - // For radial CAGradientLayer, `endPoint` sets the outer edge of - // the gradient. (1.0, 1.0) takes the bloom to the layer's - // bottom-right corner — a full 30pt radius from center. - bloom.endPoint = CGPoint(x: 1.0, y: 1.0) - bloom.frame = CGRect(x: 0, y: 0, width: containerSize, height: containerSize) - - // --- 2. Gradient-filled arrow (middle) --- - // Filled mask: any opaque color works, the mask uses alpha. The - // gradient layer above then fills the entire arrow interior. - let fillMask = CAShapeLayer() - fillMask.path = arrowPath - fillMask.fillColor = NSColor.white.cgColor - fillMask.strokeColor = NSColor.clear.cgColor - fillMask.frame = CGRect(x: 0, y: 0, width: containerSize, height: containerSize) - - let fillGradient = CAGradientLayer() - fillGradient.type = .axial - fillGradient.colors = style.strokeGradientStops.map { $0.color.cgColor } - fillGradient.locations = style.strokeGradientStops.map { NSNumber(value: Double($0.location)) } - let angleRad: CGFloat = style.strokeGradientAngleDegrees * .pi / 180 - let dx = sin(angleRad) / 2 - let dy = -cos(angleRad) / 2 - fillGradient.startPoint = CGPoint(x: 0.5 - dx, y: 0.5 - dy) - fillGradient.endPoint = CGPoint(x: 0.5 + dx, y: 0.5 + dy) - fillGradient.frame = CGRect(x: 0, y: 0, width: containerSize, height: containerSize) - fillGradient.mask = fillMask - - // --- 3. White outline stroke (front) --- - // 2pt white stroke along the arrow path. Default CA strokes are - // centered on the path — half extends outside the silhouette, - // half inside — but the gradient fill behind covers the inside - // half, so the visible effect is a clean outline around the - // gradient shape. - let border = CAShapeLayer() - border.path = arrowPath - border.fillColor = NSColor.clear.cgColor - border.strokeColor = NSColor.white.cgColor - border.lineWidth = style.strokeWidth - border.lineJoin = .round - border.lineCap = .round - border.frame = CGRect(x: 0, y: 0, width: containerSize, height: containerSize) - - // --- Container --- - // Anchor the container on the cursor's TIP — not the geometric - // center — so `layer.position = clickPoint` lands the tip - // exactly on the click, matching macOS system-cursor behavior. - // - // The tip in SVG coords is at ~(3.35, 3.35) in the 24-unit - // viewBox (upper-left corner of the path). Scale into container - // coords and convert to a normalized anchor. - let svgTip = CGPoint(x: 3.35, y: 3.35) - let shapeScale = shapeSize / 24.0 - let tipInShape = CGPoint(x: svgTip.x * shapeScale, y: svgTip.y * shapeScale) - let tipInContainer = CGPoint( - x: shapeFrame.origin.x + tipInShape.x, - y: shapeFrame.origin.y + tipInShape.y + /// Draw the cursor arrow centered on `renderer.position`, rotated to + /// `renderer.heading`. The shape is a 4-vertex pointer arrow with + /// the tip along +x before rotation, which the caller rotates by + /// `heading + π` (so the visible tip trails opposite the motion + /// vector — matching macOS cursor convention). + private func drawCursor(in ctx: GraphicsContext) { + let p = renderer.position + guard p.x > -100 else { return } // skip until first moveTo + + // Arrow path — tip at (14, 0), tail extends to the left. + var shape = Path() + shape.move(to: CGPoint(x: 14, y: 0)) + shape.addLine(to: CGPoint(x: -8, y: -9)) + shape.addLine(to: CGPoint(x: -3, y: 0)) + shape.addLine(to: CGPoint(x: -8, y: 9)) + shape.closeSubpath() + + // `renderer.heading` is the visual heading = motion_direction + π. + // The arrow path has its tip at +x, so we rotate by heading + π + // to make the tip point in the motion direction (standard cursor + // convention where the pointer leads rather than trails). + let transform = CGAffineTransform(translationX: p.x, y: p.y) + .rotated(by: CGFloat(renderer.heading + .pi)) + + let transformed = shape.applying(transform) + + // Ice-blue gradient fill (matches existing agent-cursor palette). + ctx.fill( + transformed, + with: .linearGradient( + Gradient(colors: [ + Color(red: 0xDB/255, green: 0xEE/255, blue: 0xFF/255), + Color(red: 0x5E/255, green: 0xC0/255, blue: 0xE8/255), + Color(red: 0x54/255, green: 0xCD/255, blue: 0xA0/255), + ]), + startPoint: CGPoint(x: p.x + 14, y: p.y - 9), + endPoint: CGPoint(x: p.x - 8, y: p.y + 9) + ) ) - let anchor = CGPoint( - x: tipInContainer.x / containerSize, - y: tipInContainer.y / containerSize + // White outline for legibility on any background. + ctx.stroke(transformed, with: .color(.white), lineWidth: 1.5) + + // Cyan bloom halo — radial glow that reads as agent presence. + let bloomR: CGFloat = 22 + let bloomRect = CGRect(x: p.x - bloomR, y: p.y - bloomR, + width: bloomR * 2, height: bloomR * 2) + ctx.fill( + Path(ellipseIn: bloomRect), + with: .radialGradient( + Gradient(colors: [ + Color(red: 0x5E/255, green: 0xC0/255, blue: 0xE8/255).opacity(0.45), + Color(red: 0x5E/255, green: 0xC0/255, blue: 0xE8/255).opacity(0.10), + Color(red: 0x5E/255, green: 0xC0/255, blue: 0xE8/255).opacity(0.0), + ]), + center: p, + startRadius: 0, + endRadius: bloomR + ) ) - - let container = CALayer() - container.bounds = CGRect(x: 0, y: 0, width: containerSize, height: containerSize) - container.anchorPoint = anchor - container.position = CGPoint(x: -100, y: -100) // off-screen default - container.addSublayer(bloom) - container.addSublayer(fillGradient) - container.addSublayer(border) - // NO rotation — the SVG path already has the tip at upper-left. - - return (container, bloom) - } - - /// Build the cursor-arrow path as a `CGPath`, scaled from the 24-unit - /// SVG reference in `docs/_local/references/` into `frame`. - /// - /// Shape: a classic pointer arrow with the tip at upper-left and the - /// tail extending to lower-right. All corners are rounded via short - /// cubic beziers. At 15pt `shapeSize`, the visible content occupies - /// about 11.5pt (the SVG content is ~18 units in a 24-unit viewport), - /// so the drawn arrow reads larger thanks to the 2pt white outline - /// that wraps it. - /// - /// Coordinates are in the container's coordinate system (isFlipped - /// view, so +y is down — same convention as the SVG source). - private static func makeCursorArrowPath(in frame: CGRect) -> CGPath { - let path = CGMutablePath() - - // SVG viewBox is 24×24; scale all coords uniformly to `frame.width`. - let s = frame.width / 24.0 - let ox = frame.minX - let oy = frame.minY - func pt(_ x: Double, _ y: Double) -> CGPoint { - CGPoint(x: ox + CGFloat(x) * s, y: oy + CGFloat(y) * s) - } - - // Path walk matches the SVG's M/C/L sequence. Tail corner first - // (upper-right of the shape), then the tip corner (upper-left), - // then the outer wing corner (lower-left), then the inner notch - // where the tail meets the body. - path.move(to: pt(20.5056, 10.7754)) - path.addCurve(to: pt(21.5176, 10.2459), - control1: pt(21.1225, 10.5355), - control2: pt(21.431, 10.4155)) - path.addCurve(to: pt(21.5115, 9.77954), - control1: pt(21.5926, 10.099), - control2: pt(21.5903, 9.92446)) - path.addCurve(to: pt(20.486, 9.2768), - control1: pt(21.4205, 9.61226), - control2: pt(21.109, 9.50044)) - path.addLine(to: pt(4.59629, 3.5728)) - path.addCurve(to: pt(3.66514, 3.35605), - control1: pt(4.0866, 3.38983), - control2: pt(3.83175, 3.29835)) - path.addCurve(to: pt(3.35629, 3.6649), - control1: pt(3.52029, 3.40621), - control2: pt(3.40645, 3.52004)) - path.addCurve(to: pt(3.57304, 4.59605), - control1: pt(3.29859, 3.8315), - control2: pt(3.39008, 4.08635)) - path.addLine(to: pt(9.277, 20.4858)) - path.addCurve(to: pt(9.77973, 21.5113), - control1: pt(9.50064, 21.1088), - control2: pt(9.61246, 21.4203)) - path.addCurve(to: pt(10.2461, 21.5174), - control1: pt(9.92465, 21.5901), - control2: pt(10.0991, 21.5924)) - path.addCurve(to: pt(10.7756, 20.5054), - control1: pt(10.4157, 21.4308), - control2: pt(10.5356, 21.1223)) - path.addLine(to: pt(13.3724, 13.8278)) - path.addCurve(to: pt(13.4792, 13.5957), - control1: pt(13.4194, 13.707), - control2: pt(13.4429, 13.6466)) - path.addCurve(to: pt(13.5959, 13.479), - control1: pt(13.5114, 13.5506), - control2: pt(13.5508, 13.5112)) - path.addCurve(to: pt(13.828, 13.3722), - control1: pt(13.6468, 13.4427), - control2: pt(13.7072, 13.4192)) - path.addLine(to: pt(20.5056, 10.7754)) - path.closeSubpath() - return path } } diff --git a/libs/cua-driver/Sources/CuaDriverCore/Recording/RecordingSession.swift b/libs/cua-driver/Sources/CuaDriverCore/Recording/RecordingSession.swift index c39909e3f8..eb56eefed5 100644 --- a/libs/cua-driver/Sources/CuaDriverCore/Recording/RecordingSession.swift +++ b/libs/cua-driver/Sources/CuaDriverCore/Recording/RecordingSession.swift @@ -28,6 +28,10 @@ public actor RecordingSession { /// when `videoExperimental = true` — the render-with-zoom pipeline is /// the sole consumer. Nil otherwise. private var cursorSampler: CursorSampler? + /// URL of the most recently auto-rendered post-processed video (written + /// by `teardownSession` when `videoExperimental` is true). Nil if no + /// auto-render has run yet or the last render failed. + public private(set) var lastAutoRenderURL: URL? = nil /// Monotonic anchor for this session. Captured at configure-on time /// and reused for `cursor.jsonl` `t_ms` + every turn's new /// `t_ms_from_session_start` field on `action.json`. Zero when no @@ -191,6 +195,29 @@ public actor RecordingSession { ) } + // Auto post-process: render the raw capture into a zoom-on-click MP4 + // saved next to recording.mp4 as recording_rendered.mp4. + if let dir = priorDir, priorVideoExperimental { + let renderedURL = dir.appendingPathComponent("recording_rendered.mp4") + log.info( + "auto-rendering \(dir.path, privacy: .public) -> recording_rendered.mp4" + ) + do { + try await RecordingRenderer.render(from: dir, to: renderedURL) + lastAutoRenderURL = renderedURL + log.info( + "auto-render complete: \(renderedURL.path, privacy: .public)" + ) + } catch { + lastAutoRenderURL = nil + log.error( + "auto-render failed: \(error.localizedDescription, privacy: .public)" + ) + } + } else { + lastAutoRenderURL = nil + } + sessionStartMonotonicNs = 0 sessionStartWallClock = nil sessionVideoExperimental = false @@ -199,12 +226,17 @@ public actor RecordingSession { /// Record a single turn. No-op when disabled. Never throws — every /// failure inside is logged and swallowed so the action loop stays /// on its happy path. + /// + /// `actionStartNs` is the `CLOCK_UPTIME_RAW` timestamp captured + /// **before** the tool's animation ran; 0 means "unknown" and falls + /// back to the end timestamp for both start and end fields. public func record( toolName: String, arguments: [String: Any], pid: pid_t?, clickPoint: CGPoint?, - resultSummary: String + resultSummary: String, + actionStartNs: UInt64 = 0 ) async { guard enabled, let root = outputDirectory else { return } @@ -231,7 +263,8 @@ public actor RecordingSession { arguments: arguments, pid: pid, clickPoint: clickPoint, - resultSummary: resultSummary + resultSummary: resultSummary, + actionStartNs: actionStartNs ) // AX snapshot — only when we have a pid to target. @@ -279,7 +312,8 @@ public actor RecordingSession { arguments: [String: Any], pid: pid_t?, clickPoint: CGPoint?, - resultSummary: String + resultSummary: String, + actionStartNs: UInt64 = 0 ) { // `timestamp` stays on the existing ISO-8601 wall-clock format — // other consumers (replay tooling, users reading turn folders @@ -288,10 +322,15 @@ public actor RecordingSession { // frames, so we add a sibling `t_ms_from_session_start` computed // off the same anchor `session.json` + `cursor.jsonl` use. let nowNs = clock_gettime_nsec_np(CLOCK_UPTIME_RAW) + let anchor = sessionStartMonotonicNs let tMsFromSessionStart: Int = - sessionStartMonotonicNs > 0 && nowNs >= sessionStartMonotonicNs - ? Int((nowNs - sessionStartMonotonicNs) / 1_000_000) + anchor > 0 && nowNs >= anchor + ? Int((nowNs - anchor) / 1_000_000) : 0 + let tStartMsFromSessionStart: Int = + anchor > 0 && actionStartNs >= anchor + ? Int((actionStartNs - anchor) / 1_000_000) + : tMsFromSessionStart var payload: [String: Any] = [ "tool": toolName, @@ -299,8 +338,22 @@ public actor RecordingSession { "result_summary": resultSummary, "timestamp": ISO8601DateFormatter().string(from: Date()), "t_ms_from_session_start": tMsFromSessionStart, + "t_start_ms_from_session_start": tStartMsFromSessionStart, ] - if let pid { payload["pid"] = Int(pid) } + if let pid { + payload["pid"] = Int(pid) + // Record the frontmost window bounds so the render pipeline can + // zoom to the target window without needing screen recording to + // re-analyze the video. This query is best-effort (post-action) + // — the window rarely moves during an action. + if let win = WindowEnumerator.frontmostWindow(forPid: pid) { + let b = win.bounds + payload["window_bounds"] = [ + "x": b.x, "y": b.y, + "width": b.width, "height": b.height, + ] + } + } if let clickPoint { payload["click_point"] = [ "x": clickPoint.x, @@ -453,9 +506,11 @@ public actor RecordingSession { "sample_hz": 60, "sample_count": cursorSampleCount, ] + let displayScaleFactor = ScreenInfo.mainScreenSize()?.scaleFactor ?? 1.0 return [ "schema_version": Self.sessionJSONSchemaVersion, + "display_scale_factor": displayScaleFactor, "started_at_wall_clock": isoFormatter.string(from: startedAtWallClock), "started_at_monotonic_ns": startedAtMonotonicNs, "ended_at_monotonic_ns": endedAtMonotonicNs, diff --git a/libs/cua-driver/Sources/CuaDriverCore/Recording/Render/RecordingRenderer.swift b/libs/cua-driver/Sources/CuaDriverCore/Recording/Render/RecordingRenderer.swift index c6965f2b51..531de595fe 100644 --- a/libs/cua-driver/Sources/CuaDriverCore/Recording/Render/RecordingRenderer.swift +++ b/libs/cua-driver/Sources/CuaDriverCore/Recording/Render/RecordingRenderer.swift @@ -17,6 +17,7 @@ import CoreImage import CoreMedia import CoreVideo import Foundation +import Metal import os public enum RecordingRendererError: Error, CustomStringConvertible, Sendable { @@ -74,15 +75,22 @@ public enum RecordingRenderer { /// milliseconds, not wall-clock; a 30fps input with /// `progressIntervalMs = 1000` means "every ~30 frames". public let progressIntervalMs: Double + /// When true (default), use action spans from the trajectory to + /// drive variable-speed rendering (1× inside spans, 5× outside) + /// and window-bbox zoom. Falls back to legacy click-zoom at 1× + /// speed when no action spans are present in the recording. + public let enableSpeedZones: Bool public init( noZoom: Bool = false, defaultScale: Double = 2.0, - progressIntervalMs: Double = 1000 + progressIntervalMs: Double = 1000, + enableSpeedZones: Bool = true ) { self.noZoom = noZoom self.defaultScale = defaultScale self.progressIntervalMs = progressIntervalMs + self.enableSpeedZones = enableSpeedZones } } @@ -103,17 +111,10 @@ public enum RecordingRenderer { ) async throws -> Int { // --- Load trajectory + session metadata -------------------------- let (metadata, clicks, cursorSamples) = try TrajectoryLoader.load(from: inputDirectory) - - let regions: [ZoomRegion] - if options.noZoom { - regions = [] - } else { - regions = ZoomRegionGenerator.generate( - clicks: clicks, - cursorSamples: cursorSamples, - defaultScale: options.defaultScale - ) - } + let rawActionSpans = TrajectoryLoader.loadActionSpans(from: inputDirectory) + let actionSpans = options.enableSpeedZones && !rawActionSpans.isEmpty + ? ActionSpanGenerator.generate(from: rawActionSpans) + : [] let videoURL = inputDirectory.appendingPathComponent("recording.mp4") @@ -143,6 +144,28 @@ public enum RecordingRenderer { let outputWidth = metadata.videoWidth > 0 ? metadata.videoWidth : Int(naturalSize.width) let outputHeight = metadata.videoHeight > 0 ? metadata.videoHeight : Int(naturalSize.height) let frameSize = CGSize(width: outputWidth, height: outputHeight) + // Use the recorded display scale factor to convert screen points → video + // pixels. Defaults to 1.0 for older recordings without this field. + let pointsToPixels = metadata.displayScaleFactor > 0 ? metadata.displayScaleFactor : 1.0 + + // --- Build zoom regions ------------------------------------------- + // When action spans are present (new recordings with window_bounds), + // derive zoom regions from them: each span zooms to its target + // window with eased in/out transitions. Otherwise fall back to the + // legacy click-event zoom. + let regions: [ZoomRegion] + if options.noZoom { + regions = [] + } else if !actionSpans.isEmpty { + regions = zoomRegions(from: actionSpans, frameSize: frameSize, + pointsToPixels: pointsToPixels, defaultScale: options.defaultScale) + } else { + regions = ZoomRegionGenerator.generate( + clicks: clicks, + cursorSamples: cursorSamples, + defaultScale: options.defaultScale + ) + } let reader: AVAssetReader do { @@ -237,10 +260,9 @@ public enum RecordingRenderer { // focus offset that's off by 2× — acceptable for a preview and // documented as the first follow-up. // - // TODO: embed screen scale factor in session.json so the - // renderer doesn't have to guess. Then the math becomes - // `pointsToPixels = session.displayScale`. - let pointsToPixels: Double = 2.0 + // `pointsToPixels` is now read from `session.json` (set above from + // `metadata.displayScaleFactor`). Kept as a local so the render loop + // captures it without re-reading the outer `pointsToPixels` binding. // --- Render loop ------------------------------------------------- // Pin CIContext color pipeline to sRGB so the CoreImage transform @@ -248,11 +270,21 @@ public enum RecordingRenderer { // with the Rec.709 tags on the writer input, the full path stays // tagged end-to-end (source → CIImage → pixel buffer → MP4). let srgb = CGColorSpace(name: CGColorSpace.sRGB) ?? CGColorSpaceCreateDeviceRGB() - let ciContext = CIContext(options: [ - .useSoftwareRenderer: false, - .workingColorSpace: srgb, - .outputColorSpace: srgb, - ]) + // Use a Metal-backed CIContext to keep all frame processing on the GPU, + // eliminating the CPU↔GPU round-trips that cause laggy zoom transitions. + let ciContext: CIContext + if let metalDevice = MTLCreateSystemDefaultDevice() { + ciContext = CIContext(mtlDevice: metalDevice, options: [ + .workingColorSpace: srgb, + .outputColorSpace: srgb, + ]) + } else { + ciContext = CIContext(options: [ + .useSoftwareRenderer: false, + .workingColorSpace: srgb, + .outputColorSpace: srgb, + ]) + } // Hand-off queue the writer expects its input to be fed from. // Realtime flag is off (we're batching), so we rely on @@ -270,6 +302,7 @@ public enum RecordingRenderer { cursorSamples: cursorSamples, frameSize: frameSize, pointsToPixels: pointsToPixels, + actionSpans: actionSpans, progress: progress, progressIntervalMs: options.progressIntervalMs ) @@ -300,6 +333,9 @@ private final class RenderLoopState: @unchecked Sendable { private let cursorSamples: [CursorSample] private let frameSize: CGSize private let pointsToPixels: Double + /// Merged, padded action spans used for variable-speed + window zoom. + /// Empty when the recording pre-dates window_bounds / speed-zone support. + private let actionSpans: [ActionSpan] private let progress: RecordingRendererProgress? private let progressIntervalMs: Double @@ -323,6 +359,7 @@ private final class RenderLoopState: @unchecked Sendable { cursorSamples: [CursorSample], frameSize: CGSize, pointsToPixels: Double, + actionSpans: [ActionSpan], progress: RecordingRendererProgress?, progressIntervalMs: Double ) { @@ -336,6 +373,7 @@ private final class RenderLoopState: @unchecked Sendable { self.cursorSamples = cursorSamples self.frameSize = frameSize self.pointsToPixels = pointsToPixels + self.actionSpans = actionSpans self.progress = progress self.progressIntervalMs = progressIntervalMs } @@ -364,16 +402,7 @@ private final class RenderLoopState: @unchecked Sendable { } let frameCountAtFinish = frameIndex - // `AVAssetWriter` is `NS_SWIFT_NONSENDABLE`, so - // escaping it into the `finishWriting` @Sendable - // closure below produces a warning the project-level - // `@unchecked Sendable` on `RenderLoopState` can't - // silence. The capture is safe here: `self.writer` - // is touched only at end-of-render on a single - // actor-serialized path, and `finishWriting` is - // AVFoundation's own completion callback — the - // framework owns the cross-thread transition. - nonisolated(unsafe) let capturedWriter = self.writer + let capturedWriter = self.writer capturedWriter.finishWriting { [log] in if capturedWriter.status == .completed { continuation.resume(returning: frameCountAtFinish) @@ -422,6 +451,24 @@ private final class RenderLoopState: @unchecked Sendable { pts: CMTime, tMs: Double ) throws { + // --- PTS remapping (variable speed) ---------------------------------- + // When action spans are available, remap the presentation timestamp + // so segments inside spans play at 1× and segments outside play at + // 5×. For 1×-span frames the PTS is shifted (no scale change); for + // 5×-fast frames the PTS is compressed. The mapping is monotonically + // increasing so the encoder always sees valid PTS order. + let outPts: CMTime + if !actionSpans.isEmpty { + let outMs = ActionSpanGenerator.mapPts(tMs, spans: actionSpans) + outPts = CMTimeMake(value: Int64(outMs.rounded()), timescale: 1000) + } else { + outPts = pts + } + + // --- Zoom curve sampling --------------------------------------------- + // Use the pre-built zoom regions (derived from action spans or legacy + // clicks). `sampleCurve` returns eased (scale, focusX, focusY) in + // screen-point space; multiply by pointsToPixels for FrameTransform. let curve = ZoomRegionGenerator.sampleCurve( atMs: tMs, regions: regions, @@ -462,9 +509,80 @@ private final class RenderLoopState: @unchecked Sendable { colorSpace: nil ) - if !adaptor.append(out, withPresentationTime: pts) { + if !adaptor.append(out, withPresentationTime: outPts) { let err = writer.error?.localizedDescription ?? "unknown" throw RecordingRendererError.encodeFailed("adaptor.append failed: \(err)") } } } + +// MARK: - Span → ZoomRegion conversion + +/// Build `ZoomRegion`s from merged action spans so the existing +/// `ZoomRegionGenerator.sampleCurve` machinery produces eased zoom +/// transitions to / from each window's bounding box. +/// +/// - Spans without `windowBounds` are silently dropped (no window +/// to zoom into; the video stays at scale 1.0 during that span). +/// - Scale is computed as the largest factor that fits the window +/// inside the frame while maintaining aspect ratio (letterbox). +/// - Zoom-in and zoom-out transitions use 400 ms, which sits +/// comfortably inside the ±1 s padding ActionSpanGenerator adds. +private func zoomRegions( + from spans: [ActionSpan], + frameSize: CGSize, + pointsToPixels: Double, + defaultScale: Double = 2.0 +) -> [ZoomRegion] { + return spans.compactMap { span -> ZoomRegion? in + // Waypoints are already in screen points (clickPoint / window centre); + // no pointsToPixels here — the renderer multiplies at sample time. + let waypoints = span.focusWaypoints + + // Try window-bbox fit first: largest scale that fits the window exactly. + // Focus is the window centre in screen points; FrameTransform clamps the + // crop to the video boundary, which centres the window as much as possible + // without going off the edge of the display (letterbox case). + if let wb = span.windowBounds { + let winWPx = (wb.width + 32) * pointsToPixels + let winHPx = (wb.height + 32) * pointsToPixels + if winWPx > 0, winHPx > 0 { + let scale = min( + Double(frameSize.width) / winWPx, + Double(frameSize.height) / winHPx + ) + if scale > 1.0 { + let focusX = wb.x + wb.width / 2 + let focusY = wb.y + wb.height / 2 + // Don't pass waypoints here: scale exactly fits the window, + // so any focus deviation from the window centre clips an edge. + // focusX/Y (window centre) is already the right target. + return ZoomRegion( + startMs: span.startMs, + endMs: span.endMs, + focusX: focusX, + focusY: focusY, + scale: scale, + zoomInDurationMs: ZoomDefaults.zoomInWindowMs, + zoomOutDurationMs: ZoomDefaults.transitionWindowMs, + waypoints: nil + ) + } + } + } + // Fallback: window absent or fills/exceeds frame. Zoom to click point. + if let cp = span.clickPoint { + return ZoomRegion( + startMs: span.startMs, + endMs: span.endMs, + focusX: cp.x, + focusY: cp.y, + scale: defaultScale, + zoomInDurationMs: ZoomDefaults.zoomInWindowMs, + zoomOutDurationMs: ZoomDefaults.transitionWindowMs, + waypoints: waypoints + ) + } + return nil + } +} diff --git a/libs/cua-driver/Sources/CuaDriverCore/Recording/Render/TrajectoryLoader.swift b/libs/cua-driver/Sources/CuaDriverCore/Recording/Render/TrajectoryLoader.swift index 36f645d631..1a857a4623 100644 --- a/libs/cua-driver/Sources/CuaDriverCore/Recording/Render/TrajectoryLoader.swift +++ b/libs/cua-driver/Sources/CuaDriverCore/Recording/Render/TrajectoryLoader.swift @@ -20,11 +20,17 @@ public struct SessionMetadata: Sendable, Equatable { /// Total number of cursor samples written. Informational only — /// the renderer decides from the loaded `[CursorSample]` itself. public let cursorSampleCount: Int + /// Backing scale factor of the main display at recording time + /// (e.g. 1.0 for non-Retina, 2.0 for Retina). Used by the renderer + /// to convert screen points → video pixels. Defaults to 1.0 for + /// recordings made before this field was added. + public let displayScaleFactor: Double - public init(videoWidth: Int, videoHeight: Int, cursorSampleCount: Int) { + public init(videoWidth: Int, videoHeight: Int, cursorSampleCount: Int, displayScaleFactor: Double = 1.0) { self.videoWidth = videoWidth self.videoHeight = videoHeight self.cursorSampleCount = cursorSampleCount + self.displayScaleFactor = displayScaleFactor } } @@ -103,11 +109,14 @@ public enum TrajectoryLoader { let cursor = dict["cursor"] as? [String: Any] let sampleCount = (cursor?["sample_count"] as? Int) ?? Int(cursor?["sample_count"] as? Double ?? 0) + let displayScaleFactor = (dict["display_scale_factor"] as? Double) + ?? Double(dict["display_scale_factor"] as? Int ?? 1) return SessionMetadata( videoWidth: width, videoHeight: height, - cursorSampleCount: sampleCount + cursorSampleCount: sampleCount, + displayScaleFactor: displayScaleFactor ) } @@ -231,6 +240,83 @@ public enum TrajectoryLoader { } } + // MARK: - turn-*/action.json → ActionSpans + + /// Walk every `turn-*/action.json` and project all action-class turns + /// into `ActionSpan`s (one per turn). Unlike `loadClicks`, this + /// includes keyboard / scroll / set_value turns — anything that mutates + /// UI state and was recorded with `t_start_ms_from_session_start`. + public static func loadActionSpans(from directory: URL) -> [ActionSpan] { + guard let entries = try? FileManager.default.contentsOfDirectory( + at: directory, + includingPropertiesForKeys: nil, + options: [.skipsHiddenFiles] + ) else { return [] } + + var spans: [ActionSpan] = [] + for entry in entries { + guard entry.lastPathComponent.hasPrefix("turn-") else { continue } + var isDir: ObjCBool = false + guard FileManager.default.fileExists(atPath: entry.path, isDirectory: &isDir), + isDir.boolValue else { continue } + + let actionURL = entry.appendingPathComponent("action.json") + if let span = parseActionSpan(at: actionURL) { + spans.append(span) + } + } + return spans.sorted { $0.startMs < $1.startMs } + } + + private static func isClickOrTypeClassTool(_ name: String) -> Bool { + switch name { + case "click", "double_click", "right_click", "type_text", "type_text_chars": + return true + default: + return false + } + } + + private static func parseActionSpan(at url: URL) -> ActionSpan? { + guard let data = try? Data(contentsOf: url), + let obj = try? JSONSerialization.jsonObject(with: data), + let dict = obj as? [String: Any] else { return nil } + + // Only click/type-class actions get normal-speed spans; other agent + // actions (scroll, hotkey, press_key, etc.) are fast-forwarded. + guard let tool = dict["tool"] as? String, + isClickOrTypeClassTool(tool) else { return nil } + + // Require an end timestamp so we can place the span on the timeline. + guard let endMs = doubleValue(dict["t_ms_from_session_start"]) else { return nil } + let startMs = doubleValue(dict["t_start_ms_from_session_start"]) ?? endMs + + let windowBounds: WindowBounds? + if let wb = dict["window_bounds"] as? [String: Any], + let x = doubleValue(wb["x"]), + let y = doubleValue(wb["y"]), + let w = doubleValue(wb["width"]), + let h = doubleValue(wb["height"]), + w > 0, h > 0 + { + windowBounds = WindowBounds(x: x, y: y, width: w, height: h) + } else { + windowBounds = nil + } + + let clickPoint: ClickPoint? + if let cp = dict["click_point"] as? [String: Any], + let cx = doubleValue(cp["x"]), + let cy = doubleValue(cp["y"]) + { + clickPoint = ClickPoint(x: cx, y: cy) + } else { + clickPoint = nil + } + + return ActionSpan(startMs: startMs, endMs: endMs, windowBounds: windowBounds, clickPoint: clickPoint) + } + // MARK: - helpers /// Coerce a JSON value (Int, Double, String) to Double. Returns nil diff --git a/libs/cua-driver/Sources/CuaDriverCore/Recording/Zoom/ActionSpan.swift b/libs/cua-driver/Sources/CuaDriverCore/Recording/Zoom/ActionSpan.swift new file mode 100644 index 0000000000..34149b5475 --- /dev/null +++ b/libs/cua-driver/Sources/CuaDriverCore/Recording/Zoom/ActionSpan.swift @@ -0,0 +1,171 @@ +// Action-span model: each recorded click/type produces a span covering +// its start → end times with the frontmost window's bounds at action +// time. Post-processing uses spans to drive variable-speed rendering +// (1× inside, 5× outside) and to compute per-action window-bbox zoom. +// +// Spans are padded ±1 s then merged so adjacent actions inside the +// same edit session stay at 1× with a smooth zoom that covers every +// affected window. + +import Foundation + +// MARK: - ActionSpan --------------------------------------------------- + +/// Screen-space click coordinate (points). Stored separately from +/// `WindowBounds` so the renderer can fall back to click-point zoom +/// when the window is larger than the video frame. +public struct ClickPoint: Sendable, Equatable { + public let x: Double + public let y: Double + public init(x: Double, y: Double) { self.x = x; self.y = y } +} + +/// One recorded action's timespan + the window it targeted. +public struct ActionSpan: Sendable, Equatable { + /// Session-monotonic start time of the action (ms). + public let startMs: Double + /// Session-monotonic end time of the action (ms). + public let endMs: Double + /// Frontmost window at action time (screen points, top-left origin). + /// Nil when the action had no associated window (e.g. press_key with + /// no pid, or an older recording without window_bounds in action.json). + public let windowBounds: WindowBounds? + /// Screen-space click coordinate (points) resolved by the driver at + /// dispatch time. Present on click-class actions; nil otherwise. + /// Used as fallback zoom focus when `windowBounds` is too large to + /// fit inside the video frame at any scale ≥ 1×. + public let clickPoint: ClickPoint? + /// Per-action focus waypoints, populated when two or more raw spans + /// are merged into one. Each waypoint records the focus position + /// (screen points) at its action's midpoint time so `zoomRegions` + /// can pan the camera smoothly between actions within a merged span. + /// Nil for single-action (unmerged) spans. + public let focusWaypoints: [FocusWaypoint]? + + public init( + startMs: Double, + endMs: Double, + windowBounds: WindowBounds?, + clickPoint: ClickPoint? = nil, + focusWaypoints: [FocusWaypoint]? = nil + ) { + self.startMs = startMs + self.endMs = endMs + self.windowBounds = windowBounds + self.clickPoint = clickPoint + self.focusWaypoints = focusWaypoints + } +} + +// MARK: - ActionSpanGenerator ------------------------------------------ + +public enum ActionSpanGenerator { + /// Padding added before and after each raw action span (milliseconds). + public static let padMs: Double = 500 + + /// Speed multiplier applied outside action spans in the rendered video. + public static let fastSpeed: Double = 8.0 + + /// Gap threshold: two padded spans closer than this are merged into one. + private static let mergeGapMs: Double = 5_000 + + // MARK: Span generation + + /// Pad each raw action span by ±`padMs`, then merge overlapping spans. + /// Merged spans inherit the first span's window bounds (the dominant + /// target window for the action cluster) and accumulate per-action + /// focus waypoints so the camera pans smoothly between actions. + public static func generate(from rawSpans: [ActionSpan]) -> [ActionSpan] { + let padded = rawSpans.map { + ActionSpan( + startMs: max(0, $0.startMs - padMs), + endMs: $0.endMs + padMs, + windowBounds: $0.windowBounds, + clickPoint: $0.clickPoint + ) + } + return merge(padded.sorted { $0.startMs < $1.startMs }) + } + + /// The best single focus point for a span, in screen points. + /// Prefers the click point; falls back to the window centre. + private static func focusForSpan(_ span: ActionSpan) -> (x: Double, y: Double)? { + if let cp = span.clickPoint { return (cp.x, cp.y) } + if let wb = span.windowBounds { return (wb.x + wb.width / 2, wb.y + wb.height / 2) } + return nil + } + + private static func merge(_ sorted: [ActionSpan]) -> [ActionSpan] { + var result: [ActionSpan] = [] + for span in sorted { + if let last = result.last, span.startMs <= last.endMs + mergeGapMs { + // Midpoint ≈ original action time (padding is symmetric). + let spanMid = (span.startMs + span.endMs) / 2 + + // Bootstrap waypoints from `last` on the first merge. + var waypoints = last.focusWaypoints ?? [] + if waypoints.isEmpty, let lf = focusForSpan(last) { + let lastMid = (last.startMs + last.endMs) / 2 + waypoints = [FocusWaypoint(tMs: lastMid, x: lf.x, y: lf.y)] + } + if let sf = focusForSpan(span) { + waypoints.append(FocusWaypoint(tMs: spanMid, x: sf.x, y: sf.y)) + } + + result[result.count - 1] = ActionSpan( + startMs: last.startMs, + endMs: max(last.endMs, span.endMs), + windowBounds: last.windowBounds ?? span.windowBounds, + clickPoint: last.clickPoint ?? span.clickPoint, + focusWaypoints: waypoints.isEmpty ? nil : waypoints + ) + } else { + result.append(span) + } + } + return result + } + + // MARK: PTS remapping + + /// Map an input video timestamp `inputMs` to an output timestamp that + /// plays spans at 1× speed and non-span segments at `fastSpeed`×. + /// The mapping is piecewise-linear and monotonically increasing. + public static func mapPts(_ inputMs: Double, spans: [ActionSpan]) -> Double { + var outMs: Double = 0 + var prevEnd: Double = 0 + + for span in spans { + if span.startMs > prevEnd { + // Fast segment before this span + let segEnd = min(span.startMs, inputMs) + outMs += (segEnd - prevEnd) / fastSpeed + if inputMs <= span.startMs { return outMs } + } + // 1× span segment + let start = max(span.startMs, prevEnd) + let end = min(span.endMs, inputMs) + if end > start { outMs += end - start } + if inputMs <= span.endMs { return outMs } + prevEnd = span.endMs + } + + // Fast segment after all spans + if inputMs > prevEnd { + outMs += (inputMs - prevEnd) / fastSpeed + } + return outMs + } + + // MARK: Lookup helpers + + /// True when `ms` falls inside any action span. + public static func isInSpan(_ ms: Double, spans: [ActionSpan]) -> Bool { + spans.contains { ms >= $0.startMs && ms <= $0.endMs } + } + + /// The first span that contains `ms`, or nil. + public static func span(at ms: Double, spans: [ActionSpan]) -> ActionSpan? { + spans.first { ms >= $0.startMs && ms <= $0.endMs } + } +} diff --git a/libs/cua-driver/Sources/CuaDriverServer/ToolRegistry.swift b/libs/cua-driver/Sources/CuaDriverServer/ToolRegistry.swift index 963ff90810..fbab2b4e52 100644 --- a/libs/cua-driver/Sources/CuaDriverServer/ToolRegistry.swift +++ b/libs/cua-driver/Sources/CuaDriverServer/ToolRegistry.swift @@ -54,6 +54,11 @@ public struct ToolRegistry: Sendable { guard let handler = handlers[name] else { throw MCPError.invalidParams("Unknown tool: \(name)") } + // Capture monotonic start time before any animation or side-effect + // so the recorded span brackets the full action duration. + let actionStartNs: UInt64 = Self.actionToolNames.contains(name) + ? clock_gettime_nsec_np(CLOCK_UPTIME_RAW) : 0 + let result = try await handler.invoke(arguments) // Recording hook — runs AFTER the tool's invoke. Errors inside @@ -81,7 +86,8 @@ public struct ToolRegistry: Sendable { arguments: snapshotArguments(arguments), pid: pid, clickPoint: clickPoint, - resultSummary: firstTextContent(of: result) + resultSummary: firstTextContent(of: result), + actionStartNs: actionStartNs ) } diff --git a/libs/cua-driver/Sources/CuaDriverServer/Tools/SetRecordingTool.swift b/libs/cua-driver/Sources/CuaDriverServer/Tools/SetRecordingTool.swift index 51be94b2c2..94cb5fcd8f 100644 --- a/libs/cua-driver/Sources/CuaDriverServer/Tools/SetRecordingTool.swift +++ b/libs/cua-driver/Sources/CuaDriverServer/Tools/SetRecordingTool.swift @@ -119,10 +119,16 @@ public enum SetRecordingTool { return errorResult( "Failed to disable recording: \(error.localizedDescription)") } + let renderedSuffix: String + if let rendered = await RecordingSession.shared.lastAutoRenderURL { + renderedSuffix = " Rendered: \(rendered.path)" + } else { + renderedSuffix = "" + } return CallTool.Result( content: [ .text( - text: "✅ Recording disabled.", + text: "✅ Recording disabled.\(renderedSuffix)", annotations: nil, _meta: nil )