cmux-next agent pane: render at full rate until scrolls miss frames

Packages/macOS/CmuxNext/Sources/CmuxNextAgentPane/AgentPaneFramePacing.swift

@@ -0,0 +1,70 @@
+import Foundation
+
+/// How fast an agent pane renders.
+public enum AgentPaneRenderRate: Sendable {
+    /// WebKit's default: the display-rate divisor at or above 60 fps (80 Hz
+    /// on a 160 Hz display).
+    case capped
+    /// The display's full rate.
+    case full
+    /// The full rate while scrolls keep up, capped while the machine is
+    /// loaded (``AgentPaneFramePacing``).
+    case adaptive
+}
+
+/// Chooses the agent pane's rendering rate from how its scrolls paced. The
+/// pane renders at the display's full rate until a scroll misses too many
+/// frames, then at WebKit's capped rate (the display-rate divisor at or
+/// above 60 fps) while the machine is loaded. After a backoff, a capped
+/// scroll that paces cleanly brings full rate back; the backoff doubles
+/// when full rate fails again soon after.
+nonisolated struct AgentPaneFramePacing: Equatable, Sendable {
+    /// Scrolls shorter than this many frames decide nothing.
+    static let minimumFrames = 30
+    /// A frame is late when its interval exceeds the expected one by half.
+    static let lateFactor = 1.5
+    /// Late share at full rate that drops to the capped rate.
+    static let overloaded = 0.2
+    /// Late share at the capped rate below which full rate may come back.
+    static let recovered = 0.05
+    /// The first wait at the capped rate before full rate is tried again,
+    /// and the longest.
+    static let firstBackoff: TimeInterval = 10
+    static let maximumBackoff: TimeInterval = 160
+
+    private(set) var fullRate = true
+    private var backoff = firstBackoff
+    /// When the pane last changed rate.
+    private var changedAt: Date?
+
+    /// Records one scroll's frame intervals (ms) on a display that refreshes
+    /// every `displayInterval` ms; true when the pane should render at full rate.
+    mutating func record(intervals: [Double], displayInterval: Double, at now: Date) -> Bool {
+        let capped = Self.cappedInterval(displayInterval)
+        guard intervals.count >= Self.minimumFrames, capped > displayInterval else { return fullRate }
+        let elapsed = changedAt.map { now.timeIntervalSince($0) } ?? .infinity
+        if fullRate {
+            if Self.lateShare(intervals, expected: displayInterval) > Self.overloaded {
+                // Full rate that fails soon after it came back waits longer next time.
+                backoff = elapsed < backoff ? min(backoff * 2, Self.maximumBackoff) : Self.firstBackoff
+                fullRate = false
+                changedAt = now
+            }
+        } else if elapsed >= backoff, Self.lateShare(intervals, expected: capped) <= Self.recovered {
+            fullRate = true
+            changedAt = now
+        }
+        return fullRate
+    }
+
+    /// WebKit's capped frame interval: the display-rate divisor at or above
+    /// 60 fps (80 Hz on a 160 Hz display, 60 Hz on 120 Hz).
+    static func cappedInterval(_ displayInterval: Double) -> Double {
+        guard displayInterval > 0 else { return 0 }
+        return displayInterval * max(1, ((1000.0 / 60 + 0.01) / displayInterval).rounded(.down))
+    }
+
+    private static func lateShare(_ intervals: [Double], expected: Double) -> Double {
+        Double(intervals.count { $0 > expected * lateFactor }) / Double(intervals.count)
+    }
+}

Packages/macOS/CmuxNext/Sources/CmuxNextAgentPane/AgentPaneModel.swift

@@ -15,6 +15,8 @@ public final class AgentPaneModel {
     /// Called when the page switches to or creates a session, so the App can
     /// keep it with the tab.
     @ObservationIgnored public var onSessionChange: ((String) -> Void)?
+    /// Gets each settled transcript scroll's frame intervals (milliseconds).
+    @ObservationIgnored public var onFramePacing: (([Double]) -> Void)?
 
     @ObservationIgnored private let host: any AgentPaneHostProviding
 
@@ -44,6 +46,9 @@ public final class AgentPaneModel {
                 onSessionChange?(id)
             }
             return AgentPaneReply.success()
+        case .framePacing(let intervals):
+            onFramePacing?(intervals)
+            return AgentPaneReply.success()
         case .unsupported(let method):
             return AgentPaneReply.failure(code: "unsupported", message: "Unsupported agent pane request: \(method)")
         }

Packages/macOS/CmuxNext/Sources/CmuxNextAgentPane/AgentPaneRequest.swift

@@ -12,8 +12,13 @@ public nonisolated enum AgentPaneRequest: Equatable, Sendable {
     /// The page switched to or created `sessionId`; the host keeps it so a
     /// reload or relaunch of the pane shows the same session.
     case persistSession(String)
+    /// A settled transcript scroll's frame intervals in milliseconds, at
+    /// most ``maximumPacingFrames``; the pane picks its rendering rate from them.
+    case framePacing([Double])
     case unsupported(String)
 
+    public static let maximumPacingFrames = 640
+
     public static let handlerName = "agentSession"
 
     /// Decodes a `WKScriptMessage.body` (a dictionary once bridged).
@@ -32,6 +37,12 @@ public nonisolated enum AgentPaneRequest: Equatable, Sendable {
             } else {
                 self = .unsupported(method)
             }
+        case "pane.framePacing":
+            if let intervals = params?["intervals"] as? [Double], !intervals.isEmpty {
+                self = .framePacing(Array(intervals.prefix(Self.maximumPacingFrames)))
+            } else {
+                self = .unsupported(method)
+            }
         default:
             self = .unsupported(method)
         }

Packages/macOS/CmuxNext/Sources/CmuxNextAgentPane/AgentPaneView.swift

@@ -41,18 +41,18 @@ public final class AgentPaneView: NSView {
     /// - Parameters:
     ///   - model: Answers the page's host requests.
     ///   - source: The page to load; nil loads ``bundledPage``.
-    ///   - rendersAtFullRate: Renders at the display's rate instead of
-    ///     WebKit's default, the display-rate divisor nearest 60 fps (80 Hz
-    ///     on a 160 Hz display). Off until a frame's paint fits the shorter
-    ///     interval: with it on, a fling ran unevenly at 82-99 Hz (#16471).
-    public init?(model: AgentPaneModel, source: AgentPaneSource? = nil, rendersAtFullRate: Bool = false) {
+    ///   - renderRate: How fast the page renders. Adaptive starts at the
+    ///     display's full rate and caps it while scrolls miss frames, as they
+    ///     do on a loaded machine (#16471).
+    public init?(model: AgentPaneModel, source: AgentPaneSource? = nil, renderRate: AgentPaneRenderRate = .capped) {
         guard let source = source ?? Self.bundledPage.map({ AgentPaneSource.bundled($0) }) else { return nil }
         self.model = model
         self.source = source
         let configuration = WKWebViewConfiguration()
         configuration.websiteDataStore = .nonPersistent()
-        if rendersAtFullRate {
-            configuration.preferences.setWebKitFeature("PreferPageRenderingUpdatesNear60FPSEnabled", enabled: false)
+        self.renderRate = renderRate
+        if renderRate != .capped {
+            configuration.preferences.setWebKitFeature(Self.near60FPSFeature, enabled: false)
         }
         webView = WKWebView(frame: .zero, configuration: configuration)
         super.init(frame: .zero)
@@ -66,6 +66,9 @@ public final class AgentPaneView: NSView {
         // Web Inspector and profiling for the pane (debug.agent_pane).
         webView.isInspectable = true
         #endif
+        if renderRate == .adaptive {
+            model.onFramePacing = { [weak self] intervals in self?.recordFramePacing(intervals) }
+        }
         navigation.view = self
         webView.navigationDelegate = navigation
         addSubview(webView)
@@ -80,6 +83,31 @@ public final class AgentPaneView: NSView {
         webView.frame = bounds
     }
 
+    /// WebKit's feature that renders a page at the display-rate divisor
+    /// nearest 60 fps.
+    static let near60FPSFeature = "PreferPageRenderingUpdatesNear60FPSEnabled"
+
+    public let renderRate: AgentPaneRenderRate
+    private var framePacing = AgentPaneFramePacing()
+    /// The display's refresh rate when the pane has no window screen to ask
+    /// (tests set it).
+    var displayFramesPerSecond: () -> Int = { NSScreen.main?.maximumFramesPerSecond ?? 60 }
+
+    /// An adaptive pane's settled scroll: picks the rate for the next one.
+    func recordFramePacing(_ intervals: [Double], at now: Date = Date()) {
+        let fps = window?.screen?.maximumFramesPerSecond ?? displayFramesPerSecond()
+        guard renderRate == .adaptive, fps > 0 else { return }
+        let full = framePacing.record(intervals: intervals, displayInterval: 1000 / Double(fps), at: now)
+        if full != rendersAtFullRate { rendersAtFullRate = full }
+    }
+
+    /// Whether the page renders at the display's full rate. Setting it
+    /// changes the live page's preferences.
+    public var rendersAtFullRate: Bool {
+        get { webView.configuration.preferences.isWebKitFeatureEnabled(Self.near60FPSFeature) == false }
+        set { webView.configuration.preferences.setWebKitFeature(Self.near60FPSFeature, enabled: !newValue) }
+    }
+
     /// Stops the page (and its WebSocket) for good; call when the tab closes.
     public func close() {
         webView.configuration.userContentController.removeScriptMessageHandler(forName: AgentPaneRequest.handlerName, contentWorld: .page)

Packages/macOS/CmuxNext/Sources/CmuxNextApp/AgentTabs.swift

@@ -22,9 +22,9 @@ final class AgentTabStore {
     /// the dev server `CMUX_NEXT_AGENT_PANE_DEV_URL` names (nil only when the
     /// bundled page is missing).
     private let source: AgentPaneSource?
-    /// `CMUX_NEXT_AGENT_PANE_FULL_RATE=1` (Debug builds): panes render at the
-    /// display's full rate, for measuring it (`AgentPaneView.init`).
-    private let rendersAtFullRate: Bool
+    /// Adaptive, or in Debug builds fixed by `CMUX_NEXT_AGENT_PANE_FULL_RATE`
+    /// (`1` full, `0` capped) for measuring either rate.
+    private let renderRate: AgentPaneRenderRate
     /// `~/.config/cmux/agent-pane/` hot reload, watched while any agent tab
     /// has a view.
     private let customization: AgentPaneCustomizationWatcher
@@ -54,9 +54,13 @@ final class AgentTabStore {
         let allowsDevServer = false
         #endif
         #if DEBUG
-        rendersAtFullRate = environment["CMUX_NEXT_AGENT_PANE_FULL_RATE"] == "1"
+        switch environment["CMUX_NEXT_AGENT_PANE_FULL_RATE"] {
+        case "1": renderRate = .full
+        case "0": renderRate = .capped
+        default: renderRate = .adaptive
+        }
         #else
-        rendersAtFullRate = false
+        renderRate = .adaptive
         #endif
         source = AgentPaneSource.resolve(
             environment: environment, bundledPage: AgentPaneView.bundledPage, allowsDevServer: allowsDevServer
@@ -110,7 +114,7 @@ final class AgentTabStore {
         guard tabsByPane.values.contains(where: { $0.contains(key) }) else { return nil }
         let model = AgentPaneModel(host: host, sessionId: sessions[key])
         model.onSessionChange = { [weak self] session in self?.sessions[key] = session }
-        guard let source, let view = AgentPaneView(model: model, source: source, rendersAtFullRate: rendersAtFullRate) else { return nil }
+        guard let source, let view = AgentPaneView(model: model, source: source, renderRate: renderRate) else { return nil }
         view.customization = customization.current
         views[key] = view
         customization.start()

Packages/macOS/CmuxNext/Sources/CmuxNextApp/Control/DebugAgentPane.swift

@@ -2,6 +2,7 @@
 import AppKit
 import CmuxNextAgentPane
 import CmuxNextSettings
+import ObjectiveC
 import WebKit
 
 /// `debug.agent_pane` (DEBUG builds): performance measurement of the React
@@ -13,7 +14,11 @@ import WebKit
 /// `action`: `seed_rows` (`count`, default 5000), `fling` (`seconds`,
 /// default 3; `nominal_ms`; `wait` returns the stats when the fling ends),
 /// `fling_stats`, `perf_stats` (`raw` adds every frame), `typing_stats`,
-/// `reset_typing`, or `pid` (the WebContent process, for profiling).
+/// `reset_typing`, `pid` (the WebContent process, for profiling), or
+/// `full_rate` (`enabled` turns full-rate rendering on or off on the live
+/// page; returns whether it is on). Every action first stops WebKit from
+/// pausing the page while another window covers it, so a tagged build can
+/// be measured behind the user's windows.
 @MainActor
 enum DebugAgentPane {
     /// Long enough for a 5000-row seed and a waited fling of up to ~25 s.
@@ -36,6 +41,7 @@ enum DebugAgentPane {
             return .object(["error": .string("no agent tab in the given or focused pane")])
         }
         let action = params["action"]?.stringValue ?? ""
+        keepRenderingWhenCovered(view.webView)
         if action == "pid" {
             let selector = NSSelectorFromString("_webProcessIdentifier")
             guard view.webView.responds(to: selector),
@@ -44,8 +50,12 @@ enum DebugAgentPane {
             }
             return .object(["pane": .string(pane), "pid": .number(Double(pid))])
         }
+        if action == "full_rate" {
+            if let enabled = params["enabled"]?.boolValue { view.rendersAtFullRate = enabled }
+            return .object(["pane": .string(pane), "full_rate": .bool(view.rendersAtFullRate)])
+        }
         guard let function = functions[action] else {
-            return .object(["error": .string("unknown action; use seed_rows, fling, fling_stats, perf_stats, typing_stats, reset_typing or pid")])
+            return .object(["error": .string("unknown action; use seed_rows, fling, fling_stats, perf_stats, typing_stats, reset_typing, pid or full_rate")])
         }
         do {
             let result = try await view.webView.callAsyncJavaScript(
@@ -64,6 +74,14 @@ enum DebugAgentPane {
         }
     }
 
+    /// `-[WKWebView _setWindowOcclusionDetectionEnabled:]`, when this WebKit has it.
+    private static func keepRenderingWhenCovered(_ webView: WKWebView) {
+        let selector = NSSelectorFromString("_setWindowOcclusionDetectionEnabled:")
+        guard let method = class_getInstanceMethod(WKWebView.self, selector) else { return }
+        typealias SetEnabled = @convention(c) (AnyObject, Selector, Bool) -> Void
+        unsafeBitCast(method_getImplementation(method), to: SetEnabled.self)(webView, selector, false)
+    }
+
     /// The page function's positional arguments, as Foundation values.
     private static func arguments(_ action: String, _ params: [String: JSONValue]) -> [Any] {
         switch action {

Packages/macOS/CmuxNext/Tests/CmuxNextAgentPaneTests/AgentPaneFramePacingTests.swift

@@ -0,0 +1,71 @@
+import Foundation
+import Testing
+@testable import CmuxNextAgentPane
+
+/// A 160 Hz display: 6.25 ms frames at full rate, 12.5 ms under WebKit's cap.
+@Suite struct AgentPaneFramePacingTests {
+    private let start = Date(timeIntervalSinceReferenceDate: 1_000)
+    private let display = 6.25
+    private let backoff = AgentPaneFramePacing.firstBackoff
+
+    /// A scroll's frame intervals: `late` of every 10 take `slow` ms, the rest `fast`.
+    private func scroll(_ fast: Double, late: Int = 0, slow: Double = 0, frames: Int = 120) -> [Double] {
+        (0..<frames).map { $0 % 10 < late ? slow : fast }
+    }
+
+    @Test func aSmoothScrollStaysAtFullRate() {
+        var pacing = AgentPaneFramePacing()
+        let decision1 = pacing.record(intervals: scroll(display, late: 1, slow: 8), displayInterval: display, at: start)
+        #expect(decision1)
+    }
+
+    @Test func aScrollThatMissesFramesDropsToTheCappedRate() {
+        var pacing = AgentPaneFramePacing()
+        let decision2 = pacing.record(intervals: scroll(display, late: 4, slow: 12.5), displayInterval: display, at: start)
+        #expect(!decision2)
+    }
+
+    @Test func aShortScrollDecidesNothing() {
+        var pacing = AgentPaneFramePacing()
+        let decision3 = pacing.record(intervals: scroll(12.5, frames: AgentPaneFramePacing.minimumFrames - 1), displayInterval: display, at: start)
+        #expect(decision3)
+    }
+
+    @Test func aCleanCappedScrollRestoresFullRateAfterTheBackoff() {
+        var pacing = AgentPaneFramePacing()
+        _ = pacing.record(intervals: scroll(12.5), displayInterval: display, at: start)
+        let decision4 = pacing.record(intervals: scroll(12.5), displayInterval: display, at: start.addingTimeInterval(backoff / 2))
+        #expect(!decision4)
+        let decision5 = pacing.record(intervals: scroll(12.5), displayInterval: display, at: start.addingTimeInterval(backoff + 1))
+        #expect(decision5)
+    }
+
+    @Test func aCappedScrollThatStillMissesFramesKeepsTheCap() {
+        var pacing = AgentPaneFramePacing()
+        _ = pacing.record(intervals: scroll(12.5), displayInterval: display, at: start)
+        let decision6 = pacing.record(intervals: scroll(12.5, late: 3, slow: 25), displayInterval: display, at: start.addingTimeInterval(backoff + 1))
+        #expect(!decision6)
+    }
+
+    @Test func aQuickRelapseDoublesTheBackoff() {
+        var pacing = AgentPaneFramePacing()
+        _ = pacing.record(intervals: scroll(12.5), displayInterval: display, at: start)
+        let restored = start.addingTimeInterval(backoff + 1)
+        let decision7 = pacing.record(intervals: scroll(12.5), displayInterval: display, at: restored)
+        #expect(decision7)
+        let relapse = restored.addingTimeInterval(2)
+        let decision8 = pacing.record(intervals: scroll(12.5), displayInterval: display, at: relapse)
+        #expect(!decision8)
+        let decision9 = pacing.record(intervals: scroll(12.5), displayInterval: display, at: relapse.addingTimeInterval(backoff + 1))
+        #expect(!decision9)
+        let decision10 = pacing.record(intervals: scroll(12.5), displayInterval: display, at: relapse.addingTimeInterval(2 * backoff + 1))
+        #expect(decision10)
+    }
+
+    /// At 60 Hz the cap and the full rate are the same rate.
+    @Test func aDisplayNearSixtyHertzIsLeftAtItsRate() {
+        var pacing = AgentPaneFramePacing()
+        let decision11 = pacing.record(intervals: scroll(33.3), displayInterval: 1000 / 60, at: start)
+        #expect(decision11)
+    }
+}

Packages/macOS/CmuxNext/Tests/CmuxNextAgentPaneTests/AgentPaneHandshakeTests.swift

@@ -40,6 +40,10 @@ import Testing
         #expect(AgentPaneRequest(body: ["id": "1", "method": "ready", "params": [:]]) == .ready)
         #expect(AgentPaneRequest(body: ["method": "chat.persistSession", "params": ["sessionId": "s-2"]]) == .persistSession("s-2"))
         #expect(AgentPaneRequest(body: ["method": "chat.persistSession", "params": ["sessionId": ""]]) == .unsupported("chat.persistSession"))
+        #expect(AgentPaneRequest(body: ["method": "pane.framePacing", "params": ["intervals": [6.25, 12.5]]]) == .framePacing([6.25, 12.5]))
+        #expect(AgentPaneRequest(body: ["method": "pane.framePacing", "params": ["intervals": [Double]()]]) == .unsupported("pane.framePacing"))
+        let long = AgentPaneRequest(body: ["method": "pane.framePacing", "params": ["intervals": Array(repeating: 6.25, count: 1000)]])
+        #expect(long == .framePacing(Array(repeating: 6.25, count: AgentPaneRequest.maximumPacingFrames)))
         #expect(AgentPaneRequest(body: ["method": "chat.send", "params": ["text": "hi"]]) == .unsupported("chat.send"))
         #expect(AgentPaneRequest(body: "ready") == .unsupported(""))
     }