cmux-next tests: the Extensions menu suite scales its budgets by the runner's reference workload

.github/test-determinism-allowlist.txt

@@ -80,7 +80,6 @@ Packages/macOS/CmuxNext/Tests/CmuxNextAppTests/ThemedTooltipTests.swift	sleep-th
 Packages/macOS/CmuxNext/Tests/CmuxNextAppTests/TooltipSurfaceTests.swift	assert-on-duration	feat-cmux-next debt before the gate ran there
 Packages/macOS/CmuxNext/Tests/CmuxNextAppTests/WorkspaceRemoteLocalhostTests.swift	sleep-then-assert	feat-cmux-next debt before the gate ran there
 Packages/macOS/CmuxNext/Tests/CmuxNextBrowserTests/BrowserProtectedMediaTests.swift	sleep-then-assert	feat-cmux-next debt before the gate ran there
-Packages/macOS/CmuxNext/Tests/CmuxNextBrowserTests/ExtensionsMenuPerformanceTests.swift	assert-on-duration	feat-cmux-next debt before the gate ran there
 Packages/macOS/CmuxNext/Tests/CmuxNextBrowserTests/OmniboxPhaseABenchmarkTests.swift	assert-on-duration	feat-cmux-next debt before the gate ran there
 Packages/macOS/CmuxNext/Tests/CmuxNextBrowserTests/ToolbarButtonsStateTests.swift	assert-on-duration	feat-cmux-next debt before the gate ran there
 Packages/macOS/CmuxNext/Tests/CmuxNextControlTests/BrowserPageWaitTests.swift	sleep-then-assert	feat-cmux-next debt before the gate ran there

Packages/macOS/CmuxNext/Package.swift

@@ -130,6 +130,8 @@ import PackageDescription
 //     path bar, Quick Look, drag in and out)
 //   CmuxNextCrashReporting -> CmuxSentryReporting, Sentry (crash and exception reports to Sentry:
 //     consent, channel environment, release, scrubbing; no UI, no daemon; cx-urd.58)
+//   CmuxNextTestSupport -> system frameworks only (helpers test targets share, under Tests/; no
+//     product links it)
 //   CmuxNextSessionTransfer -> Design, Settings (Move Sessions from Another cmux Build: the
 //     transferable session and peer values, the picker and the toast text; no AppKit, no daemon;
 //     the App's SessionTransferService discovers peers and moves sessions; cx-yxmkh)
@@ -1557,7 +1559,7 @@ let packageTargets: [Target] = uiLayerPreviewTargets + [
         ),
         .testTarget(
             name: "CmuxNextBrowserTests",
-            dependencies: ["CmuxNextBrowser", "CmuxNextCompat", "CmuxNextDesign", "CmuxNextRemoteFetch", "CmuxNextWakeups"],
+            dependencies: ["CmuxNextBrowser", "CmuxNextCompat", "CmuxNextDesign", "CmuxNextRemoteFetch", "CmuxNextTestSupport", "CmuxNextWakeups"],
             swiftSettings: uiSwiftSettings
         ),
         // The WebKit driver for the Rust browser host (plans/cmux-next/browser-host.md).
@@ -1646,6 +1648,13 @@ let packageTargets: [Target] = uiLayerPreviewTargets + [
             name: "CmuxNextTestExitTrace",
             path: "Tests/Support/CmuxNextTestExitTrace"
         ),
+        // Helpers test targets share, linked into no product: FrameBudgetCalibration, which
+        // scales a main-thread frame budget to the runner a suite runs on.
+        .target(
+            name: "CmuxNextTestSupport",
+            path: "Tests/CmuxNextTestSupport",
+            swiftSettings: uiSwiftSettings
+        ),
         .testTarget(
             name: "CmuxNextAppTests",
             dependencies: [ // sorted: scripts/cmux-next/check-package-dependency-lists.py
@@ -1711,6 +1720,7 @@ let packageTargets: [Target] = uiLayerPreviewTargets + [
                 "CmuxNextTasks",
                 "CmuxNextTerminalGeometry",
                 "CmuxNextTestExitTrace",
+                "CmuxNextTestSupport",
                 "CmuxNextUpdater",
                 "CmuxNextVSCodeServer",
                 "CmuxNextWakeups",

Packages/macOS/CmuxNext/Tests/CmuxNextAppTests/SidebarGroupToggleScalingTests.swift

@@ -1,6 +1,7 @@
 import AppKit
 import CmuxNextDesign
 @testable import CmuxNextSidebar
+import CmuxNextTestSupport
 import Testing
 
 /// Lawrence 2026-10-20: "opening and closing sidebar workspace groups still lags too much ... the algorithm must not

Packages/macOS/CmuxNext/Tests/CmuxNextAppTests/SidebarHoverCardRetargetBenchTests.swift

@@ -1,6 +1,7 @@
 import AppKit
 @testable import CmuxNextDesign
 @testable import CmuxNextSidebar
+import CmuxNextTestSupport
 import Darwin
 import QuartzCore
 import Testing

Packages/macOS/CmuxNext/Tests/CmuxNextAppTests/SidebarLagTests.swift

@@ -1,5 +1,6 @@
 import AppKit
 @testable import CmuxNextSidebar
+import CmuxNextTestSupport
 import Foundation
 import Testing
 

Packages/macOS/CmuxNext/Tests/CmuxNextAppTests/SidebarScrollBench.swift

@@ -1,6 +1,7 @@
 import AppKit
 import CmuxNextDesign
 @testable import CmuxNextSidebar
+import CmuxNextTestSupport
 import Darwin
 import Foundation
 import QuartzCore
@@ -250,17 +251,9 @@ final class SidebarScrollBench {
         }
     }
 
-    static func threadCPU() -> Double {
-        var ts = timespec()
-        clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts)
-        return Double(ts.tv_sec) + Double(ts.tv_nsec) / 1e9
-    }
+    static func threadCPU() -> Double { FrameBudgetCalibration.threadCPU() }
 
-    static func percentile(_ values: [Double], _ p: Double) -> Double {
-        let sorted = values.sorted()
-        guard !sorted.isEmpty else { return 0 }
-        return sorted[min(sorted.count - 1, Int((Double(sorted.count - 1) * p).rounded()))]
-    }
+    static func percentile(_ values: [Double], _ p: Double) -> Double { FrameBudgetCalibration.percentile(values, p) }
 
     static func summary(_ count: Int, _ frames: [Frame]) -> String {
         let cpu = frames.map(\.cpuMs)

Packages/macOS/CmuxNext/Tests/CmuxNextAppTests/SidebarScrollCostTests.swift

@@ -1,5 +1,6 @@
 import AppKit
 @testable import CmuxNextSidebar
+import CmuxNextTestSupport
 import Foundation
 import Testing
 

Packages/macOS/CmuxNext/Tests/CmuxNextBrowserTests/ExtensionsMenuPerformanceTests.swift

@@ -1,11 +1,13 @@
 import AppKit
 @testable import CmuxNextBrowser
+import CmuxNextTestSupport
 import Foundation
 import Testing
 
 /// The Extensions menu opens and hovers within a frame budget (u2229: "this extensions context
-/// menu is also extremely laggy", u2426: "menu visible within one frame"). Budgets: an open under 16 ms, a second open under 10 ms, a
-/// hover under 16 ms, with 40 extensions.
+/// menu is also extremely laggy", u2426: "menu visible within one frame"). Budgets: an open under 16 ms, the
+/// first open under 50 ms, a hover under 16 ms, with 40 extensions, each scaled to the runner by its reference
+/// workload (Budgets, FrameBudgetCalibration), or skipped with the reason printed on a runner too slow or noisy.
 @MainActor @Suite(.serialized) struct ExtensionsMenuPerformanceTests {
     static let count = 40
 
@@ -64,6 +66,70 @@ import Testing
         return now() - started
     }
 
+    /// The suite's budgets, ms of main-thread CPU on the hosts they were set on, held to on this runner scaled
+    /// by its reference workload (FrameBudgetCalibration): aws-m4pro-9, 3-4x slower at every sidebar frame
+    /// too, read a 51.3 ms first open against 50 ms with no change in the code.
+    struct Budgets {
+        var open = 16.0, firstOpen = 50.0, hover = 16.0
+        let calibration: FrameBudgetCalibration
+
+        /// `nominal` ms, scaled to this runner.
+        func scaled(_ nominal: Double) -> Double { calibration.budget(nominal) }
+
+        /// Each timing over its scaled budget, described; empty when every one holds.
+        func overruns(open measuredOpen: Double? = nil, firstOpen measuredFirst: Double? = nil, hover measuredHover: Double? = nil) -> [String] {
+            var found: [String] = []
+            func check(_ what: String, _ measured: Double?, _ nominal: Double) {
+                guard let measured else { return }
+                let limit = scaled(nominal)
+                if measured >= limit {
+                    found.append("\(what) took \(measured) ms, over \(nominal) ms scaled by \(calibration.factor) to \(limit) ms")
+                }
+            }
+            check("an open and close", measuredOpen, open)
+            check("the first open", measuredFirst, firstOpen)
+            check("the slowest hover", measuredHover, hover)
+            return found
+        }
+    }
+
+    /// A slow runner, stood in deterministically as in SidebarScrollCostTests, with no screen, so it runs on
+    /// every runner: every timed open and every reference sample does the reference work `slowdown` times.
+    /// This host's own reference is nominal and its own unslowed opens set the budgets, so the host's speed
+    /// cancels out. The slowed opens break those budgets as they are, and hold them once they scale by the
+    /// slowed reference.
+    @Test func aSlowRunnerScalesTheMenuBudgets() {
+        let slowdown = 3
+        func work(_ times: Int) { for _ in 0..<times { FrameBudgetCalibration.referenceWork() } }
+        func opens(_ times: Int) -> [Double] { (0..<30).map { _ in FrameBudgetCalibration.cpuMs { work(times) } } }
+        work(4) // warm up
+        let nominalMs = FrameBudgetCalibration.percentile(opens(1), 0.5)
+        let budget = 1.5 * FrameBudgetCalibration.percentile(opens(1), 0.95)
+        var slow = FrameBudgetCalibration(nominalMs: nominalMs, reference: { work(slowdown) })
+        slow.checkpoint()
+        let slowed = FrameBudgetCalibration.percentile(opens(slowdown), 0.95)
+        slow.checkpoint()
+        print(String(format: "EXTENSIONS-MENU-CALIBRATION-TEST reference %.3f ms, budget %.3f ms, slowed p95 %.3f ms | ",
+                     nominalMs, budget, slowed) + slow.summary)
+        #expect(slowed > budget, "the stand-in runner is not slow enough to break an unscaled budget")
+        if let reason = slow.skipReason { return print("EXTENSIONS-MENU-CALIBRATION-TEST skipped: \(reason)") }
+        let budgets = Budgets(open: budget, firstOpen: budget, hover: budget, calibration: slow)
+        let overruns = budgets.overruns(open: slowed, firstOpen: slowed, hover: slowed)
+        #expect(overruns.isEmpty, "a runner \(slowdown)x slower at the reference: \(overruns.joined(separator: "; "))")
+    }
+
+    /// A runner at or faster than nominal keeps the budgets as they were set: 16 ms an open, 50 ms the first,
+    /// 16 ms a hover.
+    @Test func aFastRunnerKeepsTheMenuBudgets() {
+        var fast = FrameBudgetCalibration(nominalMs: 1_000)
+        fast.checkpoint()
+        let budgets = Budgets(calibration: fast)
+        #expect(fast.factor == 1, "\(fast.summary)")
+        #expect([budgets.scaled(budgets.open), budgets.scaled(budgets.firstOpen), budgets.scaled(budgets.hover)] == [16, 50, 16])
+        #expect(budgets.overruns(open: 15.9, firstOpen: 49.9, hover: 15.9).isEmpty)
+        #expect(budgets.overruns(firstOpen: 51.3).count == 1)
+    }
+
     /// A visible window with the Extensions button at its top right.
     static func window() -> (NSWindow, NSView) {
         // Inside the main screen, so the bubble below the anchor has room (CI workers run at
@@ -92,6 +158,8 @@ import Testing
         defer { window.close() }
         let handler = ExtensionsMenu.DirectHandler(host: tab) { _ in }
         let controller = ExtensionsMenuController()
+        var calibration = FrameBudgetCalibration()
+        calibration.checkpoint()
         // The toolbar prepares the rows and the panel when a tab binds, off the click's path.
         let prepare = Self.mainThreadCPU { controller.prepare(host: tab) }
         let first = Self.mainThreadCPU { controller.show(host: tab, handler: handler, anchor: anchor) }
@@ -102,10 +170,12 @@ import Testing
             controller.show(host: tab, handler: handler, anchor: anchor)
             controller.close()
         }
+        calibration.checkpoint()
         print("TOOLBAR-PERF extensions.prepare=\(Self.milliseconds(prepare)) ms extensions.firstOpen=\(Self.milliseconds(first)) ms "
-              + "extensions.open=\(Self.milliseconds(open)) ms")
-        #expect(Self.milliseconds(open) < 16, "an open and close took \(Self.milliseconds(open)) ms")
-        #expect(Self.milliseconds(first) < 50, "the first open took \(Self.milliseconds(first)) ms")
+              + "extensions.open=\(Self.milliseconds(open)) ms | \(calibration.summary)")
+        if let reason = calibration.skipReason { return print("TOOLBAR-PERF skipped the extensions open budgets: \(reason)") }
+        let overruns = Budgets(calibration: calibration).overruns(open: Self.milliseconds(open), firstOpen: Self.milliseconds(first))
+        #expect(overruns.isEmpty, "\(overruns.joined(separator: "; "))")
     }
 
     @Test(.enabled("a bubble needs a screen to show") { await Self.hasDisplay() })
@@ -132,6 +202,8 @@ import Testing
         let controller = ExtensionsMenuController()
         controller.show(host: tab, handler: ExtensionsMenu.DirectHandler(host: tab) { _ in }, anchor: anchor)
         try #require(controller.isShown, "\(Self.environment(window))")
+        var calibration = FrameBudgetCalibration()
+        calibration.checkpoint()
         let enter = try #require(NSEvent.enterExitEvent(with: .mouseEntered, location: .zero, modifierFlags: [], timestamp: 0,
                                                         windowNumber: 0, context: nil, eventNumber: 0, trackingNumber: 0, userData: nil))
         let exit = try #require(NSEvent.enterExitEvent(with: .mouseExited, location: .zero, modifierFlags: [], timestamp: 0,
@@ -145,10 +217,13 @@ import Testing
                 row.primary.mouseExited(with: exit)
             })
         }
+        calibration.checkpoint()
         let slowest = hovers.max() ?? .zero
-        print("TOOLBAR-PERF extensions.hover.max=\(Self.milliseconds(slowest)) ms")
-        #expect(Self.milliseconds(slowest) < 16, "slowest hover took \(Self.milliseconds(slowest)) ms")
         controller.close()
+        print("TOOLBAR-PERF extensions.hover.max=\(Self.milliseconds(slowest)) ms | \(calibration.summary)")
+        if let reason = calibration.skipReason { return print("TOOLBAR-PERF skipped the extensions hover budget: \(reason)") }
+        let overruns = Budgets(calibration: calibration).overruns(hover: Self.milliseconds(slowest))
+        #expect(overruns.isEmpty, "\(overruns.joined(separator: "; "))")
     }
 
     @Test(.enabled("a bubble needs a screen to show") { await Self.hasDisplay() })

Packages/macOS/CmuxNext/Tests/CmuxNextAppTests/FrameBudgetCalibration.swift → Packages/macOS/CmuxNext/Tests/CmuxNextTestSupport/FrameBudgetCalibration.swift

@@ -1,3 +1,4 @@
+import Darwin
 import Foundation
 
 /// Frame budgets are main-thread CPU, and one runner's CPU is not another's: on aws-m4pro-9 every
@@ -10,54 +11,54 @@ import Foundation
 /// about the code (over `maxFactor`), or too noisy to read, skips the budgets: the suite prints the
 /// reason and checks nothing timed, rather than passing them.
 @MainActor
-struct FrameBudgetCalibration {
+public struct FrameBudgetCalibration {
     /// Main-thread CPU of one `referenceWork()`, ms, on the hosts the frame budgets were set on: a
     /// fact about the workload, not about the runner a test is on. Measured 1.74 ms on aws-m4pro-5
     /// (2026-10-21, a 500-row quiet scroll p50 of 0.65 ms there), rounded up so a host that fast
     /// keeps a factor of 1.
-    static let nominalReferenceMs = 2.0
+    public static let nominalReferenceMs = 2.0
     /// Past this the budgets are skipped, not scaled: a frame budget more than this many times its
     /// nominal value no longer tells a slow runner from slow code.
-    static let maxFactor = 4.0
+    public static let maxFactor = 4.0
     /// Reference samples are too noisy to scale by when, inside a checkpoint, their p90 is more than
     /// this times their p10, or the slowest checkpoint's median is more than this times the fastest's.
-    static let maxSpread = 2.0
-    static let samplesPerCheckpoint = 15
+    public static let maxSpread = 2.0
+    public static let samplesPerCheckpoint = 15
 
-    let nominalMs: Double
+    public let nominalMs: Double
     /// The reference workload (`referenceWork()`; a test stands in a slowed one).
-    let reference: @MainActor () -> Void
+    public let reference: @MainActor () -> Void
     /// Each checkpoint's samples, ms.
-    private(set) var checkpoints: [[Double]] = []
+    public private(set) var checkpoints: [[Double]] = []
 
-    init(nominalMs: Double = FrameBudgetCalibration.nominalReferenceMs,
+    public init(nominalMs: Double = FrameBudgetCalibration.nominalReferenceMs,
          reference: @escaping @MainActor () -> Void = { FrameBudgetCalibration.referenceWork() }) {
         self.nominalMs = nominalMs
         self.reference = reference
     }
 
     /// Times the reference workload `samplesPerCheckpoint` times, after one warm-up run.
-    mutating func checkpoint() {
+    public mutating func checkpoint() {
         let work = self.reference
         work()
         let samples = (0..<Self.samplesPerCheckpoint).map { _ in Self.cpuMs { work() } }
         checkpoints.append(samples)
     }
 
-    private var medians: [Double] { checkpoints.map { SidebarScrollBench.percentile($0, 0.5) } }
+    private var medians: [Double] { checkpoints.map { Self.percentile($0, 0.5) } }
 
     /// The slowest checkpoint's median over nominal, at least 1.
-    var factor: Double { max(1, (medians.max() ?? nominalMs) / nominalMs) }
+    public var factor: Double { max(1, (medians.max() ?? nominalMs) / nominalMs) }
 
     /// Why this run's timings cannot be held to a budget, or nil when they can.
-    var skipReason: String? {
+    public var skipReason: String? {
         guard !checkpoints.isEmpty else { return "the reference workload was never timed" }
         let raw = (medians.max() ?? 0) / nominalMs
         if raw > Self.maxFactor {
             return String(format: "the runner ran the reference workload %.2fx slower than nominal (over %.1fx)", raw, Self.maxFactor)
         }
         for (index, samples) in checkpoints.enumerated() {
-            let p10 = SidebarScrollBench.percentile(samples, 0.1), p90 = SidebarScrollBench.percentile(samples, 0.9)
+            let p10 = Self.percentile(samples, 0.1), p90 = Self.percentile(samples, 0.9)
             if p90 > p10 * Self.maxSpread {
                 return String(format: "reference samples at checkpoint %d spread p10 %.3f ms to p90 %.3f ms (over %.1fx)", index, p10, p90, Self.maxSpread)
             }
@@ -69,9 +70,9 @@ struct FrameBudgetCalibration {
     }
 
     /// `nominal` ms of budget, scaled to this runner.
-    func budget(_ nominal: Double) -> Double { nominal * factor }
+    public func budget(_ nominal: Double) -> Double { nominal * factor }
 
-    var summary: String {
+    public var summary: String {
         let list = medians.map { String(format: "%.3f", $0) }.joined(separator: " ")
         return String(format: "FRAME-BUDGET-CALIBRATION nominal %.3f ms | checkpoint medians ms ", nominalMs) + list
             + String(format: " | factor %.2f | ", factor) + (skipReason.map { "skip: " + $0 } ?? "scaled")
@@ -82,7 +83,7 @@ struct FrameBudgetCalibration {
 
     /// A fixed, deterministic piece of main-thread work shaped like a frame's: strings formatted,
     /// hashed into a dictionary, sorted and walked, a frame or two's worth of it.
-    static func referenceWork() {
+    public static func referenceWork() {
         var words: [String] = []
         words.reserveCapacity(1_000)
         for i in 0..<1_000 { words.append("row \((i &* 2_654_435_761) % 997) status line \(i)") }
@@ -95,9 +96,23 @@ struct FrameBudgetCalibration {
     }
 
     /// Main-thread CPU of `work`, ms, on the clock SidebarScrollBench times frames with.
-    static func cpuMs(_ work: () -> Void) -> Double {
-        let started = SidebarScrollBench.threadCPU()
+    public static func cpuMs(_ work: () -> Void) -> Double {
+        let started = threadCPU()
         work()
-        return (SidebarScrollBench.threadCPU() - started) * 1_000
+        return (threadCPU() - started) * 1_000
+    }
+
+    /// This thread's CPU time, s: work elsewhere on the host delays the thread without adding to it.
+    nonisolated public static func threadCPU() -> Double {
+        var ts = timespec()
+        clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts)
+        return Double(ts.tv_sec) + Double(ts.tv_nsec) / 1e9
+    }
+
+    /// The `p` quantile of `values` (nearest rank), 0 when there are none.
+    nonisolated public static func percentile(_ values: [Double], _ p: Double) -> Double {
+        let sorted = values.sorted()
+        guard !sorted.isEmpty else { return 0 }
+        return sorted[min(sorted.count - 1, Int((Double(sorted.count - 1) * p).rounded()))]
     }
 }