From 42f6cb0aef3653e983cf03e296207025647e74b3 Mon Sep 17 00:00:00 2001 From: Anthony Eid Date: Sun, 31 May 2026 15:34:08 -0400 Subject: [PATCH 1/5] Work in progress --- crates/editor/benches/editor_render.rs | 115 +------- crates/gpui/src/app.rs | 4 + crates/gpui/src/app/bench_context.rs | 387 +++++++++++++++++++++++++ crates/gpui/src/gpui.rs | 2 +- crates/gpui_macros/src/bench.rs | 48 +++ crates/gpui_macros/src/gpui_macros.rs | 7 + gpui_bench_plan.md | 263 +++++++++++++++++ 7 files changed, 718 insertions(+), 108 deletions(-) create mode 100644 crates/gpui/src/app/bench_context.rs create mode 100644 crates/gpui_macros/src/bench.rs create mode 100644 gpui_bench_plan.md diff --git a/crates/editor/benches/editor_render.rs b/crates/editor/benches/editor_render.rs index e93c94e1ae6e6c..cedcdd62f11cd9 100644 --- a/crates/editor/benches/editor_render.rs +++ b/crates/editor/benches/editor_render.rs @@ -1,20 +1,19 @@ -use criterion::{Bencher, BenchmarkId}; +use criterion::Bencher; use editor::{ Editor, EditorMode, MultiBuffer, actions::{DeleteToPreviousWordStart, SelectAll, SplitSelectionIntoLines}, }; -use gpui::{AppContext, Focusable as _, TestAppContext, TestDispatcher}; -use rand::{Rng as _, SeedableRng as _, rngs::StdRng}; +use gpui::{AppContext as _, BenchAppContext, Focusable as _}; use settings::SettingsStore; -use ui::IntoElement; -use util::RandomCharIter; -fn editor_input_with_1000_cursors(bencher: &mut Bencher<'_>, cx: &TestAppContext) { - let mut cx = cx.clone(); +#[gpui::bench] +fn editor_input_with_1000_cursors(bencher: &mut Bencher<'_>, cx: &mut BenchAppContext) { + init_context(cx); + let text = String::from_iter(["line:\n"; 1000]); let buffer = cx.update(|cx| MultiBuffer::build_simple(&text, cx)); - let cx = cx.add_empty_window(); + let mut cx = cx.add_empty_window(); let editor = cx.update(|window, cx| { let editor = cx.new(|cx| { let mut editor = Editor::new(EditorMode::full(), buffer, None, window, cx); @@ -58,110 +57,12 @@ fn editor_input_with_1000_cursors(bencher: &mut Bencher<'_>, cx: &TestAppContext }); } -fn open_editor_with_one_long_line(bencher: &mut Bencher<'_>, args: &(String, TestAppContext)) { - let (text, cx) = args; - let mut cx = cx.clone(); - - bencher.iter(|| { - let buffer = cx.update(|cx| MultiBuffer::build_simple(&text, cx)); - - let cx = cx.add_empty_window(); - let _ = cx.update(|window, cx| { - let editor = cx.new(|cx| { - let mut editor = Editor::new(EditorMode::full(), buffer, None, window, cx); - editor.set_style(editor::EditorStyle::default(), window, cx); - editor - }); - window.focus(&editor.focus_handle(cx), cx); - editor - }); - }); -} - -fn editor_render(bencher: &mut Bencher<'_>, cx: &TestAppContext) { - let mut cx = cx.clone(); - let buffer = cx.update(|cx| { - let mut rng = StdRng::seed_from_u64(1); - let text_len = rng.random_range(10000..90000); - if rng.random() { - let text = RandomCharIter::new(&mut rng) - .take(text_len) - .collect::(); - MultiBuffer::build_simple(&text, cx) - } else { - MultiBuffer::build_random(&mut rng, cx) - } - }); - - let cx = cx.add_empty_window(); - let editor = cx.update(|window, cx| { - let editor = cx.new(|cx| { - let mut editor = Editor::new(EditorMode::full(), buffer, None, window, cx); - editor.set_style(editor::EditorStyle::default(), window, cx); - editor - }); - window.focus(&editor.focus_handle(cx), cx); - editor - }); - - bencher.iter(|| { - cx.update(|window, cx| { - // editor.update(cx, |editor, cx| editor.move_down(&MoveDown, window, cx)); - let mut view = editor.clone().into_any_element(); - let _ = view.request_layout(window, cx); - let _ = view.prepaint(window, cx); - view.paint(window, cx); - }); - }) -} - -pub fn benches() { - let dispatcher = TestDispatcher::new(1); - let cx = gpui::TestAppContext::build(dispatcher, None); +fn init_context(cx: &mut BenchAppContext) { cx.update(|cx| { let store = SettingsStore::test(cx); cx.set_global(store); assets::Assets.load_test_fonts(cx); theme_settings::init(theme::LoadThemes::JustBase, cx); - // release_channel::init(semver::Version::new(0,0,0), cx); editor::init(cx); }); - - let mut criterion: criterion::Criterion<_> = - (criterion::Criterion::default()).configure_from_args(); - - // setup app context - let mut group = criterion.benchmark_group("Time to render"); - group.bench_with_input( - BenchmarkId::new("editor_render", "TestAppContext"), - &cx, - editor_render, - ); - - group.finish(); - - let text = String::from_iter(["char"; 1000]); - let mut group = criterion.benchmark_group("Build buffer with one long line"); - group.bench_with_input( - BenchmarkId::new("editor_with_one_long_line", "(String, TestAppContext )"), - &(text, cx.clone()), - open_editor_with_one_long_line, - ); - - group.finish(); - - let mut group = criterion.benchmark_group("multi cursor edits"); - group.bench_with_input( - BenchmarkId::new("editor_input_with_1000_cursors", "TestAppContext"), - &cx, - editor_input_with_1000_cursors, - ); - group.finish(); -} - -fn main() { - benches(); - criterion::Criterion::default() - .configure_from_args() - .final_summary(); } diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rs index c8f1032fbd931b..9da55b333b7515 100644 --- a/crates/gpui/src/app.rs +++ b/crates/gpui/src/app.rs @@ -23,6 +23,8 @@ use parking_lot::RwLock; use slotmap::SlotMap; pub use async_context::*; +#[cfg(any(test, feature = "test-support"))] +pub use bench_context::*; use collections::{FxHashMap, FxHashSet, HashMap, VecDeque}; pub use context::*; pub use entity_map::*; @@ -56,6 +58,8 @@ use crate::{ }; mod async_context; +#[cfg(any(test, feature = "test-support"))] +mod bench_context; mod context; mod entity_map; #[cfg(any(test, feature = "test-support"))] diff --git a/crates/gpui/src/app/bench_context.rs b/crates/gpui/src/app/bench_context.rs new file mode 100644 index 00000000000000..6af1b7a9fa9088 --- /dev/null +++ b/crates/gpui/src/app/bench_context.rs @@ -0,0 +1,387 @@ +use std::{future::Future, rc::Rc, sync::Arc}; + +use anyhow::{Result, anyhow}; + +use crate::{ + AnyView, AnyWindowHandle, App, AppCell, AppContext, BackgroundExecutor, Bounds, Context, Empty, + Entity, EntityId, Focusable, ForegroundExecutor, Global, Render, Reservation, Task, + TestDispatcher, TestPlatform, VisualContext, Window, WindowBounds, WindowHandle, WindowOptions, + app::{GpuiBorrow, GpuiMode}, +}; + +/// A GPUI app context for Criterion benchmarks. +/// +/// `BenchAppContext` is intentionally separate from `TestAppContext`: it owns a +/// benchmark app instance and exposes only the app/window operations needed by +/// benchmark setup. Criterion remains responsible for the measured loop via its +/// `Bencher` API. +#[derive(Clone)] +pub struct BenchAppContext { + app: Rc, + background_executor: BackgroundExecutor, + foreground_executor: ForegroundExecutor, + dispatcher: TestDispatcher, + benchmark_name: Option<&'static str>, +} + +impl BenchAppContext { + /// Creates a new benchmark app context. + pub fn new(benchmark_name: Option<&'static str>) -> Self { + Self::with_seed(benchmark_name, 0) + } + + /// Creates a new benchmark app context with the provided scheduler seed. + pub fn with_seed(benchmark_name: Option<&'static str>, seed: u64) -> Self { + Self::build(TestDispatcher::new(seed), benchmark_name) + } + + fn build(dispatcher: TestDispatcher, benchmark_name: Option<&'static str>) -> Self { + let dispatcher = Arc::new(dispatcher); + let background_executor = BackgroundExecutor::new(dispatcher.clone()); + let foreground_executor = ForegroundExecutor::new(dispatcher.clone()); + let platform = TestPlatform::new(background_executor.clone(), foreground_executor.clone()); + let asset_source = Arc::new(()); + let http_client = http_client::FakeHttpClient::with_404_response(); + let app = App::new_app(platform, asset_source, http_client); + app.borrow_mut().mode = GpuiMode::test(); + + Self { + app, + background_executor, + foreground_executor, + dispatcher: (*dispatcher).clone(), + benchmark_name, + } + } + + /// The benchmark function name that created this context. + pub fn benchmark_name(&self) -> Option<&'static str> { + self.benchmark_name + } + + /// Returns the background executor used by this benchmark app. + pub fn background_executor(&self) -> &BackgroundExecutor { + &self.background_executor + } + + /// Returns the foreground executor used by this benchmark app. + pub fn foreground_executor(&self) -> &ForegroundExecutor { + &self.foreground_executor + } + + /// Runs pending scheduled work until the benchmark app is idle. + pub fn run_until_idle(&self) { + self.dispatcher.run_until_parked(); + } + + /// Updates the app and flushes synchronous GPUI effects afterward. + pub fn update(&mut self, update: impl FnOnce(&mut App) -> R) -> R { + let mut app = self.app.borrow_mut(); + app.update(update) + } + + /// Reads app state. + pub fn read(&self, read: impl FnOnce(&App) -> R) -> R { + let app = self.app.borrow(); + read(&app) + } + + /// Adds a window with an empty root view for benchmark setup. + pub fn add_empty_window(&mut self) -> BenchWindowContext { + let window = { + let mut app = self.app.borrow_mut(); + let bounds = Bounds::maximized(None, &app); + let window: AnyWindowHandle = app + .open_window( + WindowOptions { + window_bounds: Some(WindowBounds::Windowed(bounds)), + ..Default::default() + }, + |_, cx| cx.new(|_| Empty), + ) + .expect("failed to open benchmark window") + .into(); + window + }; + + self.run_until_idle(); + BenchWindowContext { + cx: self.clone(), + window, + } + } + + /// Runs GPUI benchmark teardown. + pub fn teardown(mut self) { + self.run_until_idle(); + self.update(|cx| { + cx.background_executor().forbid_parking(); + cx.quit(); + }); + self.run_until_idle(); + } +} + +impl AppContext for BenchAppContext { + fn new(&mut self, build_entity: impl FnOnce(&mut Context) -> T) -> Entity { + let mut app = self.app.borrow_mut(); + app.new(build_entity) + } + + fn reserve_entity(&mut self) -> Reservation { + let mut app = self.app.borrow_mut(); + app.reserve_entity() + } + + fn insert_entity( + &mut self, + reservation: Reservation, + build_entity: impl FnOnce(&mut Context) -> T, + ) -> Entity { + let mut app = self.app.borrow_mut(); + app.insert_entity(reservation, build_entity) + } + + fn update_entity( + &mut self, + handle: &Entity, + update: impl FnOnce(&mut T, &mut Context) -> R, + ) -> R { + let mut app = self.app.borrow_mut(); + app.update_entity(handle, update) + } + + fn as_mut<'a, T>(&'a mut self, _: &Entity) -> GpuiBorrow<'a, T> + where + T: 'static, + { + panic!("Cannot use as_mut with BenchAppContext. Call update() instead.") + } + + fn read_entity(&self, handle: &Entity, read: impl FnOnce(&T, &App) -> R) -> R + where + T: 'static, + { + let app = self.app.borrow(); + app.read_entity(handle, read) + } + + fn update_window(&mut self, window: AnyWindowHandle, update: F) -> Result + where + F: FnOnce(AnyView, &mut Window, &mut App) -> T, + { + let mut app = self.app.borrow_mut(); + app.update_window(window, update) + } + + fn with_window( + &mut self, + entity_id: EntityId, + update: impl FnOnce(&mut Window, &mut App) -> R, + ) -> Option { + let mut app = self.app.borrow_mut(); + app.with_window(entity_id, update) + } + + fn read_window( + &self, + window: &WindowHandle, + read: impl FnOnce(Entity, &App) -> R, + ) -> Result + where + T: 'static, + { + let app = self.app.borrow(); + app.read_window(window, read) + } + + fn background_spawn(&self, future: impl Future + Send + 'static) -> Task + where + R: Send + 'static, + { + self.background_executor.spawn(future) + } + + fn read_global(&self, callback: impl FnOnce(&G, &App) -> R) -> R + where + G: Global, + { + let app = self.app.borrow(); + app.read_global(callback) + } +} + +/// A window-specific context for GPUI benchmarks. +/// +/// This is separate from `VisualTestContext`; it provides access to a benchmark +/// window without exposing test-only helpers such as input simulation. +#[derive(Clone)] +pub struct BenchWindowContext { + cx: BenchAppContext, + window: AnyWindowHandle, +} + +impl BenchWindowContext { + /// Returns the underlying benchmark app context. + pub fn app_context(&mut self) -> &mut BenchAppContext { + &mut self.cx + } + + /// Returns the window associated with this context. + pub fn window_handle(&self) -> AnyWindowHandle { + self.window + } + + /// Updates the benchmark window. + pub fn update(&mut self, update: impl FnOnce(&mut Window, &mut App) -> R) -> R { + self.cx + .update_window(self.window, |_, window, cx| update(window, cx)) + .expect("benchmark window was unexpectedly closed") + } + + /// Runs pending scheduled work until the benchmark app is idle. + pub fn run_until_idle(&self) { + self.cx.run_until_idle(); + } +} + +impl AppContext for BenchWindowContext { + fn new(&mut self, build_entity: impl FnOnce(&mut Context) -> T) -> Entity { + self.window + .update(&mut self.cx, |_, _, cx| cx.new(build_entity)) + .expect("benchmark window was unexpectedly closed") + } + + fn reserve_entity(&mut self) -> Reservation { + self.cx.reserve_entity() + } + + fn insert_entity( + &mut self, + reservation: Reservation, + build_entity: impl FnOnce(&mut Context) -> T, + ) -> Entity { + self.window + .update(&mut self.cx, |_, _, cx| { + cx.insert_entity(reservation, build_entity) + }) + .expect("benchmark window was unexpectedly closed") + } + + fn update_entity( + &mut self, + handle: &Entity, + update: impl FnOnce(&mut T, &mut Context) -> R, + ) -> R { + self.cx.update_entity(handle, update) + } + + fn as_mut<'a, T>(&'a mut self, handle: &Entity) -> GpuiBorrow<'a, T> + where + T: 'static, + { + self.cx.as_mut(handle) + } + + fn read_entity(&self, handle: &Entity, read: impl FnOnce(&T, &App) -> R) -> R + where + T: 'static, + { + self.cx.read_entity(handle, read) + } + + fn update_window(&mut self, window: AnyWindowHandle, update: F) -> Result + where + F: FnOnce(AnyView, &mut Window, &mut App) -> T, + { + self.cx.update_window(window, update) + } + + fn with_window( + &mut self, + entity_id: EntityId, + update: impl FnOnce(&mut Window, &mut App) -> R, + ) -> Option { + self.cx.with_window(entity_id, update) + } + + fn read_window( + &self, + window: &WindowHandle, + read: impl FnOnce(Entity, &App) -> R, + ) -> Result + where + T: 'static, + { + self.cx.read_window(window, read) + } + + fn background_spawn(&self, future: impl Future + Send + 'static) -> Task + where + R: Send + 'static, + { + self.cx.background_spawn(future) + } + + fn read_global(&self, callback: impl FnOnce(&G, &App) -> R) -> R + where + G: Global, + { + self.cx.read_global(callback) + } +} + +impl VisualContext for BenchWindowContext { + type Result = Result; + + fn window_handle(&self) -> AnyWindowHandle { + self.window + } + + fn update_window_entity( + &mut self, + entity: &Entity, + update: impl FnOnce(&mut T, &mut Window, &mut Context) -> R, + ) -> Result { + let entity = entity.clone(); + self.cx + .app + .borrow_mut() + .with_window(entity.entity_id(), |window, app| { + entity.update(app, |entity, cx| update(entity, window, cx)) + }) + .ok_or_else(|| { + anyhow!("entity has no current window; use `update` instead of `update_in`") + }) + } + + fn new_window_entity( + &mut self, + build_entity: impl FnOnce(&mut Window, &mut Context) -> T, + ) -> Result> { + self.window.update(&mut self.cx, |_, window, cx| { + cx.new(|cx| build_entity(window, cx)) + }) + } + + fn replace_root_view( + &mut self, + build_view: impl FnOnce(&mut Window, &mut Context) -> V, + ) -> Result> + where + V: 'static + Render, + { + self.window.update(&mut self.cx, |_, window, cx| { + window.replace_root(cx, build_view) + }) + } + + fn focus(&mut self, entity: &Entity) -> Result<()> + where + V: Focusable, + { + self.window.update(&mut self.cx, |_, window, cx| { + entity.read(cx).focus_handle(cx).focus(window, cx) + }) + } +} diff --git a/crates/gpui/src/gpui.rs b/crates/gpui/src/gpui.rs index 12aeebfb9562d6..50f0aecc00c032 100644 --- a/crates/gpui/src/gpui.rs +++ b/crates/gpui/src/gpui.rs @@ -94,7 +94,7 @@ pub use executor::*; pub use geometry::*; pub use global::*; pub use gpui_macros::{ - AppContext, IntoElement, Render, VisualContext, property_test, register_action, test, + AppContext, IntoElement, Render, VisualContext, bench, property_test, register_action, test, }; pub use gpui_shared_string::*; pub use gpui_util::arc_cow::ArcCow; diff --git a/crates/gpui_macros/src/bench.rs b/crates/gpui_macros/src/bench.rs new file mode 100644 index 00000000000000..c311b803110972 --- /dev/null +++ b/crates/gpui_macros/src/bench.rs @@ -0,0 +1,48 @@ +use proc_macro::TokenStream; +use quote::{format_ident, quote}; +use syn::{ItemFn, spanned::Spanned}; + +pub fn bench(args: TokenStream, function: TokenStream) -> TokenStream { + if !args.is_empty() { + return error_to_stream(syn::Error::new( + proc_macro2::TokenStream::from(args).span(), + "#[gpui::bench] does not accept arguments yet", + )); + } + + let mut inner_fn = match syn::parse::(function) { + Ok(function) => function, + Err(error) => return error_to_stream(error), + }; + + if let Some(asyncness) = &inner_fn.sig.asyncness { + return error_to_stream(syn::Error::new( + asyncness.span(), + "#[gpui::bench] does not support async benchmark functions yet", + )); + } + + let outer_fn_name = inner_fn.sig.ident.clone(); + let inner_fn_name = format_ident!("__gpui_bench_{}", outer_fn_name); + let criterion_group_name = format_ident!("__gpui_bench_group_{}", outer_fn_name); + inner_fn.sig.ident = inner_fn_name.clone(); + + TokenStream::from(quote! { + #inner_fn + + fn #outer_fn_name(criterion: &mut criterion::Criterion) { + criterion.bench_function(stringify!(#outer_fn_name), |bencher| { + let mut cx = gpui::BenchAppContext::new(Some(stringify!(#outer_fn_name))); + #inner_fn_name(bencher, &mut cx); + cx.teardown(); + }); + } + + criterion::criterion_group!(#criterion_group_name, #outer_fn_name); + criterion::criterion_main!(#criterion_group_name); + }) +} + +fn error_to_stream(error: syn::Error) -> TokenStream { + TokenStream::from(error.into_compile_error()) +} diff --git a/crates/gpui_macros/src/gpui_macros.rs b/crates/gpui_macros/src/gpui_macros.rs index e30c85e6edbee8..1fe3fa97eaedba 100644 --- a/crates/gpui_macros/src/gpui_macros.rs +++ b/crates/gpui_macros/src/gpui_macros.rs @@ -1,3 +1,4 @@ +mod bench; mod derive_action; mod derive_app_context; mod derive_into_element; @@ -189,6 +190,12 @@ pub fn test(args: TokenStream, function: TokenStream) -> TokenStream { test::test(args, function) } +/// `#[gpui::bench]` annotates a Criterion benchmark that runs with GPUI support. +#[proc_macro_attribute] +pub fn bench(args: TokenStream, function: TokenStream) -> TokenStream { + bench::bench(args, function) +} + /// A variant of `#[gpui::test]` that supports property-based testing. /// /// A property test, much like a standard GPUI randomized test, allows testing diff --git a/gpui_bench_plan.md b/gpui_bench_plan.md new file mode 100644 index 00000000000000..fcb9b8e95f727f --- /dev/null +++ b/gpui_bench_plan.md @@ -0,0 +1,263 @@ +# `#[gpui::bench]` + `BenchAppContext` plan + +## Motivation + +Zed has no first-class way to benchmark and regression-gate the work that actually +drops frames: synchronous main-thread cost and the fan-out a single state change +triggers. We currently reach for `criterion` + `TestAppContext`, then fall back to +`xctrace` + `dsymutil` for any real attribution. That workflow is slow, manual, and +its numbers are misleading. + +This came out of the agent `edit_file` performance work. The pathological case was +applying each `CharOperation` as its own `buffer.edit` transaction, so one edit +fanned out into hundreds of `BufferEvent::Edited` events, each triggering a +tree-sitter reparse, an LSP `didChange`, the action-log diff, and the editor's +on-edit observers (matching brackets, bracket colorization, code actions, outline). +We only found it by profiling after the fact. We want to catch this class of bug in +CI, before it ships. + +### What's wrong with the current tooling + +- **`TestScheduler` is single-threaded.** Foreground and background runnables share + one queue on one thread (`crates/scheduler/src/test_scheduler.rs`, + `schedule_background_with_priority`). So a benchmark cannot separate "blocked the + main thread" from "ran off-thread." Our `large_multi_edit` number (~919 ms) mixes + both, even though the diff and LSP work run off-thread in production. +- **No per-update timing.** To see where time went we had to `dsymutil` a 145 MB + binary and parse a Time Profiler XML export by hand. +- **No cascade visibility.** Nothing reports "this edit emitted N events and fired M + observers" — the exact metric that would have caught the footgun deterministically. +- **No frame metric.** "Would this drop a frame" had to be computed offline from + miniprof spans. +- **Harness artifacts skew absolute numbers.** Maximized window lays out far more + lines than a real pane, the headless harness repaints per edit (the real app + coalesces to one paint per frame), and per-iteration setup (project/editor/LSP + construction) pollutes the samples. + +The current bench is good for **relative** before/after comparisons, but its absolute +breakdown overstates editor rendering and conflates threads. + +## Goals + +- Measure **foreground-thread blocking time** truthfully (separate from offloaded work). +- Make the **update/effect cascade observable and assertable** (events, observers, + transactions, reparses, re-renders, allocations). +- Produce a **frame-drop metric** against a budget. +- Lean on `criterion` for statistics, sampling, baselines, and reporting rather than + reimplementing them. +- Emit **profiles viewable in Tracy / Perfetto** so drill-down doesn't require + `xctrace` + `dsymutil`. +- Enable **deterministic regression gates** in CI. + +## Non-goals + +- Replacing `criterion`'s statistics engine. +- Generating standalone Instruments `.trace` files (not a writable format; see below). +- A general-purpose APM/tracing framework. This is a test/bench harness. + +## Proposed solution + +A `#[gpui::bench]` macro that provides a `BenchAppContext`, layered on top of +`criterion`. The macro owns app/window/scheduler setup (excluded from timing), +exposes richer measurement than wall-clock, and can drop a trace next to the +criterion results. + +The core design is **one backend-agnostic span + counter recorder**, with the +measurement frontends (criterion) and the trace exporters (Tracy/Perfetto) both +reading from it. Capture once, surface many ways. + +## Capabilities + +1. **Real thread pool.** Run background work on real threads via a `BenchScheduler` + (the `Scheduler` trait already abstracts this; `PlatformScheduler` is the + production impl). This lets us measure only foreground occupancy, fixing the + single-thread conflation. +2. **Per-update closure timing.** Reuse the miniprof hook + (`TaskTiming`/`GLOBAL_THREAD_TIMINGS`, recorded in the dispatcher trampoline) and + extend it to the `cx.update` / `Context::update` / `flush_effects` entry points, + attributed by `#[track_caller]` call site. +3. **Update/effect cascade counts.** Per `flush_effects` cycle: events emitted, + observer/subscription callbacks fired, entities notified, windows invalidated, + re-renders triggered, buffer transactions, tree-sitter reparses. These are + **deterministic** and make the best regression gates. +4. **Foreground-blocking / inter-frame time / frame drops.** With a real scheduler + and a modeled frame cadence, bucket foreground occupancy into frames against a + budget (16.67 ms / 8.33 ms) and report longest contiguous span + frames dropped. +5. **Allocation counting.** Allocations/bytes per update via a counting allocator, + to catch alloc storms. +6. **Realistic frame loop + window sizing.** Model a normal pane size and coalesce + draws to a vsync cadence, removing the maximized-window / per-edit-repaint + artifacts. +7. **Trace export.** Emit Tracy (via the existing importer) and Chrome/Perfetto JSON + for timeline drill-down. +8. **Regression gates.** Count-based `assert!`s that fail the build on fan-out + regressions. + +## Criterion integration + +`criterion` only models **one scalar per iteration**, but it gives us a lot we should +not reimplement: warmup, adaptive sampling, outlier detection, summary statistics, +baseline save/compare with change detection (p-values), and CLI/HTML reporting. + +### The seam: `iter_custom` + +`iter`/`iter_batched` time wall-clock of the whole closure, which is what conflates +threads. `iter_custom` instead lets the harness return the `Duration`, so the +`BenchAppContext` runs the workload on a real pool and hands criterion only the +**foreground-blocking time**: + +```rust +b.iter_custom(|iters| { + let mut foreground = Duration::ZERO; + for _ in 0..iters { + let mut cx = BenchAppContext::new(); // setup excluded + cx.run_workload(|cx| run_streamed_edit(cx)); + foreground += cx.foreground_busy_time(); // harness-measured, not wall-clock + } + foreground +}); +``` + +Criterion then runs its statistics on the right number. This is a small step from the +`iter_batched` we already use. + +### The clean version: a custom `Measurement` + +`criterion` is generic over a `Measurement` trait (default `WallTime`; third-party +impls exist for CPU cycles / perf counters). Implementing it for "foreground busy +time" makes criterion report and graph **"foreground ms"** natively. One +`Measurement` per group, so it's one metric per run (run separate groups for +wall-time vs foreground-time vs alloc-count). + +### What stays out of criterion + +- **Counts** (events, transactions, reparses, allocations) → plain `#[test]` + assertions. Deterministic, and far less flaky as CI gates than wall-clock. +- **Frame-drop lists / per-update breakdowns / traces** → diagnostics or a + side-channel artifact, not criterion measurements. + +### Caveats + +- A real thread pool widens criterion's CIs and triggers more outlier warnings. + Manage with more samples; fine for tracked trends. +- **CI gating on criterion wall-clock is flaky** on shared runners. Gate CI on the + **counts**; use criterion's baseline/change-detection for tracked latency trends, + not hard pass/fail. + +## Trace export (Tracy / Perfetto / Instruments) + +Once the harness captures spans + counters, exporting a profile is just a serializer. +Use one in-memory model with pluggable exporters. + +- **Tracy — partly pre-built.** Zed already ships `tracy-import-miniprofiler` + (`docs/src/performance.md`) that converts `*.miniprof.json` → a Tracy capture, plus + a `tracy` feature (`ztracing/tracy`). If `BenchAppContext` emits the same miniprof + JSON schema, the existing importer and the analysis tooling work on bench output + for free. Richer Tracy use (zones/frames/plots) needs a small importer extension. +- **Chrome Trace / Perfetto — best portable default.** A plain JSON array of + `{name, ph, ts, dur, pid, tid, args}` opens directly in `ui.perfetto.dev` with no + importer and no macOS dependency. It maps cleanly onto what we care about: + - nested duration events → the update / `flush_effects` / render hierarchy, + - **flow events (`ph: "s"/"f"`) → the cascade chain** (edit → arrows to each + triggered effect), the unique thing criterion can't express, + - **counter events (`ph: "C"`) → the count metrics** as area-graph tracks. +- **Instruments — qualified.** You can't synthesize a `.trace` bundle from data. + Realistic options: record the bench binary under `xcrun xctrace record` (works + today; what we did), or emit `os_signpost` intervals that show in Instruments + _during a recording_. "Generate a file to open in Instruments" is not practical. + +## The determinism principle + +Real concurrency trades away deterministic ordering, which is what makes wall-clock +perf gates flaky. Split the two: + +- **Times** (foreground-blocking, frame timings) → advisory + tracked trends via + criterion; do not hard-gate CI on them. +- **Counts** (events, observers, transactions, reparses, allocations) → deterministic + even with a real pool; these are the actual CI gates. + +The headline value is not "faster timing"; it is **making the fan-out observable and +assertable.** + +## Phased plan + +### Phase 1 — Foundation: criterion + Tracy plumbing (do this first) + +Establish the harness scaffolding so every later metric is an incremental add, not a +rewrite. No new scheduler or metrics yet. + +- `BenchAppContext` wrapping `AppContext` or `App`, owning setup outside the timed region + and the leak-detector/window teardown. +- A backend-agnostic span recorder seeded from the existing miniprof timing data. +- `criterion` wired through `iter_custom` (start with wall-clock of the measured + region; the number gets more accurate in Phase 2). +- A `#[gpui::bench]` macro that expands to a `criterion_group!` + `bench_function`, + injects the `BenchAppContext`, and on a separate non-timed pass dumps a trace. +- Trace exporters: **miniprof JSON** (reuse `tracy-import-miniprofiler`) and + **Chrome/Perfetto JSON**. +- First consumer: port the existing `crates/agent/benches/edit_file_tool.rs` to the + macro and confirm we can open its trace in Tracy/Perfetto. + +Outcome: one command runs the bench, gets criterion stats, and drops a trace you can +open in Tracy or Perfetto. Everything below plugs into this. + +### Phase 2 — Truthful foreground time + +- `BenchScheduler` running a real background thread pool. +- `foreground_busy_time()` measurement; optionally a custom criterion `Measurement` + so criterion reports "foreground ms" natively. + +Outcome: numbers separate main-thread blocking from offloaded work (fixes the +single-thread conflation). + +### Phase 3 — Cascade instrumentation + regression gates (highest value) + +- Per-`flush_effects` counters: events emitted, observer callbacks, notifies, window + invalidations, re-renders, buffer transactions, reparses. +- Allocation counting. +- Count-based `assert!` helpers and the first regression tests (e.g. "an N-op edit + emits one `Edited` per chunk, not per op"). +- Surface counters as Perfetto counter tracks / Tracy plots. + +Outcome: the footgun class is caught deterministically in CI. + +### Phase 4 — Frame realism + +- Realistic window sizing (normal pane, not maximized). +- Modeled frame loop with coalesced draws. +- Frame-drop metric (longest foreground span per frame, frames over budget). + +Outcome: editor-render cost reflects what users feel, not a maximized-window per-edit +repaint. + +### Phase 5 — Polish + +- Macro ergonomics, parameters (window size, frame budget, pool size, seed). +- CI integration: count gates as required checks; criterion baselines tracked for + trends. +- Richer Tracy export (zones/frames/plots) if the importer extension is worth it. + +## Open questions / risks + +- How much determinism to keep with a real pool. A hybrid (real background pool + + deterministic foreground driver, optional seed) may be the sweet spot. +- Instrumentation must be zero-cost when the bench/instrumentation feature is off, + since the hooks sit on gpui hot paths. +- CI noise budget for any time-based signal; lean on counts for gating. +- Whether to extend `tracy-import-miniprofiler` for zones/plots or just rely on + Perfetto for the rich view. + +## Reference points in the existing codebase + +- Per-poll timing hook: `crates/gpui_macos/src/dispatcher.rs` (`trampoline`, + `TaskTiming`, `add_task_timing`, `GLOBAL_THREAD_TIMINGS`). +- Spawn-location attribution: `crates/scheduler/src/scheduler.rs` + (`RunnableMeta { location }`). +- Single-threaded test scheduling to replace for benches: + `crates/scheduler/src/test_scheduler.rs`. +- Effect system to instrument for cascade counts: `gpui` `App::flush_effects` / + `pending_effects` / the `Effect` enum. +- Existing miniprof + Tracy import path: `crates/miniprofiler_ui/` and + `docs/src/performance.md`. +- Current bench to port first: `crates/agent/benches/edit_file_tool.rs` + (already `harness = false`, `criterion`, `--profile-time` compatible). From 9c57459bfb03831e009fc433892f46b6cf0b1aca Mon Sep 17 00:00:00 2001 From: Anthony Eid Date: Sun, 31 May 2026 19:56:37 -0400 Subject: [PATCH 2/5] Update plan --- gpui_bench_plan.md | 351 ++++++++++++++++++++++++++++++++------------- 1 file changed, 249 insertions(+), 102 deletions(-) diff --git a/gpui_bench_plan.md b/gpui_bench_plan.md index fcb9b8e95f727f..fc001e277dd94e 100644 --- a/gpui_bench_plan.md +++ b/gpui_bench_plan.md @@ -8,7 +8,7 @@ triggers. We currently reach for `criterion` + `TestAppContext`, then fall back `xctrace` + `dsymutil` for any real attribution. That workflow is slow, manual, and its numbers are misleading. -This came out of the agent `edit_file` performance work. The pathological case was +This came out of the agent `edit_file_tool` performance work. The pathological case was applying each `CharOperation` as its own `buffer.edit` transaction, so one edit fanned out into hundreds of `BufferEvent::Edited` events, each triggering a tree-sitter reparse, an LSP `didChange`, the action-log diff, and the editor's @@ -57,91 +57,195 @@ breakdown overstates editor rendering and conflates threads. ## Proposed solution -A `#[gpui::bench]` macro that provides a `BenchAppContext`, layered on top of -`criterion`. The macro owns app/window/scheduler setup (excluded from timing), -exposes richer measurement than wall-clock, and can drop a trace next to the -criterion results. +A `#[gpui::bench]` macro that provides a `BenchAppContext` and a +`BenchBencher`, layered on top of `criterion`. `BenchAppContext` owns app, +window, scheduler, recorder, and teardown state. `BenchBencher` stays close to +Criterion's `Bencher` API while adding GPUI-specific helpers for rendering +frames, measuring foreground time, and running mounted views. The core design is **one backend-agnostic span + counter recorder**, with the -measurement frontends (criterion) and the trace exporters (Tracy/Perfetto) both -reading from it. Capture once, surface many ways. +measurement frontends (Criterion) and the trace exporters (Tracy/Perfetto) both +reading from it. Capture once, surface many ways. Criterion still owns sampling, +statistics, baselines, and reporting for a single scalar measurement at a time; +GPUI sidecar reports and traces carry the richer multi-metric data. ## Capabilities -1. **Real thread pool.** Run background work on real threads via a `BenchScheduler` - (the `Scheduler` trait already abstracts this; `PlatformScheduler` is the - production impl). This lets us measure only foreground occupancy, fixing the - single-thread conflation. -2. **Per-update closure timing.** Reuse the miniprof hook - (`TaskTiming`/`GLOBAL_THREAD_TIMINGS`, recorded in the dispatcher trampoline) and - extend it to the `cx.update` / `Context::update` / `flush_effects` entry points, - attributed by `#[track_caller]` call site. -3. **Update/effect cascade counts.** Per `flush_effects` cycle: events emitted, - observer/subscription callbacks fired, entities notified, windows invalidated, - re-renders triggered, buffer transactions, tree-sitter reparses. These are - **deterministic** and make the best regression gates. -4. **Foreground-blocking / inter-frame time / frame drops.** With a real scheduler - and a modeled frame cadence, bucket foreground occupancy into frames against a - budget (16.67 ms / 8.33 ms) and report longest contiguous span + frames dropped. -5. **Allocation counting.** Allocations/bytes per update via a counting allocator, - to catch alloc storms. -6. **Realistic frame loop + window sizing.** Model a normal pane size and coalesce - draws to a vsync cadence, removing the maximized-window / per-edit-repaint - artifacts. -7. **Trace export.** Emit Tracy (via the existing importer) and Chrome/Perfetto JSON - for timeline drill-down. -8. **Regression gates.** Count-based `assert!`s that fail the build on fan-out - regressions. +### Criterion-backed measurements + +- **Wall time first.** The initial version should use Criterion's normal wall-clock + measurement so the macro feels familiar and the PoC remains small. +- **Foreground time and missed frames next.** Phase 2 should add foreground busy time + and frame-drop / missed-frame measurements, probably via `iter_custom` before a + custom `Measurement` implementation. +- **Multiple requested measurements become multiple Criterion benchmarks.** Criterion + models one scalar per benchmark result, so a single GPUI workload requesting + multiple measurements should expand to distinct benchmark IDs such as + `render_button/wall_time`, `render_button/foreground_time`, and + `render_button/missed_frames`. +- **Rich reports stay out of Criterion's scalar model.** A `BenchReport` can contain + wall time, foreground time, missed frames, spans, counters, allocation stats, and + render details, but Criterion should analyze one chosen scalar per run. +- **Later measurement backends.** Allocation bytes, longest foreground span, + renderer-blocking time, CPU cycles, instructions, cache misses, and branch misses + can be added later where platform support exists. + +### GPUI recorder + +- **Backend-agnostic spans and counters.** Record once, then feed Criterion, + Perfetto, miniprof/Tracy, and sidecar JSON/Markdown reports. +- **Per-update timing.** Reuse the miniprof hook (`TaskTiming` / + `GLOBAL_THREAD_TIMINGS`, recorded in dispatcher trampolines) and extend it to + `App::update`, entity updates, window updates, and `flush_effects`, attributed by + `#[track_caller]` call site. +- **Update/effect cascade counts.** Per outer update / `flush_effects` cycle: update + calls, entity updates, nested update depth, effects queued/flushed, events emitted, + observer/subscription callbacks fired, entities notified, global notifications, + windows invalidated, re-renders triggered, deferred callbacks, and action dispatches. +- **Notify attribution.** Track explicit `Context::notify()` separately from total + `App::notify(...)` calls. Some paths, such as `GpuiBorrow` drop, notify implicitly + without going through `Context::notify`; the report should make that distinction + visible. +- **Extensible crate-specific counters.** GPUI owns generic app/render/frame metrics. + Editor, terminal, language, project, and agent crates should add domain-specific + counters through a generic span/counter API instead of baking Zed-specific metrics + into GPUI. +- **Low overhead when disabled.** Hooks sit on hot paths, so instrumentation must be + feature-gated and effectively zero-cost outside benchmark/instrumentation builds. + +### Frame and render instrumentation + +- **Foreground-blocking / inter-frame time / frame drops.** With a real scheduler and + a modeled frame cadence, bucket foreground occupancy into frames against a budget + (16.67 ms / 8.33 ms) and report longest contiguous span, renderer-blocking time, + missed frames, and worst frame delay. +- **Realistic frame loop + window sizing.** Model normal pane/window sizes and + coalesce draws to a vsync cadence, removing maximized-window and per-edit-repaint + artifacts. +- **Render pipeline spans.** Measure layout, prepaint, paint, scene construction, and + later backend-specific renderer work. Attribute these spans to windows, entities, + and call sites where possible. +- **Render cache/reuse metrics.** Track view cache reuse, dirty subtree size, layout + cache hits/misses, text/glyph/image/SVG/path cache hits/misses, shaped text runs, + paths built/tessellated, scene command count, and unchanged subtree skips as the + render pipeline instrumentation matures. +- **First frame vs steady state.** Make cold first-frame render, warmed steady-state + render, and incremental-update render distinct benchmark modes. + +### First-class render benchmark API + +- **Mounted views.** Provide an ergonomic `cx.mount_view(...)` / `cx.render_entity(...)` + API that mounts an `Entity` where `T: Render` in a benchmark window and returns a + `MountedView`. +- **Renderer iteration helpers.** `BenchBencher` should provide helpers such as + `bench_renderer(&mut MountedView, ...)` that run user code between frames, flush + effects, render one or more frames, and record layout/prepaint/paint/frame metrics. +- **Actions and updates between frames.** Mounted views should support dispatching + actions, updating the mounted entity, resizing the window, warming caches, rendering + a single frame, and rendering every frame in a modeled frame loop. + +### Trace export and artifacts + +- **Trace export.** Emit miniprof JSON for Tracy import and Chrome/Perfetto JSON for + timeline drill-down. +- **Artifact layout.** Store GPUI artifacts next to Criterion output so local runs and + CI uploads are easy to find: trace files, sidecar `BenchReport` JSON, and a concise + Markdown summary. +- **Regression gates.** Provide count-based and frame-count assertion helpers for + deterministic CI gates; use Criterion baselines for tracked latency trends, not hard + pass/fail wall-clock gates. + +### Optional platform counters + +- **Hardware counters.** On platforms that support it, add optional measurements for + cycles, instructions, cache misses, and branch misses. Linux can use perf counters; + macOS should initially rely on Instruments / `xctrace` rather than first-class + portable hardware-counter support. ## Criterion integration `criterion` only models **one scalar per iteration**, but it gives us a lot we should not reimplement: warmup, adaptive sampling, outlier detection, summary statistics, -baseline save/compare with change detection (p-values), and CLI/HTML reporting. +baseline save/compare with change detection (p-values), CLI/HTML reporting, and +profile-mode execution. -### The seam: `iter_custom` +The GPUI API should stay close to Criterion so existing Criterion users can onboard +quickly: -`iter`/`iter_batched` time wall-clock of the whole closure, which is what conflates -threads. `iter_custom` instead lets the harness return the `Duration`, so the -`BenchAppContext` runs the workload on a real pool and hands criterion only the -**foreground-blocking time**: +```rust +#[gpui::bench(sample_size = 20, measurement = wall_time)] +fn render_button(bencher: &mut BenchBencher<'_>, cx: &mut BenchAppContext) { + let mut view = cx.mount_view(|window, cx| ButtonView::new(window, cx)); + + bencher.bench_renderer(&mut view, |view, window, cx| { + window.dispatch_action(...); + view.update(cx, |button, cx| button.click(cx)); + }); +} +``` + +### Initial version: normal Criterion wall time + +The first implementation should inject `BenchAppContext` and `BenchBencher`, then let +`BenchBencher::iter` delegate to Criterion's normal `Bencher::iter`. This keeps the +PoC simple and produces ordinary Criterion output. + +### Phase 2 seam: `iter_custom` + +`iter` / `iter_batched` time wall-clock of the whole closure, which conflates +foreground work, background work, and harness work. `iter_custom` lets the harness +return the scalar Criterion should analyze. Phase 2 can use it for foreground time +and missed-frame measurements before committing to custom `Measurement` plumbing: ```rust b.iter_custom(|iters| { - let mut foreground = Duration::ZERO; - for _ in 0..iters { - let mut cx = BenchAppContext::new(); // setup excluded - cx.run_workload(|cx| run_streamed_edit(cx)); - foreground += cx.foreground_busy_time(); // harness-measured, not wall-clock - } - foreground + cx.iter_foreground_time(iters, |cx| { + run_workload(cx); + }) }); ``` -Criterion then runs its statistics on the right number. This is a small step from the -`iter_batched` we already use. +Criterion then runs its statistics on the right number. -### The clean version: a custom `Measurement` +### Later: custom `Measurement` `criterion` is generic over a `Measurement` trait (default `WallTime`; third-party -impls exist for CPU cycles / perf counters). Implementing it for "foreground busy -time" makes criterion report and graph **"foreground ms"** natively. One -`Measurement` per group, so it's one metric per run (run separate groups for -wall-time vs foreground-time vs alloc-count). +impls exist for CPU cycles / perf counters). Implementing measurements such as +`ForegroundTime`, `MissedFrames`, or `AllocatedBytes` makes Criterion report those +units natively. A Criterion group has one `Measurement`, so multiple requested GPUI +measurements should generate multiple Criterion benchmark IDs or groups. + +A rich `BenchReport` may contain every metric from a run, but Criterion still reduces +one selected measurement to `f64` for statistical analysis. Use sidecar reports and +traces for the full multi-metric data. + +### Later: Criterion `Profiler` / `--profile-time` + +Criterion supports `--profile-time N`, which runs each benchmark workload for about +`N` seconds without normal sampling/statistical analysis so an external or in-process +profiler can collect data. Its `Profiler` trait has `start_profiling` and +`stop_profiling` hooks with the benchmark ID and output directory. A later GPUI +integration should use this to enable the recorder for profile-mode runs and write +Perfetto/miniprof/Tracy artifacts next to Criterion's output. -### What stays out of criterion +This is a later-phase integration because `Profiler` hooks do not receive +`BenchAppContext`; we need a clean bridge between Criterion's profile lifecycle and +GPUI's active recorder. Early versions can use a simpler non-timed trace pass. -- **Counts** (events, transactions, reparses, allocations) → plain `#[test]` - assertions. Deterministic, and far less flaky as CI gates than wall-clock. -- **Frame-drop lists / per-update breakdowns / traces** → diagnostics or a - side-channel artifact, not criterion measurements. +### What stays out of Criterion + +- **Counts** (events, transactions, reparses, notifies, observer callbacks) → + deterministic assertion helpers / tests. These are less flaky as CI gates than + time-based measurements. +- **Frame-drop lists / per-update breakdowns / traces** → `BenchReport` sidecars and + trace artifacts, not Criterion's primary scalar result. ### Caveats -- A real thread pool widens criterion's CIs and triggers more outlier warnings. +- A real thread pool widens Criterion's CIs and triggers more outlier warnings. Manage with more samples; fine for tracked trends. -- **CI gating on criterion wall-clock is flaky** on shared runners. Gate CI on the - **counts**; use criterion's baseline/change-detection for tracked latency trends, +- **CI gating on Criterion wall-clock is flaky** on shared runners. Gate CI on the + **counts**; use Criterion's baseline/change-detection for tracked latency trends, not hard pass/fail. ## Trace export (Tracy / Perfetto / Instruments) @@ -157,10 +261,16 @@ Use one in-memory model with pluggable exporters. - **Chrome Trace / Perfetto — best portable default.** A plain JSON array of `{name, ph, ts, dur, pid, tid, args}` opens directly in `ui.perfetto.dev` with no importer and no macOS dependency. It maps cleanly onto what we care about: - - nested duration events → the update / `flush_effects` / render hierarchy, + - nested duration events → the update / `flush_effects` / layout / prepaint / + paint hierarchy, - **flow events (`ph: "s"/"f"`) → the cascade chain** (edit → arrows to each - triggered effect), the unique thing criterion can't express, + triggered effect), the unique thing Criterion can't express, - **counter events (`ph: "C"`) → the count metrics** as area-graph tracks. +- **Criterion profile-mode artifacts — later.** A polished implementation should use + Criterion's `Profiler` hooks during `--profile-time` runs to write GPUI artifacts + into the benchmark output directory. This is not required for the initial wall-time + PoC because bridging Criterion's profile lifecycle to the active `BenchAppContext` + recorder needs additional design. - **Instruments — qualified.** You can't synthesize a `.trace` bundle from data. Realistic options: record the bench binary under `xcrun xctrace record` (works today; what we did), or emit `os_signpost` intervals that show in Instruments @@ -172,7 +282,7 @@ Real concurrency trades away deterministic ordering, which is what makes wall-cl perf gates flaky. Split the two: - **Times** (foreground-blocking, frame timings) → advisory + tracked trends via - criterion; do not hard-gate CI on them. + Criterion; do not hard-gate CI on them. - **Counts** (events, observers, transactions, reparses, allocations) → deterministic even with a real pool; these are the actual CI gates. @@ -181,59 +291,92 @@ assertable.** ## Phased plan -### Phase 1 — Foundation: criterion + Tracy plumbing (do this first) +### Phase 1 — Foundation: Criterion wall-time PoC Establish the harness scaffolding so every later metric is an incremental add, not a -rewrite. No new scheduler or metrics yet. - -- `BenchAppContext` wrapping `AppContext` or `App`, owning setup outside the timed region - and the leak-detector/window teardown. -- A backend-agnostic span recorder seeded from the existing miniprof timing data. -- `criterion` wired through `iter_custom` (start with wall-clock of the measured - region; the number gets more accurate in Phase 2). -- A `#[gpui::bench]` macro that expands to a `criterion_group!` + `bench_function`, - injects the `BenchAppContext`, and on a separate non-timed pass dumps a trace. -- Trace exporters: **miniprof JSON** (reuse `tracy-import-miniprofiler`) and - **Chrome/Perfetto JSON**. -- First consumer: port the existing `crates/agent/benches/edit_file_tool.rs` to the - macro and confirm we can open its trace in Tracy/Perfetto. - -Outcome: one command runs the bench, gets criterion stats, and drops a trace you can -open in Tracy or Perfetto. Everything below plugs into this. - -### Phase 2 — Truthful foreground time - -- `BenchScheduler` running a real background thread pool. -- `foreground_busy_time()` measurement; optionally a custom criterion `Measurement` - so criterion reports "foreground ms" natively. - -Outcome: numbers separate main-thread blocking from offloaded work (fixes the -single-thread conflation). +rewrite. Keep this phase small and Criterion-like. + +- `BenchAppContext` as a benchmark-specific app context, distinct from + `TestAppContext`, owning app/window/scheduler setup and teardown outside the timed + region. +- `BenchBencher` as a Criterion-like wrapper around `criterion::Bencher`, initially + delegating to normal wall-time `iter` / `iter_batched` APIs. +- A `#[gpui::bench]` macro that expands to Criterion `bench_function` plumbing, + injects `BenchAppContext` and `BenchBencher`, and supports basic Criterion-like + options such as `sample_size` over time. +- Initial measurement: **wall time only**. +- First consumer: port a small existing benchmark to prove one command builds, runs, + and prints normal Criterion stats. + +Outcome: GPUI benchmarks look familiar to Criterion users and can run with a real +`BenchAppContext`, but no deep GPUI metrics are required yet. + +### Phase 2 — Mounted render benchmarks + foreground/frame measurements + +Make rendering a specific entity first-class and add the first GPUI-specific scalar +measurements. + +- `MountedView` for an `Entity` where `T: Render`, mounted in a benchmark + window with controlled size/theme/font setup. +- `cx.mount_view(...)` / `cx.render_entity(...)` helpers for quickly creating render + benchmarks without manually wiring windows. +- `BenchBencher::bench_renderer(&mut MountedView, ...)`, rendering one frame per + iteration after running user-provided code between frames. +- Helpers for dispatching actions, updating the mounted entity, resizing the window, + rendering a single frame, warming caches, and rendering a modeled frame loop. +- `BenchScheduler` or equivalent foreground/background separation sufficient to + compute foreground busy time. +- `foreground_time` and `missed_frames` measurements, initially via `iter_custom`. +- Basic frame model: frame cadence, longest foreground span, missed frames, and worst + frame delay. + +Outcome: agent panel, terminal, editor, and simple GPUI component render benchmarks +can be written ergonomically, and numbers begin to reflect main-thread blocking and +frame impact rather than only wall-clock time. ### Phase 3 — Cascade instrumentation + regression gates (highest value) -- Per-`flush_effects` counters: events emitted, observer callbacks, notifies, window - invalidations, re-renders, buffer transactions, reparses. +Add deterministic fan-out visibility and assertion helpers. + +- Per-outer-update / `flush_effects` counters: update calls, entity updates, nested + depth, effects queued/flushed, events emitted, observer/subscription callbacks, + explicit `Context::notify()` calls, total `App::notify(...)` calls, implicit notify + paths such as `GpuiBorrow` drop, window invalidations, and re-renders. +- Generic span/counter API for domain-specific metrics from editor, terminal, + language, project, and agent crates. - Allocation counting. - Count-based `assert!` helpers and the first regression tests (e.g. "an N-op edit emits one `Edited` per chunk, not per op"). - Surface counters as Perfetto counter tracks / Tracy plots. -Outcome: the footgun class is caught deterministically in CI. +Outcome: the fan-out footgun class is caught deterministically in CI. -### Phase 4 — Frame realism +### Phase 4 — Render pipeline instrumentation -- Realistic window sizing (normal pane, not maximized). -- Modeled frame loop with coalesced draws. -- Frame-drop metric (longest foreground span per frame, frames over budget). +Add the detail needed to improve GPUI's renderer itself. -Outcome: editor-render cost reflects what users feel, not a maximized-window per-edit -repaint. +- Layout, prepaint, paint, scene construction, and render-backend spans. +- Entity/window/callsite attribution for render pipeline spans. +- View cache reuse, dirty subtree size, layout cache hits/misses, text/glyph/image / + SVG/path cache hits/misses, shaped text runs, paths built/tessellated, scene command + count, and unchanged subtree skips. +- Distinct cold first-frame, warmed steady-state, and incremental-update render modes. -### Phase 5 — Polish +Outcome: GPUI render pipeline changes can be benchmarked directly and attributed to +specific phases and cache behavior. -- Macro ergonomics, parameters (window size, frame budget, pool size, seed). -- CI integration: count gates as required checks; criterion baselines tracked for +### Phase 5 — Trace/export polish and advanced measurements + +- Trace exporters: **miniprof JSON** (reuse `tracy-import-miniprofiler`) and + **Chrome/Perfetto JSON** with spans, flows, and counters. +- Criterion `Profiler` integration for seamless `--profile-time` trace artifacts. +- Macro ergonomics and Criterion-like parameters: `sample_size`, `warm_up_time`, + `measurement_time`, `app = fresh_per_sample`, `drain = after_iteration`, window + size, frame budget, pool size, and seed. +- Optional custom Criterion `Measurement` implementations for foreground time, + missed frames, allocation bytes, renderer-blocking time, and platform hardware + counters. +- CI integration: count gates as required checks; Criterion baselines tracked for trends. - Richer Tracy export (zones/frames/plots) if the importer extension is worth it. @@ -246,6 +389,8 @@ repaint. - CI noise budget for any time-based signal; lean on counts for gating. - Whether to extend `tracy-import-miniprofiler` for zones/plots or just rely on Perfetto for the rich view. +- How to bridge Criterion `Profiler` lifecycle hooks to the active `BenchAppContext` + recorder without global state that makes parallel or multi-app benchmarks fragile. ## Reference points in the existing codebase @@ -259,5 +404,7 @@ repaint. `pending_effects` / the `Effect` enum. - Existing miniprof + Tracy import path: `crates/miniprofiler_ui/` and `docs/src/performance.md`. -- Current bench to port first: `crates/agent/benches/edit_file_tool.rs` - (already `harness = false`, `criterion`, `--profile-time` compatible). +- Current PoC bench: `crates/editor/benches/editor_render.rs`. +- Near-term consumers: agent panel render benchmarks, terminal render benchmarks, + and `crates/agent/benches/edit_file_tool.rs` (already `harness = false`, + `criterion`, `--profile-time` compatible). From 555ad2c53eb423004cd1589ad5e93f4721e843d8 Mon Sep 17 00:00:00 2001 From: Anthony Eid Date: Sun, 31 May 2026 20:00:22 -0400 Subject: [PATCH 3/5] Add bench macros --- crates/editor/benches/editor_render.rs | 3 +++ crates/gpui/src/gpui.rs | 24 ++++++++++++++++++++++++ crates/gpui_macros/src/bench.rs | 3 --- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/crates/editor/benches/editor_render.rs b/crates/editor/benches/editor_render.rs index cedcdd62f11cd9..973ff16c942b51 100644 --- a/crates/editor/benches/editor_render.rs +++ b/crates/editor/benches/editor_render.rs @@ -66,3 +66,6 @@ fn init_context(cx: &mut BenchAppContext) { editor::init(cx); }); } + +gpui::bench_group!(benches, editor_input_with_1000_cursors); +gpui::bench_main!(benches); diff --git a/crates/gpui/src/gpui.rs b/crates/gpui/src/gpui.rs index 50f0aecc00c032..b792718f88120b 100644 --- a/crates/gpui/src/gpui.rs +++ b/crates/gpui/src/gpui.rs @@ -96,6 +96,30 @@ pub use global::*; pub use gpui_macros::{ AppContext, IntoElement, Render, VisualContext, bench, property_test, register_action, test, }; + +/// Defines a Criterion benchmark group for benchmarks annotated with [`gpui::bench`]. +/// +/// This mirrors `criterion::criterion_group!` so GPUI benchmark files can keep the +/// same shape as ordinary Criterion benchmarks. +/// +/// [`gpui::bench`]: crate::bench +#[macro_export] +macro_rules! bench_group { + ($($tokens:tt)*) => { + criterion::criterion_group!($($tokens)*); + }; +} + +/// Defines the entry point for GPUI Criterion benchmark groups. +/// +/// This mirrors `criterion::criterion_main!` so GPUI benchmark files can keep the +/// same shape as ordinary Criterion benchmarks. +#[macro_export] +macro_rules! bench_main { + ($($tokens:tt)*) => { + criterion::criterion_main!($($tokens)*); + }; +} pub use gpui_shared_string::*; pub use gpui_util::arc_cow::ArcCow; pub use http_client; diff --git a/crates/gpui_macros/src/bench.rs b/crates/gpui_macros/src/bench.rs index c311b803110972..7d7b2ad89399a2 100644 --- a/crates/gpui_macros/src/bench.rs +++ b/crates/gpui_macros/src/bench.rs @@ -24,7 +24,6 @@ pub fn bench(args: TokenStream, function: TokenStream) -> TokenStream { let outer_fn_name = inner_fn.sig.ident.clone(); let inner_fn_name = format_ident!("__gpui_bench_{}", outer_fn_name); - let criterion_group_name = format_ident!("__gpui_bench_group_{}", outer_fn_name); inner_fn.sig.ident = inner_fn_name.clone(); TokenStream::from(quote! { @@ -38,8 +37,6 @@ pub fn bench(args: TokenStream, function: TokenStream) -> TokenStream { }); } - criterion::criterion_group!(#criterion_group_name, #outer_fn_name); - criterion::criterion_main!(#criterion_group_name); }) } From 47fe6e707ec47b8ad553f0d84b04b98ee8f6691e Mon Sep 17 00:00:00 2001 From: Anthony Eid Date: Sun, 31 May 2026 20:08:20 -0400 Subject: [PATCH 4/5] Add back removed benches --- crates/editor/benches/editor_render.rs | 99 +++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 3 deletions(-) diff --git a/crates/editor/benches/editor_render.rs b/crates/editor/benches/editor_render.rs index 973ff16c942b51..2840d782bd594f 100644 --- a/crates/editor/benches/editor_render.rs +++ b/crates/editor/benches/editor_render.rs @@ -1,10 +1,13 @@ -use criterion::Bencher; +use criterion::{Bencher, BenchmarkId}; use editor::{ Editor, EditorMode, MultiBuffer, actions::{DeleteToPreviousWordStart, SelectAll, SplitSelectionIntoLines}, }; -use gpui::{AppContext as _, BenchAppContext, Focusable as _}; +use gpui::{AppContext as _, BenchAppContext, Focusable as _, TestAppContext, TestDispatcher}; +use rand::{Rng as _, SeedableRng as _, rngs::StdRng}; use settings::SettingsStore; +use ui::IntoElement; +use util::RandomCharIter; #[gpui::bench] fn editor_input_with_1000_cursors(bencher: &mut Bencher<'_>, cx: &mut BenchAppContext) { @@ -57,6 +60,62 @@ fn editor_input_with_1000_cursors(bencher: &mut Bencher<'_>, cx: &mut BenchAppCo }); } +fn open_editor_with_one_long_line(bencher: &mut Bencher<'_>, args: &(String, TestAppContext)) { + let (text, cx) = args; + let mut cx = cx.clone(); + + bencher.iter(|| { + let buffer = cx.update(|cx| MultiBuffer::build_simple(text, cx)); + + let cx = cx.add_empty_window(); + cx.update(|window, cx| { + let editor = cx.new(|cx| { + let mut editor = Editor::new(EditorMode::full(), buffer, None, window, cx); + editor.set_style(editor::EditorStyle::default(), window, cx); + editor + }); + window.focus(&editor.focus_handle(cx), cx); + editor + }); + }); +} + +fn editor_render(bencher: &mut Bencher<'_>, cx: &TestAppContext) { + let mut cx = cx.clone(); + let buffer = cx.update(|cx| { + let mut rng = StdRng::seed_from_u64(1); + let text_len = rng.random_range(10000..90000); + if rng.random() { + let text = RandomCharIter::new(&mut rng) + .take(text_len) + .collect::(); + MultiBuffer::build_simple(&text, cx) + } else { + MultiBuffer::build_random(&mut rng, cx) + } + }); + + let cx = cx.add_empty_window(); + let editor = cx.update(|window, cx| { + let editor = cx.new(|cx| { + let mut editor = Editor::new(EditorMode::full(), buffer, None, window, cx); + editor.set_style(editor::EditorStyle::default(), window, cx); + editor + }); + window.focus(&editor.focus_handle(cx), cx); + editor + }); + + bencher.iter(|| { + cx.update(|window, cx| { + let mut view = editor.clone().into_any_element(); + let _ = view.request_layout(window, cx); + let _ = view.prepaint(window, cx); + view.paint(window, cx); + }); + }) +} + fn init_context(cx: &mut BenchAppContext) { cx.update(|cx| { let store = SettingsStore::test(cx); @@ -67,5 +126,39 @@ fn init_context(cx: &mut BenchAppContext) { }); } -gpui::bench_group!(benches, editor_input_with_1000_cursors); +fn init_test_context(cx: &TestAppContext) { + cx.update(|cx| { + let store = SettingsStore::test(cx); + cx.set_global(store); + assets::Assets.load_test_fonts(cx); + theme_settings::init(theme::LoadThemes::JustBase, cx); + editor::init(cx); + }); +} + +fn criterion_benches(criterion: &mut criterion::Criterion) { + let dispatcher = TestDispatcher::new(1); + let cx = gpui::TestAppContext::build(dispatcher, None); + init_test_context(&cx); + + let mut group = criterion.benchmark_group("Time to render"); + group.bench_with_input( + BenchmarkId::new("editor_render", "TestAppContext"), + &cx, + editor_render, + ); + group.finish(); + + let text = String::from_iter(["char"; 1000]); + let input = (text, cx.clone()); + let mut group = criterion.benchmark_group("Build buffer with one long line"); + group.bench_with_input( + BenchmarkId::new("editor_with_one_long_line", "(String, TestAppContext )"), + &input, + open_editor_with_one_long_line, + ); + group.finish(); +} + +gpui::bench_group!(benches, editor_input_with_1000_cursors, criterion_benches); gpui::bench_main!(benches); From 9b7a28c3096d0849ceaf95ebee5696031f92b8d4 Mon Sep 17 00:00:00 2001 From: Anthony Eid Date: Sun, 31 May 2026 20:31:06 -0400 Subject: [PATCH 5/5] Remove plan doc --- crates/gpui/src/app.rs | 2 +- gpui_bench_plan.md | 410 ----------------------------------------- 2 files changed, 1 insertion(+), 411 deletions(-) delete mode 100644 gpui_bench_plan.md diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rs index 9da55b333b7515..71b5b258bdaf4c 100644 --- a/crates/gpui/src/app.rs +++ b/crates/gpui/src/app.rs @@ -24,7 +24,7 @@ use slotmap::SlotMap; pub use async_context::*; #[cfg(any(test, feature = "test-support"))] -pub use bench_context::*; +pub use bench_context::{BenchAppContext, BenchWindowContext}; use collections::{FxHashMap, FxHashSet, HashMap, VecDeque}; pub use context::*; pub use entity_map::*; diff --git a/gpui_bench_plan.md b/gpui_bench_plan.md deleted file mode 100644 index fc001e277dd94e..00000000000000 --- a/gpui_bench_plan.md +++ /dev/null @@ -1,410 +0,0 @@ -# `#[gpui::bench]` + `BenchAppContext` plan - -## Motivation - -Zed has no first-class way to benchmark and regression-gate the work that actually -drops frames: synchronous main-thread cost and the fan-out a single state change -triggers. We currently reach for `criterion` + `TestAppContext`, then fall back to -`xctrace` + `dsymutil` for any real attribution. That workflow is slow, manual, and -its numbers are misleading. - -This came out of the agent `edit_file_tool` performance work. The pathological case was -applying each `CharOperation` as its own `buffer.edit` transaction, so one edit -fanned out into hundreds of `BufferEvent::Edited` events, each triggering a -tree-sitter reparse, an LSP `didChange`, the action-log diff, and the editor's -on-edit observers (matching brackets, bracket colorization, code actions, outline). -We only found it by profiling after the fact. We want to catch this class of bug in -CI, before it ships. - -### What's wrong with the current tooling - -- **`TestScheduler` is single-threaded.** Foreground and background runnables share - one queue on one thread (`crates/scheduler/src/test_scheduler.rs`, - `schedule_background_with_priority`). So a benchmark cannot separate "blocked the - main thread" from "ran off-thread." Our `large_multi_edit` number (~919 ms) mixes - both, even though the diff and LSP work run off-thread in production. -- **No per-update timing.** To see where time went we had to `dsymutil` a 145 MB - binary and parse a Time Profiler XML export by hand. -- **No cascade visibility.** Nothing reports "this edit emitted N events and fired M - observers" — the exact metric that would have caught the footgun deterministically. -- **No frame metric.** "Would this drop a frame" had to be computed offline from - miniprof spans. -- **Harness artifacts skew absolute numbers.** Maximized window lays out far more - lines than a real pane, the headless harness repaints per edit (the real app - coalesces to one paint per frame), and per-iteration setup (project/editor/LSP - construction) pollutes the samples. - -The current bench is good for **relative** before/after comparisons, but its absolute -breakdown overstates editor rendering and conflates threads. - -## Goals - -- Measure **foreground-thread blocking time** truthfully (separate from offloaded work). -- Make the **update/effect cascade observable and assertable** (events, observers, - transactions, reparses, re-renders, allocations). -- Produce a **frame-drop metric** against a budget. -- Lean on `criterion` for statistics, sampling, baselines, and reporting rather than - reimplementing them. -- Emit **profiles viewable in Tracy / Perfetto** so drill-down doesn't require - `xctrace` + `dsymutil`. -- Enable **deterministic regression gates** in CI. - -## Non-goals - -- Replacing `criterion`'s statistics engine. -- Generating standalone Instruments `.trace` files (not a writable format; see below). -- A general-purpose APM/tracing framework. This is a test/bench harness. - -## Proposed solution - -A `#[gpui::bench]` macro that provides a `BenchAppContext` and a -`BenchBencher`, layered on top of `criterion`. `BenchAppContext` owns app, -window, scheduler, recorder, and teardown state. `BenchBencher` stays close to -Criterion's `Bencher` API while adding GPUI-specific helpers for rendering -frames, measuring foreground time, and running mounted views. - -The core design is **one backend-agnostic span + counter recorder**, with the -measurement frontends (Criterion) and the trace exporters (Tracy/Perfetto) both -reading from it. Capture once, surface many ways. Criterion still owns sampling, -statistics, baselines, and reporting for a single scalar measurement at a time; -GPUI sidecar reports and traces carry the richer multi-metric data. - -## Capabilities - -### Criterion-backed measurements - -- **Wall time first.** The initial version should use Criterion's normal wall-clock - measurement so the macro feels familiar and the PoC remains small. -- **Foreground time and missed frames next.** Phase 2 should add foreground busy time - and frame-drop / missed-frame measurements, probably via `iter_custom` before a - custom `Measurement` implementation. -- **Multiple requested measurements become multiple Criterion benchmarks.** Criterion - models one scalar per benchmark result, so a single GPUI workload requesting - multiple measurements should expand to distinct benchmark IDs such as - `render_button/wall_time`, `render_button/foreground_time`, and - `render_button/missed_frames`. -- **Rich reports stay out of Criterion's scalar model.** A `BenchReport` can contain - wall time, foreground time, missed frames, spans, counters, allocation stats, and - render details, but Criterion should analyze one chosen scalar per run. -- **Later measurement backends.** Allocation bytes, longest foreground span, - renderer-blocking time, CPU cycles, instructions, cache misses, and branch misses - can be added later where platform support exists. - -### GPUI recorder - -- **Backend-agnostic spans and counters.** Record once, then feed Criterion, - Perfetto, miniprof/Tracy, and sidecar JSON/Markdown reports. -- **Per-update timing.** Reuse the miniprof hook (`TaskTiming` / - `GLOBAL_THREAD_TIMINGS`, recorded in dispatcher trampolines) and extend it to - `App::update`, entity updates, window updates, and `flush_effects`, attributed by - `#[track_caller]` call site. -- **Update/effect cascade counts.** Per outer update / `flush_effects` cycle: update - calls, entity updates, nested update depth, effects queued/flushed, events emitted, - observer/subscription callbacks fired, entities notified, global notifications, - windows invalidated, re-renders triggered, deferred callbacks, and action dispatches. -- **Notify attribution.** Track explicit `Context::notify()` separately from total - `App::notify(...)` calls. Some paths, such as `GpuiBorrow` drop, notify implicitly - without going through `Context::notify`; the report should make that distinction - visible. -- **Extensible crate-specific counters.** GPUI owns generic app/render/frame metrics. - Editor, terminal, language, project, and agent crates should add domain-specific - counters through a generic span/counter API instead of baking Zed-specific metrics - into GPUI. -- **Low overhead when disabled.** Hooks sit on hot paths, so instrumentation must be - feature-gated and effectively zero-cost outside benchmark/instrumentation builds. - -### Frame and render instrumentation - -- **Foreground-blocking / inter-frame time / frame drops.** With a real scheduler and - a modeled frame cadence, bucket foreground occupancy into frames against a budget - (16.67 ms / 8.33 ms) and report longest contiguous span, renderer-blocking time, - missed frames, and worst frame delay. -- **Realistic frame loop + window sizing.** Model normal pane/window sizes and - coalesce draws to a vsync cadence, removing maximized-window and per-edit-repaint - artifacts. -- **Render pipeline spans.** Measure layout, prepaint, paint, scene construction, and - later backend-specific renderer work. Attribute these spans to windows, entities, - and call sites where possible. -- **Render cache/reuse metrics.** Track view cache reuse, dirty subtree size, layout - cache hits/misses, text/glyph/image/SVG/path cache hits/misses, shaped text runs, - paths built/tessellated, scene command count, and unchanged subtree skips as the - render pipeline instrumentation matures. -- **First frame vs steady state.** Make cold first-frame render, warmed steady-state - render, and incremental-update render distinct benchmark modes. - -### First-class render benchmark API - -- **Mounted views.** Provide an ergonomic `cx.mount_view(...)` / `cx.render_entity(...)` - API that mounts an `Entity` where `T: Render` in a benchmark window and returns a - `MountedView`. -- **Renderer iteration helpers.** `BenchBencher` should provide helpers such as - `bench_renderer(&mut MountedView, ...)` that run user code between frames, flush - effects, render one or more frames, and record layout/prepaint/paint/frame metrics. -- **Actions and updates between frames.** Mounted views should support dispatching - actions, updating the mounted entity, resizing the window, warming caches, rendering - a single frame, and rendering every frame in a modeled frame loop. - -### Trace export and artifacts - -- **Trace export.** Emit miniprof JSON for Tracy import and Chrome/Perfetto JSON for - timeline drill-down. -- **Artifact layout.** Store GPUI artifacts next to Criterion output so local runs and - CI uploads are easy to find: trace files, sidecar `BenchReport` JSON, and a concise - Markdown summary. -- **Regression gates.** Provide count-based and frame-count assertion helpers for - deterministic CI gates; use Criterion baselines for tracked latency trends, not hard - pass/fail wall-clock gates. - -### Optional platform counters - -- **Hardware counters.** On platforms that support it, add optional measurements for - cycles, instructions, cache misses, and branch misses. Linux can use perf counters; - macOS should initially rely on Instruments / `xctrace` rather than first-class - portable hardware-counter support. - -## Criterion integration - -`criterion` only models **one scalar per iteration**, but it gives us a lot we should -not reimplement: warmup, adaptive sampling, outlier detection, summary statistics, -baseline save/compare with change detection (p-values), CLI/HTML reporting, and -profile-mode execution. - -The GPUI API should stay close to Criterion so existing Criterion users can onboard -quickly: - -```rust -#[gpui::bench(sample_size = 20, measurement = wall_time)] -fn render_button(bencher: &mut BenchBencher<'_>, cx: &mut BenchAppContext) { - let mut view = cx.mount_view(|window, cx| ButtonView::new(window, cx)); - - bencher.bench_renderer(&mut view, |view, window, cx| { - window.dispatch_action(...); - view.update(cx, |button, cx| button.click(cx)); - }); -} -``` - -### Initial version: normal Criterion wall time - -The first implementation should inject `BenchAppContext` and `BenchBencher`, then let -`BenchBencher::iter` delegate to Criterion's normal `Bencher::iter`. This keeps the -PoC simple and produces ordinary Criterion output. - -### Phase 2 seam: `iter_custom` - -`iter` / `iter_batched` time wall-clock of the whole closure, which conflates -foreground work, background work, and harness work. `iter_custom` lets the harness -return the scalar Criterion should analyze. Phase 2 can use it for foreground time -and missed-frame measurements before committing to custom `Measurement` plumbing: - -```rust -b.iter_custom(|iters| { - cx.iter_foreground_time(iters, |cx| { - run_workload(cx); - }) -}); -``` - -Criterion then runs its statistics on the right number. - -### Later: custom `Measurement` - -`criterion` is generic over a `Measurement` trait (default `WallTime`; third-party -impls exist for CPU cycles / perf counters). Implementing measurements such as -`ForegroundTime`, `MissedFrames`, or `AllocatedBytes` makes Criterion report those -units natively. A Criterion group has one `Measurement`, so multiple requested GPUI -measurements should generate multiple Criterion benchmark IDs or groups. - -A rich `BenchReport` may contain every metric from a run, but Criterion still reduces -one selected measurement to `f64` for statistical analysis. Use sidecar reports and -traces for the full multi-metric data. - -### Later: Criterion `Profiler` / `--profile-time` - -Criterion supports `--profile-time N`, which runs each benchmark workload for about -`N` seconds without normal sampling/statistical analysis so an external or in-process -profiler can collect data. Its `Profiler` trait has `start_profiling` and -`stop_profiling` hooks with the benchmark ID and output directory. A later GPUI -integration should use this to enable the recorder for profile-mode runs and write -Perfetto/miniprof/Tracy artifacts next to Criterion's output. - -This is a later-phase integration because `Profiler` hooks do not receive -`BenchAppContext`; we need a clean bridge between Criterion's profile lifecycle and -GPUI's active recorder. Early versions can use a simpler non-timed trace pass. - -### What stays out of Criterion - -- **Counts** (events, transactions, reparses, notifies, observer callbacks) → - deterministic assertion helpers / tests. These are less flaky as CI gates than - time-based measurements. -- **Frame-drop lists / per-update breakdowns / traces** → `BenchReport` sidecars and - trace artifacts, not Criterion's primary scalar result. - -### Caveats - -- A real thread pool widens Criterion's CIs and triggers more outlier warnings. - Manage with more samples; fine for tracked trends. -- **CI gating on Criterion wall-clock is flaky** on shared runners. Gate CI on the - **counts**; use Criterion's baseline/change-detection for tracked latency trends, - not hard pass/fail. - -## Trace export (Tracy / Perfetto / Instruments) - -Once the harness captures spans + counters, exporting a profile is just a serializer. -Use one in-memory model with pluggable exporters. - -- **Tracy — partly pre-built.** Zed already ships `tracy-import-miniprofiler` - (`docs/src/performance.md`) that converts `*.miniprof.json` → a Tracy capture, plus - a `tracy` feature (`ztracing/tracy`). If `BenchAppContext` emits the same miniprof - JSON schema, the existing importer and the analysis tooling work on bench output - for free. Richer Tracy use (zones/frames/plots) needs a small importer extension. -- **Chrome Trace / Perfetto — best portable default.** A plain JSON array of - `{name, ph, ts, dur, pid, tid, args}` opens directly in `ui.perfetto.dev` with no - importer and no macOS dependency. It maps cleanly onto what we care about: - - nested duration events → the update / `flush_effects` / layout / prepaint / - paint hierarchy, - - **flow events (`ph: "s"/"f"`) → the cascade chain** (edit → arrows to each - triggered effect), the unique thing Criterion can't express, - - **counter events (`ph: "C"`) → the count metrics** as area-graph tracks. -- **Criterion profile-mode artifacts — later.** A polished implementation should use - Criterion's `Profiler` hooks during `--profile-time` runs to write GPUI artifacts - into the benchmark output directory. This is not required for the initial wall-time - PoC because bridging Criterion's profile lifecycle to the active `BenchAppContext` - recorder needs additional design. -- **Instruments — qualified.** You can't synthesize a `.trace` bundle from data. - Realistic options: record the bench binary under `xcrun xctrace record` (works - today; what we did), or emit `os_signpost` intervals that show in Instruments - _during a recording_. "Generate a file to open in Instruments" is not practical. - -## The determinism principle - -Real concurrency trades away deterministic ordering, which is what makes wall-clock -perf gates flaky. Split the two: - -- **Times** (foreground-blocking, frame timings) → advisory + tracked trends via - Criterion; do not hard-gate CI on them. -- **Counts** (events, observers, transactions, reparses, allocations) → deterministic - even with a real pool; these are the actual CI gates. - -The headline value is not "faster timing"; it is **making the fan-out observable and -assertable.** - -## Phased plan - -### Phase 1 — Foundation: Criterion wall-time PoC - -Establish the harness scaffolding so every later metric is an incremental add, not a -rewrite. Keep this phase small and Criterion-like. - -- `BenchAppContext` as a benchmark-specific app context, distinct from - `TestAppContext`, owning app/window/scheduler setup and teardown outside the timed - region. -- `BenchBencher` as a Criterion-like wrapper around `criterion::Bencher`, initially - delegating to normal wall-time `iter` / `iter_batched` APIs. -- A `#[gpui::bench]` macro that expands to Criterion `bench_function` plumbing, - injects `BenchAppContext` and `BenchBencher`, and supports basic Criterion-like - options such as `sample_size` over time. -- Initial measurement: **wall time only**. -- First consumer: port a small existing benchmark to prove one command builds, runs, - and prints normal Criterion stats. - -Outcome: GPUI benchmarks look familiar to Criterion users and can run with a real -`BenchAppContext`, but no deep GPUI metrics are required yet. - -### Phase 2 — Mounted render benchmarks + foreground/frame measurements - -Make rendering a specific entity first-class and add the first GPUI-specific scalar -measurements. - -- `MountedView` for an `Entity` where `T: Render`, mounted in a benchmark - window with controlled size/theme/font setup. -- `cx.mount_view(...)` / `cx.render_entity(...)` helpers for quickly creating render - benchmarks without manually wiring windows. -- `BenchBencher::bench_renderer(&mut MountedView, ...)`, rendering one frame per - iteration after running user-provided code between frames. -- Helpers for dispatching actions, updating the mounted entity, resizing the window, - rendering a single frame, warming caches, and rendering a modeled frame loop. -- `BenchScheduler` or equivalent foreground/background separation sufficient to - compute foreground busy time. -- `foreground_time` and `missed_frames` measurements, initially via `iter_custom`. -- Basic frame model: frame cadence, longest foreground span, missed frames, and worst - frame delay. - -Outcome: agent panel, terminal, editor, and simple GPUI component render benchmarks -can be written ergonomically, and numbers begin to reflect main-thread blocking and -frame impact rather than only wall-clock time. - -### Phase 3 — Cascade instrumentation + regression gates (highest value) - -Add deterministic fan-out visibility and assertion helpers. - -- Per-outer-update / `flush_effects` counters: update calls, entity updates, nested - depth, effects queued/flushed, events emitted, observer/subscription callbacks, - explicit `Context::notify()` calls, total `App::notify(...)` calls, implicit notify - paths such as `GpuiBorrow` drop, window invalidations, and re-renders. -- Generic span/counter API for domain-specific metrics from editor, terminal, - language, project, and agent crates. -- Allocation counting. -- Count-based `assert!` helpers and the first regression tests (e.g. "an N-op edit - emits one `Edited` per chunk, not per op"). -- Surface counters as Perfetto counter tracks / Tracy plots. - -Outcome: the fan-out footgun class is caught deterministically in CI. - -### Phase 4 — Render pipeline instrumentation - -Add the detail needed to improve GPUI's renderer itself. - -- Layout, prepaint, paint, scene construction, and render-backend spans. -- Entity/window/callsite attribution for render pipeline spans. -- View cache reuse, dirty subtree size, layout cache hits/misses, text/glyph/image / - SVG/path cache hits/misses, shaped text runs, paths built/tessellated, scene command - count, and unchanged subtree skips. -- Distinct cold first-frame, warmed steady-state, and incremental-update render modes. - -Outcome: GPUI render pipeline changes can be benchmarked directly and attributed to -specific phases and cache behavior. - -### Phase 5 — Trace/export polish and advanced measurements - -- Trace exporters: **miniprof JSON** (reuse `tracy-import-miniprofiler`) and - **Chrome/Perfetto JSON** with spans, flows, and counters. -- Criterion `Profiler` integration for seamless `--profile-time` trace artifacts. -- Macro ergonomics and Criterion-like parameters: `sample_size`, `warm_up_time`, - `measurement_time`, `app = fresh_per_sample`, `drain = after_iteration`, window - size, frame budget, pool size, and seed. -- Optional custom Criterion `Measurement` implementations for foreground time, - missed frames, allocation bytes, renderer-blocking time, and platform hardware - counters. -- CI integration: count gates as required checks; Criterion baselines tracked for - trends. -- Richer Tracy export (zones/frames/plots) if the importer extension is worth it. - -## Open questions / risks - -- How much determinism to keep with a real pool. A hybrid (real background pool + - deterministic foreground driver, optional seed) may be the sweet spot. -- Instrumentation must be zero-cost when the bench/instrumentation feature is off, - since the hooks sit on gpui hot paths. -- CI noise budget for any time-based signal; lean on counts for gating. -- Whether to extend `tracy-import-miniprofiler` for zones/plots or just rely on - Perfetto for the rich view. -- How to bridge Criterion `Profiler` lifecycle hooks to the active `BenchAppContext` - recorder without global state that makes parallel or multi-app benchmarks fragile. - -## Reference points in the existing codebase - -- Per-poll timing hook: `crates/gpui_macos/src/dispatcher.rs` (`trampoline`, - `TaskTiming`, `add_task_timing`, `GLOBAL_THREAD_TIMINGS`). -- Spawn-location attribution: `crates/scheduler/src/scheduler.rs` - (`RunnableMeta { location }`). -- Single-threaded test scheduling to replace for benches: - `crates/scheduler/src/test_scheduler.rs`. -- Effect system to instrument for cascade counts: `gpui` `App::flush_effects` / - `pending_effects` / the `Effect` enum. -- Existing miniprof + Tracy import path: `crates/miniprofiler_ui/` and - `docs/src/performance.md`. -- Current PoC bench: `crates/editor/benches/editor_render.rs`. -- Near-term consumers: agent panel render benchmarks, terminal render benchmarks, - and `crates/agent/benches/edit_file_tool.rs` (already `harness = false`, - `criterion`, `--profile-time` compatible).