From b3f8d7dd04fdcc2534bd95864a0e4e3bab8b3f40 Mon Sep 17 00:00:00 2001 From: Valentin Palkovic Date: Fri, 31 Jul 2026 22:04:37 +0200 Subject: [PATCH] refactor(docgen-harness): simplify performance measurement --- AGENTS.md | 6 + code/lib/docgen-harness/README.md | 33 +- code/lib/docgen-harness/package.json | 5 +- .../src/perf/PERF-METHODOLOGY.md | 266 +++++--- .../docgen-latency/resettable-bench.test.ts | 250 +++++++ .../perf/docgen-latency/resettable-bench.ts | 241 +++++++ .../perf/docgen-latency/vue-component-meta.ts | 194 ++++++ .../src/perf/docgen-memory/memory-harness.ts | 24 +- .../src/perf/docgen-perf/aggregate.test.ts | 122 ---- .../src/perf/docgen-perf/aggregate.ts | 95 --- .../src/perf/docgen-perf/cli.test.ts | 58 ++ .../src/perf/docgen-perf/cli.ts | 51 +- .../src/perf/docgen-perf/comparison.test.ts | 388 +++++++++++ .../src/perf/docgen-perf/comparison.ts | 617 ++++++++++++++++++ .../src/perf/docgen-perf/config.ts | 33 +- .../src/perf/docgen-perf/engine.ts | 108 +-- .../src/perf/docgen-perf/engines/compodoc.ts | 140 +--- .../perf/docgen-perf/engines/react-legacy.ts | 11 +- .../docgen-perf/engines/vue-component-meta.ts | 67 +- .../docgen-perf/engines/vue-docgen-api.ts | 9 +- .../perf/docgen-perf/engines/vue-scenario.ts | 4 +- .../src/perf/docgen-perf/ratios.test.ts | 198 ------ .../src/perf/docgen-perf/ratios.ts | 153 ----- .../src/perf/docgen-perf/registry.test.ts | 79 ++- .../src/perf/docgen-perf/registry.ts | 20 +- .../src/perf/docgen-perf/report.test.ts | 385 +++-------- .../src/perf/docgen-perf/report.ts | 181 ++--- .../src/perf/docgen-perf/run.ts | 489 +++++++++++--- .../src/perf/docgen-perf/spawn.test.ts | 102 +++ .../src/perf/docgen-perf/spawn.ts | 88 ++- .../src/perf/docgen-perf/summary.test.ts | 72 ++ .../src/perf/docgen-perf/summary.ts | 98 +++ .../src/perf/docgen-perf/types.ts | 169 ++--- .../perf/docgen-shared/latency-series.test.ts | 97 +++ .../src/perf/docgen-shared/latency-series.ts | 103 +++ .../src/perf/docgen-shared/samples.ts | 2 + .../src/perf/docgen-shared/series.ts | 55 +- scripts/knip.config.ts | 1 + yarn.lock | 8 + 39 files changed, 3396 insertions(+), 1626 deletions(-) create mode 100644 code/lib/docgen-harness/src/perf/docgen-latency/resettable-bench.test.ts create mode 100644 code/lib/docgen-harness/src/perf/docgen-latency/resettable-bench.ts create mode 100644 code/lib/docgen-harness/src/perf/docgen-latency/vue-component-meta.ts delete mode 100644 code/lib/docgen-harness/src/perf/docgen-perf/aggregate.test.ts delete mode 100644 code/lib/docgen-harness/src/perf/docgen-perf/aggregate.ts create mode 100644 code/lib/docgen-harness/src/perf/docgen-perf/cli.test.ts create mode 100644 code/lib/docgen-harness/src/perf/docgen-perf/comparison.test.ts create mode 100644 code/lib/docgen-harness/src/perf/docgen-perf/comparison.ts delete mode 100644 code/lib/docgen-harness/src/perf/docgen-perf/ratios.test.ts delete mode 100644 code/lib/docgen-harness/src/perf/docgen-perf/ratios.ts create mode 100644 code/lib/docgen-harness/src/perf/docgen-perf/spawn.test.ts create mode 100644 code/lib/docgen-harness/src/perf/docgen-perf/summary.test.ts create mode 100644 code/lib/docgen-harness/src/perf/docgen-perf/summary.ts create mode 100644 code/lib/docgen-harness/src/perf/docgen-shared/latency-series.test.ts create mode 100644 code/lib/docgen-harness/src/perf/docgen-shared/latency-series.ts diff --git a/AGENTS.md b/AGENTS.md index 38bc2d1c6ca8..a737e7edaf20 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -161,8 +161,14 @@ yarn storybook:vitest | Run sandbox E2E tests | `yarn task e2e-tests-dev --template react-vite/default-ts --start-from auto` | | Run sandbox test-runner tests | `yarn task test-runner-dev --template react-vite/default-ts --start-from auto` | | Run the docgen perf bench | `yarn workspace @storybook/docgen-harness bench:docgen-perf` | +| Run the resettable docgen bench | `yarn workspace @storybook/docgen-harness bench:docgen-latency` | | Run the docgen memory gate | `yarn workspace @storybook/docgen-harness bench:docgen-memory` | +The docgen harness keeps latency and memory as separate measurement lanes. Latency runs must not +force GC; `bench:docgen-perf` stores fresh-process raw trajectories and only gates with an explicit +seeded `--compare` budget, while `bench:docgen-latency` uses Tinybench only for its resettable Vue +version workload. Preserve the existing post-GC/RSS/OOM semantics in `bench:docgen-memory`. + ## NX and `yarn task` Use NX when you want better caching and dependency tracking. Prefer these faster defaults first, and only add `-c production` or `--no-link` when you specifically need sandbox parity or CI-like behavior. diff --git a/code/lib/docgen-harness/README.md b/code/lib/docgen-harness/README.md index ec6dfa2ef606..cd4a74373739 100644 --- a/code/lib/docgen-harness/README.md +++ b/code/lib/docgen-harness/README.md @@ -53,9 +53,10 @@ src/ ├── web-components/ # planned └── perf/ # the performance bench, see below ├── PERF-METHODOLOGY.md # the measurement contract - ├── docgen-perf/ # per-engine latency and memory suite, plus its engines/ and generators/ + ├── docgen-perf/ # fresh-process macro latency suite, engines, and generators + ├── docgen-latency/ # resettable Vue current/next Tinybench workload ├── docgen-memory/ # the docgen-server memory regression gate - └── docgen-shared/ # sampling, stats, budgets and paths shared by both + └── docgen-shared/ # latency/memory loops, stats, budgets, and shared paths ``` ## The comparator @@ -175,21 +176,39 @@ Each has a red marker in `vue3-legacy-gaps.test.ts`. `src/perf/` measures how fast the docgen engines are and how much memory they hold, which is the other half of the "docgen beyond React" question the snapshot comparator above answers for correctness. It is a set of CLIs rather than part of this package's exported API, so nothing in `src/perf/` is re-exported from `src/index.ts`. -Both commands run from `code/lib/docgen-harness`: +All commands can run from `code/lib/docgen-harness`: ```bash -yarn bench:docgen-perf # per-engine cold/warm latency and memory, full profile -yarn bench:docgen-perf --quick # smoke profile; its numbers are marked non-comparable +yarn bench:docgen-perf # descriptive fresh-process cold/warm latency suite +yarn bench:docgen-perf --quick # smaller smoke workload; never a timing gate +yarn bench:docgen-latency # resettable Vue current/next re-extraction benchmark +yarn bench:docgen-latency --quick # exact three-iteration smoke workload per pin yarn bench:docgen-memory # the docgen-server memory regression gate CI runs daily ``` -`bench:docgen-perf` generates synthetic projects under the shared sandbox directory, runs each engine in its own child process, and writes a results JSON next to them. +`bench:docgen-perf` generates synthetic projects under the shared sandbox directory, runs every repetition in a fresh child process without forced GC, and writes a Storybook-owned results JSON next to them. +The artifact keeps every cold observation and every ordered warm save; its headline warm value is the median of each process's trajectory median. +Comparisons are descriptive by default: cold and warm work are independently reported as `same-work`, `different-work`, `unknown-work`, or `same-version`, and no timing effect is calculated unless the sides prove equivalent for that metric. + +An opt-in timing gate runs a single configured pair in adjacent, seeded, balanced blocks: + +```bash +yarn bench:docgen-perf --compare vue-component-meta-version \ + --seed 42 --repetitions 10 --max-regression 0.10 +``` + +`--max-regression` is a fractional candidate slowdown and is mandatory with `--compare`. +The gate uses paired candidate/control log ratios and a 95% Student-t interval; only an interval wholly beyond the limit fails, while overlap is inconclusive. + +`bench:docgen-latency` is the narrow resettable case where Tinybench fits: each pin gets exactly K fresh engines, cold extraction primes the engine before the timer, save 1 is applied outside the timer, and only synchronous re-extraction is timed. +Its JSON maps the retained samples into this harness's own schema instead of exposing Tinybench's result objects. `bench:docgen-memory` asserts both that re-extraction is leak-free and that the program-recycle fix still flips a tight-heap run from OOM to survival. +It remains the daily performance gate and retains its post-GC, RSS, budget, and OOM semantics. Read `src/perf/PERF-METHODOLOGY.md` before changing a metric, a budget, or a version pair. It is the contract the numbers are only meaningful under. -The bench also carries unit tests for its own aggregation, reporting and generator logic, so `yarn test code/lib/docgen-harness` runs those alongside the fixture comparisons. +The bench also carries unit tests for its scheduling, work checks, statistics, result mapping, reporting, and generator logic, so `yarn test code/lib/docgen-harness` runs those alongside the fixture comparisons. ## What does not live here diff --git a/code/lib/docgen-harness/package.json b/code/lib/docgen-harness/package.json index 8d9384cc3ef4..719febfed87e 100644 --- a/code/lib/docgen-harness/package.json +++ b/code/lib/docgen-harness/package.json @@ -26,8 +26,10 @@ "./package.json": "./package.json" }, "scripts": { + "bench:docgen-latency": "node ./src/perf/docgen-latency/vue-component-meta.ts", "bench:docgen-memory": "node --import jiti/register ./src/perf/docgen-memory/gate.ts", - "bench:docgen-perf": "node ./src/perf/docgen-perf/run.ts" + "bench:docgen-perf": "node ./src/perf/docgen-perf/run.ts", + "check": "vue-tsc --noEmit" }, "devDependencies": { "@angular/common": "^21.2.14", @@ -47,6 +49,7 @@ "react-dom": "^18.2.0", "rxjs": "^7.4.0", "storybook": "workspace:*", + "tinybench": "6.1.2", "typescript": "^6.0.3", "vue": "^3.2.47", "vue-component-meta": "^3.3.9", diff --git a/code/lib/docgen-harness/src/perf/PERF-METHODOLOGY.md b/code/lib/docgen-harness/src/perf/PERF-METHODOLOGY.md index 045bb91087bc..fab4675333da 100644 --- a/code/lib/docgen-harness/src/perf/PERF-METHODOLOGY.md +++ b/code/lib/docgen-harness/src/perf/PERF-METHODOLOGY.md @@ -1,166 +1,218 @@ # Docgen performance methodology -This document sets out how the per-engine docgen performance suite measures things. It elaborates on the metrics we record, the rules that make a run repeatable, the shape a budget may take, and the CI tier a budget is intended to gate once baselines exist. +This document is the measurement contract for the docgen harness. The harness has three separate +lanes because correctness, latency, and memory need different execution rules. Do not reuse a +memory sample as a latency result or turn a descriptive latency run into a gate implicitly. -## Scope +## Scope and lanes -The performance tests run in plain Node.js, and we deliberately decided to not run them in the browser or start a Storybook dev server. We also did not involve the builders to be able to have metrics in place without huge variances between runs. This means, as a side effect, though, that some docgen engines which are currently part of builder plugins have to be called in a way where the extraction logic is called directly. This is so that we measure the docgen engines' performance rather than the performance of Storybooks' whole end-to-end docgen delivery pipeline. +The harness calls docgen engines directly in Node.js against deterministic synthetic projects. It +does not start Storybook, a browser, or a builder, so the results describe extraction rather than +the full product pipeline. -## Metrics +- **Correctness:** the framework baseline tests compare extracted argTypes and snippets. They are + the authority on whether an implementation documents at least the committed behavior. +- **Macro latency:** `bench:docgen-perf` measures cold extraction and stateful save trajectories in + fresh child processes. It is descriptive unless a comparison and budget are explicitly supplied. +- **Resettable latency:** `bench:docgen-latency` uses Tinybench for the narrow Vue current/next + operation that can create, prime, mutate, measure, and dispose a fresh engine every iteration. +- **Memory:** `bench:docgen-memory` retains the daily post-GC/RSS/OOM regression gate. Forced GC is + useful here and forbidden in both latency lanes. -We are collecting five metrics per engine: +Mitata and Vitest benchmark mode are intentionally not used. The macro suite needs process +isolation, ordered stateful saves, package-version provenance, and nonfatal tool preflights rather +than a tight in-process microbenchmark loop. Tinybench is used only where the measured operation is +fully resettable. -1. Cold extraction: that's the time a fresh process takes over its full extraction, including starting the typescript program where an engine needs one. -2. Warm extraction: the time to re-extract one changed component after a simulated save once the process is warm. In theory, an already booted typescript program's delay shouldn't be visible in the warm extraction anymore, and therefore warm extractions are faster than cold ones. -3. Whole project scan: that's the time for one batch pass over the entire project. Currently, only Compodoc works this way, so the metric applies to Compodoc for Angular alone. -4. Peak memory: the memory a save claims above the retained baseline -5. Leak detection: How much retained heap is still held after the save series, together with how steeply it climbed from one save to the next? +## Deterministic workloads -## Determinism method +The generators fix component counts, prop counts, type-import chains, and other scenario levers. +Scenario parameters are written into the result artifact. A save mutates a known component and is +applied before re-extraction; save order is part of the workload and must not be shuffled or treated +as independent samples. -### Fixed synthetic projects +Package versions are resolved from the installed package metadata and stored with results. A +current/next version comparison is invalid when both aliases resolve to the same version. -We are using a generator to generate projects for the different frameworks with a fixed set of component count props per components and also a different set of levers to, for example, cover type imports and the ability of an engine to follow these imports. This is, for example, the case if a TypeScript program is used to infer a type, whereas for Compodoc, which uses static extraction, hits a limit. +## Fresh-process macro latency -### Warmup +Run from the repository root: -Every run goes through one full cold execution pass before a warm extraction is measured. - -### Median-of-N for latency +```bash +yarn workspace @storybook/docgen-harness bench:docgen-perf +yarn workspace @storybook/docgen-harness bench:docgen-perf --quick +``` -Every latency metric records a median for cold extractions. We need to make sure that these are triggered in fresh processes, so there are n samples that come from n separate spawns, while warm extracts takes the median of the per-save durations inside one run. +The default selection and repeated `--engine` selection remain supported. Each engine/scenario +repetition runs in a fresh child process. Source children use their existing execution requirement: +native Node.js for the parser children and the jiti import only for the React OSA source child. +Missing optional tools or packages are reported as nonfatal skips; an engine that starts measuring +but does not produce every requested repetition fails. -### Fresh process per measurement +That preflight rule also applies to an explicit comparison: if either side is unavailable, neither +side runs and no gate verdict is produced. CI jobs that require a timing gate must install both +declared sides; the harness does not turn an unavailable optional tool into a performance failure. -Every measure process is spawned fresh so that it starts from a clean heap. +One raw repetition contains: -### Relative comparison on the same machine +- one timed cold extraction; +- every timed warm re-extraction in chronological save order; and +- a whole-project scan observation for engines such as Compodoc that naturally report one. -Performance questions are answered by ratios between runs executed on one machine, either on CI or on a local machine. We are not comparing across jobs, runs, or PRs. Two comparisons stand today: +Project generation, applying a save, and disposal are outside the timed region. No latency process +uses `--expose-gc` or asks V8 to collect. Compodoc is normalized to the same raw shape by running +fresh one-shot child processes rather than polling process RSS in the latency suite. -- The docgen server flag: on vs off -- A new engine against its legacy counterpart - Both are measured inside the same run. Paired runs also alternate their order across repetitions so that neither a warm cache nor a machine that has heated up can consistently favor one side. +The JSON artifact is owned and versioned by Storybook (`schemaVersion: 1`). It keeps all raw +repetitions and ordered warm observations. Summaries are conveniences, not replacements for raw +data: -The pinned N has to be even for that alternation to cancel anything. -Each side leads on half the repetitions, so an odd N gives one of them the lead once more than the other, and the cold figure is a median, which then lands on a repetition from whichever side led more often. -That would turn the very effect the alternation exists to remove into a systematic bias on the ratio, so the parity is asserted in `ratios.test.ts` rather than left to whoever next edits the constant. +- cold and scan summaries use one observation from each fresh process; +- each process contributes the median of its own warm trajectory to the warm headline; and +- the headline is the median of those process-level values. -### Comparing an engine against another release of itself +This avoids pretending saves from one evolving program are independent observations. Results are +written to the existing `docgen-perf/results.json` sandbox location unless `--json` overrides it. +`--quick` uses a smaller smoke workload and fewer descriptive repetitions; its numbers never form a +timing verdict. -There is a third comparison that follows the same two rules: two installs of one engine, measured against each other rather than against a different engine. This is how we check a version bump, because storing last week's milliseconds and diffing against them today would break the same-machine rule. Both sides run inside the same invocation, in the same alternating order, through the same control pair machinery, so nothing new was needed to compare them. +The top-level `gating` field is true only for a full paired comparison. It remains false for +descriptive runs and paired `--quick` smoke runs, even though the latter still records its method, +seed, blocks, and configured limit. -Like-for-like still applies here, and it earns its keep. A newer version that legitimately documents more members costs more, and we want that to show up as a member count mismatch rather than as a regression. +## Proving equal work -One failure is specific to this shape. -The two aliases are always separate installs, but they can still resolve to the same published version, and a single caret range is enough for that to happen. -Then we compare a version against itself and get a ratio of roughly one, which reads exactly like a clean result. -So both resolved versions are printed beside every ratio, and two equal ones are called out as not being a comparison at all. +A faster engine may simply have documented less. Each repetition therefore records documented +member counts for cold extraction and every ordered warm save. Engines that can distinguish +unresolved types also record an opaque-type count. -A bump proposed in a pull request is fully covered by this. Catching a regression on the day it ships is not, because nothing here fetches the newest published version on its own, so moving the candidate forward still waits on a person or on a scheduled job that does not exist yet. +Before comparing timing, the suite constructs stable cold and warm work profiles across all fresh +processes and reports one of these statuses for each metric: -#### Running a version comparison +- `same-work`: cold and every ordered warm signature agree; +- `different-work`: a count, save count, or ordered warm signature differs; +- `unknown-work`: a required count is missing, varies across repetitions, or only one side reports + an opaque-type count; or +- `same-version`: a version pair resolved identical package versions. -The pair that exists today is `vue-component-meta` against `vue-component-meta-next`, which are two installs of one package: the second is an alias in `code/lib/docgen-harness/package.json` pinned to an exact version. +When neither engine reports opaque-type counts, matching documented-member counts can still prove +equal work. When both report opaque-type counts, those counts must match. A timing effect is omitted +unless that metric's result is `same-work` and any required version distinction is valid. A cold +mismatch does not suppress an otherwise valid warm comparison, or vice versa. Descriptive runs +report the statuses but deliberately carry `not-configured` instead of calculating an effect. -**1. Point the alias at the version you want to test.** -Edit `code/lib/docgen-harness/package.json` and install, so the candidate is on disk: +## Explicit paired gate -```jsonc -"vue-component-meta": "^3.2.7", // the current side -"vue-component-meta-next": "npm:vue-component-meta@3.3.8" // the candidate, pinned exactly -``` +A timing gate is opt-in and runs exactly one named pair: ```bash -yarn install +yarn workspace @storybook/docgen-harness bench:docgen-perf \ + --compare vue-component-meta-version \ + --seed 42 --repetitions 10 --max-regression 0.10 ``` -Pin the candidate exactly rather than with a range. -A range on both sides can resolve to one install, and then the run compares an engine against itself. - -**2. Run both sides in one invocation, naming each explicitly.** -From `code/lib/docgen-harness/`: +Available pair names live in `docgen-perf/comparison.ts`. `--max-regression` is a required +fractional candidate slowdown; `0.10` permits at most 10%. Repetitions are paired blocks, must be +even, and must be at least 10. Omitting `--seed` creates and prints a random 32-bit seed, which is +stored in the artifact so the schedule can be replayed. -```bash -yarn bench:docgen-perf --engine vue-component-meta --engine vue-component-meta-next -``` +The Vue current/next pair is the initial gateable workload. The React pair currently has unknown +work counts, and the cross-engine Vue pair currently performs different work; selecting either as +a full gate therefore produces `invalid-gate` until its adapters can prove equivalence. -Both ids are required. -`vue-component-meta-next` is out of the default run because it carries no budget row, and a control pair only produces a ratio when both of its sides measured in the same invocation - naming one gives you a table row and no comparison. -Add `--quick` for a smoke run that proves the wiring; its numbers are marked non-comparable and must not be read as a result. +For each scenario, a stable derived seed builds two-block strata. Each stratum contains one block +that runs control then candidate and one that runs candidate then control; the seed decides which +comes first. The two sides of every block remain adjacent. Only a complete two-sided block is +published. -**3. Read the three guards on the ratio lines before reading the ratio.** -A real run prints one cold and one warm line per scenario: +For block `i`, the effect sample is: ```text -ratio cold legacy/new (vue-component-meta-version/flat): 1.04 [documented members 70 vs 70] [3.3.2 vs 3.3.8] -ratio warm legacy/new (vue-component-meta-version/flat): 1.01 [documented members 15 vs 15] [3.3.2 vs 3.3.8] +d_i = log(candidate_i / control_i) ``` -- **Two different versions.** - `[3.3.2 vs 3.3.8]` is what says two different versions were actually compared. - `[both sides resolved 3.3.8 - NOT a comparison]` means the current side's range drifted onto the pin, and the roughly-1.00 ratio beside it means nothing. - Pin the current side too, or move the candidate. -- **Equal work.** - A bare `[documented members 70 vs 70]` with no note after it is like-for-like. - A `NOT like-for-like` note means the two sides did not do equal work, and the note says which way: `documented more` / `documented less` is a difference in member counts, while `same members, but ...` is the subtler one, where the counts agree and the two versions resolved different amounts of the type graph behind them. - Either way the ratio is measuring a behaviour change rather than a cost change, and that behaviour change is the finding. -- **Above 1.00 is the candidate winning.** - The ratio is the current side's median over the candidate's, so `1.04` means the candidate was 4% faster on that scenario. - A number below 1.00 is the candidate costing more. +Cold uses the block's cold observations. Warm uses one trajectory median per side and block. The +reported candidate/control estimate is `exp(mean(d))`; its two-sided 95% interval is +`exp(mean(d) +/- t_0.975,n-1 * standardError(d))`. + +The 95% level describes each scenario/metric interval, not a family-wise confidence level for the +whole command. A future CI policy that treats many cells as one statistical decision must declare a +primary cell or add an explicit multiple-comparison correction rather than relabel these intervals. -An engine whose package does not resolve is reported as skipped with a reason rather than measured, so a forgotten `yarn install` cannot quietly turn into a missing comparison. +Let `L = 1 + maxRegression`: -#### Adding a version pair for another engine +- lower interval bound greater than `L`: `regression` and a failing exit status; +- upper interval bound at or below `L`: `within-budget`; and +- interval overlapping `L`: `inconclusive`, not a failure. -This works only for an engine whose child imports the versioned package directly, the way `engines/vue-component-meta.ts` does. -Where the harness reaches the engine through repo source instead - `react-legacy` goes via `loadReactRendererModule` into `code/renderers/react`, which imports `react-docgen` by bare specifier - no child flag can redirect that import, and a version pair needs a different approach entirely. -Declaring `@storybook/react` as a dependency of `@storybook/docgen-harness` is what makes that source reachable in the first place, but it does not make the specifier redirectable: pointing the renderer at a second `react-docgen` would take a module resolution hook registered in the child, which is not built. +Invalid configuration, incomplete data, unequal/unknown work, or a same-version pair produces an +invalid gate and fails without publishing a timing effect. A paired `--quick` run remains a smoke +result even though the CLI still requires an explicit budget. -Four data edits: +## Resettable Tinybench lane -1. An alias in `code/lib/docgen-harness/package.json` pinned to the candidate version. -2. The new id added to the `EngineId` union in `docgen-shared/engine-ids.ts`, which is hand-maintained: a registry entry naming an id that is not in the union does not compile. -3. A second registry entry in `docgen-perf/registry.ts` reusing the same child, with `inDefaultRun: false`, `versionPackage` naming the alias, and the flag that tells the child which install to load. -4. An entry in `CONTROL_PAIRS` in `docgen-perf/ratios.ts` naming the current side as `legacy` and the alias as `next`. +Run the isolated Vue version benchmark with: -Plus one code edit, unless the child already has it: the child needs a flag that selects the install, which is what `--pin` is on the vue-component-meta child. -That took a small union of allowed values, an extension of the shared option schema, a per-pin scratch directory so the two runs do not share a generated project, and a conditional import of the aliased package. +```bash +yarn workspace @storybook/docgen-harness bench:docgen-latency +yarn workspace @storybook/docgen-harness bench:docgen-latency --quick +yarn workspace @storybook/docgen-harness bench:docgen-latency --iterations 20 +``` -One easily-missed prerequisite: the *current* side's existing registry entry must declare `versionPackage` too. -`versionNote` prints nothing unless both sides resolved a version, so without it every ratio line silently loses its version note - and with it the guard against both sides resolving to the same version. +The full workload uses 15 iterations per pin and quick uses exactly 3; custom runs require at least +2 so their uncertainty fields are not derived from one observation. For every iteration and pin, +the harness: -Nothing else changes: aggregation, member counts and the ordering alternation are already shared by every pair. -A pair that shares an engine with another pair is fine, because the alternation reverses the whole engine list and so flips every pair at once. +1. constructs a fresh synchronous `vue-component-meta` engine; +2. performs cold extraction to prime it; +3. applies save 1 outside the timer; +4. times only synchronous re-extraction; and +5. verifies the work signature and disposes the engine. -## Budget shape +Tinybench is configured with `iterations: K`, `time: 0`, no warmup, retained samples, no task +concurrency, and throwing errors. Tasks explicitly set `async: false` so registration cannot invoke +the measured function while probing whether it returns a promise. The command checks exactly K +engines and K retained samples per pin, equivalent work within and across pins, complete task +results, and distinct resolved versions. -Timing budgets are ratios or slopes rather than absolute milliseconds because absolute wall clock on a shared CI executor is far too noisy to gate on. A timing ratio divides the median of one side by the median of another. An engine that has a second implementation to compare against (for example, `vue-docgen-api` against `vue-component-meta`) uses that pair as its reference. An engine without one has its reference picked when its baselines are recorded. +The output is descriptive and not a timing gate. A narrow mapper copies only Storybook's latency +summary fields and retained samples into a versioned JSON artifact; Tinybench-native task objects +are never the persistent contract. The default artifact lives at `docgen-latency/results.json` in +the shared sandbox directory. Tinybench executes the two tasks sequentially in the recorded order, +so these distributions are shadow data; the harness deliberately does not turn them into a +current/candidate ratio or confidence claim between pins. -### Like-for-like comparison +## Daily memory gate -A ratio only means something when both sides did the same amount of work. Engines resolve types to different depths, and a shallower one finishes sooner precisely because it documented less, which makes speed and thoroughness very easy to mistake for one another. +Run: -Therefore, every engine reports how many members it documented, and the suite prints those counts beside every ratio, warm as well as cold. Where the two sides disagree, we do not just flag the ratio, we say which way it went, because the direction is what tells you how to read the number. An engine that documented less is fast for the wrong reason and its ratio is worthless. An engine that documented more and still won has a ratio that undersells it. Only a pair that did equal work may ever become a budget. +```bash +yarn workspace @storybook/docgen-harness bench:docgen-memory +``` -Cold and warm get their own verdict, since a pair can document the same members on the cold pass and different ones on the save it was timed on. When either side reports no count at all we record that as unknown rather than treating it as agreement, because marking a pair equal on the strength of a number nobody measured is exactly how a bad ratio would slip through. +The memory lane intentionally samples after forced GC to detect retained heap growth and separately +tracks transient/RSS pressure. Its fresh-process gate preserves two protections: -Even the member count is not enough on its own. An engine that records a type's name without ever looking through it documents exactly as many members as one that expanded the whole chain, and it does so at a fraction of the cost. So an engine that works that way also reports how many of its documented members carry a type it never resolved, and that second count has to agree as well before the two sides count as having done equal work. +- changed-scope steady state stays below the existing retained-growth, transient-memory, and + retained-slope budgets; and +- under the fixed heap cap, recycling enabled must survive while the otherwise identical recycling + disabled negative control must fail with a V8 heap-OOM signature. -What we do not have yet is the gate. Once baselines exist, a ratio that moves the wrong way should fail CI, so that a human intercepts and decides what to do next. +Memory samples remain ordered by save. Do not alter the budgets, heap cap, RSS behavior, or OOM +positive/negative controls as part of latency work. -### Memory budgets +## Changing the harness -Memory budgets stay in absolute megabytes, with enough headroom on CI so that the gate is not flaky while still failing hard on real regressions. +When adding an engine or version pair: -## Budgets table skeleton +1. add its id and registry adapter, preserving the child's native or jiti requirement; +2. record its package version when an installed package provides the implementation; +3. emit documented-member counts for cold and every warm save, plus opaque-type counts where the + engine can compute them faithfully; +4. add the comparison pair only when both sides share identical scenarios and parameters; and +5. test scheduling, raw-result completeness, work mismatch/version collision, effect direction, + interval boundaries, and reporting. -| Engine | Cold extraction | Warm extraction | Whole-project scan | Peak memory (transient) | Retained growth | Retained slope | Tier | -| ----------------------------------------- | --------------- | --------------- | ------------------ | ----------------------- | --------------- | -------------- | ----- | -| react-legacy (react-docgen, control) | TBD | TBD | n/a | TBD | TBD | TBD | TBD | -| react-osa (ComponentMetaManager, control) | TBD | TBD | n/a | 90MB | 60MB | 3MB/save | daily | -| vue-docgen-api (legacy, current default) | TBD | TBD | n/a | TBD | TBD | TBD | TBD | -| vue-component-meta | TBD | TBD | n/a | TBD | TBD | TBD | TBD | -| compodoc | TBD | TBD | TBD | TBD | TBD | TBD | TBD | -| svelte (stretch) | TBD | TBD | n/a | TBD | TBD | TBD | TBD | -| cem (stretch) | TBD | TBD | n/a | TBD | TBD | TBD | TBD | +Never gate stored milliseconds from another machine or historical run. Timing budgets apply only +to explicit paired observations from the same invocation. Keep correctness changes in the baseline +lane and memory changes in the memory lane so each signal retains its intended meaning. diff --git a/code/lib/docgen-harness/src/perf/docgen-latency/resettable-bench.test.ts b/code/lib/docgen-harness/src/perf/docgen-latency/resettable-bench.test.ts new file mode 100644 index 000000000000..19cb4ede353d --- /dev/null +++ b/code/lib/docgen-harness/src/perf/docgen-latency/resettable-bench.test.ts @@ -0,0 +1,250 @@ +import { describe, expect, it } from 'vitest'; + +import { createFreshEngineTask, createResettableBench, mapTaskResult } from './resettable-bench.ts'; +import { packageVersion, parseOptions, runVueComponentMetaLatency } from './vue-component-meta.ts'; + +function createCompletedResult() { + const bench = createResettableBench( + [{ name: 'fake', beforeEach() {}, measure() {}, afterEach() {} }], + 2 + ); + bench.runSync(); + const result = bench.results[0]; + if (result.state !== 'completed') { + throw new Error(`expected a completed test result, received ${result.state}`); + } + return result; +} + +describe('package provenance', () => { + it('reads metadata even when the package does not export package.json', () => { + expect(packageVersion('tinybench')).toBe('6.1.2'); + }); +}); + +describe('createResettableBench', () => { + it('does not call the measured function during registration and runs exactly K iterations', () => { + const events: string[] = []; + let measurements = 0; + const bench = createResettableBench( + [ + { + name: 'fake', + beforeEach: () => events.push('beforeEach'), + measure: () => { + measurements += 1; + events.push('measure'); + }, + afterEach: () => events.push('afterEach'), + }, + ], + 3 + ); + + expect(measurements).toBe(0); + bench.runSync(); + expect(measurements).toBe(3); + expect(events).toEqual([ + 'beforeEach', + 'measure', + 'afterEach', + 'beforeEach', + 'measure', + 'afterEach', + 'beforeEach', + 'measure', + 'afterEach', + ]); + }); + + it('rejects a one-sample iteration count', () => { + expect(() => createResettableBench([], 1)).toThrow('at least 2'); + }); +}); + +describe('latency options', () => { + it('rejects one-sample CLI and programmatic runs', async () => { + expect(() => parseOptions(['--iterations', '1'])).toThrow('must be at least 2'); + await expect( + runVueComponentMetaLatency({ quick: false, iterations: 1, jsonOut: 'unused.json' }) + ).rejects.toThrow('at least 2'); + }); +}); + +describe('createFreshEngineTask', () => { + it('creates and primes one new engine for every exact iteration', () => { + const events: string[] = []; + let nextId = 0; + const entry = createFreshEngineTask( + 'current', + () => { + const id = ++nextId; + return { + cold: () => (events.push(`cold:${id}`), 10), + applySave: (save) => events.push(`save:${id}:${save}`), + reextract: () => (events.push(`warm:${id}`), 2), + dispose: () => events.push(`dispose:${id}`), + }; + }, + {} + ); + createResettableBench([entry.task], 3).runSync(); + expect(entry.state.createdEngines).toBe(3); + expect(events).toEqual([ + 'cold:1', + 'save:1:1', + 'warm:1', + 'dispose:1', + 'cold:2', + 'save:2:1', + 'warm:2', + 'dispose:2', + 'cold:3', + 'save:3:1', + 'warm:3', + 'dispose:3', + ]); + }); + + it('fails when work changes between iterations or tasks', () => { + let iteration = 0; + let disposals = 0; + const unstable = createFreshEngineTask( + 'current', + () => ({ + cold: () => 10, + applySave() {}, + reextract: () => ++iteration, + dispose: () => { + disposals += 1; + }, + }), + {} + ); + expect(() => createResettableBench([unstable.task], 2).runSync()).toThrow('work mismatch'); + expect(disposals).toBe(2); + + const shared = { value: { coldMembers: 10, warmMembers: 2 } }; + const candidate = createFreshEngineTask( + 'next', + () => ({ cold: () => 11, applySave() {}, reextract: () => 2 }), + shared + ); + expect(() => createResettableBench([candidate.task], 2).runSync()).toThrow('work mismatch'); + }); + + it.each(['cold', 'applySave', 'reextract'] as const)( + 'disposes after a %s failure without replacing the original error', + (failurePoint) => { + const originalError = new Error(`${failurePoint} failed`); + let disposals = 0; + const entry = createFreshEngineTask( + 'current', + () => ({ + cold() { + if (failurePoint === 'cold') { + throw originalError; + } + return 10; + }, + applySave() { + if (failurePoint === 'applySave') { + throw originalError; + } + }, + reextract() { + if (failurePoint === 'reextract') { + throw originalError; + } + return 2; + }, + dispose() { + disposals += 1; + throw new Error('cleanup failed'); + }, + }), + {} + ); + + let receivedError: unknown; + try { + createResettableBench([entry.task], 2).runSync(); + } catch (error) { + receivedError = error; + } + + expect(receivedError).toBe(originalError); + expect(disposals).toBe(1); + expect(entry.state.engine).toBeUndefined(); + } + ); + + it('surfaces a disposal error when measurement and validation succeeded', () => { + const cleanupError = new Error('cleanup failed'); + const entry = createFreshEngineTask( + 'current', + () => ({ + cold: () => 10, + applySave() {}, + reextract: () => 2, + dispose: () => { + throw cleanupError; + }, + }), + {} + ); + + let receivedError: unknown; + try { + createResettableBench([entry.task], 2).runSync(); + } catch (error) { + receivedError = error; + } + expect(receivedError).toBe(cleanupError); + }); +}); + +describe('mapTaskResult', () => { + it('maps only the Storybook-owned latency fields', () => { + const bench = createResettableBench( + [{ name: 'fake', beforeEach() {}, measure() {}, afterEach() {} }], + 3 + ); + bench.runSync(); + const summary = mapTaskResult(bench.results[0], 3); + expect(summary.sampleCount).toBe(3); + expect(summary.sortedSamplesMs).toHaveLength(3); + expect(summary.confidenceLevel).toBe(0.95); + expect(summary).not.toHaveProperty('throughput'); + }); + + it('rejects incomplete or non-completed results', () => { + const bench = createResettableBench( + [{ name: 'fake', beforeEach() {}, measure() {}, afterEach() {} }], + 2 + ); + expect(() => mapTaskResult(bench.results[0], 2)).toThrow('did not complete'); + bench.runSync(); + expect(() => mapTaskResult(bench.results[0], 3)).toThrow('expected exactly 3'); + }); + + it('rejects non-finite and non-positive sample durations', () => { + const nonFinite = createCompletedResult(); + nonFinite.latency.samples![0] = Number.NaN; + expect(() => mapTaskResult(nonFinite, 2)).toThrow('latency.samples[0]'); + + const nonPositive = createCompletedResult(); + nonPositive.latency.samples![0] = 0; + expect(() => mapTaskResult(nonPositive, 2)).toThrow('greater than zero'); + }); + + it('rejects non-finite location statistics and negative dispersion statistics', () => { + const nonFinite = createCompletedResult(); + nonFinite.latency.mean = Number.POSITIVE_INFINITY; + expect(() => mapTaskResult(nonFinite, 2)).toThrow('latency.mean'); + + const negative = createCompletedResult(); + negative.latency.sd = -1; + expect(() => mapTaskResult(negative, 2)).toThrow('latency.sd'); + }); +}); diff --git a/code/lib/docgen-harness/src/perf/docgen-latency/resettable-bench.ts b/code/lib/docgen-harness/src/perf/docgen-latency/resettable-bench.ts new file mode 100644 index 000000000000..d523bdf8e7fa --- /dev/null +++ b/code/lib/docgen-harness/src/perf/docgen-latency/resettable-bench.ts @@ -0,0 +1,241 @@ +import { Bench, type TaskResult } from 'tinybench'; + +export interface ResettableLatencyTask { + name: string; + beforeEach(): void; + measure(): void; + afterEach(): void; +} + +export interface LatencySummary { + sampleCount: number; + sortedSamplesMs: number[]; + meanMs: number; + medianMs: number; + minMs: number; + maxMs: number; + medianAbsoluteDeviationMs: number; + standardDeviationMs: number; + marginOfErrorMs: number; + relativeMarginOfErrorPercent: number; + confidenceLevel: 0.95; +} + +export interface ResettableEngine { + cold(): number; + applySave(save: number): void; + reextract(): number; + dispose?(): void; +} + +export interface WorkSignature { + coldMembers: number; + warmMembers: number; +} + +export interface FreshTaskState { + engine?: ResettableEngine; + pendingColdMembers?: number; + pendingWarmMembers?: number; + work?: WorkSignature; + createdEngines: number; +} + +function assertSameWork(expected: WorkSignature, actual: WorkSignature, label: string): void { + if (expected.coldMembers !== actual.coldMembers || expected.warmMembers !== actual.warmMembers) { + throw new Error( + `${label} work mismatch: expected ${JSON.stringify(expected)}, received ${JSON.stringify(actual)}` + ); + } +} + +export function createFreshEngineTask( + name: string, + createEngine: () => ResettableEngine, + sharedWork: { value?: WorkSignature } +): { task: ResettableLatencyTask; state: FreshTaskState } { + const state: FreshTaskState = { createdEngines: 0 }; + let iterationFailed = false; + let measurementCompleted = false; + return { + state, + task: { + name, + beforeEach() { + iterationFailed = false; + measurementCompleted = false; + state.engine = undefined; + state.pendingColdMembers = undefined; + state.pendingWarmMembers = undefined; + try { + state.engine = createEngine(); + state.createdEngines += 1; + state.pendingColdMembers = state.engine.cold(); + state.engine.applySave(1); + } catch (error) { + iterationFailed = true; + throw error; + } + }, + measure() { + try { + if (!state.engine) { + throw new Error(`${name} engine was not prepared before measurement`); + } + state.pendingWarmMembers = state.engine.reextract(); + measurementCompleted = true; + } catch (error) { + iterationFailed = true; + throw error; + } + }, + afterEach() { + let validationError: unknown; + if (!iterationFailed) { + try { + if ( + !measurementCompleted || + state.pendingColdMembers === undefined || + state.pendingWarmMembers === undefined + ) { + throw new Error(`${name} iteration did not report complete work`); + } + const work = { + coldMembers: state.pendingColdMembers, + warmMembers: state.pendingWarmMembers, + }; + if (state.work) { + assertSameWork(state.work, work, `${name} iteration`); + } else { + state.work = work; + } + if (sharedWork.value) { + assertSameWork(sharedWork.value, work, `${name} task`); + } else { + sharedWork.value = work; + } + } catch (error) { + validationError = error; + } + } + + const engine = state.engine; + state.engine = undefined; + state.pendingColdMembers = undefined; + state.pendingWarmMembers = undefined; + const preserveIterationError = iterationFailed; + iterationFailed = false; + measurementCompleted = false; + + let disposalError: unknown; + try { + engine?.dispose?.(); + } catch (error) { + disposalError = error; + } + + if (preserveIterationError) { + return; + } + if (validationError) { + throw validationError; + } + if (disposalError) { + throw disposalError; + } + }, + }, + }; +} + +export function assertValidIterationCount(iterations: number): void { + if (!Number.isInteger(iterations) || iterations < 2) { + throw new Error(`iterations must be an integer of at least 2, received ${iterations}`); + } +} + +/** Construct without running so registration-time invocation remains directly testable. */ +export function createResettableBench(tasks: ResettableLatencyTask[], iterations: number): Bench { + assertValidIterationCount(iterations); + + const bench = new Bench({ + concurrency: null, + iterations, + retainSamples: true, + throws: true, + time: 0, + warmup: false, + }); + for (const task of tasks) { + bench.add(task.name, task.measure, { + // Tinybench otherwise calls a synchronous function during registration to detect promises. + async: false, + beforeEach: task.beforeEach, + afterEach: task.afterEach, + }); + } + return bench; +} + +export function mapTaskResult(result: TaskResult, expectedIterations: number): LatencySummary { + assertValidIterationCount(expectedIterations); + if (result.state !== 'completed') { + throw new Error(`Tinybench task did not complete (state: ${result.state})`); + } + const { latency } = result; + if (latency.samplesCount !== expectedIterations) { + throw new Error( + `Tinybench recorded ${latency.samplesCount} samples, expected exactly ${expectedIterations}` + ); + } + if (!latency.samples || latency.samples.length !== expectedIterations) { + throw new Error('Tinybench did not retain the complete sample set'); + } + + const sortedSamplesMs = [...latency.samples]; + for (const [index, sample] of sortedSamplesMs.entries()) { + assertFinitePositive(sample, `latency.samples[${index}]`); + } + for (const [field, value] of [ + ['latency.mean', latency.mean], + ['latency.p50', latency.p50], + ['latency.min', latency.min], + ['latency.max', latency.max], + ] as const) { + assertFinitePositive(value, field); + } + for (const [field, value] of [ + ['latency.mad', latency.mad], + ['latency.sd', latency.sd], + ['latency.moe', latency.moe], + ['latency.rme', latency.rme], + ] as const) { + assertFiniteNonNegative(value, field); + } + + return { + sampleCount: latency.samplesCount, + sortedSamplesMs, + meanMs: latency.mean, + medianMs: latency.p50, + minMs: latency.min, + maxMs: latency.max, + medianAbsoluteDeviationMs: latency.mad, + standardDeviationMs: latency.sd, + marginOfErrorMs: latency.moe, + relativeMarginOfErrorPercent: latency.rme, + confidenceLevel: 0.95, + }; +} + +function assertFinitePositive(value: number, field: string): void { + if (!Number.isFinite(value) || value <= 0) { + throw new Error(`Tinybench ${field} must be finite and greater than zero, received ${value}`); + } +} + +function assertFiniteNonNegative(value: number, field: string): void { + if (!Number.isFinite(value) || value < 0) { + throw new Error(`Tinybench ${field} must be finite and non-negative, received ${value}`); + } +} diff --git a/code/lib/docgen-harness/src/perf/docgen-latency/vue-component-meta.ts b/code/lib/docgen-harness/src/perf/docgen-latency/vue-component-meta.ts new file mode 100644 index 000000000000..495d35c39031 --- /dev/null +++ b/code/lib/docgen-harness/src/perf/docgen-latency/vue-component-meta.ts @@ -0,0 +1,194 @@ +import * as fs from 'node:fs'; +import { createRequire } from 'node:module'; +import * as path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { z } from 'zod'; + +import { parseHarnessOptions } from '../docgen-shared/args.ts'; +import { SANDBOX_DIRECTORY } from '../docgen-shared/paths.ts'; +import { + type Pin, + createVueComponentMetaEngine, + loadCheckers, +} from '../docgen-perf/engines/vue-component-meta.ts'; +import { + type LatencySummary, + type WorkSignature, + assertValidIterationCount, + createFreshEngineTask, + createResettableBench, + mapTaskResult, +} from './resettable-bench.ts'; + +const require = createRequire(import.meta.url); +const PINS = ['current', 'next'] as const; +const DEFAULT_ITERATIONS = 15; +const QUICK_ITERATIONS = 3; + +const OPTIONS = { + quick: { type: 'boolean' }, + iterations: { type: 'string' }, + json: { type: 'string' }, +} as const; + +const SCHEMA = z.object({ + quick: z.boolean().default(false), + iterations: z.coerce.number().int().min(2, 'must be at least 2').optional(), + jsonOut: z.string().default(path.join(SANDBOX_DIRECTORY, 'docgen-latency', 'results.json')), +}); + +export interface Options { + quick: boolean; + iterations?: number; + jsonOut: string; +} + +interface EngineOutput { + packageName: string; + version: string; + work: WorkSignature; + latency: LatencySummary; +} + +export function parseOptions(argv: string[]): Options { + return parseHarnessOptions(argv, OPTIONS, SCHEMA, (values) => ({ + ...values, + jsonOut: values.json, + })); +} + +export function packageVersion(packageName: string): string { + let packagePath: string; + try { + packagePath = require.resolve(`${packageName}/package.json`); + } catch { + let directory = path.dirname(require.resolve(packageName)); + while (true) { + const candidate = path.join(directory, 'package.json'); + if (fs.existsSync(candidate)) { + packagePath = candidate; + break; + } + const parent = path.dirname(directory); + if (parent === directory) { + throw new Error(`could not find package metadata for ${packageName}`); + } + directory = parent; + } + } + const metadata = JSON.parse(fs.readFileSync(packagePath, 'utf8')) as { version?: string }; + if (!metadata.version) { + throw new Error(`${packageName} package metadata has no version`); + } + return metadata.version; +} + +export async function runVueComponentMetaLatency(options: Options) { + const iterations = options.iterations ?? (options.quick ? QUICK_ITERATIONS : DEFAULT_ITERATIONS); + assertValidIterationCount(iterations); + + const versions = { + current: packageVersion('vue-component-meta'), + next: packageVersion('vue-component-meta-next'), + }; + if (versions.current === versions.next) { + throw new Error(`both pins resolved vue-component-meta ${versions.current}; not a comparison`); + } + + const modules = { + current: await loadCheckers('current'), + next: await loadCheckers('next'), + }; + const scenario = { + name: 'flat' as const, + packages: 1, + componentsPerPackage: options.quick ? 5 : 20, + chainDepth: 1, + fanOut: options.quick ? 2 : 4, + heavyLib: false, + }; + const sharedWork: { value?: WorkSignature } = {}; + const taskEntries = PINS.map((pin) => + createFreshEngineTask( + pin, + () => + createVueComponentMetaEngine( + { + scenario: scenario.name, + packages: scenario.packages, + componentsPerPackage: scenario.componentsPerPackage, + chainDepth: scenario.chainDepth, + fanOut: scenario.fanOut, + heavyLib: scenario.heavyLib, + saves: 1, + outDir: path.join(SANDBOX_DIRECTORY, 'docgen-latency', pin), + }, + modules[pin] + ), + sharedWork + ) + ); + + const bench = createResettableBench( + taskEntries.map(({ task }) => task), + iterations + ); + bench.runSync(); + + const engines = Object.fromEntries( + PINS.map((pin, index) => { + const state = taskEntries[index].state; + if (state.createdEngines !== iterations || !state.work) { + throw new Error( + `${pin} created ${state.createdEngines} engines, expected exactly ${iterations}` + ); + } + const packageName = pin === 'current' ? 'vue-component-meta' : 'vue-component-meta-next'; + return [ + pin, + { + packageName, + version: versions[pin], + work: state.work, + latency: mapTaskResult(bench.results[index], iterations), + } satisfies EngineOutput, + ]; + }) + ) as Record; + + return { + schemaVersion: 1 as const, + generatedAt: new Date().toISOString(), + nodeVersion: process.version, + runner: { name: 'tinybench' as const, version: packageVersion('tinybench') }, + mode: 'descriptive' as const, + gating: false, + iterations, + taskOrder: [...PINS], + scenario, + engines, + }; +} + +async function main() { + const options = parseOptions(process.argv.slice(2)); + const results = await runVueComponentMetaLatency(options); + fs.mkdirSync(path.dirname(options.jsonOut), { recursive: true }); + fs.writeFileSync(options.jsonOut, JSON.stringify(results, null, 2)); + for (const pin of PINS) { + const result = results.engines[pin]; + console.log( + `${pin} ${result.version}: median=${result.latency.medianMs.toFixed(2)}ms ` + + `samples=${result.latency.sampleCount}` + ); + } + console.log(`wrote ${options.jsonOut}`); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/code/lib/docgen-harness/src/perf/docgen-memory/memory-harness.ts b/code/lib/docgen-harness/src/perf/docgen-memory/memory-harness.ts index f4c57ecd77e0..64e10d376853 100644 --- a/code/lib/docgen-harness/src/perf/docgen-memory/memory-harness.ts +++ b/code/lib/docgen-harness/src/perf/docgen-memory/memory-harness.ts @@ -38,6 +38,7 @@ import ts from 'typescript'; import { z } from 'zod'; import { countOption, parseHarnessOptions, positiveCountOption } from '../docgen-shared/args.ts'; +import { runLatencySeries } from '../docgen-shared/latency-series.ts'; import { SANDBOX_DIRECTORY } from '../docgen-shared/paths.ts'; import { type StoryRefLike, @@ -89,6 +90,7 @@ const OPTIONS = { out: { type: 'string' }, reuse: { type: 'boolean' }, json: { type: 'string' }, + latency: { type: 'boolean' }, } as const; const SCHEMA = z.object({ @@ -110,6 +112,7 @@ const SCHEMA = z.object({ outDir: z.string().default(path.join(SANDBOX_DIRECTORY, 'docgen-memory-stress')), reuse: z.boolean().default(false), jsonOut: z.string().optional(), + latency: z.boolean().default(false), }); type HarnessOptions = z.infer; @@ -122,6 +125,7 @@ function parseOptions(argv: string[]): HarnessOptions { forceGc: !values.noForceGc, outDir: values.out, jsonOut: values.json, + latency: values.latency, })); } @@ -260,14 +264,18 @@ function refreshEngine( harnessMain(async () => { const options = parseOptions(process.argv.slice(2)); + if (options.latency && options.mode !== 'refresh') { + throw new Error('--latency requires --mode refresh'); + } console.log('docgen-memory harness'); console.log( ` components=${options.components} variants=${options.variants} props=${options.props} ` + `saves=${options.saves} mode=${options.mode} scope=${options.scope} ` + - `forceGc=${options.forceGc && gcAvailable()}` + `lane=${options.latency ? 'latency' : 'memory'} ` + + `forceGc=${options.latency ? 'n/a' : options.forceGc && gcAvailable()}` ); - if (options.forceGc && !gcAvailable()) { + if (!options.latency && options.forceGc && !gcAvailable()) { console.log(' (run with `node --expose-gc` to measure retained heap; continuing without it)'); } @@ -276,6 +284,18 @@ harnessMain(async () => { const manager = new ComponentMetaManager(ts, options.recycleHeapPressureRatio); const entries = buildStoryRefs(project.componentPaths, project.storyPaths); + if (options.latency) { + const latency = await runLatencySeries(refreshEngine(manager, entries, project, options), { + saves: options.saves, + coldLabel: `${entries.length} components`, + }); + if (options.jsonOut) { + fs.writeFileSync(options.jsonOut, JSON.stringify({ options, ...latency }, null, 2)); + console.log(` wrote ${options.jsonOut}`); + } + return; + } + if (options.mode === 'live') { runLiveMode(manager, entries, options); return; diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/aggregate.test.ts b/code/lib/docgen-harness/src/perf/docgen-perf/aggregate.test.ts deleted file mode 100644 index 3dc61adba226..000000000000 --- a/code/lib/docgen-harness/src/perf/docgen-perf/aggregate.test.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import type { MemorySample, SaveSample } from '../docgen-shared/samples.ts'; -import type { SeriesResult } from '../docgen-shared/series.ts'; -import { summarizeSeries } from '../docgen-shared/stats.ts'; -import { designatedRep, oneShotMetrics, seriesMetrics } from './aggregate.ts'; - -function sample(save: number, durMs: number, heapUsedMb = 20, retainedHeapMb = 10): SaveSample { - return { save, durMs, rssMb: 100, heapUsedMb, retainedHeapMb }; -} - -/** Built through summarizeSeries, so the fixture cannot claim figures a real run would not produce. */ -function repOf(coldMs: number, samples: SaveSample[], baseline: MemorySample): SeriesResult { - return { coldMs, baseline, samples, ...summarizeSeries(samples, baseline) }; -} - -function rep(coldMs: number, durations: number[]): SeriesResult { - return repOf( - coldMs, - durations.map((d, i) => sample(i + 1, d)), - { rssMb: 90, heapUsedMb: 12, retainedHeapMb: 10 } - ); -} - -describe('designatedRep', () => { - it('picks the repetition whose cold sample is the median', () => { - const reps = [rep(900, [1]), rep(100, [2]), rep(300, [3]), rep(200, [4]), rep(250, [5])]; - expect(designatedRep(reps).coldMs).toBe(250); - }); - - it('never picks the slow first repetition just because it came first', () => { - const reps = [rep(5000, [999]), rep(100, [10]), rep(110, [11]), rep(120, [12]), rep(130, [13])]; - expect(designatedRep(reps).coldMs).not.toBe(5000); - expect(designatedRep(reps).coldMs).toBe(120); - }); - - it('takes the lower middle for an even count', () => { - const reps = [rep(10, [1]), rep(20, [2]), rep(30, [3]), rep(40, [4])]; - expect(designatedRep(reps).coldMs).toBe(20); - }); - - it('does not mutate its input', () => { - const reps = [rep(30, [1]), rep(10, [2]), rep(20, [3])]; - designatedRep(reps); - expect(reps.map((r) => r.coldMs)).toEqual([30, 10, 20]); - }); -}); - -describe('seriesMetrics', () => { - const reps = [rep(300, [9, 9, 9]), rep(100, [1, 2, 3]), rep(200, [4, 5, 6])]; - - it('medians cold across every repetition', () => { - expect(seriesMetrics(reps, 3).coldExtractionMs).toMatchObject({ - samples: [300, 100, 200], - median: 200, - }); - }); - - it('reads warm from the designated repetition, not the first', () => { - expect(seriesMetrics(reps, 3).warmExtractionMs).toMatchObject({ - samples: [4, 5, 6], - median: 5, - }); - }); - - it('marks whole-project scan as not applicable rather than faking one', () => { - expect(seriesMetrics(reps, 3).wholeProjectScanMs).toEqual({ status: 'n/a' }); - }); - - it('derives transient memory from the same repetition warm came from', () => { - expect(seriesMetrics(reps, 3).peakTransientMb).toMatchObject({ mean: 10 }); - }); - - it('rejects a repetition count below the pinned N', () => { - expect(() => seriesMetrics(reps, 5)).toThrow('recorded 3 repetitions, expected the pinned 5'); - }); - - it('rejects an empty set of repetitions', () => { - expect(() => seriesMetrics([], 5)).toThrow('no completed repetition recorded'); - }); - - it('rejects a run with no retained samples', () => { - const noGc = repOf(100, [{ save: 1, durMs: 5, rssMb: 100, heapUsedMb: 20 }], { - rssMb: 90, - heapUsedMb: 12, - }); - expect(() => seriesMetrics([noGc], 1)).toThrow('retained metrics missing'); - }); -}); - -describe('oneShotMetrics', () => { - const reps = [ - { coldMs: 300, warmMs: 280, peakRssMb: 500 }, - { coldMs: 100, warmMs: 120, peakRssMb: 400 }, - { coldMs: 200, warmMs: 200, peakRssMb: 450 }, - ]; - - it('reports cold and whole-project scan as the same measurement', () => { - const metrics = oneShotMetrics(reps, 3); - expect(metrics.wholeProjectScanMs).toEqual(metrics.coldExtractionMs); - }); - - it('has no retained series to report', () => { - const metrics = oneShotMetrics(reps, 3); - expect(metrics.retainedGrowthMb).toEqual({ status: 'n/a' }); - expect(metrics.retainedSlopeMbPerSave).toEqual({ status: 'n/a' }); - }); - - it('enforces the pinned repetition count', () => { - expect(() => oneShotMetrics(reps, 5)).toThrow('expected the pinned 5'); - }); - - it('reports no peak rather than a low one when a repetition went unsampled', () => { - // `ps` is POSIX-only and a short run can finish inside one poll interval, so a repetition can - // come back with nothing sampled. Averaging that in would report a peak nobody measured. - const partial = [ - { coldMs: 300, warmMs: 280, peakRssMb: 500 }, - { coldMs: 100, warmMs: 120 }, - ]; - expect(oneShotMetrics(partial, 2).peakTransientMb).toEqual({ status: 'n/a' }); - }); -}); diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/aggregate.ts b/code/lib/docgen-harness/src/perf/docgen-perf/aggregate.ts deleted file mode 100644 index 1de7582109e8..000000000000 --- a/code/lib/docgen-harness/src/perf/docgen-perf/aggregate.ts +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Turns N repetitions into the five floor metrics (see ../PERF-METHODOLOGY.md). Warm - * latency and all three memory metrics read the same designated repetition's save series, so those - * four figures always describe one run; cold latency is a median across every repetition instead. - */ -import type { SeriesResult } from '../docgen-shared/series.ts'; -import { mean, median } from '../docgen-shared/stats.ts'; -import { type EngineMetrics, NOT_APPLICABLE } from './types.ts'; - -/** - * Repetition 1 is systematically the slowest - it pays for a cold module graph and a cold OS page - * cache - so taking it would bias warm latency and all three memory metrics at once. Cold latency - * needs no such protection: it is already a median across every repetition. - */ -export function designatedRep(reps: T[]): T { - const byCold = [...reps].sort((a, b) => a.coldMs - b.coldMs); - return byCold[Math.floor((byCold.length - 1) / 2)]; -} - -/** - * An engine that failed part-way through holds fewer samples than expectedN. Reporting those would - * put numbers taken at an unrecorded N into the results file, which the comparison method rests on - * not happening. - */ -function assertRepetitionCount(reps: unknown[], expectedN: number): void { - if (reps.length === 0) { - throw new Error('no completed repetition recorded'); - } - if (reps.length !== expectedN) { - throw new Error(`recorded ${reps.length} repetitions, expected the pinned ${expectedN}`); - } -} - -export function seriesMetrics(reps: SeriesResult[], expectedN: number): EngineMetrics { - assertRepetitionCount(reps, expectedN); - const designated = designatedRep(reps); - const coldSamples = reps.map((r) => r.coldMs); - const warmSamples = designated.samples.map((s) => s.durMs); - const { transients, avgTransient } = designated; - - if ( - avgTransient === undefined || - designated.retainedGrowth === undefined || - designated.retainedSlope === undefined - ) { - throw new Error( - 'retained metrics missing: the child must run under --expose-gc, over at least two saves' - ); - } - - return { - coldExtractionMs: { status: 'measured', samples: coldSamples, median: median(coldSamples) }, - warmExtractionMs: { status: 'measured', samples: warmSamples, median: median(warmSamples) }, - // Per-component engines have no batch pass; recording one would be a faked equivalent. - wholeProjectScanMs: NOT_APPLICABLE, - peakTransientMb: { status: 'measured', samples: transients, mean: avgTransient }, - retainedGrowthMb: { status: 'measured', value: designated.retainedGrowth }, - retainedSlopeMbPerSave: { status: 'measured', value: designated.retainedSlope }, - }; -} - -export interface OneShotRepetition { - coldMs: number; - warmMs: number; - /** Undefined when the engine's external sampler never read the child's memory. */ - peakRssMb?: number; - coldMembers?: number; - warmMembers?: number; - /** Of the cold pass's members, how many carry a type the engine never resolved. */ - coldOpaqueTypes?: number; -} - -/** - * Metrics for a one-shot CLI engine: a fresh process per run, so cold extraction and the - * whole-project scan are the same full-project measurement, and there is no retained series to read. - */ -export function oneShotMetrics(reps: OneShotRepetition[], expectedN: number): EngineMetrics { - assertRepetitionCount(reps, expectedN); - const coldSamples = reps.map((r) => r.coldMs); - const warmSamples = reps.map((r) => r.warmMs); - // Every repetition must have been sampled, or the mean describes a different N than the one - // recorded. A partially-sampled series is reported as no measurement rather than as a low one. - const peaks = reps.map((r) => r.peakRssMb).filter((mb) => mb !== undefined); - return { - coldExtractionMs: { status: 'measured', samples: coldSamples, median: median(coldSamples) }, - warmExtractionMs: { status: 'measured', samples: warmSamples, median: median(warmSamples) }, - wholeProjectScanMs: { status: 'measured', samples: coldSamples, median: median(coldSamples) }, - peakTransientMb: - peaks.length === reps.length - ? { status: 'measured', samples: peaks, mean: mean(peaks) } - : NOT_APPLICABLE, - retainedGrowthMb: NOT_APPLICABLE, - retainedSlopeMbPerSave: NOT_APPLICABLE, - }; -} diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/cli.test.ts b/code/lib/docgen-harness/src/perf/docgen-perf/cli.test.ts new file mode 100644 index 000000000000..450ef9c26a1b --- /dev/null +++ b/code/lib/docgen-harness/src/perf/docgen-perf/cli.test.ts @@ -0,0 +1,58 @@ +import * as path from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +import { parseCliOptions } from './cli.ts'; +import { DEFAULT_ENGINE_IDS } from './registry.ts'; + +const workRoot = path.resolve('docgen-perf-test'); + +describe('parseCliOptions', () => { + it('preserves the descriptive command defaults and repeatable engine selection', () => { + expect(parseCliOptions([], workRoot)).toMatchObject({ + quick: false, + engines: DEFAULT_ENGINE_IDS, + jsonOut: path.join(workRoot, 'results.json'), + }); + expect( + parseCliOptions(['--engine', 'react-legacy', '--engine', 'react-osa', '--quick'], workRoot) + ).toMatchObject({ quick: true, engines: ['react-legacy', 'react-osa'] }); + }); + + it('selects exactly one pair and validates the explicit gate configuration', () => { + expect( + parseCliOptions( + [ + '--compare', + 'vue-component-meta-version', + '--seed', + '42', + '--repetitions', + '10', + '--max-regression', + '0.05', + ], + workRoot + ) + ).toMatchObject({ + engines: ['vue-component-meta', 'vue-component-meta-next'], + seed: 42, + repetitions: 10, + maxRegression: 0.05, + }); + expect(() => + parseCliOptions( + ['--compare', 'vue', '--max-regression', '0.1', '--repetitions', '9'], + workRoot + ) + ).toThrow('even --repetitions'); + expect(() => parseCliOptions(['--compare', 'vue'], workRoot)).toThrow( + 'explicit --max-regression' + ); + }); + + it('rejects gate-only flags on a descriptive run', () => { + expect(() => parseCliOptions(['--seed', '1'], workRoot)).toThrow('require --compare'); + expect(() => parseCliOptions(['--repetitions', '4'], workRoot)).toThrow('require --compare'); + }); +}); diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/cli.ts b/code/lib/docgen-harness/src/perf/docgen-perf/cli.ts index 4fdfdedff700..0c6cc4e0a4bb 100644 --- a/code/lib/docgen-harness/src/perf/docgen-perf/cli.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/cli.ts @@ -1,37 +1,82 @@ -/** Command line for the docgen perf suite. */ +/** Command line for descriptive runs and explicit paired timing gates. */ import * as path from 'node:path'; import { z } from 'zod'; import { parseHarnessOptions } from '../docgen-shared/args.ts'; +import { COMPARISON_PAIRS, type PairName } from './comparison.ts'; +import { MIN_PAIRED_REPETITIONS } from './config.ts'; import { ALL_ENGINE_IDS, DEFAULT_ENGINE_IDS } from './registry.ts'; import type { EngineId } from './types.ts'; +const PAIR_NAMES = COMPARISON_PAIRS.map(({ name }) => name) as [PairName, ...PairName[]]; + const OPTIONS = { quick: { type: 'boolean' }, engine: { type: 'string', multiple: true }, json: { type: 'string' }, + compare: { type: 'string' }, + seed: { type: 'string' }, + repetitions: { type: 'string' }, + 'max-regression': { type: 'string' }, } as const; export interface CliOptions { quick: boolean; engines: EngineId[]; jsonOut: string; + compare?: PairName; + seed?: number; + repetitions?: number; + maxRegression?: number; } export function parseCliOptions(argv: string[], workRoot: string): CliOptions { const schema = z.object({ quick: z.boolean().default(false), - // No --engine means the default set, which is why this defaults to empty rather than to - // DEFAULT_ENGINE_IDS: an explicit empty list and an absent flag are the same request. engines: z.array(z.enum(ALL_ENGINE_IDS as [EngineId, ...EngineId[]])).default([]), jsonOut: z.string().default(path.join(workRoot, 'results.json')), + compare: z.enum(PAIR_NAMES).optional(), + seed: z.coerce.number().int().min(0).max(0xffff_ffff).optional(), + repetitions: z.coerce.number().int().positive().optional(), + maxRegression: z.coerce.number().finite().nonnegative().optional(), }); const options = parseHarnessOptions(argv, OPTIONS, schema, (values) => ({ ...values, engines: values.engine, jsonOut: values.json, })); + + if (options.compare) { + const repetitions = options.repetitions ?? MIN_PAIRED_REPETITIONS; + if (repetitions < MIN_PAIRED_REPETITIONS || repetitions % 2 !== 0) { + throw new Error( + `--compare requires an even --repetitions of at least ${MIN_PAIRED_REPETITIONS}` + ); + } + if (options.maxRegression === undefined) { + throw new Error('--compare requires an explicit --max-regression fraction'); + } + const pair = COMPARISON_PAIRS.find(({ name }) => name === options.compare)!; + const selected = options.engines.length ? options.engines : [pair.control, pair.candidate]; + if (!selected.includes(pair.control) || !selected.includes(pair.candidate)) { + throw new Error( + `--compare ${pair.name} requires --engine ${pair.control} and --engine ${pair.candidate}` + ); + } + if (selected.some((engine) => engine !== pair.control && engine !== pair.candidate)) { + throw new Error('--compare runs exactly one pair; remove unrelated --engine selections'); + } + return { ...options, repetitions, engines: selected }; + } + + if ( + options.seed !== undefined || + options.repetitions !== undefined || + options.maxRegression !== undefined + ) { + throw new Error('--seed, --repetitions, and --max-regression require --compare'); + } return { ...options, engines: options.engines.length ? options.engines : DEFAULT_ENGINE_IDS, diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/comparison.test.ts b/code/lib/docgen-harness/src/perf/docgen-perf/comparison.test.ts new file mode 100644 index 000000000000..880abd2437c7 --- /dev/null +++ b/code/lib/docgen-harness/src/perf/docgen-perf/comparison.test.ts @@ -0,0 +1,388 @@ +import { describe, expect, it } from 'vitest'; + +import { + assessWork, + comparePairedTimings, + computePairedEffect, + createComparisonPlan, + deriveComparisonSeed, + executeComparisonPlan, + evaluateBudget, + type PairedComparison, + type PairedEffect, + studentTCritical95, + type WorkProfile, +} from './comparison.ts'; + +const work = (version = '1.0.0'): WorkProfile => ({ + version, + cold: { members: 20, opaqueTypes: 0 }, + warm: [ + { save: 1, members: 21, opaqueTypes: 0 }, + { save: 2, members: 22, opaqueTypes: 1 }, + ], +}); + +describe('createComparisonPlan', () => { + it('is deterministic, adjacent, and exactly AB/BA balanced', () => { + const options = { seed: 'ci-123', pair: 'vue-version', scenario: 'flat', repetitions: 10 }; + const first = createComparisonPlan(options); + const second = createComparisonPlan(options); + + expect(first).toEqual(second); + expect(first.blocks).toHaveLength(10); + expect(first.blocks.map(({ block }) => block)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + expect(first.blocks.filter(({ order }) => order[0] === 'control')).toHaveLength(5); + expect(first.blocks.filter(({ order }) => order[0] === 'candidate')).toHaveLength(5); + for (const { order } of first.blocks) { + expect(new Set(order)).toEqual(new Set(['control', 'candidate'])); + } + for (let block = 0; block < first.blocks.length; block += 2) { + expect( + first.blocks + .slice(block, block + 2) + .map(({ order }) => order[0]) + .sort() + ).toEqual(['candidate', 'control']); + } + }); + + it('derives a stable independent seed for each pair and scenario', () => { + const seed = deriveComparisonSeed(42, 'vue-version', 'flat'); + expect(seed).toBe(deriveComparisonSeed(42, 'vue-version', 'flat')); + expect(seed).not.toBe(deriveComparisonSeed(42, 'vue-version', 'workspace')); + expect(seed).not.toBe(deriveComparisonSeed(42, 'react', 'flat')); + }); + + it.each([0, 8, 9, 11, 10.5])('rejects invalid repetition count %s', (repetitions) => { + expect(() => + createComparisonPlan({ seed: 1, pair: 'vue', scenario: 'flat', repetitions }) + ).toThrow('even repetition count of at least 10'); + }); +}); + +describe('executeComparisonPlan', () => { + it('executes exact adjacent block order and retains side identity', async () => { + const plan = createComparisonPlan({ + seed: 42, + pair: 'vue-version', + scenario: 'flat', + repetitions: 10, + }); + const calls: string[] = []; + const blocks = await executeComparisonPlan(plan, async (side, block) => { + calls.push(`${block}:${side}`); + return `${side}-${block}`; + }); + expect(calls).toEqual( + plan.blocks.flatMap(({ block, order }) => order.map((side) => `${block}:${side}`)) + ); + expect(blocks[0]).toMatchObject({ + block: 1, + control: 'control-1', + candidate: 'candidate-1', + }); + }); + + it('does not emit a partial block after a side fails', async () => { + const plan = createComparisonPlan({ + seed: 1, + pair: 'vue-version', + scenario: 'flat', + repetitions: 10, + }); + const published: unknown[] = []; + await expect( + executeComparisonPlan(plan, async (side, block) => { + if (block === 2 && side === plan.blocks[1].order[1]) { + throw new Error('side failed'); + } + published.push({ side, block }); + return 1; + }) + ).rejects.toThrow('side failed'); + expect(published.filter((value) => (value as { block: number }).block === 2)).toHaveLength(1); + }); +}); + +describe('assessWork', () => { + it('requires matching cold and ordered warm member/opaque signatures', () => { + expect(assessWork(work(), work())).toEqual({ + status: 'same-work', + reason: 'matching-signatures', + }); + + const reordered = work(); + reordered.warm = [...reordered.warm!].reverse(); + expect(assessWork(work(), reordered)).toMatchObject({ status: 'different-work' }); + + const mislabeled = work(); + mislabeled.warm = [ + { save: 2, members: 21, opaqueTypes: 0 }, + { save: 1, members: 22, opaqueTypes: 1 }, + ]; + const labeled = work(); + labeled.warm = [ + { save: 1, members: 21, opaqueTypes: 0 }, + { save: 2, members: 22, opaqueTypes: 1 }, + ]; + expect(assessWork(labeled, mislabeled)).toMatchObject({ + status: 'different-work', + reason: 'different-warm-signature', + }); + }); + + it('distinguishes unequal, unknown, and colliding-version work', () => { + const different = work(); + different.cold = { members: 19, opaqueTypes: 0 }; + expect(assessWork(work(), different).status).toBe('different-work'); + + const unknown = work(); + unknown.warm = [ + { save: 1, members: 21 }, + { save: 2, members: 22, opaqueTypes: 1 }, + ]; + expect(assessWork(work(), unknown).status).toBe('unknown-work'); + + expect(assessWork(work('3.3.9'), work('3.3.9'), { versionsMustDiffer: true })).toEqual({ + status: 'same-version', + reason: 'matching-version', + }); + expect( + assessWork({ ...work(), version: undefined }, work('3.3.8'), { versionsMustDiffer: true }) + .status + ).toBe('unknown-work'); + + expect(assessWork({ ...work(), warm: [] }, { ...work(), warm: [] }).status).toBe( + 'unknown-work' + ); + }); + + it('does not calculate an effect unless work is proven equal', () => { + const timings = Array.from({ length: 10 }, (_, block) => ({ + block: block + 1, + controlMs: 100, + candidateMs: 120, + })); + const candidateWork = work(); + candidateWork.cold = { members: 21, opaqueTypes: 0 }; + + expect( + comparePairedTimings({ controlWork: work(), candidateWork, timings, expectedPairs: 10 }) + ).toEqual({ + work: { status: 'different-work', reason: 'different-cold-signature' }, + }); + }); + + it('validates cold and warm work independently', () => { + const timings = Array.from({ length: 10 }, (_, block) => ({ + block: block + 1, + controlMs: 100, + candidateMs: 105, + })); + const coldMismatch = work(); + coldMismatch.cold = { members: 21, opaqueTypes: 0 }; + + expect( + comparePairedTimings({ + controlWork: work(), + candidateWork: coldMismatch, + timings, + expectedPairs: 10, + metric: 'cold', + }) + ).not.toHaveProperty('effect'); + expect( + comparePairedTimings({ + controlWork: work(), + candidateWork: coldMismatch, + timings, + expectedPairs: 10, + metric: 'warm', + }).effect?.status + ).toBe('measured'); + + const warmMismatch = work(); + warmMismatch.warm = [ + { save: 1, members: 99 }, + { save: 2, members: 22, opaqueTypes: 1 }, + ]; + expect( + comparePairedTimings({ + controlWork: work(), + candidateWork: warmMismatch, + timings, + expectedPairs: 10, + metric: 'cold', + }).effect?.status + ).toBe('measured'); + expect( + comparePairedTimings({ + controlWork: work(), + candidateWork: warmMismatch, + timings, + expectedPairs: 10, + metric: 'warm', + }) + ).not.toHaveProperty('effect'); + }); +}); + +describe('computePairedEffect', () => { + it('uses paired candidate/control log ratios and a 95% Student-t interval', () => { + const ratios = [0.9, 0.95, 1, 1.05, 1.1, 0.92, 0.98, 1.02, 1.08, 1.12]; + const result = computePairedEffect( + ratios.map((ratio, block) => ({ + block: block + 1, + controlMs: 100, + candidateMs: 100 * ratio, + })), + { expectedPairs: 10 } + ); + + expect(result.status).toBe('measured'); + const effect = result as PairedEffect; + const expectedLogMean = ratios.reduce((sum, ratio) => sum + Math.log(ratio), 0) / ratios.length; + expect(effect.meanLogRatio).toBeCloseTo(expectedLogMean, 12); + expect(effect.candidateOverControl.estimate).toBeCloseTo(Math.exp(expectedLogMean), 12); + expect(effect.criticalValue95).toBeCloseTo(2.2621571628, 10); + expect(effect.candidateOverControl.lower95).toBeLessThan(effect.candidateOverControl.estimate); + expect(effect.candidateOverControl.upper95).toBeGreaterThan( + effect.candidateOverControl.estimate + ); + }); + + it('preserves candidate/control direction', () => { + const result = computePairedEffect([ + { block: 1, controlMs: 100, candidateMs: 200 }, + { block: 2, controlMs: 50, candidateMs: 100 }, + ]); + expect(result.status).toBe('measured'); + const effect = result as PairedEffect; + expect(effect.candidateOverControl.estimate).toBeCloseTo(2, 12); + expect(effect.candidateOverControl.lower95).toBeCloseTo(2, 12); + expect(effect.candidateOverControl.upper95).toBeCloseTo(2, 12); + }); + + it.each([ + ['incomplete-data', [{ block: 1, controlMs: 1, candidateMs: 1 }], { expectedPairs: 2 }], + [ + 'incomplete-data', + [ + { block: 1, controlMs: 1, candidateMs: 1 }, + { block: 1, controlMs: 1, candidateMs: 1 }, + ], + {}, + ], + [ + 'incomplete-data', + [ + { block: 1, controlMs: 1, candidateMs: 1 }, + { block: 3, controlMs: 1, candidateMs: 1 }, + ], + {}, + ], + [ + 'incomplete-data', + [ + { block: 1, controlMs: 1, candidateMs: 1 }, + { block: 2, controlMs: 1 }, + ], + {}, + ], + [ + 'nonfinite-data', + [ + { block: 1, controlMs: 1, candidateMs: Number.NaN }, + { block: 2, controlMs: 1, candidateMs: 1 }, + ], + {}, + ], + [ + 'nonpositive-data', + [ + { block: 1, controlMs: 1, candidateMs: 0 }, + { block: 2, controlMs: 1, candidateMs: 1 }, + ], + {}, + ], + ] as const)('rejects %s', (reason, timings, options) => { + expect(computePairedEffect(timings, options)).toEqual({ status: 'invalid', reason }); + }); + + it('uses a valid asymptotic Student-t critical value beyond the table', () => { + expect(studentTCritical95(100)).toBeCloseTo(1.983971518, 8); + }); +}); + +describe('evaluateBudget', () => { + const comparison = (lower95: number, upper95: number, pairs = 10): PairedComparison => ({ + work: { status: 'same-work', reason: 'matching-signatures' }, + effect: { + status: 'measured', + pairs, + logRatios: [], + meanLogRatio: Math.log((lower95 + upper95) / 2), + standardError: 0, + criticalValue95: studentTCritical95(Math.max(1, pairs - 1)), + candidateOverControl: { + estimate: (lower95 + upper95) / 2, + lower95, + upper95, + }, + }, + }); + + it('keeps descriptive and smoke runs out of timing gates', () => { + expect(evaluateBudget(comparison(0.9, 1.3), { repetitions: 10 }).status).toBe('not-configured'); + expect( + evaluateBudget(comparison(1.3, 1.4), { + repetitions: 10, + maxRegression: 0.1, + smoke: true, + }) + ).toMatchObject({ + status: 'smoke', + maxRegression: 0.1, + maxCandidateOverControl: 1.1, + }); + + expect( + evaluateBudget( + { work: { status: 'unknown-work', reason: 'missing-signature' } }, + { repetitions: 10, maxRegression: 0.1, smoke: true } + ) + ).toMatchObject({ status: 'invalid-gate', reason: 'work-not-comparable' }); + }); + + it('only reports regression when the whole interval exceeds the limit', () => { + expect( + evaluateBudget(comparison(1.11, 1.2), { repetitions: 10, maxRegression: 0.1 }).status + ).toBe('regression'); + expect( + evaluateBudget(comparison(1.05, 1.15), { repetitions: 10, maxRegression: 0.1 }).status + ).toBe('inconclusive'); + expect( + evaluateBudget(comparison(0.95, 1.1), { repetitions: 10, maxRegression: 0.1 }).status + ).toBe('within-budget'); + }); + + it('rejects malformed or unjustified gates', () => { + expect(evaluateBudget(comparison(1, 1), { repetitions: 9, maxRegression: 0.1 })).toMatchObject({ + status: 'invalid-gate', + reason: 'invalid-repetition-count', + }); + expect( + evaluateBudget(comparison(1, 1), { repetitions: 10, maxRegression: Number.NaN }) + ).toMatchObject({ status: 'invalid-gate', reason: 'invalid-budget' }); + expect( + evaluateBudget( + { work: { status: 'unknown-work', reason: 'missing-signature' } }, + { repetitions: 10, maxRegression: 0.1 } + ) + ).toMatchObject({ status: 'invalid-gate', reason: 'work-not-comparable' }); + expect( + evaluateBudget(comparison(1, 1, 8), { repetitions: 10, maxRegression: 0.1 }) + ).toMatchObject({ status: 'invalid-gate', reason: 'incomplete-effect' }); + }); +}); diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/comparison.ts b/code/lib/docgen-harness/src/perf/docgen-perf/comparison.ts new file mode 100644 index 000000000000..50aacbb4d652 --- /dev/null +++ b/code/lib/docgen-harness/src/perf/docgen-perf/comparison.ts @@ -0,0 +1,617 @@ +/** Pure scheduling, work-equivalence, and paired-statistics helpers for docgen comparisons. */ + +import type { EngineId } from '../docgen-shared/engine-ids.ts'; + +export const COMPARISON_PAIRS = [ + { name: 'react', control: 'react-legacy', candidate: 'react-osa' }, + { name: 'vue', control: 'vue-docgen-api', candidate: 'vue-component-meta' }, + { + name: 'vue-component-meta-version', + control: 'vue-component-meta', + candidate: 'vue-component-meta-next', + versionsMustDiffer: true, + }, +] as const satisfies ReadonlyArray<{ + name: string; + control: EngineId; + candidate: EngineId; + versionsMustDiffer?: boolean; +}>; + +export type PairName = (typeof COMPARISON_PAIRS)[number]['name']; + +export type ComparisonSide = 'control' | 'candidate'; + +export interface ComparisonBlock { + /** One-based block number, used to join the two adjacent measurements after they run. */ + block: number; + /** Both sides run once per block; the tuple records which side runs first. */ + order: readonly [ComparisonSide, ComparisonSide]; +} + +export interface ComparisonPlan { + pair: string; + scenario: string; + repetitions: number; + /** Stable seed derived from the caller's seed and this pair/scenario identity. */ + seed: number; + blocks: ComparisonBlock[]; +} + +export interface ComparisonPlanOptions { + seed: number | string; + pair: string; + scenario: string; + /** Number of paired blocks. Each side therefore runs this many times. */ + repetitions: number; +} + +export interface ExecutedComparisonBlock extends ComparisonBlock { + control: Value; + candidate: Value; +} + +/** Executes each block adjacently and publishes only complete two-sided blocks. */ +export async function executeComparisonPlan( + plan: ComparisonPlan, + measure: (side: ComparisonSide, block: number) => Promise +): Promise>> { + const completed: Array> = []; + for (const scheduled of plan.blocks) { + const values: Partial> = {}; + for (const side of scheduled.order) { + values[side] = await measure(side, scheduled.block); + } + completed.push({ + ...scheduled, + control: values.control!, + candidate: values.candidate!, + }); + } + return completed; +} + +function seedPart(value: number | string): string { + if (typeof value === 'number') { + if (!Number.isSafeInteger(value)) { + throw new Error('comparison seed must be a safe integer or a non-empty string'); + } + return `number:${value}`; + } + if (value.length === 0) { + throw new Error('comparison seed must be a safe integer or a non-empty string'); + } + return `string:${value}`; +} + +function namedPart(name: string, value: string): string { + if (value.length === 0) { + throw new Error(`${name} must not be empty`); + } + // Length-prefixing makes identities such as ("a:b", "c") and ("a", "b:c") unambiguous. + return `${value.length}:${value}`; +} + +/** + * Derives an unsigned 32-bit seed without relying on platform-specific hashing or object order. + * Pair/scenario derivation lets suites add or remove unrelated comparisons without perturbing a + * surviving comparison's plan. + */ +export function deriveComparisonSeed( + seed: number | string, + pair: string, + scenario: string +): number { + const identity = `${namedPart('seed', seedPart(seed))}|${namedPart('pair', pair)}|${namedPart( + 'scenario', + scenario + )}`; + let hash = 0x811c9dc5; + for (let i = 0; i < identity.length; i++) { + hash ^= identity.charCodeAt(i); + hash = Math.imul(hash, 0x01000193); + } + return hash >>> 0; +} + +function randomFrom(seed: number): () => number { + let state = seed; + return () => { + state = (state + 0x6d2b79f5) | 0; + let value = state; + value = Math.imul(value ^ (value >>> 15), value | 1); + value ^= value + Math.imul(value ^ (value >>> 7), value | 61); + return ((value ^ (value >>> 14)) >>> 0) / 0x1_0000_0000; + }; +} + +/** Builds deterministic adjacent blocks, balanced within every two-block stratum. */ +export function createComparisonPlan(options: ComparisonPlanOptions): ComparisonPlan { + if ( + !Number.isInteger(options.repetitions) || + options.repetitions < 10 || + options.repetitions % 2 !== 0 + ) { + throw new Error('paired comparisons require an even repetition count of at least 10'); + } + + const seed = deriveComparisonSeed(options.seed, options.pair, options.scenario); + const random = randomFrom(seed); + const firstSides: ComparisonSide[] = []; + for (let stratum = 0; stratum < options.repetitions / 2; stratum++) { + const first = random() < 0.5 ? 'control' : 'candidate'; + firstSides.push(first, first === 'control' ? 'candidate' : 'control'); + } + + return { + pair: options.pair, + scenario: options.scenario, + repetitions: options.repetitions, + seed, + blocks: firstSides.map((first, index) => ({ + block: index + 1, + order: first === 'control' ? ['control', 'candidate'] : ['candidate', 'control'], + })), + }; +} + +export interface WorkSignature { + members?: number; + opaqueTypes?: number; +} + +export interface WarmWorkSignature extends WorkSignature { + /** Present for macro trajectories; two sides must identify the same ordered save. */ + save?: number; +} + +export interface WorkProfile { + /** Required when the pair compares two pins of the same package. */ + version?: string; + cold?: WorkSignature; + /** Kept in save order because a stateful trajectory is not an IID sample set. */ + warm?: readonly WarmWorkSignature[]; +} + +export type WorkStatus = 'same-work' | 'different-work' | 'unknown-work' | 'same-version'; + +export type WorkReason = + | 'matching-signatures' + | 'different-cold-signature' + | 'different-warm-signature' + | 'different-warm-length' + | 'missing-signature' + | 'missing-version' + | 'matching-version'; + +export interface WorkAssessment { + status: WorkStatus; + reason: WorkReason; +} + +export interface WorkAssessmentOptions { + /** Enable for a current/next pair that must resolve distinct package versions. */ + versionsMustDiffer?: boolean; +} + +export type WorkMetric = 'cold' | 'warm'; + +function knownCount(value: number | undefined): value is number { + return value !== undefined && Number.isSafeInteger(value) && value >= 0; +} + +function knownSignature(signature: WorkSignature | undefined): signature is WorkSignature & { + members: number; +} { + return ( + knownCount(signature?.members) && + (signature.opaqueTypes === undefined || knownCount(signature.opaqueTypes)) + ); +} + +function knownWarmSignature( + signature: WarmWorkSignature | undefined +): signature is WarmWorkSignature & { members: number; save: number } { + return ( + signature !== undefined && + knownCount(signature.members) && + (signature.opaqueTypes === undefined || knownCount(signature.opaqueTypes)) && + knownCount(signature.save) && + signature.save > 0 + ); +} + +function compareSignature( + control: WarmWorkSignature & { members: number }, + candidate: WarmWorkSignature & { members: number } +): 'same' | 'different' | 'unknown' { + if ((control.save === undefined) !== (candidate.save === undefined)) { + return 'unknown'; + } + if (control.save !== candidate.save) { + return 'different'; + } + if (control.members !== candidate.members) { + return 'different'; + } + if ((control.opaqueTypes === undefined) !== (candidate.opaqueTypes === undefined)) { + return 'unknown'; + } + return control.opaqueTypes === candidate.opaqueTypes ? 'same' : 'different'; +} + +/** + * Proves equal work from the cold signature and every ordered warm signature. Missing or malformed + * counts remain unknown rather than being interpreted as zero or agreement. + */ +function assessVersion( + control: WorkProfile, + candidate: WorkProfile, + options: WorkAssessmentOptions +): WorkAssessment | undefined { + if (options.versionsMustDiffer) { + if (!control.version || !candidate.version) { + return { status: 'unknown-work', reason: 'missing-version' }; + } + if (control.version === candidate.version) { + return { status: 'same-version', reason: 'matching-version' }; + } + } + return undefined; +} + +function assessColdWork(control: WorkProfile, candidate: WorkProfile): WorkAssessment { + if (!knownSignature(control.cold) || !knownSignature(candidate.cold)) { + return { status: 'unknown-work', reason: 'missing-signature' }; + } + const coldComparison = compareSignature(control.cold, candidate.cold); + if (coldComparison === 'unknown') { + return { status: 'unknown-work', reason: 'missing-signature' }; + } + if (coldComparison === 'different') { + return { status: 'different-work', reason: 'different-cold-signature' }; + } + return { status: 'same-work', reason: 'matching-signatures' }; +} + +function assessWarmWork(control: WorkProfile, candidate: WorkProfile): WorkAssessment { + if (!control.warm || !candidate.warm) { + return { status: 'unknown-work', reason: 'missing-signature' }; + } + if (control.warm.length === 0 || candidate.warm.length === 0) { + return { status: 'unknown-work', reason: 'missing-signature' }; + } + if (control.warm.length !== candidate.warm.length) { + return { status: 'different-work', reason: 'different-warm-length' }; + } + for (let i = 0; i < control.warm.length; i++) { + const controlSignature = control.warm[i]; + const candidateSignature = candidate.warm[i]; + if (!knownWarmSignature(controlSignature) || !knownWarmSignature(candidateSignature)) { + return { status: 'unknown-work', reason: 'missing-signature' }; + } + const warmComparison = compareSignature(controlSignature, candidateSignature); + if (warmComparison === 'unknown') { + return { status: 'unknown-work', reason: 'missing-signature' }; + } + if (warmComparison === 'different') { + return { status: 'different-work', reason: 'different-warm-signature' }; + } + } + return { status: 'same-work', reason: 'matching-signatures' }; +} + +/** Proves equal work for one timing metric, so cold and warm comparisons fail independently. */ +export function assessMetricWork( + control: WorkProfile, + candidate: WorkProfile, + metric: WorkMetric, + options: WorkAssessmentOptions = {} +): WorkAssessment { + return ( + assessVersion(control, candidate, options) ?? + (metric === 'cold' ? assessColdWork(control, candidate) : assessWarmWork(control, candidate)) + ); +} + +/** Proves equal work across the full repetition for callers that need one aggregate status. */ +export function assessWork( + control: WorkProfile, + candidate: WorkProfile, + options: WorkAssessmentOptions = {} +): WorkAssessment { + const version = assessVersion(control, candidate, options); + if (version) { + return version; + } + const cold = assessColdWork(control, candidate); + return cold.status === 'same-work' ? assessWarmWork(control, candidate) : cold; +} + +export interface PairedTiming { + block: number; + controlMs?: number; + candidateMs?: number; +} + +export type InvalidEffectReason = + | 'incomplete-data' + | 'nonfinite-data' + | 'nonpositive-data' + | 'insufficient-pairs'; + +export interface InvalidPairedEffect { + status: 'invalid'; + reason: InvalidEffectReason; +} + +export interface PairedEffect { + status: 'measured'; + pairs: number; + /** One value per block. Positive values mean the candidate was slower. */ + logRatios: number[]; + meanLogRatio: number; + standardError: number; + criticalValue95: number; + candidateOverControl: { + estimate: number; + lower95: number; + upper95: number; + }; +} + +export type PairedEffectResult = PairedEffect | InvalidPairedEffect; + +export interface PairedEffectOptions { + /** Detects a child that stopped before every scheduled block completed. */ + expectedPairs?: number; +} + +// Two-sided 95% Student-t critical values. Larger degrees of freedom use a convergent expansion. +const T_CRITICAL_95 = [ + Number.NaN, + 12.7062047364, + 4.30265272975, + 3.18244630528, + 2.7764451052, + 2.57058183564, + 2.44691184879, + 2.36462425101, + 2.3060041352, + 2.2621571628, + 2.22813885196, + 2.20098516008, + 2.17881282966, + 2.16036865646, + 2.14478668792, + 2.13144954556, + 2.11990529922, + 2.10981557783, + 2.10092204024, + 2.09302405441, + 2.08596344727, + 2.07961384473, + 2.0738730679, + 2.06865761042, + 2.06389856163, + 2.05953855275, + 2.05552943864, + 2.05183051648, + 2.0484071418, + 2.04522964213, + 2.0422724563, +]; + +export function studentTCritical95(degreesOfFreedom: number): number { + if (!Number.isInteger(degreesOfFreedom) || degreesOfFreedom < 1) { + throw new Error('Student-t degrees of freedom must be a positive integer'); + } + if (degreesOfFreedom < T_CRITICAL_95.length) { + return T_CRITICAL_95[degreesOfFreedom]; + } + + const v = degreesOfFreedom; + const z = 1.959963984540054; + const z2 = z * z; + const z3 = z2 * z; + const z5 = z3 * z2; + const z7 = z5 * z2; + const z9 = z7 * z2; + return ( + z + + (z3 + z) / (4 * v) + + (5 * z5 + 16 * z3 + 3 * z) / (96 * v ** 2) + + (3 * z7 + 19 * z5 + 17 * z3 - 15 * z) / (384 * v ** 3) + + (79 * z9 + 776 * z7 + 1482 * z5 - 1920 * z3 - 945 * z) / (92_160 * v ** 4) + ); +} + +/** Computes the geometric candidate/control effect and a two-sided 95% Student-t interval. */ +export function computePairedEffect( + timings: readonly PairedTiming[], + options: PairedEffectOptions = {} +): PairedEffectResult { + if ( + options.expectedPairs !== undefined && + (!Number.isInteger(options.expectedPairs) || + options.expectedPairs < 1 || + timings.length !== options.expectedPairs) + ) { + return { status: 'invalid', reason: 'incomplete-data' }; + } + if (timings.length < 2) { + return { status: 'invalid', reason: 'insufficient-pairs' }; + } + + const blocks = new Set(); + const logRatios: number[] = []; + for (const timing of timings) { + if ( + !Number.isSafeInteger(timing.block) || + timing.block < 1 || + blocks.has(timing.block) || + timing.controlMs === undefined || + timing.candidateMs === undefined + ) { + return { status: 'invalid', reason: 'incomplete-data' }; + } + blocks.add(timing.block); + if (!Number.isFinite(timing.controlMs) || !Number.isFinite(timing.candidateMs)) { + return { status: 'invalid', reason: 'nonfinite-data' }; + } + if (timing.controlMs <= 0 || timing.candidateMs <= 0) { + return { status: 'invalid', reason: 'nonpositive-data' }; + } + // Subtracting logs avoids overflowing candidate/control before taking its logarithm. + const logRatio = Math.log(timing.candidateMs) - Math.log(timing.controlMs); + if (!Number.isFinite(logRatio)) { + return { status: 'invalid', reason: 'nonfinite-data' }; + } + logRatios.push(logRatio); + } + for (let block = 1; block <= timings.length; block++) { + if (!blocks.has(block)) { + return { status: 'invalid', reason: 'incomplete-data' }; + } + } + + const meanLogRatio = logRatios.reduce((sum, value) => sum + value, 0) / logRatios.length; + const squaredError = logRatios.reduce((sum, value) => sum + (value - meanLogRatio) ** 2, 0); + const standardDeviation = Math.sqrt(squaredError / (logRatios.length - 1)); + const standardError = standardDeviation / Math.sqrt(logRatios.length); + const criticalValue95 = studentTCritical95(logRatios.length - 1); + const margin = criticalValue95 * standardError; + const estimate = Math.exp(meanLogRatio); + const lower95 = Math.exp(meanLogRatio - margin); + const upper95 = Math.exp(meanLogRatio + margin); + if ( + !Number.isFinite(meanLogRatio) || + !Number.isFinite(standardError) || + !Number.isFinite(estimate) || + !Number.isFinite(lower95) || + !Number.isFinite(upper95) + ) { + return { status: 'invalid', reason: 'nonfinite-data' }; + } + + return { + status: 'measured', + pairs: logRatios.length, + logRatios, + meanLogRatio, + standardError, + criticalValue95, + candidateOverControl: { estimate, lower95, upper95 }, + }; +} + +export interface PairedComparisonInput { + controlWork: WorkProfile; + candidateWork: WorkProfile; + timings: readonly PairedTiming[]; + expectedPairs?: number; + versionsMustDiffer?: boolean; + /** Omit only when a caller intentionally requires the full cold-and-warm work profile. */ + metric?: WorkMetric; +} + +export interface PairedComparison { + work: WorkAssessment; + /** Deliberately absent unless the two sides proved they performed identical work. */ + effect?: PairedEffectResult; +} + +/** Assesses work before calculating any timing effect. */ +export function comparePairedTimings(input: PairedComparisonInput): PairedComparison { + const work = input.metric + ? assessMetricWork(input.controlWork, input.candidateWork, input.metric, { + versionsMustDiffer: input.versionsMustDiffer, + }) + : assessWork(input.controlWork, input.candidateWork, { + versionsMustDiffer: input.versionsMustDiffer, + }); + return work.status === 'same-work' + ? { + work, + effect: computePairedEffect(input.timings, { expectedPairs: input.expectedPairs }), + } + : { work }; +} + +export type BudgetStatus = + | 'not-configured' + | 'smoke' + | 'within-budget' + | 'inconclusive' + | 'regression' + | 'invalid-gate'; + +export type InvalidGateReason = + | 'invalid-budget' + | 'invalid-repetition-count' + | 'work-not-comparable' + | 'missing-effect' + | 'invalid-effect' + | 'incomplete-effect'; + +export interface BudgetVerdict { + status: BudgetStatus; + /** Fractional limit: 0.1 means the candidate may be at most 10% slower. */ + maxRegression?: number; + maxCandidateOverControl?: number; + reason?: InvalidGateReason; +} + +export interface BudgetOptions { + /** Number of blocks planned for each side. */ + repetitions: number; + /** Fractional slowdown, such as 0.1 for 10%. Undefined keeps the run descriptive. */ + maxRegression?: number; + /** Smoke results are never suitable for a timing verdict, even if a budget was supplied. */ + smoke?: boolean; +} + +/** Applies an explicit budget; only a confidence interval wholly beyond it is a regression. */ +export function evaluateBudget( + comparison: PairedComparison, + options: BudgetOptions +): BudgetVerdict { + if (options.maxRegression === undefined) { + return { status: 'not-configured' }; + } + if (!Number.isFinite(options.maxRegression) || options.maxRegression < 0) { + return { status: 'invalid-gate', reason: 'invalid-budget' }; + } + if ( + !Number.isInteger(options.repetitions) || + options.repetitions < 10 || + options.repetitions % 2 !== 0 + ) { + return { status: 'invalid-gate', reason: 'invalid-repetition-count' }; + } + if (comparison.work.status !== 'same-work') { + return { status: 'invalid-gate', reason: 'work-not-comparable' }; + } + if (!comparison.effect) { + return { status: 'invalid-gate', reason: 'missing-effect' }; + } + if (comparison.effect.status === 'invalid') { + return { status: 'invalid-gate', reason: 'invalid-effect' }; + } + if (comparison.effect.pairs !== options.repetitions) { + return { status: 'invalid-gate', reason: 'incomplete-effect' }; + } + + const maxCandidateOverControl = 1 + options.maxRegression; + const base = { maxRegression: options.maxRegression, maxCandidateOverControl }; + if (options.smoke) { + return { status: 'smoke', ...base }; + } + if (comparison.effect.candidateOverControl.lower95 > maxCandidateOverControl) { + return { status: 'regression', ...base }; + } + if (comparison.effect.candidateOverControl.upper95 <= maxCandidateOverControl) { + return { status: 'within-budget', ...base }; + } + return { status: 'inconclusive', ...base }; +} diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/config.ts b/code/lib/docgen-harness/src/perf/docgen-perf/config.ts index 170867a63516..e71559cd029b 100644 --- a/code/lib/docgen-harness/src/perf/docgen-perf/config.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/config.ts @@ -1,26 +1,18 @@ /** * Fixed measurement parameters for the per-engine docgen performance suite. * - * N is pinned here for every engine and recorded with the results; numbers taken at different N are - * not comparable. The --quick profile exists for smoke runs only and marks its results - * non-comparable. + * The main suite is descriptive. Its fixed profiles control runtime, while an opt-in paired gate + * validates and records its own repetition count. */ -/** - * Fresh-process spawns per cold/scan median. One value for all engines. - * - * Must stay even. The two sides of a control pair alternate which one runs first on odd and even - * repetitions, so an odd N gives one side the first slot once more than the other, and the cold - * figure - a median - then lands on a repetition from the majority slot. That turns the ordering - * effect the alternation exists to cancel into a systematic, directional bias on the headline ratio. - */ -export const PINNED_N = 6; +/** Fresh-process repetitions in a full descriptive run. */ +export const DEFAULT_REPETITIONS = 6; -/** Spawns for --quick smoke runs. Never comparable with PINNED_N results. */ -export const QUICK_N = 2; +/** Fresh-process repetitions for --quick smoke runs. */ +export const QUICK_REPETITIONS = 2; -/** Sampling interval for the compodoc child's externally-polled peak RSS. */ -export const RSS_POLL_INTERVAL_MS = 100; +/** Minimum block count for an opt-in paired timing gate. */ +export const MIN_PAIRED_REPETITIONS = 10; /** * How long one compodoc run may take before it is killed. Compodoc can hang rather than exit on @@ -52,16 +44,14 @@ export interface AngularScenarioConfig { } export interface SuiteProfile { - n: number; - comparable: boolean; + repetitions: number; react: ReactScenarioConfig; vue: VueScenarioConfig[]; angular: AngularScenarioConfig; } export const DEFAULT_PROFILE: SuiteProfile = { - n: PINNED_N, - comparable: true, + repetitions: DEFAULT_REPETITIONS, react: { components: 300, variants: 4, props: 10, saves: 20 }, vue: [ { @@ -96,8 +86,7 @@ export const DEFAULT_PROFILE: SuiteProfile = { }; export const QUICK_PROFILE: SuiteProfile = { - n: QUICK_N, - comparable: false, + repetitions: QUICK_REPETITIONS, react: { components: 20, variants: 2, props: 4, saves: 4 }, vue: [ { diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/engine.ts b/code/lib/docgen-harness/src/perf/docgen-perf/engine.ts index 56cd78e8bc88..1b2de6628ac9 100644 --- a/code/lib/docgen-harness/src/perf/docgen-perf/engine.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/engine.ts @@ -1,16 +1,13 @@ -/** - * What the orchestrator knows about an engine. Engines differ only in how one repetition is - * produced; everything downstream (aggregation, member counts) follows from that choice. - */ +/** Engine registry contract for one fresh-process latency repetition. */ import * as fs from 'node:fs'; import { createRequire } from 'node:module'; import * as path from 'node:path'; -import type { SeriesResult } from '../docgen-shared/series.ts'; -import { designatedRep, seriesMetrics } from './aggregate.ts'; +import type { LatencyRepetition } from '../docgen-shared/latency-series.ts'; import type { SuiteProfile } from './config.ts'; -import type { SeriesChildSpec } from './spawn.ts'; -import type { EngineId, EngineMetrics, MemberCounts, ScenarioResult } from './types.ts'; +import type { LatencyChildSpec } from './spawn.ts'; +import { assertCompleteLatencyRepetitions, summarizeLatencyRepetitions } from './summary.ts'; +import type { EngineId, ScenarioResult } from './types.ts'; const require = createRequire(import.meta.url); @@ -25,23 +22,16 @@ function resolvePackageVersion(packageName: string): string | undefined { export interface ScenarioSpec { name: string; - /** Recorded verbatim in the results so a stored run is self-describing. */ + /** Recorded verbatim so a stored run is self-describing. */ params: Record; } -/** Only what every engine needs. What one engine alone cares about stays inside that engine. */ export interface MeasureContext { - /** Scratch directory for this engine/scenario: generated project and per-repetition JSON. */ scenarioDir: string; - runSeriesChild(spec: SeriesChildSpec, outDir: string, jsonPath: string): SeriesResult; + runLatencyChild(spec: LatencyChildSpec, outDir: string, jsonPath: string): LatencyRepetition; } -/** - * `Sample` is whatever one repetition of this engine produces. Every engine's sample carries the - * cold-pass duration, because that is what picks the one repetition every single-run figure is read - * from. - */ -export abstract class BenchEngine { +export abstract class BenchEngine { abstract readonly id: EngineId; /** Engines outside the default run only measure when named with `--engine`. */ @@ -49,44 +39,33 @@ export abstract class BenchEngine; + abstract measure( + ctx: MeasureContext, + scenario: ScenarioSpec, + repetition: number + ): Promise; - abstract aggregate(samples: Sample[], expectedN: number): EngineMetrics; - - /** - * Anything that must resolve before the run starts. A returned string is the skip reason: a - * missing external tool is a partial install, not a regression, so it is reported as skipped - * rather than failed. - */ preflight(): string | undefined { return undefined; } - /** The resolved version of an externally installed engine, recorded with the results. */ version(): string | undefined { return undefined; } - /** - * What one repetition documented. An engine that cannot report a count leaves it out, which is - * what keeps a missing count from being read as a measured zero. The counts an engine does report - * are what establishes whether a ratio against it compared equal work. - */ - members(_sample: Sample): MemberCounts { - return {}; - } - - /** - * One scenario's complete result. `Sample` is still bound here, so the orchestrator never has to - * name - or cast to - the sample shape a particular engine happens to produce. - */ - assemble(samples: Sample[], expectedN: number, scenario: ScenarioSpec): ScenarioResult { - // aggregate() first: it is what rejects a run that never reached the pinned N, and there is - // nothing for designatedRep to pick from until that has passed. - const metrics = this.aggregate(samples, expectedN); - // The counts are read from the same repetition as the warm and memory metrics, so every figure - // reported for a scenario describes one run. - return { params: scenario.params, metrics, ...this.members(designatedRep(samples)) }; + assemble( + repetitions: LatencyRepetition[], + expectedRepetitions: number, + scenario: ScenarioSpec + ): ScenarioResult { + const expectedWarmObservations = + typeof scenario.params.saves === 'number' ? scenario.params.saves : undefined; + assertCompleteLatencyRepetitions(repetitions, expectedRepetitions, expectedWarmObservations); + return { + params: scenario.params, + repetitions, + summary: summarizeLatencyRepetitions(repetitions), + }; } } @@ -96,19 +75,13 @@ export interface SeriesChildConfig { child: string; scenarios(profile: SuiteProfile): ScenarioSpec[]; args(scenario: ScenarioSpec): string[]; - /** Only the reused docgen-memory harness runs under the jiti loader. */ + /** The React OSA source child still requires the jiti loader. */ jiti?: boolean; inDefaultRun?: boolean; - /** The npm package name whose resolved `package.json#version` should be reported for this engine. */ versionPackage?: string; } -/** - * An engine measured by spawning a series-harness child, one fresh process per repetition. All - * engines of this kind share the series harness, so aggregation and member counts are the same - * for each. - */ -export class SeriesChildEngine extends BenchEngine { +export class SeriesChildEngine extends BenchEngine { readonly id: EngineId; readonly #config: SeriesChildConfig; @@ -123,12 +96,6 @@ export class SeriesChildEngine extends BenchEngine { return this.#config.scenarios(profile); } - /** - * A declared `versionPackage` is one the child imports, so failing to resolve it means a partial - * install. Skipping here rather than letting `version()` quietly answer undefined is what keeps a - * version pair from running with no way to tell its two installs apart - the one failure that - * comparison exists to catch. - */ preflight(): string | undefined { const { versionPackage } = this.#config; if (versionPackage && resolvePackageVersion(versionPackage) === undefined) { @@ -143,23 +110,20 @@ export class SeriesChildEngine extends BenchEngine { : undefined; } - async measure(ctx: MeasureContext, scenario: ScenarioSpec, rep: number): Promise { - return ctx.runSeriesChild( + async measure( + ctx: MeasureContext, + scenario: ScenarioSpec, + repetition: number + ): Promise { + return ctx.runLatencyChild( { childPath: path.join(import.meta.dirname, this.#config.child), args: this.#config.args(scenario), + expectedWorkload: { ...scenario.params }, jiti: this.#config.jiti, }, path.join(ctx.scenarioDir, 'project'), - path.join(ctx.scenarioDir, `rep${rep}.json`) + path.join(ctx.scenarioDir, `rep${repetition}.json`) ); } - - aggregate(samples: SeriesResult[], expectedN: number): EngineMetrics { - return seriesMetrics(samples, expectedN); - } - - members(sample: SeriesResult): MemberCounts { - return { coldMembers: sample.coldMembers, warmMembers: sample.warmMembers }; - } } diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/engines/compodoc.ts b/code/lib/docgen-harness/src/perf/docgen-perf/engines/compodoc.ts index 91ccda77369b..7b166ebb4a41 100644 --- a/code/lib/docgen-harness/src/perf/docgen-perf/engines/compodoc.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/engines/compodoc.ts @@ -1,25 +1,19 @@ -import { spawn, spawnSync } from 'node:child_process'; +import { spawn } from 'node:child_process'; import * as fs from 'node:fs'; import { createRequire } from 'node:module'; import * as path from 'node:path'; import { outputTail } from '../../docgen-shared/child-output.ts'; -import { type OneShotRepetition, oneShotMetrics } from '../aggregate.ts'; -import { type DocumentationCounts, countDocumentation } from './compodoc-doc.ts'; -import { - type AngularScenarioConfig, - COMPODOC_TIMEOUT_MS, - RSS_POLL_INTERVAL_MS, - type SuiteProfile, -} from '../config.ts'; +import type { LatencyObservation, LatencyRepetition } from '../../docgen-shared/latency-series.ts'; +import { COMPODOC_TIMEOUT_MS, type AngularScenarioConfig, type SuiteProfile } from '../config.ts'; import { BenchEngine, type MeasureContext, type ScenarioSpec } from '../engine.ts'; +import type { EngineId } from '../types.ts'; +import { type DocumentationCounts, countDocumentation } from './compodoc-doc.ts'; import { angularComponentSource, generateAngularProject } from '../generators/angular.ts'; -import type { EngineId, EngineMetrics, MemberCounts } from '../types.ts'; const require = createRequire(import.meta.url); interface ResolvedCompodoc { - /** The CLI entry point, run as `node ` so no shell shim or exec bit is involved. */ cli: string; version: string; } @@ -27,7 +21,10 @@ interface ResolvedCompodoc { function resolveCompodoc(): ResolvedCompodoc | undefined { try { const packagePath = require.resolve('@compodoc/compodoc/package.json'); - const pkg = JSON.parse(fs.readFileSync(packagePath, 'utf8')); + const pkg = JSON.parse(fs.readFileSync(packagePath, 'utf8')) as { + bin: { compodoc: string }; + version: string; + }; return { cli: path.join(path.dirname(packagePath), pkg.bin.compodoc), version: pkg.version, @@ -38,69 +35,37 @@ function resolveCompodoc(): ResolvedCompodoc | undefined { } interface CompodocRun extends DocumentationCounts { - durMs: number; - /** - * Undefined when no poll ever read the child's RSS - `ps` is POSIX-only, and a run shorter than - * one interval is never sampled. Reporting the initial 0 instead would put a fabricated peak in - * the results, which is the one thing a floor may not become. - */ - peakRssMb: number | undefined; + durationMs: number; } function runCompodocOnce( cli: string, projectDir: string, - docsOutDir: string, - pollIntervalMs: number + docsOutDir: string ): Promise { fs.rmSync(docsOutDir, { recursive: true, force: true }); const args = [cli, '-p', 'tsconfig.json', '-e', 'json', '-d', docsOutDir, '--silent']; return new Promise((resolve, reject) => { - const start = Date.now(); + const start = performance.now(); const child = spawn(process.execPath, args, { cwd: projectDir }); - let peakRssKb: number | undefined; let output = ''; let timedOut = false; child.stdout.on('data', (chunk: Buffer) => (output += chunk.toString())); child.stderr.on('data', (chunk: Buffer) => (output += chunk.toString())); - // Killed rather than waited on: the close handler below is the only thing that settles this - // promise, so a compodoc that hangs would otherwise stall the whole suite with no way out. const timeout = setTimeout(() => { timedOut = true; child.kill('SIGKILL'); }, COMPODOC_TIMEOUT_MS); - const poll = setInterval(() => { - if (child.pid === undefined) { - return; - } - - const ps = spawnSync('ps', ['-o', 'rss=', '-p', String(child.pid)], { encoding: 'utf8' }); - if (ps.error || typeof ps.stdout !== 'string') { - return; - } - const rssKb = Number(ps.stdout.trim()); - if (Number.isFinite(rssKb) && (peakRssKb === undefined || rssKb > peakRssKb)) { - peakRssKb = rssKb; - } - }, pollIntervalMs); - - const stopWatching = () => { - clearInterval(poll); - clearTimeout(timeout); - }; - - child.on('error', (err) => { + const stopWatching = () => clearTimeout(timeout); + child.on('error', (error) => { stopWatching(); - reject(err); + reject(error); }); child.on('close', (status) => { stopWatching(); - const durMs = Date.now() - start; - // Before the status check: a killed child closes with a null status, which would otherwise be - // reported as an ordinary non-zero exit and hide why the run really ended. if (timedOut) { reject( new Error( @@ -118,28 +83,25 @@ function runCompodocOnce( reject(new Error('compodoc run produced no documentation.json')); return; } - // Counted before the next run overwrites the file, which both runs share. - const counts = countDocumentation(documentationJson); - // The polled peak misses spikes shorter than the interval; the recorded value is a floor. - resolve({ - durMs, - peakRssMb: peakRssKb === undefined ? undefined : peakRssKb / 1024, - ...counts, - }); + resolve({ durationMs: performance.now() - start, ...countDocumentation(documentationJson) }); }); }); } -/** - * One repetition: fresh project, cold full run, touch one component, warm full run. Both runs are - * fresh compodoc processes, matching the one-sample-per-fresh-process topology. - */ +function observation(run: CompodocRun): LatencyObservation { + return { + durationMs: run.durationMs, + members: run.members, + opaqueTypes: run.opaqueTypes, + }; +} + +/** One repetition uses fresh Compodoc processes for both full-project passes. */ export async function runCompodocRepetition( cli: string, scenario: AngularScenarioConfig, - workDir: string, - pollIntervalMs: number -): Promise { + workDir: string +): Promise { const projectDir = path.join(workDir, 'project'); const docsOutDir = path.join(workDir, 'docs'); const project = generateAngularProject({ @@ -148,39 +110,20 @@ export async function runCompodocRepetition( props: scenario.props, }); - const cold = await runCompodocOnce(cli, project.outDir, docsOutDir, pollIntervalMs); - - // Touch one component so the warm run sees a genuinely changed file. + const cold = observation(await runCompodocOnce(cli, project.outDir, docsOutDir)); fs.writeFileSync(project.componentPaths[0], angularComponentSource(0, scenario.props + 1)); - const warm = await runCompodocOnce(cli, project.outDir, docsOutDir, pollIntervalMs); + const warm = observation(await runCompodocOnce(cli, project.outDir, docsOutDir)); - const peaks = [cold.peakRssMb, warm.peakRssMb].filter((mb) => mb !== undefined); - - return { - coldMs: cold.durMs, - warmMs: warm.durMs, - // Nothing sampled means no peak, not a peak of zero. - peakRssMb: peaks.length ? Math.max(...peaks) : undefined, - coldMembers: cold.members, - // A second whole-project pass, so this counts the project, not the one file that changed. It is - // not comparable with a series engine's warm count, which covers only the re-extracted member. - warmMembers: warm.members, - coldOpaqueTypes: cold.opaqueTypes, - }; + return { cold, scan: cold, warm: [{ save: 1, ...warm }] }; } -export class CompodocEngine extends BenchEngine { +export class CompodocEngine extends BenchEngine { readonly id: EngineId = 'compodoc'; #resolved: ResolvedCompodoc | undefined; scenarios(profile: SuiteProfile): ScenarioSpec[] { - // The poll interval rides in params because it qualifies the peak this engine reports: the - // sample misses spikes shorter than the gap between polls, so the recorded peak is a floor and - // reading it later means knowing how wide that gap was. - return [ - { name: 'default', params: { ...profile.angular, rssPollIntervalMs: RSS_POLL_INTERVAL_MS } }, - ]; + return [{ name: 'default', params: { ...profile.angular } }]; } preflight(): string | undefined { @@ -194,29 +137,14 @@ export class CompodocEngine extends BenchEngine { return this.#resolved?.version; } - async measure(ctx: MeasureContext, scenario: ScenarioSpec): Promise { + async measure(ctx: MeasureContext, scenario: ScenarioSpec): Promise { if (!this.#resolved) { throw new Error('compodoc unresolved; preflight must run first'); } return runCompodocRepetition( this.#resolved.cli, { components: Number(scenario.params.components), props: Number(scenario.params.props) }, - ctx.scenarioDir, - // Read from the constant, not back out of `params`. `params` carries it so a stored run says - // how wide the sampling gap was; the run itself has no reason to round-trip through it. - RSS_POLL_INTERVAL_MS + ctx.scenarioDir ); } - - aggregate(samples: OneShotRepetition[], expectedN: number): EngineMetrics { - return oneShotMetrics(samples, expectedN); - } - - members(sample: OneShotRepetition): MemberCounts { - return { - coldMembers: sample.coldMembers, - warmMembers: sample.warmMembers, - coldOpaqueTypes: sample.coldOpaqueTypes, - }; - } } diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/engines/react-legacy.ts b/code/lib/docgen-harness/src/perf/docgen-perf/engines/react-legacy.ts index b02d19af96ee..6f0dde02c3b6 100644 --- a/code/lib/docgen-harness/src/perf/docgen-perf/engines/react-legacy.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/engines/react-legacy.ts @@ -1,6 +1,6 @@ /** - * Series harness for the legacy React docgen engines: `react-docgen` (the budgeted legacy control) - * and `react-docgen-typescript` (measurable, no budget row). + * Latency harness for the legacy React docgen engines: `react-docgen` (the default legacy control) + * and `react-docgen-typescript` (an opt-in diagnostic engine). * * Both cache per file for the process lifetime and expose only global invalidation, so every save * must invalidate before re-extracting or the sample is a cache hit. @@ -9,7 +9,7 @@ * require under that loader ("jsReleases.map is not a function"). * * Run from code/lib/docgen-harness: - * node --expose-gc src/perf/docgen-perf/engines/react-legacy.ts \ + * node src/perf/docgen-perf/engines/react-legacy.ts \ * --parser react-docgen --components 300 --saves 20 --json /tmp/result.json */ import * as fs from 'node:fs'; @@ -25,11 +25,12 @@ import { } from '../../docgen-memory/generate-project.ts'; import { countOption, parseHarnessOptions } from '../../docgen-shared/args.ts'; import { SANDBOX_DIRECTORY } from '../../docgen-shared/paths.ts'; +import { runLatencySeriesHarness } from '../../docgen-shared/latency-series.ts'; import { type ComponentRefLike, loadReactRendererModule, } from '../../docgen-shared/react-renderer-module.ts'; -import { type SeriesEngine, harnessMain, runSeriesHarness } from '../../docgen-shared/series.ts'; +import { type SeriesEngine, harnessMain } from '../../docgen-shared/series.ts'; /** The renderer module surfaces this harness loads, narrowed to what it calls. */ interface UtilsModule { @@ -185,7 +186,7 @@ async function createEngine(options: HarnessOptions): Promise { harnessMain(async () => { const options = parseOptions(process.argv.slice(2)); - await runSeriesHarness({ + await runLatencySeriesHarness({ title: `react-legacy harness (${options.parser})`, options, banner: { diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/engines/vue-component-meta.ts b/code/lib/docgen-harness/src/perf/docgen-perf/engines/vue-component-meta.ts index 87b0ecb79427..abbca8c3bb96 100644 --- a/code/lib/docgen-harness/src/perf/docgen-perf/engines/vue-component-meta.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/engines/vue-component-meta.ts @@ -1,5 +1,5 @@ /** - * Series harness for the vue-component-meta engine - Vue's opt-in successor to vue-docgen-api. + * Latency harness for the vue-component-meta engine - Vue's opt-in successor to vue-docgen-api. * * The checker never re-stats files on its own; a disk rewrite must be followed by * `checker.updateFile(path, content)` or the re-extraction measures a stale-cache no-op. Checker @@ -7,16 +7,18 @@ * `createCheckerByJson`, the workspace scenarios use `createChecker` on the deepest package. * * Run from code/lib/docgen-harness: - * node --expose-gc src/perf/docgen-perf/engines/vue-component-meta.ts \ + * node src/perf/docgen-perf/engines/vue-component-meta.ts \ * --scenario workspace --packages 4 --components-per-package 10 --heavy-lib --json /tmp/result.json */ import * as fs from 'node:fs'; +import { pathToFileURL } from 'node:url'; import type { MetaCheckerOptions } from 'vue-component-meta'; import { z } from 'zod'; import { parseHarnessOptions } from '../../docgen-shared/args.ts'; -import { type SeriesEngine, harnessMain, runSeriesHarness } from '../../docgen-shared/series.ts'; +import { runLatencySeriesHarness } from '../../docgen-shared/latency-series.ts'; +import { type SeriesEngine, harnessMain } from '../../docgen-shared/series.ts'; import { VUE_OPTIONS, VUE_SCHEMA, @@ -29,7 +31,7 @@ import { } from './vue-scenario.ts'; /** Both pins are the same package, so the current one's types describe either install. */ -type CheckerModule = typeof import('vue-component-meta'); +export type CheckerModule = typeof import('vue-component-meta'); type Checker = ReturnType; /** Mirrors the production Vite plugin's checker options. */ @@ -40,7 +42,7 @@ const CHECKER_OPTIONS: MetaCheckerOptions = { }; const PINS = ['current', 'next'] as const; -type Pin = (typeof PINS)[number]; +export type Pin = (typeof PINS)[number]; /** Each pin is a separate install, so each gets its own scratch directory. */ const PIN_DIR: Record = { @@ -66,14 +68,19 @@ function parseOptions(argv: string[]): VueHarnessOptions & { pin: Pin } { } /** - * Only the pin being measured is loaded. Importing both copies would leave the other one's module - * graph on the heap of every run, including the runs that feed the standing vue pair, which would - * shift a memory number that has nothing to do with this comparison. + * Only the pin being measured is loaded. Importing both copies would initialize and retain the + * other implementation before a fresh-process observation that is meant to exercise one pin. */ -function loadCheckers(pin: Pin): Promise { +export function loadCheckers(pin: Pin): Promise { return pin === 'next' ? import('vue-component-meta-next') : import('vue-component-meta'); } +export interface VueComponentMetaEngine extends SeriesEngine { + cold(): number; + applySave(save: number): void; + reextract(): number; +} + /** * Both checker calls stay inside the timed path on purpose, re-extraction included. The production * plugin's `transform` hook runs `getExportNames` then `getComponentMeta` on every transform, so @@ -95,14 +102,16 @@ function extractOne(checker: Checker, sfcPath: string): number { return meta.props.length + meta.events.length + meta.slots.length + meta.exposed.length; } -async function createEngine(options: VueHarnessOptions, pin: Pin): Promise { +export function createVueComponentMetaEngine( + options: VueHarnessOptions, + fns: CheckerModule +): VueComponentMetaEngine { const scenario = setUpVueScenario(options); - const fns = await loadCheckers(pin); let checker: Checker | undefined; let measuredPath = scenario.targetPaths[0]; return { - async cold() { + cold() { checker = options.scenario === 'flat' ? fns.createCheckerByJson(scenario.project.outDir, { include: ['**/*'] }, CHECKER_OPTIONS) @@ -116,27 +125,35 @@ async function createEngine(options: VueHarnessOptions, pin: Pin): Promise { - const options = parseOptions(process.argv.slice(2)); - await runSeriesHarness({ - title: `vue-component-meta harness (${options.scenario}, pin=${options.pin})`, - options, - banner: vueBanner(options), - saves: options.saves, - coldLabel: `${options.componentsPerPackage} components, checker creation included`, - jsonOut: options.jsonOut, - setup: async () => createEngine(options, options.pin), +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + harnessMain(async () => { + const options = parseOptions(process.argv.slice(2)); + const checkers = await loadCheckers(options.pin); + await runLatencySeriesHarness({ + title: `vue-component-meta harness (${options.scenario}, pin=${options.pin})`, + options, + banner: vueBanner(options), + saves: options.saves, + coldLabel: `${options.componentsPerPackage} components, checker creation included`, + jsonOut: options.jsonOut, + setup: () => createVueComponentMetaEngine(options, checkers), + }); }); -}); +} diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/engines/vue-docgen-api.ts b/code/lib/docgen-harness/src/perf/docgen-perf/engines/vue-docgen-api.ts index 29cab8eb0fb9..dad5b9e51ed0 100644 --- a/code/lib/docgen-harness/src/perf/docgen-perf/engines/vue-docgen-api.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/engines/vue-docgen-api.ts @@ -1,5 +1,5 @@ /** - * Series harness for the vue-docgen-api engine - Vue's legacy docgen and still the default + * Latency harness for the vue-docgen-api engine - Vue's legacy docgen and still the default * (`resolveDocgenOptions` in `code/frameworks/vue3-vite/src/preset.ts`), the Vue equivalent of the * react-docgen control. * @@ -7,14 +7,15 @@ * needs no invalidation step. * * Run from code/lib/docgen-harness: - * node --expose-gc src/perf/docgen-perf/engines/vue-docgen-api.ts \ + * node src/perf/docgen-perf/engines/vue-docgen-api.ts \ * --scenario workspace --packages 4 --components-per-package 10 --heavy-lib --json /tmp/result.json */ import * as fs from 'node:fs'; import { parse } from 'vue-docgen-api'; -import { type SeriesEngine, harnessMain, runSeriesHarness } from '../../docgen-shared/series.ts'; +import { runLatencySeriesHarness } from '../../docgen-shared/latency-series.ts'; +import { type SeriesEngine, harnessMain } from '../../docgen-shared/series.ts'; import { type VueHarnessOptions, parseVueOptions, @@ -61,7 +62,7 @@ function createEngine(options: VueHarnessOptions): SeriesEngine { harnessMain(async () => { const options = parseVueOptions(process.argv.slice(2), 'vue-docgen-api'); - await runSeriesHarness({ + await runLatencySeriesHarness({ title: `vue-docgen-api harness (${options.scenario})`, options, banner: vueBanner(options), diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/engines/vue-scenario.ts b/code/lib/docgen-harness/src/perf/docgen-perf/engines/vue-scenario.ts index 54cdb50c476d..5e34692cfe30 100644 --- a/code/lib/docgen-harness/src/perf/docgen-perf/engines/vue-scenario.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/engines/vue-scenario.ts @@ -1,8 +1,8 @@ /** * The scenario plumbing both Vue engine harnesses share: option parsing, project generation, the * measured component set, and the per-save mutation. Both engines must see identical projects and - * touch the same files in the same order, or the ratio between their medians measures the generator - * instead of the engines. + * touch the same files in the same order, or a paired effect measures different workloads instead + * of the engines. * * Scenarios: * flat single package; the measured set is every component. diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/ratios.test.ts b/code/lib/docgen-harness/src/perf/docgen-perf/ratios.test.ts deleted file mode 100644 index 71b595579c22..000000000000 --- a/code/lib/docgen-harness/src/perf/docgen-perf/ratios.test.ts +++ /dev/null @@ -1,198 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import { PINNED_N } from './config.ts'; -import { CONTROL_PAIRS, computeRatios, engineOrderForRep } from './ratios.ts'; -import { type EngineId, type EngineResult, NOT_APPLICABLE, type ScenarioResult } from './types.ts'; - -function scenario( - cold: number, - warm: number, - members?: [number, number], - coldOpaqueTypes?: number -): ScenarioResult { - return { - params: {}, - coldMembers: members?.[0], - warmMembers: members?.[1], - coldOpaqueTypes, - metrics: { - coldExtractionMs: { status: 'measured', samples: [cold], median: cold }, - warmExtractionMs: { status: 'measured', samples: [warm], median: warm }, - wholeProjectScanMs: NOT_APPLICABLE, - peakTransientMb: NOT_APPLICABLE, - retainedGrowthMb: NOT_APPLICABLE, - retainedSlopeMbPerSave: NOT_APPLICABLE, - }, - }; -} - -function measured(scenarios: Record): EngineResult { - return { status: 'measured', scenarios }; -} - -describe('engineOrderForRep', () => { - const engines: EngineId[] = ['react-legacy', 'react-osa', 'vue-docgen-api', 'vue-component-meta']; - - /** - * Every engine named by a control pair. The two properties below are asserted against this rather - * than a fixed list, so they keep holding as pairs are added - including a pair that shares an - * engine with another, which a pairwise swap cannot alternate. - */ - const paired = [...new Set(CONTROL_PAIRS.flatMap((pair) => [pair.legacy, pair.next]))]; - - /** Which of the two runs first, which is the bias the alternation exists to cancel. */ - const runsFirst = (order: EngineId[], pair: { legacy: EngineId; next: EngineId }) => { - const legacy = order.indexOf(pair.legacy); - const next = order.indexOf(pair.next); - if (legacy < 0 || next < 0) { - throw new Error(`${pair.legacy} and ${pair.next} must both be running to have an order`); - } - return legacy < next ? pair.legacy : pair.next; - }; - - it('keeps the listed order on odd repetitions', () => { - expect(engineOrderForRep(engines, 1)).toEqual(engines); - expect(engineOrderForRep(engines, 3)).toEqual(engines); - }); - - it('reverses the order on even repetitions', () => { - expect(engineOrderForRep(engines, 2)).toEqual([...engines].reverse()); - }); - - it('flips which side of every control pair goes first', () => { - for (const pair of CONTROL_PAIRS) { - expect(runsFirst(engineOrderForRep(paired, 2), pair), pair.name).not.toBe( - runsFirst(engineOrderForRep(paired, 1), pair) - ); - } - }); - - it('does not mutate its input', () => { - const input: EngineId[] = ['react-legacy', 'react-osa']; - engineOrderForRep(input, 2); - expect(input).toEqual(['react-legacy', 'react-osa']); - }); - - it('gives each side of a pair the first slot equally often across the pinned N', () => { - // The alternation only cancels the ordering effect when both sides lead the same number of - // times. An odd N leaves a 3-2 split, and the cold median - the headline figure - then lands on - // a repetition from whichever side led more often. Driven off PINNED_N so making it odd fails - // here rather than silently biasing every ratio. - for (const pair of CONTROL_PAIRS) { - const firsts = Array.from({ length: PINNED_N }, (_, i) => - runsFirst(engineOrderForRep(paired, i + 1), pair) - ); - expect(firsts.filter((id) => id === pair.legacy).length, pair.name).toBe( - firsts.filter((id) => id === pair.next).length - ); - } - }); -}); - -describe('computeRatios', () => { - it('divides the legacy median by the new engine median', () => { - const ratios = computeRatios({ - 'react-legacy': measured({ default: scenario(400, 40) }), - 'react-osa': measured({ default: scenario(100, 20) }), - }); - expect(ratios.react.default.cold).toBe(4); - expect(ratios.react.default.warm).toBe(2); - }); - - it('produces no ratio when the new engine median rounded to zero', () => { - // A warm re-extraction can finish below the clock's resolution. Dividing by it yields Infinity, - // which would render as a ratio and serialize into the results file as null. - const ratios = computeRatios({ - 'react-legacy': measured({ default: scenario(400, 40) }), - 'react-osa': measured({ default: scenario(100, 0) }), - }); - expect(ratios.react.default.warm).toBeUndefined(); - expect(ratios.react.default.cold).toBe(4); - }); - - it('produces nothing when one side did not measure', () => { - expect( - computeRatios({ - 'react-legacy': measured({ default: scenario(400, 40) }), - 'react-osa': { status: 'failed', reason: 'boom' }, - }) - ).toEqual({}); - }); - - it('produces nothing when one side was skipped', () => { - expect( - computeRatios({ - 'vue-docgen-api': measured({ flat: scenario(20, 2) }), - 'vue-component-meta': { status: 'skipped', reason: 'not installed' }, - }) - ).toEqual({}); - }); - - it('pairs scenarios by name and ignores ones only one side ran', () => { - const ratios = computeRatios({ - 'vue-docgen-api': measured({ flat: scenario(20, 2), workspace: scenario(30, 3) }), - 'vue-component-meta': measured({ flat: scenario(400, 40) }), - }); - expect(Object.keys(ratios.vue)).toEqual(['flat']); - }); - - it('marks a pair like-for-like when both documented the same members', () => { - const ratios = computeRatios({ - 'vue-docgen-api': measured({ flat: scenario(20, 2, [50, 5]) }), - 'vue-component-meta': measured({ flat: scenario(400, 40, [50, 5]) }), - }); - expect(ratios.vue.flat.coldComparability).toBe('like-for-like'); - }); - - it('marks a pair not like-for-like when the member counts disagree', () => { - // vue-docgen-api does not resolve tsconfig `paths` aliases, so it documents a fraction of what - // vue-component-meta does and is fast for the wrong reason. - const ratios = computeRatios({ - 'vue-docgen-api': measured({ flat: scenario(23, 1, [6, 0]) }), - 'vue-component-meta': measured({ flat: scenario(425, 40, [320, 32]) }), - }); - expect(ratios.vue.flat).toMatchObject({ - coldComparability: 'next-documents-more', - warmComparability: 'next-documents-more', - legacyColdMembers: 6, - nextColdMembers: 320, - legacyWarmMembers: 0, - nextWarmMembers: 32, - }); - }); - - it('separates a pair that documented more from one that documented less', () => { - // Same numbers, opposite meaning: the side that documented less is fast for the wrong reason, - // while the side that documented more has a ratio that undersells it. A boolean lost this. - const fewer = computeRatios({ - 'vue-docgen-api': measured({ flat: scenario(400, 40, [320, 32]) }), - 'vue-component-meta': measured({ flat: scenario(23, 1, [6, 0]) }), - }); - expect(fewer.vue.flat.coldComparability).toBe('next-documents-less'); - }); - - it('catches a pair that documented the same members off different resolution work', () => { - // The Angular trap: an engine that records a type's name without looking through it documents - // exactly as many members as one that expanded the chain, so counts alone read as clean. - const ratios = computeRatios({ - 'vue-docgen-api': measured({ flat: scenario(400, 40, [90, 9], 0) }), - 'vue-component-meta': measured({ flat: scenario(100, 10, [90, 9], 10) }), - }); - expect(ratios.vue.flat.coldComparability).toBe('next-resolves-less'); - expect(ratios.vue.flat.warmComparability).toBe('like-for-like'); - }); - - it('reports comparability unknown when the engines report no member counts', () => { - // Unknown is not the same claim as unequal; a missing count must not read as agreement. - const ratios = computeRatios({ - 'react-legacy': measured({ default: scenario(400, 40) }), - 'react-osa': measured({ default: scenario(100, 20) }), - }); - expect(ratios.react.default.coldComparability).toBe('unknown'); - }); - - it('names every pair it knows uniquely', () => { - const names = CONTROL_PAIRS.map((p) => p.name); - expect(new Set(names).size).toBe(names.length); - }); -}); diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/ratios.ts b/code/lib/docgen-harness/src/perf/docgen-perf/ratios.ts deleted file mode 100644 index e840594ef37e..000000000000 --- a/code/lib/docgen-harness/src/perf/docgen-perf/ratios.ts +++ /dev/null @@ -1,153 +0,0 @@ -/** - * The calibration references: each framework's legacy-engine median divided by its new-engine - * median, both measured in the same invocation. Ratios stand in for absolute milliseconds because - * wall-clock on shared CI executors is too noisy to gate (PERF-METHODOLOGY.md, "Budget shape"). - */ -import type { - Comparability, - EngineId, - EngineResult, - LatencyMetric, - NotApplicable, - RatioEntry, - Ratios, - ScenarioResult, -} from './types.ts'; - -interface ControlPair { - /** Key under which this pair's ratios appear in the results. */ - name: string; - legacy: EngineId; - next: EngineId; -} - -// A pair whose engines are not both registered yields no ratio, so a pair may be listed before the -// engines behind it land. -export const CONTROL_PAIRS: ControlPair[] = [ - { name: 'react', legacy: 'react-legacy', next: 'react-osa' }, - { name: 'vue', legacy: 'vue-docgen-api', next: 'vue-component-meta' }, - { - name: 'vue-component-meta-version', - legacy: 'vue-component-meta', - next: 'vue-component-meta-next', - }, -]; - -/** - * Even repetitions run the engines back to front, so cache warming and thermal drift do not - * consistently favour whichever side of a pair is listed first. - * - * Reversing the whole list rather than swapping each pair in place is what makes that hold for - * *every* pair at once. Pairs can share an engine - `vue-component-meta` is the new side of the vue - * pair and the legacy side of the version pair - and swapping such a chain pairwise moves the shared - * engine twice, which puts the first pair back in its original relative order. - */ -export function engineOrderForRep(engines: EngineId[], rep: number): EngineId[] { - return rep % 2 === 0 ? [...engines].reverse() : [...engines]; -} - -/** - * Undefined unless both sides measured: dividing by a skipped or failed side is not a comparison. - * - * A zero denominator is treated the same way. It means the new engine's median landed below the - * clock's resolution, and Infinity would then be rendered and stored as if it were a ratio. - */ -function medianRatio(legacy: LatencyMetric | NotApplicable, next: LatencyMetric | NotApplicable) { - if (legacy.status !== 'measured' || next.status !== 'measured') { - return undefined; - } - const ratio = legacy.median / next.median; - return Number.isFinite(ratio) ? ratio : undefined; -} - -/** - * Member counts decide first, and any difference there settles it. Only once they agree does the - * count of types an engine never looked through get a say, which is the case a member count on its - * own cannot see. - * - * A missing count on either side yields `unknown` rather than a verdict. Treating it as agreement - * would mark a pair like-for-like on the strength of a number nobody measured. - */ -function comparability( - legacyMembers: number | undefined, - nextMembers: number | undefined, - legacyOpaque?: number, - nextOpaque?: number -): Comparability { - if (legacyMembers === undefined || nextMembers === undefined) { - return 'unknown'; - } - if (nextMembers !== legacyMembers) { - return nextMembers > legacyMembers ? 'next-documents-more' : 'next-documents-less'; - } - if (legacyOpaque === undefined || nextOpaque === undefined || nextOpaque === legacyOpaque) { - return 'like-for-like'; - } - return nextOpaque > legacyOpaque ? 'next-resolves-less' : 'next-resolves-more'; -} - -function ratioFor( - legacy: ScenarioResult, - next: ScenarioResult, - versions: PairVersions -): RatioEntry { - return { - cold: medianRatio(legacy.metrics.coldExtractionMs, next.metrics.coldExtractionMs), - warm: medianRatio(legacy.metrics.warmExtractionMs, next.metrics.warmExtractionMs), - legacyColdMembers: legacy.coldMembers, - nextColdMembers: next.coldMembers, - legacyWarmMembers: legacy.warmMembers, - nextWarmMembers: next.warmMembers, - coldComparability: comparability( - legacy.coldMembers, - next.coldMembers, - legacy.coldOpaqueTypes, - next.coldOpaqueTypes - ), - // No engine reports opaque types for the re-extracted member, so warm is judged on counts alone. - warmComparability: comparability(legacy.warmMembers, next.warmMembers), - ...versions, - }; -} - -interface PairVersions { - legacyVersion?: string; - nextVersion?: string; -} - -/** - * Ratios for every control pair whose two engines both measured in this invocation. A pair with one - * failed or skipped side yields nothing: dividing a fresh median by a stale one is not a comparison. - * - * Resolved versions ride along because a pair can have both sides land on the same version - a - * range on one side is enough - and a ratio of one taken against itself must not read as a clean - * result. - */ -export function computeRatios( - results: Partial>, - engineVersions: Partial> = {} -): Ratios { - const ratios: Ratios = {}; - - for (const pair of CONTROL_PAIRS) { - const legacy = results[pair.legacy]; - const next = results[pair.next]; - if (legacy?.status !== 'measured' || next?.status !== 'measured') { - continue; - } - const versions: PairVersions = { - legacyVersion: engineVersions[pair.legacy], - nextVersion: engineVersions[pair.next], - }; - for (const [scenarioName, legacyScenario] of Object.entries(legacy.scenarios)) { - const nextScenario = next.scenarios[scenarioName]; - if (!nextScenario) { - continue; - } - ratios[pair.name] ??= {}; - ratios[pair.name][scenarioName] = ratioFor(legacyScenario, nextScenario, versions); - } - } - - return ratios; -} diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/registry.test.ts b/code/lib/docgen-harness/src/perf/docgen-perf/registry.test.ts index e5304e420145..891ad11d3651 100644 --- a/code/lib/docgen-harness/src/perf/docgen-perf/registry.test.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/registry.test.ts @@ -1,23 +1,26 @@ import * as fs from 'node:fs'; +import * as path from 'node:path'; import { describe, expect, it } from 'vitest'; -import type { SeriesResult } from '../docgen-shared/series.ts'; -import { QUICK_PROFILE } from './config.ts'; +import { COMPARISON_PAIRS } from './comparison.ts'; +import { DEFAULT_PROFILE, QUICK_PROFILE } from './config.ts'; import { type MeasureContext, type ScenarioSpec, SeriesChildEngine } from './engine.ts'; -import { CONTROL_PAIRS } from './ratios.ts'; import { ALL_ENGINE_IDS, DEFAULT_ENGINE_IDS, ENGINES, engineById } from './registry.ts'; import type { EngineId } from './types.ts'; /** Captures what an engine would have spawned, without spawning it. */ function captureSpawn() { - const calls: Array<{ spec: Parameters[0]; jsonPath: string }> = + const calls: Array<{ spec: Parameters[0]; jsonPath: string }> = []; const ctx: MeasureContext = { - scenarioDir: '/tmp/scenario', - runSeriesChild(spec, _outDir, jsonPath) { + scenarioDir: path.resolve('scenario'), + runLatencyChild(spec, _outDir, jsonPath) { calls.push({ spec, jsonPath }); - return {} as SeriesResult; + return { + cold: { durationMs: 1 }, + warm: [{ save: 1, durationMs: 1 }], + }; }, }; return { ctx, calls }; @@ -39,7 +42,7 @@ describe('the engine table', () => { expect(new Set(ALL_ENGINE_IDS).size).toBe(ALL_ENGINE_IDS.length); }); - it('keeps the unbudgeted react-docgen-typescript engine out of the default run', () => { + it('keeps the diagnostic react-docgen-typescript engine out of the default run', () => { expect(DEFAULT_ENGINE_IDS).not.toContain('react-legacy-rdt'); expect(ALL_ENGINE_IDS).toContain('react-legacy-rdt'); }); @@ -56,22 +59,40 @@ describe('the engine table', () => { } }); - it('registers both sides of every control pair', () => { - // A pair naming an unregistered engine would silently never produce a ratio. - for (const pair of CONTROL_PAIRS) { - expect(ALL_ENGINE_IDS, pair.name).toContain(pair.legacy); - expect(ALL_ENGINE_IDS, pair.name).toContain(pair.next); + it('registers both sides of every comparison pair', () => { + for (const pair of COMPARISON_PAIRS) { + expect(ALL_ENGINE_IDS, pair.name).toContain(pair.control); + expect(ALL_ENGINE_IDS, pair.name).toContain(pair.candidate); } }); - it('runs both sides of every control pair that is on by default', () => { - // A ratio needs both sides measured in the same invocation. - for (const pair of CONTROL_PAIRS) { - if (!DEFAULT_ENGINE_IDS.includes(pair.next)) { - continue; // e.g. vue-component-meta-version: opt-in, no budget row yet + it.each([ + ['quick', QUICK_PROFILE], + ['full', DEFAULT_PROFILE], + ] as const)('keeps every comparison pair on identical %s scenarios', (_name, profile) => { + for (const pair of COMPARISON_PAIRS) { + const control = engineById(pair.control).scenarios(profile); + const candidate = engineById(pair.candidate).scenarios(profile); + expect( + candidate.map(({ name }) => name), + pair.name + ).toEqual(control.map(({ name }) => name)); + for (const scenario of control) { + expect( + candidate.find(({ name }) => name === scenario.name)?.params, + `${pair.name}/${scenario.name}` + ).toEqual(scenario.params); } - expect(DEFAULT_ENGINE_IDS, pair.name).toContain(pair.legacy); - expect(DEFAULT_ENGINE_IDS, pair.name).toContain(pair.next); + } + }); + + it('runs both sides of every comparison pair that is on by default', () => { + for (const pair of COMPARISON_PAIRS) { + if (!DEFAULT_ENGINE_IDS.includes(pair.candidate)) { + continue; // e.g. vue-component-meta-version: explicitly selected with --compare + } + expect(DEFAULT_ENGINE_IDS, pair.name).toContain(pair.control); + expect(DEFAULT_ENGINE_IDS, pair.name).toContain(pair.candidate); } }); }); @@ -107,17 +128,24 @@ describe('what the series engines spawn', () => { }); it('measures both React engines on one changed component per save', async () => { - // Equal work on both sides is what makes the React ratio an engine comparison. + // Matching the scripted scope is necessary even though missing member counts keep the pair's + // timing effect disabled today. for (const id of ['react-legacy', 'react-osa'] as EngineId[]) { const { spec } = await specFor(id, reactScenario); - expect(spec.args.slice(spec.args.indexOf('--scope')), id).toEqual(['--scope', 'changed']); + const scope = spec.args.indexOf('--scope'); + expect(spec.args.slice(scope, scope + 2), id).toEqual(['--scope', 'changed']); } }); + it('puts the reused React OSA harness on the latency-only lane', async () => { + expect((await specFor('react-osa', reactScenario)).spec.args).toContain('--latency'); + }); + it('passes the scenario size through to the child', async () => { const { spec } = await specFor('react-legacy', reactScenario); const components = spec.args[spec.args.indexOf('--components') + 1]; expect(components).toBe(String(QUICK_PROFILE.react.components)); + expect(spec.expectedWorkload).toEqual(reactScenario.params); }); it('passes --heavy-lib only for scenarios that ask for it', async () => { @@ -126,9 +154,12 @@ describe('what the series engines spawn', () => { const workspace = scenarios.find((s) => s.name === 'workspace')!; expect((await specFor('vue-component-meta', flat)).spec.args).not.toContain('--heavy-lib'); expect((await specFor('vue-component-meta', workspace)).spec.args).toContain('--heavy-lib'); + expect((await specFor('vue-component-meta', flat)).spec.expectedWorkload).toEqual( + vueFlatScenario.params + ); }); - it('gives the two Vue engines identical flags, so the ratio compares engines not projects', async () => { + it('gives the two Vue engines identical flags, so a pair compares engines not projects', async () => { for (const scenario of engineById('vue-docgen-api').scenarios(QUICK_PROFILE)) { const legacy = await specFor('vue-docgen-api', scenario); const meta = await specFor('vue-component-meta', scenario); @@ -160,7 +191,7 @@ describe('what the series engines spawn', () => { it('resolves the two sides of the version pair to two different installs', () => { // The invariant, rather than the literal the alias happens to pin today: a caret range on the // current side is enough for both sides to land on one release, and the pair then compares an - // engine against itself for a ratio of 1.00 that reads exactly like a clean result. + // engine against itself and could otherwise look like a clean result. const current = engineById('vue-component-meta').version(); const next = engineById('vue-component-meta-next').version(); expect(current).toMatch(/^\d+\.\d+\.\d+/); diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/registry.ts b/code/lib/docgen-harness/src/perf/docgen-perf/registry.ts index 33c9e9cef148..cfcc995a45d2 100644 --- a/code/lib/docgen-harness/src/perf/docgen-perf/registry.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/registry.ts @@ -43,15 +43,15 @@ const vueArgs = ({ params }: ScenarioSpec): string[] => [ ]; export const ENGINES: BenchEngine[] = [ - // --scope changed on both sides of the React pair compares the engines on equal work; that is - // not what a real legacy save costs - see the calibration caveat in PERF-METHODOLOGY.md. + // Both sides mutate and extract one changed component per save. Their missing member counts keep + // timing effects disabled until the harness can also prove the extracted output is equivalent. new SeriesChildEngine({ id: 'react-legacy', child: 'engines/react-legacy.ts', scenarios: reactScenarios, args: (scenario) => [...reactArgs(scenario), '--parser', 'react-docgen', '--scope', 'changed'], }), - // Measurable through the same child, but it carries no budget row, so it stays out by default. + // Measurable through the same child, but kept as an opt-in diagnostic engine. new SeriesChildEngine({ id: 'react-legacy-rdt', child: 'engines/react-legacy.ts', @@ -70,7 +70,14 @@ export const ENGINES: BenchEngine[] = [ child: '../docgen-memory/memory-harness.ts', scenarios: reactScenarios, jiti: true, - args: (scenario) => [...reactArgs(scenario), '--mode', 'refresh', '--scope', 'changed'], + args: (scenario) => [ + ...reactArgs(scenario), + '--mode', + 'refresh', + '--scope', + 'changed', + '--latency', + ], }), new SeriesChildEngine({ id: 'vue-docgen-api', @@ -85,9 +92,8 @@ export const ENGINES: BenchEngine[] = [ args: vueArgs, versionPackage: 'vue-component-meta', }), - // Same child, pinned to a second, explicitly-versioned copy of the package - a version-pair - // control rather than an engine-vs-engine one. No budget row yet, so it stays out of the - // default run; a contributor testing a bump runs both ids with --engine explicitly. + // Same child, pinned to a second, explicitly-versioned copy of the package for the opt-in + // vue-component-meta-version comparison. It stays out of descriptive runs by default. new SeriesChildEngine({ id: 'vue-component-meta-next', child: 'engines/vue-component-meta.ts', diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/report.test.ts b/code/lib/docgen-harness/src/perf/docgen-perf/report.test.ts index ff93f8e94317..7626ee4276f5 100644 --- a/code/lib/docgen-harness/src/perf/docgen-perf/report.test.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/report.test.ts @@ -1,318 +1,139 @@ import { describe, expect, it } from 'vitest'; -import { renderRatios, renderResults } from './report.ts'; -import { type EngineMetrics, type EngineResult, NOT_APPLICABLE, type Ratios } from './types.ts'; +import { renderComparisons, renderResults } from './report.ts'; +import type { EngineResult, ScenarioResult, SuiteResults } from './types.ts'; -/** - * The report is read as a block of terminal output, not as an array of strings, so the snapshots - * below join it back into one. Column alignment and the notes that travel beside a ratio are the - * whole point of this module, and neither survives a per-line assertion. - */ const block = (lines: string[]) => lines.join('\n'); -/** A series engine: per-component, so it has no whole-project scan. */ -const seriesMetrics: EngineMetrics = { - coldExtractionMs: { status: 'measured', samples: [1204, 1180, 1191], median: 1191 }, - warmExtractionMs: { status: 'measured', samples: [42.4, 39.8, 41.1], median: 41.1 }, - wholeProjectScanMs: NOT_APPLICABLE, - peakTransientMb: { status: 'measured', samples: [61, 58, 63], mean: 60.7 }, - retainedGrowthMb: { status: 'measured', value: 12.4 }, - retainedSlopeMbPerSave: { status: 'measured', value: 0.62 }, -}; - -/** A one-shot CLI engine: a fresh process per run, so no retained series to report. */ -const oneShotMetrics: EngineMetrics = { - coldExtractionMs: { status: 'measured', samples: [8420, 8110, 8300], median: 8300 }, - warmExtractionMs: { status: 'measured', samples: [8290, 8050, 8180], median: 8180 }, - wholeProjectScanMs: { status: 'measured', samples: [8420, 8110, 8300], median: 8300 }, - peakTransientMb: { status: 'measured', samples: [980, 1010, 995], mean: 995 }, - retainedGrowthMb: NOT_APPLICABLE, - retainedSlopeMbPerSave: NOT_APPLICABLE, -}; - -const measured = (metrics: EngineMetrics): EngineResult => ({ +function scenario(cold = 10, warm = 2, scan?: number): ScenarioResult { + return { + params: {}, + repetitions: [ + { + cold: { durationMs: cold, members: 10 }, + warm: [{ save: 1, durationMs: warm, members: 2 }], + ...(scan === undefined ? {} : { scan: { durationMs: scan, members: 10 } }), + }, + ], + summary: { + cold: { processSamplesMs: [cold], medianMs: cold }, + warm: { processSamplesMs: [warm], medianMs: warm }, + ...(scan === undefined ? {} : { scan: { processSamplesMs: [scan], medianMs: scan } }), + }, + }; +} + +const measured = (scenarios: Record): EngineResult => ({ status: 'measured', - scenarios: { default: { params: {}, metrics } }, + scenarios, }); -describe('renderRatios', () => { - it('spells out which side documented more, on the cold line and the warm one', () => { - // The warm line needs the note more than the cold one: the legacy Vue parser documents nothing - // on the save it is timed on, so a bare 22.50 would read as a clean win over identical work. - expect( - block( - renderRatios({ - vue: { - flat: { - cold: 18.48, - warm: 22.5, - legacyColdMembers: 6, - nextColdMembers: 320, - legacyWarmMembers: 0, - nextWarmMembers: 32, - coldComparability: 'next-documents-more', - warmComparability: 'next-documents-more', - }, - }, - }) - ) - ).toMatchInlineSnapshot(` - " ratio cold legacy/new (vue/flat): 18.48 [documented members 6 vs 320 - NOT like-for-like - new engine documented more, so this ratio undersells it] - ratio warm legacy/new (vue/flat): 22.50 [documented members 0 vs 32 - NOT like-for-like - new engine documented more, so this ratio undersells it]" +describe('renderResults', () => { + it('renders latency-only process summaries and operational statuses', () => { + const { table, statusLines } = renderResults( + ['react-legacy', 'compodoc', 'vue-docgen-api', 'vue-component-meta'], + { + 'react-legacy': measured({ default: scenario(12.34, 1.23) }), + compodoc: measured({ default: scenario(80, 75, 80) }), + 'vue-docgen-api': { status: 'skipped', reason: 'not installed' }, + 'vue-component-meta': { status: 'failed', reason: 'child failed\nstack' }, + } + ); + expect(block(table)).toMatchInlineSnapshot(` + " engine/scenario cold warm trajectory scan processes + react-legacy/default 12.3ms 1.2ms n/a 1 + compodoc/default 80.0ms 75.0ms 80.0ms 1" + `); + expect(block(statusLines)).toMatchInlineSnapshot(` + " vue-docgen-api: SKIPPED - not installed + vue-component-meta: FAILED - child failed" `); }); - it('reads a ratio the other way when the new engine documented less', () => { - // The same shape of mismatch, the opposite meaning: this one is fast for the wrong reason. + it('renders every scenario and the header alone when none measured', () => { expect( block( - renderRatios({ - vue: { - flat: { - cold: 0.05, - legacyColdMembers: 320, - nextColdMembers: 6, - coldComparability: 'next-documents-less', - warmComparability: 'unknown', - }, - }, - }) + renderResults(['vue-component-meta'], { + 'vue-component-meta': measured({ flat: scenario(), workspace: scenario() }), + }).table ) - ).toMatchInlineSnapshot( - `" ratio cold legacy/new (vue/flat): 0.05 [documented members 320 vs 6 - NOT like-for-like - new engine documented less, so it is fast for the wrong reason]"` - ); - }); - - it('warns when the counts agree but the resolution work did not', () => { - // Equal member counts off unequal work, which is the case a count alone cannot see. + ).toContain('vue-component-meta/workspace'); expect( block( - renderRatios({ - vue: { - flat: { - cold: 4.1, - legacyColdMembers: 90, - nextColdMembers: 90, - coldComparability: 'next-resolves-less', - warmComparability: 'like-for-like', - }, - }, - }) + renderResults(['compodoc'], { compodoc: { status: 'skipped', reason: 'missing' } }).table ) - ).toMatchInlineSnapshot( - `" ratio cold legacy/new (vue/flat): 4.10 [documented members 90 vs 90 - NOT like-for-like - same members, but the new engine left more types unresolved]"` - ); + ).toBe(' engine/scenario cold warm trajectory scan processes'); }); +}); - it('prints the counts without a warning when both sides did the same work', () => { - expect( - block( - renderRatios({ - vue: { - flat: { - cold: 1.2, - warm: 1.1, - legacyColdMembers: 50, - nextColdMembers: 50, - coldComparability: 'like-for-like', - warmComparability: 'like-for-like', +describe('renderComparisons', () => { + it('renders a paired estimate with work and gate status', () => { + const comparisons: SuiteResults['comparisons'] = { + 'vue-component-meta-version': { + mode: 'paired-gate', + control: 'vue-component-meta', + candidate: 'vue-component-meta-next', + controlVersion: '3.3.9', + candidateVersion: '3.3.8', + scenarios: { + flat: { + seed: 42, + blocks: [], + cold: { + work: { status: 'same-work', reason: 'matching-signatures' }, + effect: { + status: 'measured', + pairs: 10, + logRatios: [], + meanLogRatio: 0, + standardError: 0, + criticalValue95: 2.26, + candidateOverControl: { estimate: 1.02, lower95: 0.98, upper95: 1.06 }, + }, + gate: { + status: 'inconclusive', + maxRegression: 0.05, + maxCandidateOverControl: 1.05, + }, }, - }, - }) - ) - ).toMatchInlineSnapshot(` - " ratio cold legacy/new (vue/flat): 1.20 [documented members 50 vs 50] - ratio warm legacy/new (vue/flat): 1.10" - `); - }); - - it('prints a bare ratio when neither engine reports member counts', () => { - // Unknown earns no warning: it is not a claim of inequality, only an absence of counts. - expect( - block( - renderRatios({ - react: { - default: { - cold: 4, - warm: 2, - coldComparability: 'unknown', - warmComparability: 'unknown', + warm: { + work: { status: 'same-work', reason: 'matching-signatures' }, + gate: { status: 'not-configured' }, }, }, - }) - ) - ).toMatchInlineSnapshot(` - " ratio cold legacy/new (react/default): 4.00 - ratio warm legacy/new (react/default): 2.00" - `); - }); - - it('omits the half of the pair that did not measure', () => { - expect( - block( - renderRatios({ - react: { - default: { cold: 4, coldComparability: 'unknown', warmComparability: 'unknown' }, - }, - }) - ) - ).toMatchInlineSnapshot(`" ratio cold legacy/new (react/default): 4.00"`); - }); - - it('renders every scenario of every pair', () => { - const ratios: Ratios = { - react: { - default: { - cold: 3.9, - warm: 2.1, - coldComparability: 'unknown', - warmComparability: 'unknown', }, }, - vue: { - flat: { cold: 18.5, coldComparability: 'unknown', warmComparability: 'unknown' }, - workspace: { cold: 21.2, coldComparability: 'unknown', warmComparability: 'unknown' }, - }, }; - expect(block(renderRatios(ratios))).toMatchInlineSnapshot(` - " ratio cold legacy/new (react/default): 3.90 - ratio warm legacy/new (react/default): 2.10 - ratio cold legacy/new (vue/flat): 18.50 - ratio cold legacy/new (vue/workspace): 21.20" + expect(block(renderComparisons(comparisons))).toMatchInlineSnapshot(` + " vue-component-meta-version/flat cold: candidate/control=1.020 95%=[0.980,1.060] pairs=10 work=same-work(matching-signatures) versions=3.3.9/3.3.8 gate=inconclusive limit=1.050 + vue-component-meta-version/flat warm: work=same-work(matching-signatures) versions=3.3.9/3.3.8 gate=not-configured" `); }); - it('says so when there is no ratio at all', () => { - expect(block(renderRatios({}))).toMatchInlineSnapshot( - `" no calibration ratio: it needs both sides of a control pair measured in one run"` - ); - }); - - it('names both versions when a pair compares one package against another release of it', () => { - expect( - block( - renderRatios({ - 'vue-component-meta-version': { - flat: { - cold: 1.08, - legacyVersion: '3.3.2', - nextVersion: '3.3.8', - coldComparability: 'like-for-like', - warmComparability: 'like-for-like', - }, - }, - }) - ) - ).toMatchInlineSnapshot( - `" ratio cold legacy/new (vue-component-meta-version/flat): 1.08 [3.3.2 vs 3.3.8]"` - ); - }); - - it('says a pair measured nothing when both sides resolved the same version', () => { - // A range on the current side is enough to let both land on one release. The ratio is then 1.00 - // against itself, which is the one failure a version pair must never report as a clean result. - expect( - block( - renderRatios({ - 'vue-component-meta-version': { - flat: { - cold: 1.0, - legacyVersion: '3.3.8', - nextVersion: '3.3.8', - coldComparability: 'like-for-like', - warmComparability: 'like-for-like', + it('keeps unknown work effect-free and explains an empty comparison set', () => { + const comparisons: SuiteResults['comparisons'] = { + react: { + mode: 'descriptive', + control: 'react-legacy', + candidate: 'react-osa', + scenarios: { + default: { + blocks: [], + cold: { + work: { status: 'unknown-work', reason: 'missing-signature' }, + gate: { status: 'not-configured' }, }, - }, - }) - ) - ).toMatchInlineSnapshot( - `" ratio cold legacy/new (vue-component-meta-version/flat): 1.00 [both sides resolved 3.3.8 - NOT a comparison]"` - ); - }); - - it('carries the versions onto the warm line as well as the cold one', () => { - expect( - block( - renderRatios({ - 'vue-component-meta-version': { - flat: { - cold: 1.08, - warm: 0.97, - legacyColdMembers: 320, - nextColdMembers: 320, - legacyVersion: '3.3.2', - nextVersion: '3.3.8', - coldComparability: 'like-for-like', - warmComparability: 'unknown', + warm: { + work: { status: 'unknown-work', reason: 'missing-signature' }, + gate: { status: 'not-configured' }, }, }, - }) - ) - ).toMatchInlineSnapshot(` - " ratio cold legacy/new (vue-component-meta-version/flat): 1.08 [documented members 320 vs 320] [3.3.2 vs 3.3.8] - ratio warm legacy/new (vue-component-meta-version/flat): 0.97 [3.3.2 vs 3.3.8]" - `); - }); -}); - -describe('renderResults', () => { - it('renders the table every run ends with', () => { - // One engine of each kind, plus a skip and a failure, because the column widths are computed - // across the whole table and a status line replaces a row rather than joining one. - const { table, statusLines } = renderResults( - ['react-legacy', 'react-osa', 'compodoc', 'vue-docgen-api', 'vue-component-meta'], - { - 'react-legacy': measured(seriesMetrics), - 'react-osa': measured(seriesMetrics), - compodoc: measured(oneShotMetrics), - 'vue-docgen-api': { status: 'skipped', reason: 'vue-docgen-api did not resolve' }, - 'vue-component-meta': { - status: 'failed', - reason: 'child exited with status 1:\n at createChecker (vue-component-meta)', - }, - } - ); - expect(block(table)).toMatchInlineSnapshot(` - " engine/scenario cold warm scan peak ret-growth ret-slope - react-legacy/default 1191ms 41ms n/a 61MB 12.4MB 0.62MB/save - react-osa/default 1191ms 41ms n/a 61MB 12.4MB 0.62MB/save - compodoc/default 8300ms 8180ms 8300ms 995MB n/a n/a" - `); - // Only the first line of a reason: a stack trace would push the table off the screen. - expect(block(statusLines)).toMatchInlineSnapshot(` - " vue-docgen-api: SKIPPED - vue-docgen-api did not resolve - vue-component-meta: FAILED - child exited with status 1:" - `); - }); - - it('renders one engine with several scenarios as a row each', () => { - const { table } = renderResults(['vue-component-meta'], { - 'vue-component-meta': { - status: 'measured', - scenarios: { - flat: { params: {}, metrics: seriesMetrics }, - workspace: { params: {}, metrics: seriesMetrics }, - 'base-type-touch': { params: {}, metrics: seriesMetrics }, }, }, - }); - expect(block(table)).toMatchInlineSnapshot(` - " engine/scenario cold warm scan peak ret-growth ret-slope - vue-component-meta/flat 1191ms 41ms n/a 61MB 12.4MB 0.62MB/save - vue-component-meta/workspace 1191ms 41ms n/a 61MB 12.4MB 0.62MB/save - vue-component-meta/base-type-touch 1191ms 41ms n/a 61MB 12.4MB 0.62MB/save" - `); - }); - - it('renders the header alone when nothing measured', () => { - const { table, statusLines } = renderResults(['compodoc'], { - compodoc: { status: 'skipped', reason: '@compodoc/compodoc did not resolve' }, - }); - expect(block(table)).toMatchInlineSnapshot( - `" engine/scenario cold warm scan peak ret-growth ret-slope"` - ); - expect(block(statusLines)).toMatchInlineSnapshot( - `" compodoc: SKIPPED - @compodoc/compodoc did not resolve"` + }; + expect(block(renderComparisons(comparisons))).not.toContain('candidate/control'); + expect(block(renderComparisons({}))).toBe( + ' no comparison: both sides of a configured pair must measure' ); }); }); diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/report.ts b/code/lib/docgen-harness/src/perf/docgen-perf/report.ts index 6bae84a05043..47b5cf5aca71 100644 --- a/code/lib/docgen-harness/src/perf/docgen-perf/report.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/report.ts @@ -1,42 +1,24 @@ -/** Renders the suite's terminal output. Pure string building, so it is testable without a runner. */ -import type { - Comparability, - EngineId, - EngineMetrics, - EngineResult, - RatioEntry, - Ratios, -} from './types.ts'; +/** Pure terminal rendering for Storybook-owned latency results. */ +import type { EngineId, EngineResult, SuiteResults } from './types.ts'; -const HEADER = ['engine/scenario', 'cold', 'warm', 'scan', 'peak', 'ret-growth', 'ret-slope']; +const HEADER = ['engine/scenario', 'cold', 'warm trajectory', 'scan', 'processes']; -/** - * Decimals for a single-valued metric, per unit. A slope needs two to say anything at all, while a - * sub-millisecond difference between two aggregates is noise. - */ -const VALUE_PRECISION = { ms: 0, MB: 1, 'MB/save': 2 } as const; +export interface RenderedResults { + table: string[]; + statusLines: string[]; +} -export function formatCell( - metric: EngineMetrics[keyof EngineMetrics], - unit: keyof typeof VALUE_PRECISION +function gateText( + metric: SuiteResults['comparisons'][string]['scenarios'][string]['cold'] ): string { - if (metric.status === 'n/a') { - return 'n/a'; + const parts = [`gate=${metric.gate.status}`]; + if (metric.gate.maxCandidateOverControl !== undefined) { + parts.push(`limit=${metric.gate.maxCandidateOverControl.toFixed(3)}`); } - if ('median' in metric) { - return `${metric.median.toFixed(0)}${unit}`; + if (metric.gate.reason) { + parts.push(`reason=${metric.gate.reason}`); } - if ('mean' in metric) { - return `${metric.mean.toFixed(0)}${unit}`; - } - return `${metric.value.toFixed(VALUE_PRECISION[unit])}${unit}`; -} - -export interface RenderedResults { - /** Already padded into aligned lines. */ - table: string[]; - /** One line per engine that skipped or failed, in place of a table row. */ - statusLines: string[]; + return parts.join(' '); } export function renderResults( @@ -58,106 +40,65 @@ export function renderResults( continue; } for (const [scenarioName, scenario] of Object.entries(result.scenarios)) { - const m = scenario.metrics; rows.push([ `${engineId}/${scenarioName}`, - formatCell(m.coldExtractionMs, 'ms'), - formatCell(m.warmExtractionMs, 'ms'), - formatCell(m.wholeProjectScanMs, 'ms'), - formatCell(m.peakTransientMb, 'MB'), - formatCell(m.retainedGrowthMb, 'MB'), - formatCell(m.retainedSlopeMbPerSave, 'MB/save'), + `${scenario.summary.cold.medianMs.toFixed(1)}ms`, + `${scenario.summary.warm.medianMs.toFixed(1)}ms`, + scenario.summary.scan ? `${scenario.summary.scan.medianMs.toFixed(1)}ms` : 'n/a', + String(scenario.repetitions.length), ]); } } - const widths = HEADER.map((_, col) => Math.max(...rows.map((row) => row[col].length))); - // trimEnd because the last column is padded like every other one, and a run's output should not - // carry trailing spaces on every line. - const table = rows.map((row) => - ` ${row.map((cell, col) => cell.padEnd(widths[col])).join(' ')}`.trimEnd() + const widths = HEADER.map((_, column) => Math.max(...rows.map((row) => row[column].length))); + return { + table: rows.map((row) => + ` ${row.map((cell, column) => cell.padEnd(widths[column])).join(' ')}`.trimEnd() + ), + statusLines, + }; +} + +function effectLine( + pairName: string, + scenarioName: string, + metricName: 'cold' | 'warm', + metric: SuiteResults['comparisons'][string]['scenarios'][string]['cold'], + versions: string +): string { + const label = `${pairName}/${scenarioName}`; + const work = `${metric.work.status}(${metric.work.reason})${versions}`; + const gate = gateText(metric); + if (!metric.effect) { + return ` ${label} ${metricName}: work=${work} ${gate}`; + } + if (metric.effect.status === 'invalid') { + return ( + ` ${label} ${metricName}: invalid effect (${metric.effect.reason}) ` + `work=${work} ${gate}` + ); + } + const ratio = metric.effect.candidateOverControl; + return ( + ` ${label} ${metricName}: candidate/control=${ratio.estimate.toFixed(3)} ` + + `95%=[${ratio.lower95.toFixed(3)},${ratio.upper95.toFixed(3)}] ` + + `pairs=${metric.effect.pairs} work=${work} ${gate}` ); - return { table, statusLines }; } -/** - * A ratio between engines that documented different numbers of members is not a speed comparison - - * the faster side was fast because it resolved less. Printing such a ratio bare reads as a clean - * win, so the member counts and the warning travel with it, on the warm line as much as the cold - * one. The warm line needs it more: the legacy Vue parser documents zero members on the save it is - * timed on, so a bare warm ratio would read as a clean win over identical work. - */ -export function renderRatios(ratios: Ratios): string[] { +export function renderComparisons(comparisons: SuiteResults['comparisons']): string[] { const lines: string[] = []; - - for (const [pairName, scenarios] of Object.entries(ratios)) { - for (const [scenarioName, entry] of Object.entries(scenarios)) { - const label = `${pairName}/${scenarioName}`; - const versions = versionNote(entry); - - if (entry.cold !== undefined) { - lines.push( - ` ratio cold legacy/new (${label}): ${entry.cold.toFixed(2)}` + - memberNote(entry.legacyColdMembers, entry.nextColdMembers, entry.coldComparability) + - versions - ); - } - if (entry.warm !== undefined) { - lines.push( - ` ratio warm legacy/new (${label}): ${entry.warm.toFixed(2)}` + - memberNote(entry.legacyWarmMembers, entry.nextWarmMembers, entry.warmComparability) + - versions - ); - } + for (const [pairName, comparison] of Object.entries(comparisons)) { + const versions = + comparison.controlVersion && comparison.candidateVersion + ? ` versions=${comparison.controlVersion}/${comparison.candidateVersion}` + : ''; + for (const [scenarioName, scenario] of Object.entries(comparison.scenarios)) { + lines.push(effectLine(pairName, scenarioName, 'cold', scenario.cold, versions)); + lines.push(effectLine(pairName, scenarioName, 'warm', scenario.warm, versions)); } } - if (lines.length === 0) { - lines.push(' no calibration ratio: it needs both sides of a control pair measured in one run'); + lines.push(' no comparison: both sides of a configured pair must measure'); } return lines; } - -/** - * Spelled out rather than flagged, because the direction is what a reader needs: an engine that - * documented less is fast for the wrong reason, while one that documented more has a ratio that - * undersells it. A bare "NOT like-for-like" leaves those two looking the same. - */ -const VERDICT: Record = { - // No counts is not a claim of inequality, so it earns no warning - only a direction does. - unknown: '', - 'like-for-like': '', - 'next-documents-more': - 'NOT like-for-like - new engine documented more, so this ratio undersells it', - 'next-documents-less': - 'NOT like-for-like - new engine documented less, so it is fast for the wrong reason', - 'next-resolves-more': - 'NOT like-for-like - same members, but the new engine resolved more of them', - 'next-resolves-less': - 'NOT like-for-like - same members, but the new engine left more types unresolved', -}; - -function memberNote( - legacy: number | undefined, - next: number | undefined, - verdict: Comparability -): string { - const note = VERDICT[verdict]; - if (legacy === undefined || next === undefined) { - return note ? ` [${note}]` : ''; - } - return ` [documented members ${legacy} vs ${next}${note ? ` - ${note}` : ''}]`; -} - -/** - * Two sides on the same version compare an engine against itself, which reads as a clean 1.00 and - * means nothing. Saying so beside the ratio is what stops it being read as "no regression". - */ -function versionNote({ legacyVersion, nextVersion }: RatioEntry): string { - if (legacyVersion === undefined || nextVersion === undefined) { - return ''; - } - return legacyVersion === nextVersion - ? ` [both sides resolved ${legacyVersion} - NOT a comparison]` - : ` [${legacyVersion} vs ${nextVersion}]`; -} diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/run.ts b/code/lib/docgen-harness/src/perf/docgen-perf/run.ts index f5cf1e177fa2..3b5587775c84 100644 --- a/code/lib/docgen-harness/src/perf/docgen-perf/run.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/run.ts @@ -1,116 +1,219 @@ -/** - * Orchestrator for the per-engine docgen performance suite. See ../PERF-METHODOLOGY.md for the - * measurement contract. - * - * Run from code/lib/docgen-harness: - * yarn bench:docgen-perf # full profile - * yarn bench:docgen-perf --quick # smoke profile; results marked non-comparable - * yarn bench:docgen-perf --engine react-legacy --engine react-osa - */ +/** Descriptive docgen latency suite and opt-in paired timing gate. */ +import { randomBytes } from 'node:crypto'; import * as fs from 'node:fs'; import * as path from 'node:path'; +import type { LatencyRepetition } from '../docgen-shared/latency-series.ts'; import { SANDBOX_DIRECTORY } from '../docgen-shared/paths.ts'; import { parseCliOptions } from './cli.ts'; +import { + COMPARISON_PAIRS, + assessMetricWork, + comparePairedTimings, + createComparisonPlan, + evaluateBudget, + executeComparisonPlan, + type WorkProfile, +} from './comparison.ts'; import { DEFAULT_PROFILE, QUICK_PROFILE, type SuiteProfile } from './config.ts'; -import { computeRatios, engineOrderForRep } from './ratios.ts'; +import type { ScenarioSpec } from './engine.ts'; import { engineById } from './registry.ts'; -import { renderRatios, renderResults } from './report.ts'; -import { runSeriesChild } from './spawn.ts'; -import type { EngineId, EngineResult, ScenarioResult, SuiteResults } from './types.ts'; +import { renderComparisons, renderResults } from './report.ts'; +import { runLatencyChild } from './spawn.ts'; +import { warmTrajectoryMedian } from './summary.ts'; +import type { + ComparisonResult, + EngineId, + EngineResult, + RawPairedBlock, + ScenarioResult, + SuiteResults, +} from './types.ts'; const WORK_ROOT = path.join(SANDBOX_DIRECTORY, 'docgen-perf'); - -/** - * Raw repetition samples, keyed by `engine/scenario`. Only the cold duration is common to every - * engine's sample; the rest stays known to the engine that produced it, which is what reads it back. - */ -type SampleStore = Map>; +type SampleStore = Map; function scenarioKey(engineId: EngineId, scenarioName: string): string { return `${engineId}/${scenarioName}`; } -/** One repetition of one engine, across every scenario that engine runs. */ -async function measureEngine( +async function measureScenario( engineId: EngineId, - profile: SuiteProfile, - rep: number, + scenario: ScenarioSpec, + repetition: number, + total: number, store: SampleStore -): Promise { - const engine = engineById(engineId); - for (const scenario of engine.scenarios(profile)) { - const key = scenarioKey(engineId, scenario.name); - console.log(` ${key} (rep ${rep}/${profile.n})…`); - const sample = await engine.measure( - { - scenarioDir: path.join(WORK_ROOT, engineId, scenario.name), - runSeriesChild, - }, - scenario, - rep - ); - store.set(key, [...(store.get(key) ?? []), sample]); - } -} - -async function main() { - const options = parseCliOptions(process.argv.slice(2), WORK_ROOT); - const profile: SuiteProfile = options.quick ? QUICK_PROFILE : DEFAULT_PROFILE; - - console.log('docgen-perf suite'); - console.log( - ` engines=${options.engines.join(',')} n=${profile.n} comparable=${profile.comparable}` +): Promise { + const key = scenarioKey(engineId, scenario.name); + console.log(` ${key} (rep ${repetition}/${total})…`); + const sample = await engineById(engineId).measure( + { + scenarioDir: path.join(WORK_ROOT, engineId, scenario.name), + runLatencyChild, + }, + scenario, + repetition ); - if (!profile.comparable) { - console.log(' QUICK PROFILE: results are non-comparable smoke numbers'); - } + store.set(key, [...(store.get(key) ?? []), sample]); + return sample; +} - const store: SampleStore = new Map(); - const failed = new Map(); - const skipped = new Map(); +function signatureOf(repetition: LatencyRepetition): Omit { + return { + cold: { + members: repetition.cold.members, + opaqueTypes: repetition.cold.opaqueTypes, + }, + warm: repetition.warm.map(({ save, members, opaqueTypes }) => ({ + save, + members, + opaqueTypes, + })), + }; +} - for (const engineId of options.engines) { - const reason = engineById(engineId).preflight(); - if (reason) { - skipped.set(engineId, reason); - console.log(` ${engineId}: SKIPPED - ${reason}`); - } +/** A profile is known only when the scripted work stayed identical across every fresh process. */ +function workProfile(repetitions: LatencyRepetition[], version?: string): WorkProfile { + const signatures = repetitions.map(signatureOf); + const first = signatures[0]; + if (!first) { + return { version }; } + const cold = JSON.stringify(first.cold); + const warm = JSON.stringify(first.warm); + return { + version, + ...(signatures.every((signature) => JSON.stringify(signature.cold) === cold) + ? { cold: first.cold } + : {}), + ...(signatures.every((signature) => JSON.stringify(signature.warm) === warm) + ? { warm: first.warm } + : {}), + }; +} - for (let rep = 1; rep <= profile.n; rep++) { - console.log(`\n=== repetition ${rep}/${profile.n} ===`); - for (const engineId of engineOrderForRep(options.engines, rep)) { +async function runDescriptive( + engines: EngineId[], + profile: SuiteProfile, + store: SampleStore, + failed: Map, + skipped: Map +): Promise { + for (let repetition = 1; repetition <= profile.repetitions; repetition++) { + console.log(`\n=== repetition ${repetition}/${profile.repetitions} ===`); + for (const engineId of engines) { if (failed.has(engineId) || skipped.has(engineId)) { continue; } try { - await measureEngine(engineId, profile, rep, store); - } catch (err) { - const reason = err instanceof Error ? err.message : String(err); + for (const scenario of engineById(engineId).scenarios(profile)) { + await measureScenario(engineId, scenario, repetition, profile.repetitions, store); + } + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); failed.set(engineId, reason); console.error(` ${engineId} FAILED: ${reason}`); } } } +} +async function runPaired( + pairName: string, + globalSeed: number, + profile: SuiteProfile, + store: SampleStore, + blocksByScenario: Map, + failed: Map, + skipped: Map +): Promise { + const pair = COMPARISON_PAIRS.find(({ name }) => name === pairName)!; + const controlSkip = skipped.get(pair.control); + const candidateSkip = skipped.get(pair.candidate); + if (controlSkip || candidateSkip) { + if (!controlSkip) { + skipped.set( + pair.control, + `comparison partner ${pair.candidate} was skipped: ${candidateSkip}` + ); + } + if (!candidateSkip) { + skipped.set(pair.candidate, `comparison partner ${pair.control} was skipped: ${controlSkip}`); + } + return; + } + const controlScenarios = engineById(pair.control).scenarios(profile); + const candidateScenarios = engineById(pair.candidate).scenarios(profile); + const controlScenarioNames = controlScenarios.map(({ name }) => name).sort(); + const candidateScenarioNames = candidateScenarios.map(({ name }) => name).sort(); + if (JSON.stringify(controlScenarioNames) !== JSON.stringify(candidateScenarioNames)) { + throw new Error( + `${pair.name} has mismatched scenario sets: ` + + `${pair.control}=${controlScenarioNames.join(',')} ` + + `${pair.candidate}=${candidateScenarioNames.join(',')}` + ); + } + + for (const controlScenario of controlScenarios) { + const candidateScenario = candidateScenarios.find(({ name }) => name === controlScenario.name); + if (!candidateScenario) { + throw new Error(`${pair.candidate} has no scenario named ${controlScenario.name}`); + } + if (JSON.stringify(candidateScenario.params) !== JSON.stringify(controlScenario.params)) { + throw new Error(`${pair.name}/${controlScenario.name} has mismatched scenario parameters`); + } + + const plan = createComparisonPlan({ + seed: globalSeed, + pair: pair.name, + scenario: controlScenario.name, + repetitions: profile.repetitions, + }); + console.log(`\n=== paired ${pair.name}/${controlScenario.name} seed=${plan.seed} ===`); + try { + const blocks = await executeComparisonPlan(plan, async (side, block) => { + const engineId = side === 'control' ? pair.control : pair.candidate; + const scenario = side === 'control' ? controlScenario : candidateScenario; + try { + return await measureScenario(engineId, scenario, block, profile.repetitions, store); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + failed.set(engineId, reason); + console.error(` ${engineId} FAILED: ${reason}`); + throw error; + } + }); + blocksByScenario.set(controlScenario.name, blocks); + } catch { + return; + } + } +} + +function assembleEngineResults( + engines: EngineId[], + profile: SuiteProfile, + store: SampleStore, + failed: Map, + skipped: Map +): { + engineResults: SuiteResults['engines']; + engineVersions: SuiteResults['engineVersions']; +} { const engineResults: SuiteResults['engines'] = {}; const engineVersions: SuiteResults['engineVersions'] = {}; - for (const engineId of options.engines) { + for (const engineId of engines) { const engine = engineById(engineId); const version = engine.version(); if (version) { engineVersions[engineId] = version; } - const skipReason = skipped.get(engineId); if (skipReason) { engineResults[engineId] = { status: 'skipped', reason: skipReason }; continue; } - // An engine that failed part-way holds fewer samples than the pinned N, so it stays failed - // rather than being reported as measured at an unrecorded N. const failReason = failed.get(engineId); if (failReason) { engineResults[engineId] = { status: 'failed', reason: failReason }; @@ -119,54 +222,262 @@ async function main() { try { const scenarios: Record = {}; for (const scenario of engine.scenarios(profile)) { - const samples = store.get(scenarioKey(engineId, scenario.name)) ?? []; - scenarios[scenario.name] = engine.assemble(samples, profile.n, scenario); + scenarios[scenario.name] = engine.assemble( + store.get(scenarioKey(engineId, scenario.name)) ?? [], + profile.repetitions, + scenario + ); } engineResults[engineId] = { status: 'measured', scenarios } satisfies EngineResult; - } catch (err) { - const reason = err instanceof Error ? err.message : String(err); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); failed.set(engineId, reason); engineResults[engineId] = { status: 'failed', reason }; } } + return { engineResults, engineVersions }; +} + +function descriptiveComparisons( + engineResults: SuiteResults['engines'], + engineVersions: SuiteResults['engineVersions'] +): SuiteResults['comparisons'] { + const comparisons: SuiteResults['comparisons'] = {}; + for (const pair of COMPARISON_PAIRS) { + const control = engineResults[pair.control]; + const candidate = engineResults[pair.candidate]; + if (control?.status !== 'measured' || candidate?.status !== 'measured') { + continue; + } + const scenarios: ComparisonResult['scenarios'] = {}; + for (const [name, controlScenario] of Object.entries(control.scenarios)) { + const candidateScenario = candidate.scenarios[name]; + if (!candidateScenario) { + continue; + } + const controlWork = workProfile(controlScenario.repetitions, engineVersions[pair.control]); + const candidateWork = workProfile( + candidateScenario.repetitions, + engineVersions[pair.candidate] + ); + const workOptions = { + versionsMustDiffer: 'versionsMustDiffer' in pair && pair.versionsMustDiffer, + }; + scenarios[name] = { + blocks: [], + cold: { + work: assessMetricWork(controlWork, candidateWork, 'cold', workOptions), + gate: { status: 'not-configured' }, + }, + warm: { + work: assessMetricWork(controlWork, candidateWork, 'warm', workOptions), + gate: { status: 'not-configured' }, + }, + }; + } + comparisons[pair.name] = { + mode: 'descriptive', + control: pair.control, + candidate: pair.candidate, + controlVersion: engineVersions[pair.control], + candidateVersion: engineVersions[pair.candidate], + scenarios, + }; + } + return comparisons; +} + +function pairedComparison( + pairName: string, + engineResults: SuiteResults['engines'], + profile: SuiteProfile, + blocksByScenario: Map, + engineVersions: SuiteResults['engineVersions'], + globalSeed: number, + maxRegression: number, + smoke: boolean +): SuiteResults['comparisons'] { + const pair = COMPARISON_PAIRS.find(({ name }) => name === pairName)!; + if ( + engineResults[pair.control]?.status !== 'measured' || + engineResults[pair.candidate]?.status !== 'measured' + ) { + return {}; + } + const scenarios: ComparisonResult['scenarios'] = {}; + + for (const [scenarioName, blocks] of blocksByScenario) { + const controlRepetitions = blocks.map(({ control }) => control); + const candidateRepetitions = blocks.map(({ candidate }) => candidate); + const comparisonInput = { + controlWork: workProfile(controlRepetitions, engineVersions[pair.control]), + candidateWork: workProfile(candidateRepetitions, engineVersions[pair.candidate]), + expectedPairs: profile.repetitions, + versionsMustDiffer: 'versionsMustDiffer' in pair && pair.versionsMustDiffer, + }; + const cold = comparePairedTimings({ + ...comparisonInput, + metric: 'cold', + timings: blocks.map(({ block, control, candidate }) => ({ + block, + controlMs: control.cold.durationMs, + candidateMs: candidate.cold.durationMs, + })), + }); + const warm = comparePairedTimings({ + ...comparisonInput, + metric: 'warm', + timings: blocks.map(({ block, control, candidate }) => ({ + block, + controlMs: warmTrajectoryMedian(control), + candidateMs: warmTrajectoryMedian(candidate), + })), + }); + const budgetOptions = { repetitions: profile.repetitions, maxRegression, smoke }; + scenarios[scenarioName] = { + seed: createComparisonPlan({ + seed: globalSeed, + pair: pair.name, + scenario: scenarioName, + repetitions: profile.repetitions, + }).seed, + blocks, + cold: { + work: cold.work, + effect: cold.effect, + gate: evaluateBudget(cold, budgetOptions), + }, + warm: { + work: warm.work, + effect: warm.effect, + gate: evaluateBudget(warm, budgetOptions), + }, + }; + } + return { + [pair.name]: { + mode: 'paired-gate', + control: pair.control, + candidate: pair.candidate, + controlVersion: engineVersions[pair.control], + candidateVersion: engineVersions[pair.candidate], + scenarios, + }, + }; +} +async function main() { + const options = parseCliOptions(process.argv.slice(2), WORK_ROOT); + const baseProfile = options.quick ? QUICK_PROFILE : DEFAULT_PROFILE; + const profile = { + ...baseProfile, + repetitions: options.repetitions ?? baseProfile.repetitions, + }; + const seed = options.compare ? (options.seed ?? randomBytes(4).readUInt32LE()) : undefined; + + console.log('docgen-perf suite'); + console.log( + ` engines=${options.engines.join(',')} repetitions=${profile.repetitions} ` + + `mode=${options.compare ? 'paired-gate' : 'descriptive'}` + ); + if (options.quick) { + console.log(' QUICK PROFILE: smoke workload, never a timing gate'); + } + if (seed !== undefined) { + console.log(` seed=${seed}`); + } + + const store: SampleStore = new Map(); + const failed = new Map(); + const skipped = new Map(); + const blocksByScenario = new Map(); + + for (const engineId of options.engines) { + const reason = engineById(engineId).preflight(); + if (reason) { + skipped.set(engineId, reason); + console.log(` ${engineId}: SKIPPED - ${reason}`); + } + } + + try { + if (options.compare) { + await runPaired(options.compare, seed!, profile, store, blocksByScenario, failed, skipped); + } else { + await runDescriptive(options.engines, profile, store, failed, skipped); + } + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + for (const engineId of options.engines) { + if (!skipped.has(engineId)) { + failed.set(engineId, reason); + } + } + } + + const { engineResults, engineVersions } = assembleEngineResults( + options.engines, + profile, + store, + failed, + skipped + ); + const comparisons = options.compare + ? pairedComparison( + options.compare, + engineResults, + profile, + blocksByScenario, + engineVersions, + seed!, + options.maxRegression!, + options.quick + ) + : descriptiveComparisons(engineResults, engineVersions); const results: SuiteResults = { + schemaVersion: 1, generatedAt: new Date().toISOString(), nodeVersion: process.version, - pinnedN: profile.n, - comparable: profile.comparable, + profile: options.quick ? 'quick' : 'full', + repetitions: profile.repetitions, + mode: options.compare ? 'paired-gate' : 'descriptive', + gating: options.compare !== undefined && !options.quick, + ...(options.compare ? { comparisonMethod: 'paired-log-ratio-t95-v1' as const, seed } : {}), engineVersions, engines: engineResults, - ratios: computeRatios(engineResults, engineVersions), + comparisons, }; console.log('\nresults'); const { table, statusLines } = renderResults(options.engines, engineResults); - for (const line of [...table, ...statusLines, ...renderRatios(results.ratios)]) { + for (const line of [...table, ...statusLines, ...renderComparisons(comparisons)]) { console.log(line); } - if (!profile.comparable) { - console.log(' QUICK PROFILE: results are non-comparable smoke numbers'); - } - fs.mkdirSync(path.dirname(options.jsonOut), { recursive: true }); fs.writeFileSync(options.jsonOut, JSON.stringify(results, null, 2)); console.log(` wrote ${options.jsonOut}`); - if (failed.size > 0) { - console.error('\ndocgen-perf suite FAILED:'); - for (const [engineId, reason] of failed) { - console.error(` - ${engineId}: ${reason}`); + const gateFailed = Object.values(comparisons).some((comparison) => + Object.values(comparison.scenarios).some((scenario) => + [scenario.cold.gate.status, scenario.warm.gate.status].some( + (status) => status === 'regression' || status === 'invalid-gate' + ) + ) + ); + if (failed.size > 0 || gateFailed) { + if (failed.size > 0) { + console.error('\ndocgen-perf suite FAILED:'); + for (const [engineId, reason] of failed) { + console.error(` - ${engineId}: ${reason}`); + } } - // Not `process.exit`: writes to a pipe are async, so exiting here can truncate the reasons - // printed just above - exactly the output someone is reading when the suite fails under `tee`. process.exitCode = 1; return; } console.log('\ndocgen-perf suite completed.'); } -main().catch((err) => { - console.error(err); +main().catch((error) => { + console.error(error); process.exitCode = 1; }); diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/spawn.test.ts b/code/lib/docgen-harness/src/perf/docgen-perf/spawn.test.ts new file mode 100644 index 000000000000..d16fcf9c2bcd --- /dev/null +++ b/code/lib/docgen-harness/src/perf/docgen-perf/spawn.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from 'vitest'; + +import { parseLatencyChildResult } from './spawn.ts'; + +const expectedWorkload = { + name: 'flat', + packages: 1, + componentsPerPackage: 5, + chainDepth: 1, + fanOut: 2, + heavyLib: false, + saves: 3, +}; + +function resultJson(options: Record) { + return JSON.stringify({ + options, + cold: { durationMs: 100, members: 25 }, + warm: [ + { save: 1, durationMs: 10, members: 5 }, + { save: 2, durationMs: 11, members: 5 }, + { save: 3, durationMs: 12, members: 5 }, + ], + diagnostics: { absoluteLog: '/absolute/private/child.log' }, + peakRssMb: 512, + }); +} + +describe('parseLatencyChildResult', () => { + it('validates the echoed workload and persists only latency observations', () => { + const result = parseLatencyChildResult( + resultJson({ + scenario: 'flat', + packages: 1, + componentsPerPackage: 5, + chainDepth: 1, + fanOut: 2, + heavyLib: false, + saves: 3, + outDir: '/absolute/private/project', + jsonOut: '/absolute/private/result.json', + parser: 'vue-component-meta', + scope: 'changed', + pin: 'next', + }), + expectedWorkload + ); + + expect(result).toEqual({ + cold: { durationMs: 100, members: 25 }, + warm: [ + { save: 1, durationMs: 10, members: 5 }, + { save: 2, durationMs: 11, members: 5 }, + { save: 3, durationMs: 12, members: 5 }, + ], + }); + expect(result).not.toHaveProperty('options'); + expect(result).not.toHaveProperty('diagnostics'); + expect(result).not.toHaveProperty('peakRssMb'); + expect(JSON.stringify(result)).not.toContain('/absolute/private'); + }); + + it('maps scenario param name to the child scenario option', () => { + expect(() => + parseLatencyChildResult( + resultJson({ + scenario: 'workspace', + packages: 1, + componentsPerPackage: 5, + chainDepth: 1, + fanOut: 2, + heavyLib: false, + saves: 3, + }), + expectedWorkload + ) + ).toThrow('child echoed workload option "scenario" as "workspace", expected "flat"'); + }); + + it('rejects a missing echoed workload parameter or options envelope', () => { + expect(() => + parseLatencyChildResult( + resultJson({ + scenario: 'flat', + packages: 1, + componentsPerPackage: 5, + chainDepth: 1, + fanOut: 2, + heavyLib: false, + }), + expectedWorkload + ) + ).toThrow('child result is missing echoed workload option "saves"'); + + expect(() => + parseLatencyChildResult( + JSON.stringify({ cold: { durationMs: 1 }, warm: [{ save: 1, durationMs: 1 }] }), + expectedWorkload + ) + ).toThrow('child result is missing its parsed options envelope'); + }); +}); diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/spawn.ts b/code/lib/docgen-harness/src/perf/docgen-perf/spawn.ts index b9e376957005..3bc741939baf 100644 --- a/code/lib/docgen-harness/src/perf/docgen-perf/spawn.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/spawn.ts @@ -1,42 +1,78 @@ -/** - * Spawning one measured child process and reading its result back. - * - * Every measurement runs in a fresh process so it starts from a clean heap; an engine measured after - * another engine's garbage would report that garbage as its own. - */ +/** Fresh-process execution for one latency repetition. */ import { spawnSync } from 'node:child_process'; import * as fs from 'node:fs'; import * as path from 'node:path'; import { outputTail } from '../docgen-shared/child-output.ts'; -import type { SeriesResult } from '../docgen-shared/series.ts'; +import type { LatencyRepetition } from '../docgen-shared/latency-series.ts'; -export interface SeriesChildSpec { +export interface LatencyChildSpec { childPath: string; args: string[]; - /** - * Run the child under the jiti loader. Only the reused docgen-memory harness needs it: under - * jiti, react-docgen's browserslist dependency fails on its JSON data require - * ("jsReleases.map is not a function"), so the legacy child must run on native type stripping. - */ + /** Scenario parameters the child must echo after parsing its CLI flags. */ + expectedWorkload: Record; + /** The React OSA source child requires jiti; legacy parser children must remain native. */ jiti?: boolean; } -/** - * Run one series-harness child and parse its result JSON. A non-zero exit, or a missing or - * unreadable result, is a failure - never a silently empty measurement. - */ -export function runSeriesChild( - spec: SeriesChildSpec, +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function childOptionName(scenarioParam: string): string { + return scenarioParam === 'name' ? 'scenario' : scenarioParam; +} + +/** Validates the child's echoed CLI workload and narrows its envelope to persisted latency data. */ +export function parseLatencyChildResult( + json: string, + expectedWorkload: Record +): LatencyRepetition { + const envelope: unknown = JSON.parse(json); + if (!isRecord(envelope) || !isRecord(envelope.options)) { + throw new Error('child result is missing its parsed options envelope'); + } + + for (const [scenarioParam, expected] of Object.entries(expectedWorkload)) { + const option = childOptionName(scenarioParam); + if (!Object.hasOwn(envelope.options, option)) { + throw new Error(`child result is missing echoed workload option "${option}"`); + } + const actual = envelope.options[option]; + if (actual !== expected) { + throw new Error( + `child echoed workload option "${option}" as ${JSON.stringify(actual)}, expected ${JSON.stringify(expected)}` + ); + } + } + + if (!isRecord(envelope.cold) || !Array.isArray(envelope.warm)) { + throw new Error('child result is missing cold or warm latency observations'); + } + if (envelope.scan !== undefined && !isRecord(envelope.scan)) { + throw new Error('child result has an invalid scan latency observation'); + } + + return { + cold: envelope.cold as unknown as LatencyRepetition['cold'], + warm: envelope.warm as LatencyRepetition['warm'], + ...(envelope.scan === undefined + ? {} + : { + scan: envelope.scan as unknown as NonNullable, + }), + }; +} + +export function runLatencyChild( + spec: LatencyChildSpec, outDir: string, jsonPath: string -): SeriesResult { +): LatencyRepetition { fs.mkdirSync(path.dirname(jsonPath), { recursive: true }); - // Remove any stale result so a crashed run cannot be mistaken for a prior success. fs.rmSync(jsonPath, { force: true }); const nodeArgs = [ - '--expose-gc', ...(spec.jiti ? ['--import', 'jiti/register'] : []), spec.childPath, ...spec.args, @@ -45,8 +81,10 @@ export function runSeriesChild( '--json', jsonPath, ]; - - const proc = spawnSync(process.execPath, nodeArgs, { encoding: 'utf8' }); + const proc = spawnSync(process.execPath, nodeArgs, { + encoding: 'utf8', + maxBuffer: 4 * 1024 * 1024, + }); const output = `${proc.stdout ?? ''}${proc.stderr ?? ''}`; if (proc.status !== 0) { @@ -55,5 +93,5 @@ export function runSeriesChild( if (!fs.existsSync(jsonPath)) { throw new Error(`child wrote no result JSON at ${jsonPath}:\n${outputTail(output, 4)}`); } - return JSON.parse(fs.readFileSync(jsonPath, 'utf8')) as SeriesResult; + return parseLatencyChildResult(fs.readFileSync(jsonPath, 'utf8'), spec.expectedWorkload); } diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/summary.test.ts b/code/lib/docgen-harness/src/perf/docgen-perf/summary.test.ts new file mode 100644 index 000000000000..9ce4ed9a7a84 --- /dev/null +++ b/code/lib/docgen-harness/src/perf/docgen-perf/summary.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from 'vitest'; + +import type { LatencyRepetition } from '../docgen-shared/latency-series.ts'; +import { + assertCompleteLatencyRepetitions, + assertRepetitionCount, + summarizeLatencyRepetitions, + warmTrajectoryMedian, +} from './summary.ts'; + +function repetition(cold: number, warm: number[], scan?: number): LatencyRepetition { + return { + cold: { durationMs: cold, members: 10 }, + warm: warm.map((durationMs, index) => ({ save: index + 1, durationMs, members: 2 })), + ...(scan === undefined ? {} : { scan: { durationMs: scan, members: 10 } }), + }; +} + +describe('latency summaries', () => { + it('keeps one independent warm summary per fresh process', () => { + const repetitions = [repetition(300, [1, 9, 5]), repetition(100, [20, 10, 30])]; + expect(repetitions[0].warm.map(({ save }) => save)).toEqual([1, 2, 3]); + expect(summarizeLatencyRepetitions(repetitions)).toEqual({ + cold: { processSamplesMs: [300, 100], medianMs: 200 }, + warm: { processSamplesMs: [5, 20], medianMs: 12.5 }, + }); + }); + + it('summarizes a whole-project scan only when every process reported one', () => { + expect( + summarizeLatencyRepetitions([repetition(10, [2], 10), repetition(20, [4], 20)]).scan + ).toMatchObject({ processSamplesMs: [10, 20], medianMs: 15 }); + expect(() => + summarizeLatencyRepetitions([repetition(10, [2], 10), repetition(20, [4])]) + ).toThrow('recorded 1 scan observations for 2 repetitions'); + }); + + it('does not mutate a chronological trajectory while calculating its median', () => { + const value = repetition(10, [9, 1, 5]); + expect(warmTrajectoryMedian(value)).toBe(5); + expect(value.warm.map(({ durationMs }) => durationMs)).toEqual([9, 1, 5]); + }); + + it('rejects missing or incomplete fresh-process repetitions', () => { + expect(() => assertRepetitionCount([], 2)).toThrow('no completed repetition'); + expect(() => assertRepetitionCount([{}], 2)).toThrow('recorded 1 repetitions, expected 2'); + + const missingSave = repetition(10, [2, 3]); + missingSave.warm.pop(); + expect(() => assertCompleteLatencyRepetitions([missingSave], 1, 2)).toThrow( + 'recorded 1 warm observations, expected 2' + ); + + const outOfOrder = repetition(10, [2, 3]); + outOfOrder.warm[1].save = 3; + expect(() => assertCompleteLatencyRepetitions([outOfOrder], 1, 2)).toThrow( + 'warm observation 2 is labeled save 3' + ); + + const invalidDuration = repetition(10, [2]); + invalidDuration.cold.durationMs = Number.NaN; + expect(() => assertCompleteLatencyRepetitions([invalidDuration], 1, 1)).toThrow( + 'finite positive duration' + ); + + const impossibleWork = repetition(10, [2]); + impossibleWork.cold.opaqueTypes = 11; + expect(() => assertCompleteLatencyRepetitions([impossibleWork], 1, 1)).toThrow( + 'opaque types cannot exceed documented members' + ); + }); +}); diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/summary.ts b/code/lib/docgen-harness/src/perf/docgen-perf/summary.ts new file mode 100644 index 000000000000..8c5591ccfdbc --- /dev/null +++ b/code/lib/docgen-harness/src/perf/docgen-perf/summary.ts @@ -0,0 +1,98 @@ +import type { LatencyObservation, LatencyRepetition } from '../docgen-shared/latency-series.ts'; +import { median } from '../docgen-shared/stats.ts'; +import type { ScenarioLatencySummary } from './types.ts'; + +export function assertRepetitionCount(repetitions: unknown[], expected: number): void { + if (repetitions.length === 0) { + throw new Error('no completed repetition recorded'); + } + if (repetitions.length !== expected) { + throw new Error(`recorded ${repetitions.length} repetitions, expected ${expected}`); + } +} + +function assertCount(value: number | undefined, label: string): void { + if (value !== undefined && (!Number.isSafeInteger(value) || value < 0)) { + throw new Error(`${label} must be a non-negative safe integer when present`); + } +} + +function assertObservation( + observation: LatencyObservation | undefined, + label: string +): asserts observation is LatencyObservation { + if (!observation || !Number.isFinite(observation.durationMs) || observation.durationMs <= 0) { + throw new Error(`${label} must contain a finite positive duration`); + } + assertCount(observation.members, `${label} members`); + assertCount(observation.opaqueTypes, `${label} opaque types`); + if ( + observation.members !== undefined && + observation.opaqueTypes !== undefined && + observation.opaqueTypes > observation.members + ) { + throw new Error(`${label} opaque types cannot exceed documented members`); + } +} + +/** Rejects truncated or malformed child JSON before it can become a headline or paired block. */ +export function assertCompleteLatencyRepetitions( + repetitions: LatencyRepetition[], + expectedRepetitions: number, + expectedWarmObservations?: number +): void { + assertRepetitionCount(repetitions, expectedRepetitions); + for (const [repetitionIndex, repetition] of repetitions.entries()) { + const prefix = `repetition ${repetitionIndex + 1}`; + assertObservation(repetition.cold, `${prefix} cold observation`); + if (!Array.isArray(repetition.warm) || repetition.warm.length === 0) { + throw new Error(`${prefix} recorded no warm observations`); + } + if ( + expectedWarmObservations !== undefined && + repetition.warm.length !== expectedWarmObservations + ) { + throw new Error( + `${prefix} recorded ${repetition.warm.length} warm observations, expected ${expectedWarmObservations}` + ); + } + for (const [warmIndex, observation] of repetition.warm.entries()) { + const expectedSave = warmIndex + 1; + if (observation.save !== expectedSave) { + throw new Error( + `${prefix} warm observation ${expectedSave} is labeled save ${observation.save}` + ); + } + assertObservation(observation, `${prefix} warm observation ${expectedSave}`); + } + if (repetition.scan !== undefined) { + assertObservation(repetition.scan, `${prefix} scan observation`); + } + } +} + +export function warmTrajectoryMedian(repetition: LatencyRepetition): number { + return median(repetition.warm.map((observation) => observation.durationMs)); +} + +export function summarizeLatencyRepetitions( + repetitions: LatencyRepetition[] +): ScenarioLatencySummary { + const cold = repetitions.map((repetition) => repetition.cold.durationMs); + const warm = repetitions.map(warmTrajectoryMedian); + const scans = repetitions.map((repetition) => repetition.scan?.durationMs); + const completeScans = scans.filter((duration): duration is number => duration !== undefined); + if (completeScans.length > 0 && completeScans.length !== repetitions.length) { + throw new Error( + `recorded ${completeScans.length} scan observations for ${repetitions.length} repetitions` + ); + } + + return { + cold: { processSamplesMs: cold, medianMs: median(cold) }, + warm: { processSamplesMs: warm, medianMs: median(warm) }, + ...(completeScans.length > 0 + ? { scan: { processSamplesMs: completeScans, medianMs: median(completeScans) } } + : {}), + }; +} diff --git a/code/lib/docgen-harness/src/perf/docgen-perf/types.ts b/code/lib/docgen-harness/src/perf/docgen-perf/types.ts index 71623b13c827..bc8b0c67378f 100644 --- a/code/lib/docgen-harness/src/perf/docgen-perf/types.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/types.ts @@ -1,71 +1,38 @@ -/** - * Result shapes for the per-engine docgen performance suite. The measurement contract behind these - * shapes is ../PERF-METHODOLOGY.md. - */ +/** Storybook-owned result shapes for the docgen latency suite. */ import type { EngineId } from '../docgen-shared/engine-ids.ts'; +import type { + LatencyObservation, + LatencyRepetition, + WarmLatencyObservation, +} from '../docgen-shared/latency-series.ts'; +import type { + BudgetVerdict, + ComparisonSide, + PairedEffectResult, + WorkAssessment, +} from './comparison.ts'; -export type { EngineId }; +export type { EngineId, LatencyObservation, LatencyRepetition, WarmLatencyObservation }; -/** A latency metric: median of repeated samples (fresh process each, for cold/scan). */ -export interface LatencyMetric { - status: 'measured'; - samples: number[]; - median: number; +export interface ProcessLatencySummary { + /** One independent value per fresh process. */ + processSamplesMs: number[]; + medianMs: number; } -/** A memory metric aggregated as the mean of a per-save (or per-run) series. */ -export interface SeriesMeanMetric { - status: 'measured'; - samples: number[]; - mean: number; +export interface ScenarioLatencySummary { + cold: ProcessLatencySummary; + /** Each process contributes the median of its ordered save trajectory. */ + warm: ProcessLatencySummary; + /** Only whole-project one-shot engines report this. */ + scan?: ProcessLatencySummary; } -/** A single-valued metric read from one run's series (retained growth, retained slope). */ -export interface ValueMetric { - status: 'measured'; - value: number; -} - -/** The explicit marker for a metric that does not apply to an engine; never a faked equivalent. */ -export interface NotApplicable { - status: 'n/a'; -} - -export const NOT_APPLICABLE: NotApplicable = { status: 'n/a' }; - -export interface EngineMetrics { - coldExtractionMs: LatencyMetric | NotApplicable; - warmExtractionMs: LatencyMetric | NotApplicable; - wholeProjectScanMs: LatencyMetric | NotApplicable; - peakTransientMb: SeriesMeanMetric | NotApplicable; - retainedGrowthMb: ValueMetric | NotApplicable; - retainedSlopeMbPerSave: ValueMetric | NotApplicable; -} - -/** - * What one repetition documented. Every field is optional because an engine that cannot report a - * count must leave it out; a zero would read as "documented nothing", which is a different claim. - */ -export interface MemberCounts { - /** - * Members the cold pass documented, when the engine reports it. Two engines over the same project - * can differ by an order of magnitude here, and a timing ratio between them means nothing without - * it. - */ - coldMembers?: number; - /** Members the timed re-extraction documented. Warm ratios need this for the same reason. */ - warmMembers?: number; - /** - * Of {@link coldMembers}, how many the engine documented under a type name it never resolved. - * An engine that prints `Hop19Shape` and one that expands it into its fields report the same - * member count off very different work, so equal counts alone do not make a ratio like-for-like. - */ - coldOpaqueTypes?: number; -} - -export interface ScenarioResult extends MemberCounts { +export interface ScenarioResult { params: Record; - metrics: EngineMetrics; + /** Ordered by fresh-process repetition; each warm trajectory remains ordered by save. */ + repetitions: LatencyRepetition[]; + summary: ScenarioLatencySummary; } export type EngineResult = @@ -73,61 +40,47 @@ export type EngineResult = | { status: 'skipped'; reason: string } | { status: 'failed'; reason: string }; -/** - * Whether a ratio compared equal work, and when it did not, which side did more. - * - * The direction decides how to read the number. A new engine that did less is fast for the wrong - * reason and its ratio is worthless; one that did more and still won has a ratio that understates - * it. Only `like-for-like` may become a budget. - * - * The two `resolves` cases exist because equal member counts are not enough on their own: an engine - * that records a type's name without looking through it documents exactly as many members as one - * that expanded the whole chain, off a fraction of the work. - */ -export type Comparability = - | 'unknown' - | 'like-for-like' - | 'next-documents-more' - | 'next-documents-less' - | 'next-resolves-more' - | 'next-resolves-less'; +export interface RawPairedBlock { + block: number; + order: readonly [ComparisonSide, ComparisonSide]; + control: LatencyRepetition; + candidate: LatencyRepetition; +} + +export interface ComparisonMetricResult { + work: WorkAssessment; + effect?: PairedEffectResult; + gate: BudgetVerdict; +} -/** - * One control pair's comparison for one scenario: the legacy engine's median over the new engine's, - * both measured in the same invocation. - * - * Cold and warm carry their own verdict because they are independent measurements - a pair can - * document the same members cold and different ones on the save it was timed on. - */ -export interface RatioEntry { - cold?: number; - warm?: number; - legacyColdMembers?: number; - nextColdMembers?: number; - legacyWarmMembers?: number; - nextWarmMembers?: number; - coldComparability: Comparability; - warmComparability: Comparability; - /** - * The versions the two sides resolved to. Only meaningful for a pair whose sides are the same - * package, where two equal versions mean nothing was compared at all. - */ - legacyVersion?: string; - nextVersion?: string; +export interface ComparisonScenarioResult { + seed?: number; + blocks: RawPairedBlock[]; + cold: ComparisonMetricResult; + warm: ComparisonMetricResult; } -/** Keyed by control-pair name, then by scenario name. */ -export type Ratios = Record>; +export interface ComparisonResult { + mode: 'descriptive' | 'paired-gate'; + control: EngineId; + candidate: EngineId; + controlVersion?: string; + candidateVersion?: string; + scenarios: Record; +} export interface SuiteResults { + schemaVersion: 1; generatedAt: string; nodeVersion: string; - /** The one pinned N; numbers taken at different N are not comparable. */ - pinnedN: number; - /** False for --quick smoke runs, whose numbers must never be compared against real runs. */ - comparable: boolean; - /** Resolved versions of externally-installed engines, when they ran. */ + profile: 'full' | 'quick'; + repetitions: number; + mode: 'descriptive' | 'paired-gate'; + /** False for descriptive and --quick smoke runs, even when paired blocks were requested. */ + gating: boolean; + comparisonMethod?: 'paired-log-ratio-t95-v1'; + seed?: number; engineVersions: Partial>; engines: Partial>; - ratios: Ratios; + comparisons: Record; } diff --git a/code/lib/docgen-harness/src/perf/docgen-shared/latency-series.test.ts b/code/lib/docgen-harness/src/perf/docgen-shared/latency-series.test.ts new file mode 100644 index 000000000000..23dc344bb971 --- /dev/null +++ b/code/lib/docgen-harness/src/perf/docgen-shared/latency-series.test.ts @@ -0,0 +1,97 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { runLatencySeries } from './latency-series.ts'; +import { runSeries, type SeriesEngine } from './series.ts'; + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); +}); + +describe('the separated save-series lanes', () => { + it('preserves ordered latency observations and never invokes forced GC', async () => { + vi.spyOn(console, 'log').mockImplementation(() => {}); + const gc = vi.fn(); + vi.stubGlobal('gc', gc); + const calls: string[] = []; + const engine: SeriesEngine = { + cold() { + calls.push('cold'); + return 30; + }, + applySave(save) { + calls.push(`apply-${save}`); + }, + reextract(save) { + calls.push(`extract-${save}`); + return 10 + save; + }, + dispose() { + calls.push('dispose'); + }, + }; + + const result = await runLatencySeries(engine, { saves: 3, coldLabel: 'fixture' }); + + expect(calls).toEqual([ + 'cold', + 'apply-1', + 'extract-1', + 'apply-2', + 'extract-2', + 'apply-3', + 'extract-3', + 'dispose', + ]); + expect(result.cold.members).toBe(30); + expect(result.warm.map(({ save, members }) => ({ save, members }))).toEqual([ + { save: 1, members: 11 }, + { save: 2, members: 12 }, + { save: 3, members: 13 }, + ]); + expect(gc).not.toHaveBeenCalled(); + }); + + it('retains the documented work for every memory save', async () => { + vi.spyOn(console, 'log').mockImplementation(() => {}); + const result = await runSeries( + { + cold: () => 20, + applySave: () => {}, + reextract: (save) => save * 2, + }, + { saves: 3, coldLabel: 'fixture', forceGc: false } + ); + + expect( + result.samples.map(({ save, documentedMembers }) => ({ save, documentedMembers })) + ).toEqual([ + { save: 1, documentedMembers: 2 }, + { save: 2, documentedMembers: 4 }, + { save: 3, documentedMembers: 6 }, + ]); + expect(result.warmMembers).toBe(6); + }); + + it('rejects an empty trajectory and disposes after a measurement failure', async () => { + vi.spyOn(console, 'log').mockImplementation(() => {}); + const dispose = vi.fn(); + const engine: SeriesEngine = { + cold: () => 1, + applySave: () => {}, + reextract: () => { + throw new Error('extract failed'); + }, + dispose, + }; + + await expect(runLatencySeries(engine, { saves: 1, coldLabel: 'fixture' })).rejects.toThrow( + 'extract failed' + ); + expect(dispose).toHaveBeenCalledOnce(); + await expect(runLatencySeries(engine, { saves: 0, coldLabel: 'fixture' })).rejects.toThrow( + 'positive safe integer' + ); + expect(dispose).toHaveBeenCalledTimes(2); + }); +}); diff --git a/code/lib/docgen-harness/src/perf/docgen-shared/latency-series.ts b/code/lib/docgen-harness/src/perf/docgen-shared/latency-series.ts new file mode 100644 index 000000000000..51b65b480832 --- /dev/null +++ b/code/lib/docgen-harness/src/perf/docgen-shared/latency-series.ts @@ -0,0 +1,103 @@ +/** + * Latency-only execution for stateful docgen save trajectories. Memory sampling deliberately lives + * in `series.ts`; keeping this loop separate prevents forced GC from perturbing latency samples. + */ +import { writeFileSync } from 'node:fs'; + +import type { SeriesEngine } from './series.ts'; + +export interface LatencyObservation { + durationMs: number; + members?: number; + opaqueTypes?: number; +} + +export interface WarmLatencyObservation extends LatencyObservation { + /** Save number in execution order, counting from one. */ + save: number; +} + +/** One fresh process's complete, ordered latency result. */ +export interface LatencyRepetition { + cold: LatencyObservation; + warm: WarmLatencyObservation[]; + /** Whole-project one-shot engines use this; per-component engines leave it absent. */ + scan?: LatencyObservation; +} + +export interface LatencySeriesOptions { + saves: number; + /** Describes the measured set for the cold-pass log line, e.g. "300 components". */ + coldLabel: string; +} + +export async function runLatencySeries( + engine: SeriesEngine, + options: LatencySeriesOptions +): Promise { + let failed = false; + try { + if (!Number.isSafeInteger(options.saves) || options.saves < 1) { + throw new Error( + `latency series saves must be a positive safe integer, received ${options.saves}` + ); + } + console.log(` full extraction over ${options.coldLabel} (cold pass)…`); + const coldStart = performance.now(); + const coldMembers = await engine.cold(); + const cold = { durationMs: performance.now() - coldStart, members: coldMembers }; + console.log( + ` cold pass: ${cold.durationMs.toFixed(0)}ms` + + (cold.members !== undefined ? ` (${cold.members} documented members)` : '') + ); + + const warm: WarmLatencyObservation[] = []; + for (let save = 1; save <= options.saves; save++) { + await engine.applySave(save); + const start = performance.now(); + const members = await engine.reextract(save); + const durationMs = performance.now() - start; + warm.push({ save, durationMs, members }); + console.log( + ` save ${String(save).padStart(3)}: ${durationMs.toFixed(1).padStart(7)}ms` + + (members !== undefined ? ` members=${members}` : '') + ); + } + + return { cold, warm }; + } catch (error) { + failed = true; + throw error; + } finally { + try { + engine.dispose?.(); + } catch (error) { + if (!failed) { + throw error; + } + } + } +} + +export interface LatencySeriesHarnessSpec extends LatencySeriesOptions { + title: string; + options: object; + banner?: Record; + jsonOut?: string; + setup(): SeriesEngine | Promise; +} + +export async function runLatencySeriesHarness(spec: LatencySeriesHarnessSpec): Promise { + console.log(spec.title); + console.log( + ` ${Object.entries(spec.banner ?? spec.options) + .map(([key, value]) => `${key}=${String(value)}`) + .join(' ')}` + ); + + const result = await runLatencySeries(await spec.setup(), spec); + if (spec.jsonOut) { + writeFileSync(spec.jsonOut, JSON.stringify({ options: spec.options, ...result }, null, 2)); + console.log(` wrote ${spec.jsonOut}`); + } +} diff --git a/code/lib/docgen-harness/src/perf/docgen-shared/samples.ts b/code/lib/docgen-harness/src/perf/docgen-shared/samples.ts index 84cbe5a8254e..2c604c76e11c 100644 --- a/code/lib/docgen-harness/src/perf/docgen-shared/samples.ts +++ b/code/lib/docgen-harness/src/perf/docgen-shared/samples.ts @@ -12,4 +12,6 @@ export interface SaveSample extends MemorySample { save: number; /** The duration of the save's re-extraction, in milliseconds. */ durMs: number; + /** Members documented by this exact re-extraction, when the engine reports it. */ + documentedMembers?: number; } diff --git a/code/lib/docgen-harness/src/perf/docgen-shared/series.ts b/code/lib/docgen-harness/src/perf/docgen-shared/series.ts index 4c7e290e4dc9..f0d881656745 100644 --- a/code/lib/docgen-harness/src/perf/docgen-shared/series.ts +++ b/code/lib/docgen-harness/src/perf/docgen-shared/series.ts @@ -1,27 +1,25 @@ /** - * The save-series harness every docgen engine child runs: + * The memory save-series harness: * one timed cold pass, then K simulated saves that mutate the project on disk, invalidate the * engine's caches, and re-extract, with memory sampled around forced GC after every save. Engines * differ only in what those steps mean, so they implement {@link SeriesEngine} and this module owns * the timing. */ -import { writeFileSync } from 'node:fs'; - import type { MemorySample, SaveSample } from './samples.ts'; -import { formatSampleLine, gcAvailable, sampleMemory } from './sampling.ts'; +import { formatSampleLine, sampleMemory } from './sampling.ts'; import { type SeriesSummary, summarizeSeries } from './stats.ts'; export interface SeriesEngine { /** * One full extraction over the measured set, from a cold start. Returns how many members it * documented, when the engine can report that; two engines over one project can differ by an order - * of magnitude, and a timing ratio between them means nothing without it. + * of magnitude, and a timing comparison between them means nothing without it. */ - cold(): Promise; + cold(): number | undefined | Promise; /** Mutate the project on disk for save `save` and invalidate the engine's caches. Never timed. */ - applySave(save: number): Promise; + applySave(save: number): void | Promise; /** Re-extract after save `save`. This call, and only this call, is the warm sample. */ - reextract(save: number): Promise; + reextract(save: number): number | undefined | Promise; dispose?(): void; } @@ -66,13 +64,13 @@ export async function runSeries( // performance.now(), not Date.now(): a warm re-extraction of a single component runs in single // -digit milliseconds, and at Date.now()'s 1ms granularity a whole series can median to 0 - a - // number every ratio taken against it then divides by. + // number any comparison against it would then divide by. const saveStart = performance.now(); warmMembers = await engine.reextract(save); const durMs = performance.now() - saveStart; const mem = sampleMemory(forceGc); - samples.push({ save, durMs, ...mem }); + samples.push({ save, durMs, documentedMembers: warmMembers, ...mem }); console.log(formatSampleLine(save, durMs, mem)); } @@ -105,43 +103,6 @@ export function printSeriesSummary(result: SeriesResult, saves: number): void { } } -export interface SeriesHarnessSpec extends SeriesOptions { - /** Banner line, e.g. `vue-component-meta harness (workspace)`. */ - title: string; - /** The resolved options, recorded in the result JSON so a stored run is self-describing. */ - options: object; - /** The subset echoed under the banner. Defaults to {@link options}. */ - banner?: Record; - jsonOut?: string; - /** Build the project and the engine. Runs before the cold pass, so its cost is not measured. */ - setup(): Promise; -} - -/** - * Children own only their {@link SeriesEngine}; everything the orchestrator reads back is produced - * here so the children cannot drift apart in what they report. - */ -export async function runSeriesHarness(spec: SeriesHarnessSpec): Promise { - console.log(spec.title); - console.log( - ` ${Object.entries(spec.banner ?? spec.options) - .map(([key, value]) => `${key}=${String(value)}`) - .join(' ')}` - ); - if ((spec.forceGc ?? true) && !gcAvailable()) { - console.log(' (run with `node --expose-gc` to measure retained heap; continuing without it)'); - } - - const engine = await spec.setup(); - const result = await runSeries(engine, spec); - printSeriesSummary(result, spec.saves); - - if (spec.jsonOut) { - writeFileSync(spec.jsonOut, JSON.stringify({ options: spec.options, ...result }, null, 2)); - console.log(` wrote ${spec.jsonOut}`); - } -} - /** Entry-point wrapper: any throw becomes a non-zero exit, which is how the orchestrator sees it. */ export function harnessMain(run: () => Promise): void { run().catch((err) => { diff --git a/scripts/knip.config.ts b/scripts/knip.config.ts index d010701d87e6..74d2e85f3ad0 100644 --- a/scripts/knip.config.ts +++ b/scripts/knip.config.ts @@ -68,6 +68,7 @@ const baseConfig = { 'lib/docgen-harness': { entry: [ 'src/index.ts', + 'src/perf/docgen-latency/vue-component-meta.ts', 'src/perf/docgen-perf/run.ts', 'src/perf/docgen-perf/engines/{react-legacy,vue-component-meta,vue-docgen-api}.ts', 'src/perf/docgen-perf/generators/{angular,vue}.ts', diff --git a/yarn.lock b/yarn.lock index 88d38675b0f4..205fe4bf57c0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12307,6 +12307,7 @@ __metadata: react-dom: "npm:^18.2.0" rxjs: "npm:^7.4.0" storybook: "workspace:*" + tinybench: "npm:6.1.2" typescript: "npm:^6.0.3" vue: "npm:^3.2.47" vue-component-meta: "npm:^3.3.9" @@ -37579,6 +37580,13 @@ __metadata: languageName: node linkType: hard +"tinybench@npm:6.1.2": + version: 6.1.2 + resolution: "tinybench@npm:6.1.2" + checksum: 10c0/5573a48afd54ac0506f9dbca8beb46b47032d029e5ee8ff931aba55e65146283b7ef9aa73af7060ccbb8ec88642898feee1bcf7037f45e372530bf4ce194ea4e + languageName: node + linkType: hard + "tinybench@npm:^2.9.0": version: 2.9.0 resolution: "tinybench@npm:2.9.0"