Bench: Move the docgen perf suite into the docgen harness

AGENTS.md

@@ -160,6 +160,8 @@ yarn storybook:vitest
 | Generate a sandbox              | `yarn task sandbox --template react-vite/default-ts --start-from auto`         |
 | Run sandbox E2E tests           | `yarn task e2e-tests-dev --template react-vite/default-ts --start-from auto`   |
 | Run sandbox test-runner tests   | `yarn task test-runner-dev --template react-vite/default-ts --start-from auto` |
+| Run the docgen perf bench       | `yarn workspace @storybook/docgen-harness bench:docgen-perf`                   |
+| Run the docgen memory gate      | `yarn workspace @storybook/docgen-harness bench:docgen-memory`                 |
 
 ## NX and `yarn task`
 

code/.oxlintrc.json

@@ -95,6 +95,12 @@
         "local-rules/no-uncategorized-errors": "off"
       }
     },
+    {
+      "files": ["**/lib/docgen-harness/src/perf/**"],
+      "rules": {
+        "local-rules/no-uncategorized-errors": "off"
+      }
+    },
     {
       "files": ["**/core/src/preview-errors.ts"],
       "rules": {

code/lib/docgen-harness/README.md

@@ -50,7 +50,12 @@ src/
 │   └── __testfixtures__/<case>/  # component, stories, compodoc-input.json, aot-cmp.ts (signal cases),
 │                                 # argtypes.snapshot, argtypes-filtered.snapshot, snippet-<story>.snapshot
 ├── svelte/                       # planned
-└── web-components/               # planned
+├── web-components/               # planned
+└── perf/                         # the performance bench, see below
+    ├── PERF-METHODOLOGY.md       # the measurement contract
+    ├── docgen-perf/              # per-engine latency and memory suite, plus its engines/ and generators/
+    ├── docgen-memory/            # the docgen-server memory regression gate
+    └── docgen-shared/            # sampling, stats, budgets and paths shared by both
 ```
 
 ## The comparator
@@ -165,7 +170,27 @@ Each has a red marker in `vue3-legacy-gaps.test.ts`.
 - #29697 (not reproduced) -> `signal-io/`: aliased signal inputs record under their alias at 2.0.0; regression baseline, no marker.
 - #22007 -> `properties-methods-noise/`: the filter flag's origin case, and the fixture where both flag states meaningfully differ.
 
+## The performance bench
+
+`src/perf/` measures how fast the docgen engines are and how much memory they hold, which is the other half of the "docgen beyond React" question the snapshot comparator above answers for correctness.
+It is a set of CLIs rather than part of this package's exported API, so nothing in `src/perf/` is re-exported from `src/index.ts`.
+
+Both commands run from `code/lib/docgen-harness`:
+
+```bash
+yarn bench:docgen-perf            # per-engine cold/warm latency and memory, full profile
+yarn bench:docgen-perf --quick    # smoke profile; its numbers are marked non-comparable
+yarn bench:docgen-memory          # the docgen-server memory regression gate CI runs daily
+```
+
+`bench:docgen-perf` generates synthetic projects under the shared sandbox directory, runs each engine in its own child process, and writes a results JSON next to them.
+`bench:docgen-memory` asserts both that re-extraction is leak-free and that the program-recycle fix still flips a tight-heap run from OOM to survival.
+
+Read `src/perf/PERF-METHODOLOGY.md` before changing a metric, a budget, or a version pair.
+It is the contract the numbers are only meaningful under.
+
+The bench also carries unit tests for its own aggregation, reporting and generator logic, so `yarn test code/lib/docgen-harness` runs those alongside the fixture comparisons.
+
 ## What does not live here
 
-- The performance harness stays under `scripts/`.
 - Framework provider code lives in each framework's own package.

code/lib/docgen-harness/package.json

@@ -25,22 +25,38 @@
     },
     "./package.json": "./package.json"
   },
+  "scripts": {
+    "bench:docgen-memory": "node --import jiti/register ./src/perf/docgen-memory/gate.ts",
+    "bench:docgen-perf": "node ./src/perf/docgen-perf/run.ts"
+  },
   "devDependencies": {
     "@angular/common": "^21.2.14",
     "@angular/compiler": "^21.2.14",
     "@angular/core": "^21.2.14",
     "@angular/platform-browser": "^21.2.14",
     "@angular/platform-browser-dynamic": "^21.2.14",
+    "@compodoc/compodoc": "2.0.0",
     "@storybook/angular-vite": "workspace:*",
+    "@storybook/react": "workspace:*",
     "@storybook/vue3": "workspace:*",
     "@testing-library/vue": "^8.0.0",
+    "@types/react": "^18.0.37",
     "@vitejs/plugin-vue": "^4.6.2",
+    "jiti": "^2.6.1",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
     "rxjs": "^7.4.0",
+    "storybook": "workspace:*",
     "typescript": "^6.0.3",
     "vue": "^3.2.47",
     "vue-component-meta": "^3.3.9",
+    "vue-component-meta-next": "npm:vue-component-meta@3.3.8",
     "vue-docgen-api": "^4.75.1",
     "vue-tsc": "latest",
+    "zod": "^3.25.76",
     "zone.js": "^0.16.2"
+  },
+  "nx": {
+    "includedScripts": []
   }
 }

code/lib/docgen-harness/project.json

@@ -6,6 +6,7 @@
     "compile": {},
     "check": {
       "executor": "nx:run-commands",
+      "dependsOn": [],
       "options": { "cwd": "{projectRoot}", "command": "vue-tsc --noEmit" }
     }
   },

scripts/bench/PERF-METHODOLOGY.md → code/lib/docgen-harness/src/perf/PERF-METHODOLOGY.md

@@ -61,10 +61,10 @@ A bump proposed in a pull request is fully covered by this. Catching a regressio
 
 #### Running a version comparison
 
-The pair that exists today is `vue-component-meta` against `vue-component-meta-next`, which are two installs of one package: the second is an alias in `scripts/package.json` pinned to an exact version.
+The pair that exists today is `vue-component-meta` against `vue-component-meta-next`, which are two installs of one package: the second is an alias in `code/lib/docgen-harness/package.json` pinned to an exact version.
 
 **1. Point the alias at the version you want to test.**
-Edit `scripts/package.json` and install, so the candidate is on disk:
+Edit `code/lib/docgen-harness/package.json` and install, so the candidate is on disk:
 
 ```jsonc
 "vue-component-meta": "^3.2.7",                          // the current side
@@ -79,7 +79,7 @@ Pin the candidate exactly rather than with a range.
 A range on both sides can resolve to one install, and then the run compares an engine against itself.
 
 **2. Run both sides in one invocation, naming each explicitly.**
-From `scripts/`:
+From `code/lib/docgen-harness/`:
 
 ```bash
 yarn bench:docgen-perf --engine vue-component-meta --engine vue-component-meta-next
@@ -115,11 +115,11 @@ An engine whose package does not resolve is reported as skipped with a reason ra
 
 This works only for an engine whose child imports the versioned package directly, the way `engines/vue-component-meta.ts` does.
 Where the harness reaches the engine through repo source instead - `react-legacy` goes via `loadReactRendererModule` into `code/renderers/react`, which imports `react-docgen` by bare specifier - no child flag can redirect that import, and a version pair needs a different approach entirely.
-Declaring `@storybook/react` as a dependency of `scripts` is what makes that source reachable in the first place, but it does not make the specifier redirectable: pointing the renderer at a second `react-docgen` would take a module resolution hook registered in the child, which is not built.
+Declaring `@storybook/react` as a dependency of `@storybook/docgen-harness` is what makes that source reachable in the first place, but it does not make the specifier redirectable: pointing the renderer at a second `react-docgen` would take a module resolution hook registered in the child, which is not built.
 
 Four data edits:
 
-1. An alias in `scripts/package.json` pinned to the candidate version.
+1. An alias in `code/lib/docgen-harness/package.json` pinned to the candidate version.
 2. The new id added to the `EngineId` union in `docgen-shared/engine-ids.ts`, which is hand-maintained: a registry entry naming an id that is not in the union does not compile.
 3. A second registry entry in `docgen-perf/registry.ts` reusing the same child, with `inDefaultRun: false`, `versionPackage` naming the alias, and the flag that tells the child which install to load.
 4. An entry in `CONTROL_PAIRS` in `docgen-perf/ratios.ts` naming the current side as `legacy` and the alias as `next`.

scripts/bench/docgen-memory/gate.ts → code/lib/docgen-harness/src/perf/docgen-memory/gate.ts

@@ -12,9 +12,9 @@
  *      (`--recycle off`, ratio Infinity) MUST OOM, recycle ON (default) MUST survive. Asserting
  *      the flip, not just survival, is what guards the fix rather than the cap.
  *
- * Run:
- *   yarn bench:docgen-memory          # from scripts/
- *   node --import jiti/register scripts/bench/docgen-memory/gate.ts
+ * Run from code/lib/docgen-harness:
+ *   yarn bench:docgen-memory
+ *   node --import jiti/register src/perf/docgen-memory/gate.ts
  */
 import { spawnSync } from 'node:child_process';
 import * as fs from 'node:fs';
@@ -62,12 +62,18 @@ const CONFIGS: GateConfig[] = [
     // reintroduces whole-index re-extraction.
     name: 'changed-scope steady state (N=600, props=10, 20 saves)',
     args: [
-      '--components', '600',
-      '--variants', '4',
-      '--props', '10',
-      '--saves', '20',
-      '--mode', 'refresh',
-      '--scope', 'changed',
+      '--components',
+      '600',
+      '--variants',
+      '4',
+      '--props',
+      '10',
+      '--saves',
+      '20',
+      '--mode',
+      'refresh',
+      '--scope',
+      'changed',
     ],
     outSubdir: 'changed-scope',
     budgets: memoryBudgetsFor('react-osa'),
@@ -77,12 +83,18 @@ const CONFIGS: GateConfig[] = [
     // program is recycled under heap pressure. Pair with the negative control below.
     name: 'live path survives WITH recycling (N=800, --heavy, cap 1536MB)',
     args: [
-      '--components', '800',
-      '--variants', '4',
-      '--props', '10',
-      '--saves', '1',
-      '--mode', 'live',
-      '--recycle', 'on',
+      '--components',
+      '800',
+      '--variants',
+      '4',
+      '--props',
+      '10',
+      '--saves',
+      '1',
+      '--mode',
+      'live',
+      '--recycle',
+      'on',
       '--heavy',
       '--no-force-gc',
     ],
@@ -93,14 +105,20 @@ const CONFIGS: GateConfig[] = [
   {
     // Negative control: the SAME workload + cap, with recycling disabled, MUST OOM. If this ever
     // survives, the cap is too loose (or the workload shrank), so fail and re-tune the pair.
-    name: 'live path OOMs WITHOUT recycling — negative control (N=800, --heavy, cap 1536MB)',
+    name: 'live path OOMs WITHOUT recycling - negative control (N=800, --heavy, cap 1536MB)',
     args: [
-      '--components', '800',
-      '--variants', '4',
-      '--props', '10',
-      '--saves', '1',
-      '--mode', 'live',
-      '--recycle', 'off',
+      '--components',
+      '800',
+      '--variants',
+      '4',
+      '--props',
+      '10',
+      '--saves',
+      '1',
+      '--mode',
+      'live',
+      '--recycle',
+      'off',
       '--heavy',
       '--no-force-gc',
     ],
@@ -167,9 +185,21 @@ function checkMetricConfig(config: GateConfig, outcome: RunOutcome, failures: st
 
   const { budgets, name } = config;
   const checks: Array<{ label: string; value: number | undefined; budget: number }> = [
-    { label: 'retained growth (MB)', value: outcome.result.retainedGrowth, budget: budgets.maxRetainedGrowthMb },
-    { label: 'avg transient/save (MB)', value: outcome.result.avgTransient, budget: budgets.maxTransientMb },
-    { label: 'retained slope (MB/save)', value: outcome.result.retainedSlope, budget: budgets.maxRetainedSlopeMb },
+    {
+      label: 'retained growth (MB)',
+      value: outcome.result.retainedGrowth,
+      budget: budgets.maxRetainedGrowthMb,
+    },
+    {
+      label: 'avg transient/save (MB)',
+      value: outcome.result.avgTransient,
+      budget: budgets.maxTransientMb,
+    },
+    {
+      label: 'retained slope (MB/save)',
+      value: outcome.result.retainedSlope,
+      budget: budgets.maxRetainedSlopeMb,
+    },
   ];
 
   for (const { label, value, budget } of checks) {
@@ -198,12 +228,14 @@ function checkNegativeControl(config: GateConfig, outcome: RunOutcome, failures:
     }
     if (!crashed) {
       failures.push(
-        `${config.name}: expected an OOM without recycling, but the run survived — the cap is too ` +
+        `${config.name}: expected an OOM without recycling, but the run survived - the cap is too ` +
           `loose or the workload shrank; re-tune the live config pair.`
       );
       console.log('  ✗ expected OOM, but survived');
     } else {
-      failures.push(`${config.name}: crashed (status ${outcome.status}) but without a heap-OOM signature`);
+      failures.push(
+        `${config.name}: crashed (status ${outcome.status}) but without a heap-OOM signature`
+      );
       console.log(`  ✗ crashed (status ${outcome.status}) but not a heap OOM`);
     }
     return;

scripts/bench/docgen-memory/generate-project.ts → code/lib/docgen-harness/src/perf/docgen-memory/generate-project.ts

@@ -3,16 +3,16 @@
  *
  * Produces N component files (each with a non-trivial props interface that extends React DOM types,
  * so TypeScript type resolution has realistic cost) plus a co-located `.stories.tsx` with M variants.
- * Used by both the in-process memory harness (`memory-harness.ts`) and — at larger scale — as a real
+ * Used by both the in-process memory harness (`memory-harness.ts`) and - at larger scale - as a real
  * buildable/deployable Storybook for the "+5K components" benchmark from
  * https://github.com/storybookjs/storybook/issues/34824.
  *
  * The `--heavy` lever (optionally scaled with `--heavy-factor N`) grows the TypeScript checker
  * working set by emitting inline literal-union props that `serializeType` expands into retained enum
- * arrays — the density that lets the in-process harness actually OOM.
+ * arrays - the density that lets the in-process harness actually OOM.
  *
- * Run directly:
- *   node --import jiti/register scripts/bench/docgen-memory/generate-project.ts --out ../storybook-sandboxes/docgen-stress --components 5000 --variants 5
+ * Run directly, from code/lib/docgen-harness (--out defaults into the sandbox directory):
+ *   node --import jiti/register src/perf/docgen-memory/generate-project.ts --components 5000 --variants 5
  */
 import { createRequire } from 'node:module';
 import * as fs from 'node:fs';
@@ -22,6 +22,7 @@ import { pathToFileURL } from 'node:url';
 import { z } from 'zod';
 
 import { countOption, parseHarnessOptions, positiveCountOption } from '../docgen-shared/args.ts';
+import { SANDBOX_DIRECTORY } from '../docgen-shared/paths.ts';
 import type { ComponentRefLike, StoryRefLike } from '../docgen-shared/react-renderer-module.ts';
 
 const require = createRequire(import.meta.url);
@@ -37,7 +38,7 @@ export interface GenerateOptions {
   props: number;
   /**
    * Inflate the *type-resolution* cost per component with large constructed types (big unions,
-   * mapped types, deep nesting, generics) — the lever that actually grows the TypeScript checker
+   * mapped types, deep nesting, generics) - the lever that actually grows the TypeScript checker
    * working set, mirroring complex real-world component libraries.
    */
   heavyTypes: boolean;
@@ -148,10 +149,10 @@ function propLines(componentIndex: number, extraProps: number): string {
  *
  *   - Transient resolution: a big token union + a mapped type over it + a deeply nested generic.
  *     These inflate the per-save type-checking spike but are emitted as type *aliases*, so
- *     `serializeType` only prints their name — they are not deeply retained on the doc.
+ *     `serializeType` only prints their name - they are not deeply retained on the doc.
  *   - Retained working set: `inline literal-union` props whose members are unique per component and
  *     per prop. `serializeType` expands these into `enum` value arrays stored on the emitted doc, so
- *     they stay resident for the life of the program — mirroring real libraries whose warm working
+ *     they stay resident for the life of the program - mirroring real libraries whose warm working
  *     set sits permanently near the heap cap.
  */
 function heavyTypeBlock(i: number, factor: number): { typeDefs: string; propLines: string } {
@@ -162,10 +163,7 @@ function heavyTypeBlock(i: number, factor: number): { typeDefs: string; propLine
   // Inline literal-union props: unique members per (component, prop) so the checker and the emitted
   // doc cannot dedupe them across components. These are the dominant retained-memory lever.
   const enumLines = Array.from({ length: enumProps }, (_, k) => {
-    const members = Array.from(
-      { length: tokenCount },
-      (_, t) => `'c${i}_p${k}_v${t}'`
-    ).join(' | ');
+    const members = Array.from({ length: tokenCount }, (_, t) => `'c${i}_p${k}_v${t}'`).join(' | ');
     return `  /** Inline literal union ${k}. */
   enum${k}?: ${members};`;
   }).join('\n');
@@ -203,9 +201,7 @@ export function componentSource(
   extraProps: number,
   opts: { heavyTypes?: boolean; heavyFactor?: number; base64Kb?: number } = {}
 ): string {
-  const heavy = opts.heavyTypes
-    ? heavyTypeBlock(i, opts.heavyFactor ?? 1)
-    : undefined;
+  const heavy = opts.heavyTypes ? heavyTypeBlock(i, opts.heavyFactor ?? 1) : undefined;
   const base64 = opts.base64Kb ? base64Block(i, opts.base64Kb) : '';
   return `import * as React from 'react';
 
@@ -214,7 +210,7 @@ ${propLines(i, extraProps)}${heavy ? `\n${heavy.propLines}` : ''}
 }
 
 /**
- * Comp${i} — generated stress-test component.
+ * Comp${i} - generated stress-test component.
  */
 export const Comp${i} = ({ label, ...rest }: Comp${i}Props) => {
   return <div {...rest}>{label}</div>;
@@ -314,7 +310,7 @@ function parseOptions(argv: string[]): GenerateOptions {
       'no-node-modules': { type: 'boolean' },
     } as const,
     z.object({
-      outDir: z.string().default('../storybook-sandboxes/docgen-memory-stress'),
+      outDir: z.string().default(path.join(SANDBOX_DIRECTORY, 'docgen-memory-stress')),
       components: countOption(500),
       variants: countOption(4),
       props: countOption(8),