Bench: Add the docgen perf methodology note, the per-engine harness and the shared bench plumbing

scripts/bench/docgen-memory/gate.ts

@@ -2,18 +2,15 @@
  * CI regression gate for the docgen-server memory behavior
  * (https://github.com/storybookjs/storybook/issues/35260).
  *
- * Runs {@link file://./memory-harness.ts} in fresh child processes (so each measurement starts from a
- * clean heap) and asserts two independent things:
+ * Runs {@link file://./memory-harness.ts} in fresh child processes (one clean heap per measurement)
+ * and asserts two independent things:
  *
- *   1. Steady-state churn/leak-freeness (`changed-scope` metric config): post-GC `heapUsed` must not
- *      climb across saves, and the per-save transient working set must stay under budget. A regression
- *      that re-extracts the whole index would blow past these.
- *   2. The OOM fix itself, as a **crash → survive negative control** (`live` configs): the live path
- *      (many per-component `batchExtract` calls on the shared program) is run under a tight heap cap
- *      both WITH and WITHOUT program recycling.
- *        - recycle OFF (`--recycle off`, ratio Infinity) MUST OOM.
- *        - recycle ON (default) MUST survive.
- *      Asserting the flip (not just survival) is what guards the fix rather than the cap.
+ *   1. Steady-state leak-freeness (`changed-scope` metric config): post-GC `heapUsed` must not
+ *      climb across saves, and the per-save transient working set must stay under budget.
+ *   2. The OOM fix itself, as a **crash → survive negative control** (`live` configs): the live
+ *      path is run under a tight heap cap both WITH and WITHOUT program recycling - recycle OFF
+ *      (`--recycle off`, ratio Infinity) MUST OOM, recycle ON (default) MUST survive. Asserting
+ *      the flip, not just survival, is what guards the fix rather than the cap.
  *
  * Run:
  *   yarn bench:docgen-memory          # from scripts/
@@ -23,6 +20,8 @@ import { spawnSync } from 'node:child_process';
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 
+import { type MemoryBudgets, memoryBudgetsFor } from '../docgen-shared/budgets.ts';
+import { outputTail } from '../docgen-shared/child-output.ts';
 import { SANDBOX_DIRECTORY } from '../../utils/constants.ts';
 
 interface HarnessResult {
@@ -33,15 +32,6 @@ interface HarnessResult {
   avgTransient?: number;
 }
 
-interface MetricBudgets {
-  /** Max allowed post-GC retained growth (MB) across the run. */
-  maxRetainedGrowthMb: number;
-  /** Max allowed average transient working set added per save (MB). */
-  maxTransientMb: number;
-  /** Max allowed post-GC retained-heap slope (MB/save). */
-  maxRetainedSlopeMb: number;
-}
-
 interface GateConfig {
   name: string;
   args: string[];
@@ -50,12 +40,12 @@ interface GateConfig {
   /** `--max-old-space-size` cap (MB) for the child. Omit to run uncapped. */
   capMb?: number;
   /**
-   * Crash → survive negative control. `true` ⇒ the child MUST OOM (recycle disabled); `false` ⇒ it
-   * MUST survive (recycle on). Omit for metric-only configs.
+   * Crash → survive negative control: `true` ⇒ MUST OOM (recycle disabled), `false` ⇒ MUST
+   * survive (recycle on). Omit for metric-only configs.
    */
   expectOom?: boolean;
   /** Post-GC metric budgets, asserted from `result.json` (metric configs only). */
-  budgets?: MetricBudgets;
+  budgets?: MemoryBudgets;
 }
 
 const HARNESS = path.join(import.meta.dirname, 'memory-harness.ts');
@@ -78,7 +68,7 @@ const CONFIGS: GateConfig[] = [
       '--scope', 'changed',
     ],
     outSubdir: 'changed-scope',
-    budgets: { maxRetainedGrowthMb: 60, maxTransientMb: 90, maxRetainedSlopeMb: 3 },
+    budgets: memoryBudgetsFor('react-osa'),
   },
   {
     // Positive control: the live per-edit workload survives under a tight cap BECAUSE the shared
@@ -153,15 +143,7 @@ function runHarness(config: GateConfig): RunOutcome {
 
   const proc = spawnSync(process.execPath, nodeArgs, { encoding: 'utf8' });
   const output = `${proc.stdout ?? ''}${proc.stderr ?? ''}`;
-  // Echo a compact tail so CI logs show what happened without the full V8 crash dump.
-  console.log(
-    output
-      .trim()
-      .split('\n')
-      .slice(-8)
-      .map((line) => `    ${line}`)
-      .join('\n')
-  );
+  console.log(outputTail(output, 8));
 
   const result = fs.existsSync(jsonPath)
     ? (JSON.parse(fs.readFileSync(jsonPath, 'utf8')) as HarnessResult)

scripts/bench/docgen-memory/generate-project.ts

@@ -19,6 +19,11 @@ import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { pathToFileURL } from 'node:url';
 
+import { z } from 'zod';
+
+import { countOption, parseHarnessOptions } from '../docgen-shared/args.ts';
+import type { ComponentRefLike, StoryRefLike } from '../docgen-shared/react-renderer-module.ts';
+
 const require = createRequire(import.meta.url);
 
 export interface GenerateOptions {
@@ -239,6 +244,26 @@ ${variantExports}
 `;
 }
 
+/** The renderer-side reference for the generated `Comp{i}`, matching the naming used above. */
+export function componentRef(i: number, componentPath: string): ComponentRefLike {
+  return {
+    componentName: `Comp${i}`,
+    importName: `Comp${i}`,
+    localImportName: `Comp${i}`,
+    importId: `./Comp${i}`,
+    isPackage: false,
+    path: componentPath,
+  };
+}
+
+/** Pairs {@link GeneratedProject.componentPaths} with `storyPaths`, which share an index. */
+export function buildStoryRefs(componentPaths: string[], storyPaths: string[]): StoryRefLike[] {
+  return componentPaths.map((componentPath, i) => ({
+    storyPath: storyPaths[i],
+    component: componentRef(i, componentPath),
+  }));
+}
+
 export function generateProject(options: GenerateOptions): GeneratedProject {
   const outDir = path.resolve(options.outDir);
   fs.rmSync(outDir, { recursive: true, force: true });
@@ -275,25 +300,40 @@ export function generateProject(options: GenerateOptions): GeneratedProject {
   return { outDir, configPath, componentPaths, storyPaths };
 }
 
-function parseArgs(argv: string[]): GenerateOptions {
-  const get = (flag: string, fallback: string) => {
-    const idx = argv.indexOf(flag);
-    return idx >= 0 && argv[idx + 1] ? argv[idx + 1] : fallback;
-  };
-  return {
-    outDir: get('--out', '../storybook-sandboxes/docgen-memory-stress'),
-    components: Number(get('--components', '500')),
-    variants: Number(get('--variants', '4')),
-    props: Number(get('--props', '8')),
-    heavyTypes: argv.includes('--heavy'),
-    heavyFactor: Number(get('--heavy-factor', '1')),
-    base64Kb: Number(get('--base64-kb', '0')),
-    withNodeModules: !argv.includes('--no-node-modules'),
-  };
+function parseOptions(argv: string[]): GenerateOptions {
+  return parseHarnessOptions<GenerateOptions>(
+    argv,
+    {
+      out: { type: 'string' },
+      components: { type: 'string' },
+      variants: { type: 'string' },
+      props: { type: 'string' },
+      heavy: { type: 'boolean' },
+      'heavy-factor': { type: 'string' },
+      'base64-kb': { type: 'string' },
+      'no-node-modules': { type: 'boolean' },
+    } as const,
+    z.object({
+      outDir: z.string().default('../storybook-sandboxes/docgen-memory-stress'),
+      components: countOption(500),
+      variants: countOption(4),
+      props: countOption(8),
+      heavyTypes: z.boolean().default(false),
+      heavyFactor: countOption(1),
+      base64Kb: countOption(0),
+      withNodeModules: z.boolean().default(true),
+    }),
+    (values) => ({
+      ...values,
+      outDir: values.out,
+      heavyTypes: values.heavy,
+      withNodeModules: !values.noNodeModules,
+    })
+  );
 }
 
 if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
-  const options = parseArgs(process.argv.slice(2));
+  const options = parseOptions(process.argv.slice(2));
   const start = Date.now();
   const result = generateProject(options);
   console.log(