diff --git a/AGENTS.md b/AGENTS.md index 2bf976081d14..38bc2d1c6ca8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -160,6 +160,8 @@ yarn storybook:vitest | Generate a sandbox | `yarn task sandbox --template react-vite/default-ts --start-from auto` | | Run sandbox E2E tests | `yarn task e2e-tests-dev --template react-vite/default-ts --start-from auto` | | Run sandbox test-runner tests | `yarn task test-runner-dev --template react-vite/default-ts --start-from auto` | +| Run the docgen perf bench | `yarn workspace @storybook/docgen-harness bench:docgen-perf` | +| Run the docgen memory gate | `yarn workspace @storybook/docgen-harness bench:docgen-memory` | ## NX and `yarn task` diff --git a/code/.oxlintrc.json b/code/.oxlintrc.json index 4041dcab8f40..1e450e0d2fa6 100644 --- a/code/.oxlintrc.json +++ b/code/.oxlintrc.json @@ -95,6 +95,12 @@ "local-rules/no-uncategorized-errors": "off" } }, + { + "files": ["**/lib/docgen-harness/src/perf/**"], + "rules": { + "local-rules/no-uncategorized-errors": "off" + } + }, { "files": ["**/core/src/preview-errors.ts"], "rules": { diff --git a/code/lib/docgen-harness/README.md b/code/lib/docgen-harness/README.md index d8eac7020d84..ec6dfa2ef606 100644 --- a/code/lib/docgen-harness/README.md +++ b/code/lib/docgen-harness/README.md @@ -50,7 +50,12 @@ src/ │ └── __testfixtures__// # component, stories, compodoc-input.json, aot-cmp.ts (signal cases), │ # argtypes.snapshot, argtypes-filtered.snapshot, snippet-.snapshot ├── svelte/ # planned -└── web-components/ # planned +├── web-components/ # planned +└── perf/ # the performance bench, see below + ├── PERF-METHODOLOGY.md # the measurement contract + ├── docgen-perf/ # per-engine latency and memory suite, plus its engines/ and generators/ + ├── docgen-memory/ # the docgen-server memory regression gate + └── docgen-shared/ # sampling, stats, budgets and paths shared by both ``` ## The comparator @@ -165,7 +170,27 @@ Each has a red marker in `vue3-legacy-gaps.test.ts`. - #29697 (not reproduced) -> `signal-io/`: aliased signal inputs record under their alias at 2.0.0; regression baseline, no marker. - #22007 -> `properties-methods-noise/`: the filter flag's origin case, and the fixture where both flag states meaningfully differ. +## The performance bench + +`src/perf/` measures how fast the docgen engines are and how much memory they hold, which is the other half of the "docgen beyond React" question the snapshot comparator above answers for correctness. +It is a set of CLIs rather than part of this package's exported API, so nothing in `src/perf/` is re-exported from `src/index.ts`. + +Both commands run from `code/lib/docgen-harness`: + +```bash +yarn bench:docgen-perf # per-engine cold/warm latency and memory, full profile +yarn bench:docgen-perf --quick # smoke profile; its numbers are marked non-comparable +yarn bench:docgen-memory # the docgen-server memory regression gate CI runs daily +``` + +`bench:docgen-perf` generates synthetic projects under the shared sandbox directory, runs each engine in its own child process, and writes a results JSON next to them. +`bench:docgen-memory` asserts both that re-extraction is leak-free and that the program-recycle fix still flips a tight-heap run from OOM to survival. + +Read `src/perf/PERF-METHODOLOGY.md` before changing a metric, a budget, or a version pair. +It is the contract the numbers are only meaningful under. + +The bench also carries unit tests for its own aggregation, reporting and generator logic, so `yarn test code/lib/docgen-harness` runs those alongside the fixture comparisons. + ## What does not live here -- The performance harness stays under `scripts/`. - Framework provider code lives in each framework's own package. diff --git a/code/lib/docgen-harness/package.json b/code/lib/docgen-harness/package.json index 123f14731400..8d9384cc3ef4 100644 --- a/code/lib/docgen-harness/package.json +++ b/code/lib/docgen-harness/package.json @@ -25,22 +25,38 @@ }, "./package.json": "./package.json" }, + "scripts": { + "bench:docgen-memory": "node --import jiti/register ./src/perf/docgen-memory/gate.ts", + "bench:docgen-perf": "node ./src/perf/docgen-perf/run.ts" + }, "devDependencies": { "@angular/common": "^21.2.14", "@angular/compiler": "^21.2.14", "@angular/core": "^21.2.14", "@angular/platform-browser": "^21.2.14", "@angular/platform-browser-dynamic": "^21.2.14", + "@compodoc/compodoc": "2.0.0", "@storybook/angular-vite": "workspace:*", + "@storybook/react": "workspace:*", "@storybook/vue3": "workspace:*", "@testing-library/vue": "^8.0.0", + "@types/react": "^18.0.37", "@vitejs/plugin-vue": "^4.6.2", + "jiti": "^2.6.1", + "react": "^18.2.0", + "react-dom": "^18.2.0", "rxjs": "^7.4.0", + "storybook": "workspace:*", "typescript": "^6.0.3", "vue": "^3.2.47", "vue-component-meta": "^3.3.9", + "vue-component-meta-next": "npm:vue-component-meta@3.3.8", "vue-docgen-api": "^4.75.1", "vue-tsc": "latest", + "zod": "^3.25.76", "zone.js": "^0.16.2" + }, + "nx": { + "includedScripts": [] } } diff --git a/code/lib/docgen-harness/project.json b/code/lib/docgen-harness/project.json index a3244bc0df09..31f2f7e7691e 100644 --- a/code/lib/docgen-harness/project.json +++ b/code/lib/docgen-harness/project.json @@ -6,6 +6,7 @@ "compile": {}, "check": { "executor": "nx:run-commands", + "dependsOn": [], "options": { "cwd": "{projectRoot}", "command": "vue-tsc --noEmit" } } }, diff --git a/scripts/bench/PERF-METHODOLOGY.md b/code/lib/docgen-harness/src/perf/PERF-METHODOLOGY.md similarity index 95% rename from scripts/bench/PERF-METHODOLOGY.md rename to code/lib/docgen-harness/src/perf/PERF-METHODOLOGY.md index 7b4ab814b4d7..045bb91087bc 100644 --- a/scripts/bench/PERF-METHODOLOGY.md +++ b/code/lib/docgen-harness/src/perf/PERF-METHODOLOGY.md @@ -61,10 +61,10 @@ A bump proposed in a pull request is fully covered by this. Catching a regressio #### Running a version comparison -The pair that exists today is `vue-component-meta` against `vue-component-meta-next`, which are two installs of one package: the second is an alias in `scripts/package.json` pinned to an exact version. +The pair that exists today is `vue-component-meta` against `vue-component-meta-next`, which are two installs of one package: the second is an alias in `code/lib/docgen-harness/package.json` pinned to an exact version. **1. Point the alias at the version you want to test.** -Edit `scripts/package.json` and install, so the candidate is on disk: +Edit `code/lib/docgen-harness/package.json` and install, so the candidate is on disk: ```jsonc "vue-component-meta": "^3.2.7", // the current side @@ -79,7 +79,7 @@ Pin the candidate exactly rather than with a range. A range on both sides can resolve to one install, and then the run compares an engine against itself. **2. Run both sides in one invocation, naming each explicitly.** -From `scripts/`: +From `code/lib/docgen-harness/`: ```bash yarn bench:docgen-perf --engine vue-component-meta --engine vue-component-meta-next @@ -115,11 +115,11 @@ An engine whose package does not resolve is reported as skipped with a reason ra This works only for an engine whose child imports the versioned package directly, the way `engines/vue-component-meta.ts` does. Where the harness reaches the engine through repo source instead - `react-legacy` goes via `loadReactRendererModule` into `code/renderers/react`, which imports `react-docgen` by bare specifier - no child flag can redirect that import, and a version pair needs a different approach entirely. -Declaring `@storybook/react` as a dependency of `scripts` is what makes that source reachable in the first place, but it does not make the specifier redirectable: pointing the renderer at a second `react-docgen` would take a module resolution hook registered in the child, which is not built. +Declaring `@storybook/react` as a dependency of `@storybook/docgen-harness` is what makes that source reachable in the first place, but it does not make the specifier redirectable: pointing the renderer at a second `react-docgen` would take a module resolution hook registered in the child, which is not built. Four data edits: -1. An alias in `scripts/package.json` pinned to the candidate version. +1. An alias in `code/lib/docgen-harness/package.json` pinned to the candidate version. 2. The new id added to the `EngineId` union in `docgen-shared/engine-ids.ts`, which is hand-maintained: a registry entry naming an id that is not in the union does not compile. 3. A second registry entry in `docgen-perf/registry.ts` reusing the same child, with `inDefaultRun: false`, `versionPackage` naming the alias, and the flag that tells the child which install to load. 4. An entry in `CONTROL_PAIRS` in `docgen-perf/ratios.ts` naming the current side as `legacy` and the alias as `next`. diff --git a/scripts/bench/docgen-memory/gate.ts b/code/lib/docgen-harness/src/perf/docgen-memory/gate.ts similarity index 85% rename from scripts/bench/docgen-memory/gate.ts rename to code/lib/docgen-harness/src/perf/docgen-memory/gate.ts index 79ee4c707fb4..0542e12ff494 100644 --- a/scripts/bench/docgen-memory/gate.ts +++ b/code/lib/docgen-harness/src/perf/docgen-memory/gate.ts @@ -12,9 +12,9 @@ * (`--recycle off`, ratio Infinity) MUST OOM, recycle ON (default) MUST survive. Asserting * the flip, not just survival, is what guards the fix rather than the cap. * - * Run: - * yarn bench:docgen-memory # from scripts/ - * node --import jiti/register scripts/bench/docgen-memory/gate.ts + * Run from code/lib/docgen-harness: + * yarn bench:docgen-memory + * node --import jiti/register src/perf/docgen-memory/gate.ts */ import { spawnSync } from 'node:child_process'; import * as fs from 'node:fs'; @@ -62,12 +62,18 @@ const CONFIGS: GateConfig[] = [ // reintroduces whole-index re-extraction. name: 'changed-scope steady state (N=600, props=10, 20 saves)', args: [ - '--components', '600', - '--variants', '4', - '--props', '10', - '--saves', '20', - '--mode', 'refresh', - '--scope', 'changed', + '--components', + '600', + '--variants', + '4', + '--props', + '10', + '--saves', + '20', + '--mode', + 'refresh', + '--scope', + 'changed', ], outSubdir: 'changed-scope', budgets: memoryBudgetsFor('react-osa'), @@ -77,12 +83,18 @@ const CONFIGS: GateConfig[] = [ // program is recycled under heap pressure. Pair with the negative control below. name: 'live path survives WITH recycling (N=800, --heavy, cap 1536MB)', args: [ - '--components', '800', - '--variants', '4', - '--props', '10', - '--saves', '1', - '--mode', 'live', - '--recycle', 'on', + '--components', + '800', + '--variants', + '4', + '--props', + '10', + '--saves', + '1', + '--mode', + 'live', + '--recycle', + 'on', '--heavy', '--no-force-gc', ], @@ -93,14 +105,20 @@ const CONFIGS: GateConfig[] = [ { // Negative control: the SAME workload + cap, with recycling disabled, MUST OOM. If this ever // survives, the cap is too loose (or the workload shrank), so fail and re-tune the pair. - name: 'live path OOMs WITHOUT recycling — negative control (N=800, --heavy, cap 1536MB)', + name: 'live path OOMs WITHOUT recycling - negative control (N=800, --heavy, cap 1536MB)', args: [ - '--components', '800', - '--variants', '4', - '--props', '10', - '--saves', '1', - '--mode', 'live', - '--recycle', 'off', + '--components', + '800', + '--variants', + '4', + '--props', + '10', + '--saves', + '1', + '--mode', + 'live', + '--recycle', + 'off', '--heavy', '--no-force-gc', ], @@ -167,9 +185,21 @@ function checkMetricConfig(config: GateConfig, outcome: RunOutcome, failures: st const { budgets, name } = config; const checks: Array<{ label: string; value: number | undefined; budget: number }> = [ - { label: 'retained growth (MB)', value: outcome.result.retainedGrowth, budget: budgets.maxRetainedGrowthMb }, - { label: 'avg transient/save (MB)', value: outcome.result.avgTransient, budget: budgets.maxTransientMb }, - { label: 'retained slope (MB/save)', value: outcome.result.retainedSlope, budget: budgets.maxRetainedSlopeMb }, + { + label: 'retained growth (MB)', + value: outcome.result.retainedGrowth, + budget: budgets.maxRetainedGrowthMb, + }, + { + label: 'avg transient/save (MB)', + value: outcome.result.avgTransient, + budget: budgets.maxTransientMb, + }, + { + label: 'retained slope (MB/save)', + value: outcome.result.retainedSlope, + budget: budgets.maxRetainedSlopeMb, + }, ]; for (const { label, value, budget } of checks) { @@ -198,12 +228,14 @@ function checkNegativeControl(config: GateConfig, outcome: RunOutcome, failures: } if (!crashed) { failures.push( - `${config.name}: expected an OOM without recycling, but the run survived — the cap is too ` + + `${config.name}: expected an OOM without recycling, but the run survived - the cap is too ` + `loose or the workload shrank; re-tune the live config pair.` ); console.log(' ✗ expected OOM, but survived'); } else { - failures.push(`${config.name}: crashed (status ${outcome.status}) but without a heap-OOM signature`); + failures.push( + `${config.name}: crashed (status ${outcome.status}) but without a heap-OOM signature` + ); console.log(` ✗ crashed (status ${outcome.status}) but not a heap OOM`); } return; diff --git a/scripts/bench/docgen-memory/generate-project.ts b/code/lib/docgen-harness/src/perf/docgen-memory/generate-project.ts similarity index 92% rename from scripts/bench/docgen-memory/generate-project.ts rename to code/lib/docgen-harness/src/perf/docgen-memory/generate-project.ts index 0da68921d5c5..0259b13ec6de 100644 --- a/scripts/bench/docgen-memory/generate-project.ts +++ b/code/lib/docgen-harness/src/perf/docgen-memory/generate-project.ts @@ -3,16 +3,16 @@ * * Produces N component files (each with a non-trivial props interface that extends React DOM types, * so TypeScript type resolution has realistic cost) plus a co-located `.stories.tsx` with M variants. - * Used by both the in-process memory harness (`memory-harness.ts`) and — at larger scale — as a real + * Used by both the in-process memory harness (`memory-harness.ts`) and - at larger scale - as a real * buildable/deployable Storybook for the "+5K components" benchmark from * https://github.com/storybookjs/storybook/issues/34824. * * The `--heavy` lever (optionally scaled with `--heavy-factor N`) grows the TypeScript checker * working set by emitting inline literal-union props that `serializeType` expands into retained enum - * arrays — the density that lets the in-process harness actually OOM. + * arrays - the density that lets the in-process harness actually OOM. * - * Run directly: - * node --import jiti/register scripts/bench/docgen-memory/generate-project.ts --out ../storybook-sandboxes/docgen-stress --components 5000 --variants 5 + * Run directly, from code/lib/docgen-harness (--out defaults into the sandbox directory): + * node --import jiti/register src/perf/docgen-memory/generate-project.ts --components 5000 --variants 5 */ import { createRequire } from 'node:module'; import * as fs from 'node:fs'; @@ -22,6 +22,7 @@ import { pathToFileURL } from 'node:url'; import { z } from 'zod'; import { countOption, parseHarnessOptions, positiveCountOption } from '../docgen-shared/args.ts'; +import { SANDBOX_DIRECTORY } from '../docgen-shared/paths.ts'; import type { ComponentRefLike, StoryRefLike } from '../docgen-shared/react-renderer-module.ts'; const require = createRequire(import.meta.url); @@ -37,7 +38,7 @@ export interface GenerateOptions { props: number; /** * Inflate the *type-resolution* cost per component with large constructed types (big unions, - * mapped types, deep nesting, generics) — the lever that actually grows the TypeScript checker + * mapped types, deep nesting, generics) - the lever that actually grows the TypeScript checker * working set, mirroring complex real-world component libraries. */ heavyTypes: boolean; @@ -148,10 +149,10 @@ function propLines(componentIndex: number, extraProps: number): string { * * - Transient resolution: a big token union + a mapped type over it + a deeply nested generic. * These inflate the per-save type-checking spike but are emitted as type *aliases*, so - * `serializeType` only prints their name — they are not deeply retained on the doc. + * `serializeType` only prints their name - they are not deeply retained on the doc. * - Retained working set: `inline literal-union` props whose members are unique per component and * per prop. `serializeType` expands these into `enum` value arrays stored on the emitted doc, so - * they stay resident for the life of the program — mirroring real libraries whose warm working + * they stay resident for the life of the program - mirroring real libraries whose warm working * set sits permanently near the heap cap. */ function heavyTypeBlock(i: number, factor: number): { typeDefs: string; propLines: string } { @@ -162,10 +163,7 @@ function heavyTypeBlock(i: number, factor: number): { typeDefs: string; propLine // Inline literal-union props: unique members per (component, prop) so the checker and the emitted // doc cannot dedupe them across components. These are the dominant retained-memory lever. const enumLines = Array.from({ length: enumProps }, (_, k) => { - const members = Array.from( - { length: tokenCount }, - (_, t) => `'c${i}_p${k}_v${t}'` - ).join(' | '); + const members = Array.from({ length: tokenCount }, (_, t) => `'c${i}_p${k}_v${t}'`).join(' | '); return ` /** Inline literal union ${k}. */ enum${k}?: ${members};`; }).join('\n'); @@ -203,9 +201,7 @@ export function componentSource( extraProps: number, opts: { heavyTypes?: boolean; heavyFactor?: number; base64Kb?: number } = {} ): string { - const heavy = opts.heavyTypes - ? heavyTypeBlock(i, opts.heavyFactor ?? 1) - : undefined; + const heavy = opts.heavyTypes ? heavyTypeBlock(i, opts.heavyFactor ?? 1) : undefined; const base64 = opts.base64Kb ? base64Block(i, opts.base64Kb) : ''; return `import * as React from 'react'; @@ -214,7 +210,7 @@ ${propLines(i, extraProps)}${heavy ? `\n${heavy.propLines}` : ''} } /** - * Comp${i} — generated stress-test component. + * Comp${i} - generated stress-test component. */ export const Comp${i} = ({ label, ...rest }: Comp${i}Props) => { return
{label}
; @@ -314,7 +310,7 @@ function parseOptions(argv: string[]): GenerateOptions { 'no-node-modules': { type: 'boolean' }, } as const, z.object({ - outDir: z.string().default('../storybook-sandboxes/docgen-memory-stress'), + outDir: z.string().default(path.join(SANDBOX_DIRECTORY, 'docgen-memory-stress')), components: countOption(500), variants: countOption(4), props: countOption(8), diff --git a/scripts/bench/docgen-memory/memory-harness.ts b/code/lib/docgen-harness/src/perf/docgen-memory/memory-harness.ts similarity index 94% rename from scripts/bench/docgen-memory/memory-harness.ts rename to code/lib/docgen-harness/src/perf/docgen-memory/memory-harness.ts index 6bb01c8cf6c4..f4c57ecd77e0 100644 --- a/scripts/bench/docgen-memory/memory-harness.ts +++ b/code/lib/docgen-harness/src/perf/docgen-memory/memory-harness.ts @@ -18,16 +18,16 @@ * docs-addon per-edit wave that drives the #35260 OOM. The program-recycle fix * bounds this path; use --recycle off to assert the OOM still happens without it. * - * Run (diagnose retained vs transient): - * node --expose-gc --import jiti/register scripts/bench/docgen-memory/memory-harness.ts \ + * Run from code/lib/docgen-harness (diagnose retained vs transient): + * node --expose-gc --import jiti/register src/perf/docgen-memory/memory-harness.ts \ * --components 800 --saves 25 --mode refresh * * Run (reproduce the crash / verify the fix): * NODE_OPTIONS=--max_old_space_size=1536 node --expose-gc --import jiti/register \ - * scripts/bench/docgen-memory/memory-harness.ts \ + * src/perf/docgen-memory/memory-harness.ts \ * --components 800 --props 10 --saves 1 --mode live --heavy --no-force-gc --recycle off # → OOM * NODE_OPTIONS=--max_old_space_size=1536 node --expose-gc --import jiti/register \ - * scripts/bench/docgen-memory/memory-harness.ts \ + * src/perf/docgen-memory/memory-harness.ts \ * --components 800 --props 10 --saves 1 --mode live --heavy --no-force-gc --recycle on # → survives */ import * as fs from 'node:fs'; @@ -39,7 +39,10 @@ import { z } from 'zod'; import { countOption, parseHarnessOptions, positiveCountOption } from '../docgen-shared/args.ts'; import { SANDBOX_DIRECTORY } from '../docgen-shared/paths.ts'; -import { type StoryRefLike, loadReactRendererModule } from '../docgen-shared/react-renderer-module.ts'; +import { + type StoryRefLike, + loadReactRendererModule, +} from '../docgen-shared/react-renderer-module.ts'; import { MB, gcAvailable } from '../docgen-shared/sampling.ts'; import { type SeriesEngine, @@ -52,7 +55,9 @@ import { buildStoryRefs, componentSource, generateProject } from './generate-pro interface ComponentMetaManagerLike { batchExtract(entries: StoryRefLike[]): void; - onFilesChanged(changes: Array<{ filePath: string; type: 'changed' | 'created' | 'deleted' }>): void; + onFilesChanged( + changes: Array<{ filePath: string; type: 'changed' | 'created' | 'deleted' }> + ): void; dispose(): void; } @@ -121,8 +126,8 @@ function parseOptions(argv: string[]): HarnessOptions { } /** - * Loaded via `new URL` at runtime, not a static import, so the `scripts` typecheck does not pull - * `code/renderers` source into its program. + * Resolved to a file URL and imported dynamically rather than statically, so this package's + * typecheck does not pull `code/renderers` source into its program. */ async function loadComponentMetaManager(): Promise { const mod = await loadReactRendererModule<{ ComponentMetaManager: ComponentMetaManagerCtor }>( diff --git a/scripts/bench/docgen-perf/aggregate.test.ts b/code/lib/docgen-harness/src/perf/docgen-perf/aggregate.test.ts similarity index 96% rename from scripts/bench/docgen-perf/aggregate.test.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/aggregate.test.ts index abec014abf2c..3dc61adba226 100644 --- a/scripts/bench/docgen-perf/aggregate.test.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/aggregate.test.ts @@ -57,7 +57,10 @@ describe('seriesMetrics', () => { }); it('reads warm from the designated repetition, not the first', () => { - expect(seriesMetrics(reps, 3).warmExtractionMs).toMatchObject({ samples: [4, 5, 6], median: 5 }); + expect(seriesMetrics(reps, 3).warmExtractionMs).toMatchObject({ + samples: [4, 5, 6], + median: 5, + }); }); it('marks whole-project scan as not applicable rather than faking one', () => { @@ -110,7 +113,10 @@ describe('oneShotMetrics', () => { it('reports no peak rather than a low one when a repetition went unsampled', () => { // `ps` is POSIX-only and a short run can finish inside one poll interval, so a repetition can // come back with nothing sampled. Averaging that in would report a peak nobody measured. - const partial = [{ coldMs: 300, warmMs: 280, peakRssMb: 500 }, { coldMs: 100, warmMs: 120 }]; + const partial = [ + { coldMs: 300, warmMs: 280, peakRssMb: 500 }, + { coldMs: 100, warmMs: 120 }, + ]; expect(oneShotMetrics(partial, 2).peakTransientMb).toEqual({ status: 'n/a' }); }); }); diff --git a/scripts/bench/docgen-perf/aggregate.ts b/code/lib/docgen-harness/src/perf/docgen-perf/aggregate.ts similarity index 97% rename from scripts/bench/docgen-perf/aggregate.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/aggregate.ts index a53920e05463..1de7582109e8 100644 --- a/scripts/bench/docgen-perf/aggregate.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/aggregate.ts @@ -1,5 +1,5 @@ /** - * Turns N repetitions into the five floor metrics (see scripts/bench/PERF-METHODOLOGY.md). Warm + * Turns N repetitions into the five floor metrics (see ../PERF-METHODOLOGY.md). Warm * latency and all three memory metrics read the same designated repetition's save series, so those * four figures always describe one run; cold latency is a median across every repetition instead. */ diff --git a/scripts/bench/docgen-perf/cli.ts b/code/lib/docgen-harness/src/perf/docgen-perf/cli.ts similarity index 100% rename from scripts/bench/docgen-perf/cli.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/cli.ts diff --git a/scripts/bench/docgen-perf/config.ts b/code/lib/docgen-harness/src/perf/docgen-perf/config.ts similarity index 100% rename from scripts/bench/docgen-perf/config.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/config.ts diff --git a/scripts/bench/docgen-perf/engine.ts b/code/lib/docgen-harness/src/perf/docgen-perf/engine.ts similarity index 97% rename from scripts/bench/docgen-perf/engine.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/engine.ts index fac50cba3db1..56cd78e8bc88 100644 --- a/scripts/bench/docgen-perf/engine.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/engine.ts @@ -132,13 +132,15 @@ export class SeriesChildEngine extends BenchEngine { preflight(): string | undefined { const { versionPackage } = this.#config; if (versionPackage && resolvePackageVersion(versionPackage) === undefined) { - return `${versionPackage} did not resolve; it is pinned in scripts/package.json, so run yarn install`; + return `${versionPackage} did not resolve; it is pinned in code/lib/docgen-harness/package.json, so run yarn install`; } return undefined; } version(): string | undefined { - return this.#config.versionPackage ? resolvePackageVersion(this.#config.versionPackage) : undefined; + return this.#config.versionPackage + ? resolvePackageVersion(this.#config.versionPackage) + : undefined; } async measure(ctx: MeasureContext, scenario: ScenarioSpec, rep: number): Promise { diff --git a/scripts/bench/docgen-perf/engines/compodoc-doc.test.ts b/code/lib/docgen-harness/src/perf/docgen-perf/engines/compodoc-doc.test.ts similarity index 92% rename from scripts/bench/docgen-perf/engines/compodoc-doc.test.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/engines/compodoc-doc.test.ts index 4397481b7846..363b3f10285a 100644 --- a/scripts/bench/docgen-perf/engines/compodoc-doc.test.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/engines/compodoc-doc.test.ts @@ -7,12 +7,7 @@ import { countDocumentation } from './compodoc-doc.ts'; /** Real compodoc output, committed as the Angular docgen baselines. */ const fixture = (name: string) => countDocumentation( - path.join( - import.meta.dirname, - '../../../../code/lib/docgen-harness/src/angular/__testfixtures__', - name, - 'compodoc-input.json' - ) + path.join(import.meta.dirname, '../../../angular/__testfixtures__', name, 'compodoc-input.json') ); describe('countDocumentation', () => { diff --git a/scripts/bench/docgen-perf/engines/compodoc-doc.ts b/code/lib/docgen-harness/src/perf/docgen-perf/engines/compodoc-doc.ts similarity index 92% rename from scripts/bench/docgen-perf/engines/compodoc-doc.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/engines/compodoc-doc.ts index ae0c7e890e42..5d0fd1af5e7f 100644 --- a/scripts/bench/docgen-perf/engines/compodoc-doc.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/engines/compodoc-doc.ts @@ -44,12 +44,27 @@ function documentedMembers(doc: Documentation): MemberEntry[] { * because the two spellings reach the same member field. */ const RESOLVED_TYPES = new Set([ - 'string', 'number', 'boolean', 'any', 'unknown', 'void', 'never', 'null', 'undefined', - 'object', 'symbol', 'bigint', 'function', 'Date', 'Function', 'Array', 'Object', + 'string', + 'number', + 'boolean', + 'any', + 'unknown', + 'void', + 'never', + 'null', + 'undefined', + 'object', + 'symbol', + 'bigint', + 'function', + 'Date', + 'Function', + 'Array', + 'Object', ]); /** - * A member whose type is a bare name the engine never looked through — `Hop19Shape`, not + * A member whose type is a bare name the engine never looked through - `Hop19Shape`, not * `{ x: string }`. Inline object literals, primitives and literal unions are all self-describing, * so only a lone identifier counts. * diff --git a/scripts/bench/docgen-perf/engines/compodoc.ts b/code/lib/docgen-harness/src/perf/docgen-perf/engines/compodoc.ts similarity index 96% rename from scripts/bench/docgen-perf/engines/compodoc.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/engines/compodoc.ts index e620637c24e5..91ccda77369b 100644 --- a/scripts/bench/docgen-perf/engines/compodoc.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/engines/compodoc.ts @@ -121,7 +121,11 @@ function runCompodocOnce( // Counted before the next run overwrites the file, which both runs share. const counts = countDocumentation(documentationJson); // The polled peak misses spikes shorter than the interval; the recorded value is a floor. - resolve({ durMs, peakRssMb: peakRssKb === undefined ? undefined : peakRssKb / 1024, ...counts }); + resolve({ + durMs, + peakRssMb: peakRssKb === undefined ? undefined : peakRssKb / 1024, + ...counts, + }); }); }); } @@ -183,7 +187,7 @@ export class CompodocEngine extends BenchEngine { this.#resolved = resolveCompodoc(); return this.#resolved ? undefined - : '@compodoc/compodoc did not resolve; it is pinned in scripts/package.json, so run yarn install'; + : '@compodoc/compodoc did not resolve; it is pinned in code/lib/docgen-harness/package.json, so run yarn install'; } version(): string | undefined { diff --git a/scripts/bench/docgen-perf/engines/react-legacy.ts b/code/lib/docgen-harness/src/perf/docgen-perf/engines/react-legacy.ts similarity index 96% rename from scripts/bench/docgen-perf/engines/react-legacy.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/engines/react-legacy.ts index 7b16e79fa086..b02d19af96ee 100644 --- a/scripts/bench/docgen-perf/engines/react-legacy.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/engines/react-legacy.ts @@ -8,8 +8,8 @@ * Do not run this child under jiti - react-docgen's browserslist dependency fails its JSON data * require under that loader ("jsReleases.map is not a function"). * - * Run: - * node --expose-gc scripts/bench/docgen-perf/engines/react-legacy.ts \ + * Run from code/lib/docgen-harness: + * node --expose-gc src/perf/docgen-perf/engines/react-legacy.ts \ * --parser react-docgen --components 300 --saves 20 --json /tmp/result.json */ import * as fs from 'node:fs'; @@ -29,11 +29,7 @@ import { type ComponentRefLike, loadReactRendererModule, } from '../../docgen-shared/react-renderer-module.ts'; -import { - type SeriesEngine, - harnessMain, - runSeriesHarness, -} from '../../docgen-shared/series.ts'; +import { type SeriesEngine, harnessMain, runSeriesHarness } from '../../docgen-shared/series.ts'; /** The renderer module surfaces this harness loads, narrowed to what it calls. */ interface UtilsModule { diff --git a/scripts/bench/docgen-perf/engines/vue-component-meta.ts b/code/lib/docgen-harness/src/perf/docgen-perf/engines/vue-component-meta.ts similarity index 97% rename from scripts/bench/docgen-perf/engines/vue-component-meta.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/engines/vue-component-meta.ts index d04657ba5b24..87b0ecb79427 100644 --- a/scripts/bench/docgen-perf/engines/vue-component-meta.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/engines/vue-component-meta.ts @@ -6,8 +6,8 @@ * creation runs inside the cold pass, since it is part of first-extraction cost: `flat` uses * `createCheckerByJson`, the workspace scenarios use `createChecker` on the deepest package. * - * Run: - * node --expose-gc scripts/bench/docgen-perf/engines/vue-component-meta.ts \ + * Run from code/lib/docgen-harness: + * node --expose-gc src/perf/docgen-perf/engines/vue-component-meta.ts \ * --scenario workspace --packages 4 --components-per-package 10 --heavy-lib --json /tmp/result.json */ import * as fs from 'node:fs'; diff --git a/scripts/bench/docgen-perf/engines/vue-docgen-api.ts b/code/lib/docgen-harness/src/perf/docgen-perf/engines/vue-docgen-api.ts similarity index 95% rename from scripts/bench/docgen-perf/engines/vue-docgen-api.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/engines/vue-docgen-api.ts index 255c098a3827..29cab8eb0fb9 100644 --- a/scripts/bench/docgen-perf/engines/vue-docgen-api.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/engines/vue-docgen-api.ts @@ -6,8 +6,8 @@ * The parser reads from disk on every call and keeps no cache, so one save costs one `parse` and * needs no invalidation step. * - * Run: - * node --expose-gc scripts/bench/docgen-perf/engines/vue-docgen-api.ts \ + * Run from code/lib/docgen-harness: + * node --expose-gc src/perf/docgen-perf/engines/vue-docgen-api.ts \ * --scenario workspace --packages 4 --components-per-package 10 --heavy-lib --json /tmp/result.json */ import * as fs from 'node:fs'; diff --git a/scripts/bench/docgen-perf/engines/vue-scenario.ts b/code/lib/docgen-harness/src/perf/docgen-perf/engines/vue-scenario.ts similarity index 100% rename from scripts/bench/docgen-perf/engines/vue-scenario.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/engines/vue-scenario.ts diff --git a/scripts/bench/docgen-perf/generators/angular.test.ts b/code/lib/docgen-harness/src/perf/docgen-perf/generators/angular.test.ts similarity index 100% rename from scripts/bench/docgen-perf/generators/angular.test.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/generators/angular.test.ts diff --git a/scripts/bench/docgen-perf/generators/angular.ts b/code/lib/docgen-harness/src/perf/docgen-perf/generators/angular.ts similarity index 94% rename from scripts/bench/docgen-perf/generators/angular.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/generators/angular.ts index acf254e63240..a5e74b867e1a 100644 --- a/scripts/bench/docgen-perf/generators/angular.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/generators/angular.ts @@ -4,8 +4,8 @@ * Ships a minimal fake `(at)angular/core` type surface inside its own node_modules so the tree is * hermetic - no npm install of the real framework is needed for type resolution. * - * Run directly: - * node scripts/bench/docgen-perf/generators/angular.ts --out ../storybook-sandboxes/docgen-perf-angular --components 100 + * Run directly, from code/lib/docgen-harness (--out defaults into the sandbox directory): + * node src/perf/docgen-perf/generators/angular.ts --components 100 */ import * as fs from 'node:fs'; import * as path from 'node:path'; @@ -14,6 +14,7 @@ import { pathToFileURL } from 'node:url'; import { z } from 'zod'; import { countOption, parseHarnessOptions } from '../../docgen-shared/args.ts'; +import { SANDBOX_DIRECTORY } from '../../docgen-shared/paths.ts'; export interface AngularGenerateOptions { outDir: string; @@ -147,7 +148,7 @@ function parseOptions(argv: string[]): AngularGenerateOptions { argv, { out: { type: 'string' }, components: { type: 'string' }, props: { type: 'string' } } as const, z.object({ - outDir: z.string().default('../storybook-sandboxes/docgen-perf-angular'), + outDir: z.string().default(path.join(SANDBOX_DIRECTORY, 'docgen-perf-angular')), components: countOption(100), props: countOption(8), }), diff --git a/scripts/bench/docgen-perf/generators/vue.test.ts b/code/lib/docgen-harness/src/perf/docgen-perf/generators/vue.test.ts similarity index 99% rename from scripts/bench/docgen-perf/generators/vue.test.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/generators/vue.test.ts index 2e2f6b365dd8..9dff0b709221 100644 --- a/scripts/bench/docgen-perf/generators/vue.test.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/generators/vue.test.ts @@ -99,8 +99,7 @@ describe('vueProjectFiles', () => { }); it('extends each package config from the base, and includes .vue', () => { - expect(vueProjectFiles(workspace).files['packages/pkg0/tsconfig.json']) - .toMatchInlineSnapshot(` + expect(vueProjectFiles(workspace).files['packages/pkg0/tsconfig.json']).toMatchInlineSnapshot(` "{ "extends": "../../tsconfig.base.json", "include": [ diff --git a/scripts/bench/docgen-perf/generators/vue.ts b/code/lib/docgen-harness/src/perf/docgen-perf/generators/vue.ts similarity index 97% rename from scripts/bench/docgen-perf/generators/vue.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/generators/vue.ts index 693cdea62690..9e808d698cca 100644 --- a/scripts/bench/docgen-perf/generators/vue.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/generators/vue.ts @@ -13,8 +13,8 @@ * - heavyLib: emits a fake `(at)bench/heavy-ui` package with a large `.d.ts` surface; package * props reference it when enabled. * - * Run directly: - * node scripts/bench/docgen-perf/generators/vue.ts --out ../storybook-sandboxes/docgen-perf-vue --packages 4 --components-per-package 10 + * Run directly, from code/lib/docgen-harness (--out defaults into the sandbox directory): + * node src/perf/docgen-perf/generators/vue.ts --packages 4 --components-per-package 10 */ import { createRequire } from 'node:module'; import * as fs from 'node:fs'; @@ -24,6 +24,7 @@ import { pathToFileURL } from 'node:url'; import { z } from 'zod'; import { countOption, parseHarnessOptions } from '../../docgen-shared/args.ts'; +import { SANDBOX_DIRECTORY } from '../../docgen-shared/paths.ts'; const require = createRequire(import.meta.url); @@ -299,7 +300,7 @@ function parseOptions(argv: string[]): VueGenerateOptions { 'heavy-lib': { type: 'boolean' }, } as const, z.object({ - outDir: z.string().default('../storybook-sandboxes/docgen-perf-vue'), + outDir: z.string().default(path.join(SANDBOX_DIRECTORY, 'docgen-perf-vue')), packages: countOption(4), componentsPerPackage: countOption(10), chainDepth: countOption(3), diff --git a/scripts/bench/docgen-perf/ratios.test.ts b/code/lib/docgen-harness/src/perf/docgen-perf/ratios.test.ts similarity index 100% rename from scripts/bench/docgen-perf/ratios.test.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/ratios.test.ts diff --git a/scripts/bench/docgen-perf/ratios.ts b/code/lib/docgen-harness/src/perf/docgen-perf/ratios.ts similarity index 97% rename from scripts/bench/docgen-perf/ratios.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/ratios.ts index 1570dd2c31ee..e840594ef37e 100644 --- a/scripts/bench/docgen-perf/ratios.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/ratios.ts @@ -26,7 +26,11 @@ interface ControlPair { export const CONTROL_PAIRS: ControlPair[] = [ { name: 'react', legacy: 'react-legacy', next: 'react-osa' }, { name: 'vue', legacy: 'vue-docgen-api', next: 'vue-component-meta' }, - { name: 'vue-component-meta-version', legacy: 'vue-component-meta', next: 'vue-component-meta-next' }, + { + name: 'vue-component-meta-version', + legacy: 'vue-component-meta', + next: 'vue-component-meta-next', + }, ]; /** diff --git a/scripts/bench/docgen-perf/registry.test.ts b/code/lib/docgen-harness/src/perf/docgen-perf/registry.test.ts similarity index 88% rename from scripts/bench/docgen-perf/registry.test.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/registry.test.ts index 9cb759126c58..e5304e420145 100644 --- a/scripts/bench/docgen-perf/registry.test.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/registry.test.ts @@ -1,8 +1,10 @@ +import * as fs from 'node:fs'; + import { describe, expect, it } from 'vitest'; import type { SeriesResult } from '../docgen-shared/series.ts'; import { QUICK_PROFILE } from './config.ts'; -import type { MeasureContext, ScenarioSpec } from './engine.ts'; +import { type MeasureContext, type ScenarioSpec, SeriesChildEngine } from './engine.ts'; import { CONTROL_PAIRS } from './ratios.ts'; import { ALL_ENGINE_IDS, DEFAULT_ENGINE_IDS, ENGINES, engineById } from './registry.ts'; import type { EngineId } from './types.ts'; @@ -75,6 +77,19 @@ describe('the engine table', () => { }); describe('what the series engines spawn', () => { + it('points every spawning engine at a child script that exists', async () => { + // `child` is a plain string joined onto this directory, so nothing typechecks it. A wrong + // value only surfaces at runtime, for one engine, after the full project generation has + // already been paid for. `react-osa` reaches into ../docgen-memory, so this also pins the + // sibling layout of docgen-perf, docgen-memory and docgen-shared. + const spawning = ENGINES.filter((engine) => engine instanceof SeriesChildEngine); + expect(spawning.length).toBeGreaterThan(0); + for (const engine of spawning) { + const { spec } = await specFor(engine.id, engine.scenarios(QUICK_PROFILE)[0]); + expect(fs.existsSync(spec.childPath), `${engine.id} -> ${spec.childPath}`).toBe(true); + } + }); + it('runs only the reused memory harness under jiti', async () => { // Under jiti, react-docgen's browserslist dependency fails its JSON data require, so the // legacy child must run on native type stripping. diff --git a/scripts/bench/docgen-perf/registry.ts b/code/lib/docgen-harness/src/perf/docgen-perf/registry.ts similarity index 84% rename from scripts/bench/docgen-perf/registry.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/registry.ts index 2f92ef41b1b1..33c9e9cef148 100644 --- a/scripts/bench/docgen-perf/registry.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/registry.ts @@ -16,20 +16,30 @@ const vueScenarios = (profile: SuiteProfile): ScenarioSpec[] => profile.vue.map((scenario) => ({ name: scenario.name, params: { ...scenario } })); const reactArgs = ({ params }: ScenarioSpec): string[] => [ - '--components', String(params.components), - '--variants', String(params.variants), - '--props', String(params.props), - '--saves', String(params.saves), + '--components', + String(params.components), + '--variants', + String(params.variants), + '--props', + String(params.props), + '--saves', + String(params.saves), ]; const vueArgs = ({ params }: ScenarioSpec): string[] => [ - '--scenario', String(params.name), - '--packages', String(params.packages), - '--components-per-package', String(params.componentsPerPackage), - '--chain-depth', String(params.chainDepth), - '--fan-out', String(params.fanOut), + '--scenario', + String(params.name), + '--packages', + String(params.packages), + '--components-per-package', + String(params.componentsPerPackage), + '--chain-depth', + String(params.chainDepth), + '--fan-out', + String(params.fanOut), ...(params.heavyLib ? ['--heavy-lib'] : []), - '--saves', String(params.saves), + '--saves', + String(params.saves), ]; export const ENGINES: BenchEngine[] = [ @@ -49,8 +59,10 @@ export const ENGINES: BenchEngine[] = [ inDefaultRun: false, args: (scenario) => [ ...reactArgs(scenario), - '--parser', 'react-docgen-typescript', - '--scope', 'changed', + '--parser', + 'react-docgen-typescript', + '--scope', + 'changed', ], }), new SeriesChildEngine({ @@ -88,7 +100,9 @@ export const ENGINES: BenchEngine[] = [ ]; export const ALL_ENGINE_IDS: EngineId[] = ENGINES.map((engine) => engine.id); -export const DEFAULT_ENGINE_IDS: EngineId[] = ENGINES.filter((e) => e.inDefaultRun).map((e) => e.id); +export const DEFAULT_ENGINE_IDS: EngineId[] = ENGINES.filter((e) => e.inDefaultRun).map( + (e) => e.id +); export function engineById(id: EngineId): BenchEngine { const engine = ENGINES.find((candidate) => candidate.id === id); diff --git a/scripts/bench/docgen-perf/report.test.ts b/code/lib/docgen-harness/src/perf/docgen-perf/report.test.ts similarity index 90% rename from scripts/bench/docgen-perf/report.test.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/report.test.ts index 94943bde648d..ff93f8e94317 100644 --- a/scripts/bench/docgen-perf/report.test.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/report.test.ts @@ -78,7 +78,9 @@ describe('renderRatios', () => { }, }) ) - ).toMatchInlineSnapshot(`" ratio cold legacy/new (vue/flat): 0.05 [documented members 320 vs 6 - NOT like-for-like - new engine documented less, so it is fast for the wrong reason]"`); + ).toMatchInlineSnapshot( + `" ratio cold legacy/new (vue/flat): 0.05 [documented members 320 vs 6 - NOT like-for-like - new engine documented less, so it is fast for the wrong reason]"` + ); }); it('warns when the counts agree but the resolution work did not', () => { @@ -97,7 +99,9 @@ describe('renderRatios', () => { }, }) ) - ).toMatchInlineSnapshot(`" ratio cold legacy/new (vue/flat): 4.10 [documented members 90 vs 90 - NOT like-for-like - same members, but the new engine left more types unresolved]"`); + ).toMatchInlineSnapshot( + `" ratio cold legacy/new (vue/flat): 4.10 [documented members 90 vs 90 - NOT like-for-like - same members, but the new engine left more types unresolved]"` + ); }); it('prints the counts without a warning when both sides did the same work', () => { @@ -179,7 +183,9 @@ describe('renderRatios', () => { }); it('says so when there is no ratio at all', () => { - expect(block(renderRatios({}))).toMatchInlineSnapshot(`" no calibration ratio: it needs both sides of a control pair measured in one run"`); + expect(block(renderRatios({}))).toMatchInlineSnapshot( + `" no calibration ratio: it needs both sides of a control pair measured in one run"` + ); }); it('names both versions when a pair compares one package against another release of it', () => { @@ -197,7 +203,9 @@ describe('renderRatios', () => { }, }) ) - ).toMatchInlineSnapshot(`" ratio cold legacy/new (vue-component-meta-version/flat): 1.08 [3.3.2 vs 3.3.8]"`); + ).toMatchInlineSnapshot( + `" ratio cold legacy/new (vue-component-meta-version/flat): 1.08 [3.3.2 vs 3.3.8]"` + ); }); it('says a pair measured nothing when both sides resolved the same version', () => { @@ -217,7 +225,9 @@ describe('renderRatios', () => { }, }) ) - ).toMatchInlineSnapshot(`" ratio cold legacy/new (vue-component-meta-version/flat): 1.00 [both sides resolved 3.3.8 - NOT a comparison]"`); + ).toMatchInlineSnapshot( + `" ratio cold legacy/new (vue-component-meta-version/flat): 1.00 [both sides resolved 3.3.8 - NOT a comparison]"` + ); }); it('carries the versions onto the warm line as well as the cold one', () => { @@ -298,7 +308,11 @@ describe('renderResults', () => { const { table, statusLines } = renderResults(['compodoc'], { compodoc: { status: 'skipped', reason: '@compodoc/compodoc did not resolve' }, }); - expect(block(table)).toMatchInlineSnapshot(`" engine/scenario cold warm scan peak ret-growth ret-slope"`); - expect(block(statusLines)).toMatchInlineSnapshot(`" compodoc: SKIPPED - @compodoc/compodoc did not resolve"`); + expect(block(table)).toMatchInlineSnapshot( + `" engine/scenario cold warm scan peak ret-growth ret-slope"` + ); + expect(block(statusLines)).toMatchInlineSnapshot( + `" compodoc: SKIPPED - @compodoc/compodoc did not resolve"` + ); }); }); diff --git a/scripts/bench/docgen-perf/report.ts b/code/lib/docgen-harness/src/perf/docgen-perf/report.ts similarity index 90% rename from scripts/bench/docgen-perf/report.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/report.ts index 82b944d44dbd..6bae84a05043 100644 --- a/scripts/bench/docgen-perf/report.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/report.ts @@ -74,8 +74,8 @@ export function renderResults( const widths = HEADER.map((_, col) => Math.max(...rows.map((row) => row[col].length))); // trimEnd because the last column is padded like every other one, and a run's output should not // carry trailing spaces on every line. - const table = rows.map( - (row) => ` ${row.map((cell, col) => cell.padEnd(widths[col])).join(' ')}`.trimEnd() + const table = rows.map((row) => + ` ${row.map((cell, col) => cell.padEnd(widths[col])).join(' ')}`.trimEnd() ); return { table, statusLines }; } @@ -127,10 +127,14 @@ const VERDICT: Record = { // No counts is not a claim of inequality, so it earns no warning - only a direction does. unknown: '', 'like-for-like': '', - 'next-documents-more': 'NOT like-for-like - new engine documented more, so this ratio undersells it', - 'next-documents-less': 'NOT like-for-like - new engine documented less, so it is fast for the wrong reason', - 'next-resolves-more': 'NOT like-for-like - same members, but the new engine resolved more of them', - 'next-resolves-less': 'NOT like-for-like - same members, but the new engine left more types unresolved', + 'next-documents-more': + 'NOT like-for-like - new engine documented more, so this ratio undersells it', + 'next-documents-less': + 'NOT like-for-like - new engine documented less, so it is fast for the wrong reason', + 'next-resolves-more': + 'NOT like-for-like - same members, but the new engine resolved more of them', + 'next-resolves-less': + 'NOT like-for-like - same members, but the new engine left more types unresolved', }; function memberNote( diff --git a/scripts/bench/docgen-perf/run.ts b/code/lib/docgen-harness/src/perf/docgen-perf/run.ts similarity index 95% rename from scripts/bench/docgen-perf/run.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/run.ts index e0f6c2970615..f5cf1e177fa2 100644 --- a/scripts/bench/docgen-perf/run.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/run.ts @@ -1,9 +1,9 @@ /** - * Orchestrator for the per-engine docgen performance suite. See scripts/bench/PERF-METHODOLOGY.md - * for the measurement contract. + * Orchestrator for the per-engine docgen performance suite. See ../PERF-METHODOLOGY.md for the + * measurement contract. * - * Run: - * yarn bench:docgen-perf # from scripts/, full profile + * Run from code/lib/docgen-harness: + * yarn bench:docgen-perf # full profile * yarn bench:docgen-perf --quick # smoke profile; results marked non-comparable * yarn bench:docgen-perf --engine react-legacy --engine react-osa */ @@ -59,7 +59,9 @@ async function main() { const profile: SuiteProfile = options.quick ? QUICK_PROFILE : DEFAULT_PROFILE; console.log('docgen-perf suite'); - console.log(` engines=${options.engines.join(',')} n=${profile.n} comparable=${profile.comparable}`); + console.log( + ` engines=${options.engines.join(',')} n=${profile.n} comparable=${profile.comparable}` + ); if (!profile.comparable) { console.log(' QUICK PROFILE: results are non-comparable smoke numbers'); } diff --git a/scripts/bench/docgen-perf/spawn.ts b/code/lib/docgen-harness/src/perf/docgen-perf/spawn.ts similarity index 100% rename from scripts/bench/docgen-perf/spawn.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/spawn.ts diff --git a/scripts/bench/docgen-perf/types.ts b/code/lib/docgen-harness/src/perf/docgen-perf/types.ts similarity index 99% rename from scripts/bench/docgen-perf/types.ts rename to code/lib/docgen-harness/src/perf/docgen-perf/types.ts index 756fd0a8430e..71623b13c827 100644 --- a/scripts/bench/docgen-perf/types.ts +++ b/code/lib/docgen-harness/src/perf/docgen-perf/types.ts @@ -1,6 +1,6 @@ /** * Result shapes for the per-engine docgen performance suite. The measurement contract behind these - * shapes is scripts/bench/PERF-METHODOLOGY.md. + * shapes is ../PERF-METHODOLOGY.md. */ import type { EngineId } from '../docgen-shared/engine-ids.ts'; diff --git a/scripts/bench/docgen-shared/args.test.ts b/code/lib/docgen-harness/src/perf/docgen-shared/args.test.ts similarity index 100% rename from scripts/bench/docgen-shared/args.test.ts rename to code/lib/docgen-harness/src/perf/docgen-shared/args.test.ts diff --git a/scripts/bench/docgen-shared/args.ts b/code/lib/docgen-harness/src/perf/docgen-shared/args.ts similarity index 67% rename from scripts/bench/docgen-shared/args.ts rename to code/lib/docgen-harness/src/perf/docgen-shared/args.ts index f5850f17d152..d4d0b1fa89d9 100644 --- a/scripts/bench/docgen-shared/args.ts +++ b/code/lib/docgen-harness/src/perf/docgen-shared/args.ts @@ -17,20 +17,20 @@ export const positiveCountOption = (fallback: number) => /** * parseArgs keys flags as written, so `--fan-out` arrives as `fan-out`; schema keys are camelCase. - * A schema key that is not exactly its flag's camelCase — `maxRetainedGrowthMb` for - * `--max-retained-growth` — must still be renamed in `toInput`, or Zod silently uses its default. + * A schema key that is not exactly its flag's camelCase - `maxRetainedGrowthMb` for + * `--max-retained-growth` - must still be renamed in `toInput`, or Zod silently uses its default. */ function toCamelCase(flag: string): string { return flag.replace(/-([a-z])/g, (_, letter: string) => letter.toUpperCase()); } /** - * `Out` is asserted rather than inferred from the schema, which is not the obvious choice: returning - * `z.infer` would make the two check against each other. It does not work here. `scripts` - * compiles with `strictNullChecks: false`, and under that flag Zod's `addQuestionMarks` finds no - * required keys, so every inferred field comes back optional and satisfies no caller's option type. - * Prefer `type Options = z.infer` at the call site where the schema is the whole - * contract - that removes the second declaration this cast could drift from. + * `Out` is asserted rather than inferred, because `schema` is accepted as the non-generic + * `z.ZodTypeAny` and `z.infer` of that is `any` - there is nothing for the return type to check + * against. Callers therefore name their own option type. Prefer + * `type Options = z.infer` at the call site where the schema is the whole contract, + * so the asserted type is derived from the schema rather than being a second declaration that can + * drift from it. */ export function parseHarnessOptions( argv: string[], diff --git a/scripts/bench/docgen-shared/budgets.ts b/code/lib/docgen-harness/src/perf/docgen-shared/budgets.ts similarity index 100% rename from scripts/bench/docgen-shared/budgets.ts rename to code/lib/docgen-harness/src/perf/docgen-shared/budgets.ts diff --git a/scripts/bench/docgen-shared/child-output.ts b/code/lib/docgen-harness/src/perf/docgen-shared/child-output.ts similarity index 100% rename from scripts/bench/docgen-shared/child-output.ts rename to code/lib/docgen-harness/src/perf/docgen-shared/child-output.ts diff --git a/scripts/bench/docgen-shared/engine-ids.ts b/code/lib/docgen-harness/src/perf/docgen-shared/engine-ids.ts similarity index 100% rename from scripts/bench/docgen-shared/engine-ids.ts rename to code/lib/docgen-harness/src/perf/docgen-shared/engine-ids.ts diff --git a/scripts/bench/docgen-shared/paths.ts b/code/lib/docgen-harness/src/perf/docgen-shared/paths.ts similarity index 50% rename from scripts/bench/docgen-shared/paths.ts rename to code/lib/docgen-harness/src/perf/docgen-shared/paths.ts index c9452c222b71..acf741239f19 100644 --- a/scripts/bench/docgen-shared/paths.ts +++ b/code/lib/docgen-harness/src/perf/docgen-shared/paths.ts @@ -4,9 +4,19 @@ * for its own importers; the harness entry points here run as native ESM, where importing it throws * `ReferenceError: __dirname is not defined in ES module scope`. */ -import { isAbsolute, join } from 'node:path'; +import { createRequire } from 'node:module'; +import { dirname, isAbsolute, join } from 'node:path'; -const ROOT_DIRECTORY = join(import.meta.dirname, '..', '..', '..'); +const require = createRequire(import.meta.url); + +/** + * Anchored on package resolution instead of a hand-counted `..` walk from this file. Every + * generated project and every results file hangs off SANDBOX_DIRECTORY, and a wrong root would + * write them somewhere else without raising an error, so relocating a file inside this package + * must not be able to change it. + */ +const PACKAGE_ROOT = dirname(require.resolve('@storybook/docgen-harness/package.json')); +const ROOT_DIRECTORY = join(PACKAGE_ROOT, '..', '..', '..'); export const SANDBOX_DIRECTORY = process.env.STORYBOOK_SANDBOX_ROOT && isAbsolute(process.env.STORYBOOK_SANDBOX_ROOT) diff --git a/scripts/bench/docgen-shared/react-renderer-module.test.ts b/code/lib/docgen-harness/src/perf/docgen-shared/react-renderer-module.test.ts similarity index 100% rename from scripts/bench/docgen-shared/react-renderer-module.test.ts rename to code/lib/docgen-harness/src/perf/docgen-shared/react-renderer-module.test.ts diff --git a/scripts/bench/docgen-shared/react-renderer-module.ts b/code/lib/docgen-harness/src/perf/docgen-shared/react-renderer-module.ts similarity index 90% rename from scripts/bench/docgen-shared/react-renderer-module.ts rename to code/lib/docgen-harness/src/perf/docgen-shared/react-renderer-module.ts index 4d81b39f21a4..d7ad9cb655d6 100644 --- a/scripts/bench/docgen-shared/react-renderer-module.ts +++ b/code/lib/docgen-harness/src/perf/docgen-shared/react-renderer-module.ts @@ -9,9 +9,9 @@ import { pathToFileURL } from 'node:url'; * exports map, and `files` excludes `src/**` outright. * * That reach is deliberate, but it anchors on the resolved package instead of a path relative to - * this file. `scripts` declares `@storybook/react`, so the coupling is recorded in the workspace - * graph rather than living only in a `../../../` that nothing checks, and either tree can move - * without the failure surfacing as a missing file inside an unrelated import. + * this file. `@storybook/docgen-harness` declares `@storybook/react`, so the coupling is recorded in + * the workspace graph rather than living only in a `../../../` that nothing checks, and either tree + * can move without the failure surfacing as a missing file inside an unrelated import. */ const require = createRequire(import.meta.url); diff --git a/scripts/bench/docgen-shared/samples.ts b/code/lib/docgen-harness/src/perf/docgen-shared/samples.ts similarity index 100% rename from scripts/bench/docgen-shared/samples.ts rename to code/lib/docgen-harness/src/perf/docgen-shared/samples.ts diff --git a/scripts/bench/docgen-shared/sampling.ts b/code/lib/docgen-harness/src/perf/docgen-shared/sampling.ts similarity index 100% rename from scripts/bench/docgen-shared/sampling.ts rename to code/lib/docgen-harness/src/perf/docgen-shared/sampling.ts diff --git a/scripts/bench/docgen-shared/series.ts b/code/lib/docgen-harness/src/perf/docgen-shared/series.ts similarity index 97% rename from scripts/bench/docgen-shared/series.ts rename to code/lib/docgen-harness/src/perf/docgen-shared/series.ts index b5f13406f91a..4c7e290e4dc9 100644 --- a/scripts/bench/docgen-shared/series.ts +++ b/code/lib/docgen-harness/src/perf/docgen-shared/series.ts @@ -78,7 +78,14 @@ export async function runSeries( engine.dispose?.(); - return { coldMs, coldMembers, warmMembers, baseline, samples, ...summarizeSeries(samples, baseline) }; + return { + coldMs, + coldMembers, + warmMembers, + baseline, + samples, + ...summarizeSeries(samples, baseline), + }; } export function printSeriesSummary(result: SeriesResult, saves: number): void { diff --git a/scripts/bench/docgen-shared/stats.test.ts b/code/lib/docgen-harness/src/perf/docgen-shared/stats.test.ts similarity index 100% rename from scripts/bench/docgen-shared/stats.test.ts rename to code/lib/docgen-harness/src/perf/docgen-shared/stats.test.ts diff --git a/scripts/bench/docgen-shared/stats.ts b/code/lib/docgen-harness/src/perf/docgen-shared/stats.ts similarity index 100% rename from scripts/bench/docgen-shared/stats.ts rename to code/lib/docgen-harness/src/perf/docgen-shared/stats.ts diff --git a/scripts/ci/common-jobs.ts b/scripts/ci/common-jobs.ts index 9964b1b22ac5..c2e72958cd28 100644 --- a/scripts/ci/common-jobs.ts +++ b/scripts/ci/common-jobs.ts @@ -445,7 +445,7 @@ export const docgenMemoryGate = defineJob( { run: { name: 'Docgen-server re-extraction memory gate', - working_directory: 'scripts', + working_directory: 'code/lib/docgen-harness', command: 'yarn bench:docgen-memory', }, }, diff --git a/scripts/knip.config.ts b/scripts/knip.config.ts index c159f8163cba..d010701d87e6 100644 --- a/scripts/knip.config.ts +++ b/scripts/knip.config.ts @@ -63,6 +63,18 @@ const baseConfig = { entry: ['src/index.ts', 'src/ink/steps/checks/index.tsx'], project, }, + // The perf bench is a set of CLIs and spawned child processes, reachable from `yarn + // bench:docgen-*` rather than from the package's own entry, so each root is listed here. + 'lib/docgen-harness': { + entry: [ + 'src/index.ts', + 'src/perf/docgen-perf/run.ts', + 'src/perf/docgen-perf/engines/{react-legacy,vue-component-meta,vue-docgen-api}.ts', + 'src/perf/docgen-perf/generators/{angular,vue}.ts', + 'src/perf/docgen-memory/{gate,memory-harness,generate-project}.ts', + ], + project, + }, 'lib/*': { project, }, diff --git a/scripts/package.json b/scripts/package.json index 7169081cf2aa..6b95ec9d95e3 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -5,8 +5,6 @@ "type": "module", "scripts": { "bench-packages": "jiti ./bench/bench-packages.ts", - "bench:docgen-memory": "node --import jiti/register ./bench/docgen-memory/gate.ts", - "bench:docgen-perf": "node ./bench/docgen-perf/run.ts", "build-package": "jiti ./build-package.ts", "check": "node ./check/check-package.ts", "check-package": "jiti ./check-package.ts", @@ -50,14 +48,12 @@ "dependencies": { "@actions/core": "^1.11.1", "@anthropic-ai/claude-agent-sdk": "^0.2.85", - "@compodoc/compodoc": "2.0.0", "@fal-works/esbuild-plugin-global-externals": "^2.1.2", "@google-cloud/bigquery": "^6.2.1", "@octokit/graphql": "^5.0.6", "@octokit/request": "^8.4.1", "@openai/codex-sdk": "^0.117.0", "@polka/parse": "^1.0.0-next.28", - "@storybook/react": "workspace:*", "@testing-library/dom": "^10.4.0", "@testing-library/jest-dom": "6.9.1", "@testing-library/react": "^16.0.0", @@ -157,10 +153,6 @@ "typescript-native": "npm:typescript@^7.0.2", "uuid": "^9.0.1", "vitest": "^4.1.5", - "vue": "^3.2.47", - "vue-component-meta": "^3.2.7", - "vue-component-meta-next": "npm:vue-component-meta@3.3.8", - "vue-docgen-api": "^4.75.1", "wait-on": "^8.0.3", "window-size": "^1.1.1", "yaml": "^2.8.1", diff --git a/yarn.lock b/yarn.lock index 25de2481c5ab..88d38675b0f4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12295,16 +12295,25 @@ __metadata: "@angular/core": "npm:^21.2.14" "@angular/platform-browser": "npm:^21.2.14" "@angular/platform-browser-dynamic": "npm:^21.2.14" + "@compodoc/compodoc": "npm:2.0.0" "@storybook/angular-vite": "workspace:*" + "@storybook/react": "workspace:*" "@storybook/vue3": "workspace:*" "@testing-library/vue": "npm:^8.0.0" + "@types/react": "npm:^18.0.37" "@vitejs/plugin-vue": "npm:^4.6.2" + jiti: "npm:^2.6.1" + react: "npm:^18.2.0" + react-dom: "npm:^18.2.0" rxjs: "npm:^7.4.0" + storybook: "workspace:*" typescript: "npm:^6.0.3" vue: "npm:^3.2.47" vue-component-meta: "npm:^3.3.9" + vue-component-meta-next: "npm:vue-component-meta@3.3.8" vue-docgen-api: "npm:^4.75.1" vue-tsc: "npm:latest" + zod: "npm:^3.25.76" zone.js: "npm:^0.16.2" languageName: unknown linkType: soft @@ -12758,14 +12767,12 @@ __metadata: dependencies: "@actions/core": "npm:^1.11.1" "@anthropic-ai/claude-agent-sdk": "npm:^0.2.85" - "@compodoc/compodoc": "npm:2.0.0" "@fal-works/esbuild-plugin-global-externals": "npm:^2.1.2" "@google-cloud/bigquery": "npm:^6.2.1" "@octokit/graphql": "npm:^5.0.6" "@octokit/request": "npm:^8.4.1" "@openai/codex-sdk": "npm:^0.117.0" "@polka/parse": "npm:^1.0.0-next.28" - "@storybook/react": "workspace:*" "@testing-library/dom": "npm:^10.4.0" "@testing-library/jest-dom": "npm:6.9.1" "@testing-library/react": "npm:^16.0.0" @@ -12869,10 +12876,6 @@ __metadata: verdaccio: "npm:^5.33.0" verdaccio-auth-memory: "npm:^10.3.1" vitest: "npm:^4.1.5" - vue: "npm:^3.2.47" - vue-component-meta: "npm:^3.2.7" - vue-component-meta-next: "npm:vue-component-meta@3.3.8" - vue-docgen-api: "npm:^4.75.1" wait-on: "npm:^8.0.3" window-size: "npm:^1.1.1" yaml: "npm:^2.8.1" @@ -39748,7 +39751,7 @@ __metadata: languageName: node linkType: hard -"vue-component-meta@npm:^3.2.7, vue-component-meta@npm:^3.3.9": +"vue-component-meta@npm:^3.3.9": version: 3.3.9 resolution: "vue-component-meta@npm:3.3.9" dependencies: