From 9d4df81981c8eb4cced5f1d4eab568e915b6b078 Mon Sep 17 00:00:00 2001 From: Valentin Palkovic Date: Tue, 28 Jul 2026 23:01:04 +0200 Subject: [PATCH 1/4] Bench: Add the docgen perf engine framework and the compodoc engine This is the base of the per-engine perf suite: the metric types, the fixed measurement profiles, the aggregation that turns repetitions into medians, the child-process spawn helper, and the BenchEngine base class every engine implements. It comes with the first engine, compodoc, which measures Angular. Compodoc runs as a fresh CLI process, so cold extraction and whole-project scan are the same measurement and peak memory is sampled from outside the child. The engine also reads the documentation.json compodoc writes, and reports two counts: how many members it documented, and how many of those carry a type it never resolved. The second count matters because compodoc never expands a named type. Chain a type through twenty files and it documents every member at the same speed, having looked through none of them, so a member count alone would read as identical to an engine that expanded all of them. @compodoc/compodoc is pinned at exactly 2.0.0, the version the Angular docgen baselines capture against. Its cost moves between versions. The profile and the engine id list carry Angular only. React and Vue arrive with their engines in the next PR. There is no CLI entry point yet. registry.ts, cli.ts and run.ts land in the PR stacked on top, so this cannot be run end to end; the two test files cover the aggregation and the documentation.json parsing. --- scripts/bench/docgen-perf/aggregate.test.ts | 109 +++++ scripts/bench/docgen-perf/aggregate.ts | 87 ++++ scripts/bench/docgen-perf/config.ts | 39 ++ scripts/bench/docgen-perf/engine.ts | 127 ++++++ .../docgen-perf/engines/compodoc-doc.test.ts | 53 +++ .../bench/docgen-perf/engines/compodoc-doc.ts | 76 ++++ scripts/bench/docgen-perf/engines/compodoc.ts | 184 ++++++++ .../bench/docgen-perf/generators/angular.ts | 142 ++++++ scripts/bench/docgen-perf/spawn.ts | 59 +++ scripts/bench/docgen-perf/types.ts | 120 +++++ scripts/bench/docgen-shared/engine-ids.ts | 2 +- scripts/package.json | 1 + yarn.lock | 421 +++++++++++++++++- 13 files changed, 1397 insertions(+), 23 deletions(-) create mode 100644 scripts/bench/docgen-perf/aggregate.test.ts create mode 100644 scripts/bench/docgen-perf/aggregate.ts create mode 100644 scripts/bench/docgen-perf/config.ts create mode 100644 scripts/bench/docgen-perf/engine.ts create mode 100644 scripts/bench/docgen-perf/engines/compodoc-doc.test.ts create mode 100644 scripts/bench/docgen-perf/engines/compodoc-doc.ts create mode 100644 scripts/bench/docgen-perf/engines/compodoc.ts create mode 100644 scripts/bench/docgen-perf/generators/angular.ts create mode 100644 scripts/bench/docgen-perf/spawn.ts create mode 100644 scripts/bench/docgen-perf/types.ts diff --git a/scripts/bench/docgen-perf/aggregate.test.ts b/scripts/bench/docgen-perf/aggregate.test.ts new file mode 100644 index 000000000000..cf488729ba99 --- /dev/null +++ b/scripts/bench/docgen-perf/aggregate.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it } from 'vitest'; + +import type { MemorySample, SaveSample } from '../docgen-shared/samples.ts'; +import type { SeriesResult } from '../docgen-shared/series.ts'; +import { summarizeSeries } from '../docgen-shared/stats.ts'; +import { designatedRep, oneShotMetrics, seriesMetrics } from './aggregate.ts'; + +function sample(save: number, durMs: number, heapUsedMb = 20, retainedHeapMb = 10): SaveSample { + return { save, durMs, rssMb: 100, heapUsedMb, retainedHeapMb }; +} + +/** Built through summarizeSeries, so the fixture cannot claim figures a real run would not produce. */ +function repOf(coldMs: number, samples: SaveSample[], baseline: MemorySample): SeriesResult { + return { coldMs, baseline, samples, ...summarizeSeries(samples, baseline) }; +} + +function rep(coldMs: number, durations: number[]): SeriesResult { + return repOf( + coldMs, + durations.map((d, i) => sample(i + 1, d)), + { rssMb: 90, heapUsedMb: 12, retainedHeapMb: 10 } + ); +} + +describe('designatedRep', () => { + it('picks the repetition whose cold sample is the median', () => { + const reps = [rep(900, [1]), rep(100, [2]), rep(300, [3]), rep(200, [4]), rep(250, [5])]; + expect(designatedRep(reps).coldMs).toBe(250); + }); + + it('never picks the slow first repetition just because it came first', () => { + const reps = [rep(5000, [999]), rep(100, [10]), rep(110, [11]), rep(120, [12]), rep(130, [13])]; + expect(designatedRep(reps).coldMs).not.toBe(5000); + expect(designatedRep(reps).coldMs).toBe(120); + }); + + it('takes the lower middle for an even count', () => { + const reps = [rep(10, [1]), rep(20, [2]), rep(30, [3]), rep(40, [4])]; + expect(designatedRep(reps).coldMs).toBe(20); + }); + + it('does not mutate its input', () => { + const reps = [rep(30, [1]), rep(10, [2]), rep(20, [3])]; + designatedRep(reps); + expect(reps.map((r) => r.coldMs)).toEqual([30, 10, 20]); + }); +}); + +describe('seriesMetrics', () => { + const reps = [rep(300, [9, 9, 9]), rep(100, [1, 2, 3]), rep(200, [4, 5, 6])]; + + it('medians cold across every repetition', () => { + expect(seriesMetrics(reps, 3).coldExtractionMs).toMatchObject({ + samples: [300, 100, 200], + median: 200, + }); + }); + + it('reads warm from the designated repetition, not the first', () => { + expect(seriesMetrics(reps, 3).warmExtractionMs).toMatchObject({ samples: [4, 5, 6], median: 5 }); + }); + + it('marks whole-project scan as not applicable rather than faking one', () => { + expect(seriesMetrics(reps, 3).wholeProjectScanMs).toEqual({ status: 'n/a' }); + }); + + it('derives transient memory from the same repetition warm came from', () => { + expect(seriesMetrics(reps, 3).peakTransientMb).toMatchObject({ mean: 10 }); + }); + + it('rejects a repetition count below the pinned N', () => { + expect(() => seriesMetrics(reps, 5)).toThrow('recorded 3 repetitions, expected the pinned 5'); + }); + + it('rejects an empty set of repetitions', () => { + expect(() => seriesMetrics([], 5)).toThrow('no completed repetition recorded'); + }); + + it('rejects a run with no retained samples', () => { + const noGc = repOf(100, [{ save: 1, durMs: 5, rssMb: 100, heapUsedMb: 20 }], { + rssMb: 90, + heapUsedMb: 12, + }); + expect(() => seriesMetrics([noGc], 1)).toThrow('retained metrics missing'); + }); +}); + +describe('oneShotMetrics', () => { + const reps = [ + { coldMs: 300, warmMs: 280, peakRssMb: 500 }, + { coldMs: 100, warmMs: 120, peakRssMb: 400 }, + { coldMs: 200, warmMs: 200, peakRssMb: 450 }, + ]; + + it('reports cold and whole-project scan as the same measurement', () => { + const metrics = oneShotMetrics(reps, 3); + expect(metrics.wholeProjectScanMs).toEqual(metrics.coldExtractionMs); + }); + + it('has no retained series to report', () => { + const metrics = oneShotMetrics(reps, 3); + expect(metrics.retainedGrowthMb).toEqual({ status: 'n/a' }); + expect(metrics.retainedSlopeMbPerSave).toEqual({ status: 'n/a' }); + }); + + it('enforces the pinned repetition count', () => { + expect(() => oneShotMetrics(reps, 5)).toThrow('expected the pinned 5'); + }); +}); diff --git a/scripts/bench/docgen-perf/aggregate.ts b/scripts/bench/docgen-perf/aggregate.ts new file mode 100644 index 000000000000..10762817ce50 --- /dev/null +++ b/scripts/bench/docgen-perf/aggregate.ts @@ -0,0 +1,87 @@ +/** + * Turns N repetitions into the five floor metrics (see scripts/bench/PERF-METHODOLOGY.md). Warm + * latency and all three memory metrics read the same designated repetition's save series, so those + * four figures always describe one run; cold latency is a median across every repetition instead. + */ +import type { SeriesResult } from '../docgen-shared/series.ts'; +import { mean, median } from '../docgen-shared/stats.ts'; +import { type EngineMetrics, NOT_APPLICABLE } from './types.ts'; + +/** + * Repetition 1 is systematically the slowest - it pays for a cold module graph and a cold OS page + * cache - so taking it would bias warm latency and all three memory metrics at once. Cold latency + * needs no such protection: it is already a median across every repetition. + */ +export function designatedRep(reps: T[]): T { + const byCold = [...reps].sort((a, b) => a.coldMs - b.coldMs); + return byCold[Math.floor((byCold.length - 1) / 2)]; +} + +/** + * An engine that failed part-way through holds fewer samples than expectedN. Reporting those would + * put numbers taken at an unrecorded N into the results file, which the comparison method rests on + * not happening. + */ +function assertRepetitionCount(reps: unknown[], expectedN: number): void { + if (reps.length === 0) { + throw new Error('no completed repetition recorded'); + } + if (reps.length !== expectedN) { + throw new Error(`recorded ${reps.length} repetitions, expected the pinned ${expectedN}`); + } +} + +export function seriesMetrics(reps: SeriesResult[], expectedN: number): EngineMetrics { + assertRepetitionCount(reps, expectedN); + const designated = designatedRep(reps); + const coldSamples = reps.map((r) => r.coldMs); + const warmSamples = designated.samples.map((s) => s.durMs); + const { transients, avgTransient } = designated; + + if ( + avgTransient === undefined || + designated.retainedGrowth === undefined || + designated.retainedSlope === undefined + ) { + throw new Error('retained metrics missing (child must run under --expose-gc)'); + } + + return { + coldExtractionMs: { status: 'measured', samples: coldSamples, median: median(coldSamples) }, + warmExtractionMs: { status: 'measured', samples: warmSamples, median: median(warmSamples) }, + // Per-component engines have no batch pass; recording one would be a faked equivalent. + wholeProjectScanMs: NOT_APPLICABLE, + peakTransientMb: { status: 'measured', samples: transients, mean: avgTransient }, + retainedGrowthMb: { status: 'measured', value: designated.retainedGrowth }, + retainedSlopeMbPerSave: { status: 'measured', value: designated.retainedSlope }, + }; +} + +export interface OneShotRepetition { + coldMs: number; + warmMs: number; + peakRssMb: number; + coldMembers?: number; + warmMembers?: number; + /** Of the cold pass's members, how many carry a type the engine never resolved. */ + coldOpaqueTypes?: number; +} + +/** + * Metrics for a one-shot CLI engine: a fresh process per run, so cold extraction and the + * whole-project scan are the same full-project measurement, and there is no retained series to read. + */ +export function oneShotMetrics(reps: OneShotRepetition[], expectedN: number): EngineMetrics { + assertRepetitionCount(reps, expectedN); + const coldSamples = reps.map((r) => r.coldMs); + const warmSamples = reps.map((r) => r.warmMs); + const peaks = reps.map((r) => r.peakRssMb); + return { + coldExtractionMs: { status: 'measured', samples: coldSamples, median: median(coldSamples) }, + warmExtractionMs: { status: 'measured', samples: warmSamples, median: median(warmSamples) }, + wholeProjectScanMs: { status: 'measured', samples: coldSamples, median: median(coldSamples) }, + peakTransientMb: { status: 'measured', samples: peaks, mean: mean(peaks) }, + retainedGrowthMb: NOT_APPLICABLE, + retainedSlopeMbPerSave: NOT_APPLICABLE, + }; +} diff --git a/scripts/bench/docgen-perf/config.ts b/scripts/bench/docgen-perf/config.ts new file mode 100644 index 000000000000..461a2f705ac4 --- /dev/null +++ b/scripts/bench/docgen-perf/config.ts @@ -0,0 +1,39 @@ +/** + * Fixed measurement parameters for the per-engine docgen performance suite. + * + * N is pinned here for every engine and recorded with the results; numbers taken at different N are + * not comparable. The --quick profile exists for smoke runs only and marks its results + * non-comparable. + */ + +/** Fresh-process spawns per cold/scan median. One value for all engines. */ +export const PINNED_N = 5; + +/** Spawns for --quick smoke runs. Never comparable with PINNED_N results. */ +export const QUICK_N = 2; + +/** Sampling interval for the compodoc child's externally-polled peak RSS. */ +export const RSS_POLL_INTERVAL_MS = 100; + +export interface AngularScenarioConfig { + components: number; + props: number; +} + +export interface SuiteProfile { + n: number; + comparable: boolean; + angular: AngularScenarioConfig; +} + +export const DEFAULT_PROFILE: SuiteProfile = { + n: PINNED_N, + comparable: true, + angular: { components: 100, props: 8 }, +}; + +export const QUICK_PROFILE: SuiteProfile = { + n: QUICK_N, + comparable: false, + angular: { components: 10, props: 4 }, +}; diff --git a/scripts/bench/docgen-perf/engine.ts b/scripts/bench/docgen-perf/engine.ts new file mode 100644 index 000000000000..55f9af8bace5 --- /dev/null +++ b/scripts/bench/docgen-perf/engine.ts @@ -0,0 +1,127 @@ +/** + * What the orchestrator knows about an engine. Engines differ only in how one repetition is + * produced; everything downstream (aggregation, member counts) follows from that choice. + */ +import * as path from 'node:path'; + +import type { SeriesResult } from '../docgen-shared/series.ts'; +import { seriesMetrics } from './aggregate.ts'; +import type { SuiteProfile } from './config.ts'; +import type { SeriesChildSpec } from './spawn.ts'; +import type { EngineId, EngineMetrics } from './types.ts'; + +export interface ScenarioSpec { + name: string; + /** Recorded verbatim in the results so a stored run is self-describing. */ + params: Record; +} + +/** Only what every engine needs. What one engine alone cares about stays inside that engine. */ +export interface MeasureContext { + /** Scratch directory for this engine/scenario: generated project and per-repetition JSON. */ + scenarioDir: string; + runSeriesChild(spec: SeriesChildSpec, outDir: string, jsonPath: string): SeriesResult; +} + +/** `Sample` is whatever one repetition of this engine produces. */ +export abstract class BenchEngine { + abstract readonly id: EngineId; + + /** Engines outside the default run only measure when named with `--engine`. */ + inDefaultRun = true; + + abstract scenarios(profile: SuiteProfile): ScenarioSpec[]; + + abstract measure(ctx: MeasureContext, scenario: ScenarioSpec, rep: number): Promise; + + abstract aggregate(samples: Sample[], expectedN: number): EngineMetrics; + + /** + * Anything that must resolve before the run starts. A returned string is the skip reason: a + * missing external tool is a partial install, not a regression, so it is reported as skipped + * rather than failed. + */ + preflight(): string | undefined { + return undefined; + } + + /** The resolved version of an externally installed engine, recorded with the results. */ + version(): string | undefined { + return undefined; + } + + /** Members the cold pass documented. Undefined when the engine cannot report a count. */ + coldMembers(_sample: Sample): number | undefined { + return undefined; + } + + /** Members the timed re-extraction documented. */ + warmMembers(_sample: Sample): number | undefined { + return undefined; + } + + /** + * Of the cold pass's members, how many the engine documented under a type name it never resolved. + * Two engines can agree on {@link coldMembers} and still have done entirely different work, so a + * member count alone does not establish like-for-like; this is what separates them. + */ + coldOpaqueTypes(_sample: Sample): number | undefined { + return undefined; + } +} + +export interface SeriesChildConfig { + id: EngineId; + /** Child entry point, relative to this directory. */ + child: string; + scenarios(profile: SuiteProfile): ScenarioSpec[]; + args(scenario: ScenarioSpec): string[]; + /** Only the reused docgen-memory harness runs under the jiti loader. */ + jiti?: boolean; + inDefaultRun?: boolean; +} + +/** + * An engine measured by spawning a series-harness child, one fresh process per repetition. All + * engines of this kind share the series harness, so aggregation and member counts are the same + * for each. + */ +export class SeriesChildEngine extends BenchEngine { + readonly id: EngineId; + readonly #config: SeriesChildConfig; + + constructor(config: SeriesChildConfig) { + super(); + this.id = config.id; + this.inDefaultRun = config.inDefaultRun ?? true; + this.#config = config; + } + + scenarios(profile: SuiteProfile): ScenarioSpec[] { + return this.#config.scenarios(profile); + } + + async measure(ctx: MeasureContext, scenario: ScenarioSpec, rep: number): Promise { + return ctx.runSeriesChild( + { + childPath: path.join(import.meta.dirname, this.#config.child), + args: this.#config.args(scenario), + jiti: this.#config.jiti, + }, + path.join(ctx.scenarioDir, 'project'), + path.join(ctx.scenarioDir, `rep${rep}.json`) + ); + } + + aggregate(samples: SeriesResult[], expectedN: number): EngineMetrics { + return seriesMetrics(samples, expectedN); + } + + coldMembers(sample: SeriesResult): number | undefined { + return sample.coldMembers; + } + + warmMembers(sample: SeriesResult): number | undefined { + return sample.warmMembers; + } +} diff --git a/scripts/bench/docgen-perf/engines/compodoc-doc.test.ts b/scripts/bench/docgen-perf/engines/compodoc-doc.test.ts new file mode 100644 index 000000000000..4397481b7846 --- /dev/null +++ b/scripts/bench/docgen-perf/engines/compodoc-doc.test.ts @@ -0,0 +1,53 @@ +import * as path from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +import { countDocumentation } from './compodoc-doc.ts'; + +/** Real compodoc output, committed as the Angular docgen baselines. */ +const fixture = (name: string) => + countDocumentation( + path.join( + import.meta.dirname, + '../../../../code/lib/docgen-harness/src/angular/__testfixtures__', + name, + 'compodoc-input.json' + ) + ); + +describe('countDocumentation', () => { + it('counts an inherited member once, not once per class that lists it', () => { + // The component declares `heading` and `dismissed` and inherits `dismissible`, which compodoc + // also lists separately under `classes`. Counting `classes` too would report it twice. + expect(fixture('cross-file-inheritance').members).toBe(3); + }); + + it('counts signal inputs and outputs like decorator ones', () => { + expect(fixture('signal-io').members).toBe(6); + }); + + describe('opaque types', () => { + it('separates a union written inline from the same union behind an alias', () => { + // This component's three inputs are `ButtonKind`, `"small" | "large"` and `ToneOption`. The + // inline one describes itself; the two aliases are names compodoc never looked through, and + // an engine that resolved them would document the same three members off far more work. + const counts = fixture('decorator-union-enum'); + expect(counts.members).toBe(3); + expect(counts.opaqueTypes).toBe(2); + }); + + it('counts an unresolved generic parameter', () => { + expect(fixture('decorator-generic')).toEqual({ members: 2, opaqueTypes: 2 }); + }); + + it('counts an output whose payload type was dropped', () => { + // compodoc records `EventEmitter`, losing the emitted shape. + expect(fixture('decorator-io-basics')).toEqual({ members: 5, opaqueTypes: 1 }); + }); + + it('reports none when every member describes itself', () => { + expect(fixture('properties-methods-noise').opaqueTypes).toBe(0); + expect(fixture('jsdoc-tags').opaqueTypes).toBe(0); + }); + }); +}); diff --git a/scripts/bench/docgen-perf/engines/compodoc-doc.ts b/scripts/bench/docgen-perf/engines/compodoc-doc.ts new file mode 100644 index 000000000000..ae0c7e890e42 --- /dev/null +++ b/scripts/bench/docgen-perf/engines/compodoc-doc.ts @@ -0,0 +1,76 @@ +/** + * Reading compodoc's `documentation.json`. + * + * Compodoc records how much it documented and how much of it it actually resolved, which is what + * makes its timings readable: it never expands a named type, so a chain of aliases costs it nothing + * and it still reports every member. Counting members alone would let it look equal to an engine + * that did far more work. + */ +import * as fs from 'node:fs'; + +interface MemberEntry { + type?: string; + subProperties?: unknown[]; +} + +interface MemberHolder { + inputsClass?: MemberEntry[]; + outputsClass?: MemberEntry[]; + propertiesClass?: MemberEntry[]; + methodsClass?: MemberEntry[]; +} + +interface Documentation { + components?: MemberHolder[]; + directives?: MemberHolder[]; +} + +const MEMBER_ARRAYS = ['inputsClass', 'outputsClass', 'propertiesClass', 'methodsClass'] as const; + +/** + * `classes` is deliberately left out. Compodoc copies an ancestor's members into every descendant's + * own arrays, so a base class documented under `classes` would be counted a second time. This also + * matches what Storybook reads, which is a component's own four arrays. + */ +function documentedMembers(doc: Documentation): MemberEntry[] { + return [...(doc.components ?? []), ...(doc.directives ?? [])].flatMap((holder) => + MEMBER_ARRAYS.flatMap((key) => holder[key] ?? []) + ); +} + +/** + * Names that describe themselves, so recording one is not a resolution an engine skipped. Keyword + * spellings are lowercase as compodoc emits them (`function`, not `Function`); both are listed + * because the two spellings reach the same member field. + */ +const RESOLVED_TYPES = new Set([ + 'string', 'number', 'boolean', 'any', 'unknown', 'void', 'never', 'null', 'undefined', + 'object', 'symbol', 'bigint', 'function', 'Date', 'Function', 'Array', 'Object', +]); + +/** + * A member whose type is a bare name the engine never looked through — `Hop19Shape`, not + * `{ x: string }`. Inline object literals, primitives and literal unions are all self-describing, + * so only a lone identifier counts. + * + * This is a floor: wrapped forms such as `Partial` are equally unresolved but do not match, + * so the true share of unresolved types is at least what this reports. + */ +function isOpaque(entry: MemberEntry): boolean { + if (entry.subProperties?.length || !entry.type) { + return false; + } + const named = entry.type.replace(/\[\]$/, '').trim(); + return /^[A-Za-z_$][\w$]*$/.test(named) && !RESOLVED_TYPES.has(named); +} + +export interface DocumentationCounts { + members: number; + opaqueTypes: number; +} + +export function countDocumentation(documentationJsonPath: string): DocumentationCounts { + const doc = JSON.parse(fs.readFileSync(documentationJsonPath, 'utf8')) as Documentation; + const members = documentedMembers(doc); + return { members: members.length, opaqueTypes: members.filter(isOpaque).length }; +} diff --git a/scripts/bench/docgen-perf/engines/compodoc.ts b/scripts/bench/docgen-perf/engines/compodoc.ts new file mode 100644 index 000000000000..63d72a4cc1c7 --- /dev/null +++ b/scripts/bench/docgen-perf/engines/compodoc.ts @@ -0,0 +1,184 @@ +import { spawn, spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import { createRequire } from 'node:module'; +import * as path from 'node:path'; + +import { outputTail } from '../../docgen-shared/child-output.ts'; +import { type OneShotRepetition, oneShotMetrics } from '../aggregate.ts'; +import { type DocumentationCounts, countDocumentation } from './compodoc-doc.ts'; +import { type AngularScenarioConfig, RSS_POLL_INTERVAL_MS, type SuiteProfile } from '../config.ts'; +import { BenchEngine, type MeasureContext, type ScenarioSpec } from '../engine.ts'; +import { angularComponentSource, generateAngularProject } from '../generators/angular.ts'; +import type { EngineId, EngineMetrics } from '../types.ts'; + +const require = createRequire(import.meta.url); + +interface ResolvedCompodoc { + /** The CLI entry point, run as `node ` so no shell shim or exec bit is involved. */ + cli: string; + version: string; +} + +function resolveCompodoc(): ResolvedCompodoc | undefined { + try { + const packagePath = require.resolve('@compodoc/compodoc/package.json'); + const pkg = JSON.parse(fs.readFileSync(packagePath, 'utf8')); + return { + cli: path.join(path.dirname(packagePath), pkg.bin.compodoc), + version: pkg.version, + }; + } catch { + return undefined; + } +} + +interface CompodocRun extends DocumentationCounts { + durMs: number; + peakRssMb: number; +} + +function runCompodocOnce( + cli: string, + projectDir: string, + docsOutDir: string, + pollIntervalMs: number +): Promise { + fs.rmSync(docsOutDir, { recursive: true, force: true }); + const args = [cli, '-p', 'tsconfig.json', '-e', 'json', '-d', docsOutDir, '--silent']; + + return new Promise((resolve, reject) => { + const start = Date.now(); + const child = spawn(process.execPath, args, { cwd: projectDir }); + let peakRssKb = 0; + let output = ''; + child.stdout.on('data', (chunk: Buffer) => (output += chunk.toString())); + child.stderr.on('data', (chunk: Buffer) => (output += chunk.toString())); + + const poll = setInterval(() => { + if (child.pid === undefined) { + return; + } + + const ps = spawnSync('ps', ['-o', 'rss=', '-p', String(child.pid)], { encoding: 'utf8' }); + if (ps.error || typeof ps.stdout !== 'string') { + return; + } + const rssKb = Number(ps.stdout.trim()); + if (Number.isFinite(rssKb) && rssKb > peakRssKb) { + peakRssKb = rssKb; + } + }, pollIntervalMs); + + child.on('error', (err) => { + clearInterval(poll); + reject(err); + }); + child.on('close', (status) => { + clearInterval(poll); + const durMs = Date.now() - start; + if (status !== 0) { + reject(new Error(`compodoc exited with status ${status}:\n${outputTail(output, 8)}`)); + return; + } + const documentationJson = path.join(docsOutDir, 'documentation.json'); + if (!fs.existsSync(documentationJson)) { + reject(new Error('compodoc run produced no documentation.json')); + return; + } + // Counted before the next run overwrites the file, which both runs share. + const counts = countDocumentation(documentationJson); + // The polled peak misses spikes shorter than the interval; the recorded value is a floor. + resolve({ durMs, peakRssMb: peakRssKb / 1024, ...counts }); + }); + }); +} + +/** + * One repetition: fresh project, cold full run, touch one component, warm full run. Both runs are + * fresh compodoc processes, matching the one-sample-per-fresh-process topology. + */ +export async function runCompodocRepetition( + cli: string, + scenario: AngularScenarioConfig, + workDir: string, + pollIntervalMs: number +): Promise { + const projectDir = path.join(workDir, 'project'); + const docsOutDir = path.join(workDir, 'docs'); + const project = generateAngularProject({ + outDir: projectDir, + components: scenario.components, + props: scenario.props, + }); + + const cold = await runCompodocOnce(cli, project.outDir, docsOutDir, pollIntervalMs); + + // Touch one component so the warm run sees a genuinely changed file. + fs.writeFileSync(project.componentPaths[0], angularComponentSource(0, scenario.props + 1)); + const warm = await runCompodocOnce(cli, project.outDir, docsOutDir, pollIntervalMs); + + return { + coldMs: cold.durMs, + warmMs: warm.durMs, + peakRssMb: Math.max(cold.peakRssMb, warm.peakRssMb), + coldMembers: cold.members, + // A second whole-project pass, so this counts the project, not the one file that changed. It is + // not comparable with a series engine's warm count, which covers only the re-extracted member. + warmMembers: warm.members, + coldOpaqueTypes: cold.opaqueTypes, + }; +} + +export class CompodocEngine extends BenchEngine { + readonly id: EngineId = 'compodoc'; + + #resolved: ResolvedCompodoc | undefined; + + scenarios(profile: SuiteProfile): ScenarioSpec[] { + // The poll interval rides in params because it qualifies the peak this engine reports: the + // sample misses spikes shorter than the gap between polls, so the recorded peak is a floor and + // reading it later means knowing how wide that gap was. + return [ + { name: 'default', params: { ...profile.angular, rssPollIntervalMs: RSS_POLL_INTERVAL_MS } }, + ]; + } + + preflight(): string | undefined { + this.#resolved = resolveCompodoc(); + return this.#resolved + ? undefined + : '@compodoc/compodoc did not resolve; it is pinned in scripts/package.json, so run yarn install'; + } + + version(): string | undefined { + return this.#resolved?.version; + } + + async measure(ctx: MeasureContext, scenario: ScenarioSpec): Promise { + if (!this.#resolved) { + throw new Error('compodoc unresolved; preflight must run first'); + } + return runCompodocRepetition( + this.#resolved.cli, + { components: Number(scenario.params.components), props: Number(scenario.params.props) }, + ctx.scenarioDir, + Number(scenario.params.rssPollIntervalMs) + ); + } + + aggregate(samples: OneShotRepetition[], expectedN: number): EngineMetrics { + return oneShotMetrics(samples, expectedN); + } + + coldMembers(sample: OneShotRepetition): number | undefined { + return sample.coldMembers; + } + + warmMembers(sample: OneShotRepetition): number | undefined { + return sample.warmMembers; + } + + coldOpaqueTypes(sample: OneShotRepetition): number | undefined { + return sample.coldOpaqueTypes; + } +} diff --git a/scripts/bench/docgen-perf/generators/angular.ts b/scripts/bench/docgen-perf/generators/angular.ts new file mode 100644 index 000000000000..d56351959acd --- /dev/null +++ b/scripts/bench/docgen-perf/generators/angular.ts @@ -0,0 +1,142 @@ +/** + * Generator for a synthetic Angular project consumable by a standalone Compodoc CLI run. + * + * Ships a minimal fake `(at)angular/core` type surface inside its own node_modules so the tree is + * hermetic - no npm install of the real framework is needed for type resolution. + * + * Run directly: + * node scripts/bench/docgen-perf/generators/angular.ts --out ../storybook-sandboxes/docgen-perf-angular --components 100 + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { z } from 'zod'; + +import { countOption, parseHarnessOptions } from '../../docgen-shared/args.ts'; + +export interface AngularGenerateOptions { + outDir: string; + components: number; + /** Extra `@Input()` members per component on top of the fixed baseline set. */ + props: number; +} + +export interface GeneratedAngularProject { + outDir: string; + /** Absolute component file paths, in component order. */ + componentPaths: string[]; +} + +const FAKE_ANGULAR_CORE = `export declare function Component(metadata: { + selector?: string; + template?: string; + standalone?: boolean; +}): ClassDecorator; +export declare function Input(bindingPropertyName?: string): PropertyDecorator; +export declare function Output(bindingPropertyName?: string): PropertyDecorator; +export declare class EventEmitter { + emit(value?: T): void; + subscribe(next: (value: T) => void): { unsubscribe(): void }; +} +`; + +function emitFakeAngularCore(projectDir: string): void { + const dir = path.join(projectDir, 'node_modules', '@angular', 'core'); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync( + path.join(dir, 'package.json'), + JSON.stringify({ name: '@angular/core', version: '0.0.0-bench', types: 'index.d.ts' }, null, 2) + ); + fs.writeFileSync(path.join(dir, 'index.d.ts'), FAKE_ANGULAR_CORE); +} + +/** + * `extraProps` grows the input surface by one per warm-run touch, so the second Compodoc run sees + * a genuinely changed file. + */ +export function angularComponentSource(i: number, extraProps: number): string { + const extras = Array.from( + { length: extraProps }, + (_, p) => ` /** Extra input ${p} for component ${i}. */ + @Input() extra${p}?: ${p % 3 === 0 ? `'a' | 'b' | 'c'` : p % 3 === 1 ? 'number' : 'string'};` + ).join('\n'); + + return `import { Component, EventEmitter, Input, Output } from '@angular/core'; + +/** + * Comp${i} - generated bench component. + */ +@Component({ + selector: 'bench-comp${i}', + template: '
{{ label }}
', +}) +export class Comp${i}Component { + /** Primary label shown to the user. */ + @Input() label = ''; + /** Numeric size token. */ + @Input() size?: number; + /** Visual variant. */ + @Input() variant?: 'primary' | 'secondary' | 'tertiary'; + /** Disable interaction. */ + @Input() disabled = false; +${extras ? `${extras}\n` : ''} /** Emits when the user acts on the component. */ + @Output() action = new EventEmitter<{ id: string; value: number }>(); +} +`; +} + +const TSCONFIG = { + compilerOptions: { + target: 'ES2020', + module: 'ESNext', + moduleResolution: 'Bundler', + strict: true, + skipLibCheck: true, + experimentalDecorators: true, + emitDecoratorMetadata: false, + }, + include: ['src/**/*.ts'], +}; + +export function generateAngularProject(options: AngularGenerateOptions): GeneratedAngularProject { + const outDir = path.resolve(options.outDir); + fs.rmSync(outDir, { recursive: true, force: true }); + fs.mkdirSync(path.join(outDir, 'src', 'app'), { recursive: true }); + + emitFakeAngularCore(outDir); + + // The compodoc adapter passes `-p tsconfig.json` with the project dir as cwd. + fs.writeFileSync(path.join(outDir, 'tsconfig.json'), JSON.stringify(TSCONFIG, null, 2)); + + const componentPaths: string[] = []; + for (let i = 0; i < options.components; i++) { + const componentPath = path.join(outDir, 'src', 'app', `comp${i}.component.ts`); + fs.writeFileSync(componentPath, angularComponentSource(i, options.props)); + componentPaths.push(componentPath); + } + + return { outDir, componentPaths }; +} + +function parseOptions(argv: string[]): AngularGenerateOptions { + return parseHarnessOptions( + argv, + { out: { type: 'string' }, components: { type: 'string' }, props: { type: 'string' } } as const, + z.object({ + outDir: z.string().default('../storybook-sandboxes/docgen-perf-angular'), + components: countOption(100), + props: countOption(8), + }), + (values) => ({ ...values, outDir: values.out }) + ); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const options = parseOptions(process.argv.slice(2)); + const start = Date.now(); + const result = generateAngularProject(options); + console.log( + `Generated ${options.components} Angular components into ${result.outDir} in ${Date.now() - start}ms` + ); +} diff --git a/scripts/bench/docgen-perf/spawn.ts b/scripts/bench/docgen-perf/spawn.ts new file mode 100644 index 000000000000..b9e376957005 --- /dev/null +++ b/scripts/bench/docgen-perf/spawn.ts @@ -0,0 +1,59 @@ +/** + * Spawning one measured child process and reading its result back. + * + * Every measurement runs in a fresh process so it starts from a clean heap; an engine measured after + * another engine's garbage would report that garbage as its own. + */ +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import { outputTail } from '../docgen-shared/child-output.ts'; +import type { SeriesResult } from '../docgen-shared/series.ts'; + +export interface SeriesChildSpec { + childPath: string; + args: string[]; + /** + * Run the child under the jiti loader. Only the reused docgen-memory harness needs it: under + * jiti, react-docgen's browserslist dependency fails on its JSON data require + * ("jsReleases.map is not a function"), so the legacy child must run on native type stripping. + */ + jiti?: boolean; +} + +/** + * Run one series-harness child and parse its result JSON. A non-zero exit, or a missing or + * unreadable result, is a failure - never a silently empty measurement. + */ +export function runSeriesChild( + spec: SeriesChildSpec, + outDir: string, + jsonPath: string +): SeriesResult { + fs.mkdirSync(path.dirname(jsonPath), { recursive: true }); + // Remove any stale result so a crashed run cannot be mistaken for a prior success. + fs.rmSync(jsonPath, { force: true }); + + const nodeArgs = [ + '--expose-gc', + ...(spec.jiti ? ['--import', 'jiti/register'] : []), + spec.childPath, + ...spec.args, + '--out', + outDir, + '--json', + jsonPath, + ]; + + const proc = spawnSync(process.execPath, nodeArgs, { encoding: 'utf8' }); + const output = `${proc.stdout ?? ''}${proc.stderr ?? ''}`; + + if (proc.status !== 0) { + throw new Error(`child exited with status ${proc.status}:\n${outputTail(output, 4)}`); + } + if (!fs.existsSync(jsonPath)) { + throw new Error(`child wrote no result JSON at ${jsonPath}:\n${outputTail(output, 4)}`); + } + return JSON.parse(fs.readFileSync(jsonPath, 'utf8')) as SeriesResult; +} diff --git a/scripts/bench/docgen-perf/types.ts b/scripts/bench/docgen-perf/types.ts new file mode 100644 index 000000000000..671b4865a749 --- /dev/null +++ b/scripts/bench/docgen-perf/types.ts @@ -0,0 +1,120 @@ +/** + * Result shapes for the per-engine docgen performance suite. The measurement contract behind these + * shapes is scripts/bench/PERF-METHODOLOGY.md. + */ +import type { EngineId } from '../docgen-shared/engine-ids.ts'; + +export type { EngineId }; + +/** A latency metric: median of repeated samples (fresh process each, for cold/scan). */ +export interface LatencyMetric { + status: 'measured'; + samples: number[]; + median: number; +} + +/** A memory metric aggregated as the mean of a per-save (or per-run) series. */ +export interface SeriesMeanMetric { + status: 'measured'; + samples: number[]; + mean: number; +} + +/** A single-valued metric read from one run's series (retained growth, retained slope). */ +export interface ValueMetric { + status: 'measured'; + value: number; +} + +/** The explicit marker for a metric that does not apply to an engine; never a faked equivalent. */ +export interface NotApplicable { + status: 'n/a'; +} + +export const NOT_APPLICABLE: NotApplicable = { status: 'n/a' }; + +export interface EngineMetrics { + coldExtractionMs: LatencyMetric | NotApplicable; + warmExtractionMs: LatencyMetric | NotApplicable; + wholeProjectScanMs: LatencyMetric | NotApplicable; + peakTransientMb: SeriesMeanMetric | NotApplicable; + retainedGrowthMb: ValueMetric | NotApplicable; + retainedSlopeMbPerSave: ValueMetric | NotApplicable; +} + +export interface ScenarioResult { + params: Record; + metrics: EngineMetrics; + /** + * Members the cold pass documented, when the engine reports it. Two engines over the same project + * can differ by an order of magnitude here, and a timing ratio between them means nothing without + * it. + */ + coldMembers?: number; + /** Members the timed re-extraction documented. Warm ratios need this for the same reason. */ + warmMembers?: number; + /** + * Of {@link coldMembers}, how many the engine documented under a type name it never resolved. + * An engine that prints `Hop19Shape` and one that expands it into its fields report the same + * member count off very different work, so equal counts alone do not make a ratio like-for-like. + */ + coldOpaqueTypes?: number; +} + +export type EngineResult = + | { status: 'measured'; scenarios: Record } + | { status: 'skipped'; reason: string } + | { status: 'failed'; reason: string }; + +/** + * Whether a ratio compared equal work, and when it did not, which side did more. + * + * The direction decides how to read the number. A new engine that did less is fast for the wrong + * reason and its ratio is worthless; one that did more and still won has a ratio that understates + * it. Only `like-for-like` may become a budget. + * + * The two `resolves` cases exist because equal member counts are not enough on their own: an engine + * that records a type's name without looking through it documents exactly as many members as one + * that expanded the whole chain, off a fraction of the work. + */ +export type Comparability = + | 'unknown' + | 'like-for-like' + | 'next-documents-more' + | 'next-documents-less' + | 'next-resolves-more' + | 'next-resolves-less'; + +/** + * One control pair's comparison for one scenario: the legacy engine's median over the new engine's, + * both measured in the same invocation. + * + * Cold and warm carry their own verdict because they are independent measurements - a pair can + * document the same members cold and different ones on the save it was timed on. + */ +export interface RatioEntry { + cold?: number; + warm?: number; + legacyColdMembers?: number; + nextColdMembers?: number; + legacyWarmMembers?: number; + nextWarmMembers?: number; + coldComparability: Comparability; + warmComparability: Comparability; +} + +/** Keyed by control-pair name, then by scenario name. */ +export type Ratios = Record>; + +export interface SuiteResults { + generatedAt: string; + nodeVersion: string; + /** The one pinned N; numbers taken at different N are not comparable. */ + pinnedN: number; + /** False for --quick smoke runs, whose numbers must never be compared against real runs. */ + comparable: boolean; + /** Resolved versions of externally-installed engines, when they ran. */ + engineVersions: Partial>; + engines: Partial>; + ratios: Ratios; +} diff --git a/scripts/bench/docgen-shared/engine-ids.ts b/scripts/bench/docgen-shared/engine-ids.ts index 250f83a82912..b3208bbf0e70 100644 --- a/scripts/bench/docgen-shared/engine-ids.ts +++ b/scripts/bench/docgen-shared/engine-ids.ts @@ -1,4 +1,4 @@ /** * The docgen engines under measurement. Each engine adds its own id when it lands. */ -export type EngineId = 'react-osa'; +export type EngineId = 'react-osa' | 'compodoc'; diff --git a/scripts/package.json b/scripts/package.json index 851ebb1d538c..178d7bfe28b4 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -49,6 +49,7 @@ "dependencies": { "@actions/core": "^1.11.1", "@anthropic-ai/claude-agent-sdk": "^0.2.85", + "@compodoc/compodoc": "2.0.0", "@fal-works/esbuild-plugin-global-externals": "^2.1.2", "@google-cloud/bigquery": "^6.2.1", "@octokit/graphql": "^5.0.6", diff --git a/yarn.lock b/yarn.lock index 25d3bef2eac4..4d7a3fb7f47e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -48,6 +48,13 @@ __metadata: languageName: node linkType: hard +"@aduh95/viz.js@npm:3.4.0": + version: 3.4.0 + resolution: "@aduh95/viz.js@npm:3.4.0" + checksum: 10c0/6acb40e881dc0b7ac7a6800a1508e8c420e1f6b09bd42e3289aa2199345729af8aec6ffc52dc72cfe678476fcc341f4a83bc2c3d59cd223232e8495f3a414ab0 + languageName: node + linkType: hard + "@ai-sdk/anthropic@npm:^2.0.0": version: 2.0.91 resolution: "@ai-sdk/anthropic@npm:2.0.91" @@ -488,6 +495,38 @@ __metadata: languageName: node linkType: hard +"@angular-devkit/core@npm:22.0.4": + version: 22.0.4 + resolution: "@angular-devkit/core@npm:22.0.4" + dependencies: + ajv: "npm:8.20.0" + ajv-formats: "npm:3.0.1" + jsonc-parser: "npm:3.3.1" + picomatch: "npm:4.0.4" + rxjs: "npm:7.8.2" + source-map: "npm:0.7.6" + peerDependencies: + chokidar: ^5.0.0 + peerDependenciesMeta: + chokidar: + optional: true + checksum: 10c0/0d84969ad31113a48202c644038386f7dd120af950490125d7b8d56c4ebc9ca362cff56be70f90eb3c3be1c15cbcec11aad648e46caebabba82efc91e75dff20 + languageName: node + linkType: hard + +"@angular-devkit/schematics@npm:22.0.4": + version: 22.0.4 + resolution: "@angular-devkit/schematics@npm:22.0.4" + dependencies: + "@angular-devkit/core": "npm:22.0.4" + jsonc-parser: "npm:3.3.1" + magic-string: "npm:0.30.21" + ora: "npm:9.4.0" + rxjs: "npm:7.8.2" + checksum: 10c0/02288bf8f636d3517d03e97942ac1fab4ec8e4448d0d918abd96b06b1a8d0e043482993b51fc69e6c2fe43a3459fb4bbf8c3cccc803f17649ebc2f61c656d94d + languageName: node + linkType: hard + "@angular/animations@npm:^19.1.1": version: 19.2.15 resolution: "@angular/animations@npm:19.2.15" @@ -909,6 +948,13 @@ __metadata: languageName: node linkType: hard +"@arr/every@npm:^1.0.0": + version: 1.0.1 + resolution: "@arr/every@npm:1.0.1" + checksum: 10c0/f9769bc97eca37d1c0dedfda4c17506c0e1269d95c15a7478a49c4d84d43512a3f350b71c95e045cf4113d3398ebd6d298f2eea09196a1ed61c311bc4e6964e7 + languageName: node + linkType: hard + "@aw-web-design/x-default-browser@npm:1.4.126": version: 1.4.126 resolution: "@aw-web-design/x-default-browser@npm:1.4.126" @@ -2757,6 +2803,69 @@ __metadata: languageName: node linkType: hard +"@compodoc/compodoc@npm:2.0.0": + version: 2.0.0 + resolution: "@compodoc/compodoc@npm:2.0.0" + dependencies: + "@angular-devkit/schematics": "npm:22.0.4" + "@compodoc/ngd-transformer": "npm:^2.1.3" + body-parser: "npm:^2.3.0" + bootstrap.native: "npm:^5.1.10" + cheerio: "npm:1.2.0" + chokidar: "npm:^5.0.0" + commander: "npm:^15.0.0" + cosmiconfig: "npm:^9.0.2" + es6-shim: "npm:^0.35.8" + fs-extra: "npm:^11.3.5" + handlebars: "npm:^4.7.9" + html-entities: "npm:^2.6.0" + i18next: "npm:26.3.3" + json5: "npm:^2.2.3" + loglevel: "npm:^1.9.2" + loglevel-plugin-prefix: "npm:^0.8.4" + lunr: "npm:^2.3.9" + marked: "npm:18.0.5" + neotraverse: "npm:^1.0.1" + picocolors: "npm:^1.1.1" + polka: "npm:^0.5.2" + prismjs: "npm:^1.30.0" + semver: "npm:^7.8.5" + sirv: "npm:^3.0.2" + svg-pan-zoom: "npm:^3.6.2" + tablesort: "npm:^5.7.1" + tinyglobby: "npm:^0.2.17" + ts-morph: "npm:^28.0.0" + uuid: "npm:14.0.1" + vis-network: "npm:^10.1.0" + bin: + compodoc: bin/index-cli.js + checksum: 10c0/327e6b7d173f75be1996b59b89d3832467599782530f1e2b1ff8bcc73e977833e69e35e5ab57bfd79ddfb02e31ea58e5ee9a58201829751b330601f74af0efc8 + languageName: node + linkType: hard + +"@compodoc/ngd-core@npm:~2.1.1": + version: 2.1.1 + resolution: "@compodoc/ngd-core@npm:2.1.1" + dependencies: + ansi-colors: "npm:^4.1.3" + fancy-log: "npm:^2.0.0" + typescript: "npm:^5.0.4" + checksum: 10c0/f78f4231e39d96e397bb39614cfbe8dfc6b985eb9b2795cd5188a40f65eb60eb096dc29de61baa6d914fbc86ec7f21b5a2d02e6000b26d82ae3cd682aea79275 + languageName: node + linkType: hard + +"@compodoc/ngd-transformer@npm:^2.1.3": + version: 2.1.3 + resolution: "@compodoc/ngd-transformer@npm:2.1.3" + dependencies: + "@aduh95/viz.js": "npm:3.4.0" + "@compodoc/ngd-core": "npm:~2.1.1" + dot: "npm:^2.0.0-beta.1" + fs-extra: "npm:^11.1.1" + checksum: 10c0/fb7b13860fc022cda284ca0bae55f8c6610621a060c943740838b5fa2f6be7957c99c2750024cf949b2c8f1c5f41bc64cd8e28ab333f985627f5991bcaf5cdcf + languageName: node + linkType: hard + "@cypress/request@npm:3.0.6": version: 3.0.6 resolution: "@cypress/request@npm:3.0.6" @@ -7241,6 +7350,13 @@ __metadata: languageName: node linkType: hard +"@polka/url@npm:^0.5.0": + version: 0.5.0 + resolution: "@polka/url@npm:0.5.0" + checksum: 10c0/76324126e1608f4cdc5c6b11b618ca67612c37d45401f3fbcce4967294093761b7568cc6ae74ade8afbe744547e72e9d50bc38f690ddea0904a1d66a6b8a0093 + languageName: node + linkType: hard + "@polka/url@npm:^1.0.0-next.21, @polka/url@npm:^1.0.0-next.24": version: 1.0.0-next.29 resolution: "@polka/url@npm:1.0.0-next.29" @@ -12642,6 +12758,7 @@ __metadata: dependencies: "@actions/core": "npm:^1.11.1" "@anthropic-ai/claude-agent-sdk": "npm:^0.2.85" + "@compodoc/compodoc": "npm:2.0.0" "@fal-works/esbuild-plugin-global-externals": "npm:^2.1.2" "@google-cloud/bigquery": "npm:^6.2.1" "@octokit/graphql": "npm:^5.0.6" @@ -13749,6 +13866,29 @@ __metadata: languageName: node linkType: hard +"@thednp/event-listener@npm:^2.0.15": + version: 2.0.15 + resolution: "@thednp/event-listener@npm:2.0.15" + checksum: 10c0/02ce4402985f72af44049b6557443dd96a47f37301ea2dbc82f009aa56f22d9d5f06b45e4cb07f6c3bc5ff30c0faadd118095212f908b40b9521348aee231c39 + languageName: node + linkType: hard + +"@thednp/position-observer@npm:^1.1.3": + version: 1.1.3 + resolution: "@thednp/position-observer@npm:1.1.3" + dependencies: + "@thednp/shorty": "npm:^2.0.14" + checksum: 10c0/1fc95a4d8a67e5accab98a6aac571053190bd491702d09759242357d35d3b7d035998470d86abe6f113304b1a5ea44722e5caa1832ef51c36d8654e3890d1951 + languageName: node + linkType: hard + +"@thednp/shorty@npm:^2.0.14": + version: 2.0.14 + resolution: "@thednp/shorty@npm:2.0.14" + checksum: 10c0/72a4ecdbdbeb05953649721f412c1301f950450906fe5914cf4225b83e49f18dc0573f3526aa6efaaff2e442753b5e5b897ed91f5be0792ad4766c3462a0277d + languageName: node + linkType: hard + "@tmcp/adapter-valibot@npm:^0.1.5": version: 0.1.6 resolution: "@tmcp/adapter-valibot@npm:0.1.6" @@ -13842,6 +13982,17 @@ __metadata: languageName: node linkType: hard +"@ts-morph/common@npm:~0.29.0": + version: 0.29.0 + resolution: "@ts-morph/common@npm:0.29.0" + dependencies: + minimatch: "npm:^10.0.1" + path-browserify: "npm:^1.0.1" + tinyglobby: "npm:^0.2.14" + checksum: 10c0/97ab8ca66558b817e5475a8893ee1476ab760a3ac71fa9868413d95ddbaaf909eda81716e3a8d14291cbf449e624af66de81bcddd3ec2e36525728b4edf5e8ee + languageName: node + linkType: hard + "@tybys/wasm-util@npm:^0.10.2, @tybys/wasm-util@npm:^0.10.3": version: 0.10.3 resolution: "@tybys/wasm-util@npm:0.10.3" @@ -16952,6 +17103,18 @@ __metadata: languageName: node linkType: hard +"ajv@npm:8.20.0, ajv@npm:^8.0.0, ajv@npm:^8.17.1, ajv@npm:^8.9.0": + version: 8.20.0 + resolution: "ajv@npm:8.20.0" + dependencies: + fast-deep-equal: "npm:^3.1.3" + fast-uri: "npm:^3.0.1" + json-schema-traverse: "npm:^1.0.0" + require-from-string: "npm:^2.0.2" + checksum: 10c0/5df9a1c8f83863cde1bd3a9ddb426f599718f88e3dc9153616c79fb28e0be455335830d7f21d745576519f057b371352daa31047b6a33d7036fe08777d60cf2a + languageName: node + linkType: hard + "ajv@npm:8.6.3": version: 8.6.3 resolution: "ajv@npm:8.6.3" @@ -16976,18 +17139,6 @@ __metadata: languageName: node linkType: hard -"ajv@npm:^8.0.0, ajv@npm:^8.17.1, ajv@npm:^8.9.0": - version: 8.20.0 - resolution: "ajv@npm:8.20.0" - dependencies: - fast-deep-equal: "npm:^3.1.3" - fast-uri: "npm:^3.0.1" - json-schema-traverse: "npm:^1.0.0" - require-from-string: "npm:^2.0.2" - checksum: 10c0/5df9a1c8f83863cde1bd3a9ddb426f599718f88e3dc9153616c79fb28e0be455335830d7f21d745576519f057b371352daa31047b6a33d7036fe08777d60cf2a - languageName: node - linkType: hard - "alien-signals@npm:^3.2.1": version: 3.2.1 resolution: "alien-signals@npm:3.2.1" @@ -17012,7 +17163,7 @@ __metadata: languageName: node linkType: hard -"ansi-colors@npm:4.1.3, ansi-colors@npm:^4.1.1": +"ansi-colors@npm:4.1.3, ansi-colors@npm:^4.1.1, ansi-colors@npm:^4.1.3": version: 4.1.3 resolution: "ansi-colors@npm:4.1.3" checksum: 10c0/ec87a2f59902f74e61eada7f6e6fe20094a628dab765cfdbd03c3477599368768cffccdb5d3bb19a1b6c99126783a143b1fee31aab729b31ffe5836c7e5e28b9 @@ -18205,7 +18356,7 @@ __metadata: languageName: node linkType: hard -"body-parser@npm:^2.2.1": +"body-parser@npm:^2.2.1, body-parser@npm:^2.3.0": version: 2.3.0 resolution: "body-parser@npm:2.3.0" dependencies: @@ -18259,6 +18410,17 @@ __metadata: languageName: node linkType: hard +"bootstrap.native@npm:^5.1.10": + version: 5.1.10 + resolution: "bootstrap.native@npm:5.1.10" + dependencies: + "@thednp/event-listener": "npm:^2.0.15" + "@thednp/position-observer": "npm:^1.1.3" + "@thednp/shorty": "npm:^2.0.14" + checksum: 10c0/6484411fc4ba5b7d1bdd9da039d07efc32c30b5b216b1442cea4ef235b6398b22ccf721b7216258113f8e9cca86558bf6155abb98ebe7ff55a51e4345130e997 + languageName: node + linkType: hard + "bplist-parser@npm:^0.2.0": version: 0.2.0 resolution: "bplist-parser@npm:0.2.0" @@ -19052,7 +19214,7 @@ __metadata: languageName: node linkType: hard -"cheerio@npm:^1.0.0": +"cheerio@npm:1.2.0, cheerio@npm:^1.0.0": version: 1.2.0 resolution: "cheerio@npm:1.2.0" dependencies: @@ -19447,6 +19609,13 @@ __metadata: languageName: node linkType: hard +"code-block-writer@npm:^13.0.3": + version: 13.0.3 + resolution: "code-block-writer@npm:13.0.3" + checksum: 10c0/87db97b37583f71cfd7eced8bf3f0a0a0ca53af912751a734372b36c08cd27f3e8a4878ec05591c0cd9ae11bea8add1423e132d660edd86aab952656dd41fd66 + languageName: node + linkType: hard + "code-point-at@npm:^1.0.0": version: 1.1.0 resolution: "code-point-at@npm:1.1.0" @@ -19575,6 +19744,13 @@ __metadata: languageName: node linkType: hard +"commander@npm:^15.0.0": + version: 15.0.0 + resolution: "commander@npm:15.0.0" + checksum: 10c0/539229c171914ea1ccd45ee5f10d924289a12a684ea3a7a44147abe54003c35ed6de9ae4ad198d88c29fdf403dca0428451a388234e6dc01b6faa978fb207206 + languageName: node + linkType: hard + "commander@npm:^2.18.0, commander@npm:^2.19.0, commander@npm:^2.20.0, commander@npm:^2.8.1": version: 2.20.3 resolution: "commander@npm:2.20.3" @@ -20005,9 +20181,9 @@ __metadata: languageName: node linkType: hard -"cosmiconfig@npm:^9.0.0": - version: 9.0.0 - resolution: "cosmiconfig@npm:9.0.0" +"cosmiconfig@npm:^9.0.0, cosmiconfig@npm:^9.0.2": + version: 9.0.2 + resolution: "cosmiconfig@npm:9.0.2" dependencies: env-paths: "npm:^2.2.1" import-fresh: "npm:^3.3.0" @@ -20018,7 +20194,7 @@ __metadata: peerDependenciesMeta: typescript: optional: true - checksum: 10c0/1c1703be4f02a250b1d6ca3267e408ce16abfe8364193891afc94c2d5c060b69611fdc8d97af74b7e6d5d1aac0ab2fb94d6b079573146bc2d756c2484ce5f0ee + checksum: 10c0/132d32863c0b3d9ace7010a10011e09089532b36e3b11e02004d671dcbd8b8f2f16293b82d510d9c15890f30358baf8722127c7b1c011449c90b6a178f9c6594 languageName: node linkType: hard @@ -21089,6 +21265,13 @@ __metadata: languageName: node linkType: hard +"dot@npm:^2.0.0-beta.1": + version: 2.0.0-beta.1 + resolution: "dot@npm:2.0.0-beta.1" + checksum: 10c0/28876e214576210384fd49bcf2f3c4afc6478cec7de18e923c07a085dd1f3c565ff162c53911040267fc9ff96fa732dde8676c503ffd60d5a522101bc1ad1e21 + languageName: node + linkType: hard + "dotenv-expand@npm:^10.0.0": version: 10.0.0 resolution: "dotenv-expand@npm:10.0.0" @@ -21931,6 +22114,13 @@ __metadata: languageName: node linkType: hard +"es6-shim@npm:^0.35.8": + version: 0.35.8 + resolution: "es6-shim@npm:0.35.8" + checksum: 10c0/e54e147677b9cd57c96bbc2332c3096ab861040b22daa27728204cacf9aa656d6a3eeb5367a037b276bfbe442ec866938a95b4612accf0cd24d01c209e48eaf9 + languageName: node + linkType: hard + "esast-util-from-estree@npm:^2.0.0": version: 2.0.0 resolution: "esast-util-from-estree@npm:2.0.0" @@ -23035,6 +23225,15 @@ __metadata: languageName: node linkType: hard +"fancy-log@npm:^2.0.0": + version: 2.0.0 + resolution: "fancy-log@npm:2.0.0" + dependencies: + color-support: "npm:^1.1.3" + checksum: 10c0/a6e116f3346756a7363eea343b551c1375d2cd2afc2a92d224feb78589b6b3cff85db9dc5d5df89792a0f7c1e17f731f52cb3d2807302f0516422be6269b95a8 + languageName: node + linkType: hard + "fast-content-type-parse@npm:^2.0.0": version: 2.0.1 resolution: "fast-content-type-parse@npm:2.0.1" @@ -23742,6 +23941,17 @@ __metadata: languageName: node linkType: hard +"fs-extra@npm:^11.1.1, fs-extra@npm:^11.3.5": + version: 11.4.0 + resolution: "fs-extra@npm:11.4.0" + dependencies: + graceful-fs: "npm:^4.2.0" + jsonfile: "npm:^6.0.1" + universalify: "npm:^2.0.0" + checksum: 10c0/1e311eca820a12d3d795f2043dcd5628d017f4d21e634f3f9ef314ae68bee9979758262fb9cb35ffa6f26454c3fffe8b15558733e91e8fc729b07b10bb98d8b6 + languageName: node + linkType: hard + "fs-extra@npm:^5.0.0": version: 5.0.0 resolution: "fs-extra@npm:5.0.0" @@ -24541,6 +24751,24 @@ __metadata: languageName: node linkType: hard +"handlebars@npm:^4.7.9": + version: 4.7.9 + resolution: "handlebars@npm:4.7.9" + dependencies: + minimist: "npm:^1.2.5" + neo-async: "npm:^2.6.2" + source-map: "npm:^0.6.1" + uglify-js: "npm:^3.1.4" + wordwrap: "npm:^1.0.0" + dependenciesMeta: + uglify-js: + optional: true + bin: + handlebars: bin/handlebars + checksum: 10c0/22f8105a7e68e81aff2662bb434edf05f757d21d850731d71cec886d69c10cd33d3c43e34b2892968ec62de8241611851d3d0674c8ef324ea3e01dc66262faa9 + languageName: node + linkType: hard + "happy-dom@npm:^17.6.3": version: 17.6.3 resolution: "happy-dom@npm:17.6.3" @@ -24933,7 +25161,7 @@ __metadata: languageName: node linkType: hard -"html-entities@npm:^2.1.0": +"html-entities@npm:^2.1.0, html-entities@npm:^2.6.0": version: 2.6.0 resolution: "html-entities@npm:2.6.0" checksum: 10c0/7c8b15d9ea0cd00dc9279f61bab002ba6ca8a7a0f3c36ed2db3530a67a9621c017830d1d2c1c65beb9b8e3436ea663e9cf8b230472e0e413359399413b27c8b7 @@ -25305,6 +25533,18 @@ __metadata: languageName: node linkType: hard +"i18next@npm:26.3.3": + version: 26.3.3 + resolution: "i18next@npm:26.3.3" + peerDependencies: + typescript: ^5 || ^6 + peerDependenciesMeta: + typescript: + optional: true + checksum: 10c0/7a6676a8bade1683d23c3187b92d571528eeadf9f3d6776b4ef1bd27789d2ee7594a9aa7ca05394d8e11c931c8c8f224d85b2b5059a5613cb79b04b8d46a291e + languageName: node + linkType: hard + "iconv-lite@npm:0.4.24, iconv-lite@npm:~0.4.24": version: 0.4.24 resolution: "iconv-lite@npm:0.4.24" @@ -27801,6 +28041,20 @@ __metadata: languageName: node linkType: hard +"loglevel-plugin-prefix@npm:^0.8.4": + version: 0.8.4 + resolution: "loglevel-plugin-prefix@npm:0.8.4" + checksum: 10c0/357524eec4c165ff823b5bbf72e8373ff529e5cb95c1f4b20749847bd5b5b16ab328d6d33d1a9019f1a2dc52e28fca5d595e52f2ee20e24986182a6f9552a9ec + languageName: node + linkType: hard + +"loglevel@npm:^1.9.2": + version: 1.9.2 + resolution: "loglevel@npm:1.9.2" + checksum: 10c0/1e317fa4648fe0b4a4cffef6de037340592cee8547b07d4ce97a487abe9153e704b98451100c799b032c72bb89c9366d71c9fb8192ada8703269263ae77acdc7 + languageName: node + linkType: hard + "long@npm:^5.0.0, long@npm:^5.3.2": version: 5.3.2 resolution: "long@npm:5.3.2" @@ -27920,6 +28174,13 @@ __metadata: languageName: node linkType: hard +"lunr@npm:^2.3.9": + version: 2.3.9 + resolution: "lunr@npm:2.3.9" + checksum: 10c0/77d7dbb4fbd602aac161e2b50887d8eda28c0fa3b799159cee380fbb311f1e614219126ecbbd2c3a9c685f1720a8109b3c1ca85cc893c39b6c9cc6a62a1d8a8b + languageName: node + linkType: hard + "luxon@npm:^3.4.0": version: 3.7.2 resolution: "luxon@npm:3.7.2" @@ -28093,6 +28354,15 @@ __metadata: languageName: node linkType: hard +"marked@npm:18.0.5": + version: 18.0.5 + resolution: "marked@npm:18.0.5" + bin: + marked: bin/marked.js + checksum: 10c0/22763935a0a243c41851b010a086ea85a09951e379948b6aa629b6cecdcd66e3a5e8c4acac2f6b29c183c20609d3e5102495270207dfd5f4c46ddd773a4ddb9b + languageName: node + linkType: hard + "matcher-collection@npm:^1.0.0, matcher-collection@npm:^1.1.1": version: 1.1.2 resolution: "matcher-collection@npm:1.1.2" @@ -28112,6 +28382,15 @@ __metadata: languageName: node linkType: hard +"matchit@npm:^1.0.0": + version: 1.1.0 + resolution: "matchit@npm:1.1.0" + dependencies: + "@arr/every": "npm:^1.0.0" + checksum: 10c0/f2cd8702ea37dd9859cf1df762581c822180eb72372b9240397a80320cb4ffb4828b3d8ed0beff2429daf500d1ec746c2963dbb3c6386f1b5b9eb969e0f1caaa + languageName: node + linkType: hard + "math-intrinsics@npm:^1.1.0": version: 1.1.0 resolution: "math-intrinsics@npm:1.1.0" @@ -29365,6 +29644,13 @@ __metadata: languageName: node linkType: hard +"neotraverse@npm:^1.0.1": + version: 1.0.1 + resolution: "neotraverse@npm:1.0.1" + checksum: 10c0/98e5d2d1974f91d527851a9dcfceb62701a1d31970b324b9f35fbdd244d173edc7a379ea1773f64771a9dc282486e3d3b693b24e14b1e969291612b2ff6b5054 + languageName: node + linkType: hard + "netmask@npm:^2.0.2": version: 2.1.1 resolution: "netmask@npm:2.1.1" @@ -30377,6 +30663,22 @@ __metadata: languageName: node linkType: hard +"ora@npm:9.4.0": + version: 9.4.0 + resolution: "ora@npm:9.4.0" + dependencies: + chalk: "npm:^5.6.2" + cli-cursor: "npm:^5.0.0" + cli-spinners: "npm:^3.2.0" + is-interactive: "npm:^2.0.0" + is-unicode-supported: "npm:^2.1.0" + log-symbols: "npm:^7.0.1" + stdin-discarder: "npm:^0.3.2" + string-width: "npm:^8.1.0" + checksum: 10c0/8f2d7a8869cd68607797ac0ffe9f5cdceeb6009437672510d9920aea794cdd055164e3fe804248624c4940a71b22f94f1ffd94ce8fecf0746baef97a5c121a91 + languageName: node + linkType: hard + "ordered-binary@npm:^1.5.3": version: 1.6.0 resolution: "ordered-binary@npm:1.6.0" @@ -31870,6 +32172,16 @@ __metadata: languageName: node linkType: hard +"polka@npm:^0.5.2": + version: 0.5.2 + resolution: "polka@npm:0.5.2" + dependencies: + "@polka/url": "npm:^0.5.0" + trouter: "npm:^2.0.1" + checksum: 10c0/113005fe146e1ee67bc6e1a5438c71e1b59ac05a8bc435d593be210e21f712e6b9ba4d163072e3c941f8efbb7516df9c55bf18934948f91623d03dc9657a1410 + languageName: node + linkType: hard + "polka@npm:^1.0.0-next.28": version: 1.0.0-next.28 resolution: "polka@npm:1.0.0-next.28" @@ -35069,7 +35381,7 @@ __metadata: languageName: node linkType: hard -"semver@npm:7.7.4, semver@npm:^7.0.0, semver@npm:^7.2.1, semver@npm:^7.3.4, semver@npm:^7.3.5, semver@npm:^7.5.3, semver@npm:^7.5.4, semver@npm:^7.6.2, semver@npm:^7.6.3, semver@npm:^7.7.1, semver@npm:^7.7.3": +"semver@npm:7.7.4": version: 7.7.4 resolution: "semver@npm:7.7.4" bin: @@ -35078,6 +35390,15 @@ __metadata: languageName: node linkType: hard +"semver@npm:^7.0.0, semver@npm:^7.2.1, semver@npm:^7.3.4, semver@npm:^7.3.5, semver@npm:^7.5.3, semver@npm:^7.5.4, semver@npm:^7.6.2, semver@npm:^7.6.3, semver@npm:^7.7.1, semver@npm:^7.7.3, semver@npm:^7.8.5": + version: 7.8.5 + resolution: "semver@npm:7.8.5" + bin: + semver: bin/semver.js + checksum: 10c0/b1f3127a5be8125a94f37188b361c212466c292c6910adce3ec106cff5dc211ccaedc4739c11bb70fda59d6fc1f040a9bca289f4e093451521a2372e5231fe0c + languageName: node + linkType: hard + "send@npm:0.19.0": version: 0.19.0 resolution: "send@npm:0.19.0" @@ -36075,7 +36396,7 @@ __metadata: languageName: node linkType: hard -"stdin-discarder@npm:^0.3.1": +"stdin-discarder@npm:^0.3.1, stdin-discarder@npm:^0.3.2": version: 0.3.2 resolution: "stdin-discarder@npm:0.3.2" checksum: 10c0/5dbaba9efbcb447a4450d5ae19794641ea9166abe96dc4b5547a109db1bb6e8bdb17bbe1029e02ca8d9d8ee996b7c7cbcce12b12c18c121871cd4f574292381a @@ -36824,6 +37145,13 @@ __metadata: languageName: node linkType: hard +"svg-pan-zoom@npm:^3.6.2": + version: 3.6.2 + resolution: "svg-pan-zoom@npm:3.6.2" + checksum: 10c0/dc05030bd6545d523de71e38d79d6830f780a74416a609269cc378fe8778796e794cb3e27f0f5b20bd7ee930597820e5b4463e4e4e0708127bed81e020339ec0 + languageName: node + linkType: hard + "symlink-or-copy@npm:^1.0.0, symlink-or-copy@npm:^1.0.1, symlink-or-copy@npm:^1.1.8, symlink-or-copy@npm:^1.2.0, symlink-or-copy@npm:^1.3.1": version: 1.3.1 resolution: "symlink-or-copy@npm:1.3.1" @@ -36853,6 +37181,13 @@ __metadata: languageName: node linkType: hard +"tablesort@npm:^5.7.1": + version: 5.7.1 + resolution: "tablesort@npm:5.7.1" + checksum: 10c0/be69ec398438695b87589de702861d4ca126dba0c4a8c1f4bbc28ffa4ea257a623ee764abd769ec88d7f8347dc36b5157b9d36d282c181a8c43b19413cd88854 + languageName: node + linkType: hard + "tagged-tag@npm:^1.0.0": version: 1.0.0 resolution: "tagged-tag@npm:1.0.0" @@ -37528,6 +37863,15 @@ __metadata: languageName: node linkType: hard +"trouter@npm:^2.0.1": + version: 2.0.1 + resolution: "trouter@npm:2.0.1" + dependencies: + matchit: "npm:^1.0.0" + checksum: 10c0/9d294dbe74d45649d929d3abf0d4cb8e508985a2ea0f8be41b3b097a985352b14f65dd76b68ae651731128995eceddba375b15719356c419bbfe754c5fb5673c + languageName: node + linkType: hard + "trouter@npm:^4.0.0": version: 4.0.0 resolution: "trouter@npm:4.0.0" @@ -37587,6 +37931,16 @@ __metadata: languageName: node linkType: hard +"ts-morph@npm:^28.0.0": + version: 28.0.0 + resolution: "ts-morph@npm:28.0.0" + dependencies: + "@ts-morph/common": "npm:~0.29.0" + code-block-writer: "npm:^13.0.3" + checksum: 10c0/969570a5e983b4193735fff43878ce7b78787b4860bf3d23f330c86d7de707286969e5a82d486dce333eea27bf9d373a46f9de58a80c0bf86d2a462620563a55 + languageName: node + linkType: hard + "ts-pnp@npm:^1.1.6": version: 1.2.0 resolution: "ts-pnp@npm:1.2.0" @@ -38527,6 +38881,15 @@ __metadata: languageName: node linkType: hard +"uuid@npm:14.0.1": + version: 14.0.1 + resolution: "uuid@npm:14.0.1" + bin: + uuid: dist-node/bin/uuid + checksum: 10c0/2d961289097cb68c37d93d1da0b5c3aafc1eb9948a455cca8d0ae53a099b2fe583b8933254a84097f700e6bac2e23033364904df0467c22551d5d9611febcaf6 + languageName: node + linkType: hard + "uuid@npm:^10.0.0": version: 10.0.0 resolution: "uuid@npm:10.0.0" @@ -38799,6 +39162,20 @@ __metadata: languageName: node linkType: hard +"vis-network@npm:^10.1.0": + version: 10.1.0 + resolution: "vis-network@npm:10.1.0" + peerDependencies: + "@egjs/hammerjs": ^2.0.0 + component-emitter: ^1.3.0 || ^2.0.0 + keycharm: ^0.2.0 || ^0.3.0 || ^0.4.0 + uuid: ^3.4.0 || ^7.0.0 || ^8.0.0 || ^9.0.0 || ^10.0.0 || ^11.0.0 || ^13.0.0 || ^14.0.0 + vis-data: ">=8.0.0" + vis-util: ">=6.0.0" + checksum: 10c0/c3f553f03f80e096c7f8cea4fb98011906dde67c65e7d1e8ca0ed5811dad0dc9f648dde93a6a61ce3ac24f153a7a4eb96e931104c8ad1801701a50db687f5bb6 + languageName: node + linkType: hard + "vite-dev-rpc@npm:^1.1.0": version: 1.1.0 resolution: "vite-dev-rpc@npm:1.1.0" From fa1a1548f9555b1387c71958df42f8d44ca3ad0d Mon Sep 17 00:00:00 2001 From: Valentin Palkovic Date: Thu, 30 Jul 2026 20:52:48 +0200 Subject: [PATCH 2/4] Bench: Address review feedback on the engine framework Collapse coldMembers/warmMembers/coldOpaqueTypes into one members() hook returning MemberCounts, and give BenchEngine an assemble() that turns a scenario's repetitions into its ScenarioResult. Both engines implemented the three count hooks as pass-through one-liners, and the orchestrator had to call all three plus aggregate() and pick the designated repetition itself - which meant naming a sample shape it cannot know, so the generic parameter was erased and re-asserted with a cast. assemble() keeps Sample bound inside the class, so the cast has nowhere to live, and picking the designated repetition happens in one place instead of two. Sample is now constrained to { coldMs: number }, which is what designatedRep has always required and what the erased generic hid. Pin N at 6 rather than 5. The two sides of a control pair alternate which runs first by repetition parity, so an odd N hands one side the first slot once more than the other, and the cold median then lands on a repetition from the majority slot - turning the ordering effect the alternation exists to cancel into a systematic bias on the headline ratio. Report compodoc's peak RSS as n/a when nothing was sampled. `ps` is POSIX-only and a run can finish inside one poll interval, and the initial 0 was reported as a measured peak on either path. oneShotMetrics now requires every repetition to carry a reading before it reports a mean. Also: read the compodoc RSS poll interval from its constant rather than back out of the params bag, and say in the "retained metrics missing" error that a series shorter than two saves reaches it too. --- scripts/bench/docgen-perf/aggregate.test.ts | 7 +++ scripts/bench/docgen-perf/aggregate.ts | 16 ++++-- scripts/bench/docgen-perf/config.ts | 11 ++++- scripts/bench/docgen-perf/engine.ts | 49 ++++++++++--------- scripts/bench/docgen-perf/engines/compodoc.ts | 40 ++++++++------- scripts/bench/docgen-perf/types.ts | 13 +++-- 6 files changed, 87 insertions(+), 49 deletions(-) diff --git a/scripts/bench/docgen-perf/aggregate.test.ts b/scripts/bench/docgen-perf/aggregate.test.ts index cf488729ba99..abec014abf2c 100644 --- a/scripts/bench/docgen-perf/aggregate.test.ts +++ b/scripts/bench/docgen-perf/aggregate.test.ts @@ -106,4 +106,11 @@ describe('oneShotMetrics', () => { it('enforces the pinned repetition count', () => { expect(() => oneShotMetrics(reps, 5)).toThrow('expected the pinned 5'); }); + + it('reports no peak rather than a low one when a repetition went unsampled', () => { + // `ps` is POSIX-only and a short run can finish inside one poll interval, so a repetition can + // come back with nothing sampled. Averaging that in would report a peak nobody measured. + const partial = [{ coldMs: 300, warmMs: 280, peakRssMb: 500 }, { coldMs: 100, warmMs: 120 }]; + expect(oneShotMetrics(partial, 2).peakTransientMb).toEqual({ status: 'n/a' }); + }); }); diff --git a/scripts/bench/docgen-perf/aggregate.ts b/scripts/bench/docgen-perf/aggregate.ts index 10762817ce50..a53920e05463 100644 --- a/scripts/bench/docgen-perf/aggregate.ts +++ b/scripts/bench/docgen-perf/aggregate.ts @@ -43,7 +43,9 @@ export function seriesMetrics(reps: SeriesResult[], expectedN: number): EngineMe designated.retainedGrowth === undefined || designated.retainedSlope === undefined ) { - throw new Error('retained metrics missing (child must run under --expose-gc)'); + throw new Error( + 'retained metrics missing: the child must run under --expose-gc, over at least two saves' + ); } return { @@ -60,7 +62,8 @@ export function seriesMetrics(reps: SeriesResult[], expectedN: number): EngineMe export interface OneShotRepetition { coldMs: number; warmMs: number; - peakRssMb: number; + /** Undefined when the engine's external sampler never read the child's memory. */ + peakRssMb?: number; coldMembers?: number; warmMembers?: number; /** Of the cold pass's members, how many carry a type the engine never resolved. */ @@ -75,12 +78,17 @@ export function oneShotMetrics(reps: OneShotRepetition[], expectedN: number): En assertRepetitionCount(reps, expectedN); const coldSamples = reps.map((r) => r.coldMs); const warmSamples = reps.map((r) => r.warmMs); - const peaks = reps.map((r) => r.peakRssMb); + // Every repetition must have been sampled, or the mean describes a different N than the one + // recorded. A partially-sampled series is reported as no measurement rather than as a low one. + const peaks = reps.map((r) => r.peakRssMb).filter((mb) => mb !== undefined); return { coldExtractionMs: { status: 'measured', samples: coldSamples, median: median(coldSamples) }, warmExtractionMs: { status: 'measured', samples: warmSamples, median: median(warmSamples) }, wholeProjectScanMs: { status: 'measured', samples: coldSamples, median: median(coldSamples) }, - peakTransientMb: { status: 'measured', samples: peaks, mean: mean(peaks) }, + peakTransientMb: + peaks.length === reps.length + ? { status: 'measured', samples: peaks, mean: mean(peaks) } + : NOT_APPLICABLE, retainedGrowthMb: NOT_APPLICABLE, retainedSlopeMbPerSave: NOT_APPLICABLE, }; diff --git a/scripts/bench/docgen-perf/config.ts b/scripts/bench/docgen-perf/config.ts index 461a2f705ac4..79dddbd0dd0d 100644 --- a/scripts/bench/docgen-perf/config.ts +++ b/scripts/bench/docgen-perf/config.ts @@ -6,8 +6,15 @@ * non-comparable. */ -/** Fresh-process spawns per cold/scan median. One value for all engines. */ -export const PINNED_N = 5; +/** + * Fresh-process spawns per cold/scan median. One value for all engines. + * + * Must stay even. The two sides of a control pair alternate which one runs first on odd and even + * repetitions, so an odd N gives one side the first slot once more than the other, and the cold + * figure - a median - then lands on a repetition from the majority slot. That turns the ordering + * effect the alternation exists to cancel into a systematic, directional bias on the headline ratio. + */ +export const PINNED_N = 6; /** Spawns for --quick smoke runs. Never comparable with PINNED_N results. */ export const QUICK_N = 2; diff --git a/scripts/bench/docgen-perf/engine.ts b/scripts/bench/docgen-perf/engine.ts index 55f9af8bace5..d0510715cc92 100644 --- a/scripts/bench/docgen-perf/engine.ts +++ b/scripts/bench/docgen-perf/engine.ts @@ -5,10 +5,10 @@ import * as path from 'node:path'; import type { SeriesResult } from '../docgen-shared/series.ts'; -import { seriesMetrics } from './aggregate.ts'; +import { designatedRep, seriesMetrics } from './aggregate.ts'; import type { SuiteProfile } from './config.ts'; import type { SeriesChildSpec } from './spawn.ts'; -import type { EngineId, EngineMetrics } from './types.ts'; +import type { EngineId, EngineMetrics, MemberCounts, ScenarioResult } from './types.ts'; export interface ScenarioSpec { name: string; @@ -23,8 +23,12 @@ export interface MeasureContext { runSeriesChild(spec: SeriesChildSpec, outDir: string, jsonPath: string): SeriesResult; } -/** `Sample` is whatever one repetition of this engine produces. */ -export abstract class BenchEngine { +/** + * `Sample` is whatever one repetition of this engine produces. Every engine's sample carries the + * cold-pass duration, because that is what picks the one repetition every single-run figure is read + * from. + */ +export abstract class BenchEngine { abstract readonly id: EngineId; /** Engines outside the default run only measure when named with `--engine`. */ @@ -50,23 +54,26 @@ export abstract class BenchEngine { return undefined; } - /** Members the cold pass documented. Undefined when the engine cannot report a count. */ - coldMembers(_sample: Sample): number | undefined { - return undefined; - } - - /** Members the timed re-extraction documented. */ - warmMembers(_sample: Sample): number | undefined { - return undefined; + /** + * What one repetition documented. An engine that cannot report a count leaves it out, which is + * what keeps a missing count from being read as a measured zero. The counts an engine does report + * are what establishes whether a ratio against it compared equal work. + */ + members(_sample: Sample): MemberCounts { + return {}; } /** - * Of the cold pass's members, how many the engine documented under a type name it never resolved. - * Two engines can agree on {@link coldMembers} and still have done entirely different work, so a - * member count alone does not establish like-for-like; this is what separates them. + * One scenario's complete result. `Sample` is still bound here, so the orchestrator never has to + * name - or cast to - the sample shape a particular engine happens to produce. */ - coldOpaqueTypes(_sample: Sample): number | undefined { - return undefined; + assemble(samples: Sample[], expectedN: number, scenario: ScenarioSpec): ScenarioResult { + // aggregate() first: it is what rejects a run that never reached the pinned N, and there is + // nothing for designatedRep to pick from until that has passed. + const metrics = this.aggregate(samples, expectedN); + // The counts are read from the same repetition as the warm and memory metrics, so every figure + // reported for a scenario describes one run. + return { params: scenario.params, metrics, ...this.members(designatedRep(samples)) }; } } @@ -117,11 +124,7 @@ export class SeriesChildEngine extends BenchEngine { return seriesMetrics(samples, expectedN); } - coldMembers(sample: SeriesResult): number | undefined { - return sample.coldMembers; - } - - warmMembers(sample: SeriesResult): number | undefined { - return sample.warmMembers; + members(sample: SeriesResult): MemberCounts { + return { coldMembers: sample.coldMembers, warmMembers: sample.warmMembers }; } } diff --git a/scripts/bench/docgen-perf/engines/compodoc.ts b/scripts/bench/docgen-perf/engines/compodoc.ts index 63d72a4cc1c7..247f1b03d2ef 100644 --- a/scripts/bench/docgen-perf/engines/compodoc.ts +++ b/scripts/bench/docgen-perf/engines/compodoc.ts @@ -9,7 +9,7 @@ import { type DocumentationCounts, countDocumentation } from './compodoc-doc.ts' import { type AngularScenarioConfig, RSS_POLL_INTERVAL_MS, type SuiteProfile } from '../config.ts'; import { BenchEngine, type MeasureContext, type ScenarioSpec } from '../engine.ts'; import { angularComponentSource, generateAngularProject } from '../generators/angular.ts'; -import type { EngineId, EngineMetrics } from '../types.ts'; +import type { EngineId, EngineMetrics, MemberCounts } from '../types.ts'; const require = createRequire(import.meta.url); @@ -34,7 +34,12 @@ function resolveCompodoc(): ResolvedCompodoc | undefined { interface CompodocRun extends DocumentationCounts { durMs: number; - peakRssMb: number; + /** + * Undefined when no poll ever read the child's RSS - `ps` is POSIX-only, and a run shorter than + * one interval is never sampled. Reporting the initial 0 instead would put a fabricated peak in + * the results, which is the one thing a floor may not become. + */ + peakRssMb: number | undefined; } function runCompodocOnce( @@ -49,7 +54,7 @@ function runCompodocOnce( return new Promise((resolve, reject) => { const start = Date.now(); const child = spawn(process.execPath, args, { cwd: projectDir }); - let peakRssKb = 0; + let peakRssKb: number | undefined; let output = ''; child.stdout.on('data', (chunk: Buffer) => (output += chunk.toString())); child.stderr.on('data', (chunk: Buffer) => (output += chunk.toString())); @@ -64,7 +69,7 @@ function runCompodocOnce( return; } const rssKb = Number(ps.stdout.trim()); - if (Number.isFinite(rssKb) && rssKb > peakRssKb) { + if (Number.isFinite(rssKb) && (peakRssKb === undefined || rssKb > peakRssKb)) { peakRssKb = rssKb; } }, pollIntervalMs); @@ -88,7 +93,7 @@ function runCompodocOnce( // Counted before the next run overwrites the file, which both runs share. const counts = countDocumentation(documentationJson); // The polled peak misses spikes shorter than the interval; the recorded value is a floor. - resolve({ durMs, peakRssMb: peakRssKb / 1024, ...counts }); + resolve({ durMs, peakRssMb: peakRssKb === undefined ? undefined : peakRssKb / 1024, ...counts }); }); }); } @@ -117,10 +122,13 @@ export async function runCompodocRepetition( fs.writeFileSync(project.componentPaths[0], angularComponentSource(0, scenario.props + 1)); const warm = await runCompodocOnce(cli, project.outDir, docsOutDir, pollIntervalMs); + const peaks = [cold.peakRssMb, warm.peakRssMb].filter((mb) => mb !== undefined); + return { coldMs: cold.durMs, warmMs: warm.durMs, - peakRssMb: Math.max(cold.peakRssMb, warm.peakRssMb), + // Nothing sampled means no peak, not a peak of zero. + peakRssMb: peaks.length ? Math.max(...peaks) : undefined, coldMembers: cold.members, // A second whole-project pass, so this counts the project, not the one file that changed. It is // not comparable with a series engine's warm count, which covers only the re-extracted member. @@ -162,7 +170,9 @@ export class CompodocEngine extends BenchEngine { this.#resolved.cli, { components: Number(scenario.params.components), props: Number(scenario.params.props) }, ctx.scenarioDir, - Number(scenario.params.rssPollIntervalMs) + // Read from the constant, not back out of `params`. `params` carries it so a stored run says + // how wide the sampling gap was; the run itself has no reason to round-trip through it. + RSS_POLL_INTERVAL_MS ); } @@ -170,15 +180,11 @@ export class CompodocEngine extends BenchEngine { return oneShotMetrics(samples, expectedN); } - coldMembers(sample: OneShotRepetition): number | undefined { - return sample.coldMembers; - } - - warmMembers(sample: OneShotRepetition): number | undefined { - return sample.warmMembers; - } - - coldOpaqueTypes(sample: OneShotRepetition): number | undefined { - return sample.coldOpaqueTypes; + members(sample: OneShotRepetition): MemberCounts { + return { + coldMembers: sample.coldMembers, + warmMembers: sample.warmMembers, + coldOpaqueTypes: sample.coldOpaqueTypes, + }; } } diff --git a/scripts/bench/docgen-perf/types.ts b/scripts/bench/docgen-perf/types.ts index 671b4865a749..e31d7ecdc368 100644 --- a/scripts/bench/docgen-perf/types.ts +++ b/scripts/bench/docgen-perf/types.ts @@ -42,9 +42,11 @@ export interface EngineMetrics { retainedSlopeMbPerSave: ValueMetric | NotApplicable; } -export interface ScenarioResult { - params: Record; - metrics: EngineMetrics; +/** + * What one repetition documented. Every field is optional because an engine that cannot report a + * count must leave it out; a zero would read as "documented nothing", which is a different claim. + */ +export interface MemberCounts { /** * Members the cold pass documented, when the engine reports it. Two engines over the same project * can differ by an order of magnitude here, and a timing ratio between them means nothing without @@ -61,6 +63,11 @@ export interface ScenarioResult { coldOpaqueTypes?: number; } +export interface ScenarioResult extends MemberCounts { + params: Record; + metrics: EngineMetrics; +} + export type EngineResult = | { status: 'measured'; scenarios: Record } | { status: 'skipped'; reason: string } From aa3e90dcc24d130533e820fa37aa4b946dee0d9b Mon Sep 17 00:00:00 2001 From: Valentin Palkovic Date: Thu, 30 Jul 2026 21:45:37 +0200 Subject: [PATCH 3/4] Bench: Kill a compodoc run that overruns its timeout The orchestrator settles on the child's close event, so a compodoc that hangs - which it can, on some inputs - stalled the whole suite with no way out. The run is now killed after ten minutes and reported as a timeout, so the engine fails on its own rather than taking the suite with it. Reported by CodeRabbit on #35636. --- scripts/bench/docgen-perf/config.ts | 7 ++++ scripts/bench/docgen-perf/engines/compodoc.ts | 34 +++++++++++++++++-- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/scripts/bench/docgen-perf/config.ts b/scripts/bench/docgen-perf/config.ts index 79dddbd0dd0d..3965ee78af73 100644 --- a/scripts/bench/docgen-perf/config.ts +++ b/scripts/bench/docgen-perf/config.ts @@ -22,6 +22,13 @@ export const QUICK_N = 2; /** Sampling interval for the compodoc child's externally-polled peak RSS. */ export const RSS_POLL_INTERVAL_MS = 100; +/** + * How long one compodoc run may take before it is killed. Compodoc can hang rather than exit on + * some inputs, and the orchestrator waits on the child's close event, so a hang without this would + * stall the whole suite indefinitely instead of failing the one engine. + */ +export const COMPODOC_TIMEOUT_MS = 10 * 60 * 1000; + export interface AngularScenarioConfig { components: number; props: number; diff --git a/scripts/bench/docgen-perf/engines/compodoc.ts b/scripts/bench/docgen-perf/engines/compodoc.ts index 247f1b03d2ef..e620637c24e5 100644 --- a/scripts/bench/docgen-perf/engines/compodoc.ts +++ b/scripts/bench/docgen-perf/engines/compodoc.ts @@ -6,7 +6,12 @@ import * as path from 'node:path'; import { outputTail } from '../../docgen-shared/child-output.ts'; import { type OneShotRepetition, oneShotMetrics } from '../aggregate.ts'; import { type DocumentationCounts, countDocumentation } from './compodoc-doc.ts'; -import { type AngularScenarioConfig, RSS_POLL_INTERVAL_MS, type SuiteProfile } from '../config.ts'; +import { + type AngularScenarioConfig, + COMPODOC_TIMEOUT_MS, + RSS_POLL_INTERVAL_MS, + type SuiteProfile, +} from '../config.ts'; import { BenchEngine, type MeasureContext, type ScenarioSpec } from '../engine.ts'; import { angularComponentSource, generateAngularProject } from '../generators/angular.ts'; import type { EngineId, EngineMetrics, MemberCounts } from '../types.ts'; @@ -56,9 +61,17 @@ function runCompodocOnce( const child = spawn(process.execPath, args, { cwd: projectDir }); let peakRssKb: number | undefined; let output = ''; + let timedOut = false; child.stdout.on('data', (chunk: Buffer) => (output += chunk.toString())); child.stderr.on('data', (chunk: Buffer) => (output += chunk.toString())); + // Killed rather than waited on: the close handler below is the only thing that settles this + // promise, so a compodoc that hangs would otherwise stall the whole suite with no way out. + const timeout = setTimeout(() => { + timedOut = true; + child.kill('SIGKILL'); + }, COMPODOC_TIMEOUT_MS); + const poll = setInterval(() => { if (child.pid === undefined) { return; @@ -74,13 +87,28 @@ function runCompodocOnce( } }, pollIntervalMs); - child.on('error', (err) => { + const stopWatching = () => { clearInterval(poll); + clearTimeout(timeout); + }; + + child.on('error', (err) => { + stopWatching(); reject(err); }); child.on('close', (status) => { - clearInterval(poll); + stopWatching(); const durMs = Date.now() - start; + // Before the status check: a killed child closes with a null status, which would otherwise be + // reported as an ordinary non-zero exit and hide why the run really ended. + if (timedOut) { + reject( + new Error( + `compodoc did not finish within ${COMPODOC_TIMEOUT_MS}ms:\n${outputTail(output, 8)}` + ) + ); + return; + } if (status !== 0) { reject(new Error(`compodoc exited with status ${status}:\n${outputTail(output, 8)}`)); return; From 3892cd04629213ad7b4091737a3897c2ac3510bf Mon Sep 17 00:00:00 2001 From: Valentin Palkovic Date: Fri, 31 Jul 2026 09:53:17 +0200 Subject: [PATCH 4/4] Bench: Make the Angular generator's tree readable The generator was a sequence of writes, so the only way to see what compodoc is pointed at was to run it and look at a directory. angularProjectFiles returns the project as path -> contents instead, and generateAngularProject writes that map. The bookkeeping of which paths are components stops being tracked separately from the writes that create them. The new test snapshots that map: the tree, the fake @angular/core that keeps it hermetic, the tsconfig include that decides the measured set, and a component at zero and four extra props. Generated output is byte-identical. --- .../docgen-perf/generators/angular.test.ts | 152 ++++++++++++++++++ .../bench/docgen-perf/generators/angular.ts | 63 +++++--- 2 files changed, 195 insertions(+), 20 deletions(-) create mode 100644 scripts/bench/docgen-perf/generators/angular.test.ts diff --git a/scripts/bench/docgen-perf/generators/angular.test.ts b/scripts/bench/docgen-perf/generators/angular.test.ts new file mode 100644 index 000000000000..28943dfe969a --- /dev/null +++ b/scripts/bench/docgen-perf/generators/angular.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, it } from 'vitest'; + +import { angularProjectFiles } from './angular.ts'; + +/** + * These snapshots are the generator's documentation. Compodoc reads a directory, not an API, so + * what matters is the shape of the tree and how a component reaches the decorators it is annotated + * with - neither of which is visible from the generator's source without running it. + * + * The project is asserted as data rather than generated to disk, so nothing here writes a file. + */ +const tree = (files: Record) => Object.keys(files).join('\n'); + +describe('angularProjectFiles', () => { + it('lays out a project compodoc can be pointed at', () => { + expect(tree(angularProjectFiles({ components: 3, props: 1 }).files)).toMatchInlineSnapshot(` + "node_modules/@angular/core/package.json + node_modules/@angular/core/index.d.ts + tsconfig.json + src/app/comp0.component.ts + src/app/comp1.component.ts + src/app/comp2.component.ts" + `); + }); + + it('ships a fake @angular/core so the tree needs no framework install', () => { + // Compodoc only has to resolve the decorators; it never executes them. `types` points at the + // .d.ts because that is all a static extractor reads. + const { files } = angularProjectFiles({ components: 1, props: 0 }); + expect(files['node_modules/@angular/core/package.json']).toMatchInlineSnapshot(` + "{ + "name": "@angular/core", + "version": "0.0.0-bench", + "types": "index.d.ts" + }" + `); + expect(files['node_modules/@angular/core/index.d.ts']).toMatchInlineSnapshot(` + "export declare function Component(metadata: { + selector?: string; + template?: string; + standalone?: boolean; + }): ClassDecorator; + export declare function Input(bindingPropertyName?: string): PropertyDecorator; + export declare function Output(bindingPropertyName?: string): PropertyDecorator; + export declare class EventEmitter { + emit(value?: T): void; + subscribe(next: (value: T) => void): { unsubscribe(): void }; + } + " + `); + }); + + it('points tsconfig at the components and nothing else', () => { + // `include` is what decides the measured set: compodoc documents what the program contains. + expect(angularProjectFiles({ components: 1, props: 0 }).files['tsconfig.json']) + .toMatchInlineSnapshot(` + "{ + "compilerOptions": { + "target": "ES2020", + "module": "ESNext", + "moduleResolution": "Bundler", + "strict": true, + "skipLibCheck": true, + "experimentalDecorators": true, + "emitDecoratorMetadata": false + }, + "include": [ + "src/**/*.ts" + ] + }" + `); + }); + + it('gives every component the same baseline members', () => { + // The four baseline inputs and one output are fixed, so `props` is the only size lever and two + // runs at the same props always document the same members. + expect(angularProjectFiles({ components: 2, props: 0 }).files['src/app/comp0.component.ts']) + .toMatchInlineSnapshot(` + "import { Component, EventEmitter, Input, Output } from '@angular/core'; + + /** + * Comp0 - generated bench component. + */ + @Component({ + selector: 'bench-comp0', + template: '
{{ label }}
', + }) + export class Comp0Component { + /** Primary label shown to the user. */ + @Input() label = ''; + /** Numeric size token. */ + @Input() size?: number; + /** Visual variant. */ + @Input() variant?: 'primary' | 'secondary' | 'tertiary'; + /** Disable interaction. */ + @Input() disabled = false; + /** Emits when the user acts on the component. */ + @Output() action = new EventEmitter<{ id: string; value: number }>(); + } + " + `); + }); + + it('grows a component by one input per extra prop, cycling the type', () => { + // The cycling matters: a union, a number and a string exercise different compodoc paths, and an + // all-string surface would not show whether it resolves a union. + expect(angularProjectFiles({ components: 1, props: 4 }).files['src/app/comp0.component.ts']) + .toMatchInlineSnapshot(` + "import { Component, EventEmitter, Input, Output } from '@angular/core'; + + /** + * Comp0 - generated bench component. + */ + @Component({ + selector: 'bench-comp0', + template: '
{{ label }}
', + }) + export class Comp0Component { + /** Primary label shown to the user. */ + @Input() label = ''; + /** Numeric size token. */ + @Input() size?: number; + /** Visual variant. */ + @Input() variant?: 'primary' | 'secondary' | 'tertiary'; + /** Disable interaction. */ + @Input() disabled = false; + /** Extra input 0 for component 0. */ + @Input() extra0?: 'a' | 'b' | 'c'; + /** Extra input 1 for component 0. */ + @Input() extra1?: number; + /** Extra input 2 for component 0. */ + @Input() extra2?: string; + /** Extra input 3 for component 0. */ + @Input() extra3?: 'a' | 'b' | 'c'; + /** Emits when the user acts on the component. */ + @Output() action = new EventEmitter<{ id: string; value: number }>(); + } + " + `); + }); + + it('names every component file after its index, in order', () => { + expect(angularProjectFiles({ components: 4, props: 0 }).componentPaths).toMatchInlineSnapshot(` + [ + "src/app/comp0.component.ts", + "src/app/comp1.component.ts", + "src/app/comp2.component.ts", + "src/app/comp3.component.ts", + ] + `); + }); +}); diff --git a/scripts/bench/docgen-perf/generators/angular.ts b/scripts/bench/docgen-perf/generators/angular.ts index d56351959acd..acf254e63240 100644 --- a/scripts/bench/docgen-perf/generators/angular.ts +++ b/scripts/bench/docgen-perf/generators/angular.ts @@ -41,14 +41,43 @@ export declare class EventEmitter { } `; -function emitFakeAngularCore(projectDir: string): void { - const dir = path.join(projectDir, 'node_modules', '@angular', 'core'); - fs.mkdirSync(dir, { recursive: true }); - fs.writeFileSync( - path.join(dir, 'package.json'), - JSON.stringify({ name: '@angular/core', version: '0.0.0-bench', types: 'index.d.ts' }, null, 2) - ); - fs.writeFileSync(path.join(dir, 'index.d.ts'), FAKE_ANGULAR_CORE); +/** The generated tree, as paths relative to the output directory. */ +export interface AngularProjectFiles { + /** Every file the generator emits, keyed by its path relative to the output directory. */ + files: Record; + /** The component files, in component order. */ + componentPaths: string[]; +} + +/** + * The project as data, before anything touches disk. Everything this generator emits is here, so + * the shape of the tree - and how a component reaches the fake framework types - can be read + * without generating one. + */ +export function angularProjectFiles( + options: Pick +): AngularProjectFiles { + const files: Record = { + // A fake `@angular/core` inside the project's own node_modules keeps the tree hermetic: compodoc + // resolves the decorators it needs without the real framework being installed. + 'node_modules/@angular/core/package.json': JSON.stringify( + { name: '@angular/core', version: '0.0.0-bench', types: 'index.d.ts' }, + null, + 2 + ), + 'node_modules/@angular/core/index.d.ts': FAKE_ANGULAR_CORE, + // The compodoc adapter passes `-p tsconfig.json` with the project dir as cwd. + 'tsconfig.json': JSON.stringify(TSCONFIG, null, 2), + }; + + const componentPaths: string[] = []; + for (let i = 0; i < options.components; i++) { + const componentPath = `src/app/comp${i}.component.ts`; + files[componentPath] = angularComponentSource(i, options.props); + componentPaths.push(componentPath); + } + + return { files, componentPaths }; } /** @@ -102,21 +131,15 @@ const TSCONFIG = { export function generateAngularProject(options: AngularGenerateOptions): GeneratedAngularProject { const outDir = path.resolve(options.outDir); fs.rmSync(outDir, { recursive: true, force: true }); - fs.mkdirSync(path.join(outDir, 'src', 'app'), { recursive: true }); - - emitFakeAngularCore(outDir); - // The compodoc adapter passes `-p tsconfig.json` with the project dir as cwd. - fs.writeFileSync(path.join(outDir, 'tsconfig.json'), JSON.stringify(TSCONFIG, null, 2)); - - const componentPaths: string[] = []; - for (let i = 0; i < options.components; i++) { - const componentPath = path.join(outDir, 'src', 'app', `comp${i}.component.ts`); - fs.writeFileSync(componentPath, angularComponentSource(i, options.props)); - componentPaths.push(componentPath); + const { files, componentPaths } = angularProjectFiles(options); + for (const [relativePath, contents] of Object.entries(files)) { + const filePath = path.join(outDir, relativePath); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, contents); } - return { outDir, componentPaths }; + return { outDir, componentPaths: componentPaths.map((p) => path.join(outDir, p)) }; } function parseOptions(argv: string[]): AngularGenerateOptions {