Bench: Add the docgen perf engine framework and the compodoc engine

scripts/bench/docgen-perf/aggregate.test.ts

@@ -0,0 +1,116 @@
+import { describe, expect, it } from 'vitest';
+
+import type { MemorySample, SaveSample } from '../docgen-shared/samples.ts';
+import type { SeriesResult } from '../docgen-shared/series.ts';
+import { summarizeSeries } from '../docgen-shared/stats.ts';
+import { designatedRep, oneShotMetrics, seriesMetrics } from './aggregate.ts';
+
+function sample(save: number, durMs: number, heapUsedMb = 20, retainedHeapMb = 10): SaveSample {
+  return { save, durMs, rssMb: 100, heapUsedMb, retainedHeapMb };
+}
+
+/** Built through summarizeSeries, so the fixture cannot claim figures a real run would not produce. */
+function repOf(coldMs: number, samples: SaveSample[], baseline: MemorySample): SeriesResult {
+  return { coldMs, baseline, samples, ...summarizeSeries(samples, baseline) };
+}
+
+function rep(coldMs: number, durations: number[]): SeriesResult {
+  return repOf(
+    coldMs,
+    durations.map((d, i) => sample(i + 1, d)),
+    { rssMb: 90, heapUsedMb: 12, retainedHeapMb: 10 }
+  );
+}
+
+describe('designatedRep', () => {
+  it('picks the repetition whose cold sample is the median', () => {
+    const reps = [rep(900, [1]), rep(100, [2]), rep(300, [3]), rep(200, [4]), rep(250, [5])];
+    expect(designatedRep(reps).coldMs).toBe(250);
+  });
+
+  it('never picks the slow first repetition just because it came first', () => {
+    const reps = [rep(5000, [999]), rep(100, [10]), rep(110, [11]), rep(120, [12]), rep(130, [13])];
+    expect(designatedRep(reps).coldMs).not.toBe(5000);
+    expect(designatedRep(reps).coldMs).toBe(120);
+  });
+
+  it('takes the lower middle for an even count', () => {
+    const reps = [rep(10, [1]), rep(20, [2]), rep(30, [3]), rep(40, [4])];
+    expect(designatedRep(reps).coldMs).toBe(20);
+  });
+
+  it('does not mutate its input', () => {
+    const reps = [rep(30, [1]), rep(10, [2]), rep(20, [3])];
+    designatedRep(reps);
+    expect(reps.map((r) => r.coldMs)).toEqual([30, 10, 20]);
+  });
+});
+
+describe('seriesMetrics', () => {
+  const reps = [rep(300, [9, 9, 9]), rep(100, [1, 2, 3]), rep(200, [4, 5, 6])];
+
+  it('medians cold across every repetition', () => {
+    expect(seriesMetrics(reps, 3).coldExtractionMs).toMatchObject({
+      samples: [300, 100, 200],
+      median: 200,
+    });
+  });
+
+  it('reads warm from the designated repetition, not the first', () => {
+    expect(seriesMetrics(reps, 3).warmExtractionMs).toMatchObject({ samples: [4, 5, 6], median: 5 });
+  });
+
+  it('marks whole-project scan as not applicable rather than faking one', () => {
+    expect(seriesMetrics(reps, 3).wholeProjectScanMs).toEqual({ status: 'n/a' });
+  });
+
+  it('derives transient memory from the same repetition warm came from', () => {
+    expect(seriesMetrics(reps, 3).peakTransientMb).toMatchObject({ mean: 10 });
+  });
+
+  it('rejects a repetition count below the pinned N', () => {
+    expect(() => seriesMetrics(reps, 5)).toThrow('recorded 3 repetitions, expected the pinned 5');
+  });
+
+  it('rejects an empty set of repetitions', () => {
+    expect(() => seriesMetrics([], 5)).toThrow('no completed repetition recorded');
+  });
+
+  it('rejects a run with no retained samples', () => {
+    const noGc = repOf(100, [{ save: 1, durMs: 5, rssMb: 100, heapUsedMb: 20 }], {
+      rssMb: 90,
+      heapUsedMb: 12,
+    });
+    expect(() => seriesMetrics([noGc], 1)).toThrow('retained metrics missing');
+  });
+});
+
+describe('oneShotMetrics', () => {
+  const reps = [
+    { coldMs: 300, warmMs: 280, peakRssMb: 500 },
+    { coldMs: 100, warmMs: 120, peakRssMb: 400 },
+    { coldMs: 200, warmMs: 200, peakRssMb: 450 },
+  ];
+
+  it('reports cold and whole-project scan as the same measurement', () => {
+    const metrics = oneShotMetrics(reps, 3);
+    expect(metrics.wholeProjectScanMs).toEqual(metrics.coldExtractionMs);
+  });
+
+  it('has no retained series to report', () => {
+    const metrics = oneShotMetrics(reps, 3);
+    expect(metrics.retainedGrowthMb).toEqual({ status: 'n/a' });
+    expect(metrics.retainedSlopeMbPerSave).toEqual({ status: 'n/a' });
+  });
+
+  it('enforces the pinned repetition count', () => {
+    expect(() => oneShotMetrics(reps, 5)).toThrow('expected the pinned 5');
+  });
+
+  it('reports no peak rather than a low one when a repetition went unsampled', () => {
+    // `ps` is POSIX-only and a short run can finish inside one poll interval, so a repetition can
+    // come back with nothing sampled. Averaging that in would report a peak nobody measured.
+    const partial = [{ coldMs: 300, warmMs: 280, peakRssMb: 500 }, { coldMs: 100, warmMs: 120 }];
+    expect(oneShotMetrics(partial, 2).peakTransientMb).toEqual({ status: 'n/a' });
+  });
+});

scripts/bench/docgen-perf/aggregate.ts

@@ -0,0 +1,95 @@
+/**
+ * Turns N repetitions into the five floor metrics (see scripts/bench/PERF-METHODOLOGY.md). Warm
+ * latency and all three memory metrics read the same designated repetition's save series, so those
+ * four figures always describe one run; cold latency is a median across every repetition instead.
+ */
+import type { SeriesResult } from '../docgen-shared/series.ts';
+import { mean, median } from '../docgen-shared/stats.ts';
+import { type EngineMetrics, NOT_APPLICABLE } from './types.ts';
+
+/**
+ * Repetition 1 is systematically the slowest - it pays for a cold module graph and a cold OS page
+ * cache - so taking it would bias warm latency and all three memory metrics at once. Cold latency
+ * needs no such protection: it is already a median across every repetition.
+ */
+export function designatedRep<T extends { coldMs: number }>(reps: T[]): T {
+  const byCold = [...reps].sort((a, b) => a.coldMs - b.coldMs);
+  return byCold[Math.floor((byCold.length - 1) / 2)];
+}
+
+/**
+ * An engine that failed part-way through holds fewer samples than expectedN. Reporting those would
+ * put numbers taken at an unrecorded N into the results file, which the comparison method rests on
+ * not happening.
+ */
+function assertRepetitionCount(reps: unknown[], expectedN: number): void {
+  if (reps.length === 0) {
+    throw new Error('no completed repetition recorded');
+  }
+  if (reps.length !== expectedN) {
+    throw new Error(`recorded ${reps.length} repetitions, expected the pinned ${expectedN}`);
+  }
+}
+
+export function seriesMetrics(reps: SeriesResult[], expectedN: number): EngineMetrics {
+  assertRepetitionCount(reps, expectedN);
+  const designated = designatedRep(reps);
+  const coldSamples = reps.map((r) => r.coldMs);
+  const warmSamples = designated.samples.map((s) => s.durMs);
+  const { transients, avgTransient } = designated;
+
+  if (
+    avgTransient === undefined ||
+    designated.retainedGrowth === undefined ||
+    designated.retainedSlope === undefined
+  ) {
+    throw new Error(
+      'retained metrics missing: the child must run under --expose-gc, over at least two saves'
+    );
+  }
+
+  return {
+    coldExtractionMs: { status: 'measured', samples: coldSamples, median: median(coldSamples) },
+    warmExtractionMs: { status: 'measured', samples: warmSamples, median: median(warmSamples) },
+    // Per-component engines have no batch pass; recording one would be a faked equivalent.
+    wholeProjectScanMs: NOT_APPLICABLE,
+    peakTransientMb: { status: 'measured', samples: transients, mean: avgTransient },
+    retainedGrowthMb: { status: 'measured', value: designated.retainedGrowth },
+    retainedSlopeMbPerSave: { status: 'measured', value: designated.retainedSlope },
+  };
+}
+
+export interface OneShotRepetition {
+  coldMs: number;
+  warmMs: number;
+  /** Undefined when the engine's external sampler never read the child's memory. */
+  peakRssMb?: number;
+  coldMembers?: number;
+  warmMembers?: number;
+  /** Of the cold pass's members, how many carry a type the engine never resolved. */
+  coldOpaqueTypes?: number;
+}
+
+/**
+ * Metrics for a one-shot CLI engine: a fresh process per run, so cold extraction and the
+ * whole-project scan are the same full-project measurement, and there is no retained series to read.
+ */
+export function oneShotMetrics(reps: OneShotRepetition[], expectedN: number): EngineMetrics {
+  assertRepetitionCount(reps, expectedN);
+  const coldSamples = reps.map((r) => r.coldMs);
+  const warmSamples = reps.map((r) => r.warmMs);
+  // Every repetition must have been sampled, or the mean describes a different N than the one
+  // recorded. A partially-sampled series is reported as no measurement rather than as a low one.
+  const peaks = reps.map((r) => r.peakRssMb).filter((mb) => mb !== undefined);
+  return {
+    coldExtractionMs: { status: 'measured', samples: coldSamples, median: median(coldSamples) },
+    warmExtractionMs: { status: 'measured', samples: warmSamples, median: median(warmSamples) },
+    wholeProjectScanMs: { status: 'measured', samples: coldSamples, median: median(coldSamples) },
+    peakTransientMb:
+      peaks.length === reps.length
+        ? { status: 'measured', samples: peaks, mean: mean(peaks) }
+        : NOT_APPLICABLE,
+    retainedGrowthMb: NOT_APPLICABLE,
+    retainedSlopeMbPerSave: NOT_APPLICABLE,
+  };
+}

scripts/bench/docgen-perf/config.ts

@@ -0,0 +1,53 @@
+/**
+ * Fixed measurement parameters for the per-engine docgen performance suite.
+ *
+ * N is pinned here for every engine and recorded with the results; numbers taken at different N are
+ * not comparable. The --quick profile exists for smoke runs only and marks its results
+ * non-comparable.
+ */
+
+/**
+ * Fresh-process spawns per cold/scan median. One value for all engines.
+ *
+ * Must stay even. The two sides of a control pair alternate which one runs first on odd and even
+ * repetitions, so an odd N gives one side the first slot once more than the other, and the cold
+ * figure - a median - then lands on a repetition from the majority slot. That turns the ordering
+ * effect the alternation exists to cancel into a systematic, directional bias on the headline ratio.
+ */
+export const PINNED_N = 6;
+
+/** Spawns for --quick smoke runs. Never comparable with PINNED_N results. */
+export const QUICK_N = 2;
+
+/** Sampling interval for the compodoc child's externally-polled peak RSS. */
+export const RSS_POLL_INTERVAL_MS = 100;
+
+/**
+ * How long one compodoc run may take before it is killed. Compodoc can hang rather than exit on
+ * some inputs, and the orchestrator waits on the child's close event, so a hang without this would
+ * stall the whole suite indefinitely instead of failing the one engine.
+ */
+export const COMPODOC_TIMEOUT_MS = 10 * 60 * 1000;
+
+export interface AngularScenarioConfig {
+  components: number;
+  props: number;
+}
+
+export interface SuiteProfile {
+  n: number;
+  comparable: boolean;
+  angular: AngularScenarioConfig;
+}
+
+export const DEFAULT_PROFILE: SuiteProfile = {
+  n: PINNED_N,
+  comparable: true,
+  angular: { components: 100, props: 8 },
+};
+
+export const QUICK_PROFILE: SuiteProfile = {
+  n: QUICK_N,
+  comparable: false,
+  angular: { components: 10, props: 4 },
+};

scripts/bench/docgen-perf/engine.ts

@@ -0,0 +1,130 @@
+/**
+ * What the orchestrator knows about an engine. Engines differ only in how one repetition is
+ * produced; everything downstream (aggregation, member counts) follows from that choice.
+ */
+import * as path from 'node:path';
+
+import type { SeriesResult } from '../docgen-shared/series.ts';
+import { designatedRep, seriesMetrics } from './aggregate.ts';
+import type { SuiteProfile } from './config.ts';
+import type { SeriesChildSpec } from './spawn.ts';
+import type { EngineId, EngineMetrics, MemberCounts, ScenarioResult } from './types.ts';
+
+export interface ScenarioSpec {
+  name: string;
+  /** Recorded verbatim in the results so a stored run is self-describing. */
+  params: Record<string, number | string | boolean>;
+}
+
+/** Only what every engine needs. What one engine alone cares about stays inside that engine. */
+export interface MeasureContext {
+  /** Scratch directory for this engine/scenario: generated project and per-repetition JSON. */
+  scenarioDir: string;
+  runSeriesChild(spec: SeriesChildSpec, outDir: string, jsonPath: string): SeriesResult;
+}
+
+/**
+ * `Sample` is whatever one repetition of this engine produces. Every engine's sample carries the
+ * cold-pass duration, because that is what picks the one repetition every single-run figure is read
+ * from.
+ */
+export abstract class BenchEngine<Sample extends { coldMs: number } = { coldMs: number }> {
+  abstract readonly id: EngineId;
+
+  /** Engines outside the default run only measure when named with `--engine`. */
+  inDefaultRun = true;
+
+  abstract scenarios(profile: SuiteProfile): ScenarioSpec[];
+
+  abstract measure(ctx: MeasureContext, scenario: ScenarioSpec, rep: number): Promise<Sample>;
+
+  abstract aggregate(samples: Sample[], expectedN: number): EngineMetrics;
+
+  /**
+   * Anything that must resolve before the run starts. A returned string is the skip reason: a
+   * missing external tool is a partial install, not a regression, so it is reported as skipped
+   * rather than failed.
+   */
+  preflight(): string | undefined {
+    return undefined;
+  }
+
+  /** The resolved version of an externally installed engine, recorded with the results. */
+  version(): string | undefined {
+    return undefined;
+  }
+
+  /**
+   * What one repetition documented. An engine that cannot report a count leaves it out, which is
+   * what keeps a missing count from being read as a measured zero. The counts an engine does report
+   * are what establishes whether a ratio against it compared equal work.
+   */
+  members(_sample: Sample): MemberCounts {
+    return {};
+  }
+
+  /**
+   * One scenario's complete result. `Sample` is still bound here, so the orchestrator never has to
+   * name - or cast to - the sample shape a particular engine happens to produce.
+   */
+  assemble(samples: Sample[], expectedN: number, scenario: ScenarioSpec): ScenarioResult {
+    // aggregate() first: it is what rejects a run that never reached the pinned N, and there is
+    // nothing for designatedRep to pick from until that has passed.
+    const metrics = this.aggregate(samples, expectedN);
+    // The counts are read from the same repetition as the warm and memory metrics, so every figure
+    // reported for a scenario describes one run.
+    return { params: scenario.params, metrics, ...this.members(designatedRep(samples)) };
+  }
+}
+
+export interface SeriesChildConfig {
+  id: EngineId;
+  /** Child entry point, relative to this directory. */
+  child: string;
+  scenarios(profile: SuiteProfile): ScenarioSpec[];
+  args(scenario: ScenarioSpec): string[];
+  /** Only the reused docgen-memory harness runs under the jiti loader. */
+  jiti?: boolean;
+  inDefaultRun?: boolean;
+}
+
+/**
+ * An engine measured by spawning a series-harness child, one fresh process per repetition. All
+ * engines of this kind share the series harness, so aggregation and member counts are the same
+ * for each.
+ */
+export class SeriesChildEngine extends BenchEngine<SeriesResult> {
+  readonly id: EngineId;
+  readonly #config: SeriesChildConfig;
+
+  constructor(config: SeriesChildConfig) {
+    super();
+    this.id = config.id;
+    this.inDefaultRun = config.inDefaultRun ?? true;
+    this.#config = config;
+  }
+
+  scenarios(profile: SuiteProfile): ScenarioSpec[] {
+    return this.#config.scenarios(profile);
+  }
+
+  async measure(ctx: MeasureContext, scenario: ScenarioSpec, rep: number): Promise<SeriesResult> {
+    return ctx.runSeriesChild(
+      {
+        childPath: path.join(import.meta.dirname, this.#config.child),
+        args: this.#config.args(scenario),
+        jiti: this.#config.jiti,
+      },
+      path.join(ctx.scenarioDir, 'project'),
+      path.join(ctx.scenarioDir, `rep${rep}.json`)
+    );
+  }
+
+  aggregate(samples: SeriesResult[], expectedN: number): EngineMetrics {
+    return seriesMetrics(samples, expectedN);
+  }
+
+  members(sample: SeriesResult): MemberCounts {
+    return { coldMembers: sample.coldMembers, warmMembers: sample.warmMembers };
+  }
+}