feat(telemetry): measure which lint rules fire, cost, and fail to converge

packages/cli/src/commands/check.test.ts

@@ -58,6 +58,7 @@ function cleanLint(): ProjectLintResult {
     results: [
       {
         file: "index.html",
+        contentHash: "test",
         result: {
           ok: true,
           errorCount: 0,
@@ -82,6 +83,7 @@ function lintWith(
     results: [
       {
         file: "index.html",
+        contentHash: "test",
         result: {
           ok: severity !== "error",
           errorCount: severity === "error" ? 1 : 0,

packages/cli/src/commands/lint.ts

@@ -12,6 +12,8 @@ export const examples: Example[] = [
 import { formatLintFindings } from "../utils/lintFormat.js";
 import { lintProject } from "../utils/lintProject.js";
 import { resolveProject } from "../utils/project.js";
+import { trackLintRun } from "../telemetry/lintRun.js";
+import { getRunId } from "../telemetry/runId.js";
 import { withMeta } from "../utils/updateCheck.js";
 
 export default defineCommand({
@@ -45,7 +47,13 @@ export default defineCommand({
     // (publish/transcribe/upgrade/play/present) already use.
     try {
       const project = resolveProject(args.dir);
+      const startedAt = Date.now();
       const lintResult = await lintProject(project.dir);
+      trackLintRun(project.dir, lintResult, {
+        command: "lint",
+        durationMs: Date.now() - startedAt,
+        ...(getRunId() !== undefined ? { runId: getRunId() } : {}),
+      });
 
       if (args.json) {
         const allFindings = lintResult.results.flatMap((r) => r.result.findings);

packages/cli/src/commands/validate.test.ts

@@ -232,6 +232,7 @@ describe("extractCompositionErrorsFromLint", () => {
       results: [
         {
           file: "index.html",
+          contentHash: "test",
           result: {
             ok: findings.length === 0,
             errorCount: 0,
@@ -282,6 +283,7 @@ describe("extractCompositionErrorsFromLint", () => {
       results: [
         {
           file: "index.html",
+          contentHash: "test",
           result: {
             ok: false,
             errorCount: 1,
@@ -298,6 +300,7 @@ describe("extractCompositionErrorsFromLint", () => {
         },
         {
           file: "compositions/nested.html",
+          contentHash: "test",
           result: {
             ok: false,
             errorCount: 1,

packages/cli/src/telemetry/events.ts

@@ -819,3 +819,76 @@ export function trackCheckReport(props: {
     ...runIdField(props.runId),
   });
 }
+
+/**
+ * One lint pass over a project. `code_counts` is what makes "which rules
+ * actually fire" answerable; `rule_group_ms` and `slowest_rule` are what make
+ * "which rules are expensive" answerable. Only lint rule codes and timings are
+ * sent — never file paths, project names, or composition source.
+ */
+export function trackLintReport(props: {
+  /** The command that ran the lint: "lint" or "check". */
+  command: string;
+  durationMs: number;
+  filesScanned: number;
+  errorCount: number;
+  warningCount: number;
+  infoCount: number;
+  /** Finding count keyed by lint rule code. */
+  codeCounts: Record<string, number>;
+  /** Milliseconds spent per rule-source module, summed across files. */
+  ruleGroupMs: Record<string, number>;
+  /** Slowest single rule as `<group>#<index>`, across every file in the run. */
+  slowestRule: string;
+  slowestRuleMs: number;
+  /** How many rules this build ran, so a ruleset change is visible in the data. */
+  ruleCount: number;
+  /**
+   * Rule count per group. `slowest_rule` is positional, so a group that changed
+   * size between two builds has indices that no longer mean the same thing.
+   */
+  ruleGroupCounts: Record<string, number>;
+  runId?: string;
+}): void {
+  trackEvent("lint_report", {
+    command: props.command,
+    duration_ms: Math.round(props.durationMs),
+    files_scanned: props.filesScanned,
+    error_count: props.errorCount,
+    warning_count: props.warningCount,
+    info_count: props.infoCount,
+    codes: Object.keys(props.codeCounts).sort(),
+    code_counts: props.codeCounts,
+    rule_group_ms: props.ruleGroupMs,
+    slowest_rule: props.slowestRule,
+    slowest_rule_ms: Math.round(props.slowestRuleMs),
+    rule_count: props.ruleCount,
+    rule_group_counts: props.ruleGroupCounts,
+    ...runIdField(props.runId),
+  });
+}
+
+/**
+ * A finding that survived one or more edits to the file it was reported on.
+ *
+ * `cleared: false` with a high `edits` is the signal that matters most: a rule
+ * an agent kept trying and failing to satisfy. `cleared: true` gives the
+ * distribution to compare it against — how many edits a normal finding costs.
+ */
+export function trackLintRuleStreak(props: {
+  code: string;
+  severity: string;
+  edits: number;
+  cleared: boolean;
+  command: string;
+  runId?: string;
+}): void {
+  trackEvent("lint_rule_streak", {
+    code: props.code,
+    severity: props.severity,
+    edits: props.edits,
+    cleared: props.cleared,
+    command: props.command,
+    ...runIdField(props.runId),
+  });
+}

packages/cli/src/telemetry/lintRun.e2e.test.ts

@@ -0,0 +1,134 @@
+// End-to-end: a real project on disk -> lintProject -> the exact PostHog
+// payloads. Unit tests cover the streak arithmetic; this proves the wiring
+// and pins the event shape a dashboard will be built against.
+
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+const HOME = mkdtempSync(join(tmpdir(), "hf-lintrun-"));
+vi.mock("node:os", async (importOriginal) => {
+  const actual = await importOriginal<typeof import("node:os")>();
+  return { ...actual, homedir: () => HOME };
+});
+
+// Capture at the transport boundary so everything client.ts adds
+// (cli_version, invocation_id, ...) is visible in the assertions.
+const enqueued: Array<{ event: string; properties: Record<string, unknown> }> = [];
+vi.mock("./transport.js", () => ({
+  enqueue: (event: string, properties: Record<string, unknown>) =>
+    enqueued.push({ event, properties }),
+  flush: () => Promise.resolve(),
+}));
+// shouldTrack() consults these two. A dev build disables telemetry by default,
+// which would make this test assert on an empty queue and pass for the wrong
+// reason.
+vi.mock("./policy.js", async (importOriginal) => {
+  const actual = await importOriginal<typeof import("./policy.js")>();
+  return { ...actual, telemetryRuntimeOverride: () => null };
+});
+vi.mock("./config.js", async (importOriginal) => {
+  const actual = await importOriginal<typeof import("./config.js")>();
+  return { ...actual, readConfig: () => ({ ...actual.readConfig(), telemetryEnabled: true }) };
+});
+
+const { trackLintRun } = await import("./lintRun.js");
+const { lintProject } = await import("@hyperframes/lint");
+
+const COMPOSITION = `<html><body>
+  <div id="scene" data-composition-id="main" data-width="1920" data-height="1080"
+       data-start="0" data-duration="4">
+    <video id="clip" src="clip.mp4"></video>
+  </div>
+  <script src="gsap.min.js"></script>
+  <script>
+    window.__timelines = window.__timelines || {};
+    window.__timelines["main"] = gsap.timeline({ paused: true });
+  </script>
+</body></html>`;
+
+function makeProject(html: string): string {
+  const dir = mkdtempSync(join(tmpdir(), "hf-proj-"));
+  mkdirSync(join(dir, "compositions"), { recursive: true });
+  writeFileSync(join(dir, "index.html"), html, "utf-8");
+  return dir;
+}
+
+beforeEach(() => {
+  enqueued.length = 0;
+  rmSync(join(HOME, ".hyperframes"), { recursive: true, force: true });
+});
+
+describe("trackLintRun end to end", () => {
+  it("emits one lint_report carrying codes, timings, and the ruleset fingerprint", async () => {
+    const dir = makeProject(COMPOSITION);
+    const result = await lintProject(dir);
+    trackLintRun(dir, result, { command: "lint", durationMs: 12 });
+
+    const reports = enqueued.filter((e) => e.event === "lint_report");
+    expect(reports).toHaveLength(1);
+    const props = reports[0]!.properties;
+
+    expect(props["command"]).toBe("lint");
+    expect(props["files_scanned"]).toBe(1);
+    expect(props["duration_ms"]).toBe(12);
+    // The real linter found real problems in this composition.
+    expect((props["codes"] as string[]).length).toBeGreaterThan(0);
+    expect(props["error_count"]).toBeGreaterThan(0);
+    // Timings are attributed per rule group and a slowest rule is identified.
+    expect(Object.keys(props["rule_group_ms"] as object)).toContain("gsap");
+    expect(props["slowest_rule"]).toMatch(/^[a-z]+#\d+$/);
+    // Version and ruleset fingerprint ride along.
+    expect(props["cli_version"]).toBeTruthy();
+    expect(props["rule_count"]).toBeGreaterThan(0);
+    // Per-group sizes make the positional `slowest_rule` index comparable
+    // across builds: a group that changed size renumbered its rules.
+    const groupCounts = props["rule_group_counts"] as Record<string, number>;
+    expect(groupCounts["gsap"]).toBeGreaterThan(0);
+    expect(Object.values(groupCounts).reduce((a, b) => a + b, 0)).toBe(props["rule_count"]);
+    // code_counts sums to the number of findings.
+    const counts = Object.values(props["code_counts"] as Record<string, number>);
+    expect(counts.reduce((a, b) => a + b, 0)).toBe(
+      result.results.flatMap((r) => r.result.findings).length,
+    );
+
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it("emits lint_rule_streak with cleared:true once an edit removes the finding", async () => {
+    const dir = makeProject(COMPOSITION);
+
+    const first = await lintProject(dir);
+    trackLintRun(dir, first, { command: "lint", durationMs: 1 });
+    const codes = first.results[0]!.result.findings.map((f) => f.code);
+    expect(codes).toContain("media_missing_data_start");
+
+    // Fix exactly that finding and re-lint.
+    writeFileSync(
+      join(dir, "index.html"),
+      COMPOSITION.replace('<video id="clip"', '<video id="clip" data-start="0" data-duration="4"'),
+      "utf-8",
+    );
+    enqueued.length = 0;
+    const second = await lintProject(dir);
+    trackLintRun(dir, second, { command: "lint", durationMs: 1 });
+
+    const streaks = enqueued.filter((e) => e.event === "lint_rule_streak");
+    const cleared = streaks.find((e) => e.properties["code"] === "media_missing_data_start");
+    expect(cleared?.properties).toMatchObject({
+      code: "media_missing_data_start",
+      cleared: true,
+      edits: 1,
+      command: "lint",
+    });
+
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it("never throws when the lint result is malformed", () => {
+    expect(() =>
+      trackLintRun("/nope", { results: [] } as never, { command: "lint", durationMs: 0 }),
+    ).not.toThrow();
+  });
+});

packages/cli/src/telemetry/lintRun.ts

@@ -0,0 +1,83 @@
+// One place that turns a ProjectLintResult into telemetry, so `lint` and
+// `check` report identically instead of drifting apart.
+
+import { LINT_RULE_COUNT, LINT_RULE_GROUP_COUNTS, type ProjectLintResult } from "@hyperframes/lint";
+import { trackLintReport, trackLintRuleStreak } from "./events.js";
+import { recordLintRun } from "./lintStreaks.js";
+
+/**
+ * Report one lint pass: aggregate counts and timings, plus any finding that
+ * survived an edit to its file.
+ *
+ * Never throws — a telemetry failure must not fail the command that lints.
+ */
+export function trackLintRun(
+  projectDir: string,
+  lintResult: ProjectLintResult,
+  options: { command: string; durationMs: number; runId?: string },
+): void {
+  try {
+    const runIdField = options.runId !== undefined ? { runId: options.runId } : {};
+
+    trackLintReport({
+      command: options.command,
+      durationMs: options.durationMs,
+      filesScanned: lintResult.results.length,
+      errorCount: lintResult.totalErrors,
+      warningCount: lintResult.totalWarnings,
+      infoCount: lintResult.totalInfos,
+      ruleCount: LINT_RULE_COUNT,
+      ruleGroupCounts: LINT_RULE_GROUP_COUNTS,
+      ...summarize(lintResult),
+      ...runIdField,
+    });
+
+    const streaks = recordLintRun(
+      projectDir,
+      lintResult.results.map(({ file, contentHash, result }) => ({
+        file,
+        contentHash,
+        findings: result.findings,
+      })),
+    );
+    for (const streak of streaks) {
+      trackLintRuleStreak({ ...streak, command: options.command, ...runIdField });
+    }
+  } catch {
+    // Telemetry is best-effort. A malformed result, an unwritable home
+    // directory, or a transport failure must never turn a green lint red.
+  }
+}
+
+/** Roll every file's findings and timings up into one run-level summary. */
+function summarize(lintResult: ProjectLintResult): {
+  codeCounts: Record<string, number>;
+  ruleGroupMs: Record<string, number>;
+  slowestRule: string;
+  slowestRuleMs: number;
+} {
+  const codeCounts: Record<string, number> = {};
+  const ruleGroupMs: Record<string, number> = {};
+  let slowestRule = "";
+  let slowestRuleMs = 0;
+
+  for (const { result } of lintResult.results) {
+    for (const finding of result.findings) {
+      codeCounts[finding.code] = (codeCounts[finding.code] ?? 0) + 1;
+    }
+    const timings = result.timings;
+    if (!timings) continue;
+    for (const [group, ms] of Object.entries(timings.groupMs)) {
+      ruleGroupMs[group] = (ruleGroupMs[group] ?? 0) + ms;
+    }
+    if (timings.slowestRuleMs > slowestRuleMs) {
+      slowestRuleMs = timings.slowestRuleMs;
+      slowestRule = timings.slowestRule;
+    }
+  }
+
+  for (const group of Object.keys(ruleGroupMs)) {
+    ruleGroupMs[group] = Math.round(ruleGroupMs[group]!);
+  }
+  return { codeCounts, ruleGroupMs, slowestRule, slowestRuleMs };
+}