CLI: Introduce Agentic Setup workflow

.circleci/config.yml

@@ -32,7 +32,7 @@ jobs:
   generate-and-run-config:
     executor: 
       name: node/default
-      resource_class: small
+      resource_class: large
     steps:
       - node/install:
           install-yarn: true

.gitignore

@@ -80,4 +80,13 @@ CLAUDE.local.md
 .vscode/mcp.json
 .mcp.json
 .nx/polygraph
-.omc
+
+# Eval system
+scripts/eval/.cache
+scripts/eval/results
+
+# review-pr skill output
+.pr-review
+
+# Unknown
+.omc

.vscode/settings.json

@@ -23,6 +23,10 @@
   "[jsonc]": {
     "editor.defaultFormatter": "oxc.oxc-vscode"
   },
+  "[typescript]": {
+    "editor.defaultFormatter": "oxc.oxc-vscode",
+    "editor.formatOnSave": true
+  },
   "editor.codeActionsOnSave": {
     "source.fixAll.eslint": "explicit",
     "source.fixAll.oxc": "explicit"

AGENTS.md

@@ -9,10 +9,11 @@ This file is the canonical instruction source for coding agents. Files like `CLA
 Storybook is a large TypeScript monorepo. The git root is the repo root, the main code lives in `code/`, and build tooling lives in `scripts/`. The default branch is `next`.
 
 - **Base branch**: `next` (all PRs should target `next`, not `main`)
-- **Node.js**: `22.21.1` (see `.nvmrc`)
+- **Node.js**: `22.22.1` (see `.nvmrc`) — supports `.ts` natively via type stripping (no loader needed)
 - **Package Manager**: Yarn Berry
 - **Task orchestration**: NX plus the custom `yarn task` runner
 - **CI environment**: Linux and Windows
+- **TS execution**: Migrating from `jiti` to native `node` for running `.ts` files. New scripts should use `node ./path/file.ts` with explicit `.ts` import extensions (enabled by `allowImportingTsExtensions` in tsconfig). Legacy scripts still use `jiti` but should be migrated over time.
 
 ## Repository Structure
 
@@ -236,7 +237,7 @@ When writing tests:
 
 After changing files:
 
-1. Format with `cd code && oxfmt`
+1. Format with `yarn fmt:write` (run from the repo root)
 2. Lint with `yarn --cwd code lint:js:cmd <file-relative-to-code-folder> --fix` or `cd code && yarn lint:js:cmd <file-relative-to-code-folder>`
 3. Run relevant tests before submitting a PR
 

code/addons/a11y/src/preview.tsx

@@ -20,6 +20,7 @@ export const afterEach: AfterEach<any> = async ({
 }) => {
   const a11yParameter: A11yParameters | undefined = parameters.a11y;
   const a11yGlobals = globals.a11y;
+  // we do not run a11y checks as part of ghost stories runs
   const isGhostStories = !!globals.ghostStories;
 
   const shouldRunEnvironmentIndependent =

code/addons/vitest/src/vitest-plugin/agent-telemetry-reporter.test.ts

@@ -0,0 +1,206 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { AgentTelemetryReporter } from './agent-telemetry-reporter.ts';
+
+vi.mock('storybook/internal/telemetry', () => ({
+  telemetry: vi.fn(),
+  isExampleStoryId: vi.fn(
+    (id: string) =>
+      id.startsWith('example-button--') ||
+      id.startsWith('example-header--') ||
+      id.startsWith('example-page--')
+  ),
+}));
+
+const { telemetry } = await import('storybook/internal/telemetry');
+
+function createMockTestCase({
+  storyId,
+  status,
+  reports = [],
+  errors = [],
+}: {
+  storyId?: string;
+  status: 'passed' | 'failed' | 'pending';
+  reports?: Array<{ type: string; result?: Record<string, unknown> }>;
+  errors?: Array<{ message: string; stack?: string }>;
+}) {
+  return {
+    meta: () => ({ storyId, reports }),
+    result: () => ({
+      state: status,
+      errors: status === 'failed' ? errors : [],
+    }),
+  };
+}
+
+function createMockTestModules(testCounts: { passed: number; failed: number }) {
+  const tests: Array<{ result: () => { state: string } }> = [];
+  for (let i = 0; i < testCounts.passed; i++) {
+    tests.push({ result: () => ({ state: 'passed' }) });
+  }
+  for (let i = 0; i < testCounts.failed; i++) {
+    tests.push({ result: () => ({ state: 'failed' }) });
+  }
+  return [
+    {
+      children: {
+        allTests: function* (filter?: string) {
+          for (const t of tests) {
+            if (!filter || t.result().state === filter) {
+              yield t;
+            }
+          }
+        },
+      },
+      errors: () => [],
+    },
+  ];
+}
+
+describe('AgentTelemetryReporter', () => {
+  let reporter: AgentTelemetryReporter;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    reporter = new AgentTelemetryReporter({
+      configDir: '.storybook',
+      agent: { name: 'claude' },
+    });
+  });
+
+  describe('onTestCaseResult', () => {
+    it('should collect story test results', () => {
+      const testCase = createMockTestCase({
+        storyId: 'my-story--primary',
+        status: 'passed',
+      });
+      reporter.onTestCaseResult(testCase as any);
+    });
+
+    it('should skip tests without storyId', () => {
+      const testCase = createMockTestCase({
+        storyId: undefined,
+        status: 'passed',
+      });
+      reporter.onTestCaseResult(testCase as any);
+    });
+
+    it('should skip example story IDs', () => {
+      const testCase = createMockTestCase({
+        storyId: 'example-button--primary',
+        status: 'passed',
+      });
+      reporter.onTestCaseResult(testCase as any);
+    });
+  });
+
+  describe('onTestRunEnd', () => {
+    it('should send telemetry with analysis of collected results', async () => {
+      reporter.onInit({ config: { watch: false } } as any);
+
+      reporter.onTestCaseResult(createMockTestCase({ storyId: 's1', status: 'passed' }) as any);
+      reporter.onTestCaseResult(
+        createMockTestCase({
+          storyId: 's2',
+          status: 'failed',
+          errors: [{ message: 'Error: Module not found: foo' }],
+        }) as any
+      );
+      reporter.onTestCaseResult(
+        createMockTestCase({
+          storyId: 's3',
+          status: 'passed',
+          reports: [{ type: 'render-analysis', result: { emptyRender: true } }],
+        }) as any
+      );
+
+      await reporter.onTestRunEnd(createMockTestModules({ passed: 2, failed: 1 }) as any, []);
+
+      expect(telemetry).toHaveBeenCalledWith(
+        'ai-setup-self-healing-scoring',
+        expect.objectContaining({
+          agent: { name: 'claude' },
+          analysis: expect.objectContaining({
+            total: 3,
+            passed: 2,
+            passedButEmptyRender: 1,
+            successRate: 0.67,
+            successRateWithoutEmptyRender: 0.33,
+            uniqueErrorCount: 1,
+          }),
+          unhandledErrorCount: 0,
+          watch: false,
+        }),
+        { configDir: '.storybook', stripMetadata: true }
+      );
+    });
+
+    it('should filter out example stories from analysis', async () => {
+      reporter.onInit({ config: { watch: false } } as any);
+
+      reporter.onTestCaseResult(
+        createMockTestCase({ storyId: 'my-story--primary', status: 'passed' }) as any
+      );
+      reporter.onTestCaseResult(
+        createMockTestCase({ storyId: 'example-button--primary', status: 'passed' }) as any
+      );
+
+      await reporter.onTestRunEnd(createMockTestModules({ passed: 2, failed: 0 }) as any, []);
+
+      expect(telemetry).toHaveBeenCalledWith(
+        'ai-setup-self-healing-scoring',
+        expect.objectContaining({
+          analysis: expect.objectContaining({
+            total: 1,
+            passed: 1,
+          }),
+        }),
+        expect.anything()
+      );
+    });
+
+    it('should count unhandled errors', async () => {
+      reporter.onInit({ config: { watch: false } } as any);
+
+      await reporter.onTestRunEnd(
+        createMockTestModules({ passed: 0, failed: 0 }) as any,
+        [{ message: 'unhandled' }, { message: 'another' }] as any
+      );
+
+      expect(telemetry).toHaveBeenCalledWith(
+        'ai-setup-self-healing-scoring',
+        expect.objectContaining({
+          unhandledErrorCount: 2,
+        }),
+        expect.anything()
+      );
+    });
+
+    it('should reset collected results after each run', async () => {
+      reporter.onInit({ config: { watch: false } } as any);
+
+      reporter.onTestCaseResult(createMockTestCase({ storyId: 's1', status: 'passed' }) as any);
+      await reporter.onTestRunEnd(createMockTestModules({ passed: 1, failed: 0 }) as any, []);
+
+      reporter.onTestCaseResult(
+        createMockTestCase({
+          storyId: 's2',
+          status: 'failed',
+          errors: [{ message: 'err' }],
+        }) as any
+      );
+      await reporter.onTestRunEnd(createMockTestModules({ passed: 0, failed: 1 }) as any, []);
+
+      const secondCall = vi.mocked(telemetry).mock.calls[1];
+      expect(secondCall[1]).toEqual(
+        expect.objectContaining({
+          analysis: expect.objectContaining({
+            total: 1,
+            passed: 0,
+          }),
+        })
+      );
+    });
+  });
+});

code/addons/vitest/src/vitest-plugin/agent-telemetry-reporter.ts

@@ -0,0 +1,88 @@
+import type { SerializedError } from 'vitest';
+import type { TestCase, TestModule, Vitest } from 'vitest/node';
+import type { Reporter } from 'vitest/reporters';
+
+import type { TaskMeta } from '@vitest/runner';
+import type { Report } from 'storybook/preview-api';
+import { analyzeTestResults, toStoryTestResult } from 'storybook/internal/core-server';
+import type { StoryTestResult } from 'storybook/internal/core-server';
+import { isExampleStoryId, telemetry } from 'storybook/internal/telemetry';
+import type { AgentInfo } from 'storybook/internal/telemetry';
+
+interface AgentTelemetryReporterOptions {
+  configDir: string;
+  agent: AgentInfo;
+}
+
+export class AgentTelemetryReporter implements Reporter {
+  private ctx!: Vitest;
+
+  private testResults: StoryTestResult[] = [];
+
+  private startTime = Date.now();
+
+  private configDir: string;
+
+  private agent: AgentInfo;
+
+  constructor(options: AgentTelemetryReporterOptions) {
+    this.configDir = options.configDir;
+    this.agent = options.agent;
+  }
+
+  onInit(ctx: Vitest) {
+    this.ctx = ctx;
+  }
+
+  onTestRunStart() {
+    this.startTime = Date.now();
+  }
+
+  onTestCaseResult(testCase: TestCase) {
+    const { storyId, reports } = testCase.meta() as TaskMeta &
+      Partial<{ storyId: string; reports: Report[] }>;
+
+    if (!storyId || isExampleStoryId(storyId)) {
+      return;
+    }
+
+    const testResult = testCase.result();
+    const result = toStoryTestResult({
+      storyId,
+      statusRaw: testResult.state,
+      reports,
+      errors: testResult.errors,
+    });
+
+    if (result) {
+      this.testResults.push(result);
+    }
+  }
+
+  async onTestRunEnd(
+    testModules: readonly TestModule[],
+    unhandledErrors: readonly SerializedError[]
+  ) {
+    const analysis = analyzeTestResults(this.testResults);
+    const duration = Date.now() - this.startTime;
+
+    const testModulesErrors = testModules.flatMap((t) => t.errors());
+    const unhandledErrorCount = unhandledErrors.length + testModulesErrors.length;
+
+    // Fire and forget — same pattern as the existing test-run telemetry
+    telemetry(
+      'ai-setup-self-healing-scoring',
+      {
+        agent: this.agent,
+        analysis,
+        unhandledErrorCount,
+        duration,
+        watch: this.ctx.config.watch,
+      },
+      { configDir: this.configDir, stripMetadata: true }
+    );
+
+    // Reset for next run (watch mode)
+    this.testResults = [];
+  }
+}