kasperpeulen · Oct 8, 2026 · Oct 6, 2026 · Oct 6, 2026 · Oct 6, 2026 · Oct 6, 2026
diff --git a/agent-eval/README.md b/agent-eval/README.md
@@ -37,7 +37,7 @@ yarn eval:dry
 
 ### Run Experiments
 
-Run all configured experiments:
+Run the default experiments:
 
 ```bash
 yarn eval
@@ -65,10 +65,10 @@ EVAL_ALL=1 yarn eval
 EVAL_ONLY=803-edit-component yarn eval
 ```
 
-A full `EVAL_ALL=1` run (13 workflow evals × 4 experiments + 4
-lifecycle evals × 2 plugin experiments) costs roughly **$30–45** in agent
-tokens at current per-run averages ($0.30–0.80 per workflow eval, $1–2 per
-lifecycle eval). The budget guardrail is **$75 per full run** — check the
+A full `EVAL_ALL=1` run (14 workflow evals × 4 experiments + 4
+lifecycle evals × 2 plugin experiments) costs roughly **$6** in agent
+tokens at the averages measured in October 2026 (under $0.01 per eval on Haiku 5.5 low, and about
+$0.05–0.17 per eval on GPT-6.1-Sol low and the medium MCP experiments). The budget guardrail is **$75 per full run** — check the
 usage metadata in the results playground before growing the eval set past it
 (see [storybookjs/mcp#324](https://github.com/storybookjs/mcp/pull/324)).
 
@@ -129,9 +129,16 @@ via `ANTHROPIC_API_KEY`; Codex experiments use the direct Codex API via
 `OPENAI_API_KEY`):
 
 - `cc-mcp-opus-5.5-medium`: Claude Code (Opus 5.5 at medium effort) with project-local Storybook MCP config in `.mcp.json`.
-- `cc-plugin-opus-5.5-medium`: Claude Code (Opus 5.5 at medium effort) with Storybook plugin skills copied to `.claude/skills`.
+- `cc-plugin-haiku-5.5-low`: Claude Code (Haiku 5.5 at low effort) with Storybook plugin skills copied to `.claude/skills`.
 - `codex-mcp-gpt-6.1-sol-medium`: Codex (gpt-6.1-sol at medium reasoning effort) with project-local Storybook MCP config in `.codex/config.toml` and the Storybook MCP server instructions in `AGENTS.md` (none with `EVAL_STORYBOOK_LATEST=1`).
-- `codex-plugin-gpt-6.1-sol-medium`: Codex (gpt-6.1-sol at medium reasoning effort) with Storybook plugin skills copied to `.agents/skills`.
+- `codex-plugin-gpt-6.1-sol-low`: Codex (gpt-6.1-sol at low reasoning effort) with Storybook plugin skills copied to `.agents/skills`.
+
+These four are the default set: a bare `yarn eval`, which is what CI runs, runs them and they have to stay green. Four more plugin experiments run only when you name them by their exact file name (a glob does not select them), for example `yarn exec agent-eval codex-plugin-gpt-6-luna-low`:
+
+- `cc-plugin-sonnet-5.5-low`: Claude Code (Sonnet 5.5 at low effort).
+- `cc-plugin-opus-5.5-medium`: Claude Code (Opus 5.5 at medium effort).
+- `codex-plugin-gpt-6.1-sol-medium`: Codex (gpt-6.1-sol at medium reasoning effort).
+- `codex-plugin-gpt-6-luna-low`: Codex (gpt-6-luna at low reasoning effort). Its Codex system prompt says not to run tests unless asked, so evals do not require it to run the story tests itself.
 
 ## Running evals in CI
 

diff --git a/agent-eval/evals/801-create-accessible-component/EVAL.ts b/agent-eval/evals/801-create-accessible-component/EVAL.ts
@@ -9,13 +9,17 @@ import {
   expectStoryTestsRanAndPassed,
   expectWorkflowCalls,
   getEvalContext,
+  modelRunsTestsOnlyWhenAsked,
 } from '#test-utils';
 import { transcript } from '@vercel/agent-eval/eval';
 import { describe, expect, test } from 'vitest';
 
 describe('creating an accessible ToggleSwitch', () => {
   test('runs story tests after the change and finishes with them passing', async () => {
-    await expectStoryTestsRanAndPassed({ covering: ['toggleswitch'] });
+    await expectStoryTestsRanAndPassed({
+      requireAgentRun: !modelRunsTestsOnlyWhenAsked(),
+      covering: ['toggleswitch'],
+    });
   });
 
   test('uses Storybook story instructions and publishes a display review', () => {

diff --git a/agent-eval/evals/802-create-component/EVAL.ts b/agent-eval/evals/802-create-component/EVAL.ts
@@ -8,12 +8,16 @@ import {
   expectStoryTestsRanAndPassed,
   expectWorkflowCalls,
   getEvalContext,
+  modelRunsTestsOnlyWhenAsked,
 } from '#test-utils';
 import { describe, test } from 'vitest';
 
 describe('creating a ProfileCard component', () => {
   test('runs story tests after the change and finishes with them passing', async () => {
-    await expectStoryTestsRanAndPassed({ covering: ['profilecard'] });
+    await expectStoryTestsRanAndPassed({
+      requireAgentRun: !modelRunsTestsOnlyWhenAsked(),
+      covering: ['profilecard'],
+    });
   });
 
   test('uses Storybook story instructions and publishes a display review', () => {

diff --git a/agent-eval/evals/803-edit-component/EVAL.ts b/agent-eval/evals/803-edit-component/EVAL.ts
@@ -8,12 +8,16 @@ import {
   expectStoryTestsRanAndPassed,
   expectWorkflowCalls,
   getEvalContext,
+  modelRunsTestsOnlyWhenAsked,
 } from '#test-utils';
 import { describe, test } from 'vitest';
 
 describe('editing ReviewCard to add date and optional onReport', () => {
   test('runs story tests after the change and finishes with them passing', async () => {
-    await expectStoryTestsRanAndPassed({ covering: ['reviewcard'] });
+    await expectStoryTestsRanAndPassed({
+      requireAgentRun: !modelRunsTestsOnlyWhenAsked(),
+      covering: ['reviewcard'],
+    });
   });
 
   test('uses Storybook story instructions and publishes a display review', () => {

diff --git a/agent-eval/evals/804-write-story-for-existing-component/EVAL.ts b/agent-eval/evals/804-write-story-for-existing-component/EVAL.ts
@@ -9,11 +9,15 @@ import {
   expectStoryIdsInDisplayReview,
   expectStoryTestsRanAndPassed,
   expectWorkflowCalls,
+  modelRunsTestsOnlyWhenAsked,
 } from '#test-utils';
 
 describe('writing stories for an existing AlertBanner', () => {
   test('runs story tests after the change and finishes with them passing', async () => {
-    await expectStoryTestsRanAndPassed({ covering: ['alertbanner'] });
+    await expectStoryTestsRanAndPassed({
+      requireAgentRun: !modelRunsTestsOnlyWhenAsked(),
+      covering: ['alertbanner'],
+    });
   });
 
   test('uses Storybook story instructions and publishes a display review', () => {

diff --git a/agent-eval/evals/808-shared-infra-fallback/EVAL.ts b/agent-eval/evals/808-shared-infra-fallback/EVAL.ts
@@ -9,6 +9,7 @@ import {
   getEvalContext,
   getWorkflowCalls,
   getWorkflowToolResults,
+  modelRunsTestsOnlyWhenAsked,
 } from '#test-utils';
 import { readFileSync } from 'node:fs';
 import { describe, expect, test } from 'vitest';
@@ -36,7 +37,10 @@ describe('changing a shared accent token and surfacing consumer stories', () =>
   test.skipIf(codexMcpGap)(
     'runs story tests after the change and finishes with them passing',
     async () => {
-      await expectStoryTestsRanAndPassed({ covering: ['badge', 'statuspill'] });
+      await expectStoryTestsRanAndPassed({
+        requireAgentRun: !modelRunsTestsOnlyWhenAsked(),
+        covering: ['badge', 'statuspill'],
+      });
     }
   );
 

diff --git a/agent-eval/evals/812-first-story-empty-project/EVAL.ts b/agent-eval/evals/812-first-story-empty-project/EVAL.ts
@@ -10,11 +10,15 @@ import {
   expectStoryIdsInDisplayReview,
   expectStoryTestsRanAndPassed,
   expectWorkflowCalls,
+  modelRunsTestsOnlyWhenAsked,
 } from '#test-utils';
 
 describe('writing the first Button stories in an empty Storybook', () => {
   test('runs story tests after the change and finishes with them passing', async () => {
-    await expectStoryTestsRanAndPassed({ covering: ['button'] });
+    await expectStoryTestsRanAndPassed({
+      requireAgentRun: !modelRunsTestsOnlyWhenAsked(),
+      covering: ['button'],
+    });
   });
 
   test('uses Storybook story instructions and publishes a display review', () => {

diff --git a/agent-eval/evals/813-monorepo-leaf-create-component/EVAL.ts b/agent-eval/evals/813-monorepo-leaf-create-component/EVAL.ts
@@ -10,6 +10,7 @@ import {
   expectStoryIdsInDisplayReview,
   expectStoryTestsRanAndPassed,
   expectWorkflowCalls,
+  modelRunsTestsOnlyWhenAsked,
 } from '#test-utils';
 
 describe('creating a Callout in a monorepo UI package', () => {
@@ -26,7 +27,11 @@ describe('creating a Callout in a monorepo UI package', () => {
   });
 
   test('runs story tests after the change and finishes with them passing', async () => {
-    await expectStoryTestsRanAndPassed({ covering: ['callout'], cwd: 'packages/ui' });
+    await expectStoryTestsRanAndPassed({
+      requireAgentRun: !modelRunsTestsOnlyWhenAsked(),
+      covering: ['callout'],
+      cwd: 'packages/ui',
+    });
   });
 
   test('uses Storybook story instructions and publishes a display review', () => {

diff --git a/agent-eval/evals/814-write-stories-for-fetching-component/EVAL.ts b/agent-eval/evals/814-write-stories-for-fetching-component/EVAL.ts
@@ -10,13 +10,17 @@ import {
   expectStoryTestsRanAndPassed,
   expectWorkflowCalls,
   getEvalContext,
+  modelRunsTestsOnlyWhenAsked,
 } from '#test-utils';
 
 const STORIES_PATH = 'stories/OrderHistory.stories.tsx';
 
 describe('writing stories for an existing OrderHistory that fetches its data', () => {
   test('runs story tests after the change and finishes with them passing', async () => {
-    await expectStoryTestsRanAndPassed({ covering: ['orderhistory'] });
+    await expectStoryTestsRanAndPassed({
+      requireAgentRun: !modelRunsTestsOnlyWhenAsked(),
+      covering: ['orderhistory'],
+    });
   });
 
   test('mocks the request instead of changing the component', () => {

diff --git a/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/.storybook/main.ts b/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/.storybook/main.ts
@@ -0,0 +1,8 @@
+import type { StorybookConfig } from '@storybook/react-vite';
+
+const config: StorybookConfig = {
+  stories: ['../@(stories|src)/**/*.stories.@(js|jsx|mjs|ts|tsx)'],
+  addons: ['@storybook/addon-a11y', '@storybook/addon-vitest', '@storybook/addon-docs'],
+  framework: '@storybook/react-vite',
+};
+export default config;
diff --git a/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/.storybook/preview.tsx b/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/.storybook/preview.tsx
@@ -0,0 +1,7 @@
+import type { Preview } from '@storybook/react-vite';
+
+const preview: Preview = {
+  parameters: {},
+};
+
+export default preview;
diff --git a/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/EVAL.ts b/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/EVAL.ts
@@ -0,0 +1,87 @@
+import { existsSync, readFileSync, readdirSync } from 'node:fs';
+import { describe, expect, test } from 'vitest';
+import {
+  expectDisplayReviewForVisualChange,
+  expectReviewOpenedInBrowser,
+  expectSkillInvoked,
+  expectStoryDiscoveryBeforeReview,
+  expectStoryIdsInDisplayReview,
+  expectStoryTestsRanAndPassed,
+  expectWorkflowCalls,
+  getEvalContext,
+  modelRunsTestsOnlyWhenAsked,
+} from '#test-utils';
+
+const STORIES_PATH = 'src/components/OrderHistory.stories.tsx';
+
+describe('writing stories for a fetching OrderHistory in a project without MSW', () => {
+  test('runs story tests after the change and finishes with them passing', async () => {
+    await expectStoryTestsRanAndPassed({
+      requireAgentRun: !modelRunsTestsOnlyWhenAsked(),
+      covering: ['orderhistory'],
+    });
+  });
+
+  test('installs MSW and its Storybook addon', () => {
+    const { dependencies = {}, devDependencies = {} } = JSON.parse(
+      readFileSync('package.json', 'utf8')
+    );
+    const installed = { ...dependencies, ...devDependencies };
+    expect(installed, 'Expected msw to be installed').toHaveProperty('msw');
+    expect(installed, 'Expected msw-storybook-addon to be installed').toHaveProperty(
+      'msw-storybook-addon'
+    );
+    expect(
+      existsSync('public/mockServiceWorker.js'),
+      'Expected the worker script generated by `msw init`'
+    ).toBe(true);
+    const preview = readdirSync('.storybook').find((file) => file.startsWith('preview.'));
+    expect(
+      readFileSync(`.storybook/${preview}`, 'utf8'),
+      'Expected the preview file to set up msw-storybook-addon'
+    ).toMatch(/msw-storybook-addon/);
+  });
+
+  test('mocks the request with MSW instead of a module mock or a stubbed fetch', () => {
+    const component = readFileSync('src/components/OrderHistory.tsx', 'utf8');
+    expect(component, 'Expected OrderHistory to keep taking no props').toMatch(
+      /export default function OrderHistory\s*\(\s*\)/
+    );
+    expect(component, 'Expected OrderHistory to keep fetching /api/orders').toMatch(
+      /fetch\(\s*['"`]\/api\/orders/
+    );
+
+    expect(existsSync(STORIES_PATH), `Expected the stories in ${STORIES_PATH}`).toBe(true);
+    const stories = readFileSync(STORIES_PATH, 'utf8');
+    expect(stories, 'Expected handlers registered with msw.use').toMatch(/\bmsw\.use\(/);
+    expect(stories, 'Expected the handlers to mock /api/orders').toMatch(/\/api\/orders/);
+    expect(stories, 'Expected no module mock or stubbed fetch').not.toMatch(
+      /\bsb\.mock\(|\bvi\.mock\(|\bmocked\(|\bspyOn\(|\b(globalThis|window)\.fetch\s*=/
+    );
+    expect(
+      stories.match(/^export const \w+/gm)?.length ?? 0,
+      'Expected loaded, empty, and error stories'
+    ).toBeGreaterThanOrEqual(3);
+  });
+
+  test('uses Storybook story instructions and publishes a display review', () => {
+    expectWorkflowCalls(['get-storybook-story-instructions', 'review-create']);
+    expectDisplayReviewForVisualChange();
+  });
+
+  test('opens the review in the in-app browser', () => {
+    expectReviewOpenedInBrowser();
+  });
+
+  test('the review covers the new OrderHistory stories', () => {
+    expectStoryIdsInDisplayReview(['orderhistory']);
+  });
+
+  test('discovers stories through the workflow tools before publishing the review', () => {
+    expectStoryDiscoveryBeforeReview();
+  });
+
+  test.skipIf(getEvalContext().integration === 'mcp')('invokes the stories skill', () => {
+    expectSkillInvoked('stories');
+  });
+});
diff --git a/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/PROMPT.md b/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/PROMPT.md
@@ -0,0 +1 @@
+Add stories in `src/components/OrderHistory.stories.tsx` for the existing `OrderHistory` component, covering the loaded, empty, and error states.
diff --git a/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/package.json b/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/package.json
@@ -0,0 +1,23 @@
+{
+  "name": "815-write-stories-for-fetching-component-without-msw",
+  "type": "module",
+  "scripts": {
+    "build-storybook": "storybook build",
+    "postinstall": "playwright install chromium && node ./scripts/start-storybook-mcp.mjs",
+    "storybook": "storybook dev",
+    "test:stories": "vitest run --config ./vitest.storybook.config.ts --project storybook"
+  },
+  "devDependencies": {
+    "@storybook/addon-a11y": "workspace:*",
+    "@storybook/addon-docs": "workspace:*",
+    "@storybook/addon-vitest": "workspace:*",
+    "@storybook/react": "workspace:*",
+    "@storybook/react-vite": "workspace:*",
+    "@vitest/browser-playwright": "4.0.6",
+    "playwright": "1.56.1",
+    "storybook": "workspace:*"
+  },
+  "evals": {
+    "template": "vite-app"
+  }
+}
diff --git a/...vals/815-write-stories-for-fetching-component-without-msw/src/components/OrderHistory.tsx b/...vals/815-write-stories-for-fetching-component-without-msw/src/components/OrderHistory.tsx
@@ -0,0 +1,40 @@
+import { useEffect, useState } from 'react';
+
+type Order = {
+  id: string;
+  placedAt: string;
+  total: number;
+};
+
+type State = { status: 'loading' } | { status: 'error' } | { status: 'loaded'; orders: Order[] };
+
+export default function OrderHistory() {
+  const [state, setState] = useState<State>({ status: 'loading' });
+
+  useEffect(() => {
+    fetch('/api/orders')
+      .then((response) => {
+        if (!response.ok) throw new Error(`Request failed with ${response.status}`);
+        return response.json() as Promise<Order[]>;
+      })
+      .then((orders) => setState({ status: 'loaded', orders }))
+      .catch(() => setState({ status: 'error' }));
+  }, []);
+
+  if (state.status === 'loading') return <p role="status">Loading orders…</p>;
+
+  if (state.status === 'error') return <p role="alert">Could not load your orders.</p>;
+
+  if (state.orders.length === 0) return <p>You have not placed any orders yet.</p>;
+
+  return (
+    <ul>
+      {state.orders.map((order) => (
+        <li key={order.id}>
+          Order {order.id} · {new Date(order.placedAt).toLocaleDateString('en-US')} · $
+          {order.total.toFixed(2)}
+        </li>
+      ))}
+    </ul>
+  );
+}
diff --git a/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/vitest.config.ts b/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/vitest.config.ts
@@ -0,0 +1,5 @@
+// addon-vitest's test-run starts Vitest from the project root, so the
+// storybook test project must be reachable from the default config. The eval
+// runner overwrites this file before executing EVAL.ts; it only matters while
+// the agent runs.
+export { default } from './vitest.storybook.config.ts';
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Add stories in `src/components/OrderHistory.stories.tsx` for the existing `OrderHistory` component, covering the loaded, empty, and error states.