From 9d966064511ebab161c87d50c6f3d3c783954431 Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Tue, 6 Oct 2026 15:05:46 +0200 Subject: [PATCH 01/32] feat(skills): rewrite the stories skill as one short workflow Co-Authored-By: Claude Opus 5.5 (1M context) --- .../cli/skills/content/build-stories-skill.ts | 123 ++++++++++++++++++ code/core/src/cli/skills/run.ts | 20 +-- 2 files changed, 133 insertions(+), 10 deletions(-) create mode 100644 code/core/src/cli/skills/content/build-stories-skill.ts diff --git a/code/core/src/cli/skills/content/build-stories-skill.ts b/code/core/src/cli/skills/content/build-stories-skill.ts new file mode 100644 index 000000000000..1adc5708a4f2 --- /dev/null +++ b/code/core/src/cli/skills/content/build-stories-skill.ts @@ -0,0 +1,123 @@ +import { getToolName } from '../../../shared/open-service/toolset-names.ts'; + +export type StoriesSkillInputs = { + framework: string; + docsEnabled: boolean; + testSupported: boolean; + a11yEnabled: boolean; + changeDetectionEnabled: boolean; + moduleGraphSupported: boolean; + reviewEnabled: boolean; +}; + +const ref = getToolName({ transport: 'cli' }); + +function docsSection(): string { + return `## Look up components first + +\`\`\`sh +${ref('docs.list')} # every component and docs page, with its id +${ref('docs.show')} --id # props and usage examples of one entry +\`\`\` + +Run \`docs list\` once at the start, then \`docs show\` for each component you build on or are asked about. Reuse what exists instead of building a duplicate. Answer props, API and usage questions from these commands, not from source files or \`node_modules\`: a prop that is not documented does not exist. When \`docs list\` shows several sources, add \`--storybookId \`.`; +} + +function writeSection(framework: string): string { + return `## Write the component and its stories + +Every component you create or change gets stories: one per distinct state it can reach (variants, loading, empty, error, disabled), with realistic props. An interactive component also gets a \`play\` function that drives it and asserts the visible result; a callback passed as \`fn()\` must be asserted as called. + +- Import \`Meta\` and \`StoryObj\` from \`${framework}\`, and \`fn\`, \`expect\`, \`mocked\` and \`sb\` from \`storybook/test\`. +- \`play: async ({ canvas, userEvent }) => { ... }\`: query \`canvas\` directly, by role or label. Never wrap it in \`within()\`. +- To mock a module, register it in \`.storybook/preview.ts\` with \`sb.mock(import('./api.ts'), { spy: true })\` (a relative path needs its file extension), then set the result per story in \`beforeEach\` with \`mocked(getUser).mockResolvedValue(...)\`. Always mock network and other external dependencies.`; +} + +function testSection(a11yEnabled: boolean): string { + const a11y = a11yEnabled + ? '\n\nThe run also reports accessibility violations. Fix semantic ones yourself (roles, labels, alt text, keyboard access). For visual ones such as color contrast, do not change the design: describe the problem, offer two or three options and ask the user.' + : ''; + return `## Test + +\`\`\`sh +${ref('test.run')} --stories '[{"storyId":""}]' # leave out --stories to run every story +\`\`\` + +Run this after every change, instead of a \`package.json\` test script. Use focused runs while iterating and one full run before you finish. Fix failures and rerun; never finish with failing tests.${a11y}`; +} + +function discoverSection(inputs: StoriesSkillInputs): string { + const commands = [ + inputs.changeDetectionEnabled && + `${ref('stories.changed')} # stories affected by your uncommitted changes`, + inputs.moduleGraphSupported && + `${ref('stories.findByComponent')} --componentPaths '["/abs/path/Button.tsx"]' # stories that render these files`, + inputs.docsEnabled && `${ref('docs.list')} --withStoryIds true # every story id`, + ].filter(Boolean); + if (commands.length === 0) { + return ''; + } + const sharedFiles = inputs.moduleGraphSupported + ? ' A shared file (design token, theme, util, hook) has no stories of its own: pass the components that use it to `find-by-component`.' + : ''; + return `## Find the stories + +\`\`\`sh +${commands.join('\n')} +\`\`\` + +Story ids come only from these commands. Never build one from a file name, a title or memory.${sharedFiles} When nothing is found, the component has no stories yet: say so, or write them.`; +} + +function reviewSection(): string { + return `## Finish with a review + +\`\`\`sh +${ref('review.create')} --input '{ + "title": "Rounder buttons", + "description": "The **Button** corner radius goes from 4px to 8px.", + "collections": [ + { "title": "Button variants", "rationale": "The component that changed.", "storyIds": ["button--primary", "button--disabled"] }, + { "title": "Forms that use it", "rationale": "Where the change shows up in context.", "storyIds": ["login-form--default"] } + ], + "changedFiles": ["src/Button.tsx"] +}' +\`\`\` + +Publish a review after every change the user can see, and again after each later change. It needs a running Storybook. Group the stories into two to five collections, from the changed component up to the pages that show it, and include every story you created. When the user asks to see or browse components or stories and no code changed, publish the same review with \`"changedFiles": []\`. Skip the review only when nothing visible changed, and say that instead. + +Then do what the command prints: it gives the review link your answer must end with. Do not list separate story links next to it.`; +} + +function previewSection(reviewEnabled: boolean): string { + const usage = `\`\`\`sh +${ref('stories.preview')} --stories '[{"storyId":""}]' # direct links to stories +\`\`\``; + return reviewEnabled + ? `## Link to one story + +${usage} + +Only for a look at one story while you iterate, or when the user asks for a direct link. It does not replace the review.` + : `## Finish with links + +${usage} + +After every change the user can see, end your answer with the links to the most relevant stories, at most five. It needs a running Storybook.`; +} + +export function buildStoriesSkill(inputs: StoriesSkillInputs): string { + return [ + `# Storybook workflow + +Follow this for every change to how the UI looks (components, stories, styles, themes, tokens) and for requests to show components or stories. Each command prints what to do next; follow that too. Add \`--help\` to a command only when a call fails.`, + inputs.docsEnabled && docsSection(), + writeSection(inputs.framework), + inputs.testSupported && testSection(inputs.a11yEnabled), + discoverSection(inputs), + inputs.reviewEnabled && reviewSection(), + previewSection(inputs.reviewEnabled), + ] + .filter(Boolean) + .join('\n\n'); +} diff --git a/code/core/src/cli/skills/run.ts b/code/core/src/cli/skills/run.ts index eb0e7d62399c..6c1b7ebfd462 100644 --- a/code/core/src/cli/skills/run.ts +++ b/code/core/src/cli/skills/run.ts @@ -3,7 +3,7 @@ import type { Options } from '../../types/index.ts'; import { resolveStorybookConfigDir } from '../tools/config-dir.ts'; import type { ToolsetCatalogEntry } from '../tools/sdk/types.ts'; import { renderCommandReference } from './command-reference.ts'; -import { buildServerInstructions } from './content/build-server-instructions.ts'; +import { buildStoriesSkill } from './content/build-stories-skill.ts'; import { buildStoryInstructions } from './content/build-story-instructions.ts'; import type { getSetupMarkdownOutput } from './content/setup-prompts/index.ts'; import { SKILLS, SKILL_IDS, isSkillId, type SkillId } from './content/skills.ts'; @@ -195,31 +195,31 @@ function withCommandReference( toolsets: ToolsetCatalogEntry[] ): string { const text = assemble(id, inputs); + if (id === 'stories') { + return text; + } const reference = renderCommandReference(text, toolsets); return reference ? `${text.trimEnd()}\n\n${reference}` : text; } function assemble(id: Exclude, inputs: SkillInputs): string { - const { reviewEnabled } = inputs; if (id === 'stories') { - const workflow = buildServerInstructions({ - transport: 'cli', - devEnabled: true, - testSupported: inputs.testSupported, + return buildStoriesSkill({ + framework: inputs.framework, docsEnabled: inputs.docsEnabledForCli, + testSupported: inputs.testSupported, + a11yEnabled: inputs.a11yEnabled, changeDetectionEnabled: inputs.changeDetectionEnabled, moduleGraphSupported: inputs.moduleGraphSupported, - reviewEnabled, - storyInstructionsInline: true, + reviewEnabled: inputs.reviewEnabled, }); - return `${workflow}\n\n${assemble('write-story', inputs)}`; } return buildStoryInstructions({ transport: 'cli', framework: inputs.framework, renderer: inputs.renderer, changeDetectionEnabled: inputs.changeDetectionEnabled, - reviewEnabled, + reviewEnabled: inputs.reviewEnabled, testSupported: inputs.testSupported, a11yEnabled: inputs.a11yEnabled, docsEnabled: inputs.docsEnabledForCli, From bf6749622945f43d6107c5fd7912dba50bf3b41c Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Tue, 6 Oct 2026 15:21:15 +0200 Subject: [PATCH 02/32] test(skills): cover the short stories workflow Co-Authored-By: Claude Opus 5.5 (1M context) --- code/core/src/cli/skills/run.test.ts | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/code/core/src/cli/skills/run.test.ts b/code/core/src/cli/skills/run.test.ts index 4a53181815ea..928763acbc70 100644 --- a/code/core/src/cli/skills/run.test.ts +++ b/code/core/src/cli/skills/run.test.ts @@ -125,7 +125,7 @@ describe('runSkillsCommand', () => { }); const stories = await runSkillsCommand({ tokens: ['stories'], target: {} }, d); - expect(stories.output).toContain('Documentation Workflow'); + expect(stories.output).toContain('npx storybook tools docs show --id '); const writeStory = await runSkillsCommand({ tokens: ['write-story'], target: {} }, d); expect(writeStory.output).toContain('npx storybook tools docs list'); @@ -134,7 +134,7 @@ describe('runSkillsCommand', () => { it('omits the docs workflow when the CLI docs gate is off', async () => { const d = deps(); const stories = await runSkillsCommand({ tokens: ['stories'], target: {} }, d); - expect(stories.output).not.toContain('Documentation Workflow'); + expect(stories.output).not.toContain('npx storybook tools docs'); }); it('write-story assembles CLI-transport story instructions', async () => { @@ -145,25 +145,25 @@ describe('runSkillsCommand', () => { expect(result.output).toContain('npx storybook tools stories changed'); }); - it('stories carries the write-story text and ends with a reference of the tools it names', async () => { + it('stories teaches each command inline, without a command reference', async () => { const d = deps(); + d.resolveSkillInputs.mockResolvedValue({ + ...(await d.resolveSkillInputs()), + docsEnabledForCli: true, + a11yEnabled: true, + }); const stories = await runSkillsCommand({ tokens: ['stories'], target: {} }, d); - const writeStory = await runSkillsCommand({ tokens: ['write-story'], target: {} }, d); - expect(stories.output).toContain(writeStory.output.split('# Command reference')[0]); - expect(describedTools(stories.output)).toEqual([ - 'stories preview', - 'stories changed', - 'stories find-by-component', - 'review create', - 'test run', - ]); + expect(describedTools(stories.output)).toEqual([]); + expect(stories.output).toContain("npx storybook tools test run --stories '[{"); + expect(stories.output).toContain("npx storybook tools review create --input '{"); + expect(stories.output.length).toBeLessThan(6000); }); - it('--all prints the write-story text once', async () => { + it('--all prints the stories workflow once', async () => { const result = await runSkillsCommand({ tokens: [], all: true, target: {} }, deps()); - expect(result.output.split('# Writing User Interfaces')).toHaveLength(2); + expect(result.output.split('# Storybook workflow')).toHaveLength(2); }); it('setup emits the setup markdown from the lightweight probe, without loading config', async () => { From cc5660e9b38ab8c0eb4b394d4dac5a0c8b93ef43 Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Tue, 6 Oct 2026 15:34:02 +0200 Subject: [PATCH 03/32] fix(addon-vitest): drop the document dump from failed story reports Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/node/toolset/definition.test.ts | 54 +++++++++++++++++++ code/addons/vitest/src/node/toolset/format.ts | 13 ++++- 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/code/addons/vitest/src/node/toolset/definition.test.ts b/code/addons/vitest/src/node/toolset/definition.test.ts index ae2de58a7aa7..30c35d7bfb0f 100644 --- a/code/addons/vitest/src/node/toolset/definition.test.ts +++ b/code/addons/vitest/src/node/toolset/definition.test.ts @@ -279,6 +279,60 @@ describe('test API', () => { Expected button text to be "Secondary"`); }); + it('drops the document dump Testing Library appends to a failed query', async () => { + vi.mocked(runStoryTests).mockResolvedValue( + completed({ + componentTestCount: { success: 0, error: 1 }, + componentTestStatuses: [ + componentTest( + 'alert--error', + 'status-value:error', + `Unable to find an accessible element with the role "status" + +Here are the accessible roles: + + alert: + + Name "": +
+ + -------------------------------------------------- + +Ignored nodes: comments, script, style +
+
+ Locked at 10:30:00 +
+
+ at getByRole (/workspace/stories/Alert.stories.tsx:77:54)` + ), + ], + }) + ); + + expect((await runForMcp()).markdown).toBe(`## Failing Stories + +### alert--error + +Unable to find an accessible element with the role "status" + +Here are the accessible roles: + + alert: + + Name "": +
+ + -------------------------------------------------- + at getByRole (/workspace/stories/Alert.stories.tsx:77:54)`); + }); + it('reports accessibility violations with inspect links built from the origin', async () => { vi.mocked(runStoryTests).mockResolvedValue( completed({ diff --git a/code/addons/vitest/src/node/toolset/format.ts b/code/addons/vitest/src/node/toolset/format.ts index f66e364ad88d..9045fbbabc01 100644 --- a/code/addons/vitest/src/node/toolset/format.ts +++ b/code/addons/vitest/src/node/toolset/format.ts @@ -108,12 +108,23 @@ function formatPassingStoriesSection(passingStories: ComponentTestStatus[]): str - ${passingStories.map((status) => status.storyId).join('\n- ')}`; } +/** + * Testing Library appends the whole rendered document to a failed query. The message and the list + * of accessible roles above it already say what went wrong, so the dump only costs context. + */ +function withoutDomDump(description: string): string { + return description.replace( + /\n*Ignored nodes: comments, script, style\n[\s\S]*?(?=\n\s+at [^\n]*:\d+:\d+\)?(?:\n|$)|$)/, + '' + ); +} + function formatFailingStoriesSection(statuses: ComponentTestStatus[]): string { const entries = statuses.map( (status) => `### ${status.storyId} -${status.description || 'No failure details available.'}` +${status.description ? withoutDomDump(status.description) : 'No failure details available.'}` ); return `## Failing Stories From 31cb794a5436e6aa77af43aef124147c4d0261b7 Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Tue, 6 Oct 2026 15:38:06 +0200 Subject: [PATCH 04/32] feat(skills): tighten the stories workflow after reading eval transcripts Co-Authored-By: Claude Opus 5.5 (1M context) --- code/core/src/cli/skills/content/build-stories-skill.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/code/core/src/cli/skills/content/build-stories-skill.ts b/code/core/src/cli/skills/content/build-stories-skill.ts index 1adc5708a4f2..c90d41f4918a 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.ts @@ -20,16 +20,16 @@ ${ref('docs.list')} # every component and docs page, with its id ${ref('docs.show')} --id # props and usage examples of one entry \`\`\` -Run \`docs list\` once at the start, then \`docs show\` for each component you build on or are asked about. Reuse what exists instead of building a duplicate. Answer props, API and usage questions from these commands, not from source files or \`node_modules\`: a prop that is not documented does not exist. When \`docs list\` shows several sources, add \`--storybookId \`.`; +Run \`docs list\` once at the start, then \`docs show\` for each component you build on or are asked about. Reuse what exists instead of building a duplicate. Answer props, API and usage questions from these commands, not from source files or \`node_modules\`: a prop that is not documented does not exist. When \`docs list\` groups its entries under sources (\`id: acme\`), pass the source of the entry too: \`--storybookId acme\`.`; } function writeSection(framework: string): string { return `## Write the component and its stories -Every component you create or change gets stories: one per distinct state it can reach (variants, loading, empty, error, disabled), with realistic props. An interactive component also gets a \`play\` function that drives it and asserts the visible result; a callback passed as \`fn()\` must be asserted as called. +Every component you create or change gets stories: one per distinct state it can reach (variants, loading, empty, error, disabled), with realistic props. Never export a story under the name of a global such as \`Error\`: export \`ErrorState\` and set \`name: 'Error'\`. An interactive component also gets a \`play\` function that drives it and asserts the visible result; a callback passed as \`fn()\` must be asserted as called. - Import \`Meta\` and \`StoryObj\` from \`${framework}\`, and \`fn\`, \`expect\`, \`mocked\` and \`sb\` from \`storybook/test\`. -- \`play: async ({ canvas, userEvent }) => { ... }\`: query \`canvas\` directly, by role or label. Never wrap it in \`within()\`. +- \`play: async ({ canvas, userEvent }) => { ... }\`: query \`canvas\` directly, by role or label. Never wrap it in \`within()\`. \`userEvent.click(element)\` takes no options. - To mock a module, register it in \`.storybook/preview.ts\` with \`sb.mock(import('./api.ts'), { spy: true })\` (a relative path needs its file extension), then set the result per story in \`beforeEach\` with \`mocked(getUser).mockResolvedValue(...)\`. Always mock network and other external dependencies.`; } @@ -84,7 +84,7 @@ ${ref('review.create')} --input '{ }' \`\`\` -Publish a review after every change the user can see, and again after each later change. It needs a running Storybook. Group the stories into two to five collections, from the changed component up to the pages that show it, and include every story you created. When the user asks to see or browse components or stories and no code changed, publish the same review with \`"changedFiles": []\`. Skip the review only when nothing visible changed, and say that instead. +Publish a review after every change the user can see, and again after each later change. It needs a running Storybook. Group the stories into one to five collections, from the changed component up to the pages that show it, and include every story you created. When the user asks to see or browse components or stories and no code changed, publish the same review with \`"changedFiles": []\`. Skip the review only when nothing visible changed, and say that instead. Then do what the command prints: it gives the review link your answer must end with. Do not list separate story links next to it.`; } From 0583b286148ecf1f879a789062c45d9461e6117d Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Tue, 6 Oct 2026 15:39:47 +0200 Subject: [PATCH 05/32] fix(tools): wait for the status store before reading changed stories Co-Authored-By: Claude Opus 5.5 (1M context) --- code/core/src/core-server/presets/common-preset.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/code/core/src/core-server/presets/common-preset.ts b/code/core/src/core-server/presets/common-preset.ts index 580a7e8bf539..ac4410c88e1f 100644 --- a/code/core/src/core-server/presets/common-preset.ts +++ b/code/core/src/core-server/presets/common-preset.ts @@ -55,7 +55,7 @@ import { reviewToolset } from '../../shared/open-service/toolsets/review/definit import { createStoriesToolset } from '../../shared/open-service/toolsets/stories/definition.ts'; import { GitDiffProvider } from '../change-detection/GitDiffProvider.ts'; import { getChangeDetectionReadiness } from '../change-detection/readiness.ts'; -import { getStatusStoreByTypeId } from '../stores/status.ts'; +import { getStatusStoreByTypeId, universalStatusStore } from '../stores/status.ts'; import { applyServicesPresetOnce } from '../utils/apply-services-preset-once.ts'; import { getPreviewBuilder } from '../utils/get-builders.ts'; import { getRefsFromConfig } from '../utils/get-refs-from-config.ts'; @@ -413,7 +413,12 @@ export const services = async (_value: void, options: Options): Promise => getChangedFiles: () => gitDiffProvider.getChangedFiles(), }, changeStatuses: { - getAll: () => getStatusStoreByTypeId(CHANGE_DETECTION_STATUS_TYPE_ID).getAll(), + getAll: async () => { + const store = getStatusStoreByTypeId(CHANGE_DETECTION_STATUS_TYPE_ID); + // An attached tools CLI is a follower, which starts empty until the dev server answers. + await universalStatusStore.untilReady(); + return store.getAll(); + }, }, reviewEnabled, }) From 54479feb79e70241cde657f48dff05c27af68e7f Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Tue, 6 Oct 2026 15:42:21 +0200 Subject: [PATCH 06/32] Revert "fix(tools): wait for the status store before reading changed stories" This reverts commit 0583b286148ecf1f879a789062c45d9461e6117d. --- code/core/src/core-server/presets/common-preset.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/code/core/src/core-server/presets/common-preset.ts b/code/core/src/core-server/presets/common-preset.ts index ac4410c88e1f..580a7e8bf539 100644 --- a/code/core/src/core-server/presets/common-preset.ts +++ b/code/core/src/core-server/presets/common-preset.ts @@ -55,7 +55,7 @@ import { reviewToolset } from '../../shared/open-service/toolsets/review/definit import { createStoriesToolset } from '../../shared/open-service/toolsets/stories/definition.ts'; import { GitDiffProvider } from '../change-detection/GitDiffProvider.ts'; import { getChangeDetectionReadiness } from '../change-detection/readiness.ts'; -import { getStatusStoreByTypeId, universalStatusStore } from '../stores/status.ts'; +import { getStatusStoreByTypeId } from '../stores/status.ts'; import { applyServicesPresetOnce } from '../utils/apply-services-preset-once.ts'; import { getPreviewBuilder } from '../utils/get-builders.ts'; import { getRefsFromConfig } from '../utils/get-refs-from-config.ts'; @@ -413,12 +413,7 @@ export const services = async (_value: void, options: Options): Promise => getChangedFiles: () => gitDiffProvider.getChangedFiles(), }, changeStatuses: { - getAll: async () => { - const store = getStatusStoreByTypeId(CHANGE_DETECTION_STATUS_TYPE_ID); - // An attached tools CLI is a follower, which starts empty until the dev server answers. - await universalStatusStore.untilReady(); - return store.getAll(); - }, + getAll: () => getStatusStoreByTypeId(CHANGE_DETECTION_STATUS_TYPE_ID).getAll(), }, reviewEnabled, }) From 56852373e68ad5e0fa17419c81dafb3277600ca9 Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Tue, 6 Oct 2026 15:42:58 +0200 Subject: [PATCH 07/32] Revert "fix(addon-vitest): drop the document dump from failed story reports" This reverts commit cc5660e9b38ab8c0eb4b394d4dac5a0c8b93ef43. --- .../src/node/toolset/definition.test.ts | 54 ------------------- code/addons/vitest/src/node/toolset/format.ts | 13 +---- 2 files changed, 1 insertion(+), 66 deletions(-) diff --git a/code/addons/vitest/src/node/toolset/definition.test.ts b/code/addons/vitest/src/node/toolset/definition.test.ts index 30c35d7bfb0f..ae2de58a7aa7 100644 --- a/code/addons/vitest/src/node/toolset/definition.test.ts +++ b/code/addons/vitest/src/node/toolset/definition.test.ts @@ -279,60 +279,6 @@ describe('test API', () => { Expected button text to be "Secondary"`); }); - it('drops the document dump Testing Library appends to a failed query', async () => { - vi.mocked(runStoryTests).mockResolvedValue( - completed({ - componentTestCount: { success: 0, error: 1 }, - componentTestStatuses: [ - componentTest( - 'alert--error', - 'status-value:error', - `Unable to find an accessible element with the role "status" - -Here are the accessible roles: - - alert: - - Name "": -
- - -------------------------------------------------- - -Ignored nodes: comments, script, style -
-
- Locked at 10:30:00 -
-
- at getByRole (/workspace/stories/Alert.stories.tsx:77:54)` - ), - ], - }) - ); - - expect((await runForMcp()).markdown).toBe(`## Failing Stories - -### alert--error - -Unable to find an accessible element with the role "status" - -Here are the accessible roles: - - alert: - - Name "": -
- - -------------------------------------------------- - at getByRole (/workspace/stories/Alert.stories.tsx:77:54)`); - }); - it('reports accessibility violations with inspect links built from the origin', async () => { vi.mocked(runStoryTests).mockResolvedValue( completed({ diff --git a/code/addons/vitest/src/node/toolset/format.ts b/code/addons/vitest/src/node/toolset/format.ts index 9045fbbabc01..f66e364ad88d 100644 --- a/code/addons/vitest/src/node/toolset/format.ts +++ b/code/addons/vitest/src/node/toolset/format.ts @@ -108,23 +108,12 @@ function formatPassingStoriesSection(passingStories: ComponentTestStatus[]): str - ${passingStories.map((status) => status.storyId).join('\n- ')}`; } -/** - * Testing Library appends the whole rendered document to a failed query. The message and the list - * of accessible roles above it already say what went wrong, so the dump only costs context. - */ -function withoutDomDump(description: string): string { - return description.replace( - /\n*Ignored nodes: comments, script, style\n[\s\S]*?(?=\n\s+at [^\n]*:\d+:\d+\)?(?:\n|$)|$)/, - '' - ); -} - function formatFailingStoriesSection(statuses: ComponentTestStatus[]): string { const entries = statuses.map( (status) => `### ${status.storyId} -${status.description ? withoutDomDump(status.description) : 'No failure details available.'}` +${status.description || 'No failure details available.'}` ); return `## Failing Stories From 5dd8143e76d01b8cf55eef6c0a39e7036f647e42 Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Tue, 6 Oct 2026 15:46:41 +0200 Subject: [PATCH 08/32] feat(skills): name both follow-ups of a published review Co-Authored-By: Claude Opus 5.5 (1M context) --- code/core/src/cli/skills/content/build-stories-skill.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/core/src/cli/skills/content/build-stories-skill.ts b/code/core/src/cli/skills/content/build-stories-skill.ts index c90d41f4918a..c2a39e48bb11 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.ts @@ -86,7 +86,7 @@ ${ref('review.create')} --input '{ Publish a review after every change the user can see, and again after each later change. It needs a running Storybook. Group the stories into one to five collections, from the changed component up to the pages that show it, and include every story you created. When the user asks to see or browse components or stories and no code changed, publish the same review with \`"changedFiles": []\`. Skip the review only when nothing visible changed, and say that instead. -Then do what the command prints: it gives the review link your answer must end with. Do not list separate story links next to it.`; +Then do both things the command prints, every time: open the review in the in-app browser with a browser tool, and end your answer with the review section it gives you. Do not list separate story links next to it.`; } function previewSection(reviewEnabled: boolean): string { From e352561faf855fc2511a315ec3a5bde69c951cfe Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Tue, 6 Oct 2026 15:53:43 +0200 Subject: [PATCH 09/32] docs(skills): describe the stories skill as it is now Co-Authored-By: Claude Opus 5.5 (1M context) --- code/core/src/cli/skills/content/skills.ts | 2 +- code/core/src/cli/skills/run.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/code/core/src/cli/skills/content/skills.ts b/code/core/src/cli/skills/content/skills.ts index 831dedc5d808..6be486e2f793 100644 --- a/code/core/src/cli/skills/content/skills.ts +++ b/code/core/src/cli/skills/content/skills.ts @@ -11,7 +11,7 @@ export type SkillId = (typeof SKILL_IDS)[number]; export const SKILLS: Record = { stories: { blurb: - 'The mandatory, ordered workflow for UI changes: discover affected stories, test, and present results. Includes `write-story`.', + 'The workflow for UI changes, with every command it needs: look up components, write stories, test, and publish a review.', }, 'write-story': { blurb: diff --git a/code/core/src/cli/skills/run.ts b/code/core/src/cli/skills/run.ts index 6c1b7ebfd462..8f946a441656 100644 --- a/code/core/src/cli/skills/run.ts +++ b/code/core/src/cli/skills/run.ts @@ -101,7 +101,7 @@ export async function runSkillsCommand( return { output: '', errorOutput: intent.message, exitCode: 1 }; } try { - // `stories` carries the `write-story` text, so `--all` does not print it a second time. + // `stories` covers writing stories itself, so `--all` leaves out the longer `write-story` text. const ids = intent.kind === 'all' ? SKILL_IDS.filter((id) => id !== 'write-story') : [intent.id]; const docs = await serveSkills(ids, resolveStorybookConfigDir(input.target), deps); From 70823213fcb877d4d06e01fbe748704bcd4dfe31 Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Tue, 6 Oct 2026 15:59:39 +0200 Subject: [PATCH 10/32] feat(skills): require story discovery before every review Co-Authored-By: Claude Opus 5.5 (1M context) --- code/core/src/cli/skills/content/build-stories-skill.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/core/src/cli/skills/content/build-stories-skill.ts b/code/core/src/cli/skills/content/build-stories-skill.ts index c2a39e48bb11..c260f82328ac 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.ts @@ -66,7 +66,7 @@ function discoverSection(inputs: StoriesSkillInputs): string { ${commands.join('\n')} \`\`\` -Story ids come only from these commands. Never build one from a file name, a title or memory.${sharedFiles} When nothing is found, the component has no stories yet: say so, or write them.`; +Run one of these before every review, also when a test run already listed the stories you wrote: they add the stories of other components that your change affects. Story ids come only from these commands. Never build one from a file name, a title or memory.${sharedFiles} When nothing is found, the component has no stories yet: say so, or write them.`; } function reviewSection(): string { From e67855134940ad189a52da2ccebb8385d2e6fe4c Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Tue, 6 Oct 2026 16:06:00 +0200 Subject: [PATCH 11/32] feat(skills): keep story conventions to the stories the agent writes Co-Authored-By: Claude Opus 5.5 (1M context) --- code/core/src/cli/skills/content/build-stories-skill.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/core/src/cli/skills/content/build-stories-skill.ts b/code/core/src/cli/skills/content/build-stories-skill.ts index c260f82328ac..0a021303053e 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.ts @@ -26,7 +26,7 @@ Run \`docs list\` once at the start, then \`docs show\` for each component you b function writeSection(framework: string): string { return `## Write the component and its stories -Every component you create or change gets stories: one per distinct state it can reach (variants, loading, empty, error, disabled), with realistic props. Never export a story under the name of a global such as \`Error\`: export \`ErrorState\` and set \`name: 'Error'\`. An interactive component also gets a \`play\` function that drives it and asserts the visible result; a callback passed as \`fn()\` must be asserted as called. +Every component you create or change gets stories: one per distinct state it can reach (variants, loading, empty, error, disabled), with realistic props. Never export a story under the name of a global such as \`Error\`: export \`ErrorState\` and set \`name: 'Error'\`. An interactive component also gets a \`play\` function that drives it and asserts the visible result; a callback passed as \`fn()\` must be asserted as called. The rules below are for the stories you write; do not rewrite existing stories only to match them. - Import \`Meta\` and \`StoryObj\` from \`${framework}\`, and \`fn\`, \`expect\`, \`mocked\` and \`sb\` from \`storybook/test\`. - \`play: async ({ canvas, userEvent }) => { ... }\`: query \`canvas\` directly, by role or label. Never wrap it in \`within()\`. \`userEvent.click(element)\` takes no options. From 6680df54c552867e69ad77e32959ed27de7757cc Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Tue, 6 Oct 2026 16:23:51 +0200 Subject: [PATCH 12/32] fix(skills): apply review findings to the stories workflow Co-Authored-By: Claude Opus 5.5 (1M context) --- .../content/build-server-instructions.test.ts | 19 ------ .../content/build-server-instructions.ts | 13 +---- .../content/build-stories-skill.test.ts | 58 +++++++++++++++++++ .../cli/skills/content/build-stories-skill.ts | 17 ++++-- code/core/src/cli/skills/run.test.ts | 2 +- 5 files changed, 72 insertions(+), 37 deletions(-) create mode 100644 code/core/src/cli/skills/content/build-stories-skill.test.ts diff --git a/code/core/src/cli/skills/content/build-server-instructions.test.ts b/code/core/src/cli/skills/content/build-server-instructions.test.ts index 972abfd06e15..b73a17c39bdc 100644 --- a/code/core/src/cli/skills/content/build-server-instructions.test.ts +++ b/code/core/src/cli/skills/content/build-server-instructions.test.ts @@ -290,25 +290,6 @@ describe('buildServerInstructions', () => { expect(instructions).not.toContain('get-storybook-story-instructions'); }); - it.each([true, false])( - 'points at the story instructions in the same document when they are inline (review %s)', - (reviewEnabled) => { - const instructions = buildServerInstructions({ - transport: 'cli', - devEnabled: true, - testSupported: false, - docsEnabled: false, - reviewEnabled, - storyInstructionsInline: true, - }); - - expect(instructions).toContain( - '- Before creating or editing components or stories, read **Writing User Interfaces** below; it is the source of truth for imports, story patterns, and testing conventions.' - ); - expect(instructions).not.toContain('npx storybook skills write-story'); - } - ); - it('renders review, preview, and discovery references as CLI commands', () => { const instructions = buildServerInstructions({ transport: 'cli', diff --git a/code/core/src/cli/skills/content/build-server-instructions.ts b/code/core/src/cli/skills/content/build-server-instructions.ts index dd6a82c42786..d4fda3051bbf 100644 --- a/code/core/src/cli/skills/content/build-server-instructions.ts +++ b/code/core/src/cli/skills/content/build-server-instructions.ts @@ -22,13 +22,8 @@ export type ServerInstructionsInputs = { */ moduleGraphSupported?: boolean; reviewEnabled?: boolean; - /** The story instructions follow in the same document, so the workflow points at them there. */ - storyInstructionsInline?: boolean; }; -const INLINE_STORY_INSTRUCTIONS_STEP = - 'Before creating or editing components or stories, read **Writing User Interfaces** below; it is the source of truth for imports, story patterns, and testing conventions.'; - /** * The full rule for how the agent should present links in its final * user-facing response, delivered through the @@ -75,9 +70,7 @@ export function buildServerInstructions({ legacyDevInstructions .replace( '{{STORY_INSTRUCTIONS_STEP}}', - options.storyInstructionsInline - ? INLINE_STORY_INSTRUCTIONS_STEP - : `Before creating or editing components or stories, call **${skillRef('write-story')}**.\n- Treat its output as the source of truth for imports, story patterns, and testing conventions.` + `Before creating or editing components or stories, call **${skillRef('write-story')}**.\n- Treat its output as the source of truth for imports, story patterns, and testing conventions.` ) .replaceAll('{{PREVIEW_STORIES}}', ref('stories.preview')) .trim() @@ -105,9 +98,7 @@ export function buildServerInstructions({ devInstructions .replace( '{{STORY_INSTRUCTIONS_STEP}}', - options.storyInstructionsInline - ? INLINE_STORY_INSTRUCTIONS_STEP - : `Before creating or editing components or stories, call **${skillRef('write-story')}**; its output is the source of truth for imports, story patterns, and testing conventions.` + `Before creating or editing components or stories, call **${skillRef('write-story')}**; its output is the source of truth for imports, story patterns, and testing conventions.` ) .replaceAll('{{GET_STORIES_BY_COMPONENT}}', ref('stories.findByComponent')) .replace('{{PREVIEW_STORIES_STEP}}', previewStoriesStep) diff --git a/code/core/src/cli/skills/content/build-stories-skill.test.ts b/code/core/src/cli/skills/content/build-stories-skill.test.ts new file mode 100644 index 000000000000..30459ce4f6f7 --- /dev/null +++ b/code/core/src/cli/skills/content/build-stories-skill.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest'; + +import { buildStoriesSkill, type StoriesSkillInputs } from './build-stories-skill.ts'; + +const everything: StoriesSkillInputs = { + framework: '@storybook/react-vite', + docsEnabled: true, + testSupported: true, + a11yEnabled: true, + changeDetectionEnabled: true, + moduleGraphSupported: true, + reviewEnabled: true, +}; + +describe('buildStoriesSkill', () => { + it('shows every command with its arguments when every feature is on', () => { + const text = buildStoriesSkill(everything); + + expect(text).toContain('npx storybook tools docs show --id '); + expect(text).toContain(`npx storybook tools test run --stories '[{"storyId":""}]'`); + expect(text).toContain('npx storybook tools stories changed'); + expect(text).toContain(`npx storybook tools stories find-by-component --componentPaths '["`); + expect(text).toContain(`npx storybook tools review create --input '{`); + expect(text).toContain('accessibility violations'); + }); + + it('ends with story links and never mentions a review when review is off', () => { + const text = buildStoriesSkill({ + ...everything, + changeDetectionEnabled: false, + reviewEnabled: false, + }); + + expect(text).toContain('## Finish with links'); + expect(text).not.toMatch(/\breview/i); + }); + + it('leaves out testing when the project cannot run story tests', () => { + const text = buildStoriesSkill({ ...everything, testSupported: false, a11yEnabled: false }); + + expect(text).not.toContain('test run'); + expect(text).not.toContain('accessibility'); + }); + + it('selects stories by file and export when no command lists story ids', () => { + const text = buildStoriesSkill({ + ...everything, + docsEnabled: false, + changeDetectionEnabled: false, + moduleGraphSupported: false, + reviewEnabled: false, + }); + + expect(text).toContain('"absoluteStoryPath"'); + expect(text).not.toContain('find-by-component'); + expect(text).not.toContain('docs list'); + }); +}); diff --git a/code/core/src/cli/skills/content/build-stories-skill.ts b/code/core/src/cli/skills/content/build-stories-skill.ts index 0a021303053e..bcfd372a64cd 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.ts @@ -55,10 +55,15 @@ function discoverSection(inputs: StoriesSkillInputs): string { inputs.docsEnabled && `${ref('docs.list')} --withStoryIds true # every story id`, ].filter(Boolean); if (commands.length === 0) { - return ''; + return `## Find the stories + +This project has no command that lists story ids. Select a story by its file and export instead: \`--stories '[{"absoluteStoryPath":"/abs/path/Button.stories.tsx","exportName":"Primary"}]'\`.`; } - const sharedFiles = inputs.moduleGraphSupported - ? ' A shared file (design token, theme, util, hook) has no stories of its own: pass the components that use it to `find-by-component`.' + const before = inputs.reviewEnabled ? 'before every review' : 'before you share links'; + const fallback = inputs.moduleGraphSupported + ? inputs.changeDetectionEnabled + ? ' When \`stories changed\` leaves out a file you touched, pass that file to \`find-by-component\`; for a shared file (design token, theme, util, hook), which has no stories of its own, pass the components that use it.' + : ' A shared file (design token, theme, util, hook) has no stories of its own: pass the components that use it to \`find-by-component\`.' : ''; return `## Find the stories @@ -66,7 +71,7 @@ function discoverSection(inputs: StoriesSkillInputs): string { ${commands.join('\n')} \`\`\` -Run one of these before every review, also when a test run already listed the stories you wrote: they add the stories of other components that your change affects. Story ids come only from these commands. Never build one from a file name, a title or memory.${sharedFiles} When nothing is found, the component has no stories yet: say so, or write them.`; +Run one of these ${before}, also when you already know the ids of the stories you wrote: they add the stories of other components that your change affects. Story ids come only from these commands. Never build one from a file name, a title or memory.${fallback} When none of them finds a story for a component, it has no stories yet: say so, or write them.`; } function reviewSection(): string { @@ -84,7 +89,7 @@ ${ref('review.create')} --input '{ }' \`\`\` -Publish a review after every change the user can see, and again after each later change. It needs a running Storybook. Group the stories into one to five collections, from the changed component up to the pages that show it, and include every story you created. When the user asks to see or browse components or stories and no code changed, publish the same review with \`"changedFiles": []\`. Skip the review only when nothing visible changed, and say that instead. +Publish a review after every change the user can see, and again after each later change. It needs a running Storybook. Group the stories into two to five collections, from the changed component up to the pages that show it (one is enough when a single component is affected), and include every story you created. When the user asks to see or browse components or stories and no code changed, publish the same review with \`"changedFiles": []\`. Skip the review only when nothing visible changed, and say that instead. Then do both things the command prints, every time: open the review in the in-app browser with a browser tool, and end your answer with the review section it gives you. Do not list separate story links next to it.`; } @@ -110,7 +115,7 @@ export function buildStoriesSkill(inputs: StoriesSkillInputs): string { return [ `# Storybook workflow -Follow this for every change to how the UI looks (components, stories, styles, themes, tokens) and for requests to show components or stories. Each command prints what to do next; follow that too. Add \`--help\` to a command only when a call fails.`, +Follow this for every change to how the UI looks (components, stories, styles, themes, tokens) and for requests to show components or stories. Some commands print what to do next; follow that too. Add \`--help\` to a command only when a call fails.`, inputs.docsEnabled && docsSection(), writeSection(inputs.framework), inputs.testSupported && testSection(inputs.a11yEnabled), diff --git a/code/core/src/cli/skills/run.test.ts b/code/core/src/cli/skills/run.test.ts index 928763acbc70..60650b92676f 100644 --- a/code/core/src/cli/skills/run.test.ts +++ b/code/core/src/cli/skills/run.test.ts @@ -108,7 +108,7 @@ describe('runSkillsCommand', () => { expect(d.loadStorybook).not.toHaveBeenCalled(); }); - it('stories assembles CLI-transport server instructions using the CLI review gate', async () => { + it('stories prints the workflow with CLI commands', async () => { const d = deps(); const result = await runSkillsCommand({ tokens: ['stories'], target: {} }, d); expect(result.exitCode).toBe(0); From 64551f376da5108a98511cfd1e028ec3fef8ab7f Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Tue, 6 Oct 2026 16:39:28 +0200 Subject: [PATCH 13/32] fix(skills): apply the second review to the stories workflow Co-Authored-By: Claude Opus 5.5 (1M context) --- .../cli/skills/content/build-stories-skill.ts | 33 ++++++++++++------- code/core/src/cli/skills/run.test.ts | 4 ++- code/core/src/cli/skills/run.ts | 4 +-- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/code/core/src/cli/skills/content/build-stories-skill.ts b/code/core/src/cli/skills/content/build-stories-skill.ts index bcfd372a64cd..c4d573f489a5 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.ts @@ -18,15 +18,16 @@ function docsSection(): string { \`\`\`sh ${ref('docs.list')} # every component and docs page, with its id ${ref('docs.show')} --id # props and usage examples of one entry +${ref('docs.showStory')} --storyId # the code of a story that docs show only lists \`\`\` -Run \`docs list\` once at the start, then \`docs show\` for each component you build on or are asked about. Reuse what exists instead of building a duplicate. Answer props, API and usage questions from these commands, not from source files or \`node_modules\`: a prop that is not documented does not exist. When \`docs list\` groups its entries under sources (\`id: acme\`), pass the source of the entry too: \`--storybookId acme\`.`; +Run \`docs list\` once at the start, then \`docs show\` for each component you build on or are asked about. Reuse what exists instead of building a duplicate. Answer props, API and usage questions from these commands and never invent a prop. Read source files or \`node_modules\` only when the commands return nothing relevant. When \`docs list\` groups its entries under sources (\`id: acme\`), pass the source of the entry to both commands that show it: \`--storybookId acme\`.`; } function writeSection(framework: string): string { return `## Write the component and its stories -Every component you create or change gets stories: one per distinct state it can reach (variants, loading, empty, error, disabled), with realistic props. Never export a story under the name of a global such as \`Error\`: export \`ErrorState\` and set \`name: 'Error'\`. An interactive component also gets a \`play\` function that drives it and asserts the visible result; a callback passed as \`fn()\` must be asserted as called. The rules below are for the stories you write; do not rewrite existing stories only to match them. +Every component you create or change gets stories: one per distinct state it can reach (variants, loading, empty, error, disabled), with realistic props. Never export a story under the name of a global such as \`Error\`: export \`ErrorState\` and set \`name: 'Error'\`. An interactive component also gets a \`play\` function that drives it and asserts the visible result; for a callback passed as \`fn()\`, assert whether it was called, as that state expects. The rules below are for the stories you write; do not rewrite existing stories only to match them. - Import \`Meta\` and \`StoryObj\` from \`${framework}\`, and \`fn\`, \`expect\`, \`mocked\` and \`sb\` from \`storybook/test\`. - \`play: async ({ canvas, userEvent }) => { ... }\`: query \`canvas\` directly, by role or label. Never wrap it in \`within()\`. \`userEvent.click(element)\` takes no options. @@ -40,7 +41,7 @@ function testSection(a11yEnabled: boolean): string { return `## Test \`\`\`sh -${ref('test.run')} --stories '[{"storyId":""}]' # leave out --stories to run every story +${ref('test.run')} --stories '[{"storyId":""}]' # ids: see "Find the stories"; leave out --stories to run every story \`\`\` Run this after every change, instead of a \`package.json\` test script. Use focused runs while iterating and one full run before you finish. Fix failures and rerun; never finish with failing tests.${a11y}`; @@ -60,18 +61,28 @@ function discoverSection(inputs: StoriesSkillInputs): string { This project has no command that lists story ids. Select a story by its file and export instead: \`--stories '[{"absoluteStoryPath":"/abs/path/Button.stories.tsx","exportName":"Primary"}]'\`.`; } const before = inputs.reviewEnabled ? 'before every review' : 'before you share links'; - const fallback = inputs.moduleGraphSupported - ? inputs.changeDetectionEnabled - ? ' When \`stories changed\` leaves out a file you touched, pass that file to \`find-by-component\`; for a shared file (design token, theme, util, hook), which has no stories of its own, pass the components that use it.' - : ' A shared file (design token, theme, util, hook) has no stories of its own: pass the components that use it to \`find-by-component\`.' - : ''; + const sharedFile = '(design token, theme, util, hook)'; + const fallback = !inputs.moduleGraphSupported + ? [] + : [ + inputs.changeDetectionEnabled + ? `When \`stories changed\` leaves out a file you touched, pass that file to \`find-by-component\`; for a shared file ${sharedFile}, which has no stories of its own, pass the components that use it.` + : `A shared file ${sharedFile} has no stories of its own: pass the components that use it to \`find-by-component\`.`, + 'When \`find-by-component\` reports stories hidden by \`maxDistance\`, rerun it with a higher \`--maxDistance\`.', + ]; + const guidance = [ + `Run one of these ${before}, also when you already know the ids of the stories you wrote: they add the stories of other components that your change affects.`, + 'Story ids come only from these commands. Never build one from a file name, a title or memory.', + ...fallback, + 'When none of them finds a story for a component, it has no stories yet: say so, or write them.', + ].join(' '); return `## Find the stories \`\`\`sh ${commands.join('\n')} \`\`\` -Run one of these ${before}, also when you already know the ids of the stories you wrote: they add the stories of other components that your change affects. Story ids come only from these commands. Never build one from a file name, a title or memory.${fallback} When none of them finds a story for a component, it has no stories yet: say so, or write them.`; +${guidance}`; } function reviewSection(): string { @@ -89,7 +100,7 @@ ${ref('review.create')} --input '{ }' \`\`\` -Publish a review after every change the user can see, and again after each later change. It needs a running Storybook. Group the stories into two to five collections, from the changed component up to the pages that show it (one is enough when a single component is affected), and include every story you created. When the user asks to see or browse components or stories and no code changed, publish the same review with \`"changedFiles": []\`. Skip the review only when nothing visible changed, and say that instead. +Publish a review after every change the user can see, and again after each later change. It needs a running Storybook. Group the stories into two to five collections, from the changed component up to the pages that show it (one is enough when a single component is affected), and include every story you created. When the user asks to see or browse components or stories and no code changed, publish the same review with \`"changedFiles": []\`. Skip the review only when nothing visible changed and the user did not ask to see anything, and say that instead. Then do both things the command prints, every time: open the review in the in-app browser with a browser tool, and end your answer with the review section it gives you. Do not list separate story links next to it.`; } @@ -103,7 +114,7 @@ ${ref('stories.preview')} --stories '[{"storyId":""}]' # direct links to s ${usage} -Only for a look at one story while you iterate, or when the user asks for a direct link. It does not replace the review.` +Only for a look at one story while you iterate, or when the user asks for a direct link. It does not replace the review, unless \`review create\` keeps failing after you fixed what it reported: then share these links and say why.` : `## Finish with links ${usage} diff --git a/code/core/src/cli/skills/run.test.ts b/code/core/src/cli/skills/run.test.ts index 60650b92676f..2aafe74d904b 100644 --- a/code/core/src/cli/skills/run.test.ts +++ b/code/core/src/cli/skills/run.test.ts @@ -160,10 +160,12 @@ describe('runSkillsCommand', () => { expect(stories.output.length).toBeLessThan(6000); }); - it('--all prints the stories workflow once', async () => { + it('--all prints each skill once', async () => { const result = await runSkillsCommand({ tokens: [], all: true, target: {} }, deps()); expect(result.output.split('# Storybook workflow')).toHaveLength(2); + expect(result.output.split('# Writing User Interfaces')).toHaveLength(2); + expect(result.output.split('# Storybook Setup')).toHaveLength(2); }); it('setup emits the setup markdown from the lightweight probe, without loading config', async () => { diff --git a/code/core/src/cli/skills/run.ts b/code/core/src/cli/skills/run.ts index 8f946a441656..d988fa984087 100644 --- a/code/core/src/cli/skills/run.ts +++ b/code/core/src/cli/skills/run.ts @@ -101,9 +101,7 @@ export async function runSkillsCommand( return { output: '', errorOutput: intent.message, exitCode: 1 }; } try { - // `stories` covers writing stories itself, so `--all` leaves out the longer `write-story` text. - const ids = - intent.kind === 'all' ? SKILL_IDS.filter((id) => id !== 'write-story') : [intent.id]; + const ids = intent.kind === 'all' ? SKILL_IDS : [intent.id]; const docs = await serveSkills(ids, resolveStorybookConfigDir(input.target), deps); return { output: docs.join('\n\n---\n\n'), From e46586b26c286430aebe19bcc021ba905015ba06 Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Wed, 7 Oct 2026 12:06:03 +0200 Subject: [PATCH 14/32] feat(skills): teach the project's story format in the stories workflow Detect CSF Factories from the preview file and show a matching example, with CSF 3 (typed or plain JavaScript) otherwise. Put the project's own conventions first, restore the story-writing guidance that the rewrite compressed away, and correct the mocking, portal and test-failure rules found in the Fable and Opus reviews. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../content/build-stories-skill.test.ts | 50 +++++- .../cli/skills/content/build-stories-skill.ts | 142 ++++++++++++++++-- code/core/src/cli/skills/inputs.test.ts | 66 +++++++- code/core/src/cli/skills/inputs.ts | 40 ++++- code/core/src/cli/skills/run.test.ts | 5 +- code/core/src/cli/skills/run.ts | 4 + 6 files changed, 285 insertions(+), 22 deletions(-) diff --git a/code/core/src/cli/skills/content/build-stories-skill.test.ts b/code/core/src/cli/skills/content/build-stories-skill.test.ts index 30459ce4f6f7..b5897e13bc63 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.test.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.test.ts @@ -4,6 +4,10 @@ import { buildStoriesSkill, type StoriesSkillInputs } from './build-stories-skil const everything: StoriesSkillInputs = { framework: '@storybook/react-vite', + renderer: '@storybook/react', + csfFactories: false, + previewFile: '.storybook/preview.ts', + typescript: true, docsEnabled: true, testSupported: true, a11yEnabled: true, @@ -35,11 +39,53 @@ describe('buildStoriesSkill', () => { expect(text).not.toMatch(/\breview/i); }); - it('leaves out testing when the project cannot run story tests', () => { + it('says that stories are untested when the project cannot run story tests', () => { const text = buildStoriesSkill({ ...everything, testSupported: false, a11yEnabled: false }); + expect(text).toContain('cannot run story tests'); expect(text).not.toContain('test run'); - expect(text).not.toContain('accessibility'); + expect(text).not.toContain('accessibility violations'); + }); + + it('teaches CSF Factories when the preview file uses definePreview', () => { + const text = buildStoriesSkill({ + ...everything, + csfFactories: true, + previewFile: '.storybook/preview.tsx', + }); + + expect(text).toContain('preview.meta({'); + expect(text).toContain('meta.story({'); + expect(text).not.toContain('satisfies Meta'); + expect(text).toContain('register it once in `.storybook/preview.tsx`'); + }); + + it('teaches CSF 3 with the framework types otherwise', () => { + const text = buildStoriesSkill({ ...everything, framework: '@storybook/vue3-vite' }); + + expect(text).toContain("import type { Meta, StoryObj } from '@storybook/vue3-vite';"); + expect(text).not.toContain('preview.meta({'); + }); + + it('leaves the types out in a JavaScript project', () => { + const text = buildStoriesSkill({ + ...everything, + typescript: false, + previewFile: '.storybook/preview.js', + }); + + expect(text).toContain('export default { component: Button'); + expect(text).not.toContain('StoryObj'); + }); + + it('points a Svelte project at its Svelte CSF files', () => { + const text = buildStoriesSkill({ + ...everything, + framework: '@storybook/sveltekit', + renderer: '@storybook/svelte', + }); + + expect(text).toContain('*.stories.svelte'); }); it('selects stories by file and export when no command lists story ids', () => { diff --git a/code/core/src/cli/skills/content/build-stories-skill.ts b/code/core/src/cli/skills/content/build-stories-skill.ts index c4d573f489a5..1fdea87cae50 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.ts @@ -2,6 +2,12 @@ import { getToolName } from '../../../shared/open-service/toolset-names.ts'; export type StoriesSkillInputs = { framework: string; + renderer?: string; + /** The preview file uses `definePreview`, so new story files use CSF Factories. */ + csfFactories: boolean; + /** Path of the preview file, relative to the working directory. */ + previewFile: string; + typescript: boolean; docsEnabled: boolean; testSupported: boolean; a11yEnabled: boolean; @@ -24,17 +30,121 @@ ${ref('docs.showStory')} --storyId # the code of a story that docs show o Run \`docs list\` once at the start, then \`docs show\` for each component you build on or are asked about. Reuse what exists instead of building a duplicate. Answer props, API and usage questions from these commands and never invent a prop. Read source files or \`node_modules\` only when the commands return nothing relevant. When \`docs list\` groups its entries under sources (\`id: acme\`), pass the source of the entry to both commands that show it: \`--storybookId acme\`.`; } -function writeSection(framework: string): string { +const INFERS_ARGS_FROM_COMPONENT = [ + '@storybook/react', + '@storybook/preact', + '@storybook/vue3', + '@storybook/svelte', +]; + +function storyFormat({ + framework, + renderer, + csfFactories, + previewFile, + typescript, +}: StoriesSkillInputs) { + if (csfFactories) { + return `New story files use CSF Factories, because \`${previewFile}\` uses \`definePreview\`: + +\`\`\`ts +import preview from '../.storybook/preview'; // the path to ${previewFile}, or '#.storybook/preview' when other story files import it that way +import { expect, fn } from 'storybook/test'; + +import { Button } from './Button'; + +const meta = preview.meta({ component: Button, args: { onClick: fn() } }); + +export const Primary = meta.story({ args: { label: 'Save' } }); + +export const Disabled = Primary.extend({ + args: { disabled: true }, + play: async ({ canvas, userEvent, args }) => { + await userEvent.click(canvas.getByRole('button', { name: 'Save' })); + await expect(args.onClick).not.toHaveBeenCalled(); + }, +}); +\`\`\` + +Types come from \`preview\`: no \`Meta\` or \`StoryObj\` import and no default export. A file that still uses \`Meta\` and \`StoryObj\` (CSF 3) stays that way.`; + } + const typedMeta = INFERS_ARGS_FROM_COMPONENT.includes(renderer ?? framework) + ? `const meta = { component: Button, args: { onClick: fn() } } satisfies Meta; +export default meta; +type Story = StoryObj;` + : `const meta: Meta = { component: Button, args: { onClick: fn() } }; // type Meta and StoryObj as other story files do +export default meta; +type Story = StoryObj;`; + const example = typescript + ? `import type { Meta, StoryObj } from '${framework}'; +import { expect, fn } from 'storybook/test'; + +import { Button } from './Button'; + +${typedMeta} + +export const Primary: Story = { args: { label: 'Save' } }; + +export const Disabled: Story = {` + : `import { expect, fn } from 'storybook/test'; + +import { Button } from './Button'; + +export default { component: Button, args: { onClick: fn() } }; + +export const Primary = { args: { label: 'Save' } }; + +export const Disabled = {`; + return `New story files use CSF 3: + +\`\`\`${typescript ? 'ts' : 'js'} +${example} + args: { ...Primary.args, disabled: true }, + play: async ({ canvas, userEvent, args }) => { + await userEvent.click(canvas.getByRole('button', { name: 'Save' })); + await expect(args.onClick).not.toHaveBeenCalled(); + }, +}; +\`\`\` + +Do not switch \`${previewFile}\` to \`definePreview\` (CSF Factories) yourself; when a file already uses \`preview.meta()\`, keep that format.`; +} + +function rendererNote(renderer: string | undefined): string { + if (renderer === '@storybook/svelte') { + return '\n- Svelte: when the project has `*.stories.svelte` files, write those (`defineMeta` and ``, as the existing files do) instead of the format above.'; + } + if (renderer === '@storybook/web-components') { + return '\n- Web components: `canvas` queries do not reach into shadow DOM. Query through `element.shadowRoot`, or with `shadow-dom-testing-library` when the project has it.'; + } + return ''; +} + +function writeSection(inputs: StoriesSkillInputs): string { + const { previewFile } = inputs; return `## Write the component and its stories -Every component you create or change gets stories: one per distinct state it can reach (variants, loading, empty, error, disabled), with realistic props. Never export a story under the name of a global such as \`Error\`: export \`ErrorState\` and set \`name: 'Error'\`. An interactive component also gets a \`play\` function that drives it and asserts the visible result; for a callback passed as \`fn()\`, assert whether it was called, as that state expects. The rules below are for the stories you write; do not rewrite existing stories only to match them. +Every component you create or change gets stories, in its existing story file when it has one. When you remove or rename a component, a prop or a state, update or delete the stories that used it. -- Import \`Meta\` and \`StoryObj\` from \`${framework}\`, and \`fn\`, \`expect\`, \`mocked\` and \`sb\` from \`storybook/test\`. -- \`play: async ({ canvas, userEvent }) => { ... }\`: query \`canvas\` directly, by role or label. Never wrap it in \`within()\`. \`userEvent.click(element)\` takes no options. -- To mock a module, register it in \`.storybook/preview.ts\` with \`sb.mock(import('./api.ts'), { spy: true })\` (a relative path needs its file extension), then set the result per story in \`beforeEach\` with \`mocked(getUser).mockResolvedValue(...)\`. Always mock network and other external dependencies.`; +This project's own files come first. Existing story files show the format, location, naming and \`tags\`; \`${previewFile}\` and existing stories show the setup a component needs (providers, router, theme). Follow them, and use the rules below for what they do not show. A story file keeps the format it has: never mix two formats in one file, and do not rewrite existing stories only to match these rules. + +What to cover: one story per state that changes what the user sees or can do (variants, loading, empty, error, invalid input, disabled, permissions, long or missing content), not one per prop value. Name each story after its scenario and give it realistic data. An interactive component also gets \`play\` functions that use it as a user would (click, type, keyboard, submit) and assert what the user then sees; for an arg passed as \`fn()\`, assert that it was or was not called. Build with semantic elements and labels, so stories can be queried by role. + +${storyFormat(inputs)} + +- \`fn\`, \`expect\`, \`mocked\`, \`sb\` and \`screen\` come from \`storybook/test\`. In \`play\`, take \`canvas\` and \`userEvent\` from the context and query \`canvas\` by role or label; it already has the queries, so never \`within(canvas)\`. \`await\` every \`userEvent\` call and \`expect\`, and use \`findBy*\` or \`waitFor\` for what appears later. For content rendered outside the story (dialog, popover, toast), use \`screen\`. +- A component that needs a provider, router or theme gets \`decorators\`, in \`${previewFile}\` when every story needs it. Check what that file already provides before you add one. +- A story never reaches the network or another external service. Use the mocking this project already has (for example MSW handlers); otherwise mock the module that makes the call. +- To mock a module, register it once in \`${previewFile}\` with \`sb.mock(import('../src/api.${inputs.typescript ? 'ts' : 'js'}'), { spy: true })\`: the path is relative to that file, with its extension and without an alias. Then set the result per story in \`beforeEach\` with \`mocked(getUser).mockResolvedValue(...)\`. +- Do not name an export after a global the file may use, such as \`Error\`: export \`ErrorState\` and set \`name: 'Error'\`.${rendererNote(inputs.renderer)}`; } -function testSection(a11yEnabled: boolean): string { +function testSection({ testSupported, a11yEnabled }: StoriesSkillInputs): string { + if (!testSupported) { + return `## Test + +This project cannot run story tests: \`@storybook/addon-vitest\` is not set up. Say so in your answer and do not claim the stories are tested.`; + } const a11y = a11yEnabled ? '\n\nThe run also reports accessibility violations. Fix semantic ones yourself (roles, labels, alt text, keyboard access). For visual ones such as color contrast, do not change the design: describe the problem, offer two or three options and ask the user.' : ''; @@ -44,7 +154,7 @@ function testSection(a11yEnabled: boolean): string { ${ref('test.run')} --stories '[{"storyId":""}]' # ids: see "Find the stories"; leave out --stories to run every story \`\`\` -Run this after every change, instead of a \`package.json\` test script. Use focused runs while iterating and one full run before you finish. Fix failures and rerun; never finish with failing tests.${a11y}`; +Run this after every change; a \`package.json\` script does not replace it for story tests. Use focused runs while iterating and one full run before you finish. Fix the cause of a failure, not the assertion, and rerun; never finish with failing tests that your change caused. Report failures that were there before your change, or that you cannot fix after a few attempts, instead of hiding them.${a11y}`; } function discoverSection(inputs: StoriesSkillInputs): string { @@ -58,7 +168,7 @@ function discoverSection(inputs: StoriesSkillInputs): string { if (commands.length === 0) { return `## Find the stories -This project has no command that lists story ids. Select a story by its file and export instead: \`--stories '[{"absoluteStoryPath":"/abs/path/Button.stories.tsx","exportName":"Primary"}]'\`.`; +This project has no command that lists story ids. Select a story by its file and export instead: \`--stories '[{"absoluteStoryPath":"/abs/path/Button.stories.tsx","exportName":"Primary"}]'\`. Add \`"explicitStoryName"\` when the story sets a \`name\`.`; } const before = inputs.reviewEnabled ? 'before every review' : 'before you share links'; const sharedFile = '(design token, theme, util, hook)'; @@ -68,7 +178,7 @@ This project has no command that lists story ids. Select a story by its file and inputs.changeDetectionEnabled ? `When \`stories changed\` leaves out a file you touched, pass that file to \`find-by-component\`; for a shared file ${sharedFile}, which has no stories of its own, pass the components that use it.` : `A shared file ${sharedFile} has no stories of its own: pass the components that use it to \`find-by-component\`.`, - 'When \`find-by-component\` reports stories hidden by \`maxDistance\`, rerun it with a higher \`--maxDistance\`.', + `\`find-by-component\` lists stories by distance, from the stories of the file itself up to the pages that use it${inputs.reviewEnabled ? '; those layers make good review collections' : ''}. When it reports stories hidden by \`maxDistance\`, rerun it with a higher \`--maxDistance\`.`, ]; const guidance = [ `Run one of these ${before}, also when you already know the ids of the stories you wrote: they add the stories of other components that your change affects.`, @@ -100,9 +210,11 @@ ${ref('review.create')} --input '{ }' \`\`\` -Publish a review after every change the user can see, and again after each later change. It needs a running Storybook. Group the stories into two to five collections, from the changed component up to the pages that show it (one is enough when a single component is affected), and include every story you created. When the user asks to see or browse components or stories and no code changed, publish the same review with \`"changedFiles": []\`. Skip the review only when nothing visible changed and the user did not ask to see anything, and say that instead. +Publish a review after every change the user can see, and again after each later change. It needs a running Storybook. Group the stories into two to five collections, from the changed component up to the pages that show it (one is enough when a single component is affected), and include every story you created. When the user asks to see or browse components or stories and no code changed, publish the same review with \`"changedFiles": []\`. Skip the review only when nothing visible changed and the user did not ask to see anything, or when the user asked for no review, and say that instead. + +In a follow-up review, keep the collections and their order stable. Add a story where the change should not be visible, as a control. Titles and rationales are plain text without em-dashes. To the user, say "group of stories", not "collection". -Then do both things the command prints, every time: open the review in the in-app browser with a browser tool, and end your answer with the review section it gives you. Do not list separate story links next to it.`; +Then do both things the command prints, every time: open the review in the in-app browser with a browser tool (skip this only when you have no such tool), and end your answer with the review section it describes. Do not list separate story links next to it.`; } function previewSection(reviewEnabled: boolean): string { @@ -114,12 +226,12 @@ ${ref('stories.preview')} --stories '[{"storyId":""}]' # direct links to s ${usage} -Only for a look at one story while you iterate, or when the user asks for a direct link. It does not replace the review, unless \`review create\` keeps failing after you fixed what it reported: then share these links and say why.` +Only for a look at one story while you iterate, or when the user asks for a direct link. It needs a running Storybook. Add \`"globals": {"theme": "dark"}\` or \`"props": {...}\` to an item to see the story under other settings. It does not replace the review, unless \`review create\` keeps failing after you fixed what it reported: then share these links and say why.` : `## Finish with links ${usage} -After every change the user can see, end your answer with the links to the most relevant stories, at most five. It needs a running Storybook.`; +After every change the user can see, end your answer with the links to the most relevant stories, at most five. It needs a running Storybook. Add \`"globals": {"theme": "dark"}\` or \`"props": {...}\` to an item to see the story under other settings.`; } export function buildStoriesSkill(inputs: StoriesSkillInputs): string { @@ -128,8 +240,8 @@ export function buildStoriesSkill(inputs: StoriesSkillInputs): string { Follow this for every change to how the UI looks (components, stories, styles, themes, tokens) and for requests to show components or stories. Some commands print what to do next; follow that too. Add \`--help\` to a command only when a call fails.`, inputs.docsEnabled && docsSection(), - writeSection(inputs.framework), - inputs.testSupported && testSection(inputs.a11yEnabled), + writeSection(inputs), + testSection(inputs), discoverSection(inputs), inputs.reviewEnabled && reviewSection(), previewSection(inputs.reviewEnabled), diff --git a/code/core/src/cli/skills/inputs.test.ts b/code/core/src/cli/skills/inputs.test.ts index c2829e9b636f..edbf4f5b6c5b 100644 --- a/code/core/src/cli/skills/inputs.test.ts +++ b/code/core/src/cli/skills/inputs.test.ts @@ -1,17 +1,35 @@ -import { describe, expect, it, vi } from 'vitest'; +import { existsSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { fs as memfs, vol } from 'memfs'; import type { Options } from '../../types/index.ts'; import { resolveSkillInputs } from './inputs.ts'; +vi.mock('node:fs', { spy: true }); +vi.mock('node:fs/promises', { spy: true }); + +beforeEach(() => { + vol.reset(); + vi.mocked(existsSync).mockImplementation(memfs.existsSync); + vi.mocked(readFile).mockImplementation(memfs.promises.readFile as typeof readFile); + vi.spyOn(process, 'cwd').mockReturnValue('/project'); +}); + function createMockOptions({ framework = '@storybook/react-vite', features, + configDir, }: { framework?: string | { name: string }; features?: Record; + configDir?: string; } = {}): Options { return { + configDir, presets: { apply: vi.fn(async (key: string, defaultValue?: unknown) => { if (key === 'framework') { @@ -93,4 +111,50 @@ describe('resolveSkillInputs', () => { expect(inputs.changeDetectionEnabled).toBe(true); expect(options.presets.apply).not.toHaveBeenCalledWith('features', expect.anything()); }); + + it('detects CSF Factories from a preview file that imports definePreview', async () => { + vol.fromNestedJSON({ + '/project/.storybook/preview.tsx': `import { definePreview } from '@storybook/react-vite';\nexport default definePreview({});`, + }); + + const inputs = await resolveSkillInputs( + createMockOptions({ configDir: '/project/.storybook' }) + ); + + expect(inputs).toMatchObject({ + csfFactories: true, + previewFile: '.storybook/preview.tsx', + typescript: true, + }); + }); + + it('reads a plain JavaScript preview as CSF 3 without TypeScript', async () => { + vol.fromNestedJSON({ + '/project/.storybook/preview.js': 'export default { parameters: {} };', + }); + + const inputs = await resolveSkillInputs( + createMockOptions({ configDir: '/project/.storybook' }) + ); + + expect(inputs).toMatchObject({ + csfFactories: false, + previewFile: '.storybook/preview.js', + typescript: false, + }); + }); + + it('names the preview file after the main config when the project has none', async () => { + vol.fromNestedJSON({ '/project/.storybook/main.ts': 'export default {};' }); + + const inputs = await resolveSkillInputs( + createMockOptions({ configDir: '/project/.storybook' }) + ); + + expect(inputs).toMatchObject({ + csfFactories: false, + previewFile: '.storybook/preview.ts', + typescript: true, + }); + }); }); diff --git a/code/core/src/cli/skills/inputs.ts b/code/core/src/cli/skills/inputs.ts index c80982a86c42..b8bc920dbfb5 100644 --- a/code/core/src/cli/skills/inputs.ts +++ b/code/core/src/cli/skills/inputs.ts @@ -1,5 +1,9 @@ +import { relative } from 'node:path'; + import type { Options } from '../../types/index.ts'; import { extractFrameworkPackageName } from '../../common/utils/get-framework-name.ts'; +import { findConfigFile } from '../../common/utils/get-storybook-info.ts'; +import { isCsfFactoryPreview, readConfig } from '../../csf-tools/ConfigFile.ts'; import { getToolAvailability, @@ -8,7 +12,36 @@ import { } from './availability.ts'; import { frameworkToRendererMap } from './content/framework-renderer.ts'; -export type SkillInputs = ToolAvailability & { framework: string; renderer?: string }; +export type SkillInputs = ToolAvailability & { + framework: string; + renderer?: string; + /** The preview file imports `definePreview`, the entry point of CSF Factories. */ + csfFactories: boolean; + /** Path of the preview file relative to the working directory; a default name when there is none. */ + previewFile: string; + typescript: boolean; +}; + +const isTypeScriptFile = (path: string) => /\.[cm]?tsx?$/.test(path); + +async function resolvePreview(configDir = '.storybook') { + const previewPath = findConfigFile('preview', configDir); + const typescript = isTypeScriptFile(previewPath ?? findConfigFile('main', configDir) ?? ''); + const path = previewPath ?? `${configDir}/preview.${typescript ? 'ts' : 'js'}`; + let csfFactories = false; + if (previewPath) { + try { + csfFactories = isCsfFactoryPreview(await readConfig(previewPath)); + } catch { + // An unparsable preview is treated as a non-factory one. + } + } + return { + csfFactories, + previewFile: relative(process.cwd(), path).replaceAll('\\', '/') || path, + typescript, + }; +} /** * The one probing path for skill-content assembly: everything the pure builders need, resolved @@ -19,12 +52,13 @@ export async function resolveSkillInputs( options: Options, opts: GetToolAvailabilityOptions = {} ): Promise { - const [availability, frameworkPreset] = await Promise.all([ + const [availability, frameworkPreset, preview] = await Promise.all([ getToolAvailability(options, opts), options.presets.apply('framework'), + resolvePreview(options.configDir), ]); const framework = extractFrameworkPackageName( typeof frameworkPreset === 'string' ? frameworkPreset : (frameworkPreset?.name ?? '') ); - return { ...availability, framework, renderer: frameworkToRendererMap[framework] }; + return { ...availability, ...preview, framework, renderer: frameworkToRendererMap[framework] }; } diff --git a/code/core/src/cli/skills/run.test.ts b/code/core/src/cli/skills/run.test.ts index 2aafe74d904b..c34bf44209b6 100644 --- a/code/core/src/cli/skills/run.test.ts +++ b/code/core/src/cli/skills/run.test.ts @@ -27,6 +27,9 @@ const deps = () => ({ resolveSkillInputs: vi.fn().mockResolvedValue({ framework: '@storybook/react-vite', renderer: '@storybook/react', + csfFactories: false, + previewFile: '.storybook/preview.ts', + typescript: true, changeDetectionEnabled: true, moduleGraphSupported: true, reviewEnabled: true, @@ -157,7 +160,7 @@ describe('runSkillsCommand', () => { expect(describedTools(stories.output)).toEqual([]); expect(stories.output).toContain("npx storybook tools test run --stories '[{"); expect(stories.output).toContain("npx storybook tools review create --input '{"); - expect(stories.output.length).toBeLessThan(6000); + expect(stories.output.length).toBeLessThan(10000); }); it('--all prints each skill once', async () => { diff --git a/code/core/src/cli/skills/run.ts b/code/core/src/cli/skills/run.ts index d988fa984087..6063f152dba4 100644 --- a/code/core/src/cli/skills/run.ts +++ b/code/core/src/cli/skills/run.ts @@ -204,6 +204,10 @@ function assemble(id: Exclude, inputs: SkillInputs): string { if (id === 'stories') { return buildStoriesSkill({ framework: inputs.framework, + renderer: inputs.renderer, + csfFactories: inputs.csfFactories, + previewFile: inputs.previewFile, + typescript: inputs.typescript, docsEnabled: inputs.docsEnabledForCli, testSupported: inputs.testSupported, a11yEnabled: inputs.a11yEnabled, From 39234fa102318cff82eeb4546c1a80c0ab17207a Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Wed, 7 Oct 2026 12:12:55 +0200 Subject: [PATCH 15/32] fix(skills): point at the project's own runner when Storybook has no test command Co-Authored-By: Claude Opus 5.5 (1M context) --- code/core/src/cli/skills/content/build-stories-skill.test.ts | 4 ++-- code/core/src/cli/skills/content/build-stories-skill.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/code/core/src/cli/skills/content/build-stories-skill.test.ts b/code/core/src/cli/skills/content/build-stories-skill.test.ts index b5897e13bc63..491251f343a3 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.test.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.test.ts @@ -39,10 +39,10 @@ describe('buildStoriesSkill', () => { expect(text).not.toMatch(/\breview/i); }); - it('says that stories are untested when the project cannot run story tests', () => { + it("points at the project's own runner when Storybook has no test command", () => { const text = buildStoriesSkill({ ...everything, testSupported: false, a11yEnabled: false }); - expect(text).toContain('cannot run story tests'); + expect(text).toContain('Storybook has no test command in this project'); expect(text).not.toContain('test run'); expect(text).not.toContain('accessibility violations'); }); diff --git a/code/core/src/cli/skills/content/build-stories-skill.ts b/code/core/src/cli/skills/content/build-stories-skill.ts index 1fdea87cae50..e67925c0752f 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.ts @@ -143,7 +143,7 @@ function testSection({ testSupported, a11yEnabled }: StoriesSkillInputs): string if (!testSupported) { return `## Test -This project cannot run story tests: \`@storybook/addon-vitest\` is not set up. Say so in your answer and do not claim the stories are tested.`; +Storybook has no test command in this project: \`@storybook/addon-vitest\` is not set up. When the project has its own way to run story tests (a \`package.json\` script, for example with \`@storybook/test-runner\`), run that. Otherwise say in your answer that the stories were not tested.`; } const a11y = a11yEnabled ? '\n\nThe run also reports accessibility violations. Fix semantic ones yourself (roles, labels, alt text, keyboard access). For visual ones such as color contrast, do not change the design: describe the problem, offer two or three options and ask the user.' From f86640ecb5b88109102265a510e05d742fd6bbd5 Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Wed, 7 Oct 2026 14:27:26 +0200 Subject: [PATCH 16/32] fix(skills): keep only the workflow text that a probe showed to be needed Go back to the short stories workflow and add what failed without it in a probe run: name CSF Factories when the preview uses definePreview (Codex wrote CSF 3 or a hybrid), mock network requests with MSW before a module mock (both agents chose sb.mock in a project that had MSW), and use the beforeEach({ msw }) form of msw-storybook-addon v3. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../content/build-stories-skill.test.ts | 66 +++----- .../cli/skills/content/build-stories-skill.ts | 142 +++--------------- code/core/src/cli/skills/inputs.test.ts | 5 +- code/core/src/cli/skills/inputs.ts | 2 - code/core/src/cli/skills/run.test.ts | 1 - code/core/src/cli/skills/run.ts | 2 - 6 files changed, 44 insertions(+), 174 deletions(-) diff --git a/code/core/src/cli/skills/content/build-stories-skill.test.ts b/code/core/src/cli/skills/content/build-stories-skill.test.ts index 491251f343a3..36fc5128b5c0 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.test.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.test.ts @@ -4,10 +4,8 @@ import { buildStoriesSkill, type StoriesSkillInputs } from './build-stories-skil const everything: StoriesSkillInputs = { framework: '@storybook/react-vite', - renderer: '@storybook/react', csfFactories: false, previewFile: '.storybook/preview.ts', - typescript: true, docsEnabled: true, testSupported: true, a11yEnabled: true, @@ -39,66 +37,50 @@ describe('buildStoriesSkill', () => { expect(text).not.toMatch(/\breview/i); }); - it("points at the project's own runner when Storybook has no test command", () => { + it('leaves out testing when the project cannot run story tests', () => { const text = buildStoriesSkill({ ...everything, testSupported: false, a11yEnabled: false }); - expect(text).toContain('Storybook has no test command in this project'); expect(text).not.toContain('test run'); - expect(text).not.toContain('accessibility violations'); + expect(text).not.toContain('accessibility'); }); - it('teaches CSF Factories when the preview file uses definePreview', () => { + it('selects stories by file and export when no command lists story ids', () => { const text = buildStoriesSkill({ ...everything, - csfFactories: true, - previewFile: '.storybook/preview.tsx', + docsEnabled: false, + changeDetectionEnabled: false, + moduleGraphSupported: false, + reviewEnabled: false, }); - expect(text).toContain('preview.meta({'); - expect(text).toContain('meta.story({'); - expect(text).not.toContain('satisfies Meta'); - expect(text).toContain('register it once in `.storybook/preview.tsx`'); - }); - - it('teaches CSF 3 with the framework types otherwise', () => { - const text = buildStoriesSkill({ ...everything, framework: '@storybook/vue3-vite' }); - - expect(text).toContain("import type { Meta, StoryObj } from '@storybook/vue3-vite';"); - expect(text).not.toContain('preview.meta({'); + expect(text).toContain('"absoluteStoryPath"'); + expect(text).not.toContain('find-by-component'); + expect(text).not.toContain('docs list'); }); - it('leaves the types out in a JavaScript project', () => { + it('teaches CSF Factories instead of Meta and StoryObj when the preview uses definePreview', () => { const text = buildStoriesSkill({ ...everything, - typescript: false, - previewFile: '.storybook/preview.js', + csfFactories: true, + previewFile: '.storybook/preview.tsx', }); - expect(text).toContain('export default { component: Button'); - expect(text).not.toContain('StoryObj'); + expect(text).toContain('preview.meta({'); + expect(text).toContain('Do not import `Meta` or `StoryObj`'); + expect(text).not.toContain('Import `Meta` and `StoryObj`'); }); - it('points a Svelte project at its Svelte CSF files', () => { - const text = buildStoriesSkill({ - ...everything, - framework: '@storybook/sveltekit', - renderer: '@storybook/svelte', - }); + it('teaches Meta and StoryObj from the framework package otherwise', () => { + const text = buildStoriesSkill({ ...everything, framework: '@storybook/vue3-vite' }); - expect(text).toContain('*.stories.svelte'); + expect(text).toContain('Import `Meta` and `StoryObj` from `@storybook/vue3-vite`'); + expect(text).not.toContain('preview.meta({'); }); - it('selects stories by file and export when no command lists story ids', () => { - const text = buildStoriesSkill({ - ...everything, - docsEnabled: false, - changeDetectionEnabled: false, - moduleGraphSupported: false, - reviewEnabled: false, - }); + it('puts MSW before module mocks and names the preview file of the project', () => { + const text = buildStoriesSkill({ ...everything, previewFile: 'config/preview.tsx' }); - expect(text).toContain('"absoluteStoryPath"'); - expect(text).not.toContain('find-by-component'); - expect(text).not.toContain('docs list'); + expect(text.indexOf('msw.use(')).toBeLessThan(text.indexOf('sb.mock(')); + expect(text).toContain('register it in `config/preview.tsx`'); }); }); diff --git a/code/core/src/cli/skills/content/build-stories-skill.ts b/code/core/src/cli/skills/content/build-stories-skill.ts index e67925c0752f..bc7fd9e314f8 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.ts @@ -2,12 +2,10 @@ import { getToolName } from '../../../shared/open-service/toolset-names.ts'; export type StoriesSkillInputs = { framework: string; - renderer?: string; - /** The preview file uses `definePreview`, so new story files use CSF Factories. */ + /** The preview file uses `definePreview`, so story files use CSF Factories. */ csfFactories: boolean; /** Path of the preview file, relative to the working directory. */ previewFile: string; - typescript: boolean; docsEnabled: boolean; testSupported: boolean; a11yEnabled: boolean; @@ -30,121 +28,21 @@ ${ref('docs.showStory')} --storyId # the code of a story that docs show o Run \`docs list\` once at the start, then \`docs show\` for each component you build on or are asked about. Reuse what exists instead of building a duplicate. Answer props, API and usage questions from these commands and never invent a prop. Read source files or \`node_modules\` only when the commands return nothing relevant. When \`docs list\` groups its entries under sources (\`id: acme\`), pass the source of the entry to both commands that show it: \`--storybookId acme\`.`; } -const INFERS_ARGS_FROM_COMPONENT = [ - '@storybook/react', - '@storybook/preact', - '@storybook/vue3', - '@storybook/svelte', -]; - -function storyFormat({ - framework, - renderer, - csfFactories, - previewFile, - typescript, -}: StoriesSkillInputs) { - if (csfFactories) { - return `New story files use CSF Factories, because \`${previewFile}\` uses \`definePreview\`: - -\`\`\`ts -import preview from '../.storybook/preview'; // the path to ${previewFile}, or '#.storybook/preview' when other story files import it that way -import { expect, fn } from 'storybook/test'; - -import { Button } from './Button'; - -const meta = preview.meta({ component: Button, args: { onClick: fn() } }); - -export const Primary = meta.story({ args: { label: 'Save' } }); - -export const Disabled = Primary.extend({ - args: { disabled: true }, - play: async ({ canvas, userEvent, args }) => { - await userEvent.click(canvas.getByRole('button', { name: 'Save' })); - await expect(args.onClick).not.toHaveBeenCalled(); - }, -}); -\`\`\` - -Types come from \`preview\`: no \`Meta\` or \`StoryObj\` import and no default export. A file that still uses \`Meta\` and \`StoryObj\` (CSF 3) stays that way.`; - } - const typedMeta = INFERS_ARGS_FROM_COMPONENT.includes(renderer ?? framework) - ? `const meta = { component: Button, args: { onClick: fn() } } satisfies Meta; -export default meta; -type Story = StoryObj;` - : `const meta: Meta = { component: Button, args: { onClick: fn() } }; // type Meta and StoryObj as other story files do -export default meta; -type Story = StoryObj;`; - const example = typescript - ? `import type { Meta, StoryObj } from '${framework}'; -import { expect, fn } from 'storybook/test'; - -import { Button } from './Button'; - -${typedMeta} - -export const Primary: Story = { args: { label: 'Save' } }; - -export const Disabled: Story = {` - : `import { expect, fn } from 'storybook/test'; - -import { Button } from './Button'; - -export default { component: Button, args: { onClick: fn() } }; - -export const Primary = { args: { label: 'Save' } }; - -export const Disabled = {`; - return `New story files use CSF 3: - -\`\`\`${typescript ? 'ts' : 'js'} -${example} - args: { ...Primary.args, disabled: true }, - play: async ({ canvas, userEvent, args }) => { - await userEvent.click(canvas.getByRole('button', { name: 'Save' })); - await expect(args.onClick).not.toHaveBeenCalled(); - }, -}; -\`\`\` - -Do not switch \`${previewFile}\` to \`definePreview\` (CSF Factories) yourself; when a file already uses \`preview.meta()\`, keep that format.`; -} - -function rendererNote(renderer: string | undefined): string { - if (renderer === '@storybook/svelte') { - return '\n- Svelte: when the project has `*.stories.svelte` files, write those (`defineMeta` and ``, as the existing files do) instead of the format above.'; - } - if (renderer === '@storybook/web-components') { - return '\n- Web components: `canvas` queries do not reach into shadow DOM. Query through `element.shadowRoot`, or with `shadow-dom-testing-library` when the project has it.'; - } - return ''; -} - -function writeSection(inputs: StoriesSkillInputs): string { - const { previewFile } = inputs; +function writeSection({ framework, csfFactories, previewFile }: StoriesSkillInputs): string { + const format = csfFactories + ? `This project writes stories with CSF Factories, because \`${previewFile}\` uses \`definePreview\`: \`import preview from '../.storybook/preview'\` (the path to that file), then \`const meta = preview.meta({ component: Button })\` and \`export const Primary = meta.story({ args: { ... } })\`. Do not import \`Meta\` or \`StoryObj\`. Import` + : `Import \`Meta\` and \`StoryObj\` from \`${framework}\`, and`; return `## Write the component and its stories -Every component you create or change gets stories, in its existing story file when it has one. When you remove or rename a component, a prop or a state, update or delete the stories that used it. +Every component you create or change gets stories: one per distinct state it can reach (variants, loading, empty, error, disabled), with realistic props. Never export a story under the name of a global such as \`Error\`: export \`ErrorState\` and set \`name: 'Error'\`. An interactive component also gets a \`play\` function that drives it and asserts the visible result; for a callback passed as \`fn()\`, assert whether it was called, as that state expects. The rules below are for the stories you write; do not rewrite existing stories only to match them. -This project's own files come first. Existing story files show the format, location, naming and \`tags\`; \`${previewFile}\` and existing stories show the setup a component needs (providers, router, theme). Follow them, and use the rules below for what they do not show. A story file keeps the format it has: never mix two formats in one file, and do not rewrite existing stories only to match these rules. - -What to cover: one story per state that changes what the user sees or can do (variants, loading, empty, error, invalid input, disabled, permissions, long or missing content), not one per prop value. Name each story after its scenario and give it realistic data. An interactive component also gets \`play\` functions that use it as a user would (click, type, keyboard, submit) and assert what the user then sees; for an arg passed as \`fn()\`, assert that it was or was not called. Build with semantic elements and labels, so stories can be queried by role. - -${storyFormat(inputs)} - -- \`fn\`, \`expect\`, \`mocked\`, \`sb\` and \`screen\` come from \`storybook/test\`. In \`play\`, take \`canvas\` and \`userEvent\` from the context and query \`canvas\` by role or label; it already has the queries, so never \`within(canvas)\`. \`await\` every \`userEvent\` call and \`expect\`, and use \`findBy*\` or \`waitFor\` for what appears later. For content rendered outside the story (dialog, popover, toast), use \`screen\`. -- A component that needs a provider, router or theme gets \`decorators\`, in \`${previewFile}\` when every story needs it. Check what that file already provides before you add one. -- A story never reaches the network or another external service. Use the mocking this project already has (for example MSW handlers); otherwise mock the module that makes the call. -- To mock a module, register it once in \`${previewFile}\` with \`sb.mock(import('../src/api.${inputs.typescript ? 'ts' : 'js'}'), { spy: true })\`: the path is relative to that file, with its extension and without an alias. Then set the result per story in \`beforeEach\` with \`mocked(getUser).mockResolvedValue(...)\`. -- Do not name an export after a global the file may use, such as \`Error\`: export \`ErrorState\` and set \`name: 'Error'\`.${rendererNote(inputs.renderer)}`; +- ${format} \`fn\`, \`expect\`, \`mocked\` and \`sb\` from \`storybook/test\`. +- \`play: async ({ canvas, userEvent }) => { ... }\`: query \`canvas\` directly, by role or label. Never wrap it in \`within()\`. \`userEvent.click(element)\` takes no options. +- A story never calls a real service. Mock network requests with MSW when the project has it (\`msw-storybook-addon\` in \`${previewFile}\`): give the story \`beforeEach({ msw }) { msw.use(http.get('/api/users', () => HttpResponse.json([...]))) }\`, and leave the code that calls \`fetch\` unmocked. Only version 2 of that addon has no \`msw\` there: it takes \`parameters: { msw: { handlers: [...] } }\`. +- Mock a module only for what MSW cannot reach (no MSW in the project, or a dependency that is not a network call): register it in \`${previewFile}\` with \`sb.mock(import('../src/api.ts'), { spy: true })\` (path relative to that file, with its extension), then set the result per story in \`beforeEach\` with \`mocked(getUser).mockResolvedValue(...)\`.`; } -function testSection({ testSupported, a11yEnabled }: StoriesSkillInputs): string { - if (!testSupported) { - return `## Test - -Storybook has no test command in this project: \`@storybook/addon-vitest\` is not set up. When the project has its own way to run story tests (a \`package.json\` script, for example with \`@storybook/test-runner\`), run that. Otherwise say in your answer that the stories were not tested.`; - } +function testSection(a11yEnabled: boolean): string { const a11y = a11yEnabled ? '\n\nThe run also reports accessibility violations. Fix semantic ones yourself (roles, labels, alt text, keyboard access). For visual ones such as color contrast, do not change the design: describe the problem, offer two or three options and ask the user.' : ''; @@ -154,7 +52,7 @@ Storybook has no test command in this project: \`@storybook/addon-vitest\` is no ${ref('test.run')} --stories '[{"storyId":""}]' # ids: see "Find the stories"; leave out --stories to run every story \`\`\` -Run this after every change; a \`package.json\` script does not replace it for story tests. Use focused runs while iterating and one full run before you finish. Fix the cause of a failure, not the assertion, and rerun; never finish with failing tests that your change caused. Report failures that were there before your change, or that you cannot fix after a few attempts, instead of hiding them.${a11y}`; +Run this after every change, instead of a \`package.json\` test script. Use focused runs while iterating and one full run before you finish. Fix failures and rerun; never finish with failing tests.${a11y}`; } function discoverSection(inputs: StoriesSkillInputs): string { @@ -168,7 +66,7 @@ function discoverSection(inputs: StoriesSkillInputs): string { if (commands.length === 0) { return `## Find the stories -This project has no command that lists story ids. Select a story by its file and export instead: \`--stories '[{"absoluteStoryPath":"/abs/path/Button.stories.tsx","exportName":"Primary"}]'\`. Add \`"explicitStoryName"\` when the story sets a \`name\`.`; +This project has no command that lists story ids. Select a story by its file and export instead: \`--stories '[{"absoluteStoryPath":"/abs/path/Button.stories.tsx","exportName":"Primary"}]'\`.`; } const before = inputs.reviewEnabled ? 'before every review' : 'before you share links'; const sharedFile = '(design token, theme, util, hook)'; @@ -178,7 +76,7 @@ This project has no command that lists story ids. Select a story by its file and inputs.changeDetectionEnabled ? `When \`stories changed\` leaves out a file you touched, pass that file to \`find-by-component\`; for a shared file ${sharedFile}, which has no stories of its own, pass the components that use it.` : `A shared file ${sharedFile} has no stories of its own: pass the components that use it to \`find-by-component\`.`, - `\`find-by-component\` lists stories by distance, from the stories of the file itself up to the pages that use it${inputs.reviewEnabled ? '; those layers make good review collections' : ''}. When it reports stories hidden by \`maxDistance\`, rerun it with a higher \`--maxDistance\`.`, + 'When \`find-by-component\` reports stories hidden by \`maxDistance\`, rerun it with a higher \`--maxDistance\`.', ]; const guidance = [ `Run one of these ${before}, also when you already know the ids of the stories you wrote: they add the stories of other components that your change affects.`, @@ -210,11 +108,9 @@ ${ref('review.create')} --input '{ }' \`\`\` -Publish a review after every change the user can see, and again after each later change. It needs a running Storybook. Group the stories into two to five collections, from the changed component up to the pages that show it (one is enough when a single component is affected), and include every story you created. When the user asks to see or browse components or stories and no code changed, publish the same review with \`"changedFiles": []\`. Skip the review only when nothing visible changed and the user did not ask to see anything, or when the user asked for no review, and say that instead. - -In a follow-up review, keep the collections and their order stable. Add a story where the change should not be visible, as a control. Titles and rationales are plain text without em-dashes. To the user, say "group of stories", not "collection". +Publish a review after every change the user can see, and again after each later change. It needs a running Storybook. Group the stories into two to five collections, from the changed component up to the pages that show it (one is enough when a single component is affected), and include every story you created. When the user asks to see or browse components or stories and no code changed, publish the same review with \`"changedFiles": []\`. Skip the review only when nothing visible changed and the user did not ask to see anything, and say that instead. -Then do both things the command prints, every time: open the review in the in-app browser with a browser tool (skip this only when you have no such tool), and end your answer with the review section it describes. Do not list separate story links next to it.`; +Then do both things the command prints, every time: open the review in the in-app browser with a browser tool, and end your answer with the review section it gives you. Do not list separate story links next to it.`; } function previewSection(reviewEnabled: boolean): string { @@ -226,12 +122,12 @@ ${ref('stories.preview')} --stories '[{"storyId":""}]' # direct links to s ${usage} -Only for a look at one story while you iterate, or when the user asks for a direct link. It needs a running Storybook. Add \`"globals": {"theme": "dark"}\` or \`"props": {...}\` to an item to see the story under other settings. It does not replace the review, unless \`review create\` keeps failing after you fixed what it reported: then share these links and say why.` +Only for a look at one story while you iterate, or when the user asks for a direct link. It does not replace the review, unless \`review create\` keeps failing after you fixed what it reported: then share these links and say why.` : `## Finish with links ${usage} -After every change the user can see, end your answer with the links to the most relevant stories, at most five. It needs a running Storybook. Add \`"globals": {"theme": "dark"}\` or \`"props": {...}\` to an item to see the story under other settings.`; +After every change the user can see, end your answer with the links to the most relevant stories, at most five. It needs a running Storybook.`; } export function buildStoriesSkill(inputs: StoriesSkillInputs): string { @@ -241,7 +137,7 @@ export function buildStoriesSkill(inputs: StoriesSkillInputs): string { Follow this for every change to how the UI looks (components, stories, styles, themes, tokens) and for requests to show components or stories. Some commands print what to do next; follow that too. Add \`--help\` to a command only when a call fails.`, inputs.docsEnabled && docsSection(), writeSection(inputs), - testSection(inputs), + inputs.testSupported && testSection(inputs.a11yEnabled), discoverSection(inputs), inputs.reviewEnabled && reviewSection(), previewSection(inputs.reviewEnabled), diff --git a/code/core/src/cli/skills/inputs.test.ts b/code/core/src/cli/skills/inputs.test.ts index edbf4f5b6c5b..06adcb2e5832 100644 --- a/code/core/src/cli/skills/inputs.test.ts +++ b/code/core/src/cli/skills/inputs.test.ts @@ -124,11 +124,10 @@ describe('resolveSkillInputs', () => { expect(inputs).toMatchObject({ csfFactories: true, previewFile: '.storybook/preview.tsx', - typescript: true, }); }); - it('reads a plain JavaScript preview as CSF 3 without TypeScript', async () => { + it('reads a plain JavaScript preview as CSF 3', async () => { vol.fromNestedJSON({ '/project/.storybook/preview.js': 'export default { parameters: {} };', }); @@ -140,7 +139,6 @@ describe('resolveSkillInputs', () => { expect(inputs).toMatchObject({ csfFactories: false, previewFile: '.storybook/preview.js', - typescript: false, }); }); @@ -154,7 +152,6 @@ describe('resolveSkillInputs', () => { expect(inputs).toMatchObject({ csfFactories: false, previewFile: '.storybook/preview.ts', - typescript: true, }); }); }); diff --git a/code/core/src/cli/skills/inputs.ts b/code/core/src/cli/skills/inputs.ts index b8bc920dbfb5..869c5993b224 100644 --- a/code/core/src/cli/skills/inputs.ts +++ b/code/core/src/cli/skills/inputs.ts @@ -19,7 +19,6 @@ export type SkillInputs = ToolAvailability & { csfFactories: boolean; /** Path of the preview file relative to the working directory; a default name when there is none. */ previewFile: string; - typescript: boolean; }; const isTypeScriptFile = (path: string) => /\.[cm]?tsx?$/.test(path); @@ -39,7 +38,6 @@ async function resolvePreview(configDir = '.storybook') { return { csfFactories, previewFile: relative(process.cwd(), path).replaceAll('\\', '/') || path, - typescript, }; } diff --git a/code/core/src/cli/skills/run.test.ts b/code/core/src/cli/skills/run.test.ts index c34bf44209b6..e5c16679ad0b 100644 --- a/code/core/src/cli/skills/run.test.ts +++ b/code/core/src/cli/skills/run.test.ts @@ -29,7 +29,6 @@ const deps = () => ({ renderer: '@storybook/react', csfFactories: false, previewFile: '.storybook/preview.ts', - typescript: true, changeDetectionEnabled: true, moduleGraphSupported: true, reviewEnabled: true, diff --git a/code/core/src/cli/skills/run.ts b/code/core/src/cli/skills/run.ts index 6063f152dba4..6f32a8df7907 100644 --- a/code/core/src/cli/skills/run.ts +++ b/code/core/src/cli/skills/run.ts @@ -204,10 +204,8 @@ function assemble(id: Exclude, inputs: SkillInputs): string { if (id === 'stories') { return buildStoriesSkill({ framework: inputs.framework, - renderer: inputs.renderer, csfFactories: inputs.csfFactories, previewFile: inputs.previewFile, - typescript: inputs.typescript, docsEnabled: inputs.docsEnabledForCli, testSupported: inputs.testSupported, a11yEnabled: inputs.a11yEnabled, From 893635196cde88471b470b5f1a9f4b6359d5f9f8 Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Wed, 7 Oct 2026 15:18:54 +0200 Subject: [PATCH 17/32] test(agent-eval): skip the test-run assertion for GPT-6 Luna Codex gives GPT-6 Luna, and no other model in its catalog, a system prompt that says not to run tests unless the user asks. Luna follows it over the stories skill, so the seven evals that check a test run after a UI change skip that assertion for Luna instead of the skill overruling the vendor's prompt. Experiments now pass their model to the eval context, and Sonnet 5.5 low and GPT-6 Luna low experiments are added. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../801-create-accessible-component/EVAL.ts | 10 +++++--- agent-eval/evals/802-create-component/EVAL.ts | 10 +++++--- agent-eval/evals/803-edit-component/EVAL.ts | 10 +++++--- .../EVAL.ts | 10 +++++--- .../evals/808-shared-infra-fallback/EVAL.ts | 3 ++- .../812-first-story-empty-project/EVAL.ts | 10 +++++--- .../EVAL.ts | 10 +++++--- .../experiments/cc-mcp-opus-5.5-medium.ts | 6 +++-- .../experiments/cc-plugin-opus-5.5-medium.ts | 6 +++-- .../experiments/cc-plugin-sonnet-5.5-low.ts | 23 +++++++++++++++++ .../codex-mcp-gpt-6.1-sol-medium.ts | 6 +++-- .../codex-plugin-gpt-6-luna-low.ts | 25 +++++++++++++++++++ .../codex-plugin-gpt-6.1-sol-medium.ts | 6 +++-- agent-eval/lib/templates.ts | 5 ++-- agent-eval/lib/test-utils.ts | 22 ++++++++++++++-- 15 files changed, 131 insertions(+), 31 deletions(-) create mode 100644 agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts create mode 100644 agent-eval/experiments/codex-plugin-gpt-6-luna-low.ts diff --git a/agent-eval/evals/801-create-accessible-component/EVAL.ts b/agent-eval/evals/801-create-accessible-component/EVAL.ts index 6f869f60d4f7..089384a69e2f 100644 --- a/agent-eval/evals/801-create-accessible-component/EVAL.ts +++ b/agent-eval/evals/801-create-accessible-component/EVAL.ts @@ -12,6 +12,7 @@ import { expectWorkflowCalls, getEvalContext, isReviewEnabled, + modelRunsTestsOnlyWhenAsked, } from '#test-utils'; import { transcript } from '@vercel/agent-eval/eval'; import { describe, expect, test } from 'vitest'; @@ -19,9 +20,12 @@ import { describe, expect, test } from 'vitest'; describe('creating an accessible ToggleSwitch', () => { const reviewEnabled = isReviewEnabled(); - test('runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ covering: ['toggleswitch'] }); - }); + test.skipIf(modelRunsTestsOnlyWhenAsked())( + 'runs story tests after the change and finishes with them passing', + async () => { + await expectStoryTestsRanAndPassed({ covering: ['toggleswitch'] }); + } + ); describe.runIf(reviewEnabled)('with review enabled', () => { test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/evals/802-create-component/EVAL.ts b/agent-eval/evals/802-create-component/EVAL.ts index dbc6334a8a3f..ca5b4f172426 100644 --- a/agent-eval/evals/802-create-component/EVAL.ts +++ b/agent-eval/evals/802-create-component/EVAL.ts @@ -11,15 +11,19 @@ import { expectWorkflowCalls, getEvalContext, isReviewEnabled, + modelRunsTestsOnlyWhenAsked, } from '#test-utils'; import { describe, test } from 'vitest'; describe('creating a ProfileCard component', () => { const review = isReviewEnabled(); - test('runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ covering: ['profilecard'] }); - }); + test.skipIf(modelRunsTestsOnlyWhenAsked())( + 'runs story tests after the change and finishes with them passing', + async () => { + await expectStoryTestsRanAndPassed({ covering: ['profilecard'] }); + } + ); describe.runIf(review)('when review is enabled', () => { test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/evals/803-edit-component/EVAL.ts b/agent-eval/evals/803-edit-component/EVAL.ts index 2fd1c342c76b..a0e7efed7215 100644 --- a/agent-eval/evals/803-edit-component/EVAL.ts +++ b/agent-eval/evals/803-edit-component/EVAL.ts @@ -11,15 +11,19 @@ import { expectWorkflowCalls, getEvalContext, isReviewEnabled, + modelRunsTestsOnlyWhenAsked, } from '#test-utils'; import { describe, test } from 'vitest'; describe('editing ReviewCard to add date and optional onReport', () => { const review = isReviewEnabled(); - test('runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ covering: ['reviewcard'] }); - }); + test.skipIf(modelRunsTestsOnlyWhenAsked())( + 'runs story tests after the change and finishes with them passing', + async () => { + await expectStoryTestsRanAndPassed({ covering: ['reviewcard'] }); + } + ); describe.runIf(review)('when review is enabled', () => { test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/evals/804-write-story-for-existing-component/EVAL.ts b/agent-eval/evals/804-write-story-for-existing-component/EVAL.ts index 497f4909d683..e7044afe5623 100644 --- a/agent-eval/evals/804-write-story-for-existing-component/EVAL.ts +++ b/agent-eval/evals/804-write-story-for-existing-component/EVAL.ts @@ -12,14 +12,18 @@ import { expectStoryTestsRanAndPassed, expectWorkflowCalls, isReviewEnabled, + modelRunsTestsOnlyWhenAsked, } from '#test-utils'; describe('writing stories for an existing AlertBanner', () => { const review = isReviewEnabled(); - test('runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ covering: ['alertbanner'] }); - }); + test.skipIf(modelRunsTestsOnlyWhenAsked())( + 'runs story tests after the change and finishes with them passing', + async () => { + await expectStoryTestsRanAndPassed({ covering: ['alertbanner'] }); + } + ); describe.runIf(review)('when review is enabled', () => { test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/evals/808-shared-infra-fallback/EVAL.ts b/agent-eval/evals/808-shared-infra-fallback/EVAL.ts index 525a2421942c..8494a03715ec 100644 --- a/agent-eval/evals/808-shared-infra-fallback/EVAL.ts +++ b/agent-eval/evals/808-shared-infra-fallback/EVAL.ts @@ -12,6 +12,7 @@ import { getWorkflowCalls, getWorkflowToolResults, isReviewEnabled, + modelRunsTestsOnlyWhenAsked, } from '#test-utils'; import { readFileSync } from 'node:fs'; import { describe, expect, test } from 'vitest'; @@ -39,7 +40,7 @@ describe('changing a shared accent token and surfacing consumer stories', () => expect(colors, 'Expected the old accent value #2563eb to be gone').not.toMatch(/#2563eb/i); }); - test.skipIf(codexMcpReviewGap)( + test.skipIf(codexMcpReviewGap || modelRunsTestsOnlyWhenAsked())( 'runs story tests after the change and finishes with them passing', async () => { await expectStoryTestsRanAndPassed({ covering: ['badge', 'statuspill'] }); diff --git a/agent-eval/evals/812-first-story-empty-project/EVAL.ts b/agent-eval/evals/812-first-story-empty-project/EVAL.ts index 9bced87a5ee6..783e288fcd67 100644 --- a/agent-eval/evals/812-first-story-empty-project/EVAL.ts +++ b/agent-eval/evals/812-first-story-empty-project/EVAL.ts @@ -13,14 +13,18 @@ import { expectStoryTestsRanAndPassed, expectWorkflowCalls, isReviewEnabled, + modelRunsTestsOnlyWhenAsked, } from '#test-utils'; describe('writing the first Button stories in an empty Storybook', () => { const review = isReviewEnabled(); - test('runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ covering: ['button'] }); - }); + test.skipIf(modelRunsTestsOnlyWhenAsked())( + 'runs story tests after the change and finishes with them passing', + async () => { + await expectStoryTestsRanAndPassed({ covering: ['button'] }); + } + ); describe.runIf(review)('when review is enabled', () => { test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/evals/813-monorepo-leaf-create-component/EVAL.ts b/agent-eval/evals/813-monorepo-leaf-create-component/EVAL.ts index 2fc1e458e6d3..afb87b596bca 100644 --- a/agent-eval/evals/813-monorepo-leaf-create-component/EVAL.ts +++ b/agent-eval/evals/813-monorepo-leaf-create-component/EVAL.ts @@ -13,6 +13,7 @@ import { expectStoryTestsRanAndPassed, expectWorkflowCalls, isReviewEnabled, + modelRunsTestsOnlyWhenAsked, } from '#test-utils'; describe('creating a Callout in a monorepo UI package', () => { @@ -30,9 +31,12 @@ describe('creating a Callout in a monorepo UI package', () => { ).toBe(true); }); - test('runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ covering: ['callout'], cwd: 'packages/ui' }); - }); + test.skipIf(modelRunsTestsOnlyWhenAsked())( + 'runs story tests after the change and finishes with them passing', + async () => { + await expectStoryTestsRanAndPassed({ covering: ['callout'], cwd: 'packages/ui' }); + } + ); describe.runIf(review)('when review is enabled', () => { test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/experiments/cc-mcp-opus-5.5-medium.ts b/agent-eval/experiments/cc-mcp-opus-5.5-medium.ts index eaf9a953aad3..5a263c0b167a 100644 --- a/agent-eval/experiments/cc-mcp-opus-5.5-medium.ts +++ b/agent-eval/experiments/cc-mcp-opus-5.5-medium.ts @@ -6,14 +6,16 @@ import { writeClaudeMcpConfig, } from '../lib/templates.ts'; +const model = 'claude-opus-5-5'; + export default { ...DEFAULT_EXPERIMENT_CONFIG, agent: 'claude-code', // direct Anthropic API, requires ANTHROPIC_API_KEY - model: 'claude-opus-5-5', + model, agentOptions: { effort: 'medium' }, evals: WORKFLOW_STORYBOOK_EVALS, setup: async (sandbox) => { - await setupSandbox(sandbox, { agent: 'claude-code', integration: 'mcp' }); + await setupSandbox(sandbox, { agent: 'claude-code', integration: 'mcp', model }); await writeClaudeMcpConfig(sandbox); await writeClaudeInAppBrowserMock(sandbox); }, diff --git a/agent-eval/experiments/cc-plugin-opus-5.5-medium.ts b/agent-eval/experiments/cc-plugin-opus-5.5-medium.ts index 005ef58ad7d0..dbd78b3c0c53 100644 --- a/agent-eval/experiments/cc-plugin-opus-5.5-medium.ts +++ b/agent-eval/experiments/cc-plugin-opus-5.5-medium.ts @@ -6,15 +6,17 @@ import { writeClaudePluginSkills, } from '../lib/templates.ts'; +const model = 'claude-opus-5-5'; + export default { ...DEFAULT_EXPERIMENT_CONFIG, agent: 'claude-code', // direct Anthropic API, requires ANTHROPIC_API_KEY - model: 'claude-opus-5-5', + model, agentOptions: { effort: 'medium' }, // Skipped under EVAL_STORYBOOK_LATEST=1; see PLUGIN_STORYBOOK_EVALS. evals: PLUGIN_STORYBOOK_EVALS, setup: async (sandbox) => { - await setupSandbox(sandbox, { agent: 'claude-code', integration: 'plugin' }); + await setupSandbox(sandbox, { agent: 'claude-code', integration: 'plugin', model }); await writeClaudePluginSkills(sandbox); await writeClaudeInAppBrowserMock(sandbox); }, diff --git a/agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts b/agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts new file mode 100644 index 000000000000..333864fef185 --- /dev/null +++ b/agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts @@ -0,0 +1,23 @@ +import type { ExperimentConfig } from '@vercel/agent-eval'; +import { DEFAULT_EXPERIMENT_CONFIG, PLUGIN_STORYBOOK_EVALS } from '../lib/experiment.ts'; +import { + setupSandbox, + writeClaudeInAppBrowserMock, + writeClaudePluginSkills, +} from '../lib/templates.ts'; + +const model = 'claude-sonnet-5-5'; + +export default { + ...DEFAULT_EXPERIMENT_CONFIG, + agent: 'claude-code', // direct Anthropic API, requires ANTHROPIC_API_KEY + model, + agentOptions: { effort: 'low' }, + // Skipped under EVAL_STORYBOOK_LATEST=1; see PLUGIN_STORYBOOK_EVALS. + evals: PLUGIN_STORYBOOK_EVALS, + setup: async (sandbox) => { + await setupSandbox(sandbox, { agent: 'claude-code', integration: 'plugin', model }); + await writeClaudePluginSkills(sandbox); + await writeClaudeInAppBrowserMock(sandbox); + }, +} satisfies ExperimentConfig; diff --git a/agent-eval/experiments/codex-mcp-gpt-6.1-sol-medium.ts b/agent-eval/experiments/codex-mcp-gpt-6.1-sol-medium.ts index b5722002a992..7bcb8d513b3d 100644 --- a/agent-eval/experiments/codex-mcp-gpt-6.1-sol-medium.ts +++ b/agent-eval/experiments/codex-mcp-gpt-6.1-sol-medium.ts @@ -7,16 +7,18 @@ import { writeCodexMcpConfig, } from '../lib/templates.ts'; +const model = 'gpt-6.1-sol?reasoningEffort=medium'; + export default { ...DEFAULT_EXPERIMENT_CONFIG, // Use direct Codex for MCP evals. The AI Gateway Codex path does not reliably // handle Codex's Responses namespace tool shape yet: // https://github.com/openai/codex/issues/26234 agent: 'codex', - model: 'gpt-6.1-sol?reasoningEffort=medium', + model, evals: WORKFLOW_STORYBOOK_EVALS, setup: async (sandbox) => { - await setupSandbox(sandbox, { agent: 'codex', integration: 'mcp' }); + await setupSandbox(sandbox, { agent: 'codex', integration: 'mcp', model }); await writeCodexMcpConfig(sandbox); await writeCodexAgentsMd(sandbox); await writeCodexInAppBrowserMock(sandbox); diff --git a/agent-eval/experiments/codex-plugin-gpt-6-luna-low.ts b/agent-eval/experiments/codex-plugin-gpt-6-luna-low.ts new file mode 100644 index 000000000000..4c580d546696 --- /dev/null +++ b/agent-eval/experiments/codex-plugin-gpt-6-luna-low.ts @@ -0,0 +1,25 @@ +import type { ExperimentConfig } from '@vercel/agent-eval'; +import { DEFAULT_EXPERIMENT_CONFIG, PLUGIN_STORYBOOK_EVALS } from '../lib/experiment.ts'; +import { + setupSandbox, + writeCodexInAppBrowserMock, + writeCodexPluginSkills, +} from '../lib/templates.ts'; + +const model = 'gpt-6-luna?reasoningEffort=low'; + +export default { + ...DEFAULT_EXPERIMENT_CONFIG, + // Keep Codex plugin and MCP experiments on the same direct Codex runner. + // The MCP variant cannot use the AI Gateway path yet: + // https://github.com/openai/codex/issues/26234 + agent: 'codex', + model, + // Skipped under EVAL_STORYBOOK_LATEST=1; see PLUGIN_STORYBOOK_EVALS. + evals: PLUGIN_STORYBOOK_EVALS, + setup: async (sandbox) => { + await setupSandbox(sandbox, { agent: 'codex', integration: 'plugin', model }); + await writeCodexPluginSkills(sandbox); + await writeCodexInAppBrowserMock(sandbox); + }, +} satisfies ExperimentConfig; diff --git a/agent-eval/experiments/codex-plugin-gpt-6.1-sol-medium.ts b/agent-eval/experiments/codex-plugin-gpt-6.1-sol-medium.ts index 6768399aa557..77dfa4646a77 100644 --- a/agent-eval/experiments/codex-plugin-gpt-6.1-sol-medium.ts +++ b/agent-eval/experiments/codex-plugin-gpt-6.1-sol-medium.ts @@ -6,17 +6,19 @@ import { writeCodexPluginSkills, } from '../lib/templates.ts'; +const model = 'gpt-6.1-sol?reasoningEffort=medium'; + export default { ...DEFAULT_EXPERIMENT_CONFIG, // Keep Codex plugin and MCP experiments on the same direct Codex runner. // The MCP variant cannot use the AI Gateway path yet: // https://github.com/openai/codex/issues/26234 agent: 'codex', - model: 'gpt-6.1-sol?reasoningEffort=medium', + model, // Skipped under EVAL_STORYBOOK_LATEST=1; see PLUGIN_STORYBOOK_EVALS. evals: PLUGIN_STORYBOOK_EVALS, setup: async (sandbox) => { - await setupSandbox(sandbox, { agent: 'codex', integration: 'plugin' }); + await setupSandbox(sandbox, { agent: 'codex', integration: 'plugin', model }); await writeCodexPluginSkills(sandbox); await writeCodexInAppBrowserMock(sandbox); }, diff --git a/agent-eval/lib/templates.ts b/agent-eval/lib/templates.ts index 3c1d848cc49d..9ffe28df21b4 100644 --- a/agent-eval/lib/templates.ts +++ b/agent-eval/lib/templates.ts @@ -149,7 +149,7 @@ const CODEX_PLUGIN_SKILLS_DIR = path.join( export async function setupSandbox( sandbox: Sandbox, - options: { agent: EvalAgent; integration: EvalIntegration } + options: { agent: EvalAgent; integration: EvalIntegration; model?: string } ): Promise { await writeEvalSupportFiles(sandbox, options); @@ -222,7 +222,7 @@ export async function setupSandbox( async function writeEvalSupportFiles( sandbox: Sandbox, - options: { agent: EvalAgent; integration: EvalIntegration } + options: { agent: EvalAgent; integration: EvalIntegration; model?: string } ): Promise { await sandbox.writeFiles({ [TRANSCRIPT_HELPER_SANDBOX_PATH]: await fs.readFile(TRANSCRIPT_HELPER_SOURCE_PATH, 'utf8'), @@ -231,6 +231,7 @@ async function writeEvalSupportFiles( [AGENT_CONTEXT_SANDBOX_PATH]: JSON.stringify( { agent: options.agent, + model: options.model, integration: options.integration, review: isReviewEnabledFor(options.integration), }, diff --git a/agent-eval/lib/test-utils.ts b/agent-eval/lib/test-utils.ts index 8167ce4f39ee..abad26c0bf13 100644 --- a/agent-eval/lib/test-utils.ts +++ b/agent-eval/lib/test-utils.ts @@ -29,12 +29,15 @@ const TRANSCRIPT_PATH = '__agent_eval__/transcript.txt'; type AgentContext = { agent?: unknown; + model?: unknown; integration?: unknown; review?: unknown; }; type EvalContext = { agent: string; + /** The experiment's model id, such as `gpt-6-luna?reasoningEffort=low`; absent when the experiment does not pass one. */ + model?: string; // 'none' is the agentic-reference bare control (no Storybook tooling flavor); // see lib/templates.ts. Plugin-only helpers below treat it as "not plugin". integration: 'mcp' | 'plugin' | 'none'; @@ -60,7 +63,7 @@ export type StoryInputExpectation = { export function getEvalContext(): EvalContext { const agentContext = readAgentContext(); - const { agent, integration } = agentContext; + const { agent, model, integration } = agentContext; if (typeof agent !== 'string') { throw new Error( @@ -77,7 +80,22 @@ export function getEvalContext(): EvalContext { ); } - return { agent, integration, review: agentContext.review === true }; + return { + agent, + ...(typeof model === 'string' && { model }), + integration, + review: agentContext.review === true, + }; +} + +// The Codex system prompt of GPT-6 Luna, and of no other model in Codex's +// model catalog (codex-cli 0.160.1), says "Do not add or run tests unless the +// user asks you to test or verify implementation." at every reasoning effort, +// and Luna follows it over the stories skill. That is the model vendor's +// choice, so evals skip their test-run assertion for Luna rather than having +// the skill overrule the system prompt. +export function modelRunsTestsOnlyWhenAsked(): boolean { + return getEvalContext().model?.startsWith('gpt-6-luna') === true; } // Review mode of this run (see isReviewEnabledFor in lib/templates.ts). From 25a38fd18da444c2aca87d70791d6cfb97dc84f3 Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Wed, 7 Oct 2026 15:25:52 +0200 Subject: [PATCH 18/32] fix(skills): require the commands that find affected stories before a review The stories workflow said to run one of three commands before a review, and one of them was docs list --withStoryIds, which lists every story without finding the ones a change affects. Sonnet 5.5 at low effort took that one for a shared token and picked the consumer stories by grep. Name stories changed and find-by-component instead; with that, Sonnet passed eval 808 in four of four runs. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/cli/skills/content/build-stories-skill.test.ts | 8 ++++++++ code/core/src/cli/skills/content/build-stories-skill.ts | 8 +++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/code/core/src/cli/skills/content/build-stories-skill.test.ts b/code/core/src/cli/skills/content/build-stories-skill.test.ts index 36fc5128b5c0..bcdbb92e4e5a 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.test.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.test.ts @@ -44,6 +44,14 @@ describe('buildStoriesSkill', () => { expect(text).not.toContain('accessibility'); }); + it('requires the commands that find affected stories before a review, not the full story list', () => { + const text = buildStoriesSkill(everything); + + expect(text).toContain( + 'Run `stories changed` or `find-by-component` before every review, also when you already know the ids' + ); + }); + it('selects stories by file and export when no command lists story ids', () => { const text = buildStoriesSkill({ ...everything, diff --git a/code/core/src/cli/skills/content/build-stories-skill.ts b/code/core/src/cli/skills/content/build-stories-skill.ts index bc7fd9e314f8..29d00f9d16cd 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.ts @@ -78,8 +78,14 @@ This project has no command that lists story ids. Select a story by its file and : `A shared file ${sharedFile} has no stories of its own: pass the components that use it to \`find-by-component\`.`, 'When \`find-by-component\` reports stories hidden by \`maxDistance\`, rerun it with a higher \`--maxDistance\`.', ]; + const affected = [ + inputs.changeDetectionEnabled && '`stories changed`', + inputs.moduleGraphSupported && '`find-by-component`', + ].filter(Boolean); const guidance = [ - `Run one of these ${before}, also when you already know the ids of the stories you wrote: they add the stories of other components that your change affects.`, + affected.length > 0 + ? `Run ${affected.join(' or ')} ${before}, also when you already know the ids of the stories you wrote: ${affected.length > 1 ? 'they add' : 'it adds'} the stories of other components that your change affects.` + : `Run it ${before}.`, 'Story ids come only from these commands. Never build one from a file name, a title or memory.', ...fallback, 'When none of them finds a story for a component, it has no stories yet: say so, or write them.', From 06b0c0824c46b78c16dfb879590a41a437222e50 Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Wed, 7 Oct 2026 15:39:42 +0200 Subject: [PATCH 19/32] fix(skills): look up every library component the agent uses The stories workflow said to run docs show for each component you build on. Sonnet 5.5 at low effort did not count a design-system Button it only added, skipped docs show, and failed eval 803 in three of three runs, where next's text passed three of three. Ask for docs show on every listed component the agent uses, and for only the props it documents; Sonnet then passed 801 and 803 in six of six runs. Co-Authored-By: Claude Opus 5.5 (1M context) --- code/core/src/cli/skills/content/build-stories-skill.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/core/src/cli/skills/content/build-stories-skill.ts b/code/core/src/cli/skills/content/build-stories-skill.ts index 29d00f9d16cd..683c3f47d66f 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.ts @@ -25,7 +25,7 @@ ${ref('docs.show')} --id # props and usage examples of one entry ${ref('docs.showStory')} --storyId # the code of a story that docs show only lists \`\`\` -Run \`docs list\` once at the start, then \`docs show\` for each component you build on or are asked about. Reuse what exists instead of building a duplicate. Answer props, API and usage questions from these commands and never invent a prop. Read source files or \`node_modules\` only when the commands return nothing relevant. When \`docs list\` groups its entries under sources (\`id: acme\`), pass the source of the entry to both commands that show it: \`--storybookId acme\`.`; +Run \`docs list\` once at the start, then \`docs show\` for every listed component you use, even one you only add such as a button, and for each one you are asked about. Reuse what exists instead of building a duplicate. Use only the props that \`docs show\` documents, never one you assume from a name or another library, and answer props, API and usage questions from these commands. Read source files or \`node_modules\` only when the commands return nothing relevant. When \`docs list\` groups its entries under sources (\`id: acme\`), pass the source of the entry to both commands that show it: \`--storybookId acme\`.`; } function writeSection({ framework, csfFactories, previewFile }: StoriesSkillInputs): string { From 98f5fafaa4e5e5291c15715eae6e852c093550fe Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Wed, 7 Oct 2026 15:53:15 +0200 Subject: [PATCH 20/32] fix(agent-eval): keep the harness's own story-test run for GPT-6 Luna Skipping the whole test-run test for Luna also skipped the harness's own run of the finished stories, so a Luna run with a failing story could pass. expectStoryTestsRanAndPassed now takes requireAgentRun, which drops only the checks on the agent's own test run; the seven evals pass false for Luna, and 810 and 811, where the user asks for tests, still require the agent's run. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../801-create-accessible-component/EVAL.ts | 12 ++++++------ agent-eval/evals/802-create-component/EVAL.ts | 12 ++++++------ agent-eval/evals/803-edit-component/EVAL.ts | 12 ++++++------ .../EVAL.ts | 12 ++++++------ .../evals/808-shared-infra-fallback/EVAL.ts | 7 +++++-- .../812-first-story-empty-project/EVAL.ts | 12 ++++++------ .../EVAL.ts | 13 +++++++------ agent-eval/lib/test-utils.ts | 19 ++++++++++++------- 8 files changed, 54 insertions(+), 45 deletions(-) diff --git a/agent-eval/evals/801-create-accessible-component/EVAL.ts b/agent-eval/evals/801-create-accessible-component/EVAL.ts index 089384a69e2f..0bad1107bf92 100644 --- a/agent-eval/evals/801-create-accessible-component/EVAL.ts +++ b/agent-eval/evals/801-create-accessible-component/EVAL.ts @@ -20,12 +20,12 @@ import { describe, expect, test } from 'vitest'; describe('creating an accessible ToggleSwitch', () => { const reviewEnabled = isReviewEnabled(); - test.skipIf(modelRunsTestsOnlyWhenAsked())( - 'runs story tests after the change and finishes with them passing', - async () => { - await expectStoryTestsRanAndPassed({ covering: ['toggleswitch'] }); - } - ); + test('runs story tests after the change and finishes with them passing', async () => { + await expectStoryTestsRanAndPassed({ + requireAgentRun: !modelRunsTestsOnlyWhenAsked(), + covering: ['toggleswitch'], + }); + }); describe.runIf(reviewEnabled)('with review enabled', () => { test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/evals/802-create-component/EVAL.ts b/agent-eval/evals/802-create-component/EVAL.ts index ca5b4f172426..46c3d34fdc7a 100644 --- a/agent-eval/evals/802-create-component/EVAL.ts +++ b/agent-eval/evals/802-create-component/EVAL.ts @@ -18,12 +18,12 @@ import { describe, test } from 'vitest'; describe('creating a ProfileCard component', () => { const review = isReviewEnabled(); - test.skipIf(modelRunsTestsOnlyWhenAsked())( - 'runs story tests after the change and finishes with them passing', - async () => { - await expectStoryTestsRanAndPassed({ covering: ['profilecard'] }); - } - ); + test('runs story tests after the change and finishes with them passing', async () => { + await expectStoryTestsRanAndPassed({ + requireAgentRun: !modelRunsTestsOnlyWhenAsked(), + covering: ['profilecard'], + }); + }); describe.runIf(review)('when review is enabled', () => { test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/evals/803-edit-component/EVAL.ts b/agent-eval/evals/803-edit-component/EVAL.ts index a0e7efed7215..6ba236eb8af2 100644 --- a/agent-eval/evals/803-edit-component/EVAL.ts +++ b/agent-eval/evals/803-edit-component/EVAL.ts @@ -18,12 +18,12 @@ import { describe, test } from 'vitest'; describe('editing ReviewCard to add date and optional onReport', () => { const review = isReviewEnabled(); - test.skipIf(modelRunsTestsOnlyWhenAsked())( - 'runs story tests after the change and finishes with them passing', - async () => { - await expectStoryTestsRanAndPassed({ covering: ['reviewcard'] }); - } - ); + test('runs story tests after the change and finishes with them passing', async () => { + await expectStoryTestsRanAndPassed({ + requireAgentRun: !modelRunsTestsOnlyWhenAsked(), + covering: ['reviewcard'], + }); + }); describe.runIf(review)('when review is enabled', () => { test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/evals/804-write-story-for-existing-component/EVAL.ts b/agent-eval/evals/804-write-story-for-existing-component/EVAL.ts index e7044afe5623..ee721c5568da 100644 --- a/agent-eval/evals/804-write-story-for-existing-component/EVAL.ts +++ b/agent-eval/evals/804-write-story-for-existing-component/EVAL.ts @@ -18,12 +18,12 @@ import { describe('writing stories for an existing AlertBanner', () => { const review = isReviewEnabled(); - test.skipIf(modelRunsTestsOnlyWhenAsked())( - 'runs story tests after the change and finishes with them passing', - async () => { - await expectStoryTestsRanAndPassed({ covering: ['alertbanner'] }); - } - ); + test('runs story tests after the change and finishes with them passing', async () => { + await expectStoryTestsRanAndPassed({ + requireAgentRun: !modelRunsTestsOnlyWhenAsked(), + covering: ['alertbanner'], + }); + }); describe.runIf(review)('when review is enabled', () => { test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/evals/808-shared-infra-fallback/EVAL.ts b/agent-eval/evals/808-shared-infra-fallback/EVAL.ts index 8494a03715ec..a4057491e93b 100644 --- a/agent-eval/evals/808-shared-infra-fallback/EVAL.ts +++ b/agent-eval/evals/808-shared-infra-fallback/EVAL.ts @@ -40,10 +40,13 @@ describe('changing a shared accent token and surfacing consumer stories', () => expect(colors, 'Expected the old accent value #2563eb to be gone').not.toMatch(/#2563eb/i); }); - test.skipIf(codexMcpReviewGap || modelRunsTestsOnlyWhenAsked())( + test.skipIf(codexMcpReviewGap)( 'runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ covering: ['badge', 'statuspill'] }); + await expectStoryTestsRanAndPassed({ + requireAgentRun: !modelRunsTestsOnlyWhenAsked(), + covering: ['badge', 'statuspill'], + }); } ); diff --git a/agent-eval/evals/812-first-story-empty-project/EVAL.ts b/agent-eval/evals/812-first-story-empty-project/EVAL.ts index 783e288fcd67..873bd2066173 100644 --- a/agent-eval/evals/812-first-story-empty-project/EVAL.ts +++ b/agent-eval/evals/812-first-story-empty-project/EVAL.ts @@ -19,12 +19,12 @@ import { describe('writing the first Button stories in an empty Storybook', () => { const review = isReviewEnabled(); - test.skipIf(modelRunsTestsOnlyWhenAsked())( - 'runs story tests after the change and finishes with them passing', - async () => { - await expectStoryTestsRanAndPassed({ covering: ['button'] }); - } - ); + test('runs story tests after the change and finishes with them passing', async () => { + await expectStoryTestsRanAndPassed({ + requireAgentRun: !modelRunsTestsOnlyWhenAsked(), + covering: ['button'], + }); + }); describe.runIf(review)('when review is enabled', () => { test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/evals/813-monorepo-leaf-create-component/EVAL.ts b/agent-eval/evals/813-monorepo-leaf-create-component/EVAL.ts index afb87b596bca..ad8fe2610dc5 100644 --- a/agent-eval/evals/813-monorepo-leaf-create-component/EVAL.ts +++ b/agent-eval/evals/813-monorepo-leaf-create-component/EVAL.ts @@ -31,12 +31,13 @@ describe('creating a Callout in a monorepo UI package', () => { ).toBe(true); }); - test.skipIf(modelRunsTestsOnlyWhenAsked())( - 'runs story tests after the change and finishes with them passing', - async () => { - await expectStoryTestsRanAndPassed({ covering: ['callout'], cwd: 'packages/ui' }); - } - ); + test('runs story tests after the change and finishes with them passing', async () => { + await expectStoryTestsRanAndPassed({ + requireAgentRun: !modelRunsTestsOnlyWhenAsked(), + covering: ['callout'], + cwd: 'packages/ui', + }); + }); describe.runIf(review)('when review is enabled', () => { test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/lib/test-utils.ts b/agent-eval/lib/test-utils.ts index abad26c0bf13..2f800dff1d5e 100644 --- a/agent-eval/lib/test-utils.ts +++ b/agent-eval/lib/test-utils.ts @@ -92,8 +92,8 @@ export function getEvalContext(): EvalContext { // model catalog (codex-cli 0.160.1), says "Do not add or run tests unless the // user asks you to test or verify implementation." at every reasoning effort, // and Luna follows it over the stories skill. That is the model vendor's -// choice, so evals skip their test-run assertion for Luna rather than having -// the skill overrule the system prompt. +// choice, so evals do not require Luna to run the tests itself rather than +// having the skill overrule the system prompt. export function modelRunsTestsOnlyWhenAsked(): boolean { return getEvalContext().model?.startsWith('gpt-6-luna') === true; } @@ -472,15 +472,20 @@ export type WorkflowToolResult = { // // `covering` requires one of the given substrings in the passing story ids. // `cwd` is the Storybook project, for fixtures where that is not the root. +// `requireAgentRun: false` drops the transcript checks and keeps the harness's +// own run, for a model that runs tests only when asked. export async function expectStoryTestsRanAndPassed(options?: { covering?: string[]; cwd?: string; + requireAgentRun?: boolean; }): Promise { - expectWorkflowCalls(['test-run']); - expect( - getWorkflowToolResults('test-run').at(-1)?.output ?? '', - "The agent's last test run must not report failing stories or unhandled errors" - ).not.toMatch(/## (Failing Stories|Unhandled Errors)/); + if (options?.requireAgentRun !== false) { + expectWorkflowCalls(['test-run']); + expect( + getWorkflowToolResults('test-run').at(-1)?.output ?? '', + "The agent's last test run must not report failing stories or unhandled errors" + ).not.toMatch(/## (Failing Stories|Unhandled Errors)/); + } const result = await runStoryTestsInSandbox(options?.cwd); expect( From aaabda44d11f34dcfcb3be967601768e1fc78b68 Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Wed, 7 Oct 2026 16:19:21 +0200 Subject: [PATCH 21/32] fix: apply the review of the Luna eval changes and the stories workflow - Keep checking that the agent's last test run did not end red when Luna is not required to run the tests, and test that path. - Shorten the comment on modelRunsTestsOnlyWhenAsked to its source. - Keep the Sonnet 5.5 low experiment out of CI; the Luna experiment stays, passed the lifecycle evals 820-823, and is listed in the README. - Speak of one command when only one command can find stories. Co-Authored-By: Claude Opus 5.5 (1M context) --- agent-eval/README.md | 5 ++-- .../experiments/cc-plugin-sonnet-5.5-low.ts | 23 ------------------- .../codex-plugin-gpt-6-luna-low.ts | 3 --- agent-eval/lib/test-utils.test.ts | 12 ++++++++++ agent-eval/lib/test-utils.ts | 22 ++++++++---------- .../content/build-stories-skill.test.ts | 13 +++++++++++ .../cli/skills/content/build-stories-skill.ts | 5 ++-- 7 files changed, 40 insertions(+), 43 deletions(-) delete mode 100644 agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts diff --git a/agent-eval/README.md b/agent-eval/README.md index dd3ed2c0782e..3c11971a2a1f 100644 --- a/agent-eval/README.md +++ b/agent-eval/README.md @@ -65,8 +65,8 @@ EVAL_ALL=1 yarn eval EVAL_ONLY=803-edit-component yarn eval ``` -A full `EVAL_ALL=1` run (12 workflow evals × 4 experiments + 4 -lifecycle evals × 2 plugin experiments) costs roughly **$30–45** in agent +A full `EVAL_ALL=1` run (12 workflow evals × 5 experiments + 4 +lifecycle evals × 3 plugin experiments) costs roughly **$30–45** in agent tokens at current per-run averages ($0.30–0.80 per workflow eval, $1–2 per lifecycle eval). The budget guardrail is **$75 per full run** — check the usage metadata in the results playground before growing the eval set past it @@ -137,6 +137,7 @@ via `ANTHROPIC_API_KEY`; Codex experiments use the direct Codex API via - `cc-plugin-opus-5.5-medium`: Claude Code (Opus 5.5 at medium effort) with Storybook plugin skills copied to `.claude/skills`. - `codex-mcp-gpt-6.1-sol-medium`: Codex (gpt-6.1-sol at medium reasoning effort) with project-local Storybook MCP config in `.codex/config.toml` and the Storybook MCP server instructions in `AGENTS.md` (none with `EVAL_STORYBOOK_LATEST=1`). - `codex-plugin-gpt-6.1-sol-medium`: Codex (gpt-6.1-sol at medium reasoning effort) with Storybook plugin skills copied to `.agents/skills`. +- `codex-plugin-gpt-6-luna-low`: Codex (gpt-6-luna at low reasoning effort) with Storybook plugin skills copied to `.agents/skills`. Its Codex system prompt says not to run tests unless asked, so evals do not require it to run them itself (see `modelRunsTestsOnlyWhenAsked` in `lib/test-utils.ts`). It costs about $0.01 per eval. ## Running evals in CI diff --git a/agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts b/agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts deleted file mode 100644 index 333864fef185..000000000000 --- a/agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { ExperimentConfig } from '@vercel/agent-eval'; -import { DEFAULT_EXPERIMENT_CONFIG, PLUGIN_STORYBOOK_EVALS } from '../lib/experiment.ts'; -import { - setupSandbox, - writeClaudeInAppBrowserMock, - writeClaudePluginSkills, -} from '../lib/templates.ts'; - -const model = 'claude-sonnet-5-5'; - -export default { - ...DEFAULT_EXPERIMENT_CONFIG, - agent: 'claude-code', // direct Anthropic API, requires ANTHROPIC_API_KEY - model, - agentOptions: { effort: 'low' }, - // Skipped under EVAL_STORYBOOK_LATEST=1; see PLUGIN_STORYBOOK_EVALS. - evals: PLUGIN_STORYBOOK_EVALS, - setup: async (sandbox) => { - await setupSandbox(sandbox, { agent: 'claude-code', integration: 'plugin', model }); - await writeClaudePluginSkills(sandbox); - await writeClaudeInAppBrowserMock(sandbox); - }, -} satisfies ExperimentConfig; diff --git a/agent-eval/experiments/codex-plugin-gpt-6-luna-low.ts b/agent-eval/experiments/codex-plugin-gpt-6-luna-low.ts index 4c580d546696..0d0fe84e11dd 100644 --- a/agent-eval/experiments/codex-plugin-gpt-6-luna-low.ts +++ b/agent-eval/experiments/codex-plugin-gpt-6-luna-low.ts @@ -10,9 +10,6 @@ const model = 'gpt-6-luna?reasoningEffort=low'; export default { ...DEFAULT_EXPERIMENT_CONFIG, - // Keep Codex plugin and MCP experiments on the same direct Codex runner. - // The MCP variant cannot use the AI Gateway path yet: - // https://github.com/openai/codex/issues/26234 agent: 'codex', model, // Skipped under EVAL_STORYBOOK_LATEST=1; see PLUGIN_STORYBOOK_EVALS. diff --git a/agent-eval/lib/test-utils.test.ts b/agent-eval/lib/test-utils.test.ts index 777897a6be1f..38cad74f808f 100644 --- a/agent-eval/lib/test-utils.test.ts +++ b/agent-eval/lib/test-utils.test.ts @@ -526,6 +526,18 @@ describe('expectStoryTestsRanAndPassed', () => { await expect(expectStoryTestsRanAndPassed()).rejects.toThrow(/Expected test-run to be called/); }); + + test('lets the agent skip the tests with requireAgentRun false, but still runs them in the sandbox', async () => { + givenRun({ + commands: ['npm run typecheck'], + stdout: testRunDocument({ 'example-badge--accent': 'status-value:success' }), + }); + const { expectStoryTestsRanAndPassed } = await loadTestUtils(); + + await expectStoryTestsRanAndPassed({ requireAgentRun: false, covering: ['badge'] }); + + expect(execFile).toHaveBeenCalled(); + }); }); describe('parseCodexBrowserNavigations', () => { diff --git a/agent-eval/lib/test-utils.ts b/agent-eval/lib/test-utils.ts index 2f800dff1d5e..caf474168e49 100644 --- a/agent-eval/lib/test-utils.ts +++ b/agent-eval/lib/test-utils.ts @@ -36,7 +36,7 @@ type AgentContext = { type EvalContext = { agent: string; - /** The experiment's model id, such as `gpt-6-luna?reasoningEffort=low`; absent when the experiment does not pass one. */ + // Such as `gpt-6-luna?reasoningEffort=low`; absent when the experiment passes none. model?: string; // 'none' is the agentic-reference bare control (no Storybook tooling flavor); // see lib/templates.ts. Plugin-only helpers below treat it as "not plugin". @@ -88,12 +88,8 @@ export function getEvalContext(): EvalContext { }; } -// The Codex system prompt of GPT-6 Luna, and of no other model in Codex's -// model catalog (codex-cli 0.160.1), says "Do not add or run tests unless the -// user asks you to test or verify implementation." at every reasoning effort, -// and Luna follows it over the stories skill. That is the model vendor's -// choice, so evals do not require Luna to run the tests itself rather than -// having the skill overrule the system prompt. +// Codex's system prompt for GPT-6 Luna alone says "Do not add or run tests unless the user asks +// you to test or verify implementation." export function modelRunsTestsOnlyWhenAsked(): boolean { return getEvalContext().model?.startsWith('gpt-6-luna') === true; } @@ -472,8 +468,8 @@ export type WorkflowToolResult = { // // `covering` requires one of the given substrings in the passing story ids. // `cwd` is the Storybook project, for fixtures where that is not the root. -// `requireAgentRun: false` drops the transcript checks and keeps the harness's -// own run, for a model that runs tests only when asked. +// `requireAgentRun: false` lets the agent skip the tests, for a model that runs +// them only when asked; a run it did make must still not end red. export async function expectStoryTestsRanAndPassed(options?: { covering?: string[]; cwd?: string; @@ -481,11 +477,11 @@ export async function expectStoryTestsRanAndPassed(options?: { }): Promise { if (options?.requireAgentRun !== false) { expectWorkflowCalls(['test-run']); - expect( - getWorkflowToolResults('test-run').at(-1)?.output ?? '', - "The agent's last test run must not report failing stories or unhandled errors" - ).not.toMatch(/## (Failing Stories|Unhandled Errors)/); } + expect( + getWorkflowToolResults('test-run').at(-1)?.output ?? '', + "The agent's last test run must not report failing stories or unhandled errors" + ).not.toMatch(/## (Failing Stories|Unhandled Errors)/); const result = await runStoryTestsInSandbox(options?.cwd); expect( diff --git a/code/core/src/cli/skills/content/build-stories-skill.test.ts b/code/core/src/cli/skills/content/build-stories-skill.test.ts index bcdbb92e4e5a..ceb7847c7575 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.test.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.test.ts @@ -50,6 +50,19 @@ describe('buildStoriesSkill', () => { expect(text).toContain( 'Run `stories changed` or `find-by-component` before every review, also when you already know the ids' ); + expect(text).not.toContain('Run one of these'); + }); + + it('speaks of one command when only the story list can find stories', () => { + const text = buildStoriesSkill({ + ...everything, + changeDetectionEnabled: false, + moduleGraphSupported: false, + reviewEnabled: false, + }); + + expect(text).toContain('Run it before you share links. Story ids come only from this command.'); + expect(text).toContain('When it does not find a story'); }); it('selects stories by file and export when no command lists story ids', () => { diff --git a/code/core/src/cli/skills/content/build-stories-skill.ts b/code/core/src/cli/skills/content/build-stories-skill.ts index 683c3f47d66f..efc32710295a 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.ts @@ -82,13 +82,14 @@ This project has no command that lists story ids. Select a story by its file and inputs.changeDetectionEnabled && '`stories changed`', inputs.moduleGraphSupported && '`find-by-component`', ].filter(Boolean); + const several = commands.length > 1; const guidance = [ affected.length > 0 ? `Run ${affected.join(' or ')} ${before}, also when you already know the ids of the stories you wrote: ${affected.length > 1 ? 'they add' : 'it adds'} the stories of other components that your change affects.` : `Run it ${before}.`, - 'Story ids come only from these commands. Never build one from a file name, a title or memory.', + `Story ids come only from ${several ? 'these commands' : 'this command'}. Never build one from a file name, a title or memory.`, ...fallback, - 'When none of them finds a story for a component, it has no stories yet: say so, or write them.', + `When ${several ? 'none of them finds' : 'it does not find'} a story for a component, it has no stories yet: say so, or write them.`, ].join(' '); return `## Find the stories From 197b4d49df2e1ec039ae0bfa489cbbc4ca8c63ac Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Wed, 7 Oct 2026 17:09:47 +0200 Subject: [PATCH 22/32] fix(skills): ask for story discovery at the review step, and fix two review findings - Sonnet 5.5 at low effort skipped stories changed before the review when it only wrote stories, taking the ids from its test run (eval 804: 2 of 4 runs, 0 of 3 with next's text). The review step now says to run the discovery command right before it; Sonnet then passed 804 and 806 in 6 of 6 runs. - A JavaScript project no longer gets the instruction to import the Meta and StoryObj types. - Opening the review in the in-app browser is required unless the agent has no browser tool. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../content/build-stories-skill.test.ts | 7 ++++++ .../cli/skills/content/build-stories-skill.ts | 23 +++++++++++++++---- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/code/core/src/cli/skills/content/build-stories-skill.test.ts b/code/core/src/cli/skills/content/build-stories-skill.test.ts index ceb7847c7575..1e2e79152e1b 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.test.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.test.ts @@ -98,6 +98,13 @@ describe('buildStoriesSkill', () => { expect(text).not.toContain('preview.meta({'); }); + it('leaves out the Meta and StoryObj types in a JavaScript project', () => { + const text = buildStoriesSkill({ ...everything, previewFile: '.storybook/preview.js' }); + + expect(text).not.toContain('StoryObj'); + expect(text).toContain('- Import `fn`, `expect`, `mocked` and `sb` from `storybook/test`.'); + }); + it('puts MSW before module mocks and names the preview file of the project', () => { const text = buildStoriesSkill({ ...everything, previewFile: 'config/preview.tsx' }); diff --git a/code/core/src/cli/skills/content/build-stories-skill.ts b/code/core/src/cli/skills/content/build-stories-skill.ts index efc32710295a..2003ecbe5a17 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.ts @@ -31,7 +31,9 @@ Run \`docs list\` once at the start, then \`docs show\` for every listed compone function writeSection({ framework, csfFactories, previewFile }: StoriesSkillInputs): string { const format = csfFactories ? `This project writes stories with CSF Factories, because \`${previewFile}\` uses \`definePreview\`: \`import preview from '../.storybook/preview'\` (the path to that file), then \`const meta = preview.meta({ component: Button })\` and \`export const Primary = meta.story({ args: { ... } })\`. Do not import \`Meta\` or \`StoryObj\`. Import` - : `Import \`Meta\` and \`StoryObj\` from \`${framework}\`, and`; + : /\.[cm]?tsx?$/.test(previewFile) + ? `Import \`Meta\` and \`StoryObj\` from \`${framework}\`, and` + : 'Import'; return `## Write the component and its stories Every component you create or change gets stories: one per distinct state it can reach (variants, loading, empty, error, disabled), with realistic props. Never export a story under the name of a global such as \`Error\`: export \`ErrorState\` and set \`name: 'Error'\`. An interactive component also gets a \`play\` function that drives it and asserts the visible result; for a callback passed as \`fn()\`, assert whether it was called, as that state expects. The rules below are for the stories you write; do not rewrite existing stories only to match them. @@ -100,7 +102,18 @@ ${commands.join('\n')} ${guidance}`; } -function reviewSection(): string { +function reviewSection({ + changeDetectionEnabled, + moduleGraphSupported, +}: StoriesSkillInputs): string { + const discovery = [ + changeDetectionEnabled && '`stories changed`', + moduleGraphSupported && '`find-by-component`', + ].filter(Boolean); + const discoverFirst = + discovery.length > 0 + ? ` Right before it, run ${discovery.join(' or ')} and take the story ids from there, also when you only wrote stories, not from the output of the tests.` + : ''; return `## Finish with a review \`\`\`sh @@ -115,9 +128,9 @@ ${ref('review.create')} --input '{ }' \`\`\` -Publish a review after every change the user can see, and again after each later change. It needs a running Storybook. Group the stories into two to five collections, from the changed component up to the pages that show it (one is enough when a single component is affected), and include every story you created. When the user asks to see or browse components or stories and no code changed, publish the same review with \`"changedFiles": []\`. Skip the review only when nothing visible changed and the user did not ask to see anything, and say that instead. +Publish a review after every change the user can see, and again after each later change. It needs a running Storybook.${discoverFirst} Group the stories into two to five collections, from the changed component up to the pages that show it (one is enough when a single component is affected), and include every story you created. When the user asks to see or browse components or stories and no code changed, publish the same review with \`"changedFiles": []\`. Skip the review only when nothing visible changed and the user did not ask to see anything, and say that instead. -Then do both things the command prints, every time: open the review in the in-app browser with a browser tool, and end your answer with the review section it gives you. Do not list separate story links next to it.`; +Then do both things the command prints, every time: open the review in the in-app browser with a browser tool (unless you have none), and end your answer with the review section it gives you. Do not list separate story links next to it.`; } function previewSection(reviewEnabled: boolean): string { @@ -146,7 +159,7 @@ Follow this for every change to how the UI looks (components, stories, styles, t writeSection(inputs), inputs.testSupported && testSection(inputs.a11yEnabled), discoverSection(inputs), - inputs.reviewEnabled && reviewSection(), + inputs.reviewEnabled && reviewSection(inputs), previewSection(inputs.reviewEnabled), ] .filter(Boolean) From 3e84e4a554e9f3882ef97c96b1c3c593cf825f47 Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Wed, 7 Oct 2026 18:33:40 +0200 Subject: [PATCH 23/32] fix(skills): route an outdated Storybook to the upgrade, and name docs questions in the stories trigger Three evals failed on every repeat with the plugin skills as they were: - 823 (set up a project that has Storybook 10) on Opus 5.5 and Sonnet 5.5, 0/3 each: the init skill said nothing about an existing Storybook, and the setup skill only upgraded after an explicit approval. Init now hands an existing Storybook to setup, and setup treats a setup request as approval, as the stories skill already did. 3/3 on both after. - 807 (a props question) on GPT-6-Luna low, 0/4: Luna answered from source without opening the skill. The description now names a question about a component before the browse cases. 3/3 after, twice. Co-Authored-By: Claude Opus 5.5 (1M context) --- code/lib/claude-plugin/skills/stories/SKILL.md | 2 +- code/lib/claude-plugin/skills/storybook-init/SKILL.md | 2 ++ code/lib/claude-plugin/skills/storybook-setup/SKILL.md | 2 +- code/lib/codex-plugin/plugins/storybook/skills/init/SKILL.md | 2 ++ code/lib/codex-plugin/plugins/storybook/skills/setup/SKILL.md | 2 +- code/lib/codex-plugin/plugins/storybook/skills/stories/SKILL.md | 2 +- 6 files changed, 8 insertions(+), 4 deletions(-) diff --git a/code/lib/claude-plugin/skills/stories/SKILL.md b/code/lib/claude-plugin/skills/stories/SKILL.md index e9b979035094..2e4790bb93f2 100644 --- a/code/lib/claude-plugin/skills/stories/SKILL.md +++ b/code/lib/claude-plugin/skills/stories/SKILL.md @@ -1,6 +1,6 @@ --- name: stories -description: Invoke FIRST, before creating, editing, or deleting components, stories, styles, CSS, themes, colors, or design tokens — anything that changes how the UI looks, no exceptions. Also for starting or previewing Storybook to verify UI, requests to show, browse, or list components, stories, or UI states, and docs, props, or usage lookups. +description: Invoke FIRST, before creating, editing, or deleting components, stories, styles, CSS, themes, colors, or design tokens — anything that changes how the UI looks, no exceptions. Also before answering a question about a component's props, API, or usage, to start or preview Storybook, and to show, browse, or list components, stories, or UI states. --- Prerequisites: diff --git a/code/lib/claude-plugin/skills/storybook-init/SKILL.md b/code/lib/claude-plugin/skills/storybook-init/SKILL.md index 3e270a63232a..f07b7ba88ee6 100644 --- a/code/lib/claude-plugin/skills/storybook-init/SKILL.md +++ b/code/lib/claude-plugin/skills/storybook-init/SKILL.md @@ -3,5 +3,7 @@ name: storybook-init description: Use when adding Storybook to a project that does not have Storybook configured yet. --- +If Storybook is already installed (`package.json`, `.storybook/`), do not install it again: switch to `/storybook-setup`. + 1. Run `npm create storybook@latest` inside your project's root directory to install the latest version of Storybook (if the latest stable release is below 11.0, run `npm create storybook@next` instead to get the 11.0 alpha). Use the matching package-manager command when appropriate, such as `pnpm create storybook@latest` or `yarn create storybook`. 2. Invoke the `/storybook-setup` skill to help the user set up project-specific Storybook configuration, such as the `.storybook/preview.ts` file. diff --git a/code/lib/claude-plugin/skills/storybook-setup/SKILL.md b/code/lib/claude-plugin/skills/storybook-setup/SKILL.md index 97f24d738cc9..6a5d791d0afb 100644 --- a/code/lib/claude-plugin/skills/storybook-setup/SKILL.md +++ b/code/lib/claude-plugin/skills/storybook-setup/SKILL.md @@ -6,7 +6,7 @@ description: Use this skill when Storybook is already installed and the user wan Prerequisites: 1. Confirm Storybook exists (`package.json`, `.storybook/`). If not, switch to `/storybook-init`. -2. Storybook must be 11.0 or later. An 11.0 prerelease (`11.0.0-alpha.x`) or a canary build (`0.0.0-pr-*`) also counts. If it is older, or upgrade/repair is needed first, switch to `/storybook-upgrade`, but only if the user explicitly approved a Storybook upgrade. +2. Storybook must be 11.0 or later. An 11.0 prerelease (`11.0.0-alpha.x`) or a canary build (`0.0.0-pr-*`) also counts. If it is older, or upgrade/repair is needed first, switch to `/storybook-upgrade` and come back here afterwards. Treat a request to set up or install Storybook as approval to perform any required Storybook upgrade. For other requests, switch only after the user explicitly approves an upgrade. Run `npx storybook skills setup` from the project root (or the Storybook package in a monorepo). diff --git a/code/lib/codex-plugin/plugins/storybook/skills/init/SKILL.md b/code/lib/codex-plugin/plugins/storybook/skills/init/SKILL.md index 6b06b1749c91..3407e293db0f 100644 --- a/code/lib/codex-plugin/plugins/storybook/skills/init/SKILL.md +++ b/code/lib/codex-plugin/plugins/storybook/skills/init/SKILL.md @@ -3,5 +3,7 @@ name: init description: Use when adding Storybook to a project that does not have Storybook configured yet. --- +If Storybook is already installed (`package.json`, `.storybook/`), do not install it again: switch to `$storybook:setup`. + 1. Run `npm create storybook@latest` inside your project's root directory to install the latest version of Storybook (if the latest stable release is below 11.0, run `npm create storybook@next` instead to get the 11.0 alpha). Use the matching package-manager command when appropriate, such as `pnpm create storybook@latest` or `yarn create storybook`. 2. Invoke the `$storybook:setup` skill to help the user set up project-specific Storybook configuration, such as the `.storybook/preview.ts` file. diff --git a/code/lib/codex-plugin/plugins/storybook/skills/setup/SKILL.md b/code/lib/codex-plugin/plugins/storybook/skills/setup/SKILL.md index 7197e5ee2a4b..54247ea6772d 100644 --- a/code/lib/codex-plugin/plugins/storybook/skills/setup/SKILL.md +++ b/code/lib/codex-plugin/plugins/storybook/skills/setup/SKILL.md @@ -6,7 +6,7 @@ description: Use this skill when Storybook is already installed and the user wan Prerequisites: 1. Confirm Storybook exists (`package.json`, `.storybook/`). If not, switch to `$storybook:init`. -2. Storybook must be 11.0 or later. An 11.0 prerelease (`11.0.0-alpha.x`) or a canary build (`0.0.0-pr-*`) also counts. If it is older, or upgrade/repair is needed first, switch to `$storybook:upgrade`, but only if the user explicitly approved a Storybook upgrade. +2. Storybook must be 11.0 or later. An 11.0 prerelease (`11.0.0-alpha.x`) or a canary build (`0.0.0-pr-*`) also counts. If it is older, or upgrade/repair is needed first, switch to `$storybook:upgrade` and come back here afterwards. Treat a request to set up or install Storybook as approval to perform any required Storybook upgrade. For other requests, switch only after the user explicitly approves an upgrade. Run `npx storybook skills setup` from the project root (or the Storybook package in a monorepo). diff --git a/code/lib/codex-plugin/plugins/storybook/skills/stories/SKILL.md b/code/lib/codex-plugin/plugins/storybook/skills/stories/SKILL.md index a46b5c0e9b47..c99a7929033f 100644 --- a/code/lib/codex-plugin/plugins/storybook/skills/stories/SKILL.md +++ b/code/lib/codex-plugin/plugins/storybook/skills/stories/SKILL.md @@ -1,6 +1,6 @@ --- name: stories -description: Invoke FIRST, before creating, editing, or deleting components, stories, styles, CSS, themes, colors, or design tokens — anything that changes how the UI looks, no exceptions. Also for starting or previewing Storybook to verify UI, requests to show, browse, or list components, stories, or UI states, and docs, props, or usage lookups. +description: Invoke FIRST, before creating, editing, or deleting components, stories, styles, CSS, themes, colors, or design tokens — anything that changes how the UI looks, no exceptions. Also before answering a question about a component's props, API, or usage, to start or preview Storybook, and to show, browse, or list components, stories, or UI states. --- Prerequisites: From f968f80e0d8015b3d0e0a073118d691978842f63 Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Thu, 8 Oct 2026 11:26:54 +0200 Subject: [PATCH 24/32] test(agent-eval): run the plugin experiments on Sonnet 5.5 low and GPT-6.1-Sol low The plugin skills have to work on the cheaper models at low effort, so those are the experiments that have to stay green. GPT-6 Luna low is no longer an experiment: it failed about one eval per round for a different reason each time, and its test exception in the evals goes with it. Co-Authored-By: Claude Opus 5.5 (1M context) --- agent-eval/README.md | 9 ++++---- .../801-create-accessible-component/EVAL.ts | 6 +---- agent-eval/evals/802-create-component/EVAL.ts | 6 +---- agent-eval/evals/803-edit-component/EVAL.ts | 6 +---- .../EVAL.ts | 6 +---- .../evals/808-shared-infra-fallback/EVAL.ts | 6 +---- .../812-first-story-empty-project/EVAL.ts | 6 +---- .../EVAL.ts | 7 +----- .../EVAL.ts | 6 +---- .../experiments/cc-mcp-opus-5.5-medium.ts | 6 ++--- ...-medium.ts => cc-plugin-sonnet-5.5-low.ts} | 8 +++---- .../codex-mcp-gpt-6.1-sol-medium.ts | 6 ++--- .../codex-plugin-gpt-6-luna-low.ts | 22 ------------------- ...ium.ts => codex-plugin-gpt-6.1-sol-low.ts} | 6 ++--- agent-eval/lib/templates.ts | 6 ++--- agent-eval/lib/test-utils.test.ts | 12 ---------- agent-eval/lib/test-utils.ts | 20 +++-------------- 17 files changed, 27 insertions(+), 117 deletions(-) rename agent-eval/experiments/{cc-plugin-opus-5.5-medium.ts => cc-plugin-sonnet-5.5-low.ts} (87%) delete mode 100644 agent-eval/experiments/codex-plugin-gpt-6-luna-low.ts rename agent-eval/experiments/{codex-plugin-gpt-6.1-sol-medium.ts => codex-plugin-gpt-6.1-sol-low.ts} (91%) diff --git a/agent-eval/README.md b/agent-eval/README.md index ae4a22369d3c..dc11e62941d1 100644 --- a/agent-eval/README.md +++ b/agent-eval/README.md @@ -65,8 +65,8 @@ EVAL_ALL=1 yarn eval EVAL_ONLY=803-edit-component yarn eval ``` -A full `EVAL_ALL=1` run (13 workflow evals × 5 experiments + 4 -lifecycle evals × 3 plugin experiments) costs roughly **$30–45** in agent +A full `EVAL_ALL=1` run (13 workflow evals × 4 experiments + 4 +lifecycle evals × 2 plugin experiments) costs roughly **$30–45** in agent tokens at current per-run averages ($0.30–0.80 per workflow eval, $1–2 per lifecycle eval). The budget guardrail is **$75 per full run** — check the usage metadata in the results playground before growing the eval set past it @@ -129,10 +129,9 @@ via `ANTHROPIC_API_KEY`; Codex experiments use the direct Codex API via `OPENAI_API_KEY`): - `cc-mcp-opus-5.5-medium`: Claude Code (Opus 5.5 at medium effort) with project-local Storybook MCP config in `.mcp.json`. -- `cc-plugin-opus-5.5-medium`: Claude Code (Opus 5.5 at medium effort) with Storybook plugin skills copied to `.claude/skills`. +- `cc-plugin-sonnet-5.5-low`: Claude Code (Sonnet 5.5 at low effort) with Storybook plugin skills copied to `.claude/skills`. - `codex-mcp-gpt-6.1-sol-medium`: Codex (gpt-6.1-sol at medium reasoning effort) with project-local Storybook MCP config in `.codex/config.toml` and the Storybook MCP server instructions in `AGENTS.md` (none with `EVAL_STORYBOOK_LATEST=1`). -- `codex-plugin-gpt-6.1-sol-medium`: Codex (gpt-6.1-sol at medium reasoning effort) with Storybook plugin skills copied to `.agents/skills`. -- `codex-plugin-gpt-6-luna-low`: Codex (gpt-6-luna at low reasoning effort) with Storybook plugin skills copied to `.agents/skills`. Its Codex system prompt says not to run tests unless asked, so evals do not require it to run them itself (see `modelRunsTestsOnlyWhenAsked` in `lib/test-utils.ts`). It costs about $0.01 per eval. +- `codex-plugin-gpt-6.1-sol-low`: Codex (gpt-6.1-sol at low reasoning effort) with Storybook plugin skills copied to `.agents/skills`. ## Running evals in CI diff --git a/agent-eval/evals/801-create-accessible-component/EVAL.ts b/agent-eval/evals/801-create-accessible-component/EVAL.ts index 4a0507be3c03..ad148e3a444a 100644 --- a/agent-eval/evals/801-create-accessible-component/EVAL.ts +++ b/agent-eval/evals/801-create-accessible-component/EVAL.ts @@ -9,17 +9,13 @@ import { expectStoryTestsRanAndPassed, expectWorkflowCalls, getEvalContext, - modelRunsTestsOnlyWhenAsked, } from '#test-utils'; import { transcript } from '@vercel/agent-eval/eval'; import { describe, expect, test } from 'vitest'; describe('creating an accessible ToggleSwitch', () => { test('runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ - requireAgentRun: !modelRunsTestsOnlyWhenAsked(), - covering: ['toggleswitch'], - }); + await expectStoryTestsRanAndPassed({ covering: ['toggleswitch'] }); }); test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/evals/802-create-component/EVAL.ts b/agent-eval/evals/802-create-component/EVAL.ts index 94da69f563f5..5ed0038a61c8 100644 --- a/agent-eval/evals/802-create-component/EVAL.ts +++ b/agent-eval/evals/802-create-component/EVAL.ts @@ -8,16 +8,12 @@ import { expectStoryTestsRanAndPassed, expectWorkflowCalls, getEvalContext, - modelRunsTestsOnlyWhenAsked, } from '#test-utils'; import { describe, test } from 'vitest'; describe('creating a ProfileCard component', () => { test('runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ - requireAgentRun: !modelRunsTestsOnlyWhenAsked(), - covering: ['profilecard'], - }); + await expectStoryTestsRanAndPassed({ covering: ['profilecard'] }); }); test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/evals/803-edit-component/EVAL.ts b/agent-eval/evals/803-edit-component/EVAL.ts index 54c64f84b899..17bd97f2594d 100644 --- a/agent-eval/evals/803-edit-component/EVAL.ts +++ b/agent-eval/evals/803-edit-component/EVAL.ts @@ -8,16 +8,12 @@ import { expectStoryTestsRanAndPassed, expectWorkflowCalls, getEvalContext, - modelRunsTestsOnlyWhenAsked, } from '#test-utils'; import { describe, test } from 'vitest'; describe('editing ReviewCard to add date and optional onReport', () => { test('runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ - requireAgentRun: !modelRunsTestsOnlyWhenAsked(), - covering: ['reviewcard'], - }); + await expectStoryTestsRanAndPassed({ covering: ['reviewcard'] }); }); test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/evals/804-write-story-for-existing-component/EVAL.ts b/agent-eval/evals/804-write-story-for-existing-component/EVAL.ts index 8307d4dedbfb..b80e496a8eed 100644 --- a/agent-eval/evals/804-write-story-for-existing-component/EVAL.ts +++ b/agent-eval/evals/804-write-story-for-existing-component/EVAL.ts @@ -9,15 +9,11 @@ import { expectStoryIdsInDisplayReview, expectStoryTestsRanAndPassed, expectWorkflowCalls, - modelRunsTestsOnlyWhenAsked, } from '#test-utils'; describe('writing stories for an existing AlertBanner', () => { test('runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ - requireAgentRun: !modelRunsTestsOnlyWhenAsked(), - covering: ['alertbanner'], - }); + await expectStoryTestsRanAndPassed({ covering: ['alertbanner'] }); }); test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/evals/808-shared-infra-fallback/EVAL.ts b/agent-eval/evals/808-shared-infra-fallback/EVAL.ts index 3a3435088f03..d065c71b23b5 100644 --- a/agent-eval/evals/808-shared-infra-fallback/EVAL.ts +++ b/agent-eval/evals/808-shared-infra-fallback/EVAL.ts @@ -9,7 +9,6 @@ import { getEvalContext, getWorkflowCalls, getWorkflowToolResults, - modelRunsTestsOnlyWhenAsked, } from '#test-utils'; import { readFileSync } from 'node:fs'; import { describe, expect, test } from 'vitest'; @@ -37,10 +36,7 @@ describe('changing a shared accent token and surfacing consumer stories', () => test.skipIf(codexMcpGap)( 'runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ - requireAgentRun: !modelRunsTestsOnlyWhenAsked(), - covering: ['badge', 'statuspill'], - }); + await expectStoryTestsRanAndPassed({ covering: ['badge', 'statuspill'] }); } ); diff --git a/agent-eval/evals/812-first-story-empty-project/EVAL.ts b/agent-eval/evals/812-first-story-empty-project/EVAL.ts index c13d905721d1..46b5839d7ff1 100644 --- a/agent-eval/evals/812-first-story-empty-project/EVAL.ts +++ b/agent-eval/evals/812-first-story-empty-project/EVAL.ts @@ -10,15 +10,11 @@ import { expectStoryIdsInDisplayReview, expectStoryTestsRanAndPassed, expectWorkflowCalls, - modelRunsTestsOnlyWhenAsked, } from '#test-utils'; describe('writing the first Button stories in an empty Storybook', () => { test('runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ - requireAgentRun: !modelRunsTestsOnlyWhenAsked(), - covering: ['button'], - }); + await expectStoryTestsRanAndPassed({ covering: ['button'] }); }); test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/evals/813-monorepo-leaf-create-component/EVAL.ts b/agent-eval/evals/813-monorepo-leaf-create-component/EVAL.ts index 5c745d257b49..f3a99e830a9e 100644 --- a/agent-eval/evals/813-monorepo-leaf-create-component/EVAL.ts +++ b/agent-eval/evals/813-monorepo-leaf-create-component/EVAL.ts @@ -10,7 +10,6 @@ import { expectStoryIdsInDisplayReview, expectStoryTestsRanAndPassed, expectWorkflowCalls, - modelRunsTestsOnlyWhenAsked, } from '#test-utils'; describe('creating a Callout in a monorepo UI package', () => { @@ -27,11 +26,7 @@ describe('creating a Callout in a monorepo UI package', () => { }); test('runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ - requireAgentRun: !modelRunsTestsOnlyWhenAsked(), - covering: ['callout'], - cwd: 'packages/ui', - }); + await expectStoryTestsRanAndPassed({ covering: ['callout'], cwd: 'packages/ui' }); }); test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/evals/814-write-stories-for-fetching-component/EVAL.ts b/agent-eval/evals/814-write-stories-for-fetching-component/EVAL.ts index 1d7bdb91ee44..5fbe8761e21a 100644 --- a/agent-eval/evals/814-write-stories-for-fetching-component/EVAL.ts +++ b/agent-eval/evals/814-write-stories-for-fetching-component/EVAL.ts @@ -10,17 +10,13 @@ import { expectStoryTestsRanAndPassed, expectWorkflowCalls, getEvalContext, - modelRunsTestsOnlyWhenAsked, } from '#test-utils'; const STORIES_PATH = 'stories/OrderHistory.stories.tsx'; describe('writing stories for an existing OrderHistory that fetches its data', () => { test('runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ - requireAgentRun: !modelRunsTestsOnlyWhenAsked(), - covering: ['orderhistory'], - }); + await expectStoryTestsRanAndPassed({ covering: ['orderhistory'] }); }); test('mocks the request instead of changing the component', () => { diff --git a/agent-eval/experiments/cc-mcp-opus-5.5-medium.ts b/agent-eval/experiments/cc-mcp-opus-5.5-medium.ts index 5a263c0b167a..eaf9a953aad3 100644 --- a/agent-eval/experiments/cc-mcp-opus-5.5-medium.ts +++ b/agent-eval/experiments/cc-mcp-opus-5.5-medium.ts @@ -6,16 +6,14 @@ import { writeClaudeMcpConfig, } from '../lib/templates.ts'; -const model = 'claude-opus-5-5'; - export default { ...DEFAULT_EXPERIMENT_CONFIG, agent: 'claude-code', // direct Anthropic API, requires ANTHROPIC_API_KEY - model, + model: 'claude-opus-5-5', agentOptions: { effort: 'medium' }, evals: WORKFLOW_STORYBOOK_EVALS, setup: async (sandbox) => { - await setupSandbox(sandbox, { agent: 'claude-code', integration: 'mcp', model }); + await setupSandbox(sandbox, { agent: 'claude-code', integration: 'mcp' }); await writeClaudeMcpConfig(sandbox); await writeClaudeInAppBrowserMock(sandbox); }, diff --git a/agent-eval/experiments/cc-plugin-opus-5.5-medium.ts b/agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts similarity index 87% rename from agent-eval/experiments/cc-plugin-opus-5.5-medium.ts rename to agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts index dbd78b3c0c53..835678c39ad9 100644 --- a/agent-eval/experiments/cc-plugin-opus-5.5-medium.ts +++ b/agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts @@ -6,17 +6,15 @@ import { writeClaudePluginSkills, } from '../lib/templates.ts'; -const model = 'claude-opus-5-5'; - export default { ...DEFAULT_EXPERIMENT_CONFIG, agent: 'claude-code', // direct Anthropic API, requires ANTHROPIC_API_KEY - model, - agentOptions: { effort: 'medium' }, + model: 'claude-sonnet-5-5', + agentOptions: { effort: 'low' }, // Skipped under EVAL_STORYBOOK_LATEST=1; see PLUGIN_STORYBOOK_EVALS. evals: PLUGIN_STORYBOOK_EVALS, setup: async (sandbox) => { - await setupSandbox(sandbox, { agent: 'claude-code', integration: 'plugin', model }); + await setupSandbox(sandbox, { agent: 'claude-code', integration: 'plugin' }); await writeClaudePluginSkills(sandbox); await writeClaudeInAppBrowserMock(sandbox); }, diff --git a/agent-eval/experiments/codex-mcp-gpt-6.1-sol-medium.ts b/agent-eval/experiments/codex-mcp-gpt-6.1-sol-medium.ts index 7bcb8d513b3d..b5722002a992 100644 --- a/agent-eval/experiments/codex-mcp-gpt-6.1-sol-medium.ts +++ b/agent-eval/experiments/codex-mcp-gpt-6.1-sol-medium.ts @@ -7,18 +7,16 @@ import { writeCodexMcpConfig, } from '../lib/templates.ts'; -const model = 'gpt-6.1-sol?reasoningEffort=medium'; - export default { ...DEFAULT_EXPERIMENT_CONFIG, // Use direct Codex for MCP evals. The AI Gateway Codex path does not reliably // handle Codex's Responses namespace tool shape yet: // https://github.com/openai/codex/issues/26234 agent: 'codex', - model, + model: 'gpt-6.1-sol?reasoningEffort=medium', evals: WORKFLOW_STORYBOOK_EVALS, setup: async (sandbox) => { - await setupSandbox(sandbox, { agent: 'codex', integration: 'mcp', model }); + await setupSandbox(sandbox, { agent: 'codex', integration: 'mcp' }); await writeCodexMcpConfig(sandbox); await writeCodexAgentsMd(sandbox); await writeCodexInAppBrowserMock(sandbox); diff --git a/agent-eval/experiments/codex-plugin-gpt-6-luna-low.ts b/agent-eval/experiments/codex-plugin-gpt-6-luna-low.ts deleted file mode 100644 index 0d0fe84e11dd..000000000000 --- a/agent-eval/experiments/codex-plugin-gpt-6-luna-low.ts +++ /dev/null @@ -1,22 +0,0 @@ -import type { ExperimentConfig } from '@vercel/agent-eval'; -import { DEFAULT_EXPERIMENT_CONFIG, PLUGIN_STORYBOOK_EVALS } from '../lib/experiment.ts'; -import { - setupSandbox, - writeCodexInAppBrowserMock, - writeCodexPluginSkills, -} from '../lib/templates.ts'; - -const model = 'gpt-6-luna?reasoningEffort=low'; - -export default { - ...DEFAULT_EXPERIMENT_CONFIG, - agent: 'codex', - model, - // Skipped under EVAL_STORYBOOK_LATEST=1; see PLUGIN_STORYBOOK_EVALS. - evals: PLUGIN_STORYBOOK_EVALS, - setup: async (sandbox) => { - await setupSandbox(sandbox, { agent: 'codex', integration: 'plugin', model }); - await writeCodexPluginSkills(sandbox); - await writeCodexInAppBrowserMock(sandbox); - }, -} satisfies ExperimentConfig; diff --git a/agent-eval/experiments/codex-plugin-gpt-6.1-sol-medium.ts b/agent-eval/experiments/codex-plugin-gpt-6.1-sol-low.ts similarity index 91% rename from agent-eval/experiments/codex-plugin-gpt-6.1-sol-medium.ts rename to agent-eval/experiments/codex-plugin-gpt-6.1-sol-low.ts index 77dfa4646a77..7f60604fb389 100644 --- a/agent-eval/experiments/codex-plugin-gpt-6.1-sol-medium.ts +++ b/agent-eval/experiments/codex-plugin-gpt-6.1-sol-low.ts @@ -6,19 +6,17 @@ import { writeCodexPluginSkills, } from '../lib/templates.ts'; -const model = 'gpt-6.1-sol?reasoningEffort=medium'; - export default { ...DEFAULT_EXPERIMENT_CONFIG, // Keep Codex plugin and MCP experiments on the same direct Codex runner. // The MCP variant cannot use the AI Gateway path yet: // https://github.com/openai/codex/issues/26234 agent: 'codex', - model, + model: 'gpt-6.1-sol?reasoningEffort=low', // Skipped under EVAL_STORYBOOK_LATEST=1; see PLUGIN_STORYBOOK_EVALS. evals: PLUGIN_STORYBOOK_EVALS, setup: async (sandbox) => { - await setupSandbox(sandbox, { agent: 'codex', integration: 'plugin', model }); + await setupSandbox(sandbox, { agent: 'codex', integration: 'plugin' }); await writeCodexPluginSkills(sandbox); await writeCodexInAppBrowserMock(sandbox); }, diff --git a/agent-eval/lib/templates.ts b/agent-eval/lib/templates.ts index 63a1fe8de6f7..aad227eaa16d 100644 --- a/agent-eval/lib/templates.ts +++ b/agent-eval/lib/templates.ts @@ -141,7 +141,7 @@ const CODEX_PLUGIN_SKILLS_DIR = path.join( export async function setupSandbox( sandbox: Sandbox, - options: { agent: EvalAgent; integration: EvalIntegration; model?: string } + options: { agent: EvalAgent; integration: EvalIntegration } ): Promise { await writeEvalSupportFiles(sandbox, options); @@ -214,14 +214,14 @@ export async function setupSandbox( async function writeEvalSupportFiles( sandbox: Sandbox, - options: { agent: EvalAgent; integration: EvalIntegration; model?: string } + options: { agent: EvalAgent; integration: EvalIntegration } ): Promise { await sandbox.writeFiles({ [TRANSCRIPT_HELPER_SANDBOX_PATH]: await fs.readFile(TRANSCRIPT_HELPER_SOURCE_PATH, 'utf8'), [SHELL_PARSE_SANDBOX_PATH]: await fs.readFile(SHELL_PARSE_SOURCE_PATH, 'utf8'), [TYPE_UTIL_SANDBOX_PATH]: await fs.readFile(TYPE_UTIL_SOURCE_PATH, 'utf8'), [AGENT_CONTEXT_SANDBOX_PATH]: JSON.stringify( - { agent: options.agent, model: options.model, integration: options.integration }, + { agent: options.agent, integration: options.integration }, null, 2 ).concat('\n'), diff --git a/agent-eval/lib/test-utils.test.ts b/agent-eval/lib/test-utils.test.ts index 17f8a5ccc212..661d60d469b8 100644 --- a/agent-eval/lib/test-utils.test.ts +++ b/agent-eval/lib/test-utils.test.ts @@ -525,18 +525,6 @@ describe('expectStoryTestsRanAndPassed', () => { await expect(expectStoryTestsRanAndPassed()).rejects.toThrow(/Expected test-run to be called/); }); - - test('lets the agent skip the tests with requireAgentRun false, but still runs them in the sandbox', async () => { - givenRun({ - commands: ['npm run typecheck'], - stdout: testRunDocument({ 'example-badge--accent': 'status-value:success' }), - }); - const { expectStoryTestsRanAndPassed } = await loadTestUtils(); - - await expectStoryTestsRanAndPassed({ requireAgentRun: false, covering: ['badge'] }); - - expect(execFile).toHaveBeenCalled(); - }); }); describe('parseCodexBrowserNavigations', () => { diff --git a/agent-eval/lib/test-utils.ts b/agent-eval/lib/test-utils.ts index 9cd453b00d6d..d73e3affa898 100644 --- a/agent-eval/lib/test-utils.ts +++ b/agent-eval/lib/test-utils.ts @@ -29,14 +29,11 @@ const TRANSCRIPT_PATH = '__agent_eval__/transcript.txt'; type AgentContext = { agent?: unknown; - model?: unknown; integration?: unknown; }; type EvalContext = { agent: string; - // Such as `gpt-6-luna?reasoningEffort=low`; absent when the experiment passes none. - model?: string; // 'none' is the agentic-reference bare control (no Storybook tooling flavor); // see lib/templates.ts. Plugin-only helpers below treat it as "not plugin". integration: 'mcp' | 'plugin' | 'none'; @@ -60,7 +57,7 @@ export type StoryInputExpectation = { export function getEvalContext(): EvalContext { const agentContext = readAgentContext(); - const { agent, model, integration } = agentContext; + const { agent, integration } = agentContext; if (typeof agent !== 'string') { throw new Error( @@ -77,13 +74,7 @@ export function getEvalContext(): EvalContext { ); } - return { agent, ...(typeof model === 'string' && { model }), integration }; -} - -// Codex's system prompt for GPT-6 Luna alone says "Do not add or run tests unless the user asks -// you to test or verify implementation." -export function modelRunsTestsOnlyWhenAsked(): boolean { - return getEvalContext().model?.startsWith('gpt-6-luna') === true; + return { agent, integration }; } export function getTranscript(agent = getEvalContext().agent): Transcript { @@ -401,16 +392,11 @@ export type WorkflowToolResult = { // // `covering` requires one of the given substrings in the passing story ids. // `cwd` is the Storybook project, for fixtures where that is not the root. -// `requireAgentRun: false` lets the agent skip the tests, for a model that runs -// them only when asked; a run it did make must still not end red. export async function expectStoryTestsRanAndPassed(options?: { covering?: string[]; cwd?: string; - requireAgentRun?: boolean; }): Promise { - if (options?.requireAgentRun !== false) { - expectWorkflowCalls(['test-run']); - } + expectWorkflowCalls(['test-run']); expect( getWorkflowToolResults('test-run').at(-1)?.output ?? '', "The agent's last test run must not report failing stories or unhandled errors" From b9d56c63aa9fac3a1f4838089480bd7e662c934d Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Thu, 8 Oct 2026 11:38:50 +0200 Subject: [PATCH 25/32] Revert "test(agent-eval): run the plugin experiments on Sonnet 5.5 low and GPT-6.1-Sol low" This reverts commit f968f80e0d8015b3d0e0a073118d691978842f63. --- agent-eval/README.md | 9 ++++---- .../801-create-accessible-component/EVAL.ts | 6 ++++- agent-eval/evals/802-create-component/EVAL.ts | 6 ++++- agent-eval/evals/803-edit-component/EVAL.ts | 6 ++++- .../EVAL.ts | 6 ++++- .../evals/808-shared-infra-fallback/EVAL.ts | 6 ++++- .../812-first-story-empty-project/EVAL.ts | 6 ++++- .../EVAL.ts | 7 +++++- .../EVAL.ts | 6 ++++- .../experiments/cc-mcp-opus-5.5-medium.ts | 6 +++-- ....5-low.ts => cc-plugin-opus-5.5-medium.ts} | 8 ++++--- .../codex-mcp-gpt-6.1-sol-medium.ts | 6 +++-- .../codex-plugin-gpt-6-luna-low.ts | 22 +++++++++++++++++++ ....ts => codex-plugin-gpt-6.1-sol-medium.ts} | 6 +++-- agent-eval/lib/templates.ts | 6 ++--- agent-eval/lib/test-utils.test.ts | 12 ++++++++++ agent-eval/lib/test-utils.ts | 20 ++++++++++++++--- 17 files changed, 117 insertions(+), 27 deletions(-) rename agent-eval/experiments/{cc-plugin-sonnet-5.5-low.ts => cc-plugin-opus-5.5-medium.ts} (87%) create mode 100644 agent-eval/experiments/codex-plugin-gpt-6-luna-low.ts rename agent-eval/experiments/{codex-plugin-gpt-6.1-sol-low.ts => codex-plugin-gpt-6.1-sol-medium.ts} (91%) diff --git a/agent-eval/README.md b/agent-eval/README.md index dc11e62941d1..ae4a22369d3c 100644 --- a/agent-eval/README.md +++ b/agent-eval/README.md @@ -65,8 +65,8 @@ EVAL_ALL=1 yarn eval EVAL_ONLY=803-edit-component yarn eval ``` -A full `EVAL_ALL=1` run (13 workflow evals × 4 experiments + 4 -lifecycle evals × 2 plugin experiments) costs roughly **$30–45** in agent +A full `EVAL_ALL=1` run (13 workflow evals × 5 experiments + 4 +lifecycle evals × 3 plugin experiments) costs roughly **$30–45** in agent tokens at current per-run averages ($0.30–0.80 per workflow eval, $1–2 per lifecycle eval). The budget guardrail is **$75 per full run** — check the usage metadata in the results playground before growing the eval set past it @@ -129,9 +129,10 @@ via `ANTHROPIC_API_KEY`; Codex experiments use the direct Codex API via `OPENAI_API_KEY`): - `cc-mcp-opus-5.5-medium`: Claude Code (Opus 5.5 at medium effort) with project-local Storybook MCP config in `.mcp.json`. -- `cc-plugin-sonnet-5.5-low`: Claude Code (Sonnet 5.5 at low effort) with Storybook plugin skills copied to `.claude/skills`. +- `cc-plugin-opus-5.5-medium`: Claude Code (Opus 5.5 at medium effort) with Storybook plugin skills copied to `.claude/skills`. - `codex-mcp-gpt-6.1-sol-medium`: Codex (gpt-6.1-sol at medium reasoning effort) with project-local Storybook MCP config in `.codex/config.toml` and the Storybook MCP server instructions in `AGENTS.md` (none with `EVAL_STORYBOOK_LATEST=1`). -- `codex-plugin-gpt-6.1-sol-low`: Codex (gpt-6.1-sol at low reasoning effort) with Storybook plugin skills copied to `.agents/skills`. +- `codex-plugin-gpt-6.1-sol-medium`: Codex (gpt-6.1-sol at medium reasoning effort) with Storybook plugin skills copied to `.agents/skills`. +- `codex-plugin-gpt-6-luna-low`: Codex (gpt-6-luna at low reasoning effort) with Storybook plugin skills copied to `.agents/skills`. Its Codex system prompt says not to run tests unless asked, so evals do not require it to run them itself (see `modelRunsTestsOnlyWhenAsked` in `lib/test-utils.ts`). It costs about $0.01 per eval. ## Running evals in CI diff --git a/agent-eval/evals/801-create-accessible-component/EVAL.ts b/agent-eval/evals/801-create-accessible-component/EVAL.ts index ad148e3a444a..4a0507be3c03 100644 --- a/agent-eval/evals/801-create-accessible-component/EVAL.ts +++ b/agent-eval/evals/801-create-accessible-component/EVAL.ts @@ -9,13 +9,17 @@ import { expectStoryTestsRanAndPassed, expectWorkflowCalls, getEvalContext, + modelRunsTestsOnlyWhenAsked, } from '#test-utils'; import { transcript } from '@vercel/agent-eval/eval'; import { describe, expect, test } from 'vitest'; describe('creating an accessible ToggleSwitch', () => { test('runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ covering: ['toggleswitch'] }); + await expectStoryTestsRanAndPassed({ + requireAgentRun: !modelRunsTestsOnlyWhenAsked(), + covering: ['toggleswitch'], + }); }); test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/evals/802-create-component/EVAL.ts b/agent-eval/evals/802-create-component/EVAL.ts index 5ed0038a61c8..94da69f563f5 100644 --- a/agent-eval/evals/802-create-component/EVAL.ts +++ b/agent-eval/evals/802-create-component/EVAL.ts @@ -8,12 +8,16 @@ import { expectStoryTestsRanAndPassed, expectWorkflowCalls, getEvalContext, + modelRunsTestsOnlyWhenAsked, } from '#test-utils'; import { describe, test } from 'vitest'; describe('creating a ProfileCard component', () => { test('runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ covering: ['profilecard'] }); + await expectStoryTestsRanAndPassed({ + requireAgentRun: !modelRunsTestsOnlyWhenAsked(), + covering: ['profilecard'], + }); }); test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/evals/803-edit-component/EVAL.ts b/agent-eval/evals/803-edit-component/EVAL.ts index 17bd97f2594d..54c64f84b899 100644 --- a/agent-eval/evals/803-edit-component/EVAL.ts +++ b/agent-eval/evals/803-edit-component/EVAL.ts @@ -8,12 +8,16 @@ import { expectStoryTestsRanAndPassed, expectWorkflowCalls, getEvalContext, + modelRunsTestsOnlyWhenAsked, } from '#test-utils'; import { describe, test } from 'vitest'; describe('editing ReviewCard to add date and optional onReport', () => { test('runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ covering: ['reviewcard'] }); + await expectStoryTestsRanAndPassed({ + requireAgentRun: !modelRunsTestsOnlyWhenAsked(), + covering: ['reviewcard'], + }); }); test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/evals/804-write-story-for-existing-component/EVAL.ts b/agent-eval/evals/804-write-story-for-existing-component/EVAL.ts index b80e496a8eed..8307d4dedbfb 100644 --- a/agent-eval/evals/804-write-story-for-existing-component/EVAL.ts +++ b/agent-eval/evals/804-write-story-for-existing-component/EVAL.ts @@ -9,11 +9,15 @@ import { expectStoryIdsInDisplayReview, expectStoryTestsRanAndPassed, expectWorkflowCalls, + modelRunsTestsOnlyWhenAsked, } from '#test-utils'; describe('writing stories for an existing AlertBanner', () => { test('runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ covering: ['alertbanner'] }); + await expectStoryTestsRanAndPassed({ + requireAgentRun: !modelRunsTestsOnlyWhenAsked(), + covering: ['alertbanner'], + }); }); test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/evals/808-shared-infra-fallback/EVAL.ts b/agent-eval/evals/808-shared-infra-fallback/EVAL.ts index d065c71b23b5..3a3435088f03 100644 --- a/agent-eval/evals/808-shared-infra-fallback/EVAL.ts +++ b/agent-eval/evals/808-shared-infra-fallback/EVAL.ts @@ -9,6 +9,7 @@ import { getEvalContext, getWorkflowCalls, getWorkflowToolResults, + modelRunsTestsOnlyWhenAsked, } from '#test-utils'; import { readFileSync } from 'node:fs'; import { describe, expect, test } from 'vitest'; @@ -36,7 +37,10 @@ describe('changing a shared accent token and surfacing consumer stories', () => test.skipIf(codexMcpGap)( 'runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ covering: ['badge', 'statuspill'] }); + await expectStoryTestsRanAndPassed({ + requireAgentRun: !modelRunsTestsOnlyWhenAsked(), + covering: ['badge', 'statuspill'], + }); } ); diff --git a/agent-eval/evals/812-first-story-empty-project/EVAL.ts b/agent-eval/evals/812-first-story-empty-project/EVAL.ts index 46b5839d7ff1..c13d905721d1 100644 --- a/agent-eval/evals/812-first-story-empty-project/EVAL.ts +++ b/agent-eval/evals/812-first-story-empty-project/EVAL.ts @@ -10,11 +10,15 @@ import { expectStoryIdsInDisplayReview, expectStoryTestsRanAndPassed, expectWorkflowCalls, + modelRunsTestsOnlyWhenAsked, } from '#test-utils'; describe('writing the first Button stories in an empty Storybook', () => { test('runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ covering: ['button'] }); + await expectStoryTestsRanAndPassed({ + requireAgentRun: !modelRunsTestsOnlyWhenAsked(), + covering: ['button'], + }); }); test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/evals/813-monorepo-leaf-create-component/EVAL.ts b/agent-eval/evals/813-monorepo-leaf-create-component/EVAL.ts index f3a99e830a9e..5c745d257b49 100644 --- a/agent-eval/evals/813-monorepo-leaf-create-component/EVAL.ts +++ b/agent-eval/evals/813-monorepo-leaf-create-component/EVAL.ts @@ -10,6 +10,7 @@ import { expectStoryIdsInDisplayReview, expectStoryTestsRanAndPassed, expectWorkflowCalls, + modelRunsTestsOnlyWhenAsked, } from '#test-utils'; describe('creating a Callout in a monorepo UI package', () => { @@ -26,7 +27,11 @@ describe('creating a Callout in a monorepo UI package', () => { }); test('runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ covering: ['callout'], cwd: 'packages/ui' }); + await expectStoryTestsRanAndPassed({ + requireAgentRun: !modelRunsTestsOnlyWhenAsked(), + covering: ['callout'], + cwd: 'packages/ui', + }); }); test('uses Storybook story instructions and publishes a display review', () => { diff --git a/agent-eval/evals/814-write-stories-for-fetching-component/EVAL.ts b/agent-eval/evals/814-write-stories-for-fetching-component/EVAL.ts index 5fbe8761e21a..1d7bdb91ee44 100644 --- a/agent-eval/evals/814-write-stories-for-fetching-component/EVAL.ts +++ b/agent-eval/evals/814-write-stories-for-fetching-component/EVAL.ts @@ -10,13 +10,17 @@ import { expectStoryTestsRanAndPassed, expectWorkflowCalls, getEvalContext, + modelRunsTestsOnlyWhenAsked, } from '#test-utils'; const STORIES_PATH = 'stories/OrderHistory.stories.tsx'; describe('writing stories for an existing OrderHistory that fetches its data', () => { test('runs story tests after the change and finishes with them passing', async () => { - await expectStoryTestsRanAndPassed({ covering: ['orderhistory'] }); + await expectStoryTestsRanAndPassed({ + requireAgentRun: !modelRunsTestsOnlyWhenAsked(), + covering: ['orderhistory'], + }); }); test('mocks the request instead of changing the component', () => { diff --git a/agent-eval/experiments/cc-mcp-opus-5.5-medium.ts b/agent-eval/experiments/cc-mcp-opus-5.5-medium.ts index eaf9a953aad3..5a263c0b167a 100644 --- a/agent-eval/experiments/cc-mcp-opus-5.5-medium.ts +++ b/agent-eval/experiments/cc-mcp-opus-5.5-medium.ts @@ -6,14 +6,16 @@ import { writeClaudeMcpConfig, } from '../lib/templates.ts'; +const model = 'claude-opus-5-5'; + export default { ...DEFAULT_EXPERIMENT_CONFIG, agent: 'claude-code', // direct Anthropic API, requires ANTHROPIC_API_KEY - model: 'claude-opus-5-5', + model, agentOptions: { effort: 'medium' }, evals: WORKFLOW_STORYBOOK_EVALS, setup: async (sandbox) => { - await setupSandbox(sandbox, { agent: 'claude-code', integration: 'mcp' }); + await setupSandbox(sandbox, { agent: 'claude-code', integration: 'mcp', model }); await writeClaudeMcpConfig(sandbox); await writeClaudeInAppBrowserMock(sandbox); }, diff --git a/agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts b/agent-eval/experiments/cc-plugin-opus-5.5-medium.ts similarity index 87% rename from agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts rename to agent-eval/experiments/cc-plugin-opus-5.5-medium.ts index 835678c39ad9..dbd78b3c0c53 100644 --- a/agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts +++ b/agent-eval/experiments/cc-plugin-opus-5.5-medium.ts @@ -6,15 +6,17 @@ import { writeClaudePluginSkills, } from '../lib/templates.ts'; +const model = 'claude-opus-5-5'; + export default { ...DEFAULT_EXPERIMENT_CONFIG, agent: 'claude-code', // direct Anthropic API, requires ANTHROPIC_API_KEY - model: 'claude-sonnet-5-5', - agentOptions: { effort: 'low' }, + model, + agentOptions: { effort: 'medium' }, // Skipped under EVAL_STORYBOOK_LATEST=1; see PLUGIN_STORYBOOK_EVALS. evals: PLUGIN_STORYBOOK_EVALS, setup: async (sandbox) => { - await setupSandbox(sandbox, { agent: 'claude-code', integration: 'plugin' }); + await setupSandbox(sandbox, { agent: 'claude-code', integration: 'plugin', model }); await writeClaudePluginSkills(sandbox); await writeClaudeInAppBrowserMock(sandbox); }, diff --git a/agent-eval/experiments/codex-mcp-gpt-6.1-sol-medium.ts b/agent-eval/experiments/codex-mcp-gpt-6.1-sol-medium.ts index b5722002a992..7bcb8d513b3d 100644 --- a/agent-eval/experiments/codex-mcp-gpt-6.1-sol-medium.ts +++ b/agent-eval/experiments/codex-mcp-gpt-6.1-sol-medium.ts @@ -7,16 +7,18 @@ import { writeCodexMcpConfig, } from '../lib/templates.ts'; +const model = 'gpt-6.1-sol?reasoningEffort=medium'; + export default { ...DEFAULT_EXPERIMENT_CONFIG, // Use direct Codex for MCP evals. The AI Gateway Codex path does not reliably // handle Codex's Responses namespace tool shape yet: // https://github.com/openai/codex/issues/26234 agent: 'codex', - model: 'gpt-6.1-sol?reasoningEffort=medium', + model, evals: WORKFLOW_STORYBOOK_EVALS, setup: async (sandbox) => { - await setupSandbox(sandbox, { agent: 'codex', integration: 'mcp' }); + await setupSandbox(sandbox, { agent: 'codex', integration: 'mcp', model }); await writeCodexMcpConfig(sandbox); await writeCodexAgentsMd(sandbox); await writeCodexInAppBrowserMock(sandbox); diff --git a/agent-eval/experiments/codex-plugin-gpt-6-luna-low.ts b/agent-eval/experiments/codex-plugin-gpt-6-luna-low.ts new file mode 100644 index 000000000000..0d0fe84e11dd --- /dev/null +++ b/agent-eval/experiments/codex-plugin-gpt-6-luna-low.ts @@ -0,0 +1,22 @@ +import type { ExperimentConfig } from '@vercel/agent-eval'; +import { DEFAULT_EXPERIMENT_CONFIG, PLUGIN_STORYBOOK_EVALS } from '../lib/experiment.ts'; +import { + setupSandbox, + writeCodexInAppBrowserMock, + writeCodexPluginSkills, +} from '../lib/templates.ts'; + +const model = 'gpt-6-luna?reasoningEffort=low'; + +export default { + ...DEFAULT_EXPERIMENT_CONFIG, + agent: 'codex', + model, + // Skipped under EVAL_STORYBOOK_LATEST=1; see PLUGIN_STORYBOOK_EVALS. + evals: PLUGIN_STORYBOOK_EVALS, + setup: async (sandbox) => { + await setupSandbox(sandbox, { agent: 'codex', integration: 'plugin', model }); + await writeCodexPluginSkills(sandbox); + await writeCodexInAppBrowserMock(sandbox); + }, +} satisfies ExperimentConfig; diff --git a/agent-eval/experiments/codex-plugin-gpt-6.1-sol-low.ts b/agent-eval/experiments/codex-plugin-gpt-6.1-sol-medium.ts similarity index 91% rename from agent-eval/experiments/codex-plugin-gpt-6.1-sol-low.ts rename to agent-eval/experiments/codex-plugin-gpt-6.1-sol-medium.ts index 7f60604fb389..77dfa4646a77 100644 --- a/agent-eval/experiments/codex-plugin-gpt-6.1-sol-low.ts +++ b/agent-eval/experiments/codex-plugin-gpt-6.1-sol-medium.ts @@ -6,17 +6,19 @@ import { writeCodexPluginSkills, } from '../lib/templates.ts'; +const model = 'gpt-6.1-sol?reasoningEffort=medium'; + export default { ...DEFAULT_EXPERIMENT_CONFIG, // Keep Codex plugin and MCP experiments on the same direct Codex runner. // The MCP variant cannot use the AI Gateway path yet: // https://github.com/openai/codex/issues/26234 agent: 'codex', - model: 'gpt-6.1-sol?reasoningEffort=low', + model, // Skipped under EVAL_STORYBOOK_LATEST=1; see PLUGIN_STORYBOOK_EVALS. evals: PLUGIN_STORYBOOK_EVALS, setup: async (sandbox) => { - await setupSandbox(sandbox, { agent: 'codex', integration: 'plugin' }); + await setupSandbox(sandbox, { agent: 'codex', integration: 'plugin', model }); await writeCodexPluginSkills(sandbox); await writeCodexInAppBrowserMock(sandbox); }, diff --git a/agent-eval/lib/templates.ts b/agent-eval/lib/templates.ts index aad227eaa16d..63a1fe8de6f7 100644 --- a/agent-eval/lib/templates.ts +++ b/agent-eval/lib/templates.ts @@ -141,7 +141,7 @@ const CODEX_PLUGIN_SKILLS_DIR = path.join( export async function setupSandbox( sandbox: Sandbox, - options: { agent: EvalAgent; integration: EvalIntegration } + options: { agent: EvalAgent; integration: EvalIntegration; model?: string } ): Promise { await writeEvalSupportFiles(sandbox, options); @@ -214,14 +214,14 @@ export async function setupSandbox( async function writeEvalSupportFiles( sandbox: Sandbox, - options: { agent: EvalAgent; integration: EvalIntegration } + options: { agent: EvalAgent; integration: EvalIntegration; model?: string } ): Promise { await sandbox.writeFiles({ [TRANSCRIPT_HELPER_SANDBOX_PATH]: await fs.readFile(TRANSCRIPT_HELPER_SOURCE_PATH, 'utf8'), [SHELL_PARSE_SANDBOX_PATH]: await fs.readFile(SHELL_PARSE_SOURCE_PATH, 'utf8'), [TYPE_UTIL_SANDBOX_PATH]: await fs.readFile(TYPE_UTIL_SOURCE_PATH, 'utf8'), [AGENT_CONTEXT_SANDBOX_PATH]: JSON.stringify( - { agent: options.agent, integration: options.integration }, + { agent: options.agent, model: options.model, integration: options.integration }, null, 2 ).concat('\n'), diff --git a/agent-eval/lib/test-utils.test.ts b/agent-eval/lib/test-utils.test.ts index 661d60d469b8..17f8a5ccc212 100644 --- a/agent-eval/lib/test-utils.test.ts +++ b/agent-eval/lib/test-utils.test.ts @@ -525,6 +525,18 @@ describe('expectStoryTestsRanAndPassed', () => { await expect(expectStoryTestsRanAndPassed()).rejects.toThrow(/Expected test-run to be called/); }); + + test('lets the agent skip the tests with requireAgentRun false, but still runs them in the sandbox', async () => { + givenRun({ + commands: ['npm run typecheck'], + stdout: testRunDocument({ 'example-badge--accent': 'status-value:success' }), + }); + const { expectStoryTestsRanAndPassed } = await loadTestUtils(); + + await expectStoryTestsRanAndPassed({ requireAgentRun: false, covering: ['badge'] }); + + expect(execFile).toHaveBeenCalled(); + }); }); describe('parseCodexBrowserNavigations', () => { diff --git a/agent-eval/lib/test-utils.ts b/agent-eval/lib/test-utils.ts index d73e3affa898..9cd453b00d6d 100644 --- a/agent-eval/lib/test-utils.ts +++ b/agent-eval/lib/test-utils.ts @@ -29,11 +29,14 @@ const TRANSCRIPT_PATH = '__agent_eval__/transcript.txt'; type AgentContext = { agent?: unknown; + model?: unknown; integration?: unknown; }; type EvalContext = { agent: string; + // Such as `gpt-6-luna?reasoningEffort=low`; absent when the experiment passes none. + model?: string; // 'none' is the agentic-reference bare control (no Storybook tooling flavor); // see lib/templates.ts. Plugin-only helpers below treat it as "not plugin". integration: 'mcp' | 'plugin' | 'none'; @@ -57,7 +60,7 @@ export type StoryInputExpectation = { export function getEvalContext(): EvalContext { const agentContext = readAgentContext(); - const { agent, integration } = agentContext; + const { agent, model, integration } = agentContext; if (typeof agent !== 'string') { throw new Error( @@ -74,7 +77,13 @@ export function getEvalContext(): EvalContext { ); } - return { agent, integration }; + return { agent, ...(typeof model === 'string' && { model }), integration }; +} + +// Codex's system prompt for GPT-6 Luna alone says "Do not add or run tests unless the user asks +// you to test or verify implementation." +export function modelRunsTestsOnlyWhenAsked(): boolean { + return getEvalContext().model?.startsWith('gpt-6-luna') === true; } export function getTranscript(agent = getEvalContext().agent): Transcript { @@ -392,11 +401,16 @@ export type WorkflowToolResult = { // // `covering` requires one of the given substrings in the passing story ids. // `cwd` is the Storybook project, for fixtures where that is not the root. +// `requireAgentRun: false` lets the agent skip the tests, for a model that runs +// them only when asked; a run it did make must still not end red. export async function expectStoryTestsRanAndPassed(options?: { covering?: string[]; cwd?: string; + requireAgentRun?: boolean; }): Promise { - expectWorkflowCalls(['test-run']); + if (options?.requireAgentRun !== false) { + expectWorkflowCalls(['test-run']); + } expect( getWorkflowToolResults('test-run').at(-1)?.output ?? '', "The agent's last test run must not report failing stories or unhandled errors" From f0b603ce0299b8242fd279f9dafcece16acf0641 Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Thu, 8 Oct 2026 11:41:14 +0200 Subject: [PATCH 26/32] test(agent-eval): make Sonnet 5.5 low and GPT-6.1-Sol low the default plugin experiments The plugin skills have to work on the cheaper models at low effort, so those two run by default and have to stay green. The Opus 5.5 medium, GPT-6.1-Sol medium and GPT-6 Luna low plugin experiments stay, but run only when named on the command line: a bare agent-eval, which CI runs, skips them. Co-Authored-By: Claude Opus 5.5 (1M context) --- agent-eval/README.md | 15 +++++++---- .../experiments/cc-plugin-opus-5.5-medium.ts | 10 +++++--- .../experiments/cc-plugin-sonnet-5.5-low.ts | 23 +++++++++++++++++ .../codex-plugin-gpt-6-luna-low.ts | 10 +++++--- .../codex-plugin-gpt-6.1-sol-low.ts | 25 +++++++++++++++++++ .../codex-plugin-gpt-6.1-sol-medium.ts | 10 +++++--- agent-eval/lib/experiment.test.ts | 23 ++++++++++++++++- agent-eval/lib/experiment.ts | 12 +++++++++ 8 files changed, 113 insertions(+), 15 deletions(-) create mode 100644 agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts create mode 100644 agent-eval/experiments/codex-plugin-gpt-6.1-sol-low.ts diff --git a/agent-eval/README.md b/agent-eval/README.md index ae4a22369d3c..923c6c333edd 100644 --- a/agent-eval/README.md +++ b/agent-eval/README.md @@ -65,8 +65,8 @@ EVAL_ALL=1 yarn eval EVAL_ONLY=803-edit-component yarn eval ``` -A full `EVAL_ALL=1` run (13 workflow evals × 5 experiments + 4 -lifecycle evals × 3 plugin experiments) costs roughly **$30–45** in agent +A full `EVAL_ALL=1` run (13 workflow evals × 4 experiments + 4 +lifecycle evals × 2 plugin experiments) costs roughly **$30–45** in agent tokens at current per-run averages ($0.30–0.80 per workflow eval, $1–2 per lifecycle eval). The budget guardrail is **$75 per full run** — check the usage metadata in the results playground before growing the eval set past it @@ -129,10 +129,15 @@ via `ANTHROPIC_API_KEY`; Codex experiments use the direct Codex API via `OPENAI_API_KEY`): - `cc-mcp-opus-5.5-medium`: Claude Code (Opus 5.5 at medium effort) with project-local Storybook MCP config in `.mcp.json`. -- `cc-plugin-opus-5.5-medium`: Claude Code (Opus 5.5 at medium effort) with Storybook plugin skills copied to `.claude/skills`. +- `cc-plugin-sonnet-5.5-low`: Claude Code (Sonnet 5.5 at low effort) with Storybook plugin skills copied to `.claude/skills`. - `codex-mcp-gpt-6.1-sol-medium`: Codex (gpt-6.1-sol at medium reasoning effort) with project-local Storybook MCP config in `.codex/config.toml` and the Storybook MCP server instructions in `AGENTS.md` (none with `EVAL_STORYBOOK_LATEST=1`). -- `codex-plugin-gpt-6.1-sol-medium`: Codex (gpt-6.1-sol at medium reasoning effort) with Storybook plugin skills copied to `.agents/skills`. -- `codex-plugin-gpt-6-luna-low`: Codex (gpt-6-luna at low reasoning effort) with Storybook plugin skills copied to `.agents/skills`. Its Codex system prompt says not to run tests unless asked, so evals do not require it to run them itself (see `modelRunsTestsOnlyWhenAsked` in `lib/test-utils.ts`). It costs about $0.01 per eval. +- `codex-plugin-gpt-6.1-sol-low`: Codex (gpt-6.1-sol at low reasoning effort) with Storybook plugin skills copied to `.agents/skills`. + +These four are the default set: a bare `yarn eval`, which is what CI runs, runs them and they have to stay green. Three more plugin experiments run only when you name them, for example `yarn exec agent-eval codex-plugin-gpt-6-luna-low`: + +- `cc-plugin-opus-5.5-medium`: Claude Code (Opus 5.5 at medium effort). +- `codex-plugin-gpt-6.1-sol-medium`: Codex (gpt-6.1-sol at medium reasoning effort). +- `codex-plugin-gpt-6-luna-low`: Codex (gpt-6-luna at low reasoning effort). Its Codex system prompt says not to run tests unless asked, so evals do not require it to run the story tests itself. ## Running evals in CI diff --git a/agent-eval/experiments/cc-plugin-opus-5.5-medium.ts b/agent-eval/experiments/cc-plugin-opus-5.5-medium.ts index dbd78b3c0c53..10908546f4ae 100644 --- a/agent-eval/experiments/cc-plugin-opus-5.5-medium.ts +++ b/agent-eval/experiments/cc-plugin-opus-5.5-medium.ts @@ -1,5 +1,9 @@ import type { ExperimentConfig } from '@vercel/agent-eval'; -import { DEFAULT_EXPERIMENT_CONFIG, PLUGIN_STORYBOOK_EVALS } from '../lib/experiment.ts'; +import { + DEFAULT_EXPERIMENT_CONFIG, + onlyWhenNamed, + PLUGIN_STORYBOOK_EVALS, +} from '../lib/experiment.ts'; import { setupSandbox, writeClaudeInAppBrowserMock, @@ -13,8 +17,8 @@ export default { agent: 'claude-code', // direct Anthropic API, requires ANTHROPIC_API_KEY model, agentOptions: { effort: 'medium' }, - // Skipped under EVAL_STORYBOOK_LATEST=1; see PLUGIN_STORYBOOK_EVALS. - evals: PLUGIN_STORYBOOK_EVALS, + // Not in the default set: runs only when named. Also skipped under EVAL_STORYBOOK_LATEST=1. + evals: onlyWhenNamed(import.meta.url, PLUGIN_STORYBOOK_EVALS), setup: async (sandbox) => { await setupSandbox(sandbox, { agent: 'claude-code', integration: 'plugin', model }); await writeClaudePluginSkills(sandbox); diff --git a/agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts b/agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts new file mode 100644 index 000000000000..333864fef185 --- /dev/null +++ b/agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts @@ -0,0 +1,23 @@ +import type { ExperimentConfig } from '@vercel/agent-eval'; +import { DEFAULT_EXPERIMENT_CONFIG, PLUGIN_STORYBOOK_EVALS } from '../lib/experiment.ts'; +import { + setupSandbox, + writeClaudeInAppBrowserMock, + writeClaudePluginSkills, +} from '../lib/templates.ts'; + +const model = 'claude-sonnet-5-5'; + +export default { + ...DEFAULT_EXPERIMENT_CONFIG, + agent: 'claude-code', // direct Anthropic API, requires ANTHROPIC_API_KEY + model, + agentOptions: { effort: 'low' }, + // Skipped under EVAL_STORYBOOK_LATEST=1; see PLUGIN_STORYBOOK_EVALS. + evals: PLUGIN_STORYBOOK_EVALS, + setup: async (sandbox) => { + await setupSandbox(sandbox, { agent: 'claude-code', integration: 'plugin', model }); + await writeClaudePluginSkills(sandbox); + await writeClaudeInAppBrowserMock(sandbox); + }, +} satisfies ExperimentConfig; diff --git a/agent-eval/experiments/codex-plugin-gpt-6-luna-low.ts b/agent-eval/experiments/codex-plugin-gpt-6-luna-low.ts index 0d0fe84e11dd..a67d79468392 100644 --- a/agent-eval/experiments/codex-plugin-gpt-6-luna-low.ts +++ b/agent-eval/experiments/codex-plugin-gpt-6-luna-low.ts @@ -1,5 +1,9 @@ import type { ExperimentConfig } from '@vercel/agent-eval'; -import { DEFAULT_EXPERIMENT_CONFIG, PLUGIN_STORYBOOK_EVALS } from '../lib/experiment.ts'; +import { + DEFAULT_EXPERIMENT_CONFIG, + onlyWhenNamed, + PLUGIN_STORYBOOK_EVALS, +} from '../lib/experiment.ts'; import { setupSandbox, writeCodexInAppBrowserMock, @@ -12,8 +16,8 @@ export default { ...DEFAULT_EXPERIMENT_CONFIG, agent: 'codex', model, - // Skipped under EVAL_STORYBOOK_LATEST=1; see PLUGIN_STORYBOOK_EVALS. - evals: PLUGIN_STORYBOOK_EVALS, + // Not in the default set: runs only when named. Also skipped under EVAL_STORYBOOK_LATEST=1. + evals: onlyWhenNamed(import.meta.url, PLUGIN_STORYBOOK_EVALS), setup: async (sandbox) => { await setupSandbox(sandbox, { agent: 'codex', integration: 'plugin', model }); await writeCodexPluginSkills(sandbox); diff --git a/agent-eval/experiments/codex-plugin-gpt-6.1-sol-low.ts b/agent-eval/experiments/codex-plugin-gpt-6.1-sol-low.ts new file mode 100644 index 000000000000..95fccca4bce1 --- /dev/null +++ b/agent-eval/experiments/codex-plugin-gpt-6.1-sol-low.ts @@ -0,0 +1,25 @@ +import type { ExperimentConfig } from '@vercel/agent-eval'; +import { DEFAULT_EXPERIMENT_CONFIG, PLUGIN_STORYBOOK_EVALS } from '../lib/experiment.ts'; +import { + setupSandbox, + writeCodexInAppBrowserMock, + writeCodexPluginSkills, +} from '../lib/templates.ts'; + +const model = 'gpt-6.1-sol?reasoningEffort=low'; + +export default { + ...DEFAULT_EXPERIMENT_CONFIG, + // Keep Codex plugin and MCP experiments on the same direct Codex runner. + // The MCP variant cannot use the AI Gateway path yet: + // https://github.com/openai/codex/issues/26234 + agent: 'codex', + model, + // Skipped under EVAL_STORYBOOK_LATEST=1; see PLUGIN_STORYBOOK_EVALS. + evals: PLUGIN_STORYBOOK_EVALS, + setup: async (sandbox) => { + await setupSandbox(sandbox, { agent: 'codex', integration: 'plugin', model }); + await writeCodexPluginSkills(sandbox); + await writeCodexInAppBrowserMock(sandbox); + }, +} satisfies ExperimentConfig; diff --git a/agent-eval/experiments/codex-plugin-gpt-6.1-sol-medium.ts b/agent-eval/experiments/codex-plugin-gpt-6.1-sol-medium.ts index 77dfa4646a77..ca25d368844f 100644 --- a/agent-eval/experiments/codex-plugin-gpt-6.1-sol-medium.ts +++ b/agent-eval/experiments/codex-plugin-gpt-6.1-sol-medium.ts @@ -1,5 +1,9 @@ import type { ExperimentConfig } from '@vercel/agent-eval'; -import { DEFAULT_EXPERIMENT_CONFIG, PLUGIN_STORYBOOK_EVALS } from '../lib/experiment.ts'; +import { + DEFAULT_EXPERIMENT_CONFIG, + onlyWhenNamed, + PLUGIN_STORYBOOK_EVALS, +} from '../lib/experiment.ts'; import { setupSandbox, writeCodexInAppBrowserMock, @@ -15,8 +19,8 @@ export default { // https://github.com/openai/codex/issues/26234 agent: 'codex', model, - // Skipped under EVAL_STORYBOOK_LATEST=1; see PLUGIN_STORYBOOK_EVALS. - evals: PLUGIN_STORYBOOK_EVALS, + // Not in the default set: runs only when named. Also skipped under EVAL_STORYBOOK_LATEST=1. + evals: onlyWhenNamed(import.meta.url, PLUGIN_STORYBOOK_EVALS), setup: async (sandbox) => { await setupSandbox(sandbox, { agent: 'codex', integration: 'plugin', model }); await writeCodexPluginSkills(sandbox); diff --git a/agent-eval/lib/experiment.test.ts b/agent-eval/lib/experiment.test.ts index 2581e1a00132..b94979e86cbc 100644 --- a/agent-eval/lib/experiment.test.ts +++ b/agent-eval/lib/experiment.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest'; import type { RunCompleteContext } from '@vercel/agent-eval'; -import { DEFAULT_EXPERIMENT_CONFIG } from './experiment.ts'; +import { DEFAULT_EXPERIMENT_CONFIG, onlyWhenNamed } from './experiment.ts'; describe('the run-complete hook', () => { it('records the checkout commit next to the existing metadata', async () => { @@ -18,3 +18,24 @@ describe('the run-complete hook', () => { }); }); }); + +describe('an experiment outside the default set', () => { + const experimentUrl = 'file:///repo/agent-eval/experiments/codex-plugin-gpt-6-luna-low.ts'; + const evals = ['801-create-accessible-component' as const]; + + it('runs no evals when the command line does not name it', () => { + expect(onlyWhenNamed(experimentUrl, evals, ['node', 'agent-eval'])).toEqual([]); + }); + + it('runs its evals when the command line names it', () => { + expect( + onlyWhenNamed(experimentUrl, evals, ['node', 'agent-eval', 'codex-plugin-gpt-6-luna-low']) + ).toEqual(evals); + }); + + it('does not run for another experiment whose name starts the same', () => { + expect( + onlyWhenNamed(experimentUrl, evals, ['node', 'agent-eval', 'codex-plugin-gpt-6-luna-low-x3']) + ).toEqual([]); + }); +}); diff --git a/agent-eval/lib/experiment.ts b/agent-eval/lib/experiment.ts index 7cf5c413b5e1..2d76b7c32c0a 100644 --- a/agent-eval/lib/experiment.ts +++ b/agent-eval/lib/experiment.ts @@ -136,6 +136,18 @@ export const PLUGIN_STORYBOOK_EVALS: EvalName[] = STORYBOOK_LATEST ? [] : [...ACTIVE_EVALS.core, ...ACTIVE_EVALS.lifecycle]; +// For an experiment outside the default set: its evals run only when the experiment is named on +// the command line (`agent-eval codex-plugin-gpt-6-luna-low`), so a bare `agent-eval`, which is +// what CI runs, skips it. +export function onlyWhenNamed( + experimentUrl: string, + evals: EvalName[], + argv: string[] = process.argv +): EvalName[] { + const name = path.basename(fileURLToPath(experimentUrl), '.ts'); + return argv.some((arg) => path.basename(arg, '.ts') === name) ? evals : []; +} + const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); const execFileAsync = promisify(execFile); let checkoutRevision: Promise<{ commit: string; dirty: boolean }> | undefined; From 92b24dee0d327e76f4519dc4b0e1f5b46f7a8356 Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Thu, 8 Oct 2026 12:24:04 +0200 Subject: [PATCH 27/32] fix(skills): get Haiku 5.5 at low effort through the story evals - The stories skill description names running story tests as a reason to invoke it. Haiku ran npm run test:stories before opening the skill and failed eval 810 in 3 of 3 runs; it now passes 3 of 3, and 811 went from 2 of 3 to 3 of 3. - The test step says it is the only way to run story tests, as next's guide did, also when the user asks to run the tests. - The lookup step says to run docs list and docs show before editing any file. Haiku went straight to the code and failed 803 in 2 of 3 runs; it now passes 6 of 6. - Add cc-plugin-haiku-5.5-low as an experiment that runs only when named. Co-Authored-By: Claude Opus 5.5 (1M context) --- agent-eval/README.md | 3 ++- .../experiments/cc-plugin-haiku-5.5-low.ts | 27 +++++++++++++++++++ .../cli/skills/content/build-stories-skill.ts | 4 +-- .../lib/claude-plugin/skills/stories/SKILL.md | 2 +- .../plugins/storybook/skills/stories/SKILL.md | 2 +- 5 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 agent-eval/experiments/cc-plugin-haiku-5.5-low.ts diff --git a/agent-eval/README.md b/agent-eval/README.md index 923c6c333edd..9d4c6a2cd6f6 100644 --- a/agent-eval/README.md +++ b/agent-eval/README.md @@ -133,8 +133,9 @@ via `ANTHROPIC_API_KEY`; Codex experiments use the direct Codex API via - `codex-mcp-gpt-6.1-sol-medium`: Codex (gpt-6.1-sol at medium reasoning effort) with project-local Storybook MCP config in `.codex/config.toml` and the Storybook MCP server instructions in `AGENTS.md` (none with `EVAL_STORYBOOK_LATEST=1`). - `codex-plugin-gpt-6.1-sol-low`: Codex (gpt-6.1-sol at low reasoning effort) with Storybook plugin skills copied to `.agents/skills`. -These four are the default set: a bare `yarn eval`, which is what CI runs, runs them and they have to stay green. Three more plugin experiments run only when you name them, for example `yarn exec agent-eval codex-plugin-gpt-6-luna-low`: +These four are the default set: a bare `yarn eval`, which is what CI runs, runs them and they have to stay green. Four more plugin experiments run only when you name them, for example `yarn exec agent-eval codex-plugin-gpt-6-luna-low`: +- `cc-plugin-haiku-5.5-low`: Claude Code (Haiku 5.5 at low effort). - `cc-plugin-opus-5.5-medium`: Claude Code (Opus 5.5 at medium effort). - `codex-plugin-gpt-6.1-sol-medium`: Codex (gpt-6.1-sol at medium reasoning effort). - `codex-plugin-gpt-6-luna-low`: Codex (gpt-6-luna at low reasoning effort). Its Codex system prompt says not to run tests unless asked, so evals do not require it to run the story tests itself. diff --git a/agent-eval/experiments/cc-plugin-haiku-5.5-low.ts b/agent-eval/experiments/cc-plugin-haiku-5.5-low.ts new file mode 100644 index 000000000000..336a0aee646b --- /dev/null +++ b/agent-eval/experiments/cc-plugin-haiku-5.5-low.ts @@ -0,0 +1,27 @@ +import type { ExperimentConfig } from '@vercel/agent-eval'; +import { + DEFAULT_EXPERIMENT_CONFIG, + onlyWhenNamed, + PLUGIN_STORYBOOK_EVALS, +} from '../lib/experiment.ts'; +import { + setupSandbox, + writeClaudeInAppBrowserMock, + writeClaudePluginSkills, +} from '../lib/templates.ts'; + +const model = 'claude-haiku-5-5'; + +export default { + ...DEFAULT_EXPERIMENT_CONFIG, + agent: 'claude-code', // direct Anthropic API, requires ANTHROPIC_API_KEY + model, + agentOptions: { effort: 'low' }, + // Not in the default set: runs only when named. Also skipped under EVAL_STORYBOOK_LATEST=1. + evals: onlyWhenNamed(import.meta.url, PLUGIN_STORYBOOK_EVALS), + setup: async (sandbox) => { + await setupSandbox(sandbox, { agent: 'claude-code', integration: 'plugin', model }); + await writeClaudePluginSkills(sandbox); + await writeClaudeInAppBrowserMock(sandbox); + }, +} satisfies ExperimentConfig; diff --git a/code/core/src/cli/skills/content/build-stories-skill.ts b/code/core/src/cli/skills/content/build-stories-skill.ts index cd27f111f489..60a8b653a36b 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.ts @@ -23,7 +23,7 @@ ${ref('docs.show')} --id # props and usage examples of one entry ${ref('docs.showStory')} --storyId # the code of a story that docs show only lists \`\`\` -Run \`docs list\` once at the start, then \`docs show\` for every listed component you use, even one you only add such as a button, and for each one you are asked about. Reuse what exists instead of building a duplicate. Use only the props that \`docs show\` documents, never one you assume from a name or another library, and answer props, API and usage questions from these commands. Read source files or \`node_modules\` only when the commands return nothing relevant. When \`docs list\` groups its entries under sources (\`id: acme\`), pass the source of the entry to both commands that show it: \`--storybookId acme\`.`; +Do this before you edit any file: run \`docs list\` once, then \`docs show\` for every listed component your change uses, even one you only add such as a button, and for each one you are asked about. Reuse what exists instead of building a duplicate. Use only the props that \`docs show\` documents, never one you assume from a name or another library, and answer props, API and usage questions from these commands. Read source files or \`node_modules\` only when the commands return nothing relevant. When \`docs list\` groups its entries under sources (\`id: acme\`), pass the source of the entry to both commands that show it: \`--storybookId acme\`.`; } function writeSection({ framework, csfFactories, previewFile }: StoriesSkillInputs): string { @@ -52,7 +52,7 @@ function testSection(a11yEnabled: boolean): string { ${ref('test.run')} --stories '[{"storyId":""}]' # ids: see "Find the stories"; leave out --stories to run every story \`\`\` -Run this after every change, instead of a \`package.json\` test script. Use focused runs while iterating and one full run before you finish. Fix failures and rerun; never finish with failing tests.${a11y}`; +Run this after every change. It is the only way to run story tests: never a \`package.json\` test script (such as \`npm run test:stories\`) or \`vitest\` directly, also when the user asks you to run the tests. Use focused runs while iterating and one full run before you finish. Fix failures and rerun; never finish with failing tests.${a11y}`; } const affectedStoryCommands = ({ moduleGraphSupported }: StoriesSkillInputs) => diff --git a/code/lib/claude-plugin/skills/stories/SKILL.md b/code/lib/claude-plugin/skills/stories/SKILL.md index 2e4790bb93f2..0c8177896a3c 100644 --- a/code/lib/claude-plugin/skills/stories/SKILL.md +++ b/code/lib/claude-plugin/skills/stories/SKILL.md @@ -1,6 +1,6 @@ --- name: stories -description: Invoke FIRST, before creating, editing, or deleting components, stories, styles, CSS, themes, colors, or design tokens — anything that changes how the UI looks, no exceptions. Also before answering a question about a component's props, API, or usage, to start or preview Storybook, and to show, browse, or list components, stories, or UI states. +description: Invoke FIRST, before creating, editing, or deleting components, stories, styles, CSS, themes, colors, or design tokens — anything that changes how the UI looks, no exceptions. Also before running story tests, answering a question about a component's props, API, or usage, starting Storybook, or showing or listing components, stories, or UI states. --- Prerequisites: diff --git a/code/lib/codex-plugin/plugins/storybook/skills/stories/SKILL.md b/code/lib/codex-plugin/plugins/storybook/skills/stories/SKILL.md index c99a7929033f..17e2c7276605 100644 --- a/code/lib/codex-plugin/plugins/storybook/skills/stories/SKILL.md +++ b/code/lib/codex-plugin/plugins/storybook/skills/stories/SKILL.md @@ -1,6 +1,6 @@ --- name: stories -description: Invoke FIRST, before creating, editing, or deleting components, stories, styles, CSS, themes, colors, or design tokens — anything that changes how the UI looks, no exceptions. Also before answering a question about a component's props, API, or usage, to start or preview Storybook, and to show, browse, or list components, stories, or UI states. +description: Invoke FIRST, before creating, editing, or deleting components, stories, styles, CSS, themes, colors, or design tokens — anything that changes how the UI looks, no exceptions. Also before running story tests, answering a question about a component's props, API, or usage, starting Storybook, or showing or listing components, stories, or UI states. --- Prerequisites: From 72c782239dd18877648e6a912d7c4651c0dbb127 Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Thu, 8 Oct 2026 13:07:32 +0200 Subject: [PATCH 28/32] fix(skills): say how to tell the MSW addon version and where a11y results appear - Sonnet 5.5 at low effort read msw-storybook-addon v3 as v2 from the preview and wrote parameters.msw.handlers (eval 814, 1 of 3 runs; two more runs named v3 "v2"). The workflow now says to read the version in package.json; Sonnet then read it and named v3 in 6 of 6 runs of 814. - One run claimed accessibility violations that the test run never reported. The workflow now says they are listed under ## Accessibility Violations. 24 of 24 targeted runs passed on Sonnet, Haiku and GPT-6.1-Sol low. Co-Authored-By: Claude Opus 5.5 (1M context) --- code/core/src/cli/skills/content/build-stories-skill.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/core/src/cli/skills/content/build-stories-skill.ts b/code/core/src/cli/skills/content/build-stories-skill.ts index 60a8b653a36b..fc2e0565a565 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.ts @@ -38,13 +38,13 @@ Every component you create or change gets stories: one per distinct state it can - ${format} \`fn\`, \`expect\`, \`mocked\` and \`sb\` from \`storybook/test\`. - \`play: async ({ canvas, userEvent }) => { ... }\`: query \`canvas\` directly, by role or label. Never wrap it in \`within()\`. \`userEvent.click(element)\` takes no options. -- A story never calls a real service. Mock network requests with MSW when the project has it (\`msw-storybook-addon\` in \`${previewFile}\`): give the story \`beforeEach({ msw }) { msw.use(http.get('/api/users', () => HttpResponse.json([...]))) }\`, and leave the code that calls \`fetch\` unmocked. Only version 2 of that addon has no \`msw\` there: it takes \`parameters: { msw: { handlers: [...] } }\`. +- A story never calls a real service. Mock network requests with MSW when the project has it (\`msw-storybook-addon\` in \`${previewFile}\`): give the story \`beforeEach({ msw }) { msw.use(http.get('/api/users', () => HttpResponse.json([...]))) }\`, and leave the code that calls \`fetch\` unmocked. Read its version in \`package.json\`: only version 2 has no \`msw\` there and takes \`parameters: { msw: { handlers: [...] } }\` instead. - Mock a module only for what MSW cannot reach (no MSW in the project, or a dependency that is not a network call): register it in \`${previewFile}\` with \`sb.mock(import('../src/api.ts'), { spy: true })\` (path relative to that file, with its extension), then set the result per story in \`beforeEach\` with \`mocked(getUser).mockResolvedValue(...)\`.`; } function testSection(a11yEnabled: boolean): string { const a11y = a11yEnabled - ? '\n\nThe run also reports accessibility violations. Fix semantic ones yourself (roles, labels, alt text, keyboard access). For visual ones such as color contrast, do not change the design: describe the problem, offer two or three options and ask the user.' + ? '\n\nThe run lists accessibility violations under `## Accessibility Violations`. Fix semantic ones yourself (roles, labels, alt text, keyboard access). For visual ones such as color contrast, do not change the design: describe the problem, offer two or three options and ask the user.' : ''; return `## Test From 5af8d4212a9d6465c684f7bfe7bda24d17042015 Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Thu, 8 Oct 2026 13:25:48 +0200 Subject: [PATCH 29/32] fix: apply the final review of the stories workflow and the eval README - Without a module graph neither stories changed nor find-by-component works (both need core/module-graph). The workflow no longer names them in that case and takes story ids from docs list --withStoryIds, or by file and export when docs are off too. - The discovery sentence at the review step applies when the agent changed code, not to browse requests. - The MSW version sentence says what differs before version 3 instead of "no msw there", which now read as referring to package.json. - One typescript input from inputs.ts replaces a second extension regex, and also picks the extension in the sb.mock example. - README: a bare yarn eval runs the default experiments; the cost of a full run and the exact-name rule for named-only experiments. Co-Authored-By: Claude Opus 5.5 (1M context) --- agent-eval/README.md | 11 +++-- .../content/build-stories-skill.test.ts | 27 ++++++++--- .../cli/skills/content/build-stories-skill.ts | 48 +++++++++++-------- code/core/src/cli/skills/inputs.test.ts | 3 ++ code/core/src/cli/skills/inputs.ts | 7 +-- code/core/src/cli/skills/run.test.ts | 1 + code/core/src/cli/skills/run.ts | 1 + 7 files changed, 64 insertions(+), 34 deletions(-) diff --git a/agent-eval/README.md b/agent-eval/README.md index 9d4c6a2cd6f6..6a788831be02 100644 --- a/agent-eval/README.md +++ b/agent-eval/README.md @@ -37,7 +37,7 @@ yarn eval:dry ### Run Experiments -Run all configured experiments: +Run the default experiments: ```bash yarn eval @@ -66,9 +66,10 @@ EVAL_ONLY=803-edit-component yarn eval ``` A full `EVAL_ALL=1` run (13 workflow evals × 4 experiments + 4 -lifecycle evals × 2 plugin experiments) costs roughly **$30–45** in agent -tokens at current per-run averages ($0.30–0.80 per workflow eval, $1–2 per -lifecycle eval). The budget guardrail is **$75 per full run** — check the +lifecycle evals × 2 plugin experiments) costs roughly **$7** in agent +tokens at the averages measured in October 2026 (about $0.05–0.17 per workflow +eval on the low-effort plugin experiments and the medium MCP experiments, and +$0.07–0.16 per lifecycle eval). The budget guardrail is **$75 per full run** — check the usage metadata in the results playground before growing the eval set past it (see [storybookjs/mcp#324](https://github.com/storybookjs/mcp/pull/324)). @@ -133,7 +134,7 @@ via `ANTHROPIC_API_KEY`; Codex experiments use the direct Codex API via - `codex-mcp-gpt-6.1-sol-medium`: Codex (gpt-6.1-sol at medium reasoning effort) with project-local Storybook MCP config in `.codex/config.toml` and the Storybook MCP server instructions in `AGENTS.md` (none with `EVAL_STORYBOOK_LATEST=1`). - `codex-plugin-gpt-6.1-sol-low`: Codex (gpt-6.1-sol at low reasoning effort) with Storybook plugin skills copied to `.agents/skills`. -These four are the default set: a bare `yarn eval`, which is what CI runs, runs them and they have to stay green. Four more plugin experiments run only when you name them, for example `yarn exec agent-eval codex-plugin-gpt-6-luna-low`: +These four are the default set: a bare `yarn eval`, which is what CI runs, runs them and they have to stay green. Four more plugin experiments run only when you name them by their exact file name (a glob does not select them), for example `yarn exec agent-eval codex-plugin-gpt-6-luna-low`: - `cc-plugin-haiku-5.5-low`: Claude Code (Haiku 5.5 at low effort). - `cc-plugin-opus-5.5-medium`: Claude Code (Opus 5.5 at medium effort). diff --git a/code/core/src/cli/skills/content/build-stories-skill.test.ts b/code/core/src/cli/skills/content/build-stories-skill.test.ts index 112e54095ef1..aea53de54276 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.test.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.test.ts @@ -6,6 +6,7 @@ const everything: StoriesSkillInputs = { framework: '@storybook/react-vite', csfFactories: false, previewFile: '.storybook/preview.ts', + typescript: true, docsEnabled: true, testSupported: true, a11yEnabled: true, @@ -40,18 +41,25 @@ describe('buildStoriesSkill', () => { expect(text).not.toContain('Run one of these'); }); - it('speaks of one command when only stories changed can find stories', () => { + it('takes story ids from the story list when the builder has no module graph', () => { + const text = buildStoriesSkill({ ...everything, moduleGraphSupported: false }); + + expect(text).toContain('docs list --withStoryIds true'); + expect(text).toContain('Story ids come only from this command.'); + expect(text).toContain('When it does not find a story'); + expect(text).not.toContain('stories changed'); + expect(text).not.toContain('find-by-component'); + }); + + it('selects stories by file and export when no command lists story ids', () => { const text = buildStoriesSkill({ ...everything, docsEnabled: false, moduleGraphSupported: false, }); - expect(text).toContain('Run `stories changed` before every review'); - expect(text).toContain('it adds the stories of other components'); - expect(text).toContain('Story ids come only from this command.'); - expect(text).toContain('When it does not find a story'); - expect(text).not.toContain('find-by-component'); + expect(text).toContain('"absoluteStoryPath"'); + expect(text).not.toContain('stories changed'); }); it('teaches CSF Factories instead of Meta and StoryObj when the preview uses definePreview', () => { @@ -74,10 +82,15 @@ describe('buildStoriesSkill', () => { }); it('leaves out the Meta and StoryObj types in a JavaScript project', () => { - const text = buildStoriesSkill({ ...everything, previewFile: '.storybook/preview.js' }); + const text = buildStoriesSkill({ + ...everything, + previewFile: '.storybook/preview.js', + typescript: false, + }); expect(text).not.toContain('StoryObj'); expect(text).toContain('- Import `fn`, `expect`, `mocked` and `sb` from `storybook/test`.'); + expect(text).toContain("sb.mock(import('../src/api.js')"); }); it('puts MSW before module mocks and names the preview file of the project', () => { diff --git a/code/core/src/cli/skills/content/build-stories-skill.ts b/code/core/src/cli/skills/content/build-stories-skill.ts index fc2e0565a565..75651a4f265a 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.ts @@ -6,6 +6,7 @@ export type StoriesSkillInputs = { csfFactories: boolean; /** Path of the preview file, relative to the working directory. */ previewFile: string; + typescript: boolean; docsEnabled: boolean; testSupported: boolean; a11yEnabled: boolean; @@ -26,10 +27,15 @@ ${ref('docs.showStory')} --storyId # the code of a story that docs show o Do this before you edit any file: run \`docs list\` once, then \`docs show\` for every listed component your change uses, even one you only add such as a button, and for each one you are asked about. Reuse what exists instead of building a duplicate. Use only the props that \`docs show\` documents, never one you assume from a name or another library, and answer props, API and usage questions from these commands. Read source files or \`node_modules\` only when the commands return nothing relevant. When \`docs list\` groups its entries under sources (\`id: acme\`), pass the source of the entry to both commands that show it: \`--storybookId acme\`.`; } -function writeSection({ framework, csfFactories, previewFile }: StoriesSkillInputs): string { +function writeSection({ + framework, + csfFactories, + previewFile, + typescript, +}: StoriesSkillInputs): string { const format = csfFactories ? `This project writes stories with CSF Factories, because \`${previewFile}\` uses \`definePreview\`: \`import preview from '../.storybook/preview'\` (the path to that file), then \`const meta = preview.meta({ component: Button })\` and \`export const Primary = meta.story({ args: { ... } })\`. Do not import \`Meta\` or \`StoryObj\`. Import` - : /\.[cm]?tsx?$/.test(previewFile) + : typescript ? `Import \`Meta\` and \`StoryObj\` from \`${framework}\`, and` : 'Import'; return `## Write the component and its stories @@ -38,8 +44,8 @@ Every component you create or change gets stories: one per distinct state it can - ${format} \`fn\`, \`expect\`, \`mocked\` and \`sb\` from \`storybook/test\`. - \`play: async ({ canvas, userEvent }) => { ... }\`: query \`canvas\` directly, by role or label. Never wrap it in \`within()\`. \`userEvent.click(element)\` takes no options. -- A story never calls a real service. Mock network requests with MSW when the project has it (\`msw-storybook-addon\` in \`${previewFile}\`): give the story \`beforeEach({ msw }) { msw.use(http.get('/api/users', () => HttpResponse.json([...]))) }\`, and leave the code that calls \`fetch\` unmocked. Read its version in \`package.json\`: only version 2 has no \`msw\` there and takes \`parameters: { msw: { handlers: [...] } }\` instead. -- Mock a module only for what MSW cannot reach (no MSW in the project, or a dependency that is not a network call): register it in \`${previewFile}\` with \`sb.mock(import('../src/api.ts'), { spy: true })\` (path relative to that file, with its extension), then set the result per story in \`beforeEach\` with \`mocked(getUser).mockResolvedValue(...)\`.`; +- A story never calls a real service. Mock network requests with MSW when the project has it (\`msw-storybook-addon\` in \`${previewFile}\`): give the story \`beforeEach({ msw }) { msw.use(http.get('/api/users', () => HttpResponse.json([...]))) }\`, and leave the code that calls \`fetch\` unmocked. Read the addon's version in \`package.json\`: before version 3, stories have no \`msw\` in \`beforeEach\` and take \`parameters: { msw: { handlers: [...] } }\` instead. +- Mock a module only for what MSW cannot reach (no MSW in the project, or a dependency that is not a network call): register it in \`${previewFile}\` with \`sb.mock(import('../src/api.${typescript ? 'ts' : 'js'}'), { spy: true })\` (path relative to that file, with its extension), then set the result per story in \`beforeEach\` with \`mocked(getUser).mockResolvedValue(...)\`.`; } function testSection(a11yEnabled: boolean): string { @@ -55,29 +61,30 @@ ${ref('test.run')} --stories '[{"storyId":""}]' # ids: see "Find the stori Run this after every change. It is the only way to run story tests: never a \`package.json\` test script (such as \`npm run test:stories\`) or \`vitest\` directly, also when the user asks you to run the tests. Use focused runs while iterating and one full run before you finish. Fix failures and rerun; never finish with failing tests.${a11y}`; } -const affectedStoryCommands = ({ moduleGraphSupported }: StoriesSkillInputs) => - moduleGraphSupported ? '`stories changed` or `find-by-component`' : '`stories changed`'; - function discoverSection(inputs: StoriesSkillInputs): string { const commands = [ - `${ref('stories.changed')} # stories affected by your uncommitted changes`, + inputs.moduleGraphSupported && + `${ref('stories.changed')} # stories affected by your uncommitted changes`, inputs.moduleGraphSupported && `${ref('stories.findByComponent')} --componentPaths '["/abs/path/Button.tsx"]' # stories that render these files`, inputs.docsEnabled && `${ref('docs.list')} --withStoryIds true # every story id`, ].filter(Boolean); - const fallback = !inputs.moduleGraphSupported - ? [] - : [ - 'When \`stories changed\` leaves out a file you touched, pass that file to \`find-by-component\`; for a shared file (design token, theme, util, hook), which has no stories of its own, pass the components that use it.', - 'When \`find-by-component\` reports stories hidden by \`maxDistance\`, rerun it with a higher \`--maxDistance\`.', - ]; + if (commands.length === 0) { + return `## Find the stories + +This project has no command that lists story ids. Select a story by its file and export instead: \`--stories '[{"absoluteStoryPath":"/abs/path/Button.stories.tsx","exportName":"Primary"}]'\`.`; + } const several = commands.length > 1; const guidance = [ - `Run ${affectedStoryCommands(inputs)} before every review, also when you already know the ids of the stories you wrote: ${inputs.moduleGraphSupported ? 'they add' : 'it adds'} the stories of other components that your change affects.`, + inputs.moduleGraphSupported && + 'Run \`stories changed\` or \`find-by-component\` before every review, also when you already know the ids of the stories you wrote: they add the stories of other components that your change affects.', `Story ids come only from ${several ? 'these commands' : 'this command'}. Never build one from a file name, a title or memory.`, - ...fallback, + inputs.moduleGraphSupported && + 'When \`stories changed\` leaves out a file you touched, pass that file to \`find-by-component\`; for a shared file (design token, theme, util, hook), which has no stories of its own, pass the components that use it. When \`find-by-component\` reports stories hidden by \`maxDistance\`, rerun it with a higher \`--maxDistance\`.', `When ${several ? 'none of them finds' : 'it does not find'} a story for a component, it has no stories yet: say so, or write them.`, - ].join(' '); + ] + .filter(Boolean) + .join(' '); return `## Find the stories \`\`\`sh @@ -87,7 +94,10 @@ ${commands.join('\n')} ${guidance}`; } -function reviewSection(inputs: StoriesSkillInputs): string { +function reviewSection({ moduleGraphSupported }: StoriesSkillInputs): string { + const discoverFirst = moduleGraphSupported + ? ' When you changed code, run \`stories changed\` or \`find-by-component\` right before it and take the story ids from there, also when you only wrote stories, not from the output of the tests.' + : ''; return `## Finish with a review \`\`\`sh @@ -102,7 +112,7 @@ ${ref('review.create')} --input '{ }' \`\`\` -Publish a review after every change the user can see, and again after each later change. It needs a running Storybook. Right before it, run ${affectedStoryCommands(inputs)} and take the story ids from there, also when you only wrote stories, not from the output of the tests. Group the stories into two to five collections, from the changed component up to the pages that show it (one is enough when a single component is affected), and include every story you created. When the user asks to see or browse components or stories and no code changed, publish the same review with \`"changedFiles": []\`. Skip the review only when nothing visible changed and the user did not ask to see anything, and say that instead. +Publish a review after every change the user can see, and again after each later change. It needs a running Storybook.${discoverFirst} Group the stories into two to five collections, from the changed component up to the pages that show it (one is enough when a single component is affected), and include every story you created. When the user asks to see or browse components or stories and no code changed, publish the same review with \`"changedFiles": []\`. Skip the review only when nothing visible changed and the user did not ask to see anything, and say that instead. Then do both things the command prints, every time: open the review in the in-app browser with a browser tool (unless you have none), and end your answer with the review section it gives you. Do not list separate story links next to it.`; } diff --git a/code/core/src/cli/skills/inputs.test.ts b/code/core/src/cli/skills/inputs.test.ts index 17ae7faa2a55..1822767ce357 100644 --- a/code/core/src/cli/skills/inputs.test.ts +++ b/code/core/src/cli/skills/inputs.test.ts @@ -102,6 +102,7 @@ describe('resolveSkillInputs', () => { expect(inputs).toMatchObject({ csfFactories: true, previewFile: '.storybook/preview.tsx', + typescript: true, }); }); @@ -117,6 +118,7 @@ describe('resolveSkillInputs', () => { expect(inputs).toMatchObject({ csfFactories: false, previewFile: '.storybook/preview.js', + typescript: false, }); }); @@ -130,6 +132,7 @@ describe('resolveSkillInputs', () => { expect(inputs).toMatchObject({ csfFactories: false, previewFile: '.storybook/preview.ts', + typescript: true, }); }); }); diff --git a/code/core/src/cli/skills/inputs.ts b/code/core/src/cli/skills/inputs.ts index e20ae1d65e1b..8384a6b80ae0 100644 --- a/code/core/src/cli/skills/inputs.ts +++ b/code/core/src/cli/skills/inputs.ts @@ -15,6 +15,8 @@ export type SkillInputs = ToolAvailability & { csfFactories: boolean; /** Path of the preview file relative to the working directory; a default name when there is none. */ previewFile: string; + /** The preview file, or `main` when there is none, is TypeScript. */ + typescript: boolean; }; const isTypeScriptFile = (path: string) => /\.[cm]?tsx?$/.test(path); @@ -27,12 +29,11 @@ async function resolvePreview(configDir = '.storybook') { if (previewPath) { try { csfFactories = isCsfFactoryPreview(await readConfig(previewPath)); - } catch { - // An unparsable preview is treated as a non-factory one. - } + } catch {} } return { csfFactories, + typescript, previewFile: relative(process.cwd(), path).replaceAll('\\', '/') || path, }; } diff --git a/code/core/src/cli/skills/run.test.ts b/code/core/src/cli/skills/run.test.ts index fbac94538ec9..602ecb363cf4 100644 --- a/code/core/src/cli/skills/run.test.ts +++ b/code/core/src/cli/skills/run.test.ts @@ -31,6 +31,7 @@ const deps = () => ({ renderer: '@storybook/react', csfFactories: false, previewFile: '.storybook/preview.ts', + typescript: true, moduleGraphSupported: true, docsEnabled: false, docsEnabledForCli: false, diff --git a/code/core/src/cli/skills/run.ts b/code/core/src/cli/skills/run.ts index fbed5066995f..7efce816507d 100644 --- a/code/core/src/cli/skills/run.ts +++ b/code/core/src/cli/skills/run.ts @@ -228,6 +228,7 @@ function assemble(id: Exclude, inputs: SkillInputs): string { framework: inputs.framework, csfFactories: inputs.csfFactories, previewFile: inputs.previewFile, + typescript: inputs.typescript, docsEnabled: inputs.docsEnabledForCli, testSupported: inputs.testSupported, a11yEnabled: inputs.a11yEnabled, From c237c226e9ab5bf9242c37d474ffff342a7cecbc Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Thu, 8 Oct 2026 14:05:27 +0200 Subject: [PATCH 30/32] fix(skills): speak of changed files, not code, at the review step "When you changed code" and "no code changed" let Haiku 5.5 at low effort treat a task that only wrote stories as a browse request: it published the review of eval 804 with changedFiles: [] (1 of 3 runs). Both sentences now speak of files; Haiku then passed 804 in 6 of 6 runs, and 804, 806 and 812 passed 3 of 3 on Sonnet 5.5 low and GPT-6.1-Sol low, with changedFiles filled for written stories and empty for the browse request. Co-Authored-By: Claude Opus 5.5 (1M context) --- code/core/src/cli/skills/content/build-stories-skill.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/core/src/cli/skills/content/build-stories-skill.ts b/code/core/src/cli/skills/content/build-stories-skill.ts index 75651a4f265a..28f6d129f526 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.ts @@ -96,7 +96,7 @@ ${guidance}`; function reviewSection({ moduleGraphSupported }: StoriesSkillInputs): string { const discoverFirst = moduleGraphSupported - ? ' When you changed code, run \`stories changed\` or \`find-by-component\` right before it and take the story ids from there, also when you only wrote stories, not from the output of the tests.' + ? ' When you changed files, run \`stories changed\` or \`find-by-component\` right before it and take the story ids from there, also when you only wrote stories, not from the output of the tests.' : ''; return `## Finish with a review @@ -112,7 +112,7 @@ ${ref('review.create')} --input '{ }' \`\`\` -Publish a review after every change the user can see, and again after each later change. It needs a running Storybook.${discoverFirst} Group the stories into two to five collections, from the changed component up to the pages that show it (one is enough when a single component is affected), and include every story you created. When the user asks to see or browse components or stories and no code changed, publish the same review with \`"changedFiles": []\`. Skip the review only when nothing visible changed and the user did not ask to see anything, and say that instead. +Publish a review after every change the user can see, and again after each later change. It needs a running Storybook.${discoverFirst} Group the stories into two to five collections, from the changed component up to the pages that show it (one is enough when a single component is affected), and include every story you created. When the user only asks to see or browse components or stories and you changed no files, publish the same review with \`"changedFiles": []\`. Skip the review only when nothing visible changed and the user did not ask to see anything, and say that instead. Then do both things the command prints, every time: open the review in the in-app browser with a browser tool (unless you have none), and end your answer with the review section it gives you. Do not list separate story links next to it.`; } From 18e90478ef8f15db0b25fc1a509ae60ddab9dca5 Mon Sep 17 00:00:00 2001 From: Kasper Peulen Date: Thu, 8 Oct 2026 15:21:43 +0200 Subject: [PATCH 31/32] test(agent-eval): make Haiku 5.5 low the default Claude Code plugin experiment Sonnet 5.5 low stays in the repo and runs only when named. Co-Authored-By: Claude Opus 5.5 (1M context) --- agent-eval/README.md | 13 ++++++------- agent-eval/experiments/cc-plugin-haiku-5.5-low.ts | 10 +++------- agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts | 10 +++++++--- agent-eval/lib/usage.ts | 8 ++++++++ 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/agent-eval/README.md b/agent-eval/README.md index 6a788831be02..3346db868746 100644 --- a/agent-eval/README.md +++ b/agent-eval/README.md @@ -65,11 +65,10 @@ EVAL_ALL=1 yarn eval EVAL_ONLY=803-edit-component yarn eval ``` -A full `EVAL_ALL=1` run (13 workflow evals × 4 experiments + 4 -lifecycle evals × 2 plugin experiments) costs roughly **$7** in agent -tokens at the averages measured in October 2026 (about $0.05–0.17 per workflow -eval on the low-effort plugin experiments and the medium MCP experiments, and -$0.07–0.16 per lifecycle eval). The budget guardrail is **$75 per full run** — check the +A full `EVAL_ALL=1` run (14 workflow evals × 4 experiments + 4 +lifecycle evals × 2 plugin experiments) costs roughly **$6** in agent +tokens at the averages measured in October 2026 (under $0.01 per eval on Haiku 5.5 low, and about +$0.05–0.17 per eval on GPT-6.1-Sol low and the medium MCP experiments). The budget guardrail is **$75 per full run** — check the usage metadata in the results playground before growing the eval set past it (see [storybookjs/mcp#324](https://github.com/storybookjs/mcp/pull/324)). @@ -130,13 +129,13 @@ via `ANTHROPIC_API_KEY`; Codex experiments use the direct Codex API via `OPENAI_API_KEY`): - `cc-mcp-opus-5.5-medium`: Claude Code (Opus 5.5 at medium effort) with project-local Storybook MCP config in `.mcp.json`. -- `cc-plugin-sonnet-5.5-low`: Claude Code (Sonnet 5.5 at low effort) with Storybook plugin skills copied to `.claude/skills`. +- `cc-plugin-haiku-5.5-low`: Claude Code (Haiku 5.5 at low effort) with Storybook plugin skills copied to `.claude/skills`. - `codex-mcp-gpt-6.1-sol-medium`: Codex (gpt-6.1-sol at medium reasoning effort) with project-local Storybook MCP config in `.codex/config.toml` and the Storybook MCP server instructions in `AGENTS.md` (none with `EVAL_STORYBOOK_LATEST=1`). - `codex-plugin-gpt-6.1-sol-low`: Codex (gpt-6.1-sol at low reasoning effort) with Storybook plugin skills copied to `.agents/skills`. These four are the default set: a bare `yarn eval`, which is what CI runs, runs them and they have to stay green. Four more plugin experiments run only when you name them by their exact file name (a glob does not select them), for example `yarn exec agent-eval codex-plugin-gpt-6-luna-low`: -- `cc-plugin-haiku-5.5-low`: Claude Code (Haiku 5.5 at low effort). +- `cc-plugin-sonnet-5.5-low`: Claude Code (Sonnet 5.5 at low effort). - `cc-plugin-opus-5.5-medium`: Claude Code (Opus 5.5 at medium effort). - `codex-plugin-gpt-6.1-sol-medium`: Codex (gpt-6.1-sol at medium reasoning effort). - `codex-plugin-gpt-6-luna-low`: Codex (gpt-6-luna at low reasoning effort). Its Codex system prompt says not to run tests unless asked, so evals do not require it to run the story tests itself. diff --git a/agent-eval/experiments/cc-plugin-haiku-5.5-low.ts b/agent-eval/experiments/cc-plugin-haiku-5.5-low.ts index 336a0aee646b..c3acc4557fd8 100644 --- a/agent-eval/experiments/cc-plugin-haiku-5.5-low.ts +++ b/agent-eval/experiments/cc-plugin-haiku-5.5-low.ts @@ -1,9 +1,5 @@ import type { ExperimentConfig } from '@vercel/agent-eval'; -import { - DEFAULT_EXPERIMENT_CONFIG, - onlyWhenNamed, - PLUGIN_STORYBOOK_EVALS, -} from '../lib/experiment.ts'; +import { DEFAULT_EXPERIMENT_CONFIG, PLUGIN_STORYBOOK_EVALS } from '../lib/experiment.ts'; import { setupSandbox, writeClaudeInAppBrowserMock, @@ -17,8 +13,8 @@ export default { agent: 'claude-code', // direct Anthropic API, requires ANTHROPIC_API_KEY model, agentOptions: { effort: 'low' }, - // Not in the default set: runs only when named. Also skipped under EVAL_STORYBOOK_LATEST=1. - evals: onlyWhenNamed(import.meta.url, PLUGIN_STORYBOOK_EVALS), + // Skipped under EVAL_STORYBOOK_LATEST=1; see PLUGIN_STORYBOOK_EVALS. + evals: PLUGIN_STORYBOOK_EVALS, setup: async (sandbox) => { await setupSandbox(sandbox, { agent: 'claude-code', integration: 'plugin', model }); await writeClaudePluginSkills(sandbox); diff --git a/agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts b/agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts index 333864fef185..1b761f516d2e 100644 --- a/agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts +++ b/agent-eval/experiments/cc-plugin-sonnet-5.5-low.ts @@ -1,5 +1,9 @@ import type { ExperimentConfig } from '@vercel/agent-eval'; -import { DEFAULT_EXPERIMENT_CONFIG, PLUGIN_STORYBOOK_EVALS } from '../lib/experiment.ts'; +import { + DEFAULT_EXPERIMENT_CONFIG, + onlyWhenNamed, + PLUGIN_STORYBOOK_EVALS, +} from '../lib/experiment.ts'; import { setupSandbox, writeClaudeInAppBrowserMock, @@ -13,8 +17,8 @@ export default { agent: 'claude-code', // direct Anthropic API, requires ANTHROPIC_API_KEY model, agentOptions: { effort: 'low' }, - // Skipped under EVAL_STORYBOOK_LATEST=1; see PLUGIN_STORYBOOK_EVALS. - evals: PLUGIN_STORYBOOK_EVALS, + // Not in the default set: runs only when named. Also skipped under EVAL_STORYBOOK_LATEST=1. + evals: onlyWhenNamed(import.meta.url, PLUGIN_STORYBOOK_EVALS), setup: async (sandbox) => { await setupSandbox(sandbox, { agent: 'claude-code', integration: 'plugin', model }); await writeClaudePluginSkills(sandbox); diff --git a/agent-eval/lib/usage.ts b/agent-eval/lib/usage.ts index 435c34c25d1a..c71479305d97 100644 --- a/agent-eval/lib/usage.ts +++ b/agent-eval/lib/usage.ts @@ -12,6 +12,10 @@ const MODEL_PRICING: Record Date: Thu, 8 Oct 2026 17:44:11 +0200 Subject: [PATCH 32/32] fix(skills): mock requests with MSW only, and set it up when the project has none A project without msw-storybook-addon got a stubbed fetch or a rewritten component. The workflow now tells the agent to install and configure MSW, and keeps module mocks for dependencies that are not network calls. Adds eval 815 for that case. Co-Authored-By: Claude Opus 5.5 --- .../.storybook/main.ts | 8 ++ .../.storybook/preview.tsx | 7 ++ .../EVAL.ts | 87 +++++++++++++++++++ .../PROMPT.md | 1 + .../package.json | 23 +++++ .../src/components/OrderHistory.tsx | 40 +++++++++ .../vitest.config.ts | 5 ++ .../vitest.storybook.config.ts | 35 ++++++++ agent-eval/lib/experiment.ts | 1 + .../content/build-stories-skill.test.ts | 15 ++++ .../cli/skills/content/build-stories-skill.ts | 9 +- 11 files changed, 228 insertions(+), 3 deletions(-) create mode 100644 agent-eval/evals/815-write-stories-for-fetching-component-without-msw/.storybook/main.ts create mode 100644 agent-eval/evals/815-write-stories-for-fetching-component-without-msw/.storybook/preview.tsx create mode 100644 agent-eval/evals/815-write-stories-for-fetching-component-without-msw/EVAL.ts create mode 100644 agent-eval/evals/815-write-stories-for-fetching-component-without-msw/PROMPT.md create mode 100644 agent-eval/evals/815-write-stories-for-fetching-component-without-msw/package.json create mode 100644 agent-eval/evals/815-write-stories-for-fetching-component-without-msw/src/components/OrderHistory.tsx create mode 100644 agent-eval/evals/815-write-stories-for-fetching-component-without-msw/vitest.config.ts create mode 100644 agent-eval/evals/815-write-stories-for-fetching-component-without-msw/vitest.storybook.config.ts diff --git a/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/.storybook/main.ts b/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/.storybook/main.ts new file mode 100644 index 000000000000..14702fcbc4e4 --- /dev/null +++ b/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/.storybook/main.ts @@ -0,0 +1,8 @@ +import type { StorybookConfig } from '@storybook/react-vite'; + +const config: StorybookConfig = { + stories: ['../@(stories|src)/**/*.stories.@(js|jsx|mjs|ts|tsx)'], + addons: ['@storybook/addon-a11y', '@storybook/addon-vitest', '@storybook/addon-docs'], + framework: '@storybook/react-vite', +}; +export default config; diff --git a/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/.storybook/preview.tsx b/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/.storybook/preview.tsx new file mode 100644 index 000000000000..46602baafb56 --- /dev/null +++ b/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/.storybook/preview.tsx @@ -0,0 +1,7 @@ +import type { Preview } from '@storybook/react-vite'; + +const preview: Preview = { + parameters: {}, +}; + +export default preview; diff --git a/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/EVAL.ts b/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/EVAL.ts new file mode 100644 index 000000000000..3ac0382aed36 --- /dev/null +++ b/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/EVAL.ts @@ -0,0 +1,87 @@ +import { existsSync, readFileSync, readdirSync } from 'node:fs'; +import { describe, expect, test } from 'vitest'; +import { + expectDisplayReviewForVisualChange, + expectReviewOpenedInBrowser, + expectSkillInvoked, + expectStoryDiscoveryBeforeReview, + expectStoryIdsInDisplayReview, + expectStoryTestsRanAndPassed, + expectWorkflowCalls, + getEvalContext, + modelRunsTestsOnlyWhenAsked, +} from '#test-utils'; + +const STORIES_PATH = 'src/components/OrderHistory.stories.tsx'; + +describe('writing stories for a fetching OrderHistory in a project without MSW', () => { + test('runs story tests after the change and finishes with them passing', async () => { + await expectStoryTestsRanAndPassed({ + requireAgentRun: !modelRunsTestsOnlyWhenAsked(), + covering: ['orderhistory'], + }); + }); + + test('installs MSW and its Storybook addon', () => { + const { dependencies = {}, devDependencies = {} } = JSON.parse( + readFileSync('package.json', 'utf8') + ); + const installed = { ...dependencies, ...devDependencies }; + expect(installed, 'Expected msw to be installed').toHaveProperty('msw'); + expect(installed, 'Expected msw-storybook-addon to be installed').toHaveProperty( + 'msw-storybook-addon' + ); + expect( + existsSync('public/mockServiceWorker.js'), + 'Expected the worker script generated by `msw init`' + ).toBe(true); + const preview = readdirSync('.storybook').find((file) => file.startsWith('preview.')); + expect( + readFileSync(`.storybook/${preview}`, 'utf8'), + 'Expected the preview file to set up msw-storybook-addon' + ).toMatch(/msw-storybook-addon/); + }); + + test('mocks the request with MSW instead of a module mock or a stubbed fetch', () => { + const component = readFileSync('src/components/OrderHistory.tsx', 'utf8'); + expect(component, 'Expected OrderHistory to keep taking no props').toMatch( + /export default function OrderHistory\s*\(\s*\)/ + ); + expect(component, 'Expected OrderHistory to keep fetching /api/orders').toMatch( + /fetch\(\s*['"`]\/api\/orders/ + ); + + expect(existsSync(STORIES_PATH), `Expected the stories in ${STORIES_PATH}`).toBe(true); + const stories = readFileSync(STORIES_PATH, 'utf8'); + expect(stories, 'Expected handlers registered with msw.use').toMatch(/\bmsw\.use\(/); + expect(stories, 'Expected the handlers to mock /api/orders').toMatch(/\/api\/orders/); + expect(stories, 'Expected no module mock or stubbed fetch').not.toMatch( + /\bsb\.mock\(|\bvi\.mock\(|\bmocked\(|\bspyOn\(|\b(globalThis|window)\.fetch\s*=/ + ); + expect( + stories.match(/^export const \w+/gm)?.length ?? 0, + 'Expected loaded, empty, and error stories' + ).toBeGreaterThanOrEqual(3); + }); + + test('uses Storybook story instructions and publishes a display review', () => { + expectWorkflowCalls(['get-storybook-story-instructions', 'review-create']); + expectDisplayReviewForVisualChange(); + }); + + test('opens the review in the in-app browser', () => { + expectReviewOpenedInBrowser(); + }); + + test('the review covers the new OrderHistory stories', () => { + expectStoryIdsInDisplayReview(['orderhistory']); + }); + + test('discovers stories through the workflow tools before publishing the review', () => { + expectStoryDiscoveryBeforeReview(); + }); + + test.skipIf(getEvalContext().integration === 'mcp')('invokes the stories skill', () => { + expectSkillInvoked('stories'); + }); +}); diff --git a/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/PROMPT.md b/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/PROMPT.md new file mode 100644 index 000000000000..0cfee5a96a0f --- /dev/null +++ b/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/PROMPT.md @@ -0,0 +1 @@ +Add stories in `src/components/OrderHistory.stories.tsx` for the existing `OrderHistory` component, covering the loaded, empty, and error states. diff --git a/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/package.json b/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/package.json new file mode 100644 index 000000000000..e9a8abee60e2 --- /dev/null +++ b/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/package.json @@ -0,0 +1,23 @@ +{ + "name": "815-write-stories-for-fetching-component-without-msw", + "type": "module", + "scripts": { + "build-storybook": "storybook build", + "postinstall": "playwright install chromium && node ./scripts/start-storybook-mcp.mjs", + "storybook": "storybook dev", + "test:stories": "vitest run --config ./vitest.storybook.config.ts --project storybook" + }, + "devDependencies": { + "@storybook/addon-a11y": "workspace:*", + "@storybook/addon-docs": "workspace:*", + "@storybook/addon-vitest": "workspace:*", + "@storybook/react": "workspace:*", + "@storybook/react-vite": "workspace:*", + "@vitest/browser-playwright": "4.0.6", + "playwright": "1.56.1", + "storybook": "workspace:*" + }, + "evals": { + "template": "vite-app" + } +} diff --git a/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/src/components/OrderHistory.tsx b/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/src/components/OrderHistory.tsx new file mode 100644 index 000000000000..8e2a8744bfdb --- /dev/null +++ b/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/src/components/OrderHistory.tsx @@ -0,0 +1,40 @@ +import { useEffect, useState } from 'react'; + +type Order = { + id: string; + placedAt: string; + total: number; +}; + +type State = { status: 'loading' } | { status: 'error' } | { status: 'loaded'; orders: Order[] }; + +export default function OrderHistory() { + const [state, setState] = useState({ status: 'loading' }); + + useEffect(() => { + fetch('/api/orders') + .then((response) => { + if (!response.ok) throw new Error(`Request failed with ${response.status}`); + return response.json() as Promise; + }) + .then((orders) => setState({ status: 'loaded', orders })) + .catch(() => setState({ status: 'error' })); + }, []); + + if (state.status === 'loading') return

Loading orders…

; + + if (state.status === 'error') return

Could not load your orders.

; + + if (state.orders.length === 0) return

You have not placed any orders yet.

; + + return ( +
    + {state.orders.map((order) => ( +
  • + Order {order.id} · {new Date(order.placedAt).toLocaleDateString('en-US')} · $ + {order.total.toFixed(2)} +
  • + ))} +
+ ); +} diff --git a/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/vitest.config.ts b/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/vitest.config.ts new file mode 100644 index 000000000000..86e885122ec6 --- /dev/null +++ b/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/vitest.config.ts @@ -0,0 +1,5 @@ +// addon-vitest's test-run starts Vitest from the project root, so the +// storybook test project must be reachable from the default config. The eval +// runner overwrites this file before executing EVAL.ts; it only matters while +// the agent runs. +export { default } from './vitest.storybook.config.ts'; diff --git a/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/vitest.storybook.config.ts b/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/vitest.storybook.config.ts new file mode 100644 index 000000000000..e9c1264ea5bf --- /dev/null +++ b/agent-eval/evals/815-write-stories-for-fetching-component-without-msw/vitest.storybook.config.ts @@ -0,0 +1,35 @@ +import * as path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { defineConfig } from 'vitest/config'; + +import { storybookTest } from '@storybook/addon-vitest/vitest-plugin'; + +import { playwright } from '@vitest/browser-playwright'; + +const dirname = + typeof __dirname !== 'undefined' ? __dirname : path.dirname(fileURLToPath(import.meta.url)); + +// More info at: https://storybook.js.org/docs/next/writing-tests/integrations/vitest-addon +export default defineConfig({ + optimizeDeps: { + include: ['react/jsx-dev-runtime'], + }, + test: { + projects: [ + { + extends: true, + plugins: [storybookTest({ configDir: path.join(dirname, '.storybook') })], + test: { + name: 'storybook', + browser: { + enabled: true, + headless: true, + provider: playwright({}), + instances: [{ browser: 'chromium' }], + }, + }, + }, + ], + }, +}); diff --git a/agent-eval/lib/experiment.ts b/agent-eval/lib/experiment.ts index 2d76b7c32c0a..8758289ae9a1 100644 --- a/agent-eval/lib/experiment.ts +++ b/agent-eval/lib/experiment.ts @@ -22,6 +22,7 @@ const CORE_STORYBOOK_EVALS = [ '812-first-story-empty-project', '813-monorepo-leaf-create-component', '814-write-stories-for-fetching-component', + '815-write-stories-for-fetching-component-without-msw', ] as const; // The 82x block: lifecycle-skill evals (storybook-init / storybook-upgrade). diff --git a/code/core/src/cli/skills/content/build-stories-skill.test.ts b/code/core/src/cli/skills/content/build-stories-skill.test.ts index aea53de54276..2780d711ca21 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.test.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.test.ts @@ -99,4 +99,19 @@ describe('buildStoriesSkill', () => { expect(text.indexOf('msw.use(')).toBeLessThan(text.indexOf('sb.mock(')); expect(text).toContain('register it in `config/preview.tsx`'); }); + + it('tells how to set up MSW in the format of the project', () => { + const csf3 = buildStoriesSkill({ ...everything, previewFile: 'config/preview.tsx' }); + expect(csf3).toContain('never with a module mock or a replaced `fetch`'); + expect(csf3).toContain( + "`loaders: [mswLoader()]` (`import { mswLoader } from 'msw-storybook-addon/csf3'`" + ); + expect(csf3).toContain('to `config/preview.tsx`'); + + const factories = buildStoriesSkill({ ...everything, csfFactories: true }); + expect(factories).toContain( + "`addons: [addonMsw()]` (`import addonMsw from 'msw-storybook-addon'`" + ); + expect(factories).not.toContain('mswLoader'); + }); }); diff --git a/code/core/src/cli/skills/content/build-stories-skill.ts b/code/core/src/cli/skills/content/build-stories-skill.ts index 28f6d129f526..7f5ff33323e2 100644 --- a/code/core/src/cli/skills/content/build-stories-skill.ts +++ b/code/core/src/cli/skills/content/build-stories-skill.ts @@ -38,14 +38,17 @@ function writeSection({ : typescript ? `Import \`Meta\` and \`StoryObj\` from \`${framework}\`, and` : 'Import'; + const mswSetup = csfFactories + ? "`addons: [addonMsw()]` (`import addonMsw from 'msw-storybook-addon'`, without an `initialize()` call)" + : "`loaders: [mswLoader()]` (`import { mswLoader } from 'msw-storybook-addon/csf3'`, without an `initialize()` call)"; return `## Write the component and its stories Every component you create or change gets stories: one per distinct state it can reach (variants, loading, empty, error, disabled), with realistic props. Never export a story under the name of a global such as \`Error\`: export \`ErrorState\` and set \`name: 'Error'\`. An interactive component also gets a \`play\` function that drives it and asserts the visible result; for a callback passed as \`fn()\`, assert whether it was called, as that state expects. The rules below are for the stories you write; do not rewrite existing stories only to match them. - ${format} \`fn\`, \`expect\`, \`mocked\` and \`sb\` from \`storybook/test\`. - \`play: async ({ canvas, userEvent }) => { ... }\`: query \`canvas\` directly, by role or label. Never wrap it in \`within()\`. \`userEvent.click(element)\` takes no options. -- A story never calls a real service. Mock network requests with MSW when the project has it (\`msw-storybook-addon\` in \`${previewFile}\`): give the story \`beforeEach({ msw }) { msw.use(http.get('/api/users', () => HttpResponse.json([...]))) }\`, and leave the code that calls \`fetch\` unmocked. Read the addon's version in \`package.json\`: before version 3, stories have no \`msw\` in \`beforeEach\` and take \`parameters: { msw: { handlers: [...] } }\` instead. -- Mock a module only for what MSW cannot reach (no MSW in the project, or a dependency that is not a network call): register it in \`${previewFile}\` with \`sb.mock(import('../src/api.${typescript ? 'ts' : 'js'}'), { spy: true })\` (path relative to that file, with its extension), then set the result per story in \`beforeEach\` with \`mocked(getUser).mockResolvedValue(...)\`.`; +- A story never calls a real service. Mock network requests with MSW, never with a module mock or a replaced \`fetch\`: give the story \`beforeEach({ msw }) { msw.use(http.get('/api/users', () => HttpResponse.json([...]))) }\`, and leave the code that calls \`fetch\` as it is. When the project has no \`msw-storybook-addon\` yet, set it up first: install \`msw\` and \`msw-storybook-addon\` as dev dependencies, run \`npx msw init ./public --save\` (\`../public\` has to be in \`staticDirs\` of the main config), and add ${mswSetup} to \`${previewFile}\`. Read the addon's version in \`package.json\`: only before version 3 do stories have no \`msw\` in \`beforeEach\` and take \`parameters: { msw: { handlers: [...] } }\` instead. From version 3 on, never write \`parameters.msw\`. +- Mock a module only for a dependency that is not a network call: register it in \`${previewFile}\` with \`sb.mock(import('../src/api.${typescript ? 'ts' : 'js'}'), { spy: true })\` (path relative to that file, with its extension), then set the result per story in \`beforeEach\` with \`mocked(getUser).mockResolvedValue(...)\`.`; } function testSection(a11yEnabled: boolean): string { @@ -112,7 +115,7 @@ ${ref('review.create')} --input '{ }' \`\`\` -Publish a review after every change the user can see, and again after each later change. It needs a running Storybook.${discoverFirst} Group the stories into two to five collections, from the changed component up to the pages that show it (one is enough when a single component is affected), and include every story you created. When the user only asks to see or browse components or stories and you changed no files, publish the same review with \`"changedFiles": []\`. Skip the review only when nothing visible changed and the user did not ask to see anything, and say that instead. +Publish a review after every change the user can see, and again after each later change. It needs a running Storybook.${discoverFirst} Group the stories into two to five collections, from the changed component up to the pages that show it (one is enough when a single component is affected), and include every story you created. List every file you created or changed in \`changedFiles\`, story files too. When the user only asks to see or browse components or stories and you changed no files, publish the same review with \`"changedFiles": []\`. Skip the review only when nothing visible changed and the user did not ask to see anything, and say that instead. Then do both things the command prints, every time: open the review in the in-app browser with a browser tool (unless you have none), and end your answer with the review section it gives you. Do not list separate story links next to it.`; }