From c27f4c7800f1e17354a95c85f345083eaf97a2cc Mon Sep 17 00:00:00 2001 From: yannbf Date: Tue, 28 Apr 2026 23:10:38 +0200 Subject: [PATCH 01/24] Prompt: Run vitest less times, improve play functions --- code/lib/cli-storybook/src/ai/index.ts | 85 ++-- code/lib/cli-storybook/src/ai/prompt.ts | 539 +++++++++++++++++++++++- code/lib/cli-storybook/src/ai/types.ts | 3 +- scripts/eval/prompts/optimized-tests.md | 299 +++++++++++++ 4 files changed, 863 insertions(+), 63 deletions(-) create mode 100644 scripts/eval/prompts/optimized-tests.md diff --git a/code/lib/cli-storybook/src/ai/index.ts b/code/lib/cli-storybook/src/ai/index.ts index 20cd4884ae4e..0df8f549e709 100644 --- a/code/lib/cli-storybook/src/ai/index.ts +++ b/code/lib/cli-storybook/src/ai/index.ts @@ -22,54 +22,46 @@ import type { ProjectInfo, AiSetupOptions } from './types.ts'; export async function aiSetup(options: AiSetupOptions): Promise { const { configDir: userConfigDir, packageManager: packageManagerName, output } = options; - let projectInfo: ProjectInfo; + let projectInfo: ProjectInfo | undefined; try { const data = await getStorybookData({ configDir: userConfigDir, packageManagerName: packageManagerName as PackageManagerName | undefined, }); - const majorVersion = data.versionInstalled - ? parseMajorVersion(data.versionInstalled) - : undefined; - if (!data.frameworkPackage || !data.rendererPackage || !data.builderPackage) { - logger.error( - 'Could not detect framework, renderer, or builder from your Storybook config. Make sure you are running this command from your project root, or specify --config-dir.' - ); - return; - } + if (data.frameworkPackage && data.rendererPackage && data.builderPackage) { + const majorVersion = data.versionInstalled + ? parseMajorVersion(data.versionInstalled) + : undefined; - const projectTypeService = new ProjectTypeService(data.packageManager); - const detectedLanguage = await projectTypeService.detectLanguage(); - const language = detectedLanguage === SupportedLanguage.TYPESCRIPT ? 'ts' : 'js'; + const projectTypeService = new ProjectTypeService(data.packageManager); + const detectedLanguage = await projectTypeService.detectLanguage(); + const language = detectedLanguage === SupportedLanguage.TYPESCRIPT ? 'ts' : 'js'; - projectInfo = { - storybookVersion: data.versionInstalled, - majorVersion, - framework: data.frameworkPackage, - rendererPackage: data.rendererPackage, - renderer: data.renderer, - builderPackage: data.builderPackage, - addons: data.addons ?? [], - configDir: data.configDir, - storiesPaths: data.storiesPaths, - hasCsfFactoryPreview: data.hasCsfFactoryPreview, - language, - }; - } catch (err: unknown) { - logger.error( - `Failed to read Storybook configuration: ${err instanceof Error ? err.message : String(err)}` - ); - logger.log( - 'Make sure you are running this command from your project root, or specify --config-dir.' - ); - return; + projectInfo = { + storybookVersion: data.versionInstalled, + majorVersion, + framework: data.frameworkPackage, + rendererPackage: data.rendererPackage, + renderer: data.renderer, + builderPackage: data.builderPackage, + addons: data.addons ?? [], + configDir: data.configDir, + storiesPaths: data.storiesPaths, + packageManager: packageManagerName, + language, + }; + } + } catch { + // No Storybook project (or unreadable config) — fall through and emit + // the generic prompt without a project metadata section. } if ( - projectInfo.rendererPackage !== '@storybook/react' || - projectInfo.builderPackage !== '@storybook/builder-vite' + projectInfo && + (projectInfo.rendererPackage !== '@storybook/react' || + projectInfo.builderPackage !== '@storybook/builder-vite') ) { logger.log( 'AI-assisted setup is currently only available for projects using the React renderer with Vite builder. Detected renderer: ' + @@ -86,16 +78,17 @@ export async function aiSetup(options: AiSetupOptions): Promise { await telemetry('ai-setup', { cliOptions: { output: output ? 'file' : undefined, - configDir: projectInfo.configDir, + configDir: projectInfo?.configDir, packageManager: packageManagerName, }, - project: { - framework: projectInfo.framework, - renderer: projectInfo.rendererPackage, - builder: projectInfo.builderPackage, - language: projectInfo.language, - hasCsfFactoryPreview: projectInfo.hasCsfFactoryPreview, - }, + project: projectInfo + ? { + framework: projectInfo.framework, + renderer: projectInfo.rendererPackage, + builder: projectInfo.builderPackage, + language: projectInfo.language, + } + : undefined, }); // Snapshot the preview file baseline and cache the pending setup record. @@ -103,7 +96,7 @@ export async function aiSetup(options: AiSetupOptions): Promise { // collect evidence of what the agent accomplished — but only via telemetry // (the `ai-setup-evidence` event). Skip the snapshot + cache write when // telemetry is disabled so there's nobody to read it. - if (isTelemetryModuleEnabled()) { + if (projectInfo && isTelemetryModuleEnabled()) { const resolvedConfigDir = resolve(projectInfo.configDir); const previewSnapshot = await snapshotPreviewFile(resolvedConfigDir); const sessionId = await getSessionId(); @@ -121,7 +114,7 @@ export async function aiSetup(options: AiSetupOptions): Promise { await writeFile(outputPath, markdownOutput, 'utf-8'); logger.log(`Prompt written to ${outputPath}`); } else { - logger.log(markdownOutput); + console.log(markdownOutput); } } diff --git a/code/lib/cli-storybook/src/ai/prompt.ts b/code/lib/cli-storybook/src/ai/prompt.ts index 55d9a5a2b52b..7031193fdbc2 100644 --- a/code/lib/cli-storybook/src/ai/prompt.ts +++ b/code/lib/cli-storybook/src/ai/prompt.ts @@ -1,25 +1,530 @@ +import { SupportedRenderer } from 'storybook/internal/types'; import { dedent } from 'ts-dedent'; -import type { ProjectInfo } from './types.ts'; -import { getPrompts } from './setup-prompts/index.ts'; +import type { AiPrompt, ProjectInfo } from './types.ts'; + +/** + * Builds a markdown-format docs URL with renderer and language query parameters. + * Appending .md to any Storybook docs URL returns clean markdown with code examples. + */ +export function getDocsMarkdownUrl( + path: string, + projectInfo?: Pick +): string { + const { majorVersion, renderer = 'react', language = 'ts' } = projectInfo ?? {}; + const versionSegment = majorVersion ? `/${majorVersion}` : ''; + const params = new URLSearchParams(); + if (renderer) { + params.set('renderer', renderer); + } + params.set('language', language); + const query = params.toString(); + return `https://storybook.js.org/docs${versionSegment}/${path}.md${query ? `?${query}` : ''}`; +} + +const DEFAULT_PROJECT_INFO: ProjectInfo = { + storybookVersion: undefined, + majorVersion: undefined, + framework: null, + rendererPackage: null, + renderer: SupportedRenderer.REACT, + builderPackage: null, + addons: [], + configDir: '.storybook', + storiesPaths: [], + language: 'ts', +}; + +function withDefaults(projectInfo?: ProjectInfo): ProjectInfo { + return projectInfo ?? DEFAULT_PROJECT_INFO; +} + +export function getPrompts(projectInfo?: ProjectInfo): { + prompts: AiPrompt[]; +} { + return { + prompts: [ + { + name: 'setup', + description: 'Set up Storybook for success', + instructions: getSetupInstructions(withDefaults(projectInfo)), + }, + ], + }; +} + +function ext(language: 'ts' | 'js', jsx: boolean): string { + if (language === 'ts') { + return jsx ? 'tsx' : 'ts'; + } + return jsx ? 'jsx' : 'js'; +} + +function installCommand(packageManager: string | undefined, deps: string, dev = false): string { + switch (packageManager) { + case 'yarn': + case 'yarn1': + case 'yarn2': + return `yarn add ${dev ? '--dev ' : ''}${deps}`; + case 'pnpm': + return `pnpm add ${dev ? '-D ' : ''}${deps}`; + case 'bun': + return `bun add ${dev ? '-d ' : ''}${deps}`; + case 'npm': + return `npm install ${dev ? '--save-dev ' : ''}${deps}`; + default: + return ` add ${dev ? '-D ' : ''}${deps}`; + } +} + +function packageManagerRule(packageManager: string | undefined): string { + if (packageManager) { + return `**Use \`${packageManager}\` for every install** (detected from this project's lockfile).`; + } + return '**Detect the package manager once** from the lockfile (`pnpm-lock.yaml` → pnpm, `yarn.lock` → yarn, `bun.lockb` → bun, otherwise npm) and use it for every install in this trial.'; +} + +function getSetupInstructions(projectInfo: ProjectInfo): string { + const { configDir, language, packageManager } = projectInfo; + const tsx = ext(language, true); + const ts = ext(language, false); + const docsUrl = (path: string) => getDocsMarkdownUrl(path, projectInfo); + + const mswInstall = installCommand(packageManager, 'msw msw-storybook-addon mockdate', true); -function getProjectOverview(projectInfo: ProjectInfo): string { return dedent` - ## Project Info - - | Property | Value | - |----------|-------| - | Version | ${projectInfo.storybookVersion || 'unknown'} | - | Renderer | ${projectInfo.rendererPackage || 'unknown'} | - | Framework | ${projectInfo.framework || 'unknown'} | - | Builder | ${projectInfo.builderPackage || 'unknown'} | - | Config Dir | \`${projectInfo.configDir}\` | - | CSF Format | ${projectInfo.hasCsfFactoryPreview ? 'CSF Factory' : 'CSF3'} | - | Addons | ${projectInfo.addons.length > 0 ? projectInfo.addons.join(', ') : 'none'} | + Your goal is to make Storybook fully functional in this project: configure \`${configDir}/preview.${tsx}\` with the right decorators, add MSW for data, and write up to 10 colocated \`*.stories.${tsx}\` files. Add \`play\` functions only where they prove something non-trivial. + + ## Rules of engagement (follow strictly — these are time budgets, not suggestions) + + 1. **Discover with Glob/Grep/Read, not shell.** Never use \`ls\`, \`find\`, \`cat\`, \`head\`, \`tail\`, shell \`grep\`, \`sed\`, or \`node -e\` for discovery or for editing files in bulk — these are slower per call and violate caching. Concrete substitutions, no exceptions: + - List a directory → \`Glob('src/components/*')\`, not \`ls src/components\`. + - Search a string → \`Grep('pattern', { path: 'src' })\`, not \`grep -rn ...\` or \`find ... | xargs grep\`. + - Read a file → \`Read('path/to/file')\`, not \`cat\`/\`head\`/\`tail\`. + - Bulk-edit many files → multiple \`Edit\` calls (or \`Edit\` with \`replace_all\`), not \`sed -i\`. + 2. **Never read or grep inside \`node_modules\`.** The imports shown in this prompt are correct — don't verify them by introspecting installed packages. If something seems off, re-read this prompt, not \`node_modules\`. + 3. **Read budget: ~12 files for discovery.** Before writing any code you may Read at most ~12 files (\`index.html\`, entry, App, providers, routing, root CSS, 2–3 representative pages/components, 1–2 hooks, 1 test). If you need more, summarize and move on. + 4. **Edit > Write.** For any file you've Read, use \`Edit\`. Use \`Write\` only for new files. The project already has a \`${configDir}/preview.${tsx}\` from \`storybook init\` — **Edit** it, do not overwrite. + 5. **Batch the test loop.** Write **all** stories first, then run vitest **once** across everything. No per-file vitest runs until after that first batch run reveals failures. + 6. ${packageManagerRule(packageManager)} + 7. **Prefer fixing the shared \`${configDir}/preview.${tsx}\`** over story-local workarounds when multiple stories fail the same way. + 8. **Stop when the success criteria are met** — don't keep polishing. + + ## Plan (do not skip steps, but keep each step lean) + + ### Step 1 — Discover the runtime (≤12 reads) + + Identify, in this order, using Glob/Grep first then targeted Reads: + + - \`index.html\` — \`\` tags, inline \`