diff --git a/code/core/src/builder-manager/index.ts b/code/core/src/builder-manager/index.ts index de984736d4db..d66905493469 100644 --- a/code/core/src/builder-manager/index.ts +++ b/code/core/src/builder-manager/index.ts @@ -2,6 +2,7 @@ import { cp, rm, writeFile } from 'node:fs/promises'; import { stringifyProcessEnvs } from 'storybook/internal/common'; import { logger } from 'storybook/internal/node-logger'; +import { getLastEvents, getSessionId } from 'storybook/internal/telemetry'; import { globalExternals } from '@fal-works/esbuild-plugin-global-externals'; import { resolveModulePath } from 'exsolve'; @@ -199,6 +200,9 @@ const starter: StarterFunction = async function* starterGeneratorFn({ // Build additional global values const globals: Record = await buildFrameworkGlobalsFromOptions(options); + // Pass along recent CLI events to customise user onboarding and telemetry after `sb ai` commands. + globals.STORYBOOK_LAST_EVENTS = await getLastEvents(); + globals.STORYBOOK_SESSION_ID = await getSessionId(); yield; @@ -297,6 +301,9 @@ const builder: BuilderFunction = async function* builderGeneratorFn({ startTime, // Build additional global values const globals: Record = await buildFrameworkGlobalsFromOptions(options); + // Builds are long-lasting and shouldn't account for CLI events prior to building. + globals.STORYBOOK_LAST_EVENTS = {}; + globals.STORYBOOK_SESSION_ID = undefined; yield; diff --git a/code/core/src/common/satellite-addons.ts b/code/core/src/common/satellite-addons.ts index 122d3b59cba7..98d2a89cc928 100644 --- a/code/core/src/common/satellite-addons.ts +++ b/code/core/src/common/satellite-addons.ts @@ -7,6 +7,7 @@ export default [ '@storybook/addon-coverage', '@storybook/addon-webpack5-compiler-babel', '@storybook/addon-webpack5-compiler-swc', + '@storybook/addon-mcp', // Storybook for React Native related packages // TODO: For Storybook 10, we should check about possible automigrations '@storybook/addon-ondevice-actions', diff --git a/code/core/src/core-events/index.ts b/code/core/src/core-events/index.ts index 7f281f7d9b97..4091070dfb5b 100644 --- a/code/core/src/core-events/index.ts +++ b/code/core/src/core-events/index.ts @@ -91,6 +91,9 @@ enum events { // Story discovery and testing flow GHOST_STORIES_REQUEST = 'ghostStoriesRequest', GHOST_STORIES_RESPONSE = 'ghostStoriesResponse', + // AI analytics - ai prepare command + AI_PREPARE_ANALYTICS_RESPONSE = 'aiPrepareAnalyticsResponse', + AI_PREPARE_ANALYTICS_REQUEST = 'aiPrepareAnalyticsRequest', // Open a file in the code editor OPEN_IN_EDITOR_REQUEST = 'openInEditorRequest', OPEN_IN_EDITOR_RESPONSE = 'openInEditorResponse', @@ -168,6 +171,8 @@ export const { ARGTYPES_INFO_RESPONSE, GHOST_STORIES_REQUEST, GHOST_STORIES_RESPONSE, + AI_PREPARE_ANALYTICS_RESPONSE, + AI_PREPARE_ANALYTICS_REQUEST, OPEN_IN_EDITOR_REQUEST, OPEN_IN_EDITOR_RESPONSE, MANAGER_INERT_ATTRIBUTE_CHANGED, diff --git a/code/core/src/core-server/presets/common-preset.ts b/code/core/src/core-server/presets/common-preset.ts index bcdcd8941ed3..b534317e022e 100644 --- a/code/core/src/core-server/presets/common-preset.ts +++ b/code/core/src/core-server/presets/common-preset.ts @@ -41,6 +41,7 @@ import { initializeSaveStory } from '../utils/save-story/save-story.ts'; import { parseStaticDir } from '../utils/server-statics.ts'; import { type OptionsWithRequiredCache, initializeWhatsNew } from '../utils/whats-new.ts'; import { getWsToken } from './wsToken.ts'; +import { initAIAnalyticsChannel } from '../server-channel/ai-prepare-channel.ts'; const interpolate = (string: string, data: Record = {}) => Object.entries(data).reduce((acc, [k, v]) => acc.replace(new RegExp(`%${k}%`, 'g'), v), string); @@ -278,6 +279,7 @@ export const experimental_serverChannel = async ( initializeWhatsNew(channel, options, coreOptions); initializeSaveStory(channel, options, coreOptions); + initAIAnalyticsChannel(channel, options, coreOptions, () => storyIndexGeneratorPromise); initFileSearchChannel(channel, options, coreOptions); initCreateNewStoryChannel(channel, options, coreOptions); initGhostStoriesChannel(channel, options, coreOptions); diff --git a/code/core/src/core-server/server-channel/ai-prepare-channel.ts b/code/core/src/core-server/server-channel/ai-prepare-channel.ts new file mode 100644 index 000000000000..0c08fed0f8d4 --- /dev/null +++ b/code/core/src/core-server/server-channel/ai-prepare-channel.ts @@ -0,0 +1,131 @@ +import type { Channel } from 'storybook/internal/channels'; +import { + AI_PREPARE_ANALYTICS_REQUEST, + AI_PREPARE_ANALYTICS_RESPONSE, +} from 'storybook/internal/core-events'; +import { + getLastEvents, + getStorybookMetadata, + isStoryCreatedByAIPrepare, + telemetry, +} from 'storybook/internal/telemetry'; +import type { CoreConfig, Options } from 'storybook/internal/types'; +import { logger } from 'storybook/internal/node-logger'; + +import { runGhostStories } from '../utils/ghost-stories/run-story-tests.ts'; +import type { StoryIndexGenerator } from 'storybook/internal/core-server'; +import { waitForIdleVitest } from '../utils/wait-for-idle-vitest.ts'; + +export function initAIAnalyticsChannel( + channel: Channel, + options: Options, + coreOptions: CoreConfig, + getStoryIndexGeneratorPromise?: () => Promise | undefined +) { + if (coreOptions.disableTelemetry) { + return channel; + } + + /** Send analytics about the ai prepare workflow when requested*/ + channel.on(AI_PREPARE_ANALYTICS_REQUEST, async () => { + const stats: { + fileCount?: number; + storyCount?: number; + testRunDuration?: number; + } = {}; + + try { + const lastEvents = await getLastEvents(); + const lastAIPrepare = lastEvents?.['ai-prepare']; + const lastPrepareStoryScoringRun = lastEvents?.['ai-prepare-story-scoring']; + + // Only run if sb ai prepare has been called + if (!lastAIPrepare) { + return; + } + + // Already ran once for this project — never run again + if (lastPrepareStoryScoringRun) { + return; + } + + const metadata = await getStorybookMetadata(options.configDir); + const isReactStorybook = metadata?.renderer?.includes('@storybook/react'); + const hasVitestAddon = + !!metadata?.addons && + Object.keys(metadata.addons).some((addonKey) => + addonKey.includes('@storybook/addon-vitest') + ); + + // For now this is gated by React + Vitest + if (!isReactStorybook || !hasVitestAddon) { + return; + } + + // Wait for any running tests to finish before launching scoring, so we don't + // disturb end user activities. + const isIdle = await waitForIdleVitest(); + if (!isIdle) { + logger.debug('AI_PREPARE_ANALYTICS_REQUEST timed out waiting for vitest to be available.'); + return; + } + + // Fetch AI-generated stories and score them with the ghost stories metrics, if any are found. + const generatorPromise = getStoryIndexGeneratorPromise?.(); + if (!generatorPromise) { + logger.debug( + 'AI_PREPARE_ANALYTICS_REQUEST could not proceed as the index generator is not ready.' + ); + return; + } + + const generator = await generatorPromise; + const indexAndStats = await generator.getIndexAndStats(); + if (!indexAndStats) { + logger.debug('AI_PREPARE_ANALYTICS_REQUEST could not proceed as the index is not ready.'); + return; + } + + const aiStoryFiles = new Set(); + let aiStoryCount = 0; + for (const entry of Object.values(indexAndStats.storyIndex.entries)) { + if (isStoryCreatedByAIPrepare(entry)) { + aiStoryFiles.add(entry.importPath); + aiStoryCount++; + } + } + + if (aiStoryFiles.size > 0) { + const aiTestRunResult = await runGhostStories([...aiStoryFiles]); + telemetry('ai-prepare-story-scoring', { + stats: { + fileCount: aiStoryFiles.size, + storyCount: aiStoryCount, + testRunDuration: aiTestRunResult.duration, + }, + results: aiTestRunResult.summary, + ...(aiTestRunResult.runError ? { runError: aiTestRunResult.runError } : {}), + }); + } else { + telemetry('ai-prepare-story-scoring', { + stats: { + fileCount: 0, + storyCount: 0, + testRunDuration: 0, + }, + runError: 'No stories found that were generated by ai setup', + }); + } + } catch { + telemetry('ai-prepare-story-scoring', { + stats, + runError: 'Unknown error during AI story scoring', + }); + } finally { + // we don't currently do anything with this, but will be useful in the future + channel.emit(AI_PREPARE_ANALYTICS_RESPONSE); + } + }); + + return channel; +} diff --git a/code/core/src/core-server/server-channel/ghost-stories-channel.test.ts b/code/core/src/core-server/server-channel/ghost-stories-channel.test.ts index 3420908012a9..a695f73cef3a 100644 --- a/code/core/src/core-server/server-channel/ghost-stories-channel.test.ts +++ b/code/core/src/core-server/server-channel/ghost-stories-channel.test.ts @@ -343,7 +343,7 @@ describe('ghostStoriesChannel', () => { }); expect(mockTelemetry.getLastEvents).toHaveBeenCalled(); - expect(mockTelemetry.getSessionId).toHaveBeenCalled(); + expect(mockTelemetry.getSessionId).not.toHaveBeenCalled(); expect(mockTelemetry.getStorybookMetadata).not.toHaveBeenCalled(); expect(mockStoryGeneration.getComponentCandidates).not.toHaveBeenCalled(); }); diff --git a/code/core/src/core-server/server-channel/ghost-stories-channel.ts b/code/core/src/core-server/server-channel/ghost-stories-channel.ts index 76a414b994cd..e09f574b6eb9 100644 --- a/code/core/src/core-server/server-channel/ghost-stories-channel.ts +++ b/code/core/src/core-server/server-channel/ghost-stories-channel.ts @@ -6,10 +6,12 @@ import { getStorybookMetadata, telemetry, } from 'storybook/internal/telemetry'; +import { logger } from 'storybook/internal/node-logger'; import type { CoreConfig, Options } from 'storybook/internal/types'; import { getComponentCandidates } from '../utils/ghost-stories/get-candidates.ts'; import { runGhostStories } from '../utils/ghost-stories/run-story-tests.ts'; +import { waitForIdleVitest } from '../utils/wait-for-idle-vitest.ts'; export function initGhostStoriesChannel( channel: Channel, @@ -36,16 +38,23 @@ export function initGhostStoriesChannel( const ghostRunStart = Date.now(); const lastEvents = await getLastEvents(); const lastInit = lastEvents?.init; - if (!lastEvents || !lastInit) { + const lastAIPrepare = lastEvents?.['ai-prepare']; + const lastGhostStoriesRun = lastEvents?.['ghost-stories']; + + // We only want to run ghost stories immediately after init or ai prepare. + const lastRelevantEvent = lastAIPrepare ?? lastInit; + if (!lastRelevantEvent) { + return; + } + + // Already ran once for this project — never run again + if (lastGhostStoriesRun) { return; } const sessionId = await getSessionId(); - const lastGhostStoriesRun = lastEvents['ghost-stories']; - if ( - lastGhostStoriesRun || - (lastInit.body?.sessionId && lastInit.body.sessionId !== sessionId) - ) { + // We only capture ghost stories in the first ever session since init or ai prepare. + if (lastRelevantEvent.body?.sessionId && lastRelevantEvent.body.sessionId !== sessionId) { return; } @@ -62,6 +71,14 @@ export function initGhostStoriesChannel( return; } + // Wait for any running tests to finish before launching scoring, so we don't + // disturb end user activities. + const isIdle = await waitForIdleVitest(); + if (!isIdle) { + logger.debug('GHOST_STORIES_REQUEST timed out waiting for vitest to be available.'); + return; + } + // Phase 1: find candidates from components const candidateAnalysisStart = Date.now(); const candidatesResult = await getComponentCandidates(); diff --git a/code/core/src/core-server/utils/doTelemetry.ts b/code/core/src/core-server/utils/doTelemetry.ts index 6553f6a1d1a7..f1b3374ee2aa 100644 --- a/code/core/src/core-server/utils/doTelemetry.ts +++ b/code/core/src/core-server/utils/doTelemetry.ts @@ -1,4 +1,8 @@ -import { getPrecedingUpgrade, telemetry } from 'storybook/internal/telemetry'; +import { + collectAiPrepareEvidence, + getPrecedingUpgrade, + telemetry, +} from 'storybook/internal/telemetry'; import type { CoreConfig, Options } from 'storybook/internal/types'; import type { Polka } from 'polka'; @@ -36,6 +40,14 @@ export async function doTelemetry( }); return; } + + // sb ai commands trigger side effects performed by agent harnesses, which can't be observed + // directly. This is the entry point for collecting evidence about those side effects and + // recording them in telemetry. + if (indexAndStats) { + collectAiPrepareEvidence('dev', options.configDir, indexAndStats.storyIndex); + } + const { versionCheck, versionUpdates } = options; invariant( !versionUpdates || (versionUpdates && versionCheck), diff --git a/code/core/src/core-server/utils/wait-for-idle-vitest.test.ts b/code/core/src/core-server/utils/wait-for-idle-vitest.test.ts new file mode 100644 index 000000000000..a5135656c68f --- /dev/null +++ b/code/core/src/core-server/utils/wait-for-idle-vitest.test.ts @@ -0,0 +1,114 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../stores/test-provider.ts', () => ({ + fullTestProviderStore: { + getFullState: vi.fn(), + }, +})); + +import { fullTestProviderStore } from '../stores/test-provider.ts'; +import { waitForIdleVitest } from './wait-for-idle-vitest.ts'; + +const getFullState = vi.mocked(fullTestProviderStore.getFullState); + +beforeEach(() => { + vi.useFakeTimers(); + getFullState.mockReset(); +}); + +describe('waitForIdleVitest', () => { + it('returns true immediately when no providers are running', async () => { + getFullState.mockReturnValue({ a: 'test-provider-state:pending' }); + await expect(waitForIdleVitest()).resolves.toBe(true); + }); + + it('returns true immediately when state is empty', async () => { + getFullState.mockReturnValue({}); + await expect(waitForIdleVitest()).resolves.toBe(true); + }); + + it('returns true when getFullState throws (store not initialized)', async () => { + getFullState.mockImplementation(() => { + throw new Error('not initialized'); + }); + await expect(waitForIdleVitest()).resolves.toBe(true); + }); + + it('waits and returns true when provider transitions from running to succeeded', async () => { + getFullState + .mockReturnValueOnce({ a: 'test-provider-state:running' }) + .mockReturnValueOnce({ a: 'test-provider-state:succeeded' }); + + const promise = waitForIdleVitest(60_000, 100); + + // First poll finds running, schedules a timeout + await vi.advanceTimersByTimeAsync(100); + + await expect(promise).resolves.toBe(true); + expect(getFullState).toHaveBeenCalledTimes(2); + }); + + it('waits and returns true when provider transitions from running to crashed', async () => { + getFullState + .mockReturnValueOnce({ a: 'test-provider-state:running' }) + .mockReturnValueOnce({ a: 'test-provider-state:crashed' }); + + const promise = waitForIdleVitest(60_000, 100); + + // First poll finds running, schedules a timeout + await vi.advanceTimersByTimeAsync(100); + + await expect(promise).resolves.toBe(true); + expect(getFullState).toHaveBeenCalledTimes(2); + }); + + it('returns false when maxWaitMs is exceeded', async () => { + getFullState.mockReturnValue({ a: 'test-provider-state:running' }); + + const promise = waitForIdleVitest(250, 100); + + // Advance past the deadline + await vi.advanceTimersByTimeAsync(100); + await vi.advanceTimersByTimeAsync(100); + await vi.advanceTimersByTimeAsync(100); + + await expect(promise).resolves.toBe(false); + }); + + it('treats multiple providers correctly — running if any is running', async () => { + getFullState.mockReturnValue({ + a: 'test-provider-state:pending', + b: 'test-provider-state:running', + }); + + const promise = waitForIdleVitest(50, 100); + await vi.advanceTimersByTimeAsync(100); + + await expect(promise).resolves.toBe(false); + }); + + it('returns true when all multiple providers are idle', async () => { + getFullState.mockReturnValue({ + a: 'test-provider-state:pending', + b: 'test-provider-state:pending', + }); + await expect(waitForIdleVitest()).resolves.toBe(true); + }); + + it('polls at the configured interval', async () => { + getFullState + .mockReturnValueOnce({ a: 'test-provider-state:running' }) + .mockReturnValueOnce({ a: 'test-provider-state:running' }) + .mockReturnValueOnce({ a: 'test-provider-state:pending' }); + + const promise = waitForIdleVitest(60_000, 200); + + await vi.advanceTimersByTimeAsync(200); + expect(getFullState).toHaveBeenCalledTimes(2); + + await vi.advanceTimersByTimeAsync(200); + expect(getFullState).toHaveBeenCalledTimes(3); + + await expect(promise).resolves.toBe(true); + }); +}); diff --git a/code/core/src/core-server/utils/wait-for-idle-vitest.ts b/code/core/src/core-server/utils/wait-for-idle-vitest.ts new file mode 100644 index 000000000000..5e0959bac303 --- /dev/null +++ b/code/core/src/core-server/utils/wait-for-idle-vitest.ts @@ -0,0 +1,27 @@ +import { fullTestProviderStore } from '../stores/test-provider.ts'; +/** + * Wait for the test provider to be idle (no tests running). + * Returns true if idle, false if timed out. + * Use this if you intend to run a ad-hoc vitest process to + * avoid conflicts with already running component tests. + */ +export async function waitForIdleVitest( + maxWaitMs = 30 * 60 * 1000, + pollIntervalMs = 60 * 1000 +): Promise { + const deadline = Date.now() + maxWaitMs; + while (Date.now() < deadline) { + try { + const state = fullTestProviderStore.getFullState(); + const isRunning = Object.values(state).some((s) => s === 'test-provider-state:running'); + if (!isRunning) { + return true; + } + } catch { + // Store not initialized yet — treat as idle + return true; + } + await new Promise((resolve) => setTimeout(resolve, pollIntervalMs)); + } + return false; +} diff --git a/code/core/src/core-server/withTelemetry.test.ts b/code/core/src/core-server/withTelemetry.test.ts index beef361f9c24..622b9c738738 100644 --- a/code/core/src/core-server/withTelemetry.test.ts +++ b/code/core/src/core-server/withTelemetry.test.ts @@ -2,7 +2,12 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { cache, isCI, loadAllPresets } from 'storybook/internal/common'; import { prompt } from 'storybook/internal/node-logger'; -import { ErrorCollector, oneWayHash, telemetry } from 'storybook/internal/telemetry'; +import { + ErrorCollector, + collectAiPrepareEvidence, + oneWayHash, + telemetry, +} from 'storybook/internal/telemetry'; import { getErrorLevel, sendTelemetryError, withTelemetry } from './withTelemetry.ts'; @@ -29,6 +34,7 @@ describe('withTelemetry', () => { vi.resetAllMocks(); vi.mocked(ErrorCollector.getErrors).mockReturnValue([]); vi.mocked(telemetry).mockResolvedValue(undefined); + vi.mocked(collectAiPrepareEvidence).mockResolvedValue(undefined); }); it('works in happy path', async () => { const run = vi.fn(); @@ -36,6 +42,7 @@ describe('withTelemetry', () => { await withTelemetry('dev', { cliOptions }, run); expect(telemetry).toHaveBeenCalledTimes(1); + expect(collectAiPrepareEvidence).toHaveBeenCalledTimes(1); expect(telemetry).toHaveBeenCalledWith('boot', { eventType: 'dev' }, { stripMetadata: true }); }); @@ -45,6 +52,7 @@ describe('withTelemetry', () => { await withTelemetry('dev', { cliOptions: { disableTelemetry: true } }, run); expect(telemetry).toHaveBeenCalledTimes(0); + expect(collectAiPrepareEvidence).toHaveBeenCalledTimes(0); }); describe('when command fails', () => { @@ -59,6 +67,7 @@ describe('withTelemetry', () => { ).rejects.toThrow(error); expect(telemetry).toHaveBeenCalledWith('boot', { eventType: 'dev' }, { stripMetadata: true }); + expect(collectAiPrepareEvidence).toHaveBeenCalledTimes(1); }); it('does not send boot when cli option is passed', async () => { @@ -67,6 +76,7 @@ describe('withTelemetry', () => { ).rejects.toThrow(error); expect(telemetry).toHaveBeenCalledTimes(0); + expect(collectAiPrepareEvidence).toHaveBeenCalledTimes(0); }); it('sends error message when no options are passed', async () => { @@ -75,6 +85,7 @@ describe('withTelemetry', () => { ).rejects.toThrow(error); expect(telemetry).toHaveBeenCalledTimes(2); + expect(collectAiPrepareEvidence).toHaveBeenCalledTimes(1); expect(telemetry).toHaveBeenCalledWith( 'error', expect.objectContaining({ @@ -138,6 +149,7 @@ describe('withTelemetry', () => { ).rejects.toThrow(error); expect(telemetry).toHaveBeenCalledTimes(0); + expect(collectAiPrepareEvidence).toHaveBeenCalledTimes(0); expect(telemetry).not.toHaveBeenCalledWith( 'error', expect.objectContaining({}), diff --git a/code/core/src/core-server/withTelemetry.ts b/code/core/src/core-server/withTelemetry.ts index 044ce7bd7afb..9903e34fc0a3 100644 --- a/code/core/src/core-server/withTelemetry.ts +++ b/code/core/src/core-server/withTelemetry.ts @@ -1,6 +1,7 @@ import { HandledError, cache, isCI, loadAllPresets } from 'storybook/internal/common'; import { logger, prompt } from 'storybook/internal/node-logger'; import { + collectAiPrepareEvidence, ErrorCollector, getPrecedingUpgrade, oneWayHash, @@ -185,6 +186,12 @@ export async function withTelemetry( telemetry('boot', { eventType }, { stripMetadata: true }); } + if (enableTelemetry) { + // Fire-and-forget: don't await, don't block the command + const configDir = options.cliOptions.configDir || options.presetOptions?.configDir; + collectAiPrepareEvidence(eventType, configDir); + } + try { return await run(); } catch (error: any) { diff --git a/code/core/src/manager-api/typings.d.ts b/code/core/src/manager-api/typings.d.ts index 7bf67c97fc33..e3745a310bac 100644 --- a/code/core/src/manager-api/typings.d.ts +++ b/code/core/src/manager-api/typings.d.ts @@ -10,3 +10,8 @@ declare var STORYBOOK_ADDON_STATE: Record; declare var STORYBOOK_FRAMEWORK: import('storybook/internal/types').SupportedFramework | undefined; declare var STORYBOOK_RENDERER: import('storybook/internal/types').SupportedRenderer | undefined; declare var STORYBOOK_BUILDER: import('storybook/internal/types').SupportedBuilder | undefined; +declare var STORYBOOK_LAST_EVENTS: Record< + import('storybook/telemetry').EventType, + import('storybook/telemetry').CacheEntry +>; +declare var STORYBOOK_SESSION_ID: string | undefined; diff --git a/code/core/src/manager/components/sidebar/Sidebar.tsx b/code/core/src/manager/components/sidebar/Sidebar.tsx index 9442951b82ca..0be7ba7ee3f0 100644 --- a/code/core/src/manager/components/sidebar/Sidebar.tsx +++ b/code/core/src/manager/components/sidebar/Sidebar.tsx @@ -25,6 +25,7 @@ import { SearchResults } from './SearchResults.tsx'; import { SidebarBottom } from './SidebarBottom.tsx'; import { Filter } from './Filter.tsx'; import type { CombinedDataset, Selection } from './types.ts'; +import { useDelayedAnalyticsTrigger } from './useDelayedAnalyticsTrigger.ts'; import { useLastViewed } from './useLastViewed.ts'; export const DEFAULT_REF_ID = 'storybook_internal'; @@ -137,6 +138,8 @@ export const Sidebar = React.memo(function Sidebar({ headerRef ); + useDelayedAnalyticsTrigger(); + const isPagesShown = viewMode !== undefined && viewMode !== 'story' && viewMode !== 'docs'; const skipLinkHref = isPagesShown ? '#main-content-wrapper' : '#storybook-preview-wrapper'; diff --git a/code/core/src/manager/components/sidebar/useDelayedAnalyticsTrigger.test.ts b/code/core/src/manager/components/sidebar/useDelayedAnalyticsTrigger.test.ts new file mode 100644 index 000000000000..cb10dba6793e --- /dev/null +++ b/code/core/src/manager/components/sidebar/useDelayedAnalyticsTrigger.test.ts @@ -0,0 +1,142 @@ +// @vitest-environment happy-dom +import { renderHook } from '@testing-library/react'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + AI_PREPARE_ANALYTICS_REQUEST, + GHOST_STORIES_REQUEST, + PREVIEW_INITIALIZED, +} from 'storybook/internal/core-events'; + +vi.mock('storybook/manager-api', () => ({ + useStorybookApi: vi.fn(), +})); + +import { useStorybookApi } from 'storybook/manager-api'; +import { useDelayedAnalyticsTrigger } from './useDelayedAnalyticsTrigger.ts'; + +const mockUseStorybookApi = vi.mocked(useStorybookApi); + +function createMockApi() { + const listeners = new Map void>>(); + return { + emit: vi.fn(), + once: vi.fn((event: string, cb: (...args: any[]) => void) => { + if (!listeners.has(event)) { + listeners.set(event, new Set()); + } + listeners.get(event)!.add(cb); + }), + off: vi.fn((event: string, cb: (...args: any[]) => void) => { + listeners.get(event)?.delete(cb); + }), + /** Helper to simulate an event firing. */ + _trigger(event: string) { + listeners.get(event)?.forEach((cb) => cb()); + }, + _listeners: listeners, + }; +} + +describe('useDelayedAnalyticsTrigger', () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('registers a PREVIEW_INITIALIZED listener on mount', () => { + const api = createMockApi(); + mockUseStorybookApi.mockReturnValue(api as any); + + renderHook(() => useDelayedAnalyticsTrigger()); + + expect(api.once).toHaveBeenCalledWith(PREVIEW_INITIALIZED, expect.any(Function)); + }); + + it('does not emit events before PREVIEW_INITIALIZED fires', () => { + const api = createMockApi(); + mockUseStorybookApi.mockReturnValue(api as any); + + renderHook(() => useDelayedAnalyticsTrigger()); + + vi.advanceTimersByTime(20 * 60 * 1000); + expect(api.emit).not.toHaveBeenCalled(); + }); + + it('emits GHOST_STORIES_REQUEST and AI_PREPARE_ANALYTICS_REQUEST after delay', () => { + const api = createMockApi(); + mockUseStorybookApi.mockReturnValue(api as any); + + renderHook(() => useDelayedAnalyticsTrigger()); + + api._trigger(PREVIEW_INITIALIZED); + expect(api.emit).not.toHaveBeenCalled(); + + vi.advanceTimersByTime(4 * 60 * 1000); + + expect(api.emit).toHaveBeenCalledWith(GHOST_STORIES_REQUEST); + expect(api.emit).toHaveBeenCalledWith(AI_PREPARE_ANALYTICS_REQUEST); + expect(api.emit).toHaveBeenCalledTimes(2); + }); + + it('does not emit before the 4-minute delay elapses', () => { + const api = createMockApi(); + mockUseStorybookApi.mockReturnValue(api as any); + + renderHook(() => useDelayedAnalyticsTrigger()); + + api._trigger(PREVIEW_INITIALIZED); + vi.advanceTimersByTime(3 * 60 * 1000); + + expect(api.emit).not.toHaveBeenCalled(); + }); + + it('only fires once even if PREVIEW_INITIALIZED triggers multiple re-renders', () => { + const api = createMockApi(); + mockUseStorybookApi.mockReturnValue(api as any); + + const { rerender } = renderHook(() => useDelayedAnalyticsTrigger()); + + api._trigger(PREVIEW_INITIALIZED); + vi.advanceTimersByTime(10 * 60 * 1000); + + expect(api.emit).toHaveBeenCalledTimes(2); + + // Re-render and trigger again — should not fire again + rerender(); + api._trigger(PREVIEW_INITIALIZED); + vi.advanceTimersByTime(10 * 60 * 1000); + + expect(api.emit).toHaveBeenCalledTimes(2); + }); + + it('clears the timeout on unmount before it fires', () => { + const api = createMockApi(); + mockUseStorybookApi.mockReturnValue(api as any); + + const { unmount } = renderHook(() => useDelayedAnalyticsTrigger()); + + api._trigger(PREVIEW_INITIALIZED); + vi.advanceTimersByTime(2 * 60 * 1000); + + unmount(); + vi.advanceTimersByTime(4 * 60 * 1000); + + expect(api.emit).not.toHaveBeenCalled(); + }); + + it('unregisters the PREVIEW_INITIALIZED listener on unmount', () => { + const api = createMockApi(); + mockUseStorybookApi.mockReturnValue(api as any); + + const { unmount } = renderHook(() => useDelayedAnalyticsTrigger()); + + expect(api.once).toHaveBeenCalledTimes(1); + unmount(); + + expect(api.off).toHaveBeenCalledWith(PREVIEW_INITIALIZED, expect.any(Function)); + }); +}); diff --git a/code/core/src/manager/components/sidebar/useDelayedAnalyticsTrigger.ts b/code/core/src/manager/components/sidebar/useDelayedAnalyticsTrigger.ts new file mode 100644 index 000000000000..9458a5eaf642 --- /dev/null +++ b/code/core/src/manager/components/sidebar/useDelayedAnalyticsTrigger.ts @@ -0,0 +1,58 @@ +import { useEffect, useRef } from 'react'; + +import { + AI_PREPARE_ANALYTICS_REQUEST, + GHOST_STORIES_REQUEST, + PREVIEW_INITIALIZED, +} from 'storybook/internal/core-events'; +import { global } from '@storybook/global'; +import { useStorybookApi } from 'storybook/manager-api'; + +/** Delay before firing ghost stories after PREVIEW_INITIALIZED (4 minutes). */ +const TRIGGER_DELAY_MS = 4 * 60 * 1000; + +/** + * Fires one-time analytics events 10 minutes after the preview initializes. + * The server-side handlers for those events enforce the once-ever-per-project + * gate via lastEvents cache, so this hook is fire-and-forget. + */ +export function useDelayedAnalyticsTrigger(): void { + const api = useStorybookApi(); + const fired = useRef(false); + + useEffect(() => { + if (fired.current) { + return; + } + + let timeoutId: ReturnType | undefined; + + const fire = () => { + if (fired.current) { + return; + } + fired.current = true; + + // if `ai prepare` is in the same session, we run ghost stories and ai prepare analytics. + if ( + global.STORYBOOK_LAST_EVENTS?.['ai-prepare']?.body.sessionId === global.STORYBOOK_SESSION_ID + ) { + api.emit(GHOST_STORIES_REQUEST); + api.emit(AI_PREPARE_ANALYTICS_REQUEST); + } + }; + + const onInit = () => { + timeoutId = setTimeout(fire, TRIGGER_DELAY_MS); + }; + + api.once(PREVIEW_INITIALIZED, onInit); + + return () => { + api.off(PREVIEW_INITIALIZED, onInit); + if (timeoutId) { + clearTimeout(timeoutId); + } + }; + }, [api]); +} diff --git a/code/core/src/manager/globals/exports.ts b/code/core/src/manager/globals/exports.ts index 67edbe97d873..5c4142d12462 100644 --- a/code/core/src/manager/globals/exports.ts +++ b/code/core/src/manager/globals/exports.ts @@ -574,6 +574,8 @@ export default { 'withReset', ], 'storybook/internal/core-events': [ + 'AI_PREPARE_ANALYTICS_REQUEST', + 'AI_PREPARE_ANALYTICS_RESPONSE', 'ARGTYPES_INFO_REQUEST', 'ARGTYPES_INFO_RESPONSE', 'CHANNEL_CREATED', diff --git a/code/core/src/telemetry/ai-prepare-utils.test.ts b/code/core/src/telemetry/ai-prepare-utils.test.ts new file mode 100644 index 000000000000..afd0384fcd03 --- /dev/null +++ b/code/core/src/telemetry/ai-prepare-utils.test.ts @@ -0,0 +1,309 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { IndexEntry, StoryIndex } from 'storybook/internal/types'; + +import { + checkPreviewChanged, + collectAiPrepareEvidence, + countAiAuthoredStories, + isStoryCreatedByAIPrepare, +} from './ai-prepare-utils.ts'; + +// Mock modules with spy pattern +vi.mock('storybook/internal/common', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + findConfigFile: vi.fn(), + }; +}); + +vi.mock('./detect-agent.ts', () => ({ + detectAgent: vi.fn(() => undefined), +})); + +vi.mock('./event-cache.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + getAiPreparePending: vi.fn(() => undefined), + }; +}); + +vi.mock('./index.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + telemetry: vi.fn(), + }; +}); + +// Import mocked modules for spy access +import { findConfigFile } from 'storybook/internal/common'; +import { readFile } from 'node:fs/promises'; +import { detectAgent } from './detect-agent.ts'; +import { getAiPreparePending } from './event-cache.ts'; +import { SESSION_TIMEOUT } from './session-id.ts'; +import { telemetry } from './index.ts'; + +vi.mock('node:fs/promises', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + readFile: vi.fn(), + }; +}); + +const makePendingRecord = (overrides = {}) => ({ + timestamp: Date.now() - 60_000, // 1 minute ago + sessionId: 'test-session-id', + configDir: '/test/config', + previewPath: '/test/config/preview.ts', + previewHash: 'abc123', + ...overrides, +}); + +const makeStoryIndex = (entries: Record = {}): StoryIndex => ({ + v: 5, + entries, +}); + +beforeEach(() => { + vi.resetAllMocks(); + vi.mocked(telemetry).mockResolvedValue(undefined); +}); + +describe('isStoryCreatedByAIPrepare', () => { + it('returns true for stories with the ai-generated tag', () => { + expect( + isStoryCreatedByAIPrepare({ + type: 'story', + title: 'Foo', + tags: ['ai-generated', 'dev', 'play-fn'], + } as IndexEntry) + ).toBe(true); + }); + + it('returns false for regular stories', () => { + expect(isStoryCreatedByAIPrepare({ type: 'story', title: 'Foo' } as IndexEntry)).toBe(false); + }); +}); + +describe('countAiAuthoredStories', () => { + it('counts correctly with mixed entries', () => { + const index = makeStoryIndex({ + 'ai-1': { + type: 'story', + title: 'AI Generated/Button', + tags: ['ai-generated'], + id: 'ai-1', + name: 'Default', + importPath: './ai.stories.ts', + }, + 'ai-2': { + type: 'story', + title: 'AI Generated/Card', + tags: ['ai-generated'], + id: 'ai-2', + name: 'Default', + importPath: './ai2.stories.ts', + }, + regular: { + type: 'story', + title: 'Components/Input', + id: 'regular', + name: 'Default', + importPath: './input.stories.ts', + }, + docs: { + type: 'docs', + title: 'AI Generated/Docs', + tags: ['ai-generated'], + id: 'docs', + name: 'Docs', + importPath: './docs.mdx', + storiesImports: [], + }, + }); + // Only type: 'story' entries are counted, not docs + expect(countAiAuthoredStories(index)).toBe(2); + }); + + it('returns 0 when no AI stories exist', () => { + const index = makeStoryIndex({ + regular: { + type: 'story', + title: 'Components/Button', + id: 'regular', + name: 'Default', + importPath: './button.stories.ts', + }, + }); + expect(countAiAuthoredStories(index)).toBe(0); + }); +}); + +describe('checkPreviewChanged', () => { + it('returns false when hash matches snapshot', async () => { + vi.mocked(findConfigFile).mockReturnValue('/test/config/preview.ts'); + vi.mocked(readFile).mockResolvedValue('file content'); + + // Pre-compute the expected hash + const { createHash } = await import('node:crypto'); + const expectedHash = createHash('sha256').update('file content').digest('hex'); + + const result = await checkPreviewChanged('/test/config', { + previewPath: '/test/config/preview.ts', + previewHash: expectedHash, + }); + expect(result).toBe(false); + }); + + it('returns true when hash differs from snapshot', async () => { + vi.mocked(findConfigFile).mockReturnValue('/test/config/preview.ts'); + vi.mocked(readFile).mockResolvedValue('modified content'); + + const result = await checkPreviewChanged('/test/config', { + previewPath: '/test/config/preview.ts', + previewHash: 'some-old-hash', + }); + expect(result).toBe(true); + }); + + it('returns true when preview file is missing or unreadable', async () => { + vi.mocked(findConfigFile).mockReturnValue('/test/config/preview.ts'); + vi.mocked(readFile).mockRejectedValue(new Error('ENOENT')); + + const result = await checkPreviewChanged('/test/config', { + previewPath: '/test/config/preview.ts', + previewHash: 'some-hash', + }); + expect(result).toBe(true); + }); + + it('returns true when file path changed', async () => { + vi.mocked(findConfigFile).mockReturnValue('/test/config/preview.tsx'); + + const result = await checkPreviewChanged('/test/config', { + previewPath: '/test/config/preview.ts', + previewHash: 'hash', + }); + expect(result).toBe(true); + }); +}); + +describe('collectAiPrepareEvidence', () => { + it('does not fire when no agent detected', async () => { + vi.mocked(detectAgent).mockReturnValue(undefined); + + await collectAiPrepareEvidence('dev', '/test/config'); + expect(telemetry).not.toHaveBeenCalled(); + }); + + it('does not fire when no pending record', async () => { + vi.mocked(detectAgent).mockReturnValue({ name: 'claude' }); + vi.mocked(getAiPreparePending).mockResolvedValue(undefined); + + await collectAiPrepareEvidence('dev', '/test/config'); + expect(telemetry).not.toHaveBeenCalled(); + }); + + it('does not fire when pending record is expired', async () => { + vi.mocked(detectAgent).mockReturnValue({ name: 'claude' }); + vi.mocked(getAiPreparePending).mockResolvedValue( + makePendingRecord({ timestamp: Date.now() - SESSION_TIMEOUT - 1000 }) + ); + + await collectAiPrepareEvidence('dev', '/test/config'); + expect(telemetry).not.toHaveBeenCalled(); + }); + + it('does not fire when configDir does not match', async () => { + vi.mocked(detectAgent).mockReturnValue({ name: 'claude' }); + vi.mocked(getAiPreparePending).mockResolvedValue( + makePendingRecord({ configDir: '/other/project/.storybook' }) + ); + + await collectAiPrepareEvidence('dev', '/test/config'); + expect(telemetry).not.toHaveBeenCalled(); + }); + + it('fires event with correct payload when all gates pass', async () => { + vi.mocked(detectAgent).mockReturnValue({ name: 'claude' }); + const pending = makePendingRecord({ configDir: '/test/config' }); + vi.mocked(getAiPreparePending).mockResolvedValue(pending); + vi.mocked(findConfigFile).mockReturnValue(pending.previewPath); + vi.mocked(readFile).mockRejectedValue(new Error('ENOENT')); + + await collectAiPrepareEvidence('dev', '/test/config'); + + expect(telemetry).toHaveBeenCalledWith( + 'ai-prepare-evidence', + expect.objectContaining({ + previewChanged: true, + aiAuthoredStories: undefined, + sessionId: 'test-session-id', + }), + expect.objectContaining({ + immediate: true, + configDir: '/test/config', + }) + ); + }); + + it('reports aiAuthoredStories as undefined when no story index provided', async () => { + vi.mocked(detectAgent).mockReturnValue({ name: 'claude' }); + const pending = makePendingRecord({ configDir: '/test/config' }); + vi.mocked(getAiPreparePending).mockResolvedValue(pending); + vi.mocked(findConfigFile).mockReturnValue(null); + + await collectAiPrepareEvidence('dev', '/test/config'); + + expect(telemetry).toHaveBeenCalledWith( + 'ai-prepare-evidence', + expect.objectContaining({ + aiAuthoredStories: undefined, + }), + expect.anything() + ); + }); + + it('counts aiAuthoredStories when story index provided', async () => { + vi.mocked(detectAgent).mockReturnValue({ name: 'claude' }); + const pending = makePendingRecord({ + configDir: '/test/config', + previewFile: null, + previewHash: null, + }); + vi.mocked(getAiPreparePending).mockResolvedValue(pending); + vi.mocked(findConfigFile).mockReturnValue(null); + + const storyIndex = makeStoryIndex({ + 'ai-1': { + type: 'story', + title: 'AI Generated/Button', + tags: ['ai-generated'], + id: 'ai-1', + name: 'Default', + importPath: './ai.stories.ts', + }, + regular: { + type: 'story', + title: 'Components/Input', + id: 'regular', + name: 'Default', + importPath: './input.stories.ts', + }, + }); + + await collectAiPrepareEvidence('dev', '/test/config', storyIndex); + + expect(telemetry).toHaveBeenCalledWith( + 'ai-prepare-evidence', + expect.objectContaining({ + aiAuthoredStories: 1, + }), + expect.anything() + ); + }); +}); diff --git a/code/core/src/telemetry/ai-prepare-utils.ts b/code/core/src/telemetry/ai-prepare-utils.ts new file mode 100644 index 000000000000..203c744c97d4 --- /dev/null +++ b/code/core/src/telemetry/ai-prepare-utils.ts @@ -0,0 +1,144 @@ +import { flushAiPreparePending, getAiPreparePending } from './event-cache.ts'; +import { SESSION_TIMEOUT } from './session-id.ts'; +import { createHash } from 'node:crypto'; +import { readFile } from 'node:fs/promises'; + +import { findConfigFile } from 'storybook/internal/common'; +import { detectAgent } from './detect-agent.ts'; +import { telemetry } from './index.ts'; +import type { EventType } from './types.ts'; +import type { IndexEntry, StoryIndex } from 'storybook/internal/types'; + +/** + * Determines whether a story index entry was authored by the `sb ai prepare` flow. + * Currently checks title prefix. When we migrate to a tag-based approach, + * swap this to check for the tag instead — this is the single swap point. + */ +export function isStoryCreatedByAIPrepare(entry: IndexEntry): boolean { + return entry.type === 'story' && (entry.tags?.includes('ai-generated') ?? false); +} + +/** + * Count stories in the index that were created by `sb ai prepare`. + */ +export function countAiAuthoredStories(storyIndex: StoryIndex): number { + return Object.values(storyIndex.entries).filter(isStoryCreatedByAIPrepare).length; +} + +/** + * Snapshot the preview file state for baseline comparison. + * Returns the filename and SHA-256 hash, or nulls if no preview file exists. + */ +export async function snapshotPreviewFile( + configDir: string +): Promise<{ previewPath: string | null; previewHash: string | null }> { + const previewPath = findConfigFile('preview', configDir); + if (!previewPath) { + return { previewPath: null, previewHash: null }; + } + + try { + const content = await readFile(previewPath, 'utf-8'); + const hash = createHash('sha256').update(content).digest('hex'); + return { previewPath, previewHash: hash }; + } catch { + // File found by findConfigFile but unreadable — treat as absent + return { previewPath, previewHash: null }; + } +} + +/** + * Check whether the preview file has changed from an ai-prepare baseline. + * Returns true if: hash differs, file appeared, file disappeared, or file is unreadable. + */ +export async function checkPreviewChanged( + configDir: string, + baseline: { previewPath: string | null; previewHash: string | null } +): Promise { + const currentPath = findConfigFile('preview', configDir); + if (currentPath !== baseline.previewPath) { + return true; + } + if (!currentPath) { + return false; + } + try { + const content = await readFile(currentPath, 'utf-8'); + const hash = createHash('sha256').update(content).digest('hex'); + return hash !== baseline.previewHash; + } catch { + // File unreadable — treat as changed because we expected it to be readable post init + return true; + } +} + +/** + * Check for a pending ai-prepare record and fire an evidence event if found. + * + * Called from: + * - `withTelemetry` after the boot event for non-dev/build CLI commands (no story index) + * - `doTelemetry` for dev/build commands (story index available) + * + * Gated on: agent detected → pending record exists → within session window → configDir matches. + */ +export async function collectAiPrepareEvidence( + eventType: EventType, + configDir: string | undefined, + storyIndex?: StoryIndex +): Promise { + try { + // Gate 1: Is this an agent? (cheapest check) + const agent = detectAgent(); + if (!agent) { + return; + } + + // Gate 2: Is there a pending ai-prepare record? + const pending = await getAiPreparePending(); + if (!pending) { + return; + } + + // Gate 3: Does the configDir match? (cross-project guard) + if (configDir && pending.configDir !== configDir) { + return; + } + + // Gate 4: Is it within the session window? + const timeSincePrepare = Date.now() - pending.timestamp; + if (timeSincePrepare > SESSION_TIMEOUT) { + // Session expired, clean up pending record. + await flushAiPreparePending(); + return; + } + + // Don't fire evidence for ai-prepare itself — the prepare command gives the + // prompt to the agent and exits, so we only expect changes after the agent + // has started processing it. + if (eventType === 'ai-prepare') { + return; + } + + // Check if preview file changed from baseline + const previewChanged = await checkPreviewChanged(pending.configDir, pending); + + // Count AI-authored stories if story index is available + const aiAuthoredStories = storyIndex ? countAiAuthoredStories(storyIndex) : undefined; + + await telemetry( + 'ai-prepare-evidence', + { + previewChanged, + aiAuthoredStories, + sessionId: pending.sessionId, + timeSincePrepare, + }, + { + immediate: true, + configDir, + } + ); + } catch { + // Evidence collection is best-effort — never block the actual command + } +} diff --git a/code/core/src/telemetry/detect-agent.test.ts b/code/core/src/telemetry/detect-agent.test.ts index df49df7e9143..ef162f2931e4 100644 --- a/code/core/src/telemetry/detect-agent.test.ts +++ b/code/core/src/telemetry/detect-agent.test.ts @@ -1,10 +1,39 @@ -import { afterEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { detectAgent } from './detect-agent.ts'; describe('detectAgent', () => { + // Save any ambient agent env vars that might be set by the host environment + // (e.g. OPENCODE when running inside OpenCode CLI) so we can restore them. + const agentEnvVars = [ + 'OPENCODE', + 'CLAUDECODE', + 'CLAUDE_CODE', + 'GEMINI_CLI', + 'CODEX_SANDBOX', + 'CODEX_THREAD_ID', + 'CURSOR_AGENT', + 'AI_AGENT', + ]; + const savedEnv: Record = {}; + + beforeEach(() => { + for (const key of agentEnvVars) { + savedEnv[key] = process.env[key]; + delete process.env[key]; + } + }); + afterEach(() => { vi.unstubAllEnvs(); + // Restore ambient env vars + for (const key of agentEnvVars) { + if (savedEnv[key] !== undefined) { + process.env[key] = savedEnv[key]; + } else { + delete process.env[key]; + } + } }); it('detects claude via CLAUDECODE', () => { diff --git a/code/core/src/telemetry/event-cache.test.ts b/code/core/src/telemetry/event-cache.test.ts index 4ab2c6a0fbc3..76b7a701e156 100644 --- a/code/core/src/telemetry/event-cache.test.ts +++ b/code/core/src/telemetry/event-cache.test.ts @@ -4,7 +4,13 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { cache } from 'storybook/internal/common'; import type { CacheEntry } from './event-cache.ts'; -import { getLastEvents, getPrecedingUpgrade, set } from './event-cache.ts'; +import { + flushAiPreparePending, + getAiPreparePending, + getLastEvents, + getPrecedingUpgrade, + set, +} from './event-cache.ts'; import type { TelemetryEvent } from './types.ts'; vi.mock('storybook/internal/common', { spy: true }); @@ -347,4 +353,40 @@ describe('event-cache', () => { expect(result).toEqual(afterDev); }); }); + + describe('ai-prepare pending cache', () => { + let cacheGetMock: MockInstance; + let cacheRemoveMock: MockInstance; + + beforeEach(() => { + vi.clearAllMocks(); + cacheGetMock = vi.mocked(cache.get); + cacheRemoveMock = vi.mocked(cache.remove); + }); + + it('returns cached ai-prepare pending record when present', async () => { + const pending = { + timestamp: 123, + sessionId: 'session-1', + configDir: '/tmp/.storybook', + previewPath: '/tmp/.storybook/preview.ts', + previewHash: 'abc123', + }; + + cacheGetMock.mockResolvedValueOnce(pending); + + await expect(getAiPreparePending()).resolves.toEqual(pending); + expect(cacheGetMock).toHaveBeenCalledWith('ai-prepare-pending'); + }); + + it('removes the cached ai-prepare pending record and returns undefined', async () => { + cacheRemoveMock.mockResolvedValueOnce(undefined); + cacheGetMock.mockResolvedValueOnce(undefined); + + await expect(flushAiPreparePending()).resolves.toBeUndefined(); + expect(cacheRemoveMock).toHaveBeenCalledWith('ai-prepare-pending'); + await expect(getAiPreparePending()).resolves.toBeUndefined(); + expect(cacheGetMock).toHaveBeenCalledWith('ai-prepare-pending'); + }); + }); }); diff --git a/code/core/src/telemetry/event-cache.ts b/code/core/src/telemetry/event-cache.ts index 55768ea1088f..ae3d265d10a1 100644 --- a/code/core/src/telemetry/event-cache.ts +++ b/code/core/src/telemetry/event-cache.ts @@ -81,3 +81,28 @@ export const getPrecedingUpgrade = async ( ? upgradeFields(lastUpgradeEvent) : undefined; }; +/** + * Record cached at ai-prepare time. + * Read by subsequent CLI entry points for evidence collection. + * Canonical definition — imported by event-cache.ts and prepare-requirements.ts. + */ +export interface AiPreparePendingRecord { + timestamp: number; + sessionId: string; + configDir: string; + previewPath: string | null; + previewHash: string | null; +} + +export const getAiPreparePending = async (): Promise => { + // Wait for any pending set operations to complete before reading + await processingPromise; + return (await cache.get('ai-prepare-pending')) ?? undefined; +}; + +export const flushAiPreparePending = async (): Promise => { + // Wait for any pending set operations to complete before removing + await processingPromise; + await cache.remove('ai-prepare-pending'); + return undefined; +}; diff --git a/code/core/src/telemetry/index.ts b/code/core/src/telemetry/index.ts index 3b0dee4d935d..a27b99e44e55 100644 --- a/code/core/src/telemetry/index.ts +++ b/code/core/src/telemetry/index.ts @@ -16,12 +16,22 @@ export * from './sanitize.ts'; export * from './error-collector.ts'; -export { getPrecedingUpgrade, getLastEvents, type CacheEntry } from './event-cache.ts'; +export * from './ai-prepare-utils.ts'; -export { getSessionId } from './session-id.ts'; +export { + getPrecedingUpgrade, + getLastEvents, + type CacheEntry, + getAiPreparePending, + type AiPreparePendingRecord, +} from './event-cache.ts'; + +export { getSessionId, SESSION_TIMEOUT } from './session-id.ts'; export { addToGlobalContext } from './telemetry.ts'; +export { detectAgent } from './detect-agent.ts'; + /** Is this story part of the CLI generated examples, including user-created stories in those files */ export const isExampleStoryId = (storyId: string) => storyId.startsWith('example-button--') || @@ -54,8 +64,6 @@ export const telemetry = async ( } } finally { const { error } = payload; - // make sure to anonymise possible paths from error messages - // make sure to anonymise possible paths from error messages if (error) { payload.error = sanitizeError(error); diff --git a/code/core/src/telemetry/storybook-metadata.test.ts b/code/core/src/telemetry/storybook-metadata.test.ts index 595029ebb043..8429bfbb329e 100644 --- a/code/core/src/telemetry/storybook-metadata.test.ts +++ b/code/core/src/telemetry/storybook-metadata.test.ts @@ -15,6 +15,7 @@ import { import { detect } from 'package-manager-detector'; import { type Settings, globalSettings } from '../cli/globalSettings.ts'; +import { detectAgent } from './detect-agent.ts'; import { getApplicationFileCount } from '../telemetry/get-application-file-count.ts'; import { analyzeEcosystemPackages } from '../telemetry/get-known-packages.ts'; import { getMonorepoType } from '../telemetry/get-monorepo-type.ts'; @@ -32,6 +33,9 @@ import { } from './storybook-metadata.ts'; vi.mock(import('../cli/globalSettings.ts'), { spy: true }); +vi.mock('./detect-agent.ts', () => ({ + detectAgent: vi.fn().mockReturnValue(undefined), +})); vi.mock(import('./package-json.ts'), { spy: true }); vi.mock(import('./get-monorepo-type.ts'), { spy: true }); vi.mock(import('./get-framework-info.ts'), { spy: true }); @@ -562,7 +566,7 @@ describe('storybook-metadata', () => { expect(res.userSince).toEqual(1717334400000); }); - it('should not detect userSince info in CI', async () => { + it('should not detect userSince info in CI when agent is not detected', async () => { vi.mocked(isCI).mockImplementation(() => true); vi.mocked(globalSettings).mockResolvedValue({} as Settings); @@ -577,6 +581,26 @@ describe('storybook-metadata', () => { expect(res.userSince).not.toBeDefined(); }); + it('should detect userSince info in CI when agent is detected', async () => { + vi.mocked(isCI).mockImplementation(() => true); + vi.mocked(detectAgent).mockReturnValue({ name: 'claude' }); + vi.mocked(globalSettings).mockResolvedValue({ + value: { + userSince: 1717334400000, + }, + } as Settings); + + const res = await computeStorybookMetadata({ + configDir: '.storybook', + packageJson: packageJsonMock, + packageJsonPath, + mainConfig: mainJsMock, + }); + + expect(globalSettings).toHaveBeenCalled(); + expect(res.userSince).toEqual(1717334400000); + }); + it('should include knownPackages in metadata', async () => { const res = await computeStorybookMetadata({ configDir: '.storybook', diff --git a/code/core/src/telemetry/storybook-metadata.ts b/code/core/src/telemetry/storybook-metadata.ts index 9f58597f3ce3..3cbe341bb270 100644 --- a/code/core/src/telemetry/storybook-metadata.ts +++ b/code/core/src/telemetry/storybook-metadata.ts @@ -18,6 +18,7 @@ import * as pkg from 'empathic/package'; import { version } from '../../package.json'; import { globalSettings } from '../cli/globalSettings.ts'; +import { detectAgent } from './detect-agent.ts'; import { getApplicationFileCount } from './get-application-file-count.ts'; import { getChromaticVersionSpecifier } from './get-chromatic-version.ts'; import { getFrameworkInfo } from './get-framework-info.ts'; @@ -113,7 +114,7 @@ export const computeStorybookMetadata = async ({ mainConfig?: StorybookConfig & Record; configDir: string; }): Promise => { - const settings = isCI() ? undefined : await globalSettings(); + const settings = isCI() && !detectAgent() ? undefined : await globalSettings(); const metadata: Partial = { generatedAt: new Date().getTime(), userSince: settings?.value.userSince, diff --git a/code/core/src/telemetry/types.ts b/code/core/src/telemetry/types.ts index 17743b78315a..de4945851310 100644 --- a/code/core/src/telemetry/types.ts +++ b/code/core/src/telemetry/types.ts @@ -46,7 +46,10 @@ export type EventType = | 'share' | 'ghost-stories' | 'ai-prepare' + | 'ai-prepare-evidence' + | 'ai-prepare-story-scoring' | 'ai-prompt-nudge'; + export interface Dependency { version: string | undefined; versionSpecifier?: string; diff --git a/code/core/src/typings.d.ts b/code/core/src/typings.d.ts index 512933868d83..1bfca3505d3d 100644 --- a/code/core/src/typings.d.ts +++ b/code/core/src/typings.d.ts @@ -17,6 +17,11 @@ declare var STORYBOOK_RENDERER: import('./types/modules/renderers').SupportedRen declare var STORYBOOK_HOOKS_CONTEXT: any; declare var STORYBOOK_CURRENT_TASK_LOG: undefined | null | Array; +declare var STORYBOOK_LAST_EVENTS: Record< + import('./telemetry').EventType, + import('./telemetry').CacheEntry +>; +declare var STORYBOOK_SESSION_ID: string | undefined; declare var STORYBOOK_NETWORK_ADDRESS: string | undefined; declare var PREVIEW_URL: string | undefined; diff --git a/code/lib/cli-storybook/src/ai/index.ts b/code/lib/cli-storybook/src/ai/index.ts index a307c3a9aaad..5110b798b050 100644 --- a/code/lib/cli-storybook/src/ai/index.ts +++ b/code/lib/cli-storybook/src/ai/index.ts @@ -2,7 +2,14 @@ import { writeFile } from 'node:fs/promises'; import { resolve } from 'node:path'; import type { PackageManagerName } from 'storybook/internal/common'; +import { cache } from 'storybook/internal/common'; import { logger } from 'storybook/internal/node-logger'; +import { + getSessionId, + snapshotPreviewFile, + telemetry, + type AiPreparePendingRecord, +} from 'storybook/internal/telemetry'; import { SupportedLanguage } from 'storybook/internal/types'; import { ProjectTypeService } from '../../../create-storybook/src/services/ProjectTypeService.ts'; @@ -72,7 +79,37 @@ export async function aiPrepare(options: AiPrepareOptions): Promise { return; } - const markdownOutput = generateMarkdownOutput(projectInfo); + const result = generateMarkdownOutput(projectInfo); + const markdownOutput = result.markdown; + + await telemetry('ai-prepare', { + cliOptions: { + output: output ? 'file' : undefined, + configDir: projectInfo.configDir, + packageManager: packageManagerName, + }, + project: { + framework: projectInfo.framework, + renderer: projectInfo.rendererPackage, + builder: projectInfo.builderPackage, + language: projectInfo.language, + hasCsfFactoryPreview: projectInfo.hasCsfFactoryPreview, + }, + }); + + // Snapshot the preview file baseline and cache the pending setup record. + // Subsequent CLI entry points (dev, build, doctor, etc.) read this to + // collect evidence of what the agent accomplished. + const resolvedConfigDir = resolve(projectInfo.configDir); + const previewSnapshot = await snapshotPreviewFile(resolvedConfigDir); + const sessionId = await getSessionId(); + const pendingRecord: AiPreparePendingRecord = { + timestamp: Date.now(), + sessionId, + configDir: resolvedConfigDir, + ...previewSnapshot, + }; + await cache.set('ai-prepare-pending', pendingRecord); if (output) { const outputPath = resolve(output); diff --git a/code/lib/cli-storybook/src/ai/prompt.ts b/code/lib/cli-storybook/src/ai/prompt.ts index 28759ac175bc..1ad37f97de71 100644 --- a/code/lib/cli-storybook/src/ai/prompt.ts +++ b/code/lib/cli-storybook/src/ai/prompt.ts @@ -21,7 +21,9 @@ export function getDocsMarkdownUrl( return `https://storybook.js.org/docs${versionSegment}/${path}.md${query ? `?${query}` : ''}`; } -export function getPrompts(projectInfo: ProjectInfo): AiPrompt[] { +export function getPrompts(projectInfo: ProjectInfo): { + prompts: AiPrompt[]; +} { const aiPrompts: AiPrompt[] = []; aiPrompts.push({ @@ -30,7 +32,7 @@ export function getPrompts(projectInfo: ProjectInfo): AiPrompt[] { instructions: getSetupInstructions(projectInfo), }); - return aiPrompts; + return { prompts: aiPrompts }; } function getTypeImportSource(projectInfo: ProjectInfo): string { @@ -718,8 +720,10 @@ function getProjectOverview(projectInfo: ProjectInfo): string { `; } -export function generateMarkdownOutput(projectInfo: ProjectInfo): string { - const aiPrompts = getPrompts(projectInfo); +export function generateMarkdownOutput(projectInfo: ProjectInfo): { + markdown: string; +} { + const { prompts: aiPrompts } = getPrompts(projectInfo); const sections: string[] = []; @@ -733,5 +737,5 @@ export function generateMarkdownOutput(projectInfo: ProjectInfo): string { sections.push(aiPrompt.instructions); } - return sections.join('\n\n'); + return { markdown: sections.join('\n\n') }; } diff --git a/scripts/event-log-collector.ts b/scripts/event-log-collector.ts index 1e4a656960b1..6c7e8da2ae21 100644 --- a/scripts/event-log-collector.ts +++ b/scripts/event-log-collector.ts @@ -1,23 +1,87 @@ -import { json } from '@polka/parse'; -import polka from 'polka'; +#!/usr/bin/env node -const PORT = process.env.PORT || 6007; +/** + * Telemetry event log collector for local development and testing. + * + * Usage: + * node scripts/event-log-collector.ts + * + * Then point Storybook at it: + * STORYBOOK_TELEMETRY_URL=http://localhost:6007/event-log yarn storybook + * + * Endpoints: + * POST /event-log — receives telemetry events (logs + stores) + * GET /event-log — returns all received events as JSON array + * GET /events — alias: returns all received events as JSON array + * GET /events/:type — returns events filtered by eventType + */ -const server = polka(); -server.use(json()); +import { createServer } from 'node:http'; +import { writeFile, mkdir } from 'node:fs/promises'; +import { resolve } from 'node:path'; -const events: Record[] = []; -server.post('/event-log', (req, res) => { - console.log(`Received event ${req.body.eventType}`); - events.push(req.body); - res.end('OK'); -}); +const PORT = Number(process.env.PORT || 6007); +const LOG_DIR = resolve(process.env.LOG_DIR || '.cache/telemetry-debug'); +const events: Array<{ receivedAt: string; [key: string]: unknown }> = []; + +await mkdir(LOG_DIR, { recursive: true }); + +const server = createServer(async (req, res) => { + // POST /event-log — receive a telemetry event + if (req.method === 'POST' && req.url === '/event-log') { + let body = ''; + req.on('data', (chunk: Buffer) => { + body += chunk; + }); + req.on('end', async () => { + try { + const data = JSON.parse(body); + const entry = { receivedAt: new Date().toISOString(), ...data }; + events.push(entry); + + console.log(`\n[telemetry] ${data.eventType || 'unknown'}`); + console.log(JSON.stringify(data, null, 2)); + + await writeFile( + resolve(LOG_DIR, `events-${new Date().toISOString().slice(0, 10)}.jsonl`), + JSON.stringify(entry) + '\n', + { flag: 'a' } + ); + + res.statusCode = 200; + res.end('OK'); + } catch { + res.statusCode = 400; + res.end('bad json'); + } + }); + return; + } + + // GET /event-log — return all events (used by event-log-checker) + if (req.method === 'GET' && req.url === '/event-log') { + console.log(`Sending ${events.length} events`); + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify(events)); + return; + } + + // GET /events or GET /events/:type — return all or filtered events + if (req.method === 'GET' && req.url?.startsWith('/events')) { + const typeFilter = req.url.split('/events/')[1]; + const filtered = typeFilter ? events.filter((e) => e.eventType === typeFilter) : events; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify(filtered)); + return; + } -server.get('/event-log', (_req, res) => { - console.log(`Sending ${events.length} events`); - res.end(JSON.stringify(events)); + res.statusCode = 404; + res.end('not found'); }); server.listen(PORT, () => { - console.log(`Event log listening on ${PORT}`); + console.log(`Event log collector listening on http://localhost:${PORT}/event-log`); + console.log(`GET http://localhost:${PORT}/events to see all received events`); + console.log(`GET http://localhost:${PORT}/events/ to filter by event type`); + console.log(`Logs written to ${LOG_DIR}`); });