Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion code/core/src/cli/globalSettings.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ const userSettingSchema = z.object({
items: z
.object({
accessibilityTests: statusValue,
aiPrepare: statusValue,
aiSetup: statusValue,
Comment thread
Sidnioulz marked this conversation as resolved.
autodocs: statusValue,
ciTests: statusValue,
controls: statusValue,
Expand Down
10 changes: 5 additions & 5 deletions code/core/src/core-events/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,9 @@ enum events {
// Story discovery and testing flow
GHOST_STORIES_REQUEST = 'ghostStoriesRequest',
GHOST_STORIES_RESPONSE = 'ghostStoriesResponse',
// AI analytics - ai prepare command
AI_PREPARE_ANALYTICS_RESPONSE = 'aiPrepareAnalyticsResponse',
AI_PREPARE_ANALYTICS_REQUEST = 'aiPrepareAnalyticsRequest',
// AI analytics - ai setup command
AI_SETUP_ANALYTICS_RESPONSE = 'aiSetupAnalyticsResponse',
AI_SETUP_ANALYTICS_REQUEST = 'aiSetupAnalyticsRequest',
// Open a file in the code editor
OPEN_IN_EDITOR_REQUEST = 'openInEditorRequest',
OPEN_IN_EDITOR_RESPONSE = 'openInEditorResponse',
Expand Down Expand Up @@ -173,8 +173,8 @@ export const {
ARGTYPES_INFO_RESPONSE,
GHOST_STORIES_REQUEST,
GHOST_STORIES_RESPONSE,
AI_PREPARE_ANALYTICS_RESPONSE,
AI_PREPARE_ANALYTICS_REQUEST,
AI_SETUP_ANALYTICS_RESPONSE,
AI_SETUP_ANALYTICS_REQUEST,
OPEN_IN_EDITOR_REQUEST,
OPEN_IN_EDITOR_RESPONSE,
MANAGER_INERT_ATTRIBUTE_CHANGED,
Expand Down
2 changes: 1 addition & 1 deletion code/core/src/core-server/presets/common-preset.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ import { initializeSaveStory } from '../utils/save-story/save-story.ts';
import { parseStaticDir } from '../utils/server-statics.ts';
import { type OptionsWithRequiredCache, initializeWhatsNew } from '../utils/whats-new.ts';
import { getWsToken } from './wsToken.ts';
import { initAIAnalyticsChannel } from '../server-channel/ai-prepare-channel.ts';
import { initAIAnalyticsChannel } from '../server-channel/ai-setup-channel.ts';

const interpolate = (string: string, data: Record<string, string> = {}) =>
Object.entries(data).reduce((acc, [k, v]) => acc.replace(new RegExp(`%${k}%`, 'g'), v), string);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
import type { Channel } from 'storybook/internal/channels';
import {
AI_PREPARE_ANALYTICS_REQUEST,
AI_PREPARE_ANALYTICS_RESPONSE,
AI_SETUP_ANALYTICS_REQUEST,
AI_SETUP_ANALYTICS_RESPONSE,
} from 'storybook/internal/core-events';
import {
getLastEvents,
getStorybookMetadata,
isStoryCreatedByAIPrepare,
isStoryCreatedByAISetup,
telemetry,
} from 'storybook/internal/telemetry';
import type { CoreConfig, Options } from 'storybook/internal/types';
Expand All @@ -26,8 +26,8 @@ export function initAIAnalyticsChannel(
return channel;
}

/** Send analytics about the ai prepare workflow when requested*/
channel.on(AI_PREPARE_ANALYTICS_REQUEST, async () => {
/** Send analytics about the ai setup workflow when requested*/
channel.on(AI_SETUP_ANALYTICS_REQUEST, async () => {
const stats: {
fileCount?: number;
storyCount?: number;
Expand All @@ -36,16 +36,16 @@ export function initAIAnalyticsChannel(

try {
const lastEvents = await getLastEvents();
const lastAIPrepare = lastEvents?.['ai-prepare'];
const lastPrepareStoryScoringRun = lastEvents?.['ai-prepare-story-scoring'];
const lastAISetup = lastEvents?.['ai-setup'];
const lastSetupStoryScoringRun = lastEvents?.['ai-setup-story-scoring'];

// Only run if sb ai prepare has been called
if (!lastAIPrepare) {
// Only run if sb ai setup has been called
if (!lastAISetup) {
return;
}

// Already ran once for this project — never run again
if (lastPrepareStoryScoringRun) {
if (lastSetupStoryScoringRun) {
return;
}

Expand All @@ -66,38 +66,38 @@ export function initAIAnalyticsChannel(
// disturb end user activities.
const isIdle = await waitForIdleVitest();
if (!isIdle) {
logger.debug('AI_PREPARE_ANALYTICS_REQUEST timed out waiting for vitest to be available.');
logger.debug('AI_SETUP_ANALYTICS_REQUEST timed out waiting for vitest to be available.');
return;
}

// Fetch AI-generated stories and score them with the ghost stories metrics, if any are found.
const generatorPromise = getStoryIndexGeneratorPromise?.();
if (!generatorPromise) {
logger.debug(
'AI_PREPARE_ANALYTICS_REQUEST could not proceed as the index generator is not ready.'
'AI_SETUP_ANALYTICS_REQUEST could not proceed as the index generator is not ready.'
);
return;
}

const generator = await generatorPromise;
const indexAndStats = await generator.getIndexAndStats();
if (!indexAndStats) {
logger.debug('AI_PREPARE_ANALYTICS_REQUEST could not proceed as the index is not ready.');
logger.debug('AI_SETUP_ANALYTICS_REQUEST could not proceed as the index is not ready.');
return;
}

const aiStoryFiles = new Set<string>();
let aiStoryCount = 0;
for (const entry of Object.values(indexAndStats.storyIndex.entries)) {
if (isStoryCreatedByAIPrepare(entry)) {
if (isStoryCreatedByAISetup(entry)) {
aiStoryFiles.add(entry.importPath);
aiStoryCount++;
}
}

if (aiStoryFiles.size > 0) {
const aiTestRunResult = await runGhostStories([...aiStoryFiles]);
telemetry('ai-prepare-story-scoring', {
telemetry('ai-setup-story-scoring', {
stats: {
fileCount: aiStoryFiles.size,
storyCount: aiStoryCount,
Expand All @@ -107,7 +107,7 @@ export function initAIAnalyticsChannel(
...(aiTestRunResult.runError ? { runError: aiTestRunResult.runError } : {}),
});
} else {
telemetry('ai-prepare-story-scoring', {
telemetry('ai-setup-story-scoring', {
stats: {
fileCount: 0,
storyCount: 0,
Expand All @@ -117,13 +117,13 @@ export function initAIAnalyticsChannel(
});
}
} catch {
telemetry('ai-prepare-story-scoring', {
telemetry('ai-setup-story-scoring', {
stats,
runError: 'Unknown error during AI story scoring',
});
} finally {
// we don't currently do anything with this, but will be useful in the future
channel.emit(AI_PREPARE_ANALYTICS_RESPONSE);
channel.emit(AI_SETUP_ANALYTICS_RESPONSE);
}
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,11 @@ export function initGhostStoriesChannel(
const ghostRunStart = Date.now();
const lastEvents = await getLastEvents();
const lastInit = lastEvents?.init;
const lastAIPrepare = lastEvents?.['ai-prepare'];
const lastAISetup = lastEvents?.['ai-setup'];
const lastGhostStoriesRun = lastEvents?.['ghost-stories'];

// We only want to run ghost stories immediately after init or ai prepare.
const lastRelevantEvent = lastAIPrepare ?? lastInit;
// We only want to run ghost stories immediately after init or ai setup.
const lastRelevantEvent = lastAISetup ?? lastInit;
if (!lastRelevantEvent) {
return;
}
Expand All @@ -53,7 +53,7 @@ export function initGhostStoriesChannel(
}

const sessionId = await getSessionId();
// We only capture ghost stories in the first ever session since init or ai prepare.
// We only capture ghost stories in the first ever session since init or ai setup.
if (lastRelevantEvent.body?.sessionId && lastRelevantEvent.body.sessionId !== sessionId) {
return;
}
Expand Down
22 changes: 11 additions & 11 deletions code/core/src/core-server/utils/checklist.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,49 +69,49 @@ describe('initializeChecklist', () => {
} as unknown as Awaited<ReturnType<typeof globalSettings>>);
});

it('keeps aiPrepare as open when no ai-prepare event exists in cache', async () => {
it('keeps aiSetup as open when no ai-setup event exists in cache', async () => {
const { get: getEventCacheEntry } = await import('../../telemetry/event-cache.ts');
vi.mocked(getEventCacheEntry).mockResolvedValue(undefined);

const { initializeChecklist } = await import('./checklist.ts');
await initializeChecklist();

const state = mockStore.getState();
expect(state.items.aiPrepare.status).toBe('open');
expect(state.items.aiSetup.status).toBe('open');
});

it('marks aiPrepare as done when ai-prepare event exists in cache', async () => {
it('marks aiSetup as done when ai-setup event exists in cache', async () => {
const { get: getEventCacheEntry } = await import('../../telemetry/event-cache.ts');
vi.mocked(getEventCacheEntry).mockResolvedValue({
timestamp: Date.now(),
body: { eventType: 'ai-prepare' } as TelemetryEvent,
body: { eventType: 'ai-setup' } as TelemetryEvent,
} satisfies CacheEntry);

const { initializeChecklist } = await import('./checklist.ts');
await initializeChecklist();

const state = mockStore.getState();
expect(state.items.aiPrepare.status).toBe('done');
expect(state.items.aiSetup.status).toBe('done');
});

it('does not overwrite aiPrepare status if already done from persisted state', async () => {
// Simulate persisted user state where aiPrepare is already 'skipped'
it('does not overwrite aiSetup status if already done from persisted state', async () => {
// Simulate persisted user state where aiSetup is already 'skipped'
mockSettingsValue.checklist = {
items: { aiPrepare: { status: 'skipped' } },
items: { aiSetup: { status: 'skipped' } },
widget: {},
};

const { get: getEventCacheEntry } = await import('../../telemetry/event-cache.ts');
vi.mocked(getEventCacheEntry).mockResolvedValue({
timestamp: Date.now(),
body: { eventType: 'ai-prepare' } as TelemetryEvent,
body: { eventType: 'ai-setup' } as TelemetryEvent,
} satisfies CacheEntry);

const { initializeChecklist } = await import('./checklist.ts');
await initializeChecklist();

const state = mockStore.getState();
// The ai-prepare event was found, but status was 'skipped' (not 'open'), so it stays 'skipped'
expect(state.items.aiPrepare.status).toBe('skipped');
// The ai-setup event was found, but status was 'skipped' (not 'open'), so it stays 'skipped'
expect(state.items.aiSetup.status).toBe('skipped');
});
});
10 changes: 5 additions & 5 deletions code/core/src/core-server/utils/checklist.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,14 +63,14 @@ export async function initializeChecklist() {
}) satisfies StoreState
);

// Check if ai-prepare has ever been run and mark it as done
const aiPrepareEvent = await getEventCacheEntry('ai-prepare');
if (aiPrepareEvent) {
// Check if ai-setup has ever been run and mark it as done
const aiSetupEvent = await getEventCacheEntry('ai-setup');
if (aiSetupEvent) {
const currentState = store.getState();
if (currentState.items.aiPrepare?.status === 'open') {
if (currentState.items.aiSetup?.status === 'open') {
store.setState((state) => ({
...state,
items: { ...state.items, aiPrepare: { ...state.items.aiPrepare, status: 'done' } },
items: { ...state.items, aiSetup: { ...state.items.aiSetup, status: 'done' } },
}));
Comment thread
Sidnioulz marked this conversation as resolved.
}
}
Expand Down
4 changes: 2 additions & 2 deletions code/core/src/core-server/utils/doTelemetry.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import {
collectAiPrepareEvidence,
collectAiSetupEvidence,
getPrecedingUpgrade,
telemetry,
} from 'storybook/internal/telemetry';
Expand Down Expand Up @@ -45,7 +45,7 @@ export async function doTelemetry(
// directly. This is the entry point for collecting evidence about those side effects and
// recording them in telemetry.
if (indexAndStats) {
collectAiPrepareEvidence('dev', options.configDir, indexAndStats.storyIndex);
collectAiSetupEvidence('dev', options.configDir, indexAndStats.storyIndex);
}

const { versionCheck, versionUpdates } = options;
Expand Down
16 changes: 8 additions & 8 deletions code/core/src/core-server/withTelemetry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { cache, isCI, loadAllPresets } from 'storybook/internal/common';
import { prompt } from 'storybook/internal/node-logger';
import {
ErrorCollector,
collectAiPrepareEvidence,
collectAiSetupEvidence,
oneWayHash,
telemetry,
} from 'storybook/internal/telemetry';
Expand Down Expand Up @@ -34,15 +34,15 @@ describe('withTelemetry', () => {
vi.resetAllMocks();
vi.mocked(ErrorCollector.getErrors).mockReturnValue([]);
vi.mocked(telemetry).mockResolvedValue(undefined);
vi.mocked(collectAiPrepareEvidence).mockResolvedValue(undefined);
vi.mocked(collectAiSetupEvidence).mockResolvedValue(undefined);
});
it('works in happy path', async () => {
const run = vi.fn();

await withTelemetry('dev', { cliOptions }, run);

expect(telemetry).toHaveBeenCalledTimes(1);
expect(collectAiPrepareEvidence).toHaveBeenCalledTimes(1);
expect(collectAiSetupEvidence).toHaveBeenCalledTimes(1);
expect(telemetry).toHaveBeenCalledWith('boot', { eventType: 'dev' }, { stripMetadata: true });
});

Expand All @@ -52,7 +52,7 @@ describe('withTelemetry', () => {
await withTelemetry('dev', { cliOptions: { disableTelemetry: true } }, run);

expect(telemetry).toHaveBeenCalledTimes(0);
expect(collectAiPrepareEvidence).toHaveBeenCalledTimes(0);
expect(collectAiSetupEvidence).toHaveBeenCalledTimes(0);
});

describe('when command fails', () => {
Expand All @@ -67,7 +67,7 @@ describe('withTelemetry', () => {
).rejects.toThrow(error);

expect(telemetry).toHaveBeenCalledWith('boot', { eventType: 'dev' }, { stripMetadata: true });
expect(collectAiPrepareEvidence).toHaveBeenCalledTimes(1);
expect(collectAiSetupEvidence).toHaveBeenCalledTimes(1);
});

it('does not send boot when cli option is passed', async () => {
Expand All @@ -76,7 +76,7 @@ describe('withTelemetry', () => {
).rejects.toThrow(error);

expect(telemetry).toHaveBeenCalledTimes(0);
expect(collectAiPrepareEvidence).toHaveBeenCalledTimes(0);
expect(collectAiSetupEvidence).toHaveBeenCalledTimes(0);
});

it('sends error message when no options are passed', async () => {
Expand All @@ -85,7 +85,7 @@ describe('withTelemetry', () => {
).rejects.toThrow(error);

expect(telemetry).toHaveBeenCalledTimes(2);
expect(collectAiPrepareEvidence).toHaveBeenCalledTimes(1);
expect(collectAiSetupEvidence).toHaveBeenCalledTimes(1);
expect(telemetry).toHaveBeenCalledWith(
'error',
expect.objectContaining({
Expand Down Expand Up @@ -149,7 +149,7 @@ describe('withTelemetry', () => {
).rejects.toThrow(error);

expect(telemetry).toHaveBeenCalledTimes(0);
expect(collectAiPrepareEvidence).toHaveBeenCalledTimes(0);
expect(collectAiSetupEvidence).toHaveBeenCalledTimes(0);
expect(telemetry).not.toHaveBeenCalledWith(
'error',
expect.objectContaining({}),
Expand Down
4 changes: 2 additions & 2 deletions code/core/src/core-server/withTelemetry.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { HandledError, cache, isCI, loadAllPresets } from 'storybook/internal/common';
import { logger, prompt } from 'storybook/internal/node-logger';
import {
collectAiPrepareEvidence,
collectAiSetupEvidence,
ErrorCollector,
getPrecedingUpgrade,
oneWayHash,
Expand Down Expand Up @@ -189,7 +189,7 @@ export async function withTelemetry<T>(
if (enableTelemetry) {
// Fire-and-forget: don't await, don't block the command
const configDir = options.cliOptions.configDir || options.presetOptions?.configDir;
collectAiPrepareEvidence(eventType, configDir);
collectAiSetupEvidence(eventType, configDir);
}

try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,14 +95,14 @@ export const Narrow = meta.story({
play,
});

export const WithAiPrepare = meta.story({
export const WithAiSetup = meta.story({
beforeEach: async () => {
mockStore.setState({
loaded: true,
widget: {},
items: {
...initialState.items,
// aiPrepare is intentionally left 'open' so it appears in the widget's task list
// aiSetup is intentionally left 'open' so it appears in the widget's task list
controls: { status: 'accepted' },
renderComponent: { status: 'done' },
},
Expand Down
Loading
Loading