Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion code/core/src/cli/globalSettings.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ const userSettingSchema = z.object({
items: z
.object({
accessibilityTests: statusValue,
aiPrepare: statusValue,
aiSetup: statusValue,
Comment thread
Sidnioulz marked this conversation as resolved.
autodocs: statusValue,
ciTests: statusValue,
controls: statusValue,
Expand Down
10 changes: 5 additions & 5 deletions code/core/src/core-events/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,9 @@ enum events {
// Story discovery and testing flow
GHOST_STORIES_REQUEST = 'ghostStoriesRequest',
GHOST_STORIES_RESPONSE = 'ghostStoriesResponse',
// AI analytics - ai prepare command
AI_PREPARE_ANALYTICS_RESPONSE = 'aiPrepareAnalyticsResponse',
AI_PREPARE_ANALYTICS_REQUEST = 'aiPrepareAnalyticsRequest',
// AI analytics - ai setup command
AI_SETUP_ANALYTICS_RESPONSE = 'aiSetupAnalyticsResponse',
AI_SETUP_ANALYTICS_REQUEST = 'aiSetupAnalyticsRequest',
// Open a file in the code editor
OPEN_IN_EDITOR_REQUEST = 'openInEditorRequest',
OPEN_IN_EDITOR_RESPONSE = 'openInEditorResponse',
Expand Down Expand Up @@ -173,8 +173,8 @@ export const {
ARGTYPES_INFO_RESPONSE,
GHOST_STORIES_REQUEST,
GHOST_STORIES_RESPONSE,
AI_PREPARE_ANALYTICS_RESPONSE,
AI_PREPARE_ANALYTICS_REQUEST,
AI_SETUP_ANALYTICS_RESPONSE,
AI_SETUP_ANALYTICS_REQUEST,
OPEN_IN_EDITOR_REQUEST,
OPEN_IN_EDITOR_RESPONSE,
MANAGER_INERT_ATTRIBUTE_CHANGED,
Expand Down
2 changes: 1 addition & 1 deletion code/core/src/core-server/presets/common-preset.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ import { initializeSaveStory } from '../utils/save-story/save-story.ts';
import { parseStaticDir } from '../utils/server-statics.ts';
import { type OptionsWithRequiredCache, initializeWhatsNew } from '../utils/whats-new.ts';
import { getWsToken } from './wsToken.ts';
import { initAIAnalyticsChannel } from '../server-channel/ai-prepare-channel.ts';
import { initAIAnalyticsChannel } from '../server-channel/ai-setup-channel.ts';

const interpolate = (string: string, data: Record<string, string> = {}) =>
Object.entries(data).reduce((acc, [k, v]) => acc.replace(new RegExp(`%${k}%`, 'g'), v), string);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
import type { Channel } from 'storybook/internal/channels';
import {
AI_PREPARE_ANALYTICS_REQUEST,
AI_PREPARE_ANALYTICS_RESPONSE,
AI_SETUP_ANALYTICS_REQUEST,
AI_SETUP_ANALYTICS_RESPONSE,
} from 'storybook/internal/core-events';
import {
getLastEvents,
getStorybookMetadata,
isStoryCreatedByAIPrepare,
isStoryCreatedByAISetup,
telemetry,
} from 'storybook/internal/telemetry';
import type { CoreConfig, Options } from 'storybook/internal/types';
Expand All @@ -26,8 +26,8 @@ export function initAIAnalyticsChannel(
return channel;
}

/** Send analytics about the ai prepare workflow when requested*/
channel.on(AI_PREPARE_ANALYTICS_REQUEST, async () => {
/** Send analytics about the ai setup workflow when requested*/
channel.on(AI_SETUP_ANALYTICS_REQUEST, async () => {
const stats: {
fileCount?: number;
storyCount?: number;
Expand All @@ -36,16 +36,16 @@ export function initAIAnalyticsChannel(

try {
const lastEvents = await getLastEvents();
const lastAIPrepare = lastEvents?.['ai-prepare'];
const lastPrepareStoryScoringRun = lastEvents?.['ai-prepare-story-scoring'];
const lastAISetup = lastEvents?.['ai-setup'];
const lastSetupStoryScoringRun = lastEvents?.['ai-setup-story-scoring'];

// Only run if sb ai prepare has been called
if (!lastAIPrepare) {
// Only run if sb ai setup has been called
if (!lastAISetup) {
return;
}

// Already ran once for this project — never run again
if (lastPrepareStoryScoringRun) {
if (lastSetupStoryScoringRun) {
return;
}

Expand All @@ -66,38 +66,38 @@ export function initAIAnalyticsChannel(
// disturb end user activities.
const isIdle = await waitForIdleVitest();
if (!isIdle) {
logger.debug('AI_PREPARE_ANALYTICS_REQUEST timed out waiting for vitest to be available.');
logger.debug('AI_SETUP_ANALYTICS_REQUEST timed out waiting for vitest to be available.');
return;
}

// Fetch AI-generated stories and score them with the ghost stories metrics, if any are found.
const generatorPromise = getStoryIndexGeneratorPromise?.();
if (!generatorPromise) {
logger.debug(
'AI_PREPARE_ANALYTICS_REQUEST could not proceed as the index generator is not ready.'
'AI_SETUP_ANALYTICS_REQUEST could not proceed as the index generator is not ready.'
);
return;
}

const generator = await generatorPromise;
const indexAndStats = await generator.getIndexAndStats();
if (!indexAndStats) {
logger.debug('AI_PREPARE_ANALYTICS_REQUEST could not proceed as the index is not ready.');
logger.debug('AI_SETUP_ANALYTICS_REQUEST could not proceed as the index is not ready.');
return;
}

const aiStoryFiles = new Set<string>();
let aiStoryCount = 0;
for (const entry of Object.values(indexAndStats.storyIndex.entries)) {
if (isStoryCreatedByAIPrepare(entry)) {
if (isStoryCreatedByAISetup(entry)) {
aiStoryFiles.add(entry.importPath);
aiStoryCount++;
}
}

if (aiStoryFiles.size > 0) {
const aiTestRunResult = await runGhostStories([...aiStoryFiles]);
telemetry('ai-prepare-story-scoring', {
telemetry('ai-setup-story-scoring', {
stats: {
fileCount: aiStoryFiles.size,
storyCount: aiStoryCount,
Expand All @@ -107,7 +107,7 @@ export function initAIAnalyticsChannel(
...(aiTestRunResult.runError ? { runError: aiTestRunResult.runError } : {}),
});
} else {
telemetry('ai-prepare-story-scoring', {
telemetry('ai-setup-story-scoring', {
stats: {
fileCount: 0,
storyCount: 0,
Expand All @@ -117,13 +117,13 @@ export function initAIAnalyticsChannel(
});
}
} catch {
telemetry('ai-prepare-story-scoring', {
telemetry('ai-setup-story-scoring', {
stats,
runError: 'Unknown error during AI story scoring',
});
} finally {
// we don't currently do anything with this, but will be useful in the future
channel.emit(AI_PREPARE_ANALYTICS_RESPONSE);
channel.emit(AI_SETUP_ANALYTICS_RESPONSE);
}
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,11 @@ export function initGhostStoriesChannel(
const ghostRunStart = Date.now();
const lastEvents = await getLastEvents();
const lastInit = lastEvents?.init;
const lastAIPrepare = lastEvents?.['ai-prepare'];
const lastAISetup = lastEvents?.['ai-setup'];
const lastGhostStoriesRun = lastEvents?.['ghost-stories'];

// We only want to run ghost stories immediately after init or ai prepare.
const lastRelevantEvent = lastAIPrepare ?? lastInit;
// We only want to run ghost stories immediately after init or ai setup.
const lastRelevantEvent = lastAISetup ?? lastInit;
if (!lastRelevantEvent) {
return;
}
Expand All @@ -53,7 +53,7 @@ export function initGhostStoriesChannel(
}

const sessionId = await getSessionId();
// We only capture ghost stories in the first ever session since init or ai prepare.
// We only capture ghost stories in the first ever session since init or ai setup.
if (lastRelevantEvent.body?.sessionId && lastRelevantEvent.body.sessionId !== sessionId) {
return;
}
Expand Down
22 changes: 11 additions & 11 deletions code/core/src/core-server/utils/checklist.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,49 +69,49 @@ describe('initializeChecklist', () => {
} as unknown as Awaited<ReturnType<typeof globalSettings>>);
});

it('keeps aiPrepare as open when no ai-prepare event exists in cache', async () => {
it('keeps aiSetup as open when no ai-setup event exists in cache', async () => {
const { get: getEventCacheEntry } = await import('../../telemetry/event-cache.ts');
vi.mocked(getEventCacheEntry).mockResolvedValue(undefined);

const { initializeChecklist } = await import('./checklist.ts');
await initializeChecklist();

const state = mockStore.getState();
expect(state.items.aiPrepare.status).toBe('open');
expect(state.items.aiSetup.status).toBe('open');
});

it('marks aiPrepare as done when ai-prepare event exists in cache', async () => {
it('marks aiSetup as done when ai-setup event exists in cache', async () => {
const { get: getEventCacheEntry } = await import('../../telemetry/event-cache.ts');
vi.mocked(getEventCacheEntry).mockResolvedValue({
timestamp: Date.now(),
body: { eventType: 'ai-prepare' } as TelemetryEvent,
body: { eventType: 'ai-setup' } as TelemetryEvent,
} satisfies CacheEntry);

const { initializeChecklist } = await import('./checklist.ts');
await initializeChecklist();

const state = mockStore.getState();
expect(state.items.aiPrepare.status).toBe('done');
expect(state.items.aiSetup.status).toBe('done');
});

it('does not overwrite aiPrepare status if already done from persisted state', async () => {
// Simulate persisted user state where aiPrepare is already 'skipped'
it('does not overwrite aiSetup status if already done from persisted state', async () => {
// Simulate persisted user state where aiSetup is already 'skipped'
mockSettingsValue.checklist = {
items: { aiPrepare: { status: 'skipped' } },
items: { aiSetup: { status: 'skipped' } },
widget: {},
};

const { get: getEventCacheEntry } = await import('../../telemetry/event-cache.ts');
vi.mocked(getEventCacheEntry).mockResolvedValue({
timestamp: Date.now(),
body: { eventType: 'ai-prepare' } as TelemetryEvent,
body: { eventType: 'ai-setup' } as TelemetryEvent,
} satisfies CacheEntry);

const { initializeChecklist } = await import('./checklist.ts');
await initializeChecklist();

const state = mockStore.getState();
// The ai-prepare event was found, but status was 'skipped' (not 'open'), so it stays 'skipped'
expect(state.items.aiPrepare.status).toBe('skipped');
// The ai-setup event was found, but status was 'skipped' (not 'open'), so it stays 'skipped'
expect(state.items.aiSetup.status).toBe('skipped');
});
});
10 changes: 5 additions & 5 deletions code/core/src/core-server/utils/checklist.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,14 +63,14 @@ export async function initializeChecklist() {
}) satisfies StoreState
);

// Check if ai-prepare has ever been run and mark it as done
const aiPrepareEvent = await getEventCacheEntry('ai-prepare');
if (aiPrepareEvent) {
// Check if ai-setup has ever been run and mark it as done
const aiSetupEvent = await getEventCacheEntry('ai-setup');
if (aiSetupEvent) {
const currentState = store.getState();
if (currentState.items.aiPrepare?.status === 'open') {
if (currentState.items.aiSetup?.status === 'open') {
store.setState((state) => ({
...state,
items: { ...state.items, aiPrepare: { ...state.items.aiPrepare, status: 'done' } },
items: { ...state.items, aiSetup: { ...state.items.aiSetup, status: 'done' } },
}));
Comment thread
Sidnioulz marked this conversation as resolved.
}
}
Expand Down
4 changes: 2 additions & 2 deletions code/core/src/core-server/utils/doTelemetry.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import {
collectAiPrepareEvidence,
collectAiSetupEvidence,
getPrecedingUpgrade,
telemetry,
} from 'storybook/internal/telemetry';
Expand Down Expand Up @@ -45,7 +45,7 @@ export async function doTelemetry(
// directly. This is the entry point for collecting evidence about those side effects and
// recording them in telemetry.
if (indexAndStats) {
collectAiPrepareEvidence('dev', options.configDir, indexAndStats.storyIndex);
collectAiSetupEvidence('dev', options.configDir, indexAndStats.storyIndex);
}

const { versionCheck, versionUpdates } = options;
Expand Down
16 changes: 8 additions & 8 deletions code/core/src/core-server/withTelemetry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { cache, isCI, loadAllPresets } from 'storybook/internal/common';
import { prompt } from 'storybook/internal/node-logger';
import {
ErrorCollector,
collectAiPrepareEvidence,
collectAiSetupEvidence,
oneWayHash,
telemetry,
} from 'storybook/internal/telemetry';
Expand Down Expand Up @@ -34,15 +34,15 @@ describe('withTelemetry', () => {
vi.resetAllMocks();
vi.mocked(ErrorCollector.getErrors).mockReturnValue([]);
vi.mocked(telemetry).mockResolvedValue(undefined);
vi.mocked(collectAiPrepareEvidence).mockResolvedValue(undefined);
vi.mocked(collectAiSetupEvidence).mockResolvedValue(undefined);
});
it('works in happy path', async () => {
const run = vi.fn();

await withTelemetry('dev', { cliOptions }, run);

expect(telemetry).toHaveBeenCalledTimes(1);
expect(collectAiPrepareEvidence).toHaveBeenCalledTimes(1);
expect(collectAiSetupEvidence).toHaveBeenCalledTimes(1);
expect(telemetry).toHaveBeenCalledWith('boot', { eventType: 'dev' }, { stripMetadata: true });
});

Expand All @@ -52,7 +52,7 @@ describe('withTelemetry', () => {
await withTelemetry('dev', { cliOptions: { disableTelemetry: true } }, run);

expect(telemetry).toHaveBeenCalledTimes(0);
expect(collectAiPrepareEvidence).toHaveBeenCalledTimes(0);
expect(collectAiSetupEvidence).toHaveBeenCalledTimes(0);
});

describe('when command fails', () => {
Expand All @@ -67,7 +67,7 @@ describe('withTelemetry', () => {
).rejects.toThrow(error);

expect(telemetry).toHaveBeenCalledWith('boot', { eventType: 'dev' }, { stripMetadata: true });
expect(collectAiPrepareEvidence).toHaveBeenCalledTimes(1);
expect(collectAiSetupEvidence).toHaveBeenCalledTimes(1);
});

it('does not send boot when cli option is passed', async () => {
Expand All @@ -76,7 +76,7 @@ describe('withTelemetry', () => {
).rejects.toThrow(error);

expect(telemetry).toHaveBeenCalledTimes(0);
expect(collectAiPrepareEvidence).toHaveBeenCalledTimes(0);
expect(collectAiSetupEvidence).toHaveBeenCalledTimes(0);
});

it('sends error message when no options are passed', async () => {
Expand All @@ -85,7 +85,7 @@ describe('withTelemetry', () => {
).rejects.toThrow(error);

expect(telemetry).toHaveBeenCalledTimes(2);
expect(collectAiPrepareEvidence).toHaveBeenCalledTimes(1);
expect(collectAiSetupEvidence).toHaveBeenCalledTimes(1);
expect(telemetry).toHaveBeenCalledWith(
'error',
expect.objectContaining({
Expand Down Expand Up @@ -149,7 +149,7 @@ describe('withTelemetry', () => {
).rejects.toThrow(error);

expect(telemetry).toHaveBeenCalledTimes(0);
expect(collectAiPrepareEvidence).toHaveBeenCalledTimes(0);
expect(collectAiSetupEvidence).toHaveBeenCalledTimes(0);
expect(telemetry).not.toHaveBeenCalledWith(
'error',
expect.objectContaining({}),
Expand Down
4 changes: 2 additions & 2 deletions code/core/src/core-server/withTelemetry.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { HandledError, cache, isCI, loadAllPresets } from 'storybook/internal/common';
import { logger, prompt } from 'storybook/internal/node-logger';
import {
collectAiPrepareEvidence,
collectAiSetupEvidence,
ErrorCollector,
getPrecedingUpgrade,
oneWayHash,
Expand Down Expand Up @@ -189,7 +189,7 @@ export async function withTelemetry<T>(
if (enableTelemetry) {
// Fire-and-forget: don't await, don't block the command
const configDir = options.cliOptions.configDir || options.presetOptions?.configDir;
collectAiPrepareEvidence(eventType, configDir);
collectAiSetupEvidence(eventType, configDir);
}

try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,14 +95,14 @@ export const Narrow = meta.story({
play,
});

export const WithAiPrepare = meta.story({
export const WithAiSetup = meta.story({
beforeEach: async () => {
mockStore.setState({
loaded: true,
widget: {},
items: {
...initialState.items,
// aiPrepare is intentionally left 'open' so it appears in the widget's task list
// aiSetup is intentionally left 'open' so it appears in the widget's task list
controls: { status: 'accepted' },
renderComponent: { status: 'done' },
},
Expand Down
Loading
Loading