Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,6 @@ describe('ghostStoriesChannel', () => {
mockFs.existsSync.mockReturnValue(true);
mockFs.readFile.mockResolvedValue(
JSON.stringify({
success: true,
numTotalTests: 2,
numPassedTests: 2,
numFailedTests: 0,
Expand Down Expand Up @@ -186,7 +185,6 @@ describe('ghostStoriesChannel', () => {

// Telemetry is called with the correct data
expect(mockTelemetry.telemetry).toHaveBeenCalledWith('ghost-stories', {
success: true,
stats: {
globMatchCount: 10,
candidateAnalysisDuration: expect.any(Number),
Expand All @@ -199,8 +197,6 @@ describe('ghostStoriesChannel', () => {
results: {
total: 2,
passed: 2,
failed: 0,
failureRate: 0,
successRate: 1,
successRateWithoutEmptyRender: 1,
categorizedErrors: expect.any(Object),
Expand Down Expand Up @@ -238,7 +234,6 @@ describe('ghostStoriesChannel', () => {
mockFs.existsSync.mockReturnValue(true);
mockFs.readFile.mockResolvedValue(
JSON.stringify({
success: false,
numTotalTests: 2,
numPassedTests: 0,
numFailedTests: 2,
Expand Down Expand Up @@ -291,7 +286,6 @@ describe('ghostStoriesChannel', () => {
expect(mockTelemetry.telemetry).toHaveBeenCalledWith(
'ghost-stories',
expect.objectContaining({
success: false,
stats: {
globMatchCount: 10,
candidateAnalysisDuration: expect.any(Number),
Expand All @@ -304,8 +298,6 @@ describe('ghostStoriesChannel', () => {
results: expect.objectContaining({
total: 2,
passed: 0,
failed: 2,
failureRate: 1,
successRate: 0,
// categorizedErrors is now an object with categories as keys
categorizedErrors: expect.any(Object),
Expand Down Expand Up @@ -437,8 +429,7 @@ describe('ghostStoriesChannel', () => {

expect(mockStoryGeneration.getComponentCandidates).toHaveBeenCalled();
expect(mockTelemetry.telemetry).toHaveBeenCalledWith('ghost-stories', {
success: false,
error: 'Failed to analyze components',
runError: 'Failed to analyze components',
stats: {
globMatchCount: 0,
candidateAnalysisDuration: 0,
Expand Down Expand Up @@ -478,8 +469,7 @@ describe('ghostStoriesChannel', () => {

expect(mockStoryGeneration.getComponentCandidates).toHaveBeenCalled();
expect(mockTelemetry.telemetry).toHaveBeenCalledWith('ghost-stories', {
success: false,
error: 'No candidates found',
runError: 'No candidates found',
stats: {
globMatchCount: 5,
candidateAnalysisDuration: expect.any(Number),
Expand Down Expand Up @@ -523,8 +513,7 @@ describe('ghostStoriesChannel', () => {
});

expect(mockTelemetry.telemetry).toHaveBeenCalledWith('ghost-stories', {
success: false,
error: 'JSON report not found',
runError: 'JSON report not found',
stats: {
globMatchCount: 5,
candidateAnalysisDuration: 0,
Expand Down Expand Up @@ -561,7 +550,6 @@ describe('ghostStoriesChannel', () => {
mockFs.existsSync.mockReturnValue(true);
mockFs.readFile.mockResolvedValue(
JSON.stringify({
success: false,
numTotalTests: 2,
numPassedTests: 0,
numFailedTests: 2,
Expand All @@ -578,8 +566,7 @@ describe('ghostStoriesChannel', () => {
});

expect(mockTelemetry.telemetry).toHaveBeenCalledWith('ghost-stories', {
success: false,
error: 'Startup Error',
runError: 'Startup Error',
stats: {
globMatchCount: 5,
candidateAnalysisDuration: 0,
Expand All @@ -605,8 +592,7 @@ describe('ghostStoriesChannel', () => {
});

expect(mockTelemetry.telemetry).toHaveBeenCalledWith('ghost-stories', {
success: false,
error: 'Cache error',
runError: 'Unknown error during ghost run',
stats: {},
});
});
Expand Down
33 changes: 19 additions & 14 deletions code/core/src/core-server/server-channel/ghost-stories-channel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,16 @@ export function initGhostStoriesChannel(
try {
const ghostRunStart = Date.now();
const lastEvents = await getLastEvents();
const sessionId = await getSessionId();
const lastInit = lastEvents?.init;
const lastGhostStoriesRun = lastEvents?.['ghost-stories'];
if (!lastEvents || !lastInit) {
return;
}

const sessionId = await getSessionId();
const lastGhostStoriesRun = lastEvents['ghost-stories'];
if (
lastGhostStoriesRun ||
(lastInit?.body?.sessionId && lastInit?.body?.sessionId !== sessionId)
(lastInit.body?.sessionId && lastInit.body.sessionId !== sessionId)
) {
return;
}
Expand Down Expand Up @@ -70,19 +74,17 @@ export function initGhostStoriesChannel(
if (candidatesResult.error) {
stats.totalRunDuration = Date.now() - ghostRunStart;
telemetry('ghost-stories', {
success: false,
error: candidatesResult.error,
stats,
runError: candidatesResult.error,
});
return;
}

if (candidatesResult.candidates.length === 0) {
stats.totalRunDuration = Date.now() - ghostRunStart;
telemetry('ghost-stories', {
success: false,
error: 'No candidates found',
stats,
runError: 'No candidates found',
});
return;
}
Expand All @@ -92,19 +94,22 @@ export function initGhostStoriesChannel(
const testRunResult = await runStoryTests(candidatesResult.candidates);
stats.totalRunDuration = Date.now() - ghostRunStart;
stats.testRunDuration = testRunResult.duration;
if (testRunResult.runError) {
telemetry('ghost-stories', {
stats,
runError: testRunResult.runError,
});
return;
}

telemetry('ghost-stories', {
...(testRunResult.error !== undefined ? { error: testRunResult.error } : {}),
success: testRunResult.success,
stats,
results: testRunResult.summary,
});
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : String(error);

} catch {
telemetry('ghost-stories', {
success: false,
error: errorMessage,
stats,
runError: 'Unknown error during ghost run',
});
} finally {
// we don't currently do anything with this, but will be useful in the future
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -185,8 +185,7 @@ export async function getComponentCandidates({
candidates,
globMatchCount,
};
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : String(error);
} catch {
return {
candidates: [],
error: 'Failed to find candidates',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,12 @@ describe('parse-vitest-report', () => {

const result = parseVitestResults(mockVitestResults);

expect(result.success).toBe(true);
expect(result.summary).toEqual({
total: 3,
passed: 3,
passedButEmptyRender: 0,
failed: 0,
successRate: 1.0,
successRateWithoutEmptyRender: 1.0,
failureRate: 0.0,
uniqueErrorCount: 0,
categorizedErrors: {},
});
Expand Down Expand Up @@ -88,12 +85,9 @@ describe('parse-vitest-report', () => {

const result = parseVitestResults(mockVitestResults);

expect(result.success).toBe(false);
expect(result.summary?.total).toBe(3);
expect(result.summary?.passed).toBe(1);
expect(result.summary?.failed).toBe(2);
expect(result.summary?.successRate).toBe(0.33);
expect(result.summary?.failureRate).toBe(0.67);
expect(result.summary?.uniqueErrorCount).toBe(2);
});

Expand Down Expand Up @@ -121,11 +115,18 @@ describe('parse-vitest-report', () => {
{
fullName: 'Story3',
status: 'failed',
failureMessages: ['Error: Module not found: react-router\n at import statement'],
failureMessages: [
'Error: Cannot read property "x" of undefined\n at /deps/styled-components.js:1168:14',
],
},
{
fullName: 'Story4',
status: 'failed',
failureMessages: ['Error: Module not found: react-router\n at import statement'],
},
{
fullName: 'Story5',
status: 'failed',
failureMessages: ['Error: Invalid hook call\n at useEffect'],
},
],
Expand All @@ -135,25 +136,23 @@ describe('parse-vitest-report', () => {

const result = parseVitestResults(mockVitestResults);

expect(result.success).toBe(false);
expect(result.summary?.total).toBe(4);
expect(result.summary?.passed).toBe(1);
expect(result.summary?.failed).toBe(3);
expect(result.summary?.uniqueErrorCount).toBe(3);
expect(result.summary?.categorizedErrors).toEqual({
HOOK_USAGE_ERROR: {
uniqueCount: 1,
count: 1,
description: 'React hook was used incorrectly',
matchedDependencies: [],
},
MISSING_THEME_PROVIDER: {
count: 1,
description: 'Component attempted to access theme values without a theme provider',
uniqueCount: 1,
count: 2,
matchedDependencies: ['styled-components'],
},
MODULE_IMPORT_ERROR: {
uniqueCount: 1,
count: 1,
description: 'A required dependency could not be resolved',
matchedDependencies: [],
},
});
Expand Down Expand Up @@ -246,7 +245,6 @@ describe('parse-vitest-report', () => {

expect(result.summary?.total).toBe(4);
expect(result.summary?.passed).toBe(3);
expect(result.summary?.failed).toBe(1);
});

it('should handle zero total tests', () => {
Expand All @@ -262,7 +260,6 @@ describe('parse-vitest-report', () => {

expect(result.summary?.total).toBe(0);
expect(result.summary?.successRate).toBe(0);
expect(result.summary?.failureRate).toBe(0);
});
});
});
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
import type { ErrorCategory } from '../../../shared/utils/categorize-render-errors';
import {
categorizeError,
getCategoryDescription,
} from '../../../shared/utils/categorize-render-errors';
import { categorizeError } from '../../../shared/utils/categorize-render-errors';
import { type ErrorCategorizationResult, type StoryTestResult, type TestRunSummary } from './types';

/**
Expand All @@ -16,7 +13,11 @@ import { type ErrorCategorizationResult, type StoryTestResult, type TestRunSumma
function extractCategorizedErrors(testResults: StoryTestResult[]): ErrorCategorizationResult {
const failed = testResults.filter((r) => r.status === 'FAIL' && r.error);

const map = new Map<ErrorCategory, { count: number; matchedDependencies: Set<string> }>();
// Map: category -> { count, uniqueErrors: Set<string>, matchedDependencies }
const map = new Map<
ErrorCategory,
{ count: number; uniqueErrors: Set<string>; matchedDependencies: Set<string> }
>();

// To count unique error messages (by their message, not by category)
const uniqueErrorMessages = new Set<string>();
Expand All @@ -25,7 +26,7 @@ function extractCategorizedErrors(testResults: StoryTestResult[]): ErrorCategori
const { category, matchedDependencies } = categorizeError(r.error!, r.stack);

if (!map.has(category)) {
map.set(category, { count: 0, matchedDependencies: new Set() });
map.set(category, { count: 0, uniqueErrors: new Set(), matchedDependencies: new Set() });
}

const data = map.get(category)!;
Expand All @@ -34,12 +35,13 @@ function extractCategorizedErrors(testResults: StoryTestResult[]): ErrorCategori

// Use the full error message for unique error message counting
uniqueErrorMessages.add(r.error!);
data.uniqueErrors.add(r.error!);
}

const categorizedErrors = Array.from(map.entries()).reduce<Record<string, any>>(
(acc, [category, data]) => {
acc[category] = {
description: getCategoryDescription(category),
uniqueCount: data.uniqueErrors.size,
count: data.count,
matchedDependencies: Array.from(data.matchedDependencies).sort(),
};
Expand All @@ -56,12 +58,12 @@ function extractCategorizedErrors(testResults: StoryTestResult[]): ErrorCategori
}

/** Transform the Vitest test results to our expected format and return a TestRunSummary */
export function parseVitestResults(testResults: any): TestRunSummary {
export function parseVitestResults(report: any): TestRunSummary {
// Transform the Vitest test results to our expected format
const storyTestResults: StoryTestResult[] = [];
let passedButEmptyRender = 0;

for (const testSuite of testResults.testResults) {
for (const testSuite of report.testResults) {
for (const assertion of testSuite.assertionResults) {
const storyId = assertion.meta?.storyId || assertion.fullName;

Expand Down Expand Up @@ -95,34 +97,25 @@ export function parseVitestResults(testResults: any): TestRunSummary {
}
}

const total = testResults.numTotalTests;
const passed = testResults.numPassedTests;
const failed = testResults.numFailedTests;
const total = report.numTotalTests;
const passed = report.numPassedTests;
const successRate = total > 0 ? parseFloat((passed / total).toFixed(2)) : 0;
const failureRate = total > 0 ? parseFloat((failed / total).toFixed(2)) : 0;
const successRateWithoutEmptyRender =
total > 0 ? parseFloat(((passed - passedButEmptyRender) / total).toFixed(2)) : 0;

// Extract and categorize unique errors
const errorClassification = extractCategorizedErrors(storyTestResults);
const categorizedErrors = errorClassification.categorizedErrors;

const summary = {
total,
passed,
passedButEmptyRender,
failed,
successRate,
successRateWithoutEmptyRender,
failureRate,
uniqueErrorCount: errorClassification.uniqueErrorCount,
categorizedErrors,
};

const enhancedResponse: TestRunSummary = {
success: testResults.success,
summary,
return {
summary: {
total,
passed,
passedButEmptyRender,
successRate,
successRateWithoutEmptyRender,
uniqueErrorCount: errorClassification.uniqueErrorCount,
categorizedErrors,
},
};

return enhancedResponse;
}
Loading