Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,24 @@ import type { GenerateContentParameters } from '@google/genai';
import { EnhancedErrorHandler } from './errorHandler.js';
import type { RequestContext } from './types.js';

const debugLoggerSpy = vi.hoisted(() => ({
error: vi.fn(),
}));

vi.mock('../../utils/debugLogger.js', () => ({
createDebugLogger: () => ({
error: debugLoggerSpy.error,
}),
}));

describe('EnhancedErrorHandler', () => {
const fixedNow = 10_000;
let errorHandler: EnhancedErrorHandler;
let mockContext: RequestContext;
let mockRequest: GenerateContentParameters;

beforeEach(() => {
debugLoggerSpy.error.mockReset();
vi.spyOn(Date, 'now').mockReturnValue(fixedNow);
mockContext = {
model: 'test-model',
Expand Down Expand Up @@ -59,6 +70,79 @@ describe('EnhancedErrorHandler', () => {
}).toThrow(originalError);
});

it('logs structured API diagnostics without request contents', () => {
const apiError = Object.assign(
new Error(
'event:error\n:HTTP_STATUS/429\ndata:{"request_id":"req-123","code":"Throttling.AllocationQuota","message":"Allocated quota exceeded"}',
),
{ type: 'rate_limit_error' },
);

expect(() => {
errorHandler.handle(apiError, mockContext, mockRequest);
}).toThrow(apiError);

expect(debugLoggerSpy.error).toHaveBeenCalledWith(
'OpenAI API Error:',
expect.any(String),
{
durationMs: 5000,
errorType: 'rate_limit_error',
model: 'test-model',
providerCode: 'Throttling.AllocationQuota',
providerMessage: 'Allocated quota exceeded',
requestId: 'req-123',
statusCode: 429,
transport: 'sse',
},
);
expect(JSON.stringify(debugLoggerSpy.error.mock.calls[0])).not.toContain(
'test prompt',
);
});

it('prefers top-level request ids before parsed provider details', () => {
const apiError = Object.assign(new Error('API failure'), {
requestID: 'req-top-level',
request_id: 'req-snake-case',
response_id: 'resp-id',
status: 500,
});

expect(() => {
errorHandler.handle(apiError, mockContext, mockRequest);
}).toThrow(apiError);

expect(debugLoggerSpy.error).toHaveBeenCalledWith(
'OpenAI API Error:',
expect.any(String),
expect.objectContaining({
requestId: 'req-top-level',
statusCode: 500,
}),
);
});

it('skips empty request ids and falls back to later request id fields', () => {
const apiError = Object.assign(new Error('API failure'), {
requestID: '',
request_id: '',
response_id: 'resp-id',
});

expect(() => {
errorHandler.handle(apiError, mockContext, mockRequest);
}).toThrow(apiError);

expect(debugLoggerSpy.error).toHaveBeenCalledWith(
'OpenAI API Error:',
expect.any(String),
expect.objectContaining({
requestId: 'resp-id',
}),
);
});

it('should throw enhanced error message for timeout errors', () => {
const timeoutError = new Error('Request timeout');

Expand Down
63 changes: 62 additions & 1 deletion packages/core/src/core/openaiContentGenerator/errorHandler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,25 @@

import type { GenerateContentParameters } from '@google/genai';
import { createDebugLogger } from '../../utils/debugLogger.js';
import { getErrorStatus, getErrorType } from '../../utils/errors.js';
import { getRateLimitErrorDetails } from '../../utils/rateLimit.js';
import { redactProxyError } from '../../utils/runtimeFetchOptions.js';
import type { ErrorHandler, RequestContext } from './types.js';

const debugLogger = createDebugLogger('OPENAI_ERROR');
export type { ErrorHandler } from './types.js';

interface ApiErrorDiagnostics {
model: string;
durationMs: number;
errorType: string;
statusCode?: number;
providerCode?: string;
providerMessage?: string;
requestId?: string;
transport?: 'http' | 'sse' | 'unknown';
}

export class EnhancedErrorHandler implements ErrorHandler {
constructor(
private shouldSuppressLogging: (
Expand All @@ -35,7 +48,11 @@ export class EnhancedErrorHandler implements ErrorHandler {

// Allow subclasses to suppress error logging for specific scenarios
if (!this.shouldSuppressErrorLogging(redactedError, request)) {
debugLogger.error('OpenAI API Error:', errorMessage);
debugLogger.error(
'OpenAI API Error:',
errorMessage,
this.buildDiagnostics(redactedError, context),
);
}

// Provide helpful timeout-specific error message
Expand Down Expand Up @@ -98,6 +115,50 @@ export class EnhancedErrorHandler implements ErrorHandler {
return error instanceof Error ? error.message : String(error);
}

private buildDiagnostics(
error: unknown,
context: RequestContext,
): ApiErrorDiagnostics {
const details = getRateLimitErrorDetails(error);
Comment thread
yiliang114 marked this conversation as resolved.
Resolved
const requestId = this.getRequestId(error) ?? details.requestId;
const statusCode = getErrorStatus(error);
return {
model: context.model,
durationMs: Date.now() - context.startTime,
errorType: getErrorType(error),
...(statusCode !== undefined ? { statusCode } : {}),
...(details.providerCode !== undefined
? { providerCode: details.providerCode }
: {}),
...(details.providerMessage !== undefined
? { providerMessage: details.providerMessage }
: {}),
...(requestId !== undefined ? { requestId } : {}),
...(details.transport !== 'unknown'
? { transport: details.transport }
: {}),
};
}

private getRequestId(error: unknown): string | undefined {
if (!error || typeof error !== 'object') return undefined;
const source = error as {
requestID?: unknown;
request_id?: unknown;
response_id?: unknown;
};
for (const value of [
source.requestID,
source.request_id,
source.response_id,
]) {
if (typeof value === 'string' && value) {
return value;
}
}
return undefined;
}

private getTimeoutTroubleshootingTips(): string {
const tips = [
'- Reduce input length or complexity',
Expand Down
5 changes: 5 additions & 0 deletions packages/core/src/core/turn.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -422,6 +422,7 @@ describe('Turn', () => {
},
},
'Turn.run-sendMessageStream',
{ contextAlreadySummarized: true },
);
});

Expand Down Expand Up @@ -461,6 +462,7 @@ describe('Turn', () => {
},
},
'Turn.run-sendMessageStream',
{ contextAlreadySummarized: true },
);
});

Expand Down Expand Up @@ -540,6 +542,7 @@ describe('Turn', () => {
},
},
'Turn.run-sendMessageStream',
{ contextAlreadySummarized: true },
);
});

Expand Down Expand Up @@ -579,6 +582,7 @@ describe('Turn', () => {
},
},
'Turn.run-sendMessageStream',
{ contextAlreadySummarized: true },
);
});

Expand Down Expand Up @@ -620,6 +624,7 @@ describe('Turn', () => {
},
},
'Turn.run-sendMessageStream',
{ contextAlreadySummarized: true },
);
});

Expand Down
1 change: 1 addition & 0 deletions packages/core/src/core/turn.ts
Original file line number Diff line number Diff line change
Expand Up @@ -578,6 +578,7 @@ export class Turn {
'Error when talking to API',
contextForReport,
'Turn.run-sendMessageStream',
{ contextAlreadySummarized: true },
);
const status =
typeof error === 'object' &&
Expand Down
10 changes: 6 additions & 4 deletions packages/core/src/telemetry/sdk.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,9 @@ describe('Telemetry SDK', () => {
const previousDebugLogFileEnv = process.env['QWEN_DEBUG_LOG_FILE'];
try {
process.env['QWEN_DEBUG_LOG_FILE'] = '1';
setDebugLogSession({ getSessionId: () => 'otel-diag-test-session' });
setDebugLogSession({
getSessionId: () => '11111111-2222-4333-8444-555555555555',
});

diag.error(
JSON.stringify({
Expand All @@ -215,20 +217,20 @@ describe('Telemetry SDK', () => {
expect(consoleWarnSpy).not.toHaveBeenCalled();
expect(mkdirSpy).toHaveBeenCalled();
expect(appendFileSpy).toHaveBeenCalledWith(
expect.stringContaining('otel-diag-test-session'),
expect.stringContaining('11111111-2222-4333-8444-555555555555'),
expectOtelDebugLogLine(
'ERROR',
'{"message":"Error: PeriodicExportingMetricReader: metrics export failed (error Error: connect ECONNREFUSED)"}',
),
'utf8',
);
expect(appendFileSpy).toHaveBeenCalledWith(
expect.stringContaining('otel-diag-test-session'),
expect.stringContaining('11111111-2222-4333-8444-555555555555'),
expectOtelDebugLogLine('ERROR', 'A different OpenTelemetry diagnostic'),
'utf8',
);
expect(appendFileSpy).toHaveBeenCalledWith(
expect.stringContaining('otel-diag-test-session'),
expect.stringContaining('11111111-2222-4333-8444-555555555555'),
expectOtelDebugLogLine('WARN', 'An OpenTelemetry warning'),
'utf8',
);
Expand Down
32 changes: 27 additions & 5 deletions packages/core/src/utils/debugLogger.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,17 @@ describe('debugLogger', () => {

const previousDebugLogFileEnv = process.env['QWEN_DEBUG_LOG_FILE'];

beforeEach(() => {
beforeEach(async () => {
process.env['QWEN_DEBUG_LOG_FILE'] = '1';
Storage.setRuntimeBaseDir(null);
vi.clearAllMocks();
vi.useFakeTimers();
vi.setSystemTime(new Date('2026-01-24T10:30:00.000Z'));
resetDebugLoggingState();
setDebugLogSession(mockSession);
await vi.runAllTimersAsync();
resetDebugLoggingState();
vi.clearAllMocks();
vi.mocked(getTraceContext).mockReturnValue(null);
});

Expand Down Expand Up @@ -300,20 +303,39 @@ describe('debugLogger', () => {

describe('latest debug log symlink', () => {
const expectedLatestPath = path.join(Storage.getGlobalDebugDir(), 'latest');
const uuidSession: DebugLogSession = {
getSessionId: () => '92ec0176-d354-4147-848b-5cd2d80609c4',
};

it('creates a symlink to the current session log file', async () => {
resetDebugLoggingState();
setDebugLogSession(mockSession);
setDebugLogSession(uuidSession);

await vi.runAllTimersAsync();

expect(fs.unlink).toHaveBeenCalledWith(expectedLatestPath);
expect(fs.symlink).toHaveBeenCalledWith(
'test-session-123.txt',
'92ec0176-d354-4147-848b-5cd2d80609c4.txt',
expectedLatestPath,
);
});

it('does not point latest at non-session debug logs', async () => {
resetDebugLoggingState();
setDebugLogSession({ getSessionId: () => 'log-to-span-sink-test' });

await vi.runAllTimersAsync();

expect(fs.symlink).not.toHaveBeenCalled();
expect(fs.appendFile).toHaveBeenCalledWith(
Storage.getDebugLogPath('log-to-span-sink-test'),
expect.stringContaining(
'[DEBUG] [DEBUG_LOG] Skipping latest debug log alias for non-UUID session id: log-to-span-sink-test',
),
'utf8',
);
});

it('does not create symlink when session is cleared', async () => {
vi.clearAllMocks();
resetDebugLoggingState();
Expand All @@ -328,7 +350,7 @@ describe('debugLogger', () => {
resetDebugLoggingState();
vi.mocked(fs.symlink).mockRejectedValueOnce(new Error('EPERM'));

setDebugLogSession(mockSession);
setDebugLogSession(uuidSession);

await vi.runAllTimersAsync();

Expand All @@ -339,7 +361,7 @@ describe('debugLogger', () => {
process.env['QWEN_DEBUG_LOG_FILE'] = '0';
vi.clearAllMocks();
resetDebugLoggingState();
setDebugLogSession(mockSession);
setDebugLogSession(uuidSession);

await vi.runAllTimersAsync();

Expand Down
9 changes: 9 additions & 0 deletions packages/core/src/utils/debugLogger.ts
Original file line number Diff line number Diff line change
Expand Up @@ -144,11 +144,20 @@ export function resetDebugLoggingState(): void {
}

const DEBUG_LATEST_ALIAS = 'latest';
const SESSION_ID_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;

function updateLatestDebugLogAlias(sessionId: string): void {
if (!isDebugLogFileEnabled()) {
return;
}
if (!SESSION_ID_PATTERN.test(sessionId)) {
createDebugLogger('DEBUG_LOG').debug(
Comment thread
yiliang114 marked this conversation as resolved.
Resolved
'Skipping latest debug log alias for non-UUID session id:',
sessionId,
);
return;
Comment thread
yiliang114 marked this conversation as resolved.
Resolved
}

const aliasPath = path.join(Storage.getGlobalDebugDir(), DEBUG_LATEST_ALIAS);
const targetPath = Storage.getDebugLogPath(sessionId);
Expand Down
Loading
Loading