diff --git a/.env.example b/.env.example index c754d19b80..689bc3e7de 100644 --- a/.env.example +++ b/.env.example @@ -46,10 +46,15 @@ TIMEOUT_MS=5000 # Defaults to 300000ms (5 minutes) # GATEWAY_TIMEOUT_MS=300000 -# AI API request timeout in milliseconds - maximum time for upstream provider calls +# AI API request timeout in milliseconds for streaming requests # Should be shorter than GATEWAY_TIMEOUT_MS to allow for error handling # Defaults to 240000ms (4 minutes) or 80% of GATEWAY_TIMEOUT_MS, whichever is smaller -# AI_REQUEST_TIMEOUT_MS=240000 +# AI_STREAMING_TIMEOUT_MS=240000 + +# AI API request timeout in milliseconds for plain (non-streaming) requests +# Non-streaming requests use a shorter timeout since they don't benefit from incremental responses +# Defaults to 80000ms (80 seconds) +# AI_TIMEOUT_MS=80000 # ============================================================================= # WORKER CONFIGURATION diff --git a/apps/gateway/src/api.spec.ts b/apps/gateway/src/api.spec.ts index 087a273164..e9e1bc0b3b 100644 --- a/apps/gateway/src/api.spec.ts +++ b/apps/gateway/src/api.spec.ts @@ -793,20 +793,28 @@ describe("test", () => { // Timeout tests - use a short timeout via env var to test timeout handling describe("Timeout handling", () => { let originalTimeout: string | undefined; + let originalStreamingTimeout: string | undefined; beforeAll(() => { - // Save original env value - originalTimeout = process.env.AI_REQUEST_TIMEOUT_MS; + // Save original env values + originalTimeout = process.env.AI_TIMEOUT_MS; + originalStreamingTimeout = process.env.AI_STREAMING_TIMEOUT_MS; // Set a short timeout for testing (2 seconds) - process.env.AI_REQUEST_TIMEOUT_MS = "2000"; + process.env.AI_TIMEOUT_MS = "2000"; + process.env.AI_STREAMING_TIMEOUT_MS = "2000"; }); afterAll(() => { - // Restore original env value + // Restore original env values if (originalTimeout !== undefined) { - process.env.AI_REQUEST_TIMEOUT_MS = originalTimeout; + process.env.AI_TIMEOUT_MS = originalTimeout; } else { - delete process.env.AI_REQUEST_TIMEOUT_MS; + delete process.env.AI_TIMEOUT_MS; + } + if (originalStreamingTimeout !== undefined) { + process.env.AI_STREAMING_TIMEOUT_MS = originalStreamingTimeout; + } else { + delete process.env.AI_STREAMING_TIMEOUT_MS; } }); diff --git a/apps/gateway/src/chat/chat.ts b/apps/gateway/src/chat/chat.ts index 83b088992b..03324b6dab 100644 --- a/apps/gateway/src/chat/chat.ts +++ b/apps/gateway/src/chat/chat.ts @@ -18,7 +18,11 @@ import { isCodingModel } from "@/lib/coding-models.js"; import { calculateCosts, shouldBillCancelledRequests } from "@/lib/costs.js"; import { throwIamException, validateModelAccess } from "@/lib/iam.js"; import { calculateDataStorageCost, insertLog } from "@/lib/logs.js"; -import { createCombinedSignal, isTimeoutError } from "@/lib/timeout-config.js"; +import { + createCombinedSignal, + createStreamingCombinedSignal, + isTimeoutError, +} from "@/lib/timeout-config.js"; import { getCheapestFromAvailableProviders, @@ -2141,7 +2145,7 @@ chat.openapi(completions, async (c) => { } // Create a combined signal for both timeout and cancellation - const fetchSignal = createCombinedSignal( + const fetchSignal = createStreamingCombinedSignal( requestCanBeCanceled ? controller : undefined, ); @@ -4113,6 +4117,7 @@ chat.openapi(completions, async (c) => { } // Create a combined signal for both timeout and cancellation + // Non-streaming requests use a shorter timeout (default 80s) const fetchSignal = createCombinedSignal( requestCanBeCanceled ? controller : undefined, ); diff --git a/apps/gateway/src/lib/timeout-config.ts b/apps/gateway/src/lib/timeout-config.ts index b4b56e3f98..add7b577e1 100644 --- a/apps/gateway/src/lib/timeout-config.ts +++ b/apps/gateway/src/lib/timeout-config.ts @@ -16,12 +16,12 @@ export function getGatewayTimeoutMs(): number { } /** - * Gets the AI API request timeout - the maximum time for upstream provider calls. + * Gets the AI API request timeout for streaming requests - the maximum time for upstream provider calls. * Should be shorter than gateway timeout to allow for error handling. * Default: 4 minutes (240000ms) or 80% of gateway timeout, whichever is smaller */ -export function getAIRequestTimeoutMs(): number { - const envValue = Number(process.env.AI_REQUEST_TIMEOUT_MS); +export function getStreamingTimeoutMs(): number { + const envValue = Number(process.env.AI_STREAMING_TIMEOUT_MS); if (envValue > 0) { return envValue; } @@ -29,24 +29,64 @@ export function getAIRequestTimeoutMs(): number { return Math.min(240000, getGatewayTimeoutMs() * 0.8); } +/** + * Gets the AI API request timeout for non-streaming (plain) requests. + * Non-streaming requests have a shorter default timeout since they don't benefit + * from incremental responses and long waits are usually indicative of issues. + * Default: 80 seconds (80000ms) + */ +export function getTimeoutMs(): number { + const envValue = Number(process.env.AI_TIMEOUT_MS); + if (envValue > 0) { + return envValue; + } + // Default: 80 seconds for non-streaming requests + return 80000; +} + // Legacy exports for backwards compatibility (read at module load time) // These should be avoided in new code - use the getter functions instead export const GATEWAY_TIMEOUT_MS = getGatewayTimeoutMs(); -export const AI_REQUEST_TIMEOUT_MS = getAIRequestTimeoutMs(); +export const AI_STREAMING_TIMEOUT_MS = getStreamingTimeoutMs(); +export const AI_TIMEOUT_MS = getTimeoutMs(); + +/** + * Creates an AbortSignal that will abort after the streaming request timeout. + * Can be combined with other signals (e.g., client cancellation) using AbortSignal.any(). + */ +export function createStreamingTimeoutSignal(): AbortSignal { + return AbortSignal.timeout(getStreamingTimeoutMs()); +} /** - * Creates an AbortSignal that will abort after the AI request timeout. + * Creates an AbortSignal that will abort after the plain (non-streaming) request timeout. * Can be combined with other signals (e.g., client cancellation) using AbortSignal.any(). */ export function createTimeoutSignal(): AbortSignal { - return AbortSignal.timeout(getAIRequestTimeoutMs()); + return AbortSignal.timeout(getTimeoutMs()); } /** - * Combines a timeout signal with an optional cancellation signal. + * Combines a streaming timeout signal with an optional cancellation signal. * If the cancellation signal is provided, the request will abort on either timeout or cancellation. * If no cancellation signal is provided, only the timeout will cause an abort. */ +export function createStreamingCombinedSignal( + cancellationController?: AbortController, +): AbortSignal { + const timeoutSignal = createStreamingTimeoutSignal(); + + if (cancellationController) { + return AbortSignal.any([timeoutSignal, cancellationController.signal]); + } + + return timeoutSignal; +} + +/** + * Combines a plain (non-streaming) timeout signal with an optional cancellation signal. + * Uses the shorter timeout (default 80s) for non-streaming requests. + */ export function createCombinedSignal( cancellationController?: AbortController, ): AbortSignal {