Repository navigation
feat(cli): add API preconnect to reduce first-call latency #3318
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from 9 commits
Commits
Show all changes
15 commits
Select commit
Hold shift + click to select a range
a52a7b3
feat(cli): add API preconnect to reduce first-call latency (#3223)
doudouOUC a8f1e4e
fix(cli): address bot review comments for API preconnect
doudouOUC 6832534
fix(cli): use resolved base URL and shared dispatcher for API preconnect
doudouOUC 6fd7451
fix(cli): honor configured proxy in API preconnect
doudouOUC 7317a4b
fix(cli): remove env proxy skip in preconnect, add runtime guard
doudouOUC 3049436
refactor(cli): remove redundant runtime check in preconnect dispatcher
doudouOUC 99d742d
fix(cli): remove ineffective gemini/vertex preconnect and extend keep…
doudouOUC 22c44c5
merge: resolve conflict with main branch in gemini.tsx
doudouOUC 0824a7e
fix(cli): avoid importing real core module in apiPreconnect test mock
doudouOUC d032916
fix(cli): defer preconnectFired flag so missing targetUrl allows retry
doudouOUC 70a2f5b
fix(cli): address review comments for API preconnect
doudouOUC 950960f
fix(cli): add dashscope to preconnect whitelist and replace benchmark…
doudouOUC ac82e37
fix(lint): add .mjs and browser globals to scripts ESLint config
doudouOUC b208963
fix(cli): extend preconnect allowlist to cover all DashScope regional…
doudouOUC bdbb51e
fix(cli): treat non-URL sentinel as unresolved in preconnect
doudouOUC File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Some comments aren't visible on the classic Files Changed page.
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,161 @@ | ||
| /** | ||
| * @license | ||
| * Copyright 2026 Qwen Team | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; | ||
| import { preconnectApi, resetPreconnectState } from './apiPreconnect.js'; | ||
|
|
||
| // Mock fetch | ||
| const mockFetch = vi.fn().mockResolvedValue(undefined); | ||
| global.fetch = mockFetch; | ||
|
|
||
| // Mock the shared dispatcher functions from core | ||
| const { mockGetOrCreateSharedDispatcher, mockDebugLogger } = vi.hoisted(() => { | ||
| const dispatcher = { fake: 'dispatcher' }; | ||
| const mockDebugLogger = { debug: vi.fn() }; | ||
| return { | ||
| mockGetOrCreateSharedDispatcher: vi.fn(() => dispatcher), | ||
| mockDebugLogger, | ||
| }; | ||
| }); | ||
| vi.mock('@qwen-code/qwen-code-core', () => ({ | ||
| createDebugLogger: () => mockDebugLogger, | ||
| detectRuntime: () => 'node', | ||
| getOrCreateSharedDispatcher: mockGetOrCreateSharedDispatcher, | ||
| })); | ||
|
|
||
| describe('apiPreconnect', () => { | ||
| beforeEach(() => { | ||
| resetPreconnectState(); | ||
| mockFetch.mockClear(); | ||
| mockFetch.mockResolvedValue(undefined); | ||
| mockGetOrCreateSharedDispatcher.mockClear(); | ||
| delete process.env['HTTPS_PROXY']; | ||
| delete process.env['https_proxy']; | ||
| delete process.env['HTTP_PROXY']; | ||
| delete process.env['http_proxy']; | ||
| delete process.env['QWEN_CODE_DISABLE_PRECONNECT']; | ||
| delete process.env['NODE_EXTRA_CA_CERTS']; | ||
| delete process.env['SANDBOX']; | ||
| }); | ||
|
|
||
| afterEach(() => { | ||
| vi.restoreAllMocks(); | ||
| }); | ||
|
|
||
| describe('shouldSkipPreconnect', () => { | ||
| it('should skip when NODE_EXTRA_CA_CERTS is set', () => { | ||
| process.env['NODE_EXTRA_CA_CERTS'] = '/path/to/ca.pem'; | ||
| preconnectApi('qwen-oauth'); | ||
| expect(mockFetch).not.toHaveBeenCalled(); | ||
| }); | ||
| }); | ||
|
|
||
| describe('resolvedBaseUrl handling', () => { | ||
| it('should use resolvedBaseUrl when it is a default URL', () => { | ||
| preconnectApi('openai', { | ||
| resolvedBaseUrl: 'https://api.openai.com/v1', | ||
| }); | ||
| expect(mockFetch).toHaveBeenCalledWith( | ||
| 'https://api.openai.com/v1', | ||
| expect.objectContaining({ method: 'HEAD' }), | ||
| ); | ||
| }); | ||
|
|
||
| it('should skip when resolvedBaseUrl is a custom (non-default) URL', () => { | ||
| preconnectApi('openai', { | ||
| resolvedBaseUrl: 'https://custom.api.com/v1', | ||
| }); | ||
| expect(mockFetch).not.toHaveBeenCalled(); | ||
| }); | ||
|
|
||
| it('should skip when resolvedBaseUrl is a subdomain-spoofed URL', () => { | ||
| preconnectApi('openai', { | ||
| resolvedBaseUrl: 'https://api.openai.com.malicious.com/v1', | ||
| }); | ||
| expect(mockFetch).not.toHaveBeenCalled(); | ||
| }); | ||
|
|
||
| it('should fall back to default URL when resolvedBaseUrl is undefined', () => { | ||
| preconnectApi('qwen-oauth'); | ||
| expect(mockFetch).toHaveBeenCalledWith( | ||
| 'https://coding.dashscope.aliyuncs.com', | ||
| expect.objectContaining({ method: 'HEAD' }), | ||
| ); | ||
| }); | ||
| }); | ||
|
|
||
| describe('preconnect behavior', () => { | ||
| it('should use default baseUrl for qwen-oauth', () => { | ||
| preconnectApi('qwen-oauth'); | ||
| expect(mockFetch).toHaveBeenCalledWith( | ||
| 'https://coding.dashscope.aliyuncs.com', | ||
| expect.objectContaining({ method: 'HEAD' }), | ||
| ); | ||
| }); | ||
|
|
||
| it('should use default baseUrl for openai', () => { | ||
| preconnectApi('openai'); | ||
| expect(mockFetch).toHaveBeenCalledWith( | ||
| 'https://api.openai.com', | ||
| expect.objectContaining({ method: 'HEAD' }), | ||
| ); | ||
| }); | ||
|
|
||
| it('should use default baseUrl for anthropic', () => { | ||
| preconnectApi('anthropic'); | ||
| expect(mockFetch).toHaveBeenCalledWith( | ||
| 'https://api.anthropic.com', | ||
| expect.objectContaining({ method: 'HEAD' }), | ||
| ); | ||
| }); | ||
|
|
||
| it('should pass shared dispatcher on Node.js runtime', () => { | ||
| preconnectApi('qwen-oauth'); | ||
| expect(mockFetch).toHaveBeenCalledWith( | ||
| expect.any(String), | ||
| expect.objectContaining({ | ||
| dispatcher: { fake: 'dispatcher' }, | ||
| }), | ||
| ); | ||
| }); | ||
|
|
||
| it('should pass undefined proxy to shared dispatcher by default', () => { | ||
| preconnectApi('qwen-oauth'); | ||
| expect(mockGetOrCreateSharedDispatcher).toHaveBeenCalledWith(undefined); | ||
| }); | ||
|
|
||
| it('should pass configured proxy to shared dispatcher', () => { | ||
| preconnectApi('qwen-oauth', { proxy: 'http://proxy.example.com:8080' }); | ||
| expect(mockGetOrCreateSharedDispatcher).toHaveBeenCalledWith( | ||
| 'http://proxy.example.com:8080', | ||
| ); | ||
| }); | ||
|
|
||
| it('should not fire twice', () => { | ||
| preconnectApi('qwen-oauth'); | ||
| preconnectApi('openai'); | ||
| expect(mockFetch).toHaveBeenCalledTimes(1); | ||
| }); | ||
|
|
||
| it('should handle fetch errors gracefully', async () => { | ||
|
doudouOUC marked this conversation as resolved.
Resolved
|
||
| mockFetch.mockRejectedValue(new Error('Network error')); | ||
| // Should not throw | ||
| expect(() => preconnectApi('qwen-oauth')).not.toThrow(); | ||
| }); | ||
|
|
||
| it('should skip when QWEN_CODE_DISABLE_PRECONNECT is set', () => { | ||
| process.env['QWEN_CODE_DISABLE_PRECONNECT'] = '1'; | ||
| preconnectApi('qwen-oauth'); | ||
| expect(mockFetch).not.toHaveBeenCalled(); | ||
| }); | ||
|
|
||
| it('should skip in sandbox mode', () => { | ||
| process.env['SANDBOX'] = '1'; | ||
| preconnectApi('qwen-oauth'); | ||
| expect(mockFetch).not.toHaveBeenCalled(); | ||
| }); | ||
| }); | ||
| }); | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,191 @@ | ||
| /** | ||
| * @license | ||
| * Copyright 2026 Qwen Team | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| /** | ||
| * API Preconnect - Warm API connections to reduce TCP+TLS handshake latency | ||
| * | ||
| * Principle: Fire a fire-and-forget HEAD request early in startup to warm | ||
| * the TCP+TLS connection. Subsequent actual API calls reuse this connection, | ||
| * saving 100-200ms. | ||
| * | ||
| * The preconnect uses the same shared undici dispatcher as the SDK clients, | ||
| * ensuring the warmed TCP+TLS connection is reused by subsequent API calls. | ||
| */ | ||
|
|
||
| import { | ||
| createDebugLogger, | ||
| detectRuntime, | ||
| getOrCreateSharedDispatcher, | ||
| } from '@qwen-code/qwen-code-core'; | ||
|
|
||
| const debugLogger = createDebugLogger('PRECONNECT'); | ||
|
|
||
| let preconnectFired = false; | ||
|
|
||
| /** | ||
| * Default API base URLs by AuthType | ||
| */ | ||
| const DEFAULT_BASE_URLS: Record<string, string> = { | ||
| openai: 'https://api.openai.com', | ||
|
doudouOUC marked this conversation as resolved.
Resolved
|
||
| 'qwen-oauth': 'https://coding.dashscope.aliyuncs.com', | ||
| anthropic: 'https://api.anthropic.com', | ||
| }; | ||
|
|
||
| /** | ||
| * Check if preconnect should be skipped due to environment conditions | ||
| */ | ||
| function shouldSkipPreconnect(): boolean { | ||
| // Skip for custom CA certificate (enterprise TLS inspection may interfere) | ||
| if (process.env['NODE_EXTRA_CA_CERTS']) { | ||
| debugLogger.debug('Skipping preconnect: custom CA certificate configured'); | ||
| return true; | ||
| } | ||
|
|
||
| return false; | ||
| } | ||
|
|
||
| /** | ||
| * Check if running in sandbox mode | ||
| * In sandbox mode, preconnect is ineffective because the process will restart | ||
| */ | ||
| function isInSandboxMode(): boolean { | ||
| return process.env['SANDBOX'] !== undefined; | ||
| } | ||
|
|
||
| /** | ||
| * Check if baseUrl is a default URL | ||
| */ | ||
| function isDefaultBaseUrl(baseUrl: string): boolean { | ||
| const normalizedInput = baseUrl | ||
| .toLowerCase() | ||
| .replace(/^https?:\/\//, '') | ||
| .replace(/\/+$/, ''); | ||
| return Object.values(DEFAULT_BASE_URLS).some((defaultUrl) => { | ||
| const normalizedDefault = defaultUrl | ||
| .toLowerCase() | ||
| .replace(/^https?:\/\//, '') | ||
| .replace(/\/+$/, ''); | ||
| return ( | ||
| normalizedInput === normalizedDefault || | ||
| normalizedInput.startsWith(normalizedDefault + '/') | ||
| ); | ||
| }); | ||
| } | ||
|
|
||
| /** | ||
| * Get the target URL for preconnect. | ||
| * Uses the already-resolved base URL from the model config, falling back | ||
| * to default URLs by authType. | ||
| * | ||
| * Only preconnects to known default URLs — custom URLs may not accept HEAD | ||
| * requests or may require mTLS / private deployment configurations. | ||
| */ | ||
| function getPreconnectTargetUrl( | ||
| authType: string | undefined, | ||
| resolvedBaseUrl: string | undefined, | ||
| ): string | undefined { | ||
|
doudouOUC marked this conversation as resolved.
Resolved
|
||
| // 1. Use the resolved base URL from model config (already incorporates | ||
| // modelProviders > cli > env > settings priority chain) | ||
| if (resolvedBaseUrl) { | ||
| if (isDefaultBaseUrl(resolvedBaseUrl)) { | ||
| return resolvedBaseUrl; | ||
| } | ||
| debugLogger.debug( | ||
| 'Skipping preconnect: resolved baseUrl is not a default URL', | ||
| ); | ||
| return undefined; | ||
| } | ||
|
|
||
| // 2. Fall back to default value by authType | ||
| if (authType && DEFAULT_BASE_URLS[authType]) { | ||
| return DEFAULT_BASE_URLS[authType]; | ||
| } | ||
|
|
||
| return undefined; | ||
| } | ||
|
|
||
| /** | ||
| * Execute API preconnect | ||
| * Use HEAD request to establish TCP+TLS connection without sending actual request body. | ||
| * Uses the shared undici dispatcher to ensure connection pool is shared with SDK clients. | ||
| * | ||
| * @param authType - Authentication type (openai, qwen-oauth, anthropic, etc.) | ||
| * @param options - Configuration options | ||
| */ | ||
| export function preconnectApi( | ||
| authType: string | undefined, | ||
| options: { | ||
| resolvedBaseUrl?: string; | ||
| proxy?: string; | ||
| } = {}, | ||
| ): void { | ||
| if (preconnectFired) { | ||
| return; | ||
| } | ||
| preconnectFired = true; | ||
|
|
||
| // Check if disabled | ||
| if (process.env['QWEN_CODE_DISABLE_PRECONNECT'] === '1') { | ||
| debugLogger.debug('Preconnect disabled by environment variable'); | ||
| return; | ||
| } | ||
|
|
||
| // Check if in sandbox mode (process will restart, preconnect is ineffective) | ||
| if (isInSandboxMode()) { | ||
| debugLogger.debug('Skipping preconnect: sandbox mode detected'); | ||
| return; | ||
| } | ||
|
|
||
| // Check environment skip conditions (custom CA) | ||
| if (shouldSkipPreconnect()) { | ||
| return; | ||
| } | ||
|
|
||
| // Skip on non-Node runtimes (e.g. Bun) — they use independent connection | ||
| // pools, so warming undici's pool provides no benefit. | ||
| if (detectRuntime() !== 'node') { | ||
| debugLogger.debug('Skipping preconnect: unsupported runtime'); | ||
| return; | ||
| } | ||
|
|
||
| const targetUrl = getPreconnectTargetUrl(authType, options.resolvedBaseUrl); | ||
|
doudouOUC marked this conversation as resolved.
Resolved
|
||
|
|
||
| if (!targetUrl) { | ||
| debugLogger.debug('No target URL for preconnect'); | ||
| return; | ||
| } | ||
|
|
||
| debugLogger.debug(`Preconnecting to: ${targetUrl}`); | ||
|
|
||
| // Use the same shared undici dispatcher that SDK clients will use, | ||
| // so the warmed TCP+TLS connection is reused by subsequent API calls. | ||
| const dispatcher = getOrCreateSharedDispatcher(options.proxy); | ||
|
|
||
| // Fire HEAD request to warm connection (fire-and-forget) | ||
| fetch(targetUrl, { | ||
| method: 'HEAD', | ||
| signal: AbortSignal.timeout(5_000), | ||
| headers: { | ||
| 'User-Agent': 'QwenCode-Preconnect/1.0', | ||
| }, | ||
| dispatcher, | ||
| } as RequestInit) | ||
| .then(() => { | ||
| debugLogger.debug('Preconnect completed'); | ||
| }) | ||
| .catch((error) => { | ||
| // Preconnect failure doesn't affect main flow | ||
| debugLogger.debug(`Preconnect failed (ignored): ${error}`); | ||
| }); | ||
| } | ||
|
|
||
| /** | ||
| * Reset preconnect state (for testing only) | ||
| * @internal | ||
| */ | ||
| export function resetPreconnectState(): void { | ||
| preconnectFired = false; | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.