diff --git a/docs/users/configuration/model-providers.md b/docs/users/configuration/model-providers.md index 4ee6dc1b224..5570494123b 100644 --- a/docs/users/configuration/model-providers.md +++ b/docs/users/configuration/model-providers.md @@ -317,6 +317,7 @@ Most local inference servers (vLLM, Ollama, LM Studio, etc.) provide an OpenAI-c "baseUrl": "http://localhost:11434/v1", "generationConfig": { "timeout": 300000, + "streamIdleTimeoutMs": 600000, "maxRetries": 1, "contextWindowSize": 32768, "samplingParams": { @@ -358,6 +359,13 @@ Most local inference servers (vLLM, Ollama, LM Studio, etc.) provide an OpenAI-c } ``` +For queued or slow local OpenAI-compatible servers, `streamIdleTimeoutMs` +controls how long this model may stay silent between streamed chunks. It +overrides the global `QWEN_STREAM_IDLE_TIMEOUT_MS` value for the selected +provider entry; set it to `0` to disable the idle guard. The separate 15-minute +stream lifetime cap still applies unless `QWEN_STREAM_MAX_LIFETIME_MS` is raised +or disabled. + For local servers that don't require authentication, you can use any placeholder value for the API key: ```bash diff --git a/docs/users/configuration/settings.md b/docs/users/configuration/settings.md index 3d03cacb290..e4e1c979f93 100644 --- a/docs/users/configuration/settings.md +++ b/docs/users/configuration/settings.md @@ -202,6 +202,7 @@ These settings are read from operator scopes only (User, System, and SystemDefau "model": { "generationConfig": { "timeout": 60000, + "streamIdleTimeoutMs": 300000, "contextWindowSize": 128000, "modalities": { "image": true @@ -232,10 +233,10 @@ These settings are read from operator scopes only (User, System, and SystemDefau Two guards bound a streaming response, each accepting `0` to disable. Neither is implemented by the Anthropic/Gemini generators, which leave the drip-fed shape below unbounded. -- `QWEN_STREAM_IDLE_TIMEOUT_MS` (default `240000`) bounds inactivity _between_ streamed chunks: a stream that goes silent for this long is aborted as a retryable `ETIMEDOUT`. +- `streamIdleTimeoutMs` (default `240000`) bounds inactivity _between_ streamed chunks: a stream that goes silent for this long is aborted as a retryable `ETIMEDOUT`. For provider-backed models, set it under the matching `modelProviders[providerId][].generationConfig`; for runtime models, use `model.generationConfig`. An explicit model value takes precedence over `QWEN_STREAM_IDLE_TIMEOUT_MS`, and `0` disables the idle guard. - `QWEN_STREAM_MAX_LIFETIME_MS` (default `900000`) caps the _total_ upstream-wait time of one streaming response regardless of chunk flow — the bound a drip-fed stream that never completes cannot reset. -These are **environment variables (or, for embedders, `ContentGeneratorConfig.streamIdleTimeoutMs` / `streamMaxLifetimeMs`) only — there is no settings.json key**; writing `"streamMaxLifetimeMs"` into settings.json has no effect. Upgrade notes: a deployment that previously set `QWEN_STREAM_IDLE_TIMEOUT_MS=0` — or passed `streamIdleTimeoutMs: 0` in `ContentGeneratorConfig` — to opt out of stream aborts now also needs `QWEN_STREAM_MAX_LIFETIME_MS=0` (or `streamMaxLifetimeMs: 0`) to keep that; and the 15-minute lifetime cap bounds even a stream whose idle timeout you raised above it (e.g. `QWEN_STREAM_IDLE_TIMEOUT_MS=1800000`) — raise the cap likewise, or set it to `0`, if you rely on a longer window. +`streamMaxLifetimeMs` remains available only through `QWEN_STREAM_MAX_LIFETIME_MS` or, for embedders, `ContentGeneratorConfig.streamMaxLifetimeMs`; writing it into `settings.json` has no effect. The 15-minute lifetime cap still bounds a stream whose idle timeout you raise above it. Raise the lifetime environment variable likewise, or set it to `0`, if you rely on a longer window. Disabling `streamIdleTimeoutMs` alone does not disable this lifetime cap. **max_tokens (output token limit):** diff --git a/packages/cli/src/config/settingsSchema.test.ts b/packages/cli/src/config/settingsSchema.test.ts index 788a3513e0e..e719410d125 100644 --- a/packages/cli/src/config/settingsSchema.test.ts +++ b/packages/cli/src/config/settingsSchema.test.ts @@ -278,6 +278,21 @@ describe('SettingsSchema', () => { expect(model.maxToolCallsPerTurn.type).toBe('integer'); }); + it('should define streamIdleTimeoutMs as a bounded generation setting', () => { + const streamIdleTimeout = + getSettingsSchema().model.properties.generationConfig.properties + ?.streamIdleTimeoutMs; + + expect(streamIdleTimeout).toMatchObject({ + type: 'integer', + default: undefined, + minimum: 0, + maximum: 2_147_483_647, + requiresRestart: false, + showInDialog: false, + }); + }); + it('should define stopHookBlockingCap schema override as a positive integer', () => { expect( getSettingsSchema().stopHookBlockingCap.jsonSchemaOverride, diff --git a/packages/cli/src/config/settingsSchema.ts b/packages/cli/src/config/settingsSchema.ts index 108ada587fa..329687775e4 100644 --- a/packages/cli/src/config/settingsSchema.ts +++ b/packages/cli/src/config/settingsSchema.ts @@ -1707,6 +1707,19 @@ const SETTINGS_SCHEMA = { parentKey: 'generationConfig', showInDialog: false, }, + streamIdleTimeoutMs: { + type: 'integer', + label: 'Stream Idle Timeout', + category: 'Generation Configuration', + requiresRestart: false, + default: undefined as number | undefined, + description: + 'Maximum inactivity between streamed chunks for OpenAI-compatible models, in milliseconds. Set to 0 to disable the idle guard. For provider-backed models, configure this field in the selected modelProviders entry.', + minimum: 0, + maximum: 2_147_483_647, + parentKey: 'generationConfig', + showInDialog: false, + }, maxRetries: { type: 'number', label: 'Max Retries', diff --git a/packages/core/src/models/constants.ts b/packages/core/src/models/constants.ts index 4f55526cdfd..e6418a35323 100644 --- a/packages/core/src/models/constants.ts +++ b/packages/core/src/models/constants.ts @@ -21,6 +21,7 @@ type ContentGeneratorConfig = export const MODEL_GENERATION_CONFIG_FIELDS = [ 'samplingParams', 'timeout', + 'streamIdleTimeoutMs', 'maxRetries', 'retryInitialDelayMs', 'retryMaxDelayMs', diff --git a/packages/core/src/models/content-generator-config.test.ts b/packages/core/src/models/content-generator-config.test.ts index f410eeb9696..da4d2f3f177 100644 --- a/packages/core/src/models/content-generator-config.test.ts +++ b/packages/core/src/models/content-generator-config.test.ts @@ -46,6 +46,7 @@ describe('buildAgentContentGeneratorConfig', () => { samplingParams: { temperature: 0.7, top_p: 0.9 }, reasoning: { effort: 'high' as const }, timeout: 30000, + streamIdleTimeoutMs: 300000, maxRetries: 3, contextWindowSize: 128000, extra_body: { custom: 'value' }, @@ -68,6 +69,7 @@ describe('buildAgentContentGeneratorConfig', () => { expect(result.samplingParams).toEqual({ temperature: 0.7, top_p: 0.9 }); expect(result.reasoning).toEqual({ effort: 'high' }); expect(result.timeout).toBe(30000); + expect(result.streamIdleTimeoutMs).toBe(300000); expect(result.maxRetries).toBe(3); expect(result.contextWindowSize).toBe(128000); expect(result.extra_body).toEqual({ custom: 'value' }); @@ -101,6 +103,7 @@ describe('buildAgentContentGeneratorConfig', () => { expect(result.samplingParams).toBeUndefined(); expect(result.reasoning).toBeUndefined(); expect(result.timeout).toBeUndefined(); + expect(result.streamIdleTimeoutMs).toBeUndefined(); expect(result.maxRetries).toBeUndefined(); expect(result.contextWindowSize).toBeUndefined(); expect(result.extra_body).toBeUndefined(); @@ -152,6 +155,7 @@ describe('buildAgentContentGeneratorConfig', () => { envKey: 'REGISTRY_API_KEY', generationConfig: { samplingParams: { temperature: 0.5 }, + streamIdleTimeoutMs: 600000, contextWindowSize: 200000, reasoning: { effort: 'medium' as const }, }, @@ -182,6 +186,7 @@ describe('buildAgentContentGeneratorConfig', () => { expect(result.apiKeyEnvKey).toBe('REGISTRY_API_KEY'); // Registry generation config applied expect(result.samplingParams).toEqual({ temperature: 0.5 }); + expect(result.streamIdleTimeoutMs).toBe(600000); expect(result.contextWindowSize).toBe(200000); expect(result.reasoning).toEqual({ effort: 'medium' }); // Fields not in registry stay cleared (cross-provider) diff --git a/packages/core/src/models/modelConfigResolver.test.ts b/packages/core/src/models/modelConfigResolver.test.ts index f0c8a840cd2..7d8f1aa7cab 100644 --- a/packages/core/src/models/modelConfigResolver.test.ts +++ b/packages/core/src/models/modelConfigResolver.test.ts @@ -328,6 +328,7 @@ describe('modelConfigResolver', () => { apiKey: 'key', generationConfig: { timeout: 60000, + streamIdleTimeoutMs: 300000, maxRetries: 5, samplingParams: { temperature: 0.7, @@ -338,12 +339,14 @@ describe('modelConfigResolver', () => { }); expect(result.config.timeout).toBe(60000); + expect(result.config.streamIdleTimeoutMs).toBe(300000); expect(result.config.maxRetries).toBe(5); expect(result.config.retryInitialDelayMs).toBeUndefined(); expect(result.config.retryMaxDelayMs).toBeUndefined(); expect(result.config.samplingParams?.temperature).toBe(0.7); expect(result.sources['timeout'].kind).toBe('settings'); + expect(result.sources['streamIdleTimeoutMs'].kind).toBe('settings'); expect(result.sources['samplingParams'].kind).toBe('settings'); }); @@ -354,6 +357,7 @@ describe('modelConfigResolver', () => { settings: { generationConfig: { timeout: 30000, + streamIdleTimeoutMs: 300000, retryInitialDelayMs: 60_000, retryMaxDelayMs: 300_000, }, @@ -368,6 +372,7 @@ describe('modelConfigResolver', () => { baseUrl: 'https://api.example.com', generationConfig: { timeout: 60000, + streamIdleTimeoutMs: 0, retryInitialDelayMs: 3_000, retryMaxDelayMs: 30_000, }, @@ -375,9 +380,13 @@ describe('modelConfigResolver', () => { }); expect(result.config.timeout).toBe(60000); + expect(result.config.streamIdleTimeoutMs).toBe(0); expect(result.config.retryInitialDelayMs).toBe(3_000); expect(result.config.retryMaxDelayMs).toBe(30_000); expect(result.sources['timeout'].kind).toBe('modelProviders'); + expect(result.sources['streamIdleTimeoutMs'].kind).toBe( + 'modelProviders', + ); expect(result.sources['retryInitialDelayMs'].kind).toBe( 'modelProviders', ); diff --git a/packages/core/src/models/modelRegistry.test.ts b/packages/core/src/models/modelRegistry.test.ts index f8d94c171f2..967cf4ee2b1 100644 --- a/packages/core/src/models/modelRegistry.test.ts +++ b/packages/core/src/models/modelRegistry.test.ts @@ -150,6 +150,7 @@ describe('ModelRegistry', () => { name: 'GPT-4 Turbo', baseUrl: 'https://api.openai.com/v1', generationConfig: { + streamIdleTimeoutMs: 600000, samplingParams: { temperature: 0.8, max_tokens: 4096, @@ -176,6 +177,7 @@ describe('ModelRegistry', () => { expect(model?.generationConfig.samplingParams?.temperature).toBe(0.8); expect(model?.generationConfig.samplingParams?.max_tokens).toBe(4096); + expect(model?.generationConfig.streamIdleTimeoutMs).toBe(600000); // No defaults are applied - only the configured values are present expect(model?.generationConfig.samplingParams?.top_p).toBeUndefined(); expect(model?.generationConfig.timeout).toBeUndefined(); diff --git a/packages/core/src/models/types.ts b/packages/core/src/models/types.ts index 07a3e24e94b..d3a5c5355ed 100644 --- a/packages/core/src/models/types.ts +++ b/packages/core/src/models/types.ts @@ -31,6 +31,7 @@ export type ModelGenerationConfig = Pick< ContentGeneratorConfig, | 'samplingParams' | 'timeout' + | 'streamIdleTimeoutMs' | 'maxRetries' | 'retryInitialDelayMs' | 'retryMaxDelayMs' diff --git a/packages/vscode-ide-companion/schemas/settings.schema.json b/packages/vscode-ide-companion/schemas/settings.schema.json index 01ac3fac426..794577203b5 100644 --- a/packages/vscode-ide-companion/schemas/settings.schema.json +++ b/packages/vscode-ide-companion/schemas/settings.schema.json @@ -763,6 +763,12 @@ "description": "Request timeout in milliseconds.", "type": "number" }, + "streamIdleTimeoutMs": { + "description": "Maximum inactivity between streamed chunks for OpenAI-compatible models, in milliseconds. Set to 0 to disable the idle guard. For provider-backed models, configure this field in the selected modelProviders entry.", + "type": "integer", + "minimum": 0, + "maximum": 2147483647 + }, "maxRetries": { "description": "Maximum number of retries for failed requests.", "type": "number"