From 6b56d115d729002e572e7a628bcf88341873b382 Mon Sep 17 00:00:00 2001 From: Landon Cox Date: Fri, 29 May 2026 14:28:26 -0700 Subject: [PATCH 1/2] feat(api-proxy): pre-startup model validation via requestedModel config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add startup-time model validation to the API proxy. When apiProxy.requestedModel is configured, the proxy validates after fetching provider model catalogues that the requested model is available. Configuration (stdin config, non-sensitive): { "apiProxy": { "requestedModel": "gpt-4o" } } Behavior: - After fetchStartupModels() completes, checks AWF_REQUESTED_MODEL against all cached provider model lists - Emits 'model_unavailable_at_startup' error with available models if not found (clear diagnostic for retired/misspelled models) - Emits 'model_validation' info log on success - Non-blocking: proxy continues serving regardless of outcome - Supports model alias resolution across all providers Changes: - src/awf-config-schema.json: add requestedModel string field - src/config-file.ts: type + mapping for requestedModel - src/types/api-proxy-options.ts: requestedModel option with docs - src/services/api-proxy-service.ts: wire to AWF_REQUESTED_MODEL env - containers/api-proxy/server.js: validateRequestedModel() function - docs/awf-config-spec.md: §12.6 documenting behavior and config Closes #4021 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- containers/api-proxy/server.js | 62 ++++++++++++ .../server.startup-model-validation.test.js | 99 +++++++++++++++++++ docs/awf-config-spec.md | 33 +++++++ docs/awf-config.schema.json | 4 + src/awf-config-schema.json | 4 + src/config-file-mapping.test.ts | 5 + src/config-file.ts | 2 + src/schema.test.ts | 5 + src/services/api-proxy-service.ts | 3 + src/types/api-proxy-options.ts | 15 +++ 10 files changed, 232 insertions(+) create mode 100644 containers/api-proxy/server.startup-model-validation.test.js diff --git a/containers/api-proxy/server.js b/containers/api-proxy/server.js index fcbd705fa..3fd236444 100644 --- a/containers/api-proxy/server.js +++ b/containers/api-proxy/server.js @@ -484,6 +484,66 @@ async function fetchStartupModels(adapters = []) { modelFetchComplete = true; } +/** + * After model fetch, validate that the requested model (if specified via + * AWF_REQUESTED_MODEL) is available in the cached model list. + * Emits a clear diagnostic log if the model is not found. + */ +function validateRequestedModel() { + const requestedModel = (process.env.AWF_REQUESTED_MODEL || '').trim(); + if (!requestedModel) return; + + // Collect all known models across providers + const allModels = []; + for (const models of Object.values(cachedModels)) { + if (Array.isArray(models)) allModels.push(...models); + } + + if (allModels.length === 0) { + // No models fetched — cannot validate + logRequest('warn', 'model_validation_skipped', { + requested_model: requestedModel, + message: 'Cannot validate requested model — no model lists available from providers', + }); + return; + } + + // Check if the model or any alias of it resolves to an available model + const normalizedRequested = requestedModel.toLowerCase(); + const found = allModels.some(m => m.toLowerCase() === normalizedRequested); + + // Also check through model aliases — try resolution across all providers + let aliasResolved = false; + if (!found && MODEL_ALIASES) { + const { resolveModel } = require('./model-resolver'); + for (const provider of Object.keys(cachedModels)) { + const result = resolveModel(requestedModel, MODEL_ALIASES.models, cachedModels, provider); + if (result) { + aliasResolved = true; + break; + } + } + } + + if (!found && !aliasResolved) { + const availableModels = allModels.slice(0, 20).join(', '); + const truncated = allModels.length > 20 ? ` (and ${allModels.length - 20} more)` : ''; + logRequest('error', 'model_unavailable_at_startup', { + requested_model: requestedModel, + available_count: allModels.length, + message: `Requested model '${requestedModel}' is not available in any configured provider's model list. ` + + `This typically means the model is retired, restricted, or misspelled. ` + + `Available models: ${availableModels}${truncated}`, + }); + } else { + logRequest('info', 'model_validation', { + requested_model: requestedModel, + resolved_via: aliasResolved ? 'alias' : 'direct', + message: `Requested model '${requestedModel}' is available`, + }); + } +} + // ── Generic provider server factory ────────────────────────────────────────── /** * Create a health-check request handler for a provider adapter. @@ -689,6 +749,7 @@ if (require.main === module) { }); fetchStartupModels(adaptersToStart).then(() => { writeModelsJson(); + validateRequestedModel(); }).catch((err) => { logRequest('error', 'model_fetch_error', { message: 'Unexpected error fetching startup models', error: String(err) }); modelFetchComplete = true; @@ -744,6 +805,7 @@ module.exports = { probeProvider, httpProbe, fetchStartupModels, + validateRequestedModel, // State keyValidationResults, resetKeyValidationState, diff --git a/containers/api-proxy/server.startup-model-validation.test.js b/containers/api-proxy/server.startup-model-validation.test.js new file mode 100644 index 000000000..e0f49e89c --- /dev/null +++ b/containers/api-proxy/server.startup-model-validation.test.js @@ -0,0 +1,99 @@ +/** + * Tests for pre-startup model validation (AWF_REQUESTED_MODEL). + */ + +const { validateRequestedModel, cachedModels, resetModelCacheState } = require('./server'); +const { logRequest } = require('./logging'); + +jest.mock('./logging', () => ({ + logRequest: jest.fn(), +})); + +beforeEach(() => { + jest.clearAllMocks(); + resetModelCacheState(); + delete process.env.AWF_REQUESTED_MODEL; +}); + +describe('validateRequestedModel', () => { + it('does nothing when AWF_REQUESTED_MODEL is not set', () => { + validateRequestedModel(); + expect(logRequest).not.toHaveBeenCalled(); + }); + + it('does nothing when AWF_REQUESTED_MODEL is empty', () => { + process.env.AWF_REQUESTED_MODEL = ' '; + validateRequestedModel(); + expect(logRequest).not.toHaveBeenCalled(); + }); + + it('emits model_validation_skipped when no models are cached', () => { + process.env.AWF_REQUESTED_MODEL = 'gpt-4o'; + validateRequestedModel(); + expect(logRequest).toHaveBeenCalledWith('warn', 'model_validation_skipped', expect.objectContaining({ + requested_model: 'gpt-4o', + })); + }); + + it('emits model_unavailable_at_startup when model is not found', () => { + process.env.AWF_REQUESTED_MODEL = 'gpt-5-codex'; + cachedModels.copilot = ['gpt-4o', 'gpt-4o-mini', 'claude-sonnet-4-5']; + validateRequestedModel(); + expect(logRequest).toHaveBeenCalledWith('error', 'model_unavailable_at_startup', expect.objectContaining({ + requested_model: 'gpt-5-codex', + available_count: 3, + })); + expect(logRequest.mock.calls[0][2].message).toContain("not available"); + expect(logRequest.mock.calls[0][2].message).toContain("gpt-4o"); + }); + + it('emits model_validation success when model is found directly', () => { + process.env.AWF_REQUESTED_MODEL = 'gpt-4o'; + cachedModels.copilot = ['gpt-4o', 'gpt-4o-mini']; + validateRequestedModel(); + expect(logRequest).toHaveBeenCalledWith('info', 'model_validation', expect.objectContaining({ + requested_model: 'gpt-4o', + resolved_via: 'direct', + })); + }); + + it('matches model case-insensitively', () => { + process.env.AWF_REQUESTED_MODEL = 'GPT-4o'; + cachedModels.copilot = ['gpt-4o', 'gpt-4o-mini']; + validateRequestedModel(); + expect(logRequest).toHaveBeenCalledWith('info', 'model_validation', expect.objectContaining({ + requested_model: 'GPT-4o', + resolved_via: 'direct', + })); + }); + + it('searches across multiple providers', () => { + process.env.AWF_REQUESTED_MODEL = 'claude-sonnet-4-5'; + cachedModels.copilot = ['gpt-4o']; + cachedModels.anthropic = ['claude-sonnet-4-5', 'claude-haiku-3']; + validateRequestedModel(); + expect(logRequest).toHaveBeenCalledWith('info', 'model_validation', expect.objectContaining({ + requested_model: 'claude-sonnet-4-5', + resolved_via: 'direct', + })); + }); + + it('skips providers with null model lists', () => { + process.env.AWF_REQUESTED_MODEL = 'gpt-4o'; + cachedModels.copilot = null; // fetch failed + cachedModels.openai = ['gpt-4o', 'gpt-4o-mini']; + validateRequestedModel(); + expect(logRequest).toHaveBeenCalledWith('info', 'model_validation', expect.objectContaining({ + requested_model: 'gpt-4o', + })); + }); + + it('lists available models in the error diagnostic', () => { + process.env.AWF_REQUESTED_MODEL = 'nonexistent-model'; + cachedModels.copilot = ['gpt-4o', 'gpt-4o-mini', 'o3']; + validateRequestedModel(); + const message = logRequest.mock.calls[0][2].message; + expect(message).toContain('gpt-4o'); + expect(message).toContain('retired, restricted, or misspelled'); + }); +}); diff --git a/docs/awf-config-spec.md b/docs/awf-config-spec.md index 5383c1b87..24fa905ea 100644 --- a/docs/awf-config-spec.md +++ b/docs/awf-config-spec.md @@ -101,6 +101,7 @@ the corresponding CLI flag. - `apiProxy.maxEffectiveTokens` → *(config-only; no CLI equivalent)* - `apiProxy.modelMultipliers` → `--max-model-multiplier ` - `apiProxy.maxRuns` → *(config-only; no CLI equivalent)* +- `apiProxy.requestedModel` → *(config-only; maps to `AWF_REQUESTED_MODEL` for pre-startup validation)* - `apiProxy.modelFallback` → *(config-only; model fallback strategy)* - `apiProxy.models` → *(config-only; model alias rewriting)* - `apiProxy.logging.debugTokens` → *(config-only; maps to `AWF_DEBUG_TOKENS`)* @@ -883,6 +884,38 @@ response: The `/reflect` endpoint does not include fallback state by design (it is static per run). +### 12.6 Pre-Startup Model Validation + +When `apiProxy.requestedModel` is configured, the API proxy validates at startup +that the specified model is available in at least one provider's model catalogue. + +**Configuration:** + +```json +{ + "apiProxy": { + "requestedModel": "gpt-4o" + } +} +``` + +**Mapping:** `apiProxy.requestedModel` → `AWF_REQUESTED_MODEL` *(config-only; set by AWF CLI)* + +**Behavior:** + +1. After `fetchStartupModels()` completes, the proxy checks `AWF_REQUESTED_MODEL` + against all cached provider model lists. +2. If the model is found directly or resolves via model aliases, a confirmation + `model_validation` log is emitted. +3. If the model is NOT found, a `model_unavailable_at_startup` error log is + emitted listing available models as a diagnostic aid. +4. Validation is **non-blocking** — the proxy continues serving requests regardless + of the outcome, so agents that ignore the model hint are not affected. + +This enables workflow authors to get clear, early feedback when a retired or +misspelled model is specified, rather than waiting for the first API request to +fail with an opaque error. + ## 13. Model Alias Logging The API proxy emits structured logging events during model alias resolution. diff --git a/docs/awf-config.schema.json b/docs/awf-config.schema.json index 7c113e940..5cab0b17e 100644 --- a/docs/awf-config.schema.json +++ b/docs/awf-config.schema.json @@ -85,6 +85,10 @@ "minimum": 1, "description": "Maximum number of LLM invocations allowed for a run. When reached, the API proxy rejects subsequent requests with HTTP 429 and error type 'max_runs_exceeded'. See spec §11." }, + "requestedModel": { + "type": "string", + "description": "Expected model name for pre-startup validation. When set, the API proxy validates at startup that this model is available in at least one provider's model catalogue. Emits a clear diagnostic if the model is retired, restricted, or misspelled. Does not block startup." + }, "modelFallback": { "type": "object", "description": "Model fallback policy for unresolved model selections. Enabled by default as a safety net.", diff --git a/src/awf-config-schema.json b/src/awf-config-schema.json index 7c113e940..5cab0b17e 100644 --- a/src/awf-config-schema.json +++ b/src/awf-config-schema.json @@ -85,6 +85,10 @@ "minimum": 1, "description": "Maximum number of LLM invocations allowed for a run. When reached, the API proxy rejects subsequent requests with HTTP 429 and error type 'max_runs_exceeded'. See spec §11." }, + "requestedModel": { + "type": "string", + "description": "Expected model name for pre-startup validation. When set, the API proxy validates at startup that this model is available in at least one provider's model catalogue. Emits a clear diagnostic if the model is retired, restricted, or misspelled. Does not block startup." + }, "modelFallback": { "type": "object", "description": "Model fallback policy for unresolved model selections. Enabled by default as a safety net.", diff --git a/src/config-file-mapping.test.ts b/src/config-file-mapping.test.ts index 6cb01642b..2b272ea15 100644 --- a/src/config-file-mapping.test.ts +++ b/src/config-file-mapping.test.ts @@ -154,6 +154,11 @@ describe('mapAwfFileConfigToCliOptions', () => { expect(result.maxRuns).toBe(42); }); + it('maps requestedModel field', () => { + const result = mapAwfFileConfigToCliOptions({ apiProxy: { requestedModel: 'gpt-4o' } }); + expect(result.requestedModel).toBe('gpt-4o'); + }); + it('maps modelFallback field', () => { const result = mapAwfFileConfigToCliOptions({ apiProxy: { modelFallback: { enabled: false, strategy: 'middle_power' } }, diff --git a/src/config-file.ts b/src/config-file.ts index 909b0f43d..d11641435 100644 --- a/src/config-file.ts +++ b/src/config-file.ts @@ -20,6 +20,7 @@ interface AwfFileConfig { maxEffectiveTokens?: number; modelMultipliers?: Record; maxRuns?: number; + requestedModel?: string; modelFallback?: { enabled?: boolean; strategy?: 'middle_power'; @@ -179,6 +180,7 @@ export function mapAwfFileConfigToCliOptions(config: AwfFileConfig): Record { expect(validate({ apiProxy: { modelMultipliers: { 'gpt-4o': 0 } } })).toBe(false); }); + it('accepts apiProxy.requestedModel as a string', () => { + expect(validate({ apiProxy: { requestedModel: 'gpt-4o' } })).toBe(true); + expect(validate({ apiProxy: { requestedModel: 123 } })).toBe(false); + }); + it('rejects invalid logging.logLevel values', () => { expect(validate({ logging: { logLevel: 'verbose' } })).toBe(false); expect(validate({ logging: { logLevel: 'debug' } })).toBe(true); diff --git a/src/services/api-proxy-service.ts b/src/services/api-proxy-service.ts index 5ebe41bf9..2bb2f1db9 100644 --- a/src/services/api-proxy-service.ts +++ b/src/services/api-proxy-service.ts @@ -60,6 +60,9 @@ function buildProviderTargetEnv(config: WrapperConfig): Record { if (copilotProviderBaseUrl) env.COPILOT_PROVIDER_BASE_URL = copilotProviderBaseUrl; if (copilotProviderApiKey) env.COPILOT_PROVIDER_API_KEY = copilotProviderApiKey; + // Pre-startup model validation (non-sensitive config value) + if (config.requestedModel) env.AWF_REQUESTED_MODEL = config.requestedModel; + return env; } diff --git a/src/types/api-proxy-options.ts b/src/types/api-proxy-options.ts index 042037d48..1768e4186 100644 --- a/src/types/api-proxy-options.ts +++ b/src/types/api-proxy-options.ts @@ -337,6 +337,21 @@ export interface ApiProxyOptions { */ modelAliases?: Record; + /** + * Expected model name for pre-startup validation. + * + * When set, the API proxy validates at startup that this model is available + * in at least one configured provider's model catalogue. If the model is not + * found (retired, restricted, or misspelled), a clear `model_unavailable_at_startup` + * diagnostic is emitted. This does not block proxy startup. + * + * - Config: `apiProxy.requestedModel` + * - Environment variable: `AWF_REQUESTED_MODEL` (internal; set by AWF CLI) + * + * @example 'gpt-4o' + */ + requestedModel?: string; + /** * Enable detailed token and model-alias diagnostic logging. * From fa05b5031910f1a1428e2be20fe85b81c859646b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 29 May 2026 21:57:34 +0000 Subject: [PATCH 2/2] fix(api-proxy): address review feedback on startup model validation - Pass `{ enabled: false }` to resolveModel in validateRequestedModel so middle-power fallback cannot produce a false-positive alias match for a model that doesn't exist in any provider catalogue (fixes comment on server.js:515-526) - Wire `requestedModel` from options into the WrapperConfig returned by buildConfig so AWF_REQUESTED_MODEL is forwarded when using the normal stdin-config CLI path (fixes comment on api-proxy-service.ts:63-65) - Add two alias-focused tests using jest.isolateModules: one verifying resolved_via 'alias' for a genuine alias match, one confirming model_unavailable_at_startup when only fallback would have matched (fixes comment on server.startup-model-validation.test.js:18-22) --- containers/api-proxy/server.js | 6 +- .../server.startup-model-validation.test.js | 59 +++++++++++++++++++ src/commands/build-config.ts | 1 + 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/containers/api-proxy/server.js b/containers/api-proxy/server.js index 3fd236444..867a51832 100644 --- a/containers/api-proxy/server.js +++ b/containers/api-proxy/server.js @@ -512,12 +512,14 @@ function validateRequestedModel() { const normalizedRequested = requestedModel.toLowerCase(); const found = allModels.some(m => m.toLowerCase() === normalizedRequested); - // Also check through model aliases — try resolution across all providers + // Also check through model aliases — try resolution across all providers. + // Disable fallback so that middle-power fallback does not produce a non-null + // result for a model that does not actually exist in any provider catalogue. let aliasResolved = false; if (!found && MODEL_ALIASES) { const { resolveModel } = require('./model-resolver'); for (const provider of Object.keys(cachedModels)) { - const result = resolveModel(requestedModel, MODEL_ALIASES.models, cachedModels, provider); + const result = resolveModel(requestedModel, MODEL_ALIASES.models, cachedModels, provider, [], { enabled: false }); if (result) { aliasResolved = true; break; diff --git a/containers/api-proxy/server.startup-model-validation.test.js b/containers/api-proxy/server.startup-model-validation.test.js index e0f49e89c..a70c040a6 100644 --- a/containers/api-proxy/server.startup-model-validation.test.js +++ b/containers/api-proxy/server.startup-model-validation.test.js @@ -96,4 +96,63 @@ describe('validateRequestedModel', () => { expect(message).toContain('gpt-4o'); expect(message).toContain('retired, restricted, or misspelled'); }); + + it('resolves AWF_REQUESTED_MODEL via model alias and logs resolved_via alias', () => { + const prevAliases = process.env.AWF_MODEL_ALIASES; + process.env.AWF_MODEL_ALIASES = JSON.stringify({ models: { sonnet: ['copilot/*sonnet*'] } }); + + let isolatedServer; + jest.isolateModules(() => { + jest.mock('./logging', () => ({ logRequest: jest.fn() })); + isolatedServer = require('./server'); + }); + + const { logRequest: isolatedLog } = require('./logging'); + + try { + isolatedServer.resetModelCacheState(); + isolatedServer.cachedModels.copilot = ['claude-sonnet-4-5', 'gpt-4o']; + process.env.AWF_REQUESTED_MODEL = 'sonnet'; + isolatedServer.validateRequestedModel(); + expect(isolatedLog).toHaveBeenCalledWith('info', 'model_validation', expect.objectContaining({ + requested_model: 'sonnet', + resolved_via: 'alias', + })); + } finally { + if (prevAliases === undefined) delete process.env.AWF_MODEL_ALIASES; + else process.env.AWF_MODEL_ALIASES = prevAliases; + } + }); + + it('does not emit model_validation via alias when fallback would fire but model is absent', () => { + const prevAliases = process.env.AWF_MODEL_ALIASES; + const prevFallback = process.env.AWF_MODEL_FALLBACK; + process.env.AWF_MODEL_ALIASES = JSON.stringify({ models: { sonnet: ['copilot/*sonnet*'] } }); + process.env.AWF_MODEL_FALLBACK = JSON.stringify({ enabled: true, strategy: 'middle_power' }); + + let isolatedServer; + jest.isolateModules(() => { + jest.mock('./logging', () => ({ logRequest: jest.fn() })); + isolatedServer = require('./server'); + }); + + const { logRequest: isolatedLog } = require('./logging'); + + try { + isolatedServer.resetModelCacheState(); + // No models matching the alias pattern — only a non-matching model is present + isolatedServer.cachedModels.copilot = ['gpt-4o']; + process.env.AWF_REQUESTED_MODEL = 'sonnet'; + isolatedServer.validateRequestedModel(); + // middle-power fallback is disabled during validation, so model_unavailable_at_startup is expected + expect(isolatedLog).toHaveBeenCalledWith('error', 'model_unavailable_at_startup', expect.objectContaining({ + requested_model: 'sonnet', + })); + } finally { + if (prevAliases === undefined) delete process.env.AWF_MODEL_ALIASES; + else process.env.AWF_MODEL_ALIASES = prevAliases; + if (prevFallback === undefined) delete process.env.AWF_MODEL_FALLBACK; + else process.env.AWF_MODEL_FALLBACK = prevFallback; + } + }); }); diff --git a/src/commands/build-config.ts b/src/commands/build-config.ts index 3e8a0bfcd..690456c24 100644 --- a/src/commands/build-config.ts +++ b/src/commands/build-config.ts @@ -101,6 +101,7 @@ export function buildConfig(inputs: BuildConfigInputs): WrapperConfig { allowedUrls, enableApiProxy: options.enableApiProxy as boolean, modelFallback: options.modelFallback as { enabled?: boolean; strategy?: 'middle_power' } | undefined, + requestedModel: options.requestedModel as string | undefined, anthropicAutoCache: options.anthropicAutoCache as boolean, anthropicCacheTailTtl: options.anthropicCacheTailTtl as '5m' | '1h' | undefined, modelAliases,