From 54b9aa4dc9f05f0e41d695f8a411a80b74391a36 Mon Sep 17 00:00:00 2001 From: Joob1n Date: Sun, 30 Aug 2026 13:43:53 +0800 Subject: [PATCH] fix(core): refuse models whose declared output has no text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `isModelExplicitlyUnsupportedForChat` guards four call sites — the model catalog, connection readiness, the execution model authority, and session catalog selection — and could not fire on real data. Its image branch keys on `capabilities.imageGeneration`, and nothing in production has ever set that flag: the generated metadata carries the fact as `modalities.output` instead, where 128 entries declare an output that is not text and 0 declare `imageGeneration`. So `gpt-image-2` and every other image or speech model was selectable as a chat model, and the failure surfaced as a provider error on the first request rather than a refusal at selection. The guard now also refuses a model whose declared output names modalities but not text, which is the form the fact actually arrives in. Only an explicit `chat: true` outranks it; `reasoning` and `functionCalling` do not, because a TTS model carrying `reasoning: true` is describing how it composes speech and still cannot answer in text. An empty output list stays allowed. `modalities.output` is typed to text, image, and audio, so a video model's real output has no representation and serializes as `[]` — the same shape a generator bug would produce. Blocking on it would be guessing, so the rule reads only non-empty lists. `makeEntry` now passes the merged `modalities` into the availability derivation beside the merged `capabilities`. It passed only `normalizedModel.modalities`, so a bundled image-only model reached the guard with no output declaration at all. `toModelInfo` stopped dropping `output_modalities`. It was validated and discarded, so a relay that advertised an image-only model handed back a row indistinguishable from a chat model's. The fact is recorded as a capability rather than as modalities because `decodeConnectionModel` is an exact record that does not persist `modalities` — emitting it would throw `unknown field` on the next catalog read. Generated-by: Claude Opus 5 via Claude Code --- .../core/src/__tests__/model-catalog.test.ts | 56 +++++++++++++++ packages/core/src/model-catalog.ts | 32 ++++++++- .../src/__tests__/model-fetcher.test.ts | 69 +++++++++++++++++++ packages/runtime/src/model-fetcher.ts | 30 ++++++++ 4 files changed, 186 insertions(+), 1 deletion(-) diff --git a/packages/core/src/__tests__/model-catalog.test.ts b/packages/core/src/__tests__/model-catalog.test.ts index 81b36bbfff..6db41f0da2 100644 --- a/packages/core/src/__tests__/model-catalog.test.ts +++ b/packages/core/src/__tests__/model-catalog.test.ts @@ -82,6 +82,62 @@ test('chat-default validation blocks image-only models but accepts merged partia assert.deepEqual(verdict(partial), { ok: true }); }); +test('a declared output modality without text rules a model out of chat', () => { + // The shape this exists for: `gpt-image-2` on a relay. Bundled metadata + // records `modalities.output: ["image"]` and has never set + // `capabilities.imageGeneration` for any model, so before this the guard + // could not fire and an image model was selectable as a chat model. + const imageOnly = { + providerType: 'openai' as const, + defaultModel: 'gpt-image-2', + models: [{ id: 'gpt-image-2' }], + modelSource: 'fetched' as const, + }; + assert.deepEqual(verdict(imageOnly), { ok: false, reason: 'unsupported_for_chat' }); + + // Audio-only too, and a stray `reasoning: true` on a TTS model does not + // rescue it: reasoning describes how it composes speech, not that it can + // answer in text. + const audioOnly = { + providerType: 'google' as const, + defaultModel: 'gemini-3.1-flash-tts-preview', + models: [{ id: 'gemini-3.1-flash-tts-preview' }], + modelSource: 'fetched' as const, + }; + assert.deepEqual(verdict(audioOnly), { ok: false, reason: 'unsupported_for_chat' }); +}); + +test('an empty output modality list is not evidence against chat', () => { + // `modalities.output` is typed to text, image, and audio, so a video model's + // real output has no representation and serializes as `[]` — the same shape + // a generator bug would produce. Blocking on it would be guessing. + const video = { + providerType: 'google' as const, + defaultModel: 'gemini-omni-flash-preview', + models: [{ id: 'gemini-omni-flash-preview' }], + modelSource: 'fetched' as const, + }; + assert.deepEqual(verdict(video), { ok: true }); +}); + +test('an explicit chat capability outranks the declared output modality', () => { + // A provider that says both is contradicting itself, and the direct claim + // about chat is the more specific one. + const contradictory = { + providerType: 'openai-compatible' as const, + defaultModel: 'relay-omni', + models: [ + { + id: 'relay-omni', + capabilities: { chat: true }, + modalities: { input: ['text' as const], output: ['image' as const] }, + }, + ], + modelSource: 'fetched' as const, + }; + assert.deepEqual(verdict(contradictory), { ok: true }); +}); + test('catalog entries preserve advertised parallel tool-call support', () => { const [entry] = buildModelCatalogEntries({ providerType: 'openai-compatible', diff --git a/packages/core/src/model-catalog.ts b/packages/core/src/model-catalog.ts index 72a56c67a4..588e2dfd54 100644 --- a/packages/core/src/model-catalog.ts +++ b/packages/core/src/model-catalog.ts @@ -345,9 +345,15 @@ function makeEntry( const lastUpdated = normalizedModel.lastUpdated ?? metadata.lastUpdated; const modalities = normalizedModel.modalities ?? metadata.modalities; const capabilities = mergeCapabilities(normalizedModel.capabilities, metadata.capabilities); + // `modalities` too, not just `capabilities`: both are merged from the + // provider row and the bundled metadata a few lines up, and the chat guard + // reads the modality. Passing the unmerged `normalizedModel.modalities` + // meant a bundled image-only model reached the guard with no output + // declaration at all. const unavailableReason = deriveModelUnavailableReason(input, { ...normalizedModel, capabilities, + ...(modalities !== undefined ? { modalities } : {}), }); return { id: normalizedModel.id, @@ -626,10 +632,34 @@ function isStale( return now - input.modelsFetchedAt > staleAfterMs; } +/** + * Whether a declared output modality rules the model out of chat. + * + * A model that answers only in images or only in audio cannot hold a + * conversation, and this is the form that fact actually arrives in: the + * generated metadata records `modalities.output` for every such model and has + * never set `capabilities.imageGeneration` for any of them, so the capability + * check below could not fire on bundled data. + * + * An EMPTY list is not evidence. `modalities.output` is typed to text, image, + * and audio, so a video model's real output has no representation and + * serializes as `[]` — the same shape a future generator bug would produce. + * Only a non-empty list says something, and what it says is what it lists. + */ +function declaresNoTextOutput(model: ModelInfo): boolean { + const output = model.modalities?.output; + if (output === undefined || output.length === 0) return false; + return !output.includes('text'); +} + export function isModelExplicitlyUnsupportedForChat(model: ModelInfo): boolean { const caps = model.capabilities; + if (caps?.chat === false) return true; + // Only an explicit `chat: true` outranks the modality. `reasoning` and + // `functionCalling` do not: a TTS model carrying `reasoning: true` is + // describing how it composes speech, and it still cannot answer in text. + if (caps?.chat !== true && declaresNoTextOutput(model)) return true; if (!caps) return false; - if (caps.chat === false) return true; return ( caps.imageGeneration === true && caps.chat !== true && diff --git a/packages/runtime/src/__tests__/model-fetcher.test.ts b/packages/runtime/src/__tests__/model-fetcher.test.ts index 9b81d427c8..5f067d6245 100644 --- a/packages/runtime/src/__tests__/model-fetcher.test.ts +++ b/packages/runtime/src/__tests__/model-fetcher.test.ts @@ -384,6 +384,75 @@ describe('fetchProviderModels', () => { assert.equal(JSON.stringify(outcome).includes(secret), false); } }); + + test('a declared output modality without text is recorded as a capability', async () => { + // `output_modalities` was validated and then dropped, so a relay that + // advertised an image-only model handed back a row indistinguishable from + // a chat model's and nothing downstream could refuse it. + const server = await startJsonServer((_request, response) => { + respondJson(response, 200, { + data: [ + { id: 'relay-image', input_modalities: ['text'], output_modalities: ['image'] }, + { id: 'relay-speech', input_modalities: ['text'], output_modalities: ['audio'] }, + { id: 'relay-chat', input_modalities: ['text'], output_modalities: ['text', 'image'] }, + { id: 'relay-video', input_modalities: ['text'], output_modalities: [] }, + { id: 'relay-silent', input_modalities: ['text'] }, + ], + }); + }); + + const models = await fetchProviderModels( + { ...zaiConnection(), baseUrl: server.url }, + 'zai-live-secret', + ); + const capabilitiesOf = (id: string) => models.find((model) => model.id === id)?.capabilities; + + assert.equal(capabilitiesOf('relay-image')?.chat, false); + assert.equal(capabilitiesOf('relay-image')?.imageGeneration, true); + // Audio-only is equally unable to answer in text, but it is not an image + // generator and must not be labelled one. + assert.equal(capabilitiesOf('relay-speech')?.chat, false); + assert.equal(capabilitiesOf('relay-speech')?.imageGeneration, undefined); + // Text among the outputs is a chat model whatever else it also emits. + assert.equal(capabilitiesOf('relay-chat')?.chat, undefined); + // An empty list and an absent one both say nothing, and nothing is not a + // refusal: a video model's output has no representation in this union. + assert.equal(capabilitiesOf('relay-video')?.chat, undefined); + assert.equal(capabilitiesOf('relay-silent')?.chat, undefined); + }); + + test('an unrecognized output modality never disables a model', async () => { + // The array is validated as an array and never item-by-item, so these + // reach the modality read intact. Every other modality read here ADDS a + // capability and an unrecognized value merely costs a fact; this one + // REMOVES chat, where the same miss would silently disable a model that + // works. Unrecognized has to mean "said nothing", not "said not text". + const server = await startJsonServer((_request, response) => { + respondJson(response, 200, { + data: [ + { id: 'relay-cased', output_modalities: ['Text'] }, + { id: 'relay-null', output_modalities: [null] }, + { id: 'relay-numeric', output_modalities: [42] }, + { id: 'relay-future', output_modalities: ['hologram'] }, + // A recognized value alongside an unrecognized one still counts: + // the provider named a modality this build understands. + { id: 'relay-mixed', output_modalities: ['image', 'hologram'] }, + ], + }); + }); + + const models = await fetchProviderModels( + { ...zaiConnection(), baseUrl: server.url }, + 'zai-live-secret', + ); + const capabilitiesOf = (id: string) => models.find((model) => model.id === id)?.capabilities; + + for (const id of ['relay-cased', 'relay-null', 'relay-numeric', 'relay-future']) { + assert.equal(capabilitiesOf(id)?.chat, undefined, id); + } + assert.equal(capabilitiesOf('relay-mixed')?.chat, false); + assert.equal(capabilitiesOf('relay-mixed')?.imageGeneration, true); + }); }); async function startJsonServer( diff --git a/packages/runtime/src/model-fetcher.ts b/packages/runtime/src/model-fetcher.ts index 3b4e478509..dd7845fb86 100644 --- a/packages/runtime/src/model-fetcher.ts +++ b/packages/runtime/src/model-fetcher.ts @@ -801,6 +801,22 @@ function toModelInfo(model: RawProviderModel): ModelInfo | null { if (model.tags?.includes('vision')) capabilities.vision = true; if (model.tags?.includes('reasoning')) capabilities.reasoning = true; if (model.tags?.includes('tool-use')) capabilities.functionCalling = true; + // `output_modalities` was validated above and then dropped, so a relay that + // advertised an image-only model handed back a row indistinguishable from a + // chat model's. Declared output that names modalities but not text is the + // provider stating the model cannot answer in text; record it the way the + // rest of this function records modality facts, as a capability. + // + // Read through `knownOutputModalities` rather than the raw array. Every + // other modality read here ADDS a capability, so a value this code fails to + // recognize costs a fact; this one REMOVES chat, where the same miss would + // silently disable a working model. `assertOptionalArray` checks the + // container and not its items, so `['Text']` or `[null]` reach here intact. + const declaredOutput = knownOutputModalities(model.output_modalities); + if (declaredOutput.length > 0 && !declaredOutput.includes('text')) { + capabilities.chat = false; + if (declaredOutput.includes('image')) capabilities.imageGeneration = true; + } if (model.providers) { capabilities.functionCalling = providers.some( (provider) => provider.status === 'live' && provider.supports_tools === true, @@ -853,6 +869,20 @@ function providerObjectArray( return value as T[]; } +/** + * The declared output modalities this build understands, in the provider's + * order. Anything else — a value from a newer spec, a capitalized spelling, a + * non-string — is dropped rather than guessed at, so an unrecognized list + * reads as "said nothing" instead of "said not text". + */ +function knownOutputModalities(declared: readonly unknown[] | undefined): string[] { + if (declared === undefined) return []; + return declared.filter( + (value): value is 'text' | 'image' | 'audio' => + value === 'text' || value === 'image' || value === 'audio', + ); +} + function assertOptionalArray( value: unknown, label: string,