From 9f738da0f0588e60334c3be2e503acab8848f01d Mon Sep 17 00:00:00 2001 From: Tom Date: Wed, 23 Sep 2026 02:20:42 +0100 Subject: [PATCH 1/2] fix(auto-combo): keep non-chat models out of auto/* pools The credentialed branch of prepareVirtualAutoComboInputs() adds every synced/custom model id to the auto/* candidate pool without looking at what surface it serves; only no-auth providers are gated (serviceKinds). Image, video, music and TTS models that a provider lists next to its chat models (x.ai /v1/models returns grok-imagine-*, local nodes sync Wan/TTS, suno chirp-*) therefore get scored and dispatched for chat turns and can only fail, burning a fallback hop each time. Skip synced/custom models whose modelType is non-chat or whose supportedEndpoints name no chat-capable surface (chat/responses), and tag x.ai grok-imagine-* during discovery since /v1/models carries no type field. --- open-sse/services/autoCombo/virtualFactory.ts | 44 +++++-- .../models/discovery/providerModelsConfig.ts | 17 ++- .../auto-pool-skip-non-chat-models.test.ts | 122 ++++++++++++++++++ 3 files changed, 174 insertions(+), 9 deletions(-) create mode 100644 tests/unit/auto-pool-skip-non-chat-models.test.ts diff --git a/open-sse/services/autoCombo/virtualFactory.ts b/open-sse/services/autoCombo/virtualFactory.ts index d8de0dd6859c..4cf8a1af5ef8 100644 --- a/open-sse/services/autoCombo/virtualFactory.ts +++ b/open-sse/services/autoCombo/virtualFactory.ts @@ -609,6 +609,23 @@ export async function attachPreparedCapabilityValues( return prepared; } +// Endpoints that can serve a chat turn. A synced/custom model that declares only other +// surfaces (images, videos, audio, embeddings, rerank) can never answer an auto/* chat +// request, so it must not enter the candidate pool (the no-auth path already gates on +// `serviceKinds`; this is the credentialed-provider equivalent). +const CHAT_CAPABLE_ENDPOINTS = new Set(["chat", "responses"]); + +export function isDeclaredNonChatModel(model: { + modelType?: unknown; + supportedEndpoints?: unknown; +}): boolean { + if (typeof model.modelType === "string" && model.modelType !== "chat") return true; + const endpoints = Array.isArray(model.supportedEndpoints) + ? model.supportedEndpoints.filter((e): e is string => typeof e === "string") + : []; + return endpoints.length > 0 && !endpoints.some((e) => CHAT_CAPABLE_ENDPOINTS.has(e)); +} + export async function prepareVirtualAutoComboInputs( options: { includeResolvedCapabilities?: boolean; @@ -694,17 +711,27 @@ export async function prepareVirtualAutoComboInputs( // already filters those out (catalog.ts, "Add custom models"); without the same // filter here every read below null-derefs and the whole auto/* pool fails to // materialize ("Could not materialize built-in auto model auto/"). - const customModels: Array<{ id?: string }> = ( - Array.isArray(rawCustomModels) ? rawCustomModels : [] - ).filter( - (model: unknown): model is { id?: string } => - !!model && typeof model === "object" && !Array.isArray(model) - ); + const customModels: Array<{ id?: string; modelType?: unknown; supportedEndpoints?: unknown }> = + (Array.isArray(rawCustomModels) ? rawCustomModels : []).filter( + (model: unknown): model is { id?: string } => + !!model && typeof model === "object" && !Array.isArray(model) + ); const userVisibleIds = new Set(); + const nonChatIds = new Set(); for (const models of Object.values(syncedByConnection)) { - for (const m of models) if (m.id && !hiddenModels?.has(m.id)) userVisibleIds.add(m.id); + for (const m of models) { + if (!m.id || hiddenModels?.has(m.id)) continue; + userVisibleIds.add(m.id); + if (isDeclaredNonChatModel(m)) nonChatIds.add(m.id); + } + } + for (const m of customModels) { + if (!m.id || hiddenModels?.has(m.id)) continue; + userVisibleIds.add(m.id); + if (isDeclaredNonChatModel(m)) nonChatIds.add(m.id); } - for (const m of customModels) if (m.id && !hiddenModels?.has(m.id)) userVisibleIds.add(m.id); + // Decided before dropping non-chat ids: an image-only sync must not fall back to + // the provider's static catalog. const hasUserModels = userVisibleIds.size > 0; const modelIds = hasUserModels ? Array.from(userVisibleIds) @@ -717,6 +744,7 @@ export async function prepareVirtualAutoComboInputs( await yieldVirtualAutoPreparationTurn(); } if (hiddenModels?.has(modelId)) continue; + if (nonChatIds.has(modelId)) continue; const allowedConnectionIds = providerConnections .filter((conn) => { diff --git a/src/app/api/providers/[id]/models/discovery/providerModelsConfig.ts b/src/app/api/providers/[id]/models/discovery/providerModelsConfig.ts index c7dff93bd9e5..3921e7fd616e 100644 --- a/src/app/api/providers/[id]/models/discovery/providerModelsConfig.ts +++ b/src/app/api/providers/[id]/models/discovery/providerModelsConfig.ts @@ -401,13 +401,28 @@ const KIMI_CODING_MODELS_CONFIG: ProviderModelsConfigEntry = { // xai-oauth is not registered in PROVIDER_MODELS_CONFIG below and stays on // its frozen static seed (open-sse/config/providers/registry/xai/index.ts) // unless the flag is explicitly turned on. +// x.ai /v1/models lists Grok Imagine media models next to the chat models without a +// type field. Tag them so they stay out of chat catalogs and auto/* pools. +function tagXaiMediaModel(model: unknown) { + if (!model || typeof model !== "object") return model; + const id = typeof (model as { id?: unknown }).id === "string" ? (model as { id: string }).id : ""; + if (/^grok-imagine-image/i.test(id)) { + return { ...model, supportedEndpoints: ["images"], modelType: "image" }; + } + if (/^grok-imagine-video/i.test(id)) return { ...model, supportedEndpoints: ["videos"] }; + return model; +} + export const XAI_MODELS_CONFIG: ProviderModelsConfigEntry = { url: "https://api.x.ai/v1/models", method: "GET", headers: { "Content-Type": "application/json" }, authHeader: "Authorization", authPrefix: "Bearer ", - parseResponse: (data) => data.data || data.models || [], + parseResponse: (data) => { + const models = data.data || data.models || []; + return Array.isArray(models) ? models.map(tagXaiMediaModel) : models; + }, }; /** diff --git a/tests/unit/auto-pool-skip-non-chat-models.test.ts b/tests/unit/auto-pool-skip-non-chat-models.test.ts new file mode 100644 index 000000000000..3d87c89b3af3 --- /dev/null +++ b/tests/unit/auto-pool-skip-non-chat-models.test.ts @@ -0,0 +1,122 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +// Credentialed providers whose synced/custom catalog mixes chat and media models +// (x.ai /v1/models lists grok-imagine-* next to grok-4.x; local nodes sync TTS/ASR/video) +// must not put the media models into auto/* chat pools: they can only ever fail there. + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-auto-non-chat-")); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const modelsDb = await import("../../src/lib/db/models.ts"); +const virtualFactory = await import("../../open-sse/services/autoCombo/virtualFactory.ts"); +const { XAI_MODELS_CONFIG } = + await import("../../src/app/api/providers/[id]/models/discovery/providerModelsConfig.ts"); + +function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +test.beforeEach(() => resetStorage()); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = ORIGINAL_DATA_DIR; +}); + +test("isDeclaredNonChatModel only flags models that declare no chat-capable surface", () => { + const { isDeclaredNonChatModel } = virtualFactory; + assert.equal(isDeclaredNonChatModel({}), false); + assert.equal(isDeclaredNonChatModel({ supportedEndpoints: ["chat"] }), false); + assert.equal(isDeclaredNonChatModel({ supportedEndpoints: ["responses"] }), false); + assert.equal(isDeclaredNonChatModel({ supportedEndpoints: ["chat", "images"] }), false); + assert.equal(isDeclaredNonChatModel({ modelType: "chat" }), false); + assert.equal(isDeclaredNonChatModel({ supportedEndpoints: ["images"] }), true); + assert.equal(isDeclaredNonChatModel({ supportedEndpoints: ["videos"] }), true); + assert.equal(isDeclaredNonChatModel({ modelType: "embedding" }), true); +}); + +test("auto pool drops synced/custom models that declare only non-chat surfaces", async () => { + const conn = await providersDb.createProviderConnection({ + provider: "openrouter", + authType: "apikey", + name: "OpenRouter", + apiKey: "sk-test-openrouter", + }); + const connectionId = (conn as { id?: string }).id; + assert.ok(connectionId); + await modelsDb.replaceSyncedAvailableModelsForConnection("openrouter", connectionId, [ + { id: "chat-model", name: "Chat", source: "imported" as const }, + { + id: "responses-model", + name: "Responses", + source: "imported" as const, + supportedEndpoints: ["responses"], + }, + { + id: "image-model", + name: "Image", + source: "imported" as const, + supportedEndpoints: ["images"], + }, + { id: "embed-model", name: "Embed", source: "imported" as const, modelType: "embedding" }, + ]); + await modelsDb.addCustomModel("openrouter", "video-custom", "Video", "manual", "video", [ + "videos", + ]); + + const prepared = await virtualFactory.prepareVirtualAutoComboInputs(); + const ids = prepared.regularCandidates + .filter((c) => c.provider === "openrouter") + .map((c) => c.model) + .sort(); + assert.deepEqual(ids, ["chat-model", "responses-model"]); +}); + +test("an image-only sync does not fall back to the static catalog", async () => { + const conn = await providersDb.createProviderConnection({ + provider: "openrouter", + authType: "apikey", + name: "OpenRouter", + apiKey: "sk-test-openrouter", + }); + const connectionId = (conn as { id?: string }).id; + assert.ok(connectionId); + await modelsDb.replaceSyncedAvailableModelsForConnection("openrouter", connectionId, [ + { + id: "image-model", + name: "Image", + source: "imported" as const, + supportedEndpoints: ["images"], + }, + ]); + + const prepared = await virtualFactory.prepareVirtualAutoComboInputs(); + assert.equal(prepared.regularCandidates.filter((c) => c.provider === "openrouter").length, 0); +}); + +test("xAI discovery tags grok-imagine media models and leaves chat models untouched", () => { + const parsed = XAI_MODELS_CONFIG.parseResponse({ + data: [ + { id: "grok-4.7" }, + { id: "grok-imagine-image" }, + { id: "grok-imagine-image-quality" }, + { id: "grok-imagine-video-1.5" }, + ], + }) as Array>; + assert.deepEqual(parsed[0], { id: "grok-4.7" }); + assert.deepEqual(parsed[1].supportedEndpoints, ["images"]); + assert.deepEqual(parsed[2].supportedEndpoints, ["images"]); + assert.deepEqual(parsed[3].supportedEndpoints, ["videos"]); + assert.deepEqual(XAI_MODELS_CONFIG.parseResponse({}), []); +}); From 61191eba6a496ac30660352813bea4f9da41fa2b Mon Sep 17 00:00:00 2001 From: Tom Date: Wed, 23 Sep 2026 02:22:17 +0100 Subject: [PATCH 2/2] chore(changelog): add fragment for #14584 --- changelog.d/fixes/14584-auto-pool-skip-non-chat-models.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/fixes/14584-auto-pool-skip-non-chat-models.md diff --git a/changelog.d/fixes/14584-auto-pool-skip-non-chat-models.md b/changelog.d/fixes/14584-auto-pool-skip-non-chat-models.md new file mode 100644 index 000000000000..d59d720855a7 --- /dev/null +++ b/changelog.d/fixes/14584-auto-pool-skip-non-chat-models.md @@ -0,0 +1 @@ +- **fix(auto-combo):** keep models whose catalog declares only non-chat surfaces (image, video, audio, embedding, rerank) out of every `auto/*` candidate pool. Tag x.ai `grok-imagine-*` during discovery so they hit the same filter. Fixes the long-running leak of `grok-imagine-*`/`Wan2.2`/`suno/chirp-*` into `auto/coding` and `auto/smart` (#14584) — thanks @VIPKaiser