fix(auto-combo): skip non-chat models in virtual auto candidate pools

changelog.d/fixes/14584-auto-pool-skip-non-chat-models.md

@@ -0,0 +1 @@
+- **fix(auto-combo):** keep models whose catalog declares only non-chat surfaces (image, video, audio, embedding, rerank) out of every `auto/*` candidate pool. Tag x.ai `grok-imagine-*` during discovery so they hit the same filter. Fixes the long-running leak of `grok-imagine-*`/`Wan2.2`/`suno/chirp-*` into `auto/coding` and `auto/smart` (#14584) — thanks @VIPKaiser

open-sse/services/autoCombo/virtualFactory.ts

@@ -609,6 +609,23 @@ export async function attachPreparedCapabilityValues(
   return prepared;
 }
 
+// Endpoints that can serve a chat turn. A synced/custom model that declares only other
+// surfaces (images, videos, audio, embeddings, rerank) can never answer an auto/* chat
+// request, so it must not enter the candidate pool (the no-auth path already gates on
+// `serviceKinds`; this is the credentialed-provider equivalent).
+const CHAT_CAPABLE_ENDPOINTS = new Set(["chat", "responses"]);
+
+export function isDeclaredNonChatModel(model: {
+  modelType?: unknown;
+  supportedEndpoints?: unknown;
+}): boolean {
+  if (typeof model.modelType === "string" && model.modelType !== "chat") return true;
+  const endpoints = Array.isArray(model.supportedEndpoints)
+    ? model.supportedEndpoints.filter((e): e is string => typeof e === "string")
+    : [];
+  return endpoints.length > 0 && !endpoints.some((e) => CHAT_CAPABLE_ENDPOINTS.has(e));
+}
+
 export async function prepareVirtualAutoComboInputs(
   options: {
     includeResolvedCapabilities?: boolean;
@@ -694,17 +711,27 @@ export async function prepareVirtualAutoComboInputs(
     // already filters those out (catalog.ts, "Add custom models"); without the same
     // filter here every read below null-derefs and the whole auto/* pool fails to
     // materialize ("Could not materialize built-in auto model auto/<id>").
-    const customModels: Array<{ id?: string }> = (
-      Array.isArray(rawCustomModels) ? rawCustomModels : []
-    ).filter(
-      (model: unknown): model is { id?: string } =>
-        !!model && typeof model === "object" && !Array.isArray(model)
-    );
+    const customModels: Array<{ id?: string; modelType?: unknown; supportedEndpoints?: unknown }> =
+      (Array.isArray(rawCustomModels) ? rawCustomModels : []).filter(
+        (model: unknown): model is { id?: string } =>
+          !!model && typeof model === "object" && !Array.isArray(model)
+      );
     const userVisibleIds = new Set<string>();
+    const nonChatIds = new Set<string>();
     for (const models of Object.values(syncedByConnection)) {
-      for (const m of models) if (m.id && !hiddenModels?.has(m.id)) userVisibleIds.add(m.id);
+      for (const m of models) {
+        if (!m.id || hiddenModels?.has(m.id)) continue;
+        userVisibleIds.add(m.id);
+        if (isDeclaredNonChatModel(m)) nonChatIds.add(m.id);
+      }
+    }
+    for (const m of customModels) {
+      if (!m.id || hiddenModels?.has(m.id)) continue;
+      userVisibleIds.add(m.id);
+      if (isDeclaredNonChatModel(m)) nonChatIds.add(m.id);
     }
-    for (const m of customModels) if (m.id && !hiddenModels?.has(m.id)) userVisibleIds.add(m.id);
+    // Decided before dropping non-chat ids: an image-only sync must not fall back to
+    // the provider's static catalog.
     const hasUserModels = userVisibleIds.size > 0;
     const modelIds = hasUserModels
       ? Array.from(userVisibleIds)
@@ -717,6 +744,7 @@ export async function prepareVirtualAutoComboInputs(
         await yieldVirtualAutoPreparationTurn();
       }
       if (hiddenModels?.has(modelId)) continue;
+      if (nonChatIds.has(modelId)) continue;
 
       const allowedConnectionIds = providerConnections
         .filter((conn) => {

src/app/api/providers/[id]/models/discovery/providerModelsConfig.ts

@@ -401,13 +401,28 @@ const KIMI_CODING_MODELS_CONFIG: ProviderModelsConfigEntry = {
 // xai-oauth is not registered in PROVIDER_MODELS_CONFIG below and stays on
 // its frozen static seed (open-sse/config/providers/registry/xai/index.ts)
 // unless the flag is explicitly turned on.
+// x.ai /v1/models lists Grok Imagine media models next to the chat models without a
+// type field. Tag them so they stay out of chat catalogs and auto/* pools.
+function tagXaiMediaModel(model: unknown) {
+  if (!model || typeof model !== "object") return model;
+  const id = typeof (model as { id?: unknown }).id === "string" ? (model as { id: string }).id : "";
+  if (/^grok-imagine-image/i.test(id)) {
+    return { ...model, supportedEndpoints: ["images"], modelType: "image" };
+  }
+  if (/^grok-imagine-video/i.test(id)) return { ...model, supportedEndpoints: ["videos"] };
+  return model;
+}
+
 export const XAI_MODELS_CONFIG: ProviderModelsConfigEntry = {
   url: "https://api.x.ai/v1/models",
   method: "GET",
   headers: { "Content-Type": "application/json" },
   authHeader: "Authorization",
   authPrefix: "Bearer ",
-  parseResponse: (data) => data.data || data.models || [],
+  parseResponse: (data) => {
+    const models = data.data || data.models || [];
+    return Array.isArray(models) ? models.map(tagXaiMediaModel) : models;
+  },
 };
 
 /**

tests/unit/auto-pool-skip-non-chat-models.test.ts

@@ -0,0 +1,122 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+// Credentialed providers whose synced/custom catalog mixes chat and media models
+// (x.ai /v1/models lists grok-imagine-* next to grok-4.x; local nodes sync TTS/ASR/video)
+// must not put the media models into auto/* chat pools: they can only ever fail there.
+
+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-auto-non-chat-"));
+const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
+process.env.DATA_DIR = TEST_DATA_DIR;
+
+const core = await import("../../src/lib/db/core.ts");
+const providersDb = await import("../../src/lib/db/providers.ts");
+const modelsDb = await import("../../src/lib/db/models.ts");
+const virtualFactory = await import("../../open-sse/services/autoCombo/virtualFactory.ts");
+const { XAI_MODELS_CONFIG } =
+  await import("../../src/app/api/providers/[id]/models/discovery/providerModelsConfig.ts");
+
+function resetStorage() {
+  core.resetDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
+  fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
+}
+
+test.beforeEach(() => resetStorage());
+
+test.after(() => {
+  core.resetDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
+  if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR;
+  else process.env.DATA_DIR = ORIGINAL_DATA_DIR;
+});
+
+test("isDeclaredNonChatModel only flags models that declare no chat-capable surface", () => {
+  const { isDeclaredNonChatModel } = virtualFactory;
+  assert.equal(isDeclaredNonChatModel({}), false);
+  assert.equal(isDeclaredNonChatModel({ supportedEndpoints: ["chat"] }), false);
+  assert.equal(isDeclaredNonChatModel({ supportedEndpoints: ["responses"] }), false);
+  assert.equal(isDeclaredNonChatModel({ supportedEndpoints: ["chat", "images"] }), false);
+  assert.equal(isDeclaredNonChatModel({ modelType: "chat" }), false);
+  assert.equal(isDeclaredNonChatModel({ supportedEndpoints: ["images"] }), true);
+  assert.equal(isDeclaredNonChatModel({ supportedEndpoints: ["videos"] }), true);
+  assert.equal(isDeclaredNonChatModel({ modelType: "embedding" }), true);
+});
+
+test("auto pool drops synced/custom models that declare only non-chat surfaces", async () => {
+  const conn = await providersDb.createProviderConnection({
+    provider: "openrouter",
+    authType: "apikey",
+    name: "OpenRouter",
+    apiKey: "sk-test-openrouter",
+  });
+  const connectionId = (conn as { id?: string }).id;
+  assert.ok(connectionId);
+  await modelsDb.replaceSyncedAvailableModelsForConnection("openrouter", connectionId, [
+    { id: "chat-model", name: "Chat", source: "imported" as const },
+    {
+      id: "responses-model",
+      name: "Responses",
+      source: "imported" as const,
+      supportedEndpoints: ["responses"],
+    },
+    {
+      id: "image-model",
+      name: "Image",
+      source: "imported" as const,
+      supportedEndpoints: ["images"],
+    },
+    { id: "embed-model", name: "Embed", source: "imported" as const, modelType: "embedding" },
+  ]);
+  await modelsDb.addCustomModel("openrouter", "video-custom", "Video", "manual", "video", [
+    "videos",
+  ]);
+
+  const prepared = await virtualFactory.prepareVirtualAutoComboInputs();
+  const ids = prepared.regularCandidates
+    .filter((c) => c.provider === "openrouter")
+    .map((c) => c.model)
+    .sort();
+  assert.deepEqual(ids, ["chat-model", "responses-model"]);
+});
+
+test("an image-only sync does not fall back to the static catalog", async () => {
+  const conn = await providersDb.createProviderConnection({
+    provider: "openrouter",
+    authType: "apikey",
+    name: "OpenRouter",
+    apiKey: "sk-test-openrouter",
+  });
+  const connectionId = (conn as { id?: string }).id;
+  assert.ok(connectionId);
+  await modelsDb.replaceSyncedAvailableModelsForConnection("openrouter", connectionId, [
+    {
+      id: "image-model",
+      name: "Image",
+      source: "imported" as const,
+      supportedEndpoints: ["images"],
+    },
+  ]);
+
+  const prepared = await virtualFactory.prepareVirtualAutoComboInputs();
+  assert.equal(prepared.regularCandidates.filter((c) => c.provider === "openrouter").length, 0);
+});
+
+test("xAI discovery tags grok-imagine media models and leaves chat models untouched", () => {
+  const parsed = XAI_MODELS_CONFIG.parseResponse({
+    data: [
+      { id: "grok-4.7" },
+      { id: "grok-imagine-image" },
+      { id: "grok-imagine-image-quality" },
+      { id: "grok-imagine-video-1.5" },
+    ],
+  }) as Array<Record<string, unknown>>;
+  assert.deepEqual(parsed[0], { id: "grok-4.7" });
+  assert.deepEqual(parsed[1].supportedEndpoints, ["images"]);
+  assert.deepEqual(parsed[2].supportedEndpoints, ["images"]);
+  assert.deepEqual(parsed[3].supportedEndpoints, ["videos"]);
+  assert.deepEqual(XAI_MODELS_CONFIG.parseResponse({}), []);
+});