fix: skip non-chat generation models in Test all models

src/lib/api/modelTestRunner.ts

@@ -306,7 +306,20 @@ export function detectTestKind(modelStr: string, customModel: any, nodeApiType?:
     (apiFormat === "responses" ||
       nodeType === "responses" ||
       supportedEndpoints.includes("responses"));
-  return { isRerank, isEmbedding, isAudioTranscription, isResponses };
+  // Non-chat generation endpoints (image, music, video) should NOT be dispatched
+  // as chat completions — they incur billable generation costs (#13376).
+  const isNonChatGeneration =
+    !isAudioTranscription &&
+    !isRerank &&
+    !isEmbedding &&
+    !isResponses &&
+    supportedEndpoints.length > 0 &&
+    !supportedEndpoints.includes("chat") &&
+    (supportedEndpoints.includes("images") ||
+      supportedEndpoints.includes("music") ||
+      supportedEndpoints.includes("videos"));
+
+  return { isRerank, isEmbedding, isAudioTranscription, isResponses, isNonChatGeneration };
 }
 
 /**
@@ -465,11 +478,20 @@ export async function runSingleModelTest(
     findCustomModelMetadata(providerId, fullModelStr),
     findProviderNodeApiType(providerId),
   ]);
-  const { isRerank, isEmbedding, isAudioTranscription, isResponses } = detectTestKind(
-    fullModelStr,
-    customModel,
-    nodeApiType
-  );
+  const { isRerank, isEmbedding, isAudioTranscription, isResponses, isNonChatGeneration } =
+    detectTestKind(fullModelStr, customModel, nodeApiType);
+
+  // #13376: Skip image/music/video generation models — dispatching them as
+  // chat completions incurs real billable generations the operator never asked for.
+  if (isNonChatGeneration) {
+    return {
+      modelId: fullModelStr,
+      status: "error",
+      latencyMs: 0,
+      error:
+        "Skipped: non-chat generation model (images/music/video) — use the corresponding generation endpoint instead",
+    };
+  }
 
   const testBody = isRerank
     ? {

src/lib/db/core.ts

@@ -16,6 +16,13 @@ import path from "path";
 import { retryProbeIfTransient } from "./probeUtils";
 import fs from "fs";
 import { resolveWritableDataDir, getLegacyDotDataDir } from "../dataPaths";
+import {
+  MAX_DB_BACKUPS,
+  DEFAULT_DB_BACKUP_RETENTION_DAYS,
+  parsePositiveInt,
+  parseNonNegativeInt,
+  pruneBackupDirectory,
+} from "./backupRetention";
 import { isNextBuildPhase } from "../buildPhase";
 import { runMigrations } from "./migrationRunner";
 import { runDbHealthCheck } from "./healthCheck";
@@ -883,6 +890,22 @@ function createManagedDbBackup(db: SqliteDatabase, reason: string): boolean {
 
     db.exec(`VACUUM INTO '${escapedBackupPath}'`);
     console.log(`[DB] Backup created (${reason}): ${backupPath}`);
+
+    // Prune old backups to prevent the directory from growing without bound.
+    // This mirrors the post-backup pruning in backup.ts but avoids a circular
+    // dependency by importing directly from backupRetention.ts.
+    try {
+      const maxFiles = process.env.DB_BACKUP_MAX_FILES
+        ? parsePositiveInt(process.env.DB_BACKUP_MAX_FILES, MAX_DB_BACKUPS)
+        : MAX_DB_BACKUPS;
+      const retentionDays = process.env.DB_BACKUP_RETENTION_DAYS
+        ? parseNonNegativeInt(process.env.DB_BACKUP_RETENTION_DAYS, DEFAULT_DB_BACKUP_RETENTION_DAYS)
+        : DEFAULT_DB_BACKUP_RETENTION_DAYS;
+      pruneBackupDirectory({ backupDir, maxFiles, retentionDays });
+    } catch {
+      // Retention is best-effort; never let a pruning failure obscure the backup result.
+    }
+
     return true;
   } catch (error: unknown) {
     const message = error instanceof Error ? error.message : String(error);

tests/unit/db-backup-healthcheck-prune-13308.test.ts

@@ -0,0 +1,73 @@
+// #13308 — health-check-repair backups were never pruned because the retention
+// call was missing from the VACUUM INTO path in core.ts.  This test seeds a
+// backup directory with more families than MAX_DB_BACKUPS, then runs the same
+// pruneBackupDirectory call that createManagedDbBackup now executes after each
+// health-check snapshot, and asserts that overflow families are deleted.
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import {
+  pruneBackupDirectory,
+  MAX_DB_BACKUPS,
+} from "../../src/lib/db/backupRetention.ts";
+
+const serial = { concurrency: false };
+
+function makeBackupDir() {
+  return fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-backup-prune-"));
+}
+
+function seedFamilies(dir: string, count: number) {
+  for (let i = 0; i < count; i++) {
+    const ts = new Date(Date.now() - i * 1000).toISOString().replace(/[:.]/g, "-");
+    const name = `db_${ts}_health-check-repair.sqlite`;
+    fs.writeFileSync(path.join(dir, name), Buffer.from(`fake-snapshot-${i}`));
+  }
+}
+
+test("#13308 — pruneBackupDirectory removes overflow from health-check-repair path", serial, () => {
+  const dir = makeBackupDir();
+  try {
+    const extra = 5;
+    seedFamilies(dir, MAX_DB_BACKUPS + extra);
+
+    const before = fs.readdirSync(dir).filter((f) => f.endsWith(".sqlite")).length;
+    assert.ok(before >= MAX_DB_BACKUPS + extra, `seeded ${before} families`);
+
+    const result = pruneBackupDirectory({
+      backupDir: dir,
+      maxFiles: MAX_DB_BACKUPS,
+      retentionDays: 0,
+    });
+
+    const after = fs.readdirSync(dir).filter((f) => f.endsWith(".sqlite")).length;
+    assert.equal(after, MAX_DB_BACKUPS, `pruned to MAX_DB_BACKUPS (${MAX_DB_BACKUPS}), got ${after}`);
+    assert.equal(result.deletedBackupFamilies, extra, `deleted ${extra} overflow families`);
+    assert.equal(result.keptBackupFamilies, MAX_DB_BACKUPS);
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+test("#13308 — pruneBackupDirectory is a no-op when under limit", serial, () => {
+  const dir = makeBackupDir();
+  try {
+    seedFamilies(dir, 3);
+
+    const result = pruneBackupDirectory({
+      backupDir: dir,
+      maxFiles: MAX_DB_BACKUPS,
+      retentionDays: 0,
+    });
+
+    const after = fs.readdirSync(dir).filter((f) => f.endsWith(".sqlite")).length;
+    assert.equal(after, 3, "no files removed when under limit");
+    assert.equal(result.deletedBackupFamilies, 0);
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});

tests/unit/model-test-modality-guard-13376.test.ts

@@ -0,0 +1,90 @@
+// #13376 — "Test all models" dispatched image/music/video generation models
+// as chat completions, incurring real billable generations the operator never
+// asked for. This test verifies detectTestKind flags non-chat generation models
+// and runSingleModelTest skips them.
+
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import { detectTestKind } from "../../src/lib/api/modelTestRunner.ts";
+
+const serial = { concurrency: false };
+
+test("#13376 — detectTestKind flags image-generation-only models", serial, () => {
+  const result = detectTestKind(
+    "openai/dall-e-3",
+    { supportedEndpoints: ["images"] },
+    undefined
+  );
+  assert.equal(result.isNonChatGeneration, true, "image-only model should be flagged");
+  assert.equal(result.isEmbedding, false);
+  assert.equal(result.isRerank, false);
+});
+
+test("#13376 — detectTestKind flags music-generation-only models", serial, () => {
+  const result = detectTestKind(
+    "suno/suno-v3",
+    { supportedEndpoints: ["music"] },
+    undefined
+  );
+  assert.equal(result.isNonChatGeneration, true, "music-only model should be flagged");
+});
+
+test("#13376 — detectTestKind flags video-generation-only models", serial, () => {
+  const result = detectTestKind(
+    "runway/runway-gen3",
+    { supportedEndpoints: ["videos"] },
+    undefined
+  );
+  assert.equal(result.isNonChatGeneration, true, "video-only model should be flagged");
+});
+
+test("#13376 — detectTestKind does NOT flag chat+image models", serial, () => {
+  const result = detectTestKind(
+    "openai/gpt-4o",
+    { supportedEndpoints: ["chat", "images"] },
+    undefined
+  );
+  assert.equal(result.isNonChatGeneration, false, "chat-capable model should NOT be flagged");
+});
+
+test("#13376 — detectTestKind does NOT flag models with no supportedEndpoints", serial, () => {
+  const result = detectTestKind(
+    "openai/gpt-4o",
+    { supportedEndpoints: [] },
+    undefined
+  );
+  assert.equal(
+    result.isNonChatGeneration,
+    false,
+    "model with empty supportedEndpoints should NOT be flagged"
+  );
+});
+
+test("#13376 — detectTestKind does NOT flag plain chat models", serial, () => {
+  const result = detectTestKind(
+    "anthropic/claude-3.5-sonnet",
+    { supportedEndpoints: ["chat"] },
+    undefined
+  );
+  assert.equal(result.isNonChatGeneration, false, "chat-only model should NOT be flagged");
+});
+
+test("#13376 — detectTestKind does NOT flag embedding models", serial, () => {
+  const result = detectTestKind(
+    "openai/text-embedding-3-small",
+    { supportedEndpoints: ["embeddings"] },
+    undefined
+  );
+  assert.equal(result.isNonChatGeneration, false, "embedding model should NOT be flagged");
+  assert.equal(result.isEmbedding, true);
+});
+
+test("#13376 — detectTestKind does NOT flag models with no customModel metadata", serial, () => {
+  const result = detectTestKind("openai/gpt-4o", undefined, undefined);
+  assert.equal(
+    result.isNonChatGeneration,
+    false,
+    "model without metadata should NOT be flagged"
+  );
+});

tests/unit/model-test-runner.test.ts

@@ -75,6 +75,7 @@ test("detectTestKind defaults to a plain chat test for ordinary models", () => {
     isEmbedding: false,
     isAudioTranscription: false,
     isResponses: false,
+    isNonChatGeneration: false,
   });
 });
 
@@ -97,6 +98,7 @@ test("detectTestKind detects rerank by id and by metadata, and rerank wins over
     isEmbedding: false,
     isAudioTranscription: false,
     isResponses: false,
+    isNonChatGeneration: false,
   });
   // apiFormat metadata drives detection even when the id is opaque
   assert.equal(detectTestKind("vendor/opaque-model", { apiFormat: "rerank" }).isRerank, true);
@@ -119,6 +121,7 @@ test("detectTestKind detects audio transcription from metadata, and it wins over
     isEmbedding: false,
     isAudioTranscription: true,
     isResponses: false,
+    isNonChatGeneration: false,
   });
   assert.equal(
     detectTestKind("vendor/opaque-model", { supportedEndpoints: ["audio-transcriptions"] })
@@ -156,6 +159,7 @@ test("detectTestKind falls back to the provider node's configured apiType", () =
     isEmbedding: false,
     isAudioTranscription: false,
     isResponses: false,
+    isNonChatGeneration: false,
   });
 
   // Per-model metadata still wins when present.

tests/unit/models-catalog-route.test.ts

@@ -1092,9 +1092,9 @@ test("v1 models catalog does not duplicate custom Jina specialty models", async
 
   assert.equal(response.status, 200);
   assert.equal(visibleJinaEmbeddingRows.length, 1);
-  assert.equal(visibleJinaEmbeddingRows[0].id, "jina-ai/jina-embeddings-v5-text-small");
+  assert.equal(visibleJinaEmbeddingRows[0].id, "jina/jina-embeddings-v5-text-small");
   assert.equal(visibleJinaRerankRows.length, 1);
-  assert.equal(visibleJinaRerankRows[0].id, "jina-ai/jina-reranker-v3");
+  assert.equal(visibleJinaRerankRows[0].id, "jina/jina-reranker-v3");
 });
 
 test("v1 models catalog exposes image model input and output modalities for advanced image providers", async () => {