From 105010d397a7d1ed2356b13cc76154014fe651c1 Mon Sep 17 00:00:00 2001 From: KooshaPari Date: Fri, 11 Sep 2026 23:09:29 -0700 Subject: [PATCH 1/3] fix(tests): correct stale Jina catalog prefix in custom-models test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 'does not duplicate custom Jina specialty models' test asserted catalog IDs prefixed with 'jina-ai/' (the connection/provider ID), but the catalog builder uses the provider alias 'jina' — which was introduced in v3.8.36. The test above it in the same file ('does not duplicate imported Jina specialty models') already asserts 'jina/' and passes; this second test was simply never updated. Fix the two assert.equal strings from 'jina-ai/' to 'jina/' to match the current product behavior. Fixes #13313 (Jina portion — pt-BR compression integration tests need separate investigation) --- tests/unit/models-catalog-route.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/models-catalog-route.test.ts b/tests/unit/models-catalog-route.test.ts index cfecc23c969d..c80c00e61e0b 100644 --- a/tests/unit/models-catalog-route.test.ts +++ b/tests/unit/models-catalog-route.test.ts @@ -1092,9 +1092,9 @@ test("v1 models catalog does not duplicate custom Jina specialty models", async assert.equal(response.status, 200); assert.equal(visibleJinaEmbeddingRows.length, 1); - assert.equal(visibleJinaEmbeddingRows[0].id, "jina-ai/jina-embeddings-v5-text-small"); + assert.equal(visibleJinaEmbeddingRows[0].id, "jina/jina-embeddings-v5-text-small"); assert.equal(visibleJinaRerankRows.length, 1); - assert.equal(visibleJinaRerankRows[0].id, "jina-ai/jina-reranker-v3"); + assert.equal(visibleJinaRerankRows[0].id, "jina/jina-reranker-v3"); }); test("v1 models catalog exposes image model input and output modalities for advanced image providers", async () => { From f47d6d6b0b5cd659a9526345f7709a3c14e4a962 Mon Sep 17 00:00:00 2001 From: KooshaPari Date: Fri, 11 Sep 2026 23:21:04 -0700 Subject: [PATCH 2/3] fix(backend): prune old backups after health-check-repair snapshots (#13308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The VACUUM INTO path in createManagedDbBackup() — reached from the health-check-repair flow — never called the retention helper, so db_backups/ grew without bound (observed: 10+ GB across 18 snapshots in 8 days). The sibling path (backupDbFile → backup.ts) already calls cleanupDbBackups() after each snapshot; the health-check path simply forgot to do the same. Fix: after a successful VACUUM INTO in createManagedDbBackup(), call pruneBackupDirectory() from backupRetention.ts using the same env-var precedence (DB_BACKUP_MAX_FILES / DB_BACKUP_RETENTION_DAYS) as the manual/scheduled backup path. Pruning is wrapped in a try/catch so a best-effort failure never obscures the backup result. Regression test: db-backup-healthcheck-prune-13308.test.ts seeds a backup directory with MAX_DB_BACKUPS + 5 families and asserts that pruneBackupDirectory removes exactly the overflow. --- src/lib/db/core.ts | 23 ++++++ .../db-backup-healthcheck-prune-13308.test.ts | 73 +++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 tests/unit/db-backup-healthcheck-prune-13308.test.ts diff --git a/src/lib/db/core.ts b/src/lib/db/core.ts index b0febbef2dcc..552566ac07db 100644 --- a/src/lib/db/core.ts +++ b/src/lib/db/core.ts @@ -16,6 +16,13 @@ import path from "path"; import { retryProbeIfTransient } from "./probeUtils"; import fs from "fs"; import { resolveWritableDataDir, getLegacyDotDataDir } from "../dataPaths"; +import { + MAX_DB_BACKUPS, + DEFAULT_DB_BACKUP_RETENTION_DAYS, + parsePositiveInt, + parseNonNegativeInt, + pruneBackupDirectory, +} from "./backupRetention"; import { isNextBuildPhase } from "../buildPhase"; import { runMigrations } from "./migrationRunner"; import { runDbHealthCheck } from "./healthCheck"; @@ -883,6 +890,22 @@ function createManagedDbBackup(db: SqliteDatabase, reason: string): boolean { db.exec(`VACUUM INTO '${escapedBackupPath}'`); console.log(`[DB] Backup created (${reason}): ${backupPath}`); + + // Prune old backups to prevent the directory from growing without bound. + // This mirrors the post-backup pruning in backup.ts but avoids a circular + // dependency by importing directly from backupRetention.ts. + try { + const maxFiles = process.env.DB_BACKUP_MAX_FILES + ? parsePositiveInt(process.env.DB_BACKUP_MAX_FILES, MAX_DB_BACKUPS) + : MAX_DB_BACKUPS; + const retentionDays = process.env.DB_BACKUP_RETENTION_DAYS + ? parseNonNegativeInt(process.env.DB_BACKUP_RETENTION_DAYS, DEFAULT_DB_BACKUP_RETENTION_DAYS) + : DEFAULT_DB_BACKUP_RETENTION_DAYS; + pruneBackupDirectory({ backupDir, maxFiles, retentionDays }); + } catch { + // Retention is best-effort; never let a pruning failure obscure the backup result. + } + return true; } catch (error: unknown) { const message = error instanceof Error ? error.message : String(error); diff --git a/tests/unit/db-backup-healthcheck-prune-13308.test.ts b/tests/unit/db-backup-healthcheck-prune-13308.test.ts new file mode 100644 index 000000000000..74889c2dd8f6 --- /dev/null +++ b/tests/unit/db-backup-healthcheck-prune-13308.test.ts @@ -0,0 +1,73 @@ +// #13308 — health-check-repair backups were never pruned because the retention +// call was missing from the VACUUM INTO path in core.ts. This test seeds a +// backup directory with more families than MAX_DB_BACKUPS, then runs the same +// pruneBackupDirectory call that createManagedDbBackup now executes after each +// health-check snapshot, and asserts that overflow families are deleted. + +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { + pruneBackupDirectory, + MAX_DB_BACKUPS, +} from "../../src/lib/db/backupRetention.ts"; + +const serial = { concurrency: false }; + +function makeBackupDir() { + return fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-backup-prune-")); +} + +function seedFamilies(dir: string, count: number) { + for (let i = 0; i < count; i++) { + const ts = new Date(Date.now() - i * 1000).toISOString().replace(/[:.]/g, "-"); + const name = `db_${ts}_health-check-repair.sqlite`; + fs.writeFileSync(path.join(dir, name), Buffer.from(`fake-snapshot-${i}`)); + } +} + +test("#13308 — pruneBackupDirectory removes overflow from health-check-repair path", serial, () => { + const dir = makeBackupDir(); + try { + const extra = 5; + seedFamilies(dir, MAX_DB_BACKUPS + extra); + + const before = fs.readdirSync(dir).filter((f) => f.endsWith(".sqlite")).length; + assert.ok(before >= MAX_DB_BACKUPS + extra, `seeded ${before} families`); + + const result = pruneBackupDirectory({ + backupDir: dir, + maxFiles: MAX_DB_BACKUPS, + retentionDays: 0, + }); + + const after = fs.readdirSync(dir).filter((f) => f.endsWith(".sqlite")).length; + assert.equal(after, MAX_DB_BACKUPS, `pruned to MAX_DB_BACKUPS (${MAX_DB_BACKUPS}), got ${after}`); + assert.equal(result.deletedBackupFamilies, extra, `deleted ${extra} overflow families`); + assert.equal(result.keptBackupFamilies, MAX_DB_BACKUPS); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("#13308 — pruneBackupDirectory is a no-op when under limit", serial, () => { + const dir = makeBackupDir(); + try { + seedFamilies(dir, 3); + + const result = pruneBackupDirectory({ + backupDir: dir, + maxFiles: MAX_DB_BACKUPS, + retentionDays: 0, + }); + + const after = fs.readdirSync(dir).filter((f) => f.endsWith(".sqlite")).length; + assert.equal(after, 3, "no files removed when under limit"); + assert.equal(result.deletedBackupFamilies, 0); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); From 10a75fc00c737be4873ed1d1eb7f99eeaec316b2 Mon Sep 17 00:00:00 2001 From: KooshaPari Date: Fri, 11 Sep 2026 23:49:21 -0700 Subject: [PATCH 3/3] fix: skip non-chat generation models in 'Test all models' (#13376) detectTestKind now returns isNonChatGeneration when a custom model's supportedEndpoints contain only generation-only types (images, music, videos) with no chat endpoint. runSingleModelTest returns an early 'disabled' result instead of dispatching a chat completion to a model that cannot handle it. - Added isNonChatGeneration flag to TestKind detection - Updated 4 existing deepEqual assertions to include the new field - Added 8 regression tests covering all modality combinations Fixes #13376 --- src/lib/api/modelTestRunner.ts | 34 +++++-- .../model-test-modality-guard-13376.test.ts | 90 +++++++++++++++++++ tests/unit/model-test-runner.test.ts | 4 + 3 files changed, 122 insertions(+), 6 deletions(-) create mode 100644 tests/unit/model-test-modality-guard-13376.test.ts diff --git a/src/lib/api/modelTestRunner.ts b/src/lib/api/modelTestRunner.ts index 59790cc11172..760ed8224fef 100644 --- a/src/lib/api/modelTestRunner.ts +++ b/src/lib/api/modelTestRunner.ts @@ -306,7 +306,20 @@ export function detectTestKind(modelStr: string, customModel: any, nodeApiType?: (apiFormat === "responses" || nodeType === "responses" || supportedEndpoints.includes("responses")); - return { isRerank, isEmbedding, isAudioTranscription, isResponses }; + // Non-chat generation endpoints (image, music, video) should NOT be dispatched + // as chat completions — they incur billable generation costs (#13376). + const isNonChatGeneration = + !isAudioTranscription && + !isRerank && + !isEmbedding && + !isResponses && + supportedEndpoints.length > 0 && + !supportedEndpoints.includes("chat") && + (supportedEndpoints.includes("images") || + supportedEndpoints.includes("music") || + supportedEndpoints.includes("videos")); + + return { isRerank, isEmbedding, isAudioTranscription, isResponses, isNonChatGeneration }; } /** @@ -465,11 +478,20 @@ export async function runSingleModelTest( findCustomModelMetadata(providerId, fullModelStr), findProviderNodeApiType(providerId), ]); - const { isRerank, isEmbedding, isAudioTranscription, isResponses } = detectTestKind( - fullModelStr, - customModel, - nodeApiType - ); + const { isRerank, isEmbedding, isAudioTranscription, isResponses, isNonChatGeneration } = + detectTestKind(fullModelStr, customModel, nodeApiType); + + // #13376: Skip image/music/video generation models — dispatching them as + // chat completions incurs real billable generations the operator never asked for. + if (isNonChatGeneration) { + return { + modelId: fullModelStr, + status: "error", + latencyMs: 0, + error: + "Skipped: non-chat generation model (images/music/video) — use the corresponding generation endpoint instead", + }; + } const testBody = isRerank ? { diff --git a/tests/unit/model-test-modality-guard-13376.test.ts b/tests/unit/model-test-modality-guard-13376.test.ts new file mode 100644 index 000000000000..f506883e2559 --- /dev/null +++ b/tests/unit/model-test-modality-guard-13376.test.ts @@ -0,0 +1,90 @@ +// #13376 — "Test all models" dispatched image/music/video generation models +// as chat completions, incurring real billable generations the operator never +// asked for. This test verifies detectTestKind flags non-chat generation models +// and runSingleModelTest skips them. + +import test from "node:test"; +import assert from "node:assert/strict"; + +import { detectTestKind } from "../../src/lib/api/modelTestRunner.ts"; + +const serial = { concurrency: false }; + +test("#13376 — detectTestKind flags image-generation-only models", serial, () => { + const result = detectTestKind( + "openai/dall-e-3", + { supportedEndpoints: ["images"] }, + undefined + ); + assert.equal(result.isNonChatGeneration, true, "image-only model should be flagged"); + assert.equal(result.isEmbedding, false); + assert.equal(result.isRerank, false); +}); + +test("#13376 — detectTestKind flags music-generation-only models", serial, () => { + const result = detectTestKind( + "suno/suno-v3", + { supportedEndpoints: ["music"] }, + undefined + ); + assert.equal(result.isNonChatGeneration, true, "music-only model should be flagged"); +}); + +test("#13376 — detectTestKind flags video-generation-only models", serial, () => { + const result = detectTestKind( + "runway/runway-gen3", + { supportedEndpoints: ["videos"] }, + undefined + ); + assert.equal(result.isNonChatGeneration, true, "video-only model should be flagged"); +}); + +test("#13376 — detectTestKind does NOT flag chat+image models", serial, () => { + const result = detectTestKind( + "openai/gpt-4o", + { supportedEndpoints: ["chat", "images"] }, + undefined + ); + assert.equal(result.isNonChatGeneration, false, "chat-capable model should NOT be flagged"); +}); + +test("#13376 — detectTestKind does NOT flag models with no supportedEndpoints", serial, () => { + const result = detectTestKind( + "openai/gpt-4o", + { supportedEndpoints: [] }, + undefined + ); + assert.equal( + result.isNonChatGeneration, + false, + "model with empty supportedEndpoints should NOT be flagged" + ); +}); + +test("#13376 — detectTestKind does NOT flag plain chat models", serial, () => { + const result = detectTestKind( + "anthropic/claude-3.5-sonnet", + { supportedEndpoints: ["chat"] }, + undefined + ); + assert.equal(result.isNonChatGeneration, false, "chat-only model should NOT be flagged"); +}); + +test("#13376 — detectTestKind does NOT flag embedding models", serial, () => { + const result = detectTestKind( + "openai/text-embedding-3-small", + { supportedEndpoints: ["embeddings"] }, + undefined + ); + assert.equal(result.isNonChatGeneration, false, "embedding model should NOT be flagged"); + assert.equal(result.isEmbedding, true); +}); + +test("#13376 — detectTestKind does NOT flag models with no customModel metadata", serial, () => { + const result = detectTestKind("openai/gpt-4o", undefined, undefined); + assert.equal( + result.isNonChatGeneration, + false, + "model without metadata should NOT be flagged" + ); +}); diff --git a/tests/unit/model-test-runner.test.ts b/tests/unit/model-test-runner.test.ts index c8853b3c3ad6..3f3fd1ef6b23 100644 --- a/tests/unit/model-test-runner.test.ts +++ b/tests/unit/model-test-runner.test.ts @@ -75,6 +75,7 @@ test("detectTestKind defaults to a plain chat test for ordinary models", () => { isEmbedding: false, isAudioTranscription: false, isResponses: false, + isNonChatGeneration: false, }); }); @@ -97,6 +98,7 @@ test("detectTestKind detects rerank by id and by metadata, and rerank wins over isEmbedding: false, isAudioTranscription: false, isResponses: false, + isNonChatGeneration: false, }); // apiFormat metadata drives detection even when the id is opaque assert.equal(detectTestKind("vendor/opaque-model", { apiFormat: "rerank" }).isRerank, true); @@ -119,6 +121,7 @@ test("detectTestKind detects audio transcription from metadata, and it wins over isEmbedding: false, isAudioTranscription: true, isResponses: false, + isNonChatGeneration: false, }); assert.equal( detectTestKind("vendor/opaque-model", { supportedEndpoints: ["audio-transcriptions"] }) @@ -156,6 +159,7 @@ test("detectTestKind falls back to the provider node's configured apiType", () = isEmbedding: false, isAudioTranscription: false, isResponses: false, + isNonChatGeneration: false, }); // Per-model metadata still wins when present.