diff --git a/changelog.d/fixes/11822-playground-persisted-combo-simulation.md b/changelog.d/fixes/11822-playground-persisted-combo-simulation.md new file mode 100644 index 000000000000..7fc58cf57f86 --- /dev/null +++ b/changelog.d/fixes/11822-playground-persisted-combo-simulation.md @@ -0,0 +1 @@ +- Fix persisted model steps returning an error in the playground route simulator, and warn when structural steps cannot be simulated. diff --git a/src/app/api/playground/simulate-route/route.ts b/src/app/api/playground/simulate-route/route.ts index 184803d13ad1..993f73d2e9b6 100644 --- a/src/app/api/playground/simulate-route/route.ts +++ b/src/app/api/playground/simulate-route/route.ts @@ -152,7 +152,33 @@ export async function POST(request: Request) { errors.push(`Combo "${body.comboId}" not found.`); return NextResponse.json({ error: "Combo not found" }, { status: 404 }); } - const targets = typeof combo.targets === "string" ? JSON.parse(combo.targets) : combo.targets; + const persistedSteps = Array.isArray(combo.models) ? combo.models : []; + const modelSteps = persistedSteps.filter( + (step: any) => + typeof step === "string" || + ((step.kind === undefined || step.kind === "model") && typeof step.model === "string") + ); + const unsupportedStepCount = persistedSteps.length - modelSteps.length; + if (unsupportedStepCount > 0) { + warnings.push( + `Skipped ${unsupportedStepCount} unsupported persisted combo ${unsupportedStepCount === 1 ? "step" : "steps"}.` + ); + } + const targets = modelSteps.map((step: any) => { + const value = typeof step === "string" ? step : step.model; + const separator = value.indexOf("/"); + const parsedProvider = separator === -1 ? undefined : value.slice(0, separator); + const parsedModel = separator === -1 ? value : value.slice(separator + 1); + + return { + provider: + typeof step === "string" + ? parsedProvider || "unknown" + : step.providerId || step.provider || parsedProvider || "unknown", + model: parsedModel, + weight: typeof step === "string" ? undefined : step.weight, + }; + }); comboInfo = { name: combo.name, strategy: combo.strategy, targets }; } else if (body.combo) { comboInfo = body.combo; @@ -172,7 +198,11 @@ export async function POST(request: Request) { for (let i = 0; i < comboInfo.targets.length; i++) { const t = comboInfo.targets[i]; const conn = connections.find( - (c: any) => c.id === t.provider || c.name === t.provider || c.displayName === t.provider + (c: any) => + c.provider === t.provider || + c.id === t.provider || + c.name === t.provider || + c.displayName === t.provider ); const cost = estimateCost(t.model, promptTokens); const latency = estimateLatency(t.model); diff --git a/tests/unit/playground-simulate-route-persisted-combo.test.ts b/tests/unit/playground-simulate-route-persisted-combo.test.ts new file mode 100644 index 000000000000..180f0793db6c --- /dev/null +++ b/tests/unit/playground-simulate-route-persisted-combo.test.ts @@ -0,0 +1,104 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-playground-simulate-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const combosDb = await import("../../src/lib/db/combos.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const { POST } = await import("../../src/app/api/playground/simulate-route/route.ts"); + +let persistedComboId: string; + +test.beforeEach(async () => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); + await providersDb.createProviderConnection({ + provider: "cc", + authType: "no-auth", + name: "Claude Code", + }); + await providersDb.createProviderConnection({ + provider: "anthropic", + authType: "no-auth", + name: "Anthropic", + }); + const combo = await combosDb.createCombo({ + name: "persisted combo", + strategy: "priority", + models: [ + { kind: "model", model: "cc/claude-opus-5", fallbackOnlyOnQuotaExhaustion: true }, + { kind: "model", providerId: "anthropic", model: "anthropic/claude-opus-5" }, + { kind: "combo-ref", comboName: "nested combo" }, + ], + }); + persistedComboId = combo.id; +}); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +function request(body: unknown): Request { + return new Request("http://localhost/api/playground/simulate-route", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); +} + +test("simulates persisted combo model steps in order", async () => { + const response = await POST(request({ comboId: persistedComboId, promptTokens: 500 })); + const body = await response.json(); + + assert.equal(response.status, 200); + assert.deepEqual( + body.targets.map(({ provider, model, rank }: Record) => ({ + provider, + model, + rank, + })), + [ + { provider: "cc", model: "claude-opus-5", rank: 1 }, + { provider: "anthropic", model: "claude-opus-5", rank: 2 }, + ] + ); + assert.deepEqual( + body.targets.map(({ status }: Record) => status), + ["available", "available"] + ); + assert.ok(body.warnings.includes("Skipped 1 unsupported persisted combo step.")); + assert.ok(body.warnings.every((warning: string) => !warning.includes("not configured"))); +}); + +test("returns 404 for a missing persisted combo", async () => { + const response = await POST(request({ comboId: "missing" })); + + assert.equal(response.status, 404); + assert.deepEqual(await response.json(), { error: "Combo not found" }); +}); + +test("preserves inline combo simulation", async () => { + const response = await POST( + request({ + combo: { + name: "inline combo", + strategy: "priority", + targets: [{ provider: "cc", model: "claude-opus-5" }], + }, + }) + ); + const body = await response.json(); + + assert.equal(response.status, 200); + assert.deepEqual( + body.targets.map(({ provider, model }: Record) => ({ provider, model })), + [{ provider: "cc", model: "claude-opus-5" }] + ); +});