diff --git a/.gitignore b/.gitignore index efdd25c7e552..bb05ae4c4429 100644 --- a/.gitignore +++ b/.gitignore @@ -229,3 +229,6 @@ docs/prompts/AGENT-OWNERSHIP-PROTOCOL.md docs/prompts/AGENT-OWNERSHIP-PROTOCOL.omniroute-mim.md docs/prompts/AGENT-OWNERSHIP-PROTOCOL.omniroute-mid.md omniroute.md + +# mise configuration +mise.toml diff --git a/open-sse/executors/commandCode.ts b/open-sse/executors/commandCode.ts index 70bebeedf194..afd9811dd415 100644 --- a/open-sse/executors/commandCode.ts +++ b/open-sse/executors/commandCode.ts @@ -1,5 +1,6 @@ import { randomUUID } from "node:crypto"; +import { isVisionModelId } from "@/shared/constants/visionModels"; import { REGISTRY } from "../config/providerRegistry.ts"; import { BaseExecutor, mergeUpstreamExtraHeaders, type ExecuteInput } from "./base.ts"; @@ -55,6 +56,99 @@ function normalizeContentText(content: unknown): string { .join("\n"); } +/** + * Model id patterns for Command Code models that have `text, vision` + * capability per the official CC model registry, but are NOT caught + * by the shared {@link isVisionModelId} heuristic. Kept as a local + * set because these are CC-specific model IDs (vendor-prefix shapes + * like "moonshotai/Kimi-K2.6" or CC aliases like "gpt-5.4-mini"). + * + * Source: Command Code /alpha/generate model registry (docs). + */ +const CC_VISION_MODEL_PATTERNS: readonly RegExp[] = [ + // Open Source + /kimi-k2/i, // moonshotai/Kimi-K2.6, Kimi-K2.7-Code, Kimi-K2.5 + /qwen3\.\d/i, // Qwen/Qwen3.6-Plus, Qwen/Qwen3.7-Plus + /step-?3/i, // stepfun/Step-3.7-Flash + // Anthropic + /claude-fable/i, // claude-fable-5 (not covered by claude-opus/sonnet/haiku-4) + // OpenAI + /gpt-5/i, // gpt-5.5, gpt-5.4, gpt-5.3-codex, gpt-5.4-mini + // Sakana + /fugu/i, // sakana/fugu-ultra +]; + +/** + * Whether a model id routed through the Command Code executor is + * vision-capable. Checks Mimo-specific rules first, then CC-specific + * patterns, then falls through to the shared {@link isVisionModelId} + * heuristic (which covers minimax-m3, claude-3/4 families, gemini, + * gpt-4o/4.1, mistral-medium-3, and general "-vision" / "multimodal"). + */ +function isCommandCodeVisionModel(model?: string | null): boolean { + if (!model) return false; + // mimo-v2.5-pro is text-only — exclude before any positive check + if (/(?:^|\/)mimo-v2\.5-pro$/i.test(model)) return false; + // Only mimo-v2.5 and mimo-v2-omni accept images per Xiaomi vendor docs + if (/(?:^|\/)mimo-v2\.5$/i.test(model)) return true; + if (/(?:^|\/)mimo-v2-omni$/i.test(model)) return true; + // CC-specific patterns: Kimi K2, Qwen 3.x, Stepfun, Claude Fable, + // GPT-5, Sakana Fugu — not covered by the shared heuristic + if (CC_VISION_MODEL_PATTERNS.some((pattern) => pattern.test(model))) return true; + // Fall through: minimax-m3, claude-3/4, gemini-2/3, gpt-4o, -vision, multimodal + return isVisionModelId(model); +} + +/** + * Extract the image URL from an OpenAI-compatible or Command Code + * content part, returning undefined for non-image parts. + * + * OpenAI-compatible: { type: "image_url", image_url: { url: "..." } } + * Command Code CLI: { type: "image", image: "..." } + */ +function extractImageUrl(part: JsonRecord): string | undefined { + if (part.type === "image") return stringValue(part.image); + if (part.type === "image_url") { + if (isRecord(part.image_url)) return stringValue(part.image_url.url); + return stringValue(part.image_url); + } + return undefined; +} + +/** + * Convert an OpenAI-format content array to Command Code's internal + * CLI format. For vision-capable models (MiniMax M3, MiMo v2.5, etc.) + * this also preserves image parts alongside text. + */ +function convertUserContentParts(content: unknown, isVisionModel: boolean): string | unknown[] { + // For non-vision models or string content, extract text only. + if (!isVisionModel || typeof content === "string") { + return normalizeContentText(content); + } + + const parts: unknown[] = []; + for (const part of asRecordArray(content)) { + if (part.type === "text") { + const text = stringValue(part.text); + if (text) parts.push({ type: "text", text }); + continue; + } + const imgUrl = extractImageUrl(part); + if (imgUrl) { + parts.push({ type: "image", image: imgUrl }); + continue; + } + // Always drop tool_use / tool_result / thinking parts from user + // messages (Command Code doesn't accept them for role:"user"). + } + + // When every part was stripped, fall back to empty text so the + // message is still valid JSON (Command Code rejects empty content). + if (parts.length === 0) parts.push({ type: "text", text: "" }); + + return parts; +} + function convertTools(tools: unknown): unknown[] { return asRecordArray(tools).map((tool) => { const fn = isRecord(tool.function) ? tool.function : tool; @@ -86,11 +180,15 @@ function completeToolCallIds(messages: JsonRecord[]): Set { return new Set([...callIds].filter((id) => resultIds.has(id))); } -function convertMessages(messages: unknown): { system: string; messages: unknown[] } { +function convertMessages( + messages: unknown, + model?: string | null +): { system: string; messages: unknown[] } { const source = asRecordArray(messages); const pairedToolCallIds = completeToolCallIds(source); const out: unknown[] = []; const system: string[] = []; + const isVision = isCommandCodeVisionModel(model); for (const message of source) { const role = stringValue(message.role); @@ -101,7 +199,7 @@ function convertMessages(messages: unknown): { system: string; messages: unknown } if (role === "user") { - out.push({ role: "user", content: normalizeContentText(message.content) }); + out.push({ role: "user", content: convertUserContentParts(message.content, isVision) }); continue; } @@ -175,9 +273,6 @@ const COMMAND_CODE_PASSTHROUGH_FIELDS = [ function buildCommandCodeBody(model: string, body: unknown, stream = false): JsonRecord { const input = isRecord(body) ? body : {}; - const converted = convertMessages(input.messages); - const explicitSystem = typeof input.system === "string" ? input.system : ""; - const system = [converted.system, explicitSystem].filter(Boolean).join("\n\n"); // Payload rules may rewrite `body.model` (e.g. deepseek-v4-pro-max → // deepseek/deepseek-v4-pro for the command-code provider). Prefer the @@ -185,6 +280,10 @@ function buildCommandCodeBody(model: string, body: unknown, stream = false): Jso const resolvedModel = typeof input.model === "string" && input.model.trim().length > 0 ? input.model : model; + const converted = convertMessages(input.messages, resolvedModel); + const explicitSystem = typeof input.system === "string" ? input.system : ""; + const system = [converted.system, explicitSystem].filter(Boolean).join("\n\n"); + const params: JsonRecord = { model: resolvedModel, messages: converted.messages, diff --git a/tests/unit/command-code-vision.test.ts b/tests/unit/command-code-vision.test.ts new file mode 100644 index 000000000000..63fd4ceea34c --- /dev/null +++ b/tests/unit/command-code-vision.test.ts @@ -0,0 +1,508 @@ +/** + * Vision / multimodal support tests for the Command Code executor. + * + * Verifies that vision-capable models (MiniMax M3, MiMo V2.5, Kimi K2, Qwen 3.x, GPT-5, Claude 3/4, Fable 5, Gemini 3.x, Stepfun, Fugu, etc.) + * receive image parts in Command Code CLI format, while text-only + * models strip images as before (no regression). + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-cmd-code-vision-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const { getExecutor } = await import("../../open-sse/executors/index.ts"); +const core = await import("../../src/lib/db/core.ts"); + +const originalFetch = globalThis.fetch; + +function commandCodeStream(lines: unknown[]) { + const text = lines.map((l) => JSON.stringify(l)).join("\n") + "\n"; + return new Response(text, { status: 200, headers: { "Content-Type": "application/x-ndjson" } }); +} + +test.after(() => { + globalThis.fetch = originalFetch; + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test.afterEach(() => { + globalThis.fetch = originalFetch; +}); + +// ── helpers ──────────────────────────────────────────────────────────── + +type FetchCall = { url: string; init: Record; body: Record }; + +function captureFetch(response: Response) { + const calls: FetchCall[] = []; + globalThis.fetch = async (url, init: RequestInit = {}) => { + calls.push({ + url: String(url), + init: init as Record, + body: JSON.parse(String(init.body)), + }); + return response; + }; + return calls; +} + +function userContent(calls: FetchCall[]): unknown { + return ( + (calls[0].body.params as Record).messages as Record[] + )[0].content; +} + +// ── vision models: image parts preserved in CC CLI format ───────────── + +test("vision model minimax-m3 preserves image_url part as CC CLI {type:image}", async () => { + const calls = captureFetch( + commandCodeStream([{ type: "text-delta", text: "ok" }, { type: "finish" }]) + ); + + await getExecutor("command-code").execute({ + model: "MiniMaxAI/MiniMax-M3", + stream: false, + credentials: { apiKey: "cc_test_key" }, + body: { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "What's in this?" }, + { + type: "image_url", + image_url: { url: "data:image/png;base64,iVBORw0KGgo=" }, + }, + ], + }, + ], + }, + }); + + const content = userContent(calls); + assert.ok(Array.isArray(content), "vision model user content must be an array"); + const parts = content as Record[]; + assert.equal(parts.length, 2); + + // Text part preserved + assert.equal(parts[0].type, "text"); + assert.equal(parts[0].text, "What's in this?"); + + // Image part converted to CC CLI format + assert.equal(parts[1].type, "image"); + assert.equal(parts[1].image, "data:image/png;base64,iVBORw0KGgo="); +}); + +test("vision model minimax-m3 preserves image_url with HTTP URL", async () => { + const calls = captureFetch( + commandCodeStream([{ type: "text-delta", text: "ok" }, { type: "finish" }]) + ); + + await getExecutor("command-code").execute({ + model: "minimax-m3", + stream: false, + credentials: { apiKey: "cc_test_key" }, + body: { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Describe" }, + { + type: "image_url", + image_url: { url: "https://example.com/photo.jpg" }, + }, + ], + }, + ], + }, + }); + + const content = userContent(calls); + assert.ok(Array.isArray(content)); + const parts = content as Record[]; + assert.equal(parts.length, 2); + assert.equal(parts[1].type, "image"); + assert.equal(parts[1].image, "https://example.com/photo.jpg"); +}); + +test("vision model mimo-v2.5 preserves image parts", async () => { + const calls = captureFetch( + commandCodeStream([{ type: "text-delta", text: "ok" }, { type: "finish" }]) + ); + + await getExecutor("command-code").execute({ + model: "mimo-v2.5", + stream: false, + credentials: { apiKey: "cc_test_key" }, + body: { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Analyze" }, + { + type: "image_url", + image_url: { url: "https://example.com/img.png" }, + }, + ], + }, + ], + }, + }); + + const content = userContent(calls); + assert.ok(Array.isArray(content)); + const parts = content as Record[]; + assert.equal(parts.length, 2); + assert.equal(parts[1].type, "image"); +}); + +test("vision model mimo-v2.5-pro is text-only (no image parts)", async () => { + const calls = captureFetch( + commandCodeStream([{ type: "text-delta", text: "ok" }, { type: "finish" }]) + ); + + await getExecutor("command-code").execute({ + model: "mimo-v2.5-pro", + stream: false, + credentials: { apiKey: "cc_test_key" }, + body: { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Hi" }, + { + type: "image_url", + image_url: { url: "https://example.com/img.png" }, + }, + ], + }, + ], + }, + }); + + const content = userContent(calls); + // mimo-v2.5-pro is text-only — content must be flattened to a plain string + assert.equal(typeof content, "string"); + assert.equal(content, "Hi"); +}); + +test("vision model mimo-v2-omni preserves image parts", async () => { + const calls = captureFetch( + commandCodeStream([{ type: "text-delta", text: "ok" }, { type: "finish" }]) + ); + + await getExecutor("command-code").execute({ + model: "mimo-v2-omni", + stream: false, + credentials: { apiKey: "cc_test_key" }, + body: { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Check" }, + { + type: "image_url", + image_url: { url: "data:image/jpeg;base64,/9j/4AAQ=" }, + }, + ], + }, + ], + }, + }); + + const content = userContent(calls); + assert.ok(Array.isArray(content)); + const parts = content as Record[]; + assert.equal(parts.length, 2); + assert.equal(parts[1].type, "image"); + assert.equal(parts[1].image, "data:image/jpeg;base64,/9j/4AAQ="); +}); + +// ── non-vision models: images still stripped (no regression) ────────── + +test("text-only model deepseek-v4-pro strips image_url parts", async () => { + const calls = captureFetch( + commandCodeStream([{ type: "text-delta", text: "ok" }, { type: "finish" }]) + ); + + await getExecutor("command-code").execute({ + model: "deepseek/deepseek-v4-pro", + stream: false, + credentials: { apiKey: "cc_test_key" }, + body: { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Hello" }, + { + type: "image_url", + image_url: { url: "https://example.com/img.png" }, + }, + ], + }, + ], + }, + }); + + const content = userContent(calls); + // Non-vision model: content is a plain string, images stripped + assert.equal(typeof content, "string"); + assert.equal(content, "Hello"); +}); + +test("text-only model deepseek-v4-flash strips image_url parts (no regression)", async () => { + const calls = captureFetch( + commandCodeStream([{ type: "text-delta", text: "ok" }, { type: "finish" }]) + ); + + await getExecutor("command-code").execute({ + model: "deepseek/deepseek-v4-flash", + stream: false, + credentials: { apiKey: "cc_test_key" }, + body: { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Text only" }, + { + type: "image_url", + image_url: { url: "data:image/png;base64,AAA=" }, + }, + ], + }, + ], + }, + }); + + const content = userContent(calls); + assert.equal(typeof content, "string"); + assert.equal(content, "Text only"); +}); + +// ── edge cases ──────────────────────────────────────────────────────── + +test("vision model with only image content emits empty text fallback", async () => { + const calls = captureFetch( + commandCodeStream([{ type: "text-delta", text: "ok" }, { type: "finish" }]) + ); + + await getExecutor("command-code").execute({ + model: "minimax-m3", + stream: false, + credentials: { apiKey: "cc_test_key" }, + body: { + messages: [ + { + role: "user", + content: [ + { + type: "image_url", + image_url: { url: "data:image/png;base64,iVBOR=" }, + }, + ], + }, + ], + }, + }); + + const content = userContent(calls); + assert.ok(Array.isArray(content)); + const parts = content as Record[]; + // Single image part preserved — no empty text injected because + // the image itself keeps content non-empty. + assert.equal(parts.length, 1); + assert.equal(parts[0].type, "image"); + assert.equal(parts[0].image, "data:image/png;base64,iVBOR="); +}); + +test("vision model passes plain string content through unchanged", async () => { + const calls = captureFetch( + commandCodeStream([{ type: "text-delta", text: "ok" }, { type: "finish" }]) + ); + + await getExecutor("command-code").execute({ + model: "minimax-m3", + stream: false, + credentials: { apiKey: "cc_test_key" }, + body: { + messages: [{ role: "user", content: "Plain string message" }], + }, + }); + + const content = userContent(calls); + assert.equal(typeof content, "string"); + assert.equal(content, "Plain string message"); +}); + +test("vision model honors body.model rewrite for vision detection", async () => { + // #5166 scenario: body.model overwrites the execute model arg. + // Vision detection must use the rewritten model id. + const calls = captureFetch( + commandCodeStream([{ type: "text-delta", text: "ok" }, { type: "finish" }]) + ); + + // execute() gets a non-vision combo model, body.model rewrites to a vision model + await getExecutor("command-code").execute({ + model: "gpt-5.4-mini", + stream: false, + credentials: { apiKey: "cc_test_key" }, + body: { + model: "MiniMaxAI/MiniMax-M3", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Describe" }, + { + type: "image_url", + image_url: { url: "https://example.com/img.png" }, + }, + ], + }, + ], + }, + }); + + const content = userContent(calls); + // body.model = MiniMax-M3 (vision) → images preserved + assert.ok(Array.isArray(content)); + const parts = content as Record[]; + assert.equal(parts.length, 2); + assert.equal(parts[1].type, "image"); +}); + +test("vision model with multiple image parts preserves all of them", async () => { + const calls = captureFetch( + commandCodeStream([{ type: "text-delta", text: "ok" }, { type: "finish" }]) + ); + + await getExecutor("command-code").execute({ + model: "minimax-m3", + stream: false, + credentials: { apiKey: "cc_test_key" }, + body: { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Compare" }, + { + type: "image_url", + image_url: { url: "https://example.com/a.jpg" }, + }, + { + type: "image_url", + image_url: { url: "https://example.com/b.jpg" }, + }, + ], + }, + ], + }, + }); + + const content = userContent(calls); + assert.ok(Array.isArray(content)); + const parts = content as Record[]; + assert.equal(parts.length, 3); + assert.equal(parts[0].type, "text"); + assert.equal(parts[1].type, "image"); + assert.equal(parts[1].image, "https://example.com/a.jpg"); + assert.equal(parts[2].type, "image"); + assert.equal(parts[2].image, "https://example.com/b.jpg"); +}); + +test("vision model with image_url as plain string (no object wrapper) still works", async () => { + const calls = captureFetch( + commandCodeStream([{ type: "text-delta", text: "ok" }, { type: "finish" }]) + ); + + await getExecutor("command-code").execute({ + model: "minimax-m3", + stream: false, + credentials: { apiKey: "cc_test_key" }, + body: { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Look" }, + { + type: "image_url", + image_url: "https://example.com/img.png", + }, + ], + }, + ], + }, + }); + + const content = userContent(calls); + assert.ok(Array.isArray(content)); + const parts = content as Record[]; + assert.equal(parts.length, 2); + assert.equal(parts[1].type, "image"); + assert.equal(parts[1].image, "https://example.com/img.png"); +}); + +// ── CC vision models (Command Code docs registry) ────────────────── + +const VISION_CASES = [ + ["Kimi K2.6", "moonshotai/Kimi-K2.6"], + ["Kimi K2.7 Code", "moonshotai/Kimi-K2.7-Code"], + ["Kimi K2.5", "moonshotai/Kimi-K2.5"], + ["Qwen 3.6 Plus", "Qwen/Qwen3.6-Plus"], + ["Qwen 3.7 Plus", "Qwen/Qwen3.7-Plus"], + ["Step 3.7 Flash", "stepfun/Step-3.7-Flash"], + ["GPT-5.5", "gpt-5.5"], + ["GPT-5.4", "gpt-5.4"], + ["GPT-5.3 Codex", "gpt-5.3-codex"], + ["GPT-5.4 Mini", "gpt-5.4-mini"], + ["Claude Fable 5", "claude-fable-5"], + ["Sakana Fugu Ultra", "sakana/fugu-ultra"], + ["Claude Opus 4.7 (isVisionModelId)", "claude-opus-4-7"], + ["Claude Sonnet 4.6 (isVisionModelId)", "claude-sonnet-4-6"], + ["Gemini 3.5 Flash (isVisionModelId)", "google/gemini-3.5-flash"], +]; + +for (const [name, model] of VISION_CASES) { + test(`vision model ${name} preserves image parts`, async () => { + const calls = captureFetch( + commandCodeStream([{ type: "text-delta", text: "ok" }, { type: "finish" }]) + ); + await getExecutor("command-code").execute({ + model, + stream: false, + credentials: { apiKey: "cc_test_key" }, + body: { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Check" }, + { + type: "image_url", + image_url: { url: "https://example.com/img.png" }, + }, + ], + }, + ], + }, + }); + const content = userContent(calls); + assert.ok(Array.isArray(content), `${name} user content must be an array`); + const parts = content; + assert.equal(parts.length, 2); + assert.equal(parts[1].type, "image"); + }); +}