diff --git a/tests/unit/characterization-cloudAgent.test.ts b/tests/unit/characterization-cloudAgent.test.ts new file mode 100644 index 000000000000..79cb067cca06 --- /dev/null +++ b/tests/unit/characterization-cloudAgent.test.ts @@ -0,0 +1,646 @@ +/** + * Characterization — src/lib/cloudAgent (rail 3.8.55, pre-v4 extraction to @omniroute/mod-*). + * + * Pins the PUBLIC surface and the observable behaviour of the cloud-agent subsystem as it is + * today, so the v4 extraction can be proven byte-for-byte. These tests describe current + * behaviour, not desired behaviour: when a quirk is pinned the test name says so + * ("characterization: … currently …"). Updating a surface snapshot is a conscious decision — + * never regenerate the literal lists automatically. + * + * External consumers (the real boundary): src/app/api/v1/agents/{tasks,tasks/[id],credentials, + * health}/route.ts, the orchestration dashboard (type-only imports of CloudAgentTask) and + * src/lib/providers/validation/webProvidersB.ts (buildJulesApiUrl). + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-char-cloudagent-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "char-cloudagent-test-secret"; + +const core = await import("../../src/lib/db/core.ts"); +const indexMod = await import("../../src/lib/cloudAgent/index.ts"); +const registryMod = await import("../../src/lib/cloudAgent/registry.ts"); +const baseAgentMod = await import("../../src/lib/cloudAgent/baseAgent.ts"); +const typesMod = await import("../../src/lib/cloudAgent/types.ts"); +const dbMod = await import("../../src/lib/cloudAgent/db.ts"); +const apiMod = await import("../../src/lib/cloudAgent/api.ts"); +const credentialsMod = await import("../../src/lib/cloudAgent/credentials.ts"); +const julesApiMod = await import("../../src/lib/cloudAgent/julesApi.ts"); +const julesMod = await import("../../src/lib/cloudAgent/agents/jules.ts"); +const devinMod = await import("../../src/lib/cloudAgent/agents/devin.ts"); +const codexMod = await import("../../src/lib/cloudAgent/agents/codex.ts"); +const cursorMod = await import("../../src/lib/cloudAgent/agents/cursor.ts"); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +const CREDS = { apiKey: "key_char_123" }; +const SOURCE = { repoName: "acme/widgets", repoUrl: "https://github.com/acme/widgets" }; + +type FetchCall = { url: string; init?: RequestInit }; + +function withFakeFetch( + handler: (url: string, init?: RequestInit) => Response | Promise +): { calls: FetchCall[]; restore: () => void } { + const original = globalThis.fetch; + const calls: FetchCall[] = []; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = String(input); + calls.push({ url, init }); + return handler(url, init); + }) as typeof fetch; + return { + calls, + restore: () => { + globalThis.fetch = original; + }, + }; +} + +function headerOf(init: RequestInit | undefined, name: string): string | undefined { + const headers = (init?.headers ?? {}) as Record; + return headers[name]; +} + +// ─── (a) Surface snapshots ─────────────────────────────────────────────────── + +test("surface: cloudAgent/index.ts barrel exports", () => { + assert.deepEqual(Object.keys(indexMod).sort(), [ + "CLOUD_AGENT_STATUS", + "CloudAgentBase", + "CloudAgentSourceSchema", + "CloudAgentStatusSchema", + "CloudAgentTaskOptionsSchema", + "CodexCloudAgent", + "CreateCloudAgentTaskSchema", + "CursorCloudAgent", + "DevinAgent", + "JulesAgent", + "UpdateCloudAgentTaskSchema", + "createCloudAgentTaskTable", + "deleteCloudAgentTask", + "getAgent", + "getAllCloudAgentTasks", + "getAvailableAgents", + "getCloudAgentTaskById", + "getCloudAgentTasksByProvider", + "getCloudAgentTasksByStatus", + "insertCloudAgentTask", + "isCloudAgentProvider", + "updateCloudAgentTask", + ]); +}); + +test("surface: cloudAgent per-module runtime exports", () => { + const surfaces: Record = { + registry: Object.keys(registryMod).sort(), + baseAgent: Object.keys(baseAgentMod).sort(), + types: Object.keys(typesMod).sort(), + db: Object.keys(dbMod).sort(), + api: Object.keys(apiMod).sort(), + credentials: Object.keys(credentialsMod).sort(), + julesApi: Object.keys(julesApiMod).sort(), + jules: Object.keys(julesMod).sort(), + devin: Object.keys(devinMod).sort(), + codex: Object.keys(codexMod).sort(), + cursor: Object.keys(cursorMod).sort(), + }; + assert.deepEqual(surfaces, { + registry: [ + "CodexCloudAgent", + "CursorCloudAgent", + "DevinAgent", + "JulesAgent", + "getAgent", + "getAvailableAgents", + "isCloudAgentProvider", + ], + baseAgent: ["CloudAgentBase"], + types: [ + "CLOUD_AGENT_STATUS", + "CloudAgentSourceSchema", + "CloudAgentStatusSchema", + "CloudAgentTaskOptionsSchema", + "CreateCloudAgentTaskSchema", + "UpdateCloudAgentTaskSchema", + ], + db: [ + "createCloudAgentTaskTable", + "deleteCloudAgentTask", + "getAllCloudAgentTasks", + "getCloudAgentTaskById", + "getCloudAgentTasksByProvider", + "getCloudAgentTasksByStatus", + "insertCloudAgentTask", + "updateCloudAgentTask", + ], + api: [ + "getCloudAgentCorsHeaders", + "getCloudAgentCredentials", + "requireCloudAgentManagementAuth", + "serializeCloudAgentTask", + "withCloudAgentCors", + ], + credentials: [ + "deleteCloudAgentCredential", + "getCloudAgentCredentialFromDb", + "listCloudAgentCredentials", + "maskApiKey", + "saveCloudAgentCredential", + ], + julesApi: ["JULES_API_BASE_URL", "buildJulesApiUrl"], + jules: ["JulesAgent"], + devin: ["DevinAgent"], + codex: ["CodexCloudAgent"], + cursor: ["CursorCloudAgent"], + }); +}); + +test("surface: CLOUD_AGENT_STATUS values and status schema options", () => { + assert.deepEqual(typesMod.CLOUD_AGENT_STATUS, { + QUEUED: "queued", + RUNNING: "running", + AWAITING_APPROVAL: "awaiting_approval", + COMPLETED: "completed", + FAILED: "failed", + CANCELLED: "cancelled", + }); + assert.deepEqual(typesMod.CloudAgentStatusSchema.options, [ + "queued", + "running", + "awaiting_approval", + "completed", + "failed", + "cancelled", + ]); +}); + +// ─── (b) Contracts — registry ──────────────────────────────────────────────── + +test("registry: available agents, ids and base URLs", () => { + assert.deepEqual(registryMod.getAvailableAgents(), [ + "jules", + "devin", + "codex-cloud", + "cursor-cloud", + ]); + const shape = registryMod.getAvailableAgents().map((id) => { + const agent = registryMod.getAgent(id); + return [id, agent?.providerId, agent?.baseUrl, agent instanceof baseAgentMod.CloudAgentBase]; + }); + assert.deepEqual(shape, [ + ["jules", "jules", "https://jules.googleapis.com/v1alpha", true], + ["devin", "devin", "https://api.devin.ai/v1", true], + ["codex-cloud", "codex-cloud", "https://api.openai.com/v1", true], + ["cursor-cloud", "cursor-cloud", "https://api.cursor.com/v0", true], + ]); + // getAgent returns the same singleton on every call. + assert.equal(registryMod.getAgent("jules"), registryMod.getAgent("jules")); +}); + +test("registry: unknown provider → getAgent null, isCloudAgentProvider false", () => { + assert.equal(registryMod.getAgent("nope"), null); + assert.equal(registryMod.isCloudAgentProvider("nope"), false); + // The OAuth chat provider `cursor` is NOT a cloud agent (only `cursor-cloud` is). + assert.equal(registryMod.isCloudAgentProvider("cursor"), false); +}); + +test("characterization: isCloudAgentProvider currently matches Object.prototype keys (uses `in`)", () => { + // `providerId in AGENTS` walks the prototype chain, so inherited names report true while + // getAgent() (property read || null) returns a non-agent function for them. + assert.equal(registryMod.isCloudAgentProvider("toString"), true); + assert.equal(typeof registryMod.getAgent("toString"), "function"); +}); + +test("julesApi: buildJulesApiUrl normalises the leading slash", () => { + assert.equal(julesApiMod.JULES_API_BASE_URL, "https://jules.googleapis.com/v1alpha"); + assert.equal( + julesApiMod.buildJulesApiUrl("sessions"), + "https://jules.googleapis.com/v1alpha/sessions" + ); + assert.equal( + julesApiMod.buildJulesApiUrl("/sources"), + "https://jules.googleapis.com/v1alpha/sources" + ); +}); + +// ─── (b) Contracts — listSources (fake fetch) ──────────────────────────────── + +test("listSources(jules): GET /sources with X-Goog-Api-Key, maps owner/repo/branch", async () => { + const fake = withFakeFetch(() => + Response.json({ + sources: [ + { + name: "sources/github/acme/widgets", + githubRepo: { owner: "acme", repo: "widgets", defaultBranch: "main" }, + }, + { + githubRepo: { owner: "acme", repo: "gears" }, + githubRepoContext: { startingBranch: " dev " }, + }, + { name: "orphan" }, + ], + }) + ); + try { + const sources = await registryMod.getAgent("jules")!.listSources(CREDS); + assert.deepEqual(sources, [ + { + name: "sources/github/acme/widgets", + url: "https://github.com/acme/widgets", + branch: "main", + }, + { name: "acme/gears", url: "https://github.com/acme/gears", branch: "dev" }, + { name: "orphan", url: "" }, + ]); + assert.equal(fake.calls.length, 1); + assert.equal(fake.calls[0].url, "https://jules.googleapis.com/v1alpha/sources"); + assert.equal(headerOf(fake.calls[0].init, "X-Goog-Api-Key"), "key_char_123"); + } finally { + fake.restore(); + } +}); + +test("listSources(cursor-cloud): GET /repositories with Bearer, drops url-less entries", async () => { + const fake = withFakeFetch(() => + Response.json({ + repositories: [ + { url: "https://github.com/acme/widgets" }, + { repository: "https://github.com/acme/gears", name: "Gears" }, + { name: "no-url" }, + ], + }) + ); + try { + const sources = await registryMod + .getAgent("cursor-cloud")! + .listSources({ apiKey: "k", baseUrl: "https://cursor.example/v9/" }); + assert.deepEqual(sources, [ + { name: "acme/widgets", url: "https://github.com/acme/widgets" }, + { name: "Gears", url: "https://github.com/acme/gears" }, + ]); + // Per-credential baseUrl override wins and its trailing slash is stripped. + assert.equal(fake.calls[0].url, "https://cursor.example/v9/repositories"); + assert.equal(headerOf(fake.calls[0].init, "Authorization"), "Bearer k"); + } finally { + fake.restore(); + } +}); + +test("listSources(devin|codex-cloud): static empty list, no network call", async () => { + const fake = withFakeFetch(() => Response.json({})); + try { + assert.deepEqual(await registryMod.getAgent("devin")!.listSources(CREDS), []); + assert.deepEqual(await registryMod.getAgent("codex-cloud")!.listSources(CREDS), []); + assert.equal(fake.calls.length, 0); + } finally { + fake.restore(); + } +}); + +test("characterization: listSources(cursor-cloud) currently swallows a non-OK upstream as []", async () => { + const fake = withFakeFetch(() => new Response("denied", { status: 401 })); + try { + assert.deepEqual(await registryMod.getAgent("cursor-cloud")!.listSources(CREDS), []); + } finally { + fake.restore(); + } +}); + +test("listSources(jules): non-OK upstream throws with status + body", async () => { + const fake = withFakeFetch(() => new Response("denied", { status: 403 })); + try { + await assert.rejects(registryMod.getAgent("jules")!.listSources(CREDS), { + message: "Jules list sources failed: 403 denied", + }); + } finally { + fake.restore(); + } +}); + +// ─── (b) Contracts — createTask (fake fetch) ───────────────────────────────── + +const TASK_ID_RE = /^task_\d+_[0-9a-f]{16}$/; + +test("createTask(jules): POST /sessions body shape and returned task", async () => { + const fake = withFakeFetch(() => Response.json({ name: "sessions/abc123" })); + try { + const task = await registryMod.getAgent("jules")!.createTask( + { + prompt: "fix it", + source: { ...SOURCE, repoUrl: "github.com/acme/widgets.git" }, + options: { autoCreatePr: true, planApprovalRequired: true }, + }, + CREDS + ); + assert.match(task.id, TASK_ID_RE); + assert.equal(task.providerId, "jules"); + assert.equal(task.externalId, "abc123", "name `sessions/` is normalised to "); + assert.equal(task.status, "queued"); + assert.deepEqual(task.activities, []); + assert.equal(task.createdAt, task.updatedAt); + + assert.equal(fake.calls[0].url, "https://jules.googleapis.com/v1alpha/sessions"); + assert.equal(fake.calls[0].init?.method, "POST"); + assert.equal(headerOf(fake.calls[0].init, "Content-Type"), "application/json"); + assert.deepEqual(JSON.parse(String(fake.calls[0].init?.body)), { + prompt: "fix it", + title: "acme/widgets", + sourceContext: { + source: "sources/github/acme/widgets", + githubRepoContext: { startingBranch: "main" }, + }, + automationMode: "AUTO_CREATE_PR", + requirePlanApproval: true, + }); + } finally { + fake.restore(); + } +}); + +test("createTask(devin): POST /sessions, status mapped through base mapStatus", async () => { + const fake = withFakeFetch(() => Response.json({ id: "devin-1", status: "running" })); + try { + const task = await registryMod + .getAgent("devin")! + .createTask({ prompt: "p", source: { ...SOURCE, branch: "feat" }, options: {} }, CREDS); + assert.equal(task.externalId, "devin-1"); + assert.equal(task.status, "running"); + assert.equal(fake.calls[0].url, "https://api.devin.ai/v1/sessions"); + assert.equal(headerOf(fake.calls[0].init, "Authorization"), "Bearer key_char_123"); + assert.deepEqual(JSON.parse(String(fake.calls[0].init?.body)), { + prompt: "p", + repo_url: "https://github.com/acme/widgets", + branch: "feat", + }); + } finally { + fake.restore(); + } +}); + +test("createTask: non-OK upstream throws ' create task failed: '", async () => { + const fake = withFakeFetch(() => new Response("bad key", { status: 401 })); + try { + await assert.rejects( + registryMod + .getAgent("jules")! + .createTask({ prompt: "p", source: SOURCE, options: {} }, CREDS), + { message: "Jules create task failed: 401 bad key" } + ); + await assert.rejects( + registryMod + .getAgent("devin")! + .createTask({ prompt: "p", source: SOURCE, options: {} }, CREDS), + { message: "Devin create task failed: 401 bad key" } + ); + await assert.rejects( + registryMod + .getAgent("codex-cloud")! + .createTask({ prompt: "p", source: SOURCE, options: {} }, CREDS), + { message: "Codex Cloud create task failed: 401 bad key" } + ); + await assert.rejects( + registryMod + .getAgent("cursor-cloud")! + .createTask({ prompt: "p", source: SOURCE, options: {} }, CREDS), + { message: "Cursor create agent failed: 401 bad key" } + ); + } finally { + fake.restore(); + } +}); + +// ─── (b) Contracts — getStatus / approvePlan ───────────────────────────────── + +test("getStatus(jules): plan generated but not approved → awaiting_approval", async () => { + const fake = withFakeFetch((url) => + url.includes("/activities") + ? Response.json({ + activities: [ + { + id: "a1", + createTime: "2026-01-01T00:00:00Z", + planGenerated: { plan: { steps: [{ title: "Step A" }, { title: "Step B" }] } }, + }, + ], + }) + : Response.json({ state: "RUNNING" }) + ); + try { + const status = await registryMod.getAgent("jules")!.getStatus("sessions/s1", CREDS); + assert.deepEqual(status, { + status: "awaiting_approval", + externalId: "s1", + result: undefined, + activities: [ + { id: "a1", type: "plan", content: "Step A\nStep B", timestamp: "2026-01-01T00:00:00Z" }, + ], + error: undefined, + }); + assert.deepEqual(fake.calls.map((c) => c.url).sort(), [ + "https://jules.googleapis.com/v1alpha/sessions/s1", + "https://jules.googleapis.com/v1alpha/sessions/s1/activities?pageSize=30", + ]); + } finally { + fake.restore(); + } +}); + +test("getStatus(jules): a pull-request output wins → completed with result", async () => { + const fake = withFakeFetch((url) => + url.includes("/activities") + ? new Response("nope", { status: 500 }) // activities failure is tolerated + : Response.json({ + state: "FAILED", + outputs: [{ pullRequest: { url: "https://gh/pr/1", title: "T", description: "D" } }], + }) + ); + try { + const status = await registryMod.getAgent("jules")!.getStatus("s2", CREDS); + assert.equal(status.status, "completed"); + assert.deepEqual(status.result, { prUrl: "https://gh/pr/1", commitMessage: "D", summary: "T" }); + assert.deepEqual(status.activities, []); + } finally { + fake.restore(); + } +}); + +test("characterization: base mapStatus currently checks 'pending' before 'approval'", async () => { + // Devin routes its raw status through CloudAgentBase.mapStatus (substring matcher). + const cases: Array<[string, string]> = [ + ["completed", "completed"], + ["DONE", "completed"], + ["errored", "failed"], + ["canceled", "cancelled"], + ["executing", "running"], + ["waiting", "queued"], + ["needs plan review", "awaiting_approval"], + ["pending approval", "queued"], // 'pending' matches first — never awaiting_approval + ["something-else", "queued"], + ]; + const seen: Array<[string, string]> = []; + for (const [raw] of cases) { + const fake = withFakeFetch(() => Response.json({ status: raw })); + try { + const status = await registryMod.getAgent("devin")!.getStatus("x", CREDS); + seen.push([raw, status.status]); + } finally { + fake.restore(); + } + } + assert.deepEqual(seen, cases); +}); + +test("approvePlan: unsupported agents reject with a fixed message", async () => { + await assert.rejects(registryMod.getAgent("devin")!.approvePlan("x", CREDS), { + message: "Devin does not support plan approval - it auto-plans", + }); + await assert.rejects(registryMod.getAgent("codex-cloud")!.approvePlan("x", CREDS), { + message: "Codex Cloud does not support plan approval - it auto-plans", + }); + await assert.rejects(registryMod.getAgent("cursor-cloud")!.approvePlan("x", CREDS), { + message: "Cursor Cloud Agents run autonomously and do not support plan approval", + }); +}); + +// ─── (b) Contracts — db + api serialisation ────────────────────────────────── + +function row(overrides: Partial> = {}) { + return { + id: "t-1", + provider_id: "jules", + external_id: "ext-1", + status: "queued", + prompt: "p", + source: JSON.stringify(SOURCE), + options: "{}", + result: null, + activities: "[]", + error: null, + created_at: "2026-01-01 00:00:00", + updated_at: "2026-01-01 00:00:00", + completed_at: null, + ...overrides, + } as Parameters[0]; +} + +test("db: insert/get/list/update/delete round trip", () => { + dbMod.createCloudAgentTaskTable(); // idempotent (index.ts already ran it on import) + dbMod.insertCloudAgentTask(row()); + assert.equal(dbMod.getCloudAgentTaskById("t-1")?.status, "queued"); + assert.equal(dbMod.getCloudAgentTasksByProvider("jules").length, 1); + assert.equal(dbMod.getCloudAgentTasksByStatus("queued").length, 1); + assert.equal(dbMod.getAllCloudAgentTasks().length, 1); + + dbMod.updateCloudAgentTask("t-1", { status: "running", error: "e" }); + const updated = dbMod.getCloudAgentTaskById("t-1"); + assert.equal(updated?.status, "running"); + assert.equal(updated?.error, "e"); + + dbMod.deleteCloudAgentTask("t-1"); + assert.equal(dbMod.getCloudAgentTaskById("t-1") ?? null, null); +}); + +test("characterization: updateCloudAgentTask silently ignores non-whitelisted columns", () => { + dbMod.insertCloudAgentTask(row({ id: "t-2" })); + dbMod.updateCloudAgentTask("t-2", { + provider_id: "devin", + external_id: "other", + } as Parameters[1]); + const after = dbMod.getCloudAgentTaskById("t-2"); + assert.equal(after?.provider_id, "jules"); + assert.equal(after?.external_id, "ext-1"); + // Updating a missing id is a silent no-op as well. + assert.doesNotThrow(() => dbMod.updateCloudAgentTask("missing", { status: "failed" })); + dbMod.deleteCloudAgentTask("t-2"); +}); + +test("serializeCloudAgentTask: camelCase shape, malformed JSON falls back to defaults", () => { + const out = apiMod.serializeCloudAgentTask( + row({ source: "{not json", options: "", activities: "oops", result: "also bad" }) + ); + assert.deepEqual(out, { + id: "t-1", + providerId: "jules", + externalId: "ext-1", + status: "queued", + prompt: "p", + source: {}, + options: {}, + result: {}, + activities: [], + error: null, + createdAt: "2026-01-01 00:00:00", + updatedAt: "2026-01-01 00:00:00", + completedAt: null, + }); + assert.equal(apiMod.serializeCloudAgentTask(row()).result, null); +}); + +test("getCloudAgentCorsHeaders: no Origin → no ACAO / credentials headers", () => { + assert.deepEqual(apiMod.getCloudAgentCorsHeaders(), { + "Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, OPTIONS", + "Access-Control-Allow-Headers": "Content-Type, Authorization", + }); +}); + +test("getCloudAgentCredentials: no active provider connection → null", async () => { + assert.equal(await apiMod.getCloudAgentCredentials("jules"), null); +}); + +// ─── (b)+(c) credentials + schemas ─────────────────────────────────────────── + +test("credentials: maskApiKey keeps the last 4 chars, short/empty → ****", () => { + assert.equal(credentialsMod.maskApiKey("sk-abcdef1234"), "****1234"); + assert.equal(credentialsMod.maskApiKey("1234"), "****"); + assert.equal(credentialsMod.maskApiKey(""), "****"); +}); + +test("credentials: save → get (decrypted) → list (masked) → delete", () => { + assert.equal(credentialsMod.getCloudAgentCredentialFromDb("devin"), null); + credentialsMod.saveCloudAgentCredential("devin", "devin-secret-9999", "https://d.example"); + assert.deepEqual(credentialsMod.getCloudAgentCredentialFromDb("devin"), { + apiKey: "devin-secret-9999", + baseUrl: "https://d.example", + }); + credentialsMod.saveCloudAgentCredential("devin", "devin-secret-0000"); // upsert, baseUrl cleared + assert.deepEqual(credentialsMod.getCloudAgentCredentialFromDb("devin"), { + apiKey: "devin-secret-0000", + }); + const listed = credentialsMod.listCloudAgentCredentials(); + assert.deepEqual( + listed.map(({ providerId, apiKey, baseUrl }) => ({ providerId, apiKey, baseUrl })), + [{ providerId: "devin", apiKey: "****0000", baseUrl: null }] + ); + credentialsMod.deleteCloudAgentCredential("devin"); + assert.equal(credentialsMod.getCloudAgentCredentialFromDb("devin"), null); +}); + +test("CreateCloudAgentTaskSchema: invalid input is rejected (no throw on safeParse)", () => { + const ok = typesMod.CreateCloudAgentTaskSchema.safeParse({ + providerId: "jules", + prompt: "p", + source: SOURCE, + }); + assert.equal(ok.success, true); + + const bad = [ + { providerId: "cursor", prompt: "p", source: SOURCE }, + { providerId: "jules", prompt: "", source: SOURCE }, + { providerId: "jules", prompt: "x".repeat(10001), source: SOURCE }, + { providerId: "jules", prompt: "p", source: { repoName: "r", repoUrl: "not-a-url" } }, + ]; + for (const input of bad) { + assert.equal(typesMod.CreateCloudAgentTaskSchema.safeParse(input).success, false); + } + assert.equal( + typesMod.UpdateCloudAgentTaskSchema.safeParse({ id: "t", action: "pause" }).success, + false + ); +}); diff --git a/tests/unit/characterization-evals.test.ts b/tests/unit/characterization-evals.test.ts new file mode 100644 index 000000000000..22f5ce73146c --- /dev/null +++ b/tests/unit/characterization-evals.test.ts @@ -0,0 +1,397 @@ +/** + * Characterization — src/lib/evals (rail 3.8.55, pre-v4 extraction to @omniroute/mod-*). + * + * Pins the PUBLIC surface and observable behaviour of the eval framework as it is today so + * the v4 extraction can be proven byte-for-byte. Quirks are pinned, not fixed — their test + * names start with "characterization: … currently …". Surface snapshots are literal lists: + * update them consciously, never by regeneration. + * + * External consumers (the real boundary): src/app/api/evals/route.ts (listSuites, runSuite, + * createScorecard, buildEvalTargetOptions, runEvalSuiteAgainstTarget) and + * src/app/api/evals/[suiteId]/route.ts (getSuite). + * + * "Fake judge": the grading step is exercised with the `custom` strategy, whose `fn` is the + * judge — no model is called. The model-calling half (runEvalSuiteAgainstTarget → + * chat-completions route) is only pinned up to its pre-flight validation. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-char-evals-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const runner = await import("../../src/lib/evals/evalRunner.ts"); +const runtime = await import("../../src/lib/evals/runtime.ts"); +const builtins = await import("../../src/lib/evals/evalRunner/builtinSuites.ts"); +const evalsDb = await import("../../src/lib/db/evals.ts"); + +test.after(() => { + runner.resetSuites(); + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +// ─── (a) Surface snapshots ─────────────────────────────────────────────────── + +test("surface: evals module runtime exports", () => { + assert.deepEqual( + { + evalRunner: Object.keys(runner).sort(), + runtime: Object.keys(runtime).sort(), + builtinSuites: Object.keys(builtins).sort(), + }, + { + evalRunner: [ + "createScorecard", + "evaluateCase", + "getSuite", + "listSuites", + "registerSuite", + "resetSuites", + "runSuite", + ], + runtime: [ + "buildEvalCaseRequest", + "buildEvalTargetOptions", + "getEvalTargetLabel", + "normalizeEvalTarget", + "runEvalSuiteAgainstTarget", + ], + builtinSuites: [ + "builtInSuites", + "codexComparisonSuite", + "codingSuite", + "goldenSet", + "instructionSuite", + "multilingualSuite", + "reasoningSuite", + "safetySuite", + ], + } + ); +}); + +test("surface: built-in suites registered at load (id, name, source, caseCount)", () => { + const listed = runner + .listSuites() + .map((s) => [s.id, s.name, s.source, s.caseCount] as [string, string, string, number]); + assert.deepEqual(listed, [ + ["golden-set", "OmniRoute Golden Set", "built-in", 10], + ["coding-proficiency", "Coding Proficiency", "built-in", 5], + ["reasoning-logic", "Reasoning & Logic", "built-in", 5], + ["multilingual", "Multilingual", "built-in", 5], + ["safety-guardrails", "Safety & Guardrails", "built-in", 6], + ["instruction-following", "Instruction Following", "built-in", 5], + ["codex-comparison", "Codex Comparison", "built-in", 8], + ]); + assert.deepEqual( + builtins.builtInSuites.map((s) => s.id), + listed.map(([id]) => id) + ); +}); + +// ─── (b) Contracts — run a minimal suite with a fake judge ─────────────────── + +const judgeCalls: Array<{ output: string; caseId: string }> = []; +const fakeJudge = (output: string, evalCase: { id: string }) => { + judgeCalls.push({ output, caseId: evalCase.id }); + return output.includes("42"); +}; + +const MINI_SUITE = { + id: "char-mini", + name: "Characterization Mini", + description: "fake-judge suite", + cases: [ + { + id: "c1", + name: "judge pass", + model: "m-a", + input: {}, + expected: { strategy: "custom", fn: fakeJudge }, + }, + { + id: "c2", + name: "judge fail", + model: "m-a", + input: {}, + expected: { strategy: "custom", fn: fakeJudge }, + }, + { + id: "c3", + name: "contains", + model: "m-b", + input: {}, + expected: { strategy: "contains", value: "PARIS" }, + }, + { + id: "c4", + name: "errored", + model: "m-b", + input: {}, + expected: { strategy: "regex", value: "error" }, + }, + ], +}; + +test("runSuite: fake-judge suite → per-case results + rounded summary", () => { + runner.registerSuite(MINI_SUITE); + judgeCalls.length = 0; + const run = runner.runSuite( + "char-mini", + { c1: "the answer is 42", c2: "no idea", c3: "It is paris.", c4: "[ERROR] upstream error" }, + { c1: { durationMs: 12.6 }, c4: { durationMs: -5, error: "upstream error" } } + ); + + assert.equal(run.suiteId, "char-mini"); + assert.equal(run.suiteName, "Characterization Mini"); + assert.deepEqual( + run.results.map((r) => [r.caseId, r.caseName, r.passed, r.error]), + [ + ["c1", "judge pass", true, undefined], + ["c2", "judge fail", false, undefined], + ["c3", "contains", true, undefined], + // A case whose call errored is forced to failed even though /error/ matched (#13137). + ["c4", "errored", false, "upstream error"], + ] + ); + assert.deepEqual(judgeCalls, [ + { output: "the answer is 42", caseId: "c1" }, + { output: "no idea", caseId: "c2" }, + ]); + // Case metrics override the measured duration: rounded, clamped at 0. + assert.equal(run.results[0].durationMs, 13); + assert.equal(run.results[3].durationMs, 0); + assert.deepEqual(run.summary, { total: 4, passed: 2, failed: 2, passRate: 50 }); +}); + +test("runSuite: missing output is graded as the empty string", () => { + judgeCalls.length = 0; + const run = runner.runSuite("char-mini", {}); + assert.deepEqual( + judgeCalls.map((c) => c.output), + ["", ""] + ); + assert.deepEqual(run.summary, { total: 4, passed: 0, failed: 4, passRate: 0 }); +}); + +test("createScorecard: aggregates runs; empty input → zeros", () => { + const a = runner.runSuite("char-mini", { c1: "42", c3: "paris" }); + const card = runner.createScorecard([a, { ...a, suiteId: "x", suiteName: "X" }]); + assert.deepEqual(card, { + suites: 2, + totalCases: 8, + totalPassed: 4, + overallPassRate: 50, + perSuite: [ + { id: "char-mini", name: "Characterization Mini", passRate: 50 }, + { id: "x", name: "X", passRate: 50 }, + ], + }); + assert.deepEqual(runner.createScorecard([]), { + suites: 0, + totalCases: 0, + totalPassed: 0, + overallPassRate: 0, + perSuite: [], + }); +}); + +test("resetSuites: drops runtime-registered suites, keeps the built-ins", () => { + assert.ok(runner.getSuite("char-mini")); + runner.resetSuites(); + assert.equal(runner.getSuite("char-mini"), null); + assert.equal(runner.listSuites().length, 7); +}); + +test("custom suites persisted in the DB are listed (source=custom) and runnable", () => { + evalsDb.saveCustomEvalSuite({ + id: "char-custom", + name: "Char Custom", + cases: [ + { + id: "k1", + name: "says hi", + model: "m-c", + input: { messages: [{ role: "user", content: "hi" }] }, + expected: { strategy: "contains", value: "hello" }, + }, + ], + }); + const listed = runner.listSuites().find((s) => s.id === "char-custom"); + assert.equal(listed?.source, "custom"); + assert.equal(listed?.caseCount, 1); + assert.equal(runner.getSuite("char-custom")?.name, "Char Custom"); + const run = runner.runSuite("char-custom", { k1: "Hello there" }); + assert.deepEqual(run.summary, { total: 1, passed: 1, failed: 0, passRate: 100 }); +}); + +// ─── (b) Contracts — evaluateCase strategies ───────────────────────────────── + +function evalWith(expected: Record, output: string) { + const r = runner.evaluateCase({ id: "e", name: "E", expected }, output); + return { passed: r.passed, error: r.error, details: r.details }; +} + +test("evaluateCase: exact / contains / regex semantics", () => { + assert.equal(evalWith({ strategy: "exact", value: "A" }, "A").passed, true); + assert.equal(evalWith({ strategy: "exact", value: "A" }, "a").passed, false); + assert.equal(evalWith({ strategy: "contains", value: "WORLD" }, "hello world").passed, true); + assert.equal(evalWith({ strategy: "contains", value: 5 }, "5").passed, false); + // String patterns compile with dotAll, so "." spans newlines (#13138). + assert.equal(evalWith({ strategy: "regex", value: "a.b" }, "a\nb").passed, true); + assert.equal(evalWith({ strategy: "regex", value: /a.b/ }, "a\nb").passed, true); + assert.equal(evalWith({ strategy: "regex", value: /^x$/g }, "x").passed, true); + assert.equal( + evalWith({ strategy: "exact", value: "A" }, "A".repeat(300)).details?.actualSnippet.length, + 240 + ); +}); + +// ─── (c) Errors / invalid input ────────────────────────────────────────────── + +test("evaluateCase: unknown strategy → passed=false with an error, no throw", () => { + const r = evalWith({ strategy: "semantic" }, "x"); + assert.deepEqual( + [r.passed, r.error, r.details], + [false, "Unknown strategy: semantic", undefined] + ); +}); + +test("evaluateCase: unsafe / oversized / missing regex is rejected, not executed", () => { + assert.deepEqual( + evalWith({ strategy: "regex", value: "(a+)+$" }, "aaaa").details?.error, + "Regex pattern rejected as potentially unsafe (catastrophic backtracking risk). Simplify the pattern." + ); + assert.equal( + evalWith({ strategy: "regex", value: "a".repeat(600) }, "a").details?.error, + "Regex pattern too large for safe evaluation." + ); + assert.equal( + evalWith({ strategy: "regex" }, "a").details?.error, + "No regex value provided for evaluation." + ); +}); + +test("characterization: evaluateCase 'custom' currently passes the judge's raw return value through", () => { + // No fn → silently false (no error). + assert.deepEqual( + [evalWith({ strategy: "custom" }, "x").passed, evalWith({ strategy: "custom" }, "x").error], + [false, undefined] + ); + // A judge returning a truthy non-boolean is not coerced: `passed` is that value. + assert.equal(evalWith({ strategy: "custom", fn: () => "yes" }, "x").passed, "yes"); + // A throwing judge is caught and reported as a failed case. + const thrown = evalWith( + { + strategy: "custom", + fn: () => { + throw new Error("judge exploded"); + }, + }, + "x" + ); + assert.deepEqual([thrown.passed, thrown.error], [false, "judge exploded"]); +}); + +test("evaluateCase: a case without `expected` is caught and reported, not thrown", () => { + const r = runner.evaluateCase({ id: "bad", name: "Bad" }, "x"); + assert.equal(r.passed, false); + assert.match(String(r.error), /Cannot read properties of undefined/); +}); + +test("runSuite: unknown suite id throws 'Suite not found: '", () => { + assert.throws(() => runner.runSuite("does-not-exist", {}), { + message: "Suite not found: does-not-exist", + }); + assert.equal(runner.getSuite("does-not-exist"), null); +}); + +// ─── (b) Contracts — runtime helpers ───────────────────────────────────────── + +test("normalizeEvalTarget / getEvalTargetLabel", () => { + assert.deepEqual(runtime.normalizeEvalTarget(null), { type: "suite-default", id: null }); + assert.deepEqual(runtime.normalizeEvalTarget({ type: "combo", id: " fast " }), { + type: "combo", + id: "fast", + }); + // Any non-combo, non-default type collapses to "model". + assert.deepEqual(runtime.normalizeEvalTarget({ type: "weird" as "model", id: "" }), { + type: "model", + id: null, + }); + assert.equal(runtime.getEvalTargetLabel({ type: "combo", id: "fast" }), "Combo: fast"); + assert.equal(runtime.getEvalTargetLabel({ type: "model", id: null }), "Model: Unknown"); + assert.equal(runtime.getEvalTargetLabel({ type: "suite-default" }), "Suite defaults"); +}); + +test("buildEvalCaseRequest: model resolution, defaults and headers", async () => { + const caseDefault = runtime.buildEvalCaseRequest( + { model: "m-a", input: { messages: [{ role: "user", content: "q" }] } }, + { type: "suite-default" }, + null + ); + assert.equal(caseDefault.url, "http://localhost/api/v1/chat/completions"); + assert.equal(caseDefault.method, "POST"); + assert.equal(caseDefault.headers.get("authorization"), null); + assert.deepEqual(JSON.parse(await caseDefault.text()), { + messages: [{ role: "user", content: "q" }], + model: "m-a", + stream: false, + max_tokens: 512, + }); + + const forced = runtime.buildEvalCaseRequest( + { model: "m-a", input: { max_tokens: 64, stream: true } }, + { type: "combo", id: "fast" }, + "sk-x" + ); + assert.equal(forced.headers.get("authorization"), "Bearer sk-x"); + assert.deepEqual(JSON.parse(await forced.text()), { + stream: false, + model: "fast", + max_tokens: 64, + }); + + const fallback = runtime.buildEvalCaseRequest({ input: [] }, { type: "suite-default" }, null); + assert.deepEqual(JSON.parse(await fallback.text()), { + model: "gpt-4o", + stream: false, + max_tokens: 512, + }); +}); + +test("buildEvalTargetOptions: default option, then sorted case models, then combos", async () => { + const options = await runtime.buildEvalTargetOptions(); + assert.deepEqual(options[0], { + key: "suite-default:__default__", + type: "suite-default", + id: null, + label: "Suite defaults", + description: "Use each case's built-in model", + }); + assert.deepEqual( + options.slice(1).map((o) => o.key), + [ + "model:claude-sonnet-4-20250514", + "model:codex", + "model:gemini-2.5-flash", + "model:gpt-4o", + "model:m-c", // from the persisted custom suite above + ] + ); +}); + +test("runEvalSuiteAgainstTarget: pre-flight errors reject before any model call", async () => { + await assert.rejects(runtime.runEvalSuiteAgainstTarget({ suiteId: "nope" }), { + message: "Suite not found: nope", + }); + await assert.rejects( + runtime.runEvalSuiteAgainstTarget({ suiteId: "golden-set", apiKeyId: "missing-key" }), + { message: "Selected API key was not found" } + ); +}); diff --git a/tests/unit/characterization-gamification.test.ts b/tests/unit/characterization-gamification.test.ts new file mode 100644 index 000000000000..60e227f1c98b --- /dev/null +++ b/tests/unit/characterization-gamification.test.ts @@ -0,0 +1,461 @@ +/** + * Characterization — src/lib/gamification (rail 3.8.55, pre-v4 extraction to @omniroute/mod-*). + * + * Pins the PUBLIC surface and the observable behaviour (XP curve, levels/tiers, the award + * pipeline's points + badges for a fixed fixture, streaks, anti-cheat, sharing, invites, + * servers) as it is today, so the v4 extraction can be proven byte-for-byte. Quirks are pinned, + * not fixed — their names start with "characterization: … currently …". Surface snapshots are + * literal lists: update them consciously, never by regeneration. + * + * External consumers (the real boundary): open-sse/handlers/chatCore/gamificationEvent.ts and + * src/lib/radar/intelSync.ts (emitGamificationEvent), open-sse/mcp-server/tools/ + * gamificationTools.ts (leaderboard/xp/streaks/sharing/invites/servers/antiCheat), + * src/app/api/gamification/** and the dashboard profile page (xp helpers). Note: every + * consumer imports the individual modules — none imports the index.ts barrel. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-char-gamification-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const gamificationDb = await import("../../src/lib/db/gamification.ts"); +const MODULES = { + index: await import("../../src/lib/gamification/index.ts"), + antiCheat: await import("../../src/lib/gamification/antiCheat.ts"), + badges: await import("../../src/lib/gamification/badges.ts"), + events: await import("../../src/lib/gamification/events.ts"), + invites: await import("../../src/lib/gamification/invites.ts"), + leaderboard: await import("../../src/lib/gamification/leaderboard.ts"), + notifications: await import("../../src/lib/gamification/notifications.ts"), + servers: await import("../../src/lib/gamification/servers.ts"), + sharing: await import("../../src/lib/gamification/sharing.ts"), + streaks: await import("../../src/lib/gamification/streaks.ts"), + xp: await import("../../src/lib/gamification/xp.ts"), +}; +const { antiCheat, badges, events, invites, leaderboard, notifications, servers, sharing } = + MODULES; +const { streaks, xp } = MODULES; + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +const todayUtc = () => new Date().toISOString().split("T")[0]; +const yesterdayUtc = () => new Date(Date.now() - 86_400_000).toISOString().split("T")[0]; + +// ─── (a) Surface snapshots ─────────────────────────────────────────────────── + +test("surface: gamification runtime exports (barrel + every module)", () => { + const surface = Object.fromEntries( + Object.entries(MODULES).map(([name, mod]) => [name, Object.keys(mod).sort()]) + ); + assert.deepEqual(surface, { + index: [ + "BUILTIN_BADGES", + "XP_REWARDS", + "calculateLevel", + "connectServer", + "consumeBadgeUnlocks", + "createBadgeNotificationStream", + "createInvite", + "cumulativeXpForLevel", + "disconnectServer", + "emitGamificationEvent", + "getAggregateStreak", + "getAnomalies", + "getBalance", + "getHistory", + "getLevelTier", + "getLevelTitle", + "getNeighbors", + "getRank", + "getStreak", + "getTopN", + "listServers", + "recordBadgeUnlock", + "redeemInviteCode", + "rotateScope", + "transferTokens", + "updateScore", + "updateStreak", + "validateScoreChange", + "xpForLevel", + "xpToNextLevel", + ], + antiCheat: ["getAnomalies", "validateScoreChange"], + badges: ["BUILTIN_BADGES", "evaluateBadges", "seedBuiltinBadges"], + events: ["emitGamificationEvent"], + invites: ["createInvite", "listInvites", "redeemInvite", "revokeInvite"], + leaderboard: ["getNeighbors", "getRank", "getTopN", "rotateScope", "updateScore"], + notifications: ["consumeBadgeUnlocks", "createBadgeNotificationStream", "recordBadgeUnlock"], + servers: [ + "connectServer", + "disconnectServer", + "healthCheck", + "listServers", + "pushScore", + "syncLeaderboard", + ], + sharing: ["getBalance", "getHistory", "transferTokens"], + streaks: ["advanceStreak", "getAggregateStreak", "getStreak", "updateStreak"], + xp: [ + "XP_REWARDS", + "calculateLevel", + "cumulativeXpForLevel", + "getLevelTier", + "getLevelTitle", + "xpForLevel", + "xpToNextLevel", + ], + }); + // The barrel re-exports the module functions themselves (same references). + assert.equal(MODULES.index.emitGamificationEvent, events.emitGamificationEvent); + assert.equal(MODULES.index.redeemInviteCode, invites.redeemInvite); +}); + +test("surface: XP_REWARDS table", () => { + assert.deepEqual(xp.XP_REWARDS, { + request: 1, + provider_switch: 5, + model_switch: 3, + combo_create: 10, + combo_use: 2, + token_share: 1, + invite_redeem: 50, + daily_login: 5, + streak_bonus: 2, + badge_unlock: 10, + }); +}); + +test("surface: BUILTIN_BADGES catalogue (id, category, rarity, criteria type, hidden)", () => { + const catalogue = badges.BUILTIN_BADGES.map((b) => [ + b.id, + b.category, + b.rarity, + JSON.parse(b.criteria ?? "null")?.type, + b.hidden, + ]); + assert.deepEqual(catalogue, [ + ["first-token", "usage", "common", "action_count", 0], + ["token-consumer", "usage", "uncommon", "action_count", 0], + ["token-machine", "usage", "rare", "action_count", 0], + ["token-whale", "usage", "legendary", "action_count", 0], + ["generous", "sharing", "common", "action_count", 0], + ["philanthropist", "sharing", "uncommon", "action_count", 0], + ["token-santa", "sharing", "rare", "action_count", 0], + ["community-hero", "sharing", "legendary", "action_count", 0], + ["explorer", "contribution", "uncommon", "unique_count", 0], + ["polyglot", "contribution", "rare", "unique_count", 0], + ["architect", "contribution", "uncommon", "action_count", 0], + ["speedster", "contribution", "rare", "threshold", 0], + ["resilient", "contribution", "rare", "threshold", 0], + ["radar-supporter", "contribution", "rare", "action_count", 0], + ["daily-user", "streak", "common", "streak", 0], + ["weekly-warrior", "streak", "uncommon", "streak", 0], + ["monthly-master", "streak", "rare", "streak", 0], + ["unstoppable", "streak", "legendary", "streak", 0], + ["early-adopter", "rare", "legendary", "first", 0], + ["bug-hunter", "rare", "rare", "action_count", 0], + ["contributor", "rare", "rare", "action_count", 0], + ["community-leader", "rare", "rare", "rank", 0], + ["secret-badge", "rare", "legendary", "hidden", 1], + ]); +}); + +// ─── (b) Contracts — pure XP / level math ──────────────────────────────────── + +test("xp: curve, cumulative thresholds and level inversion", () => { + assert.deepEqual( + [1, 2, 3, 10, 50, 100].map((l) => [l, xp.xpForLevel(l), xp.cumulativeXpForLevel(l)]), + [ + [1, 0, 0], + [2, 282, 282], + [3, 519, 801], + [10, 3162, 14164], + [50, 35355, 724749], + [100, 100000, 4049979], + ] + ); + assert.deepEqual( + [0, 281, 282, 283, 800, 5000, 1_000_000, -5].map((v) => [ + v, + xp.calculateLevel(v), + xp.xpToNextLevel(v), + ]), + [ + [0, 1, 282], + [281, 1, 1], + [282, 2, 519], + [283, 2, 518], + [800, 2, 1], + [5000, 6, 1040], + [1_000_000, 56, 2661], + [-5, 1, 287], + ] + ); +}); + +test("xp: level titles and tiers at every boundary", () => { + assert.deepEqual( + [0, 10, 11, 25, 26, 50, 51, 75, 76, 1000].map((l) => [ + l, + xp.getLevelTitle(l), + xp.getLevelTier(l), + ]), + [ + [0, "Beginner", "bronze"], + [10, "Beginner", "bronze"], + [11, "Explorer", "silver"], + [25, "Explorer", "silver"], + [26, "Expert", "gold"], + [50, "Expert", "gold"], + [51, "Master", "platinum"], + [75, "Master", "platinum"], + [76, "Legend", "diamond"], + [1000, "Legend", "diamond"], + ] + ); +}); + +test("characterization: xp currently maps non-finite totals to level 1 and leaks NaN/-Infinity", () => { + assert.equal(xp.calculateLevel(Number.NaN), 1); + assert.equal(xp.calculateLevel(Number.POSITIVE_INFINITY), 1); + assert.ok(Number.isNaN(xp.xpToNextLevel(Number.NaN))); + assert.equal(xp.xpToNextLevel(Number.POSITIVE_INFINITY), Number.NEGATIVE_INFINITY); +}); + +// ─── (b) Contracts — award pipeline (deterministic fixture) ────────────────── + +test("emitGamificationEvent(request) on a fresh key: 1 XP + first-token badge (+10)", async () => { + await events.emitGamificationEvent({ apiKeyId: "fx-fresh", action: "request" }); + + const level = gamificationDb.getXp("fx-fresh"); + assert.deepEqual([level?.totalXp, level?.currentLevel], [11, 1]); + assert.equal(gamificationDb.hasBadge("fx-fresh", "first-token"), true); + assert.equal(await leaderboard.getRank("fx-fresh", "global"), 1); + const scores = (await leaderboard.getTopN("global")).map((r) => [r.apiKeyId, r.score]); + assert.deepEqual(scores, [["fx-fresh", 11]]); + for (const scope of ["weekly", "monthly"] as const) { + assert.deepEqual( + (await leaderboard.getTopN(scope)).map((r) => [r.apiKeyId, r.score]), + [["fx-fresh", 11]] + ); + } + assert.deepEqual(await streaks.getStreak("fx-fresh"), { + currentStreak: 1, + longestStreak: 1, + lastActiveDate: todayUtc(), + streakStartDate: todayUtc(), + }); + + // Badge toast buffer: definitions are not seeded here, so name falls back to the id. + const toasts = notifications.consumeBadgeUnlocks("fx-fresh"); + assert.deepEqual( + toasts.map(({ unlockedAt: _u, ...rest }) => rest), + [ + { + badgeId: "first-token", + badgeName: "first-token", + badgeDescription: "", + badgeIcon: "award", + badgeRarity: "common", + }, + ] + ); + assert.deepEqual(notifications.consumeBadgeUnlocks("fx-fresh"), [], "consume drains the buffer"); + + // A second request the same UTC day: +1 XP, no new badge, no streak bonus. + await events.emitGamificationEvent({ apiKeyId: "fx-fresh", action: "request" }); + assert.equal(gamificationDb.getXp("fx-fresh")?.totalXp, 12); + assert.equal((await streaks.getStreak("fx-fresh")).currentStreak, 1); +}); + +test("emitGamificationEvent(request) extending a 2-day streak: streak bonus 2×3 + daily-user", async () => { + const db = core.getDbInstance(); + db.prepare("INSERT OR REPLACE INTO key_value (namespace, key, value) VALUES (?, ?, ?)").run( + "gamification:streaks", + "fx-streak", + JSON.stringify({ + currentStreak: 2, + longestStreak: 2, + lastActiveDate: yesterdayUtc(), + streakStartDate: "2000-01-01", + }) + ); + await events.emitGamificationEvent({ apiKeyId: "fx-streak", action: "request" }); + + // 1 (request) + 6 (streak_bonus 2×3) + 10 (daily-user) + 10 (first-token) = 27. + assert.equal(gamificationDb.getXp("fx-streak")?.totalXp, 27); + assert.equal(gamificationDb.hasBadge("fx-streak", "daily-user"), true); + assert.equal(gamificationDb.hasBadge("fx-streak", "first-token"), true); + assert.deepEqual(await streaks.getStreak("fx-streak"), { + currentStreak: 3, + longestStreak: 3, + lastActiveDate: todayUtc(), + streakStartDate: "2000-01-01", + }); + notifications.consumeBadgeUnlocks("fx-streak"); +}); + +test("emitGamificationEvent(combo_create / radar_supporter): XP-only vs recognition-only", async () => { + await events.emitGamificationEvent({ apiKeyId: "fx-combo", action: "combo_create" }); + assert.equal(gamificationDb.getXp("fx-combo")?.totalXp, 10); + assert.equal((await streaks.getStreak("fx-combo")).currentStreak, 0, "only `request` streaks"); + + await events.emitGamificationEvent({ apiKeyId: "fx-radar", action: "radar_supporter" }); + assert.equal(gamificationDb.getXp("fx-radar"), null, "recognition earns no XP"); + assert.equal(gamificationDb.hasBadge("fx-radar", "radar-supporter"), true); + assert.equal(await leaderboard.getRank("fx-radar", "global"), 0, "and no leaderboard row"); + notifications.consumeBadgeUnlocks("fx-radar"); +}); + +test("characterization: evaluateBadges (no production caller) unlocks first/rank badges", async () => { + await badges.seedBuiltinBadges(); + await badges.seedBuiltinBadges(); // idempotent + const count = core + .getDbInstance() + .prepare("SELECT COUNT(*) AS n FROM badge_definitions") + .get() as { + n: number; + }; + assert.equal(count.n, 23); + // fx-fresh already holds first-token; it is inside the 30-day window and ranked. + assert.deepEqual(await badges.evaluateBadges("fx-fresh", "request"), [ + "early-adopter", + "community-leader", + ]); +}); + +// ─── (c) Errors / invalid input ────────────────────────────────────────────── + +test("emitGamificationEvent: empty apiKeyId is a silent no-op", async () => { + const before = (await leaderboard.getTopN("global")).length; + await assert.doesNotReject(events.emitGamificationEvent({ apiKeyId: "", action: "request" })); + assert.equal((await leaderboard.getTopN("global")).length, before); +}); + +test("characterization: emitGamificationEvent(unknown action) currently writes a 0-score leaderboard row", async () => { + await events.emitGamificationEvent({ + apiKeyId: "fx-unknown", + action: "not-an-action" as "request", + }); + assert.equal(gamificationDb.getXp("fx-unknown"), null); + const row = (await leaderboard.getTopN("global")).find((r) => r.apiKeyId === "fx-unknown"); + assert.equal(row?.score, 0); +}); + +test("validateScoreChange: >1000 XP/min is rejected with a reason; small awards allowed", async () => { + assert.deepEqual(await antiCheat.validateScoreChange("fx-ac", "request", 1001), { + allowed: false, + reason: "Rate limit exceeded: 1001 > 1000 XP/min", + }); + assert.deepEqual(await antiCheat.validateScoreChange("fx-ac", "request", 1), { allowed: true }); + assert.ok(Array.isArray(await antiCheat.getAnomalies())); +}); + +test("transferTokens: validation errors and insufficient balance", async () => { + assert.deepEqual(await sharing.transferTokens("a", "a", 5), { + success: false, + idempotencyKey: "", + error: "Cannot transfer to yourself", + }); + assert.deepEqual(await sharing.transferTokens("a", "b", 0), { + success: false, + idempotencyKey: "", + error: "Amount must be positive", + }); + const broke = await sharing.transferTokens("a", "b", 5, undefined, "idem-1"); + assert.deepEqual(broke, { + success: false, + idempotencyKey: "idem-1", + error: "insufficient_balance", + }); + assert.equal(await sharing.getBalance("a"), 0); + assert.deepEqual(await sharing.getHistory("a"), []); +}); + +test("characterization: transferTokens(NaN) currently passes validation and surfaces the raw SQLite error", async () => { + const result = await sharing.transferTokens("a", "b", Number.NaN, undefined, "idem-nan"); + assert.deepEqual(result, { + success: false, + idempotencyKey: "idem-nan", + error: "NOT NULL constraint failed: token_ledger.amount", + }); +}); + +test("invites: create → redeem rules → revoke", async () => { + const invite = await invites.createInvite("creator", "https://srv.example", 1); + assert.match(invite.code, /^[A-Za-z0-9]{8}$/); + assert.match(invite.token, /^[A-Za-z0-9_-]{43}$/); + + assert.deepEqual(await invites.redeemInvite("nope1234", "x"), { + success: false, + error: "Invalid invite code", + }); + assert.deepEqual(await invites.redeemInvite(invite.code, "creator"), { + success: false, + error: "Cannot redeem your own invite", + }); + assert.deepEqual(await invites.redeemInvite(invite.code, "guest"), { + success: true, + serverUrl: "https://srv.example", + }); + assert.deepEqual(await invites.redeemInvite(invite.code, "guest-2"), { + success: false, + error: "Invite has been fully redeemed", + }); + + const second = await invites.createInvite("creator"); + const listed = await invites.listInvites("creator"); + const secondRow = listed.find((i) => i.code === second.code)!; + assert.deepEqual([secondRow.serverUrl, secondRow.maxUses, secondRow.useCount], [null, 1, 0]); + assert.equal(await invites.revokeInvite(secondRow.id), true); + assert.deepEqual(await invites.redeemInvite(second.code, "guest"), { + success: false, + error: "Invite has been revoked", + }); +}); + +test("characterization: revokeInvite currently returns true for an unknown invite id", async () => { + assert.equal(await invites.revokeInvite("does-not-exist"), true); +}); + +test("servers: connect → list (no key hash exposed) → disconnect", async () => { + const conn = await servers.connectServer("Peer", "https://peer.example", "peer-key"); + assert.deepEqual( + { ...conn, id: typeof conn.id }, + { + id: "string", + name: "Peer", + url: "https://peer.example", + status: "connected", + lastSyncAt: null, + errorMessage: null, + } + ); + const listed = await servers.listServers(); + const mine = listed.find((s) => s.id === conn.id) as Record | undefined; + assert.ok(mine); + assert.equal("apiKeyHash" in mine, false); + await servers.disconnectServer(conn.id); +}); + +test("characterization: evaluateBadges currently rejects with SyntaxError when any definition has malformed criteria", async () => { + // The outer loop skips a malformed definition, but the `hidden` (secret-badge) branch + // re-parses EVERY definition without a guard, so one bad row aborts the whole evaluation. + core + .getDbInstance() + .prepare( + `INSERT INTO badge_definitions (id, name, description, icon, category, rarity, criteria, hidden) + VALUES ('char-broken', 'Broken', '', 'x', 'usage', 'common', 'not json', 0)` + ) + .run(); + await assert.rejects(badges.evaluateBadges("fx-fresh", "request"), SyntaxError); +}); diff --git a/tests/unit/characterization-webhooks.test.ts b/tests/unit/characterization-webhooks.test.ts new file mode 100644 index 000000000000..133757cd28b9 --- /dev/null +++ b/tests/unit/characterization-webhooks.test.ts @@ -0,0 +1,327 @@ +/** + * Characterization — src/lib/webhookDispatcher.ts + src/lib/webhooks/ (rail 3.8.55, + * pre-v4 extraction to @omniroute/mod-*). + * + * Pins the PUBLIC surface and the observable delivery behaviour (payload, HMAC signature, + * retries, per-kind routing, delivery bookkeeping) as it is today, against a real HTTP server + * on an ephemeral loopback port. Quirks are pinned, not fixed — their names start with + * "characterization: … currently …". Surface snapshots are literal: update them consciously. + * + * External consumers (the real boundary): open-sse/services/combo/{comboAttemptLoop, + * executeTargetAttempt}.ts + src/lib/quota/enforce.ts + src/lib/proxyEvents/ + * proxyTransitionBridge.ts (notifyWebhookEvent), src/app/api/webhooks/** (encryptMetadata, + * decryptMetadata, WEBHOOK_EVENT_VALUES, build{Slack,Telegram,Discord}Payload, + * buildTelegramUrl). + * + * Loopback delivery needs the private-URL opt-in (OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS): + * without it the SSRF guard (#12569) blocks 127.0.0.1 before any socket is opened. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import crypto from "node:crypto"; +import fs from "node:fs"; +import http from "node:http"; +import type { AddressInfo } from "node:net"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-char-webhooks-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS = "true"; + +const core = await import("../../src/lib/db/core.ts"); +const dispatcher = await import("../../src/lib/webhookDispatcher.ts"); +const eventDescriptions = await import("../../src/lib/webhooks/eventDescriptions.ts"); +const slack = await import("../../src/lib/webhooks/integrations/slack.ts"); +const telegram = await import("../../src/lib/webhooks/integrations/telegram.ts"); +const discord = await import("../../src/lib/webhooks/integrations/discord.ts"); +const webhooksDb = await import("../../src/lib/db/webhooks.ts"); +const deliveriesDb = await import("../../src/lib/db/webhookDeliveries.ts"); + +// ─── Fake receiver on an ephemeral port ────────────────────────────────────── + +type Received = { path: string; headers: http.IncomingHttpHeaders; body: string }; +const received: Received[] = []; +const statusQueue: number[] = []; + +const server = http.createServer((req, res) => { + let body = ""; + req.on("data", (chunk) => (body += chunk)); + req.on("end", () => { + received.push({ path: req.url ?? "", headers: req.headers, body }); + res.statusCode = statusQueue.shift() ?? 200; + res.end("ok"); + }); +}); +await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); +const BASE = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + +test.after(async () => { + await new Promise((resolve) => server.close(() => resolve())); + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +function reset() { + received.length = 0; + statusQueue.length = 0; +} + +const PING = { + event: "test.ping" as const, + timestamp: "2026-10-09T00:00:00.000Z", + data: { message: "hi" }, +}; + +// ─── (a) Surface snapshots ─────────────────────────────────────────────────── + +test("surface: webhook modules runtime exports", () => { + assert.deepEqual( + { + webhookDispatcher: Object.keys(dispatcher).sort(), + eventDescriptions: Object.keys(eventDescriptions).sort(), + slack: Object.keys(slack).sort(), + telegram: Object.keys(telegram).sort(), + discord: Object.keys(discord).sort(), + }, + { + webhookDispatcher: [ + "decryptMetadata", + "deliverWebhook", + "dispatchEvent", + "encryptMetadata", + "notifyWebhookEvent", + ], + eventDescriptions: ["EVENT_DESCRIPTIONS", "WEBHOOK_EVENT_VALUES"], + slack: ["buildSlackPayload"], + telegram: ["buildTelegramPayload", "buildTelegramUrl"], + discord: ["buildDiscordPayload"], + } + ); +}); + +test("surface: event catalogue (values ↔ descriptions)", () => { + assert.deepEqual( + [...eventDescriptions.WEBHOOK_EVENT_VALUES], + [ + "request.completed", + "request.failed", + "quota.exceeded", + "proxy.set_aside", + "proxy.pool.exhausted", + "test.ping", + ] + ); + assert.deepEqual(Object.keys(eventDescriptions.EVENT_DESCRIPTIONS), [ + ...eventDescriptions.WEBHOOK_EVENT_VALUES, + ]); + for (const desc of Object.values(eventDescriptions.EVENT_DESCRIPTIONS)) { + assert.deepEqual(Object.keys(desc).sort(), ["description", "emoji", "exampleData", "label"]); + } +}); + +// ─── (b) Contracts — deliverWebhook against a real HTTP server ─────────────── + +test("deliverWebhook: POSTs the JSON payload with event headers and an HMAC-SHA256 signature", async () => { + reset(); + const result = await dispatcher.deliverWebhook(`${BASE}/signed`, PING, "s3cret"); + assert.deepEqual(result, { success: true, status: 200 }); + assert.equal(received.length, 1); + const [hit] = received; + assert.equal(hit.path, "/signed"); + assert.equal(hit.body, JSON.stringify(PING)); + assert.equal(hit.headers["content-type"], "application/json"); + assert.equal(hit.headers["user-agent"], "OmniRoute-Webhook/1.0"); + assert.equal(hit.headers["x-webhook-event"], "test.ping"); + assert.equal(hit.headers["x-webhook-timestamp"], PING.timestamp); + const expected = `sha256=${crypto.createHmac("sha256", "s3cret").update(hit.body).digest("hex")}`; + assert.equal(hit.headers["x-webhook-signature"], expected); +}); + +test("deliverWebhook: no secret → no signature header", async () => { + reset(); + await dispatcher.deliverWebhook(`${BASE}/unsigned`, PING, null); + assert.equal(received[0].headers["x-webhook-signature"], undefined); +}); + +test("deliverWebhook: 4xx is final — no retry, success=false with the status", async () => { + reset(); + statusQueue.push(404); + const result = await dispatcher.deliverWebhook(`${BASE}/gone`, PING, null, 3); + assert.deepEqual(result, { success: false, status: 404 }); + assert.equal(received.length, 1); +}); + +test("deliverWebhook: 5xx is retried with backoff and can recover", async () => { + reset(); + statusQueue.push(503, 200); + const result = await dispatcher.deliverWebhook(`${BASE}/flaky`, PING, null, 1); + assert.deepEqual(result, { success: true, status: 200 }); + assert.equal(received.length, 2); +}); + +test("characterization: deliverWebhook currently reports status 0 'Max retries exceeded' after persistent 5xx", async () => { + reset(); + statusQueue.push(500, 502); + const result = await dispatcher.deliverWebhook(`${BASE}/down`, PING, null, 1); + // The last upstream status (502) is not surfaced. + assert.deepEqual(result, { success: false, status: 0, error: "Max retries exceeded" }); + assert.equal(received.length, 2); +}); + +// ─── (c) Errors ────────────────────────────────────────────────────────────── + +test("deliverWebhook: cloud-metadata target fails closed immediately (no retries)", async () => { + const started = Date.now(); + const result = await dispatcher.deliverWebhook("http://169.254.169.254/hook", PING, null, 3); + assert.equal(result.success, false); + assert.equal(result.status, 0); + assert.equal(typeof result.error, "string"); + // Three retries would have cost ≥7s of backoff. + assert.ok(Date.now() - started < 3000, "blocked URL must not burn retries"); +}); + +test("deliverWebhook: connection refused with maxRetries=0 → success=false, status 0", async () => { + const probe = http.createServer(); + await new Promise((resolve) => probe.listen(0, "127.0.0.1", resolve)); + const port = (probe.address() as AddressInfo).port; + await new Promise((resolve) => probe.close(() => resolve())); + const result = await dispatcher.deliverWebhook(`http://127.0.0.1:${port}/x`, PING, null, 0); + assert.equal(result.success, false); + assert.equal(result.status, 0); + assert.equal(typeof result.error, "string"); +}); + +test("encrypt/decryptMetadata: round trip; null / garbage → null", () => { + const enc = dispatcher.encryptMetadata({ botToken: "t", chatId: "c" }); + assert.deepEqual(dispatcher.decryptMetadata(enc), { botToken: "t", chatId: "c" }); + assert.equal(dispatcher.decryptMetadata(null), null); + assert.equal(dispatcher.decryptMetadata(""), null); + assert.equal(dispatcher.decryptMetadata("not-json"), null); +}); + +// ─── (b) Contracts — dispatchEvent routing + bookkeeping ───────────────────── + +test("dispatchEvent: routes by subscription and kind, records deliveries and status", async () => { + reset(); + const custom = webhooksDb.createWebhook({ + url: `${BASE}/custom`, + events: ["test.ping"], + secret: "whsec_char", + }); + const otherEvent = webhooksDb.createWebhook({ url: `${BASE}/other`, events: ["request.failed"] }); + const slackHook = webhooksDb.createWebhook({ url: `${BASE}/slack`, kind: "slack" }); // events ["*"] + const tg = webhooksDb.createWebhook({ url: "-100123", events: ["test.ping"], kind: "telegram" }); + + await dispatcher.dispatchEvent("test.ping", { model: "m1" }); + + assert.deepEqual(received.map((r) => r.path).sort(), ["/custom", "/slack"]); + const customHit = received.find((r) => r.path === "/custom")!; + const customBody = JSON.parse(customHit.body); + assert.deepEqual(Object.keys(customBody).sort(), ["data", "event", "timestamp"]); + assert.equal(customBody.event, "test.ping"); + assert.deepEqual(customBody.data, { model: "m1" }); + assert.equal( + customHit.headers["x-webhook-signature"], + `sha256=${crypto.createHmac("sha256", "whsec_char").update(customHit.body).digest("hex")}` + ); + const slackHit = received.find((r) => r.path === "/slack")!; + assert.equal(slackHit.headers["x-webhook-signature"], undefined, "slack is not HMAC-wrapped"); + assert.match(JSON.parse(slackHit.body).text, /Test Ping\* on `m1`/); + + const summary = (id: string) => + deliveriesDb.getDeliveries(id, 10).map((d) => [d.event_type, d.status, d.http_status, d.error]); + assert.deepEqual(summary(custom.id), [["test.ping", "success", 200, null]]); + assert.deepEqual(summary(slackHook.id), [["test.ping", "success", 200, null]]); + assert.deepEqual(summary(otherEvent.id), []); + assert.deepEqual(summary(tg.id), [ + ["test.ping", "failed", null, "Missing Telegram botToken in metadata"], + ]); + + assert.equal(webhooksDb.getWebhook(custom.id)?.last_status, 200); + assert.equal(webhooksDb.getWebhook(custom.id)?.failure_count, 0); + assert.equal(webhooksDb.getWebhook(tg.id)?.failure_count, 1); + assert.equal(webhooksDb.getWebhook(tg.id)?.last_status, 0); + + for (const wh of [custom, otherEvent, slackHook, tg]) webhooksDb.deleteWebhook(wh.id); +}); + +test("dispatchEvent: a webhook reaching 10 consecutive failures is auto-disabled", async () => { + reset(); + const wh = webhooksDb.createWebhook({ url: `${BASE}/failing`, events: ["request.failed"] }); + for (let i = 0; i < 9; i++) webhooksDb.recordWebhookDelivery(wh.id, 0, false); + statusQueue.push(400); + await dispatcher.dispatchEvent("request.failed", {}); + const after = webhooksDb.getWebhook(wh.id); + assert.equal(after?.failure_count, 10); + assert.equal(after?.enabled, false); + webhooksDb.deleteWebhook(wh.id); +}); + +test("notifyWebhookEvent: fire-and-forget — returns undefined synchronously, never throws", async () => { + assert.equal(dispatcher.notifyWebhookEvent("request.completed", { model: "m" }), undefined); + // Let the detached dispatch settle so it never races the DB teardown in test.after. + await new Promise((resolve) => setTimeout(resolve, 100)); +}); + +// ─── (b)+(c) Integration payload builders ──────────────────────────────────── + +test("buildSlackPayload: title line + error, provider only shown without a model", () => { + const p = slack.buildSlackPayload("request.failed", { model: "gpt", error: "503" }); + assert.equal(p.text, "🚨 *Request Failed* on `gpt`\n*Error:* `503`"); + assert.equal( + (p.blocks as Array<{ type: string }>).map((b) => b.type).join(","), + "section,context" + ); + assert.equal( + slack.buildSlackPayload("quota.exceeded", { provider: "claude" }).text, + "📊 *Quota Exceeded*\n*Provider:* claude" + ); +}); + +test("buildDiscordPayload: embed with per-event colour, description fallback", () => { + const p = discord.buildDiscordPayload("request.completed", { model: "m" }); + assert.equal(p.embeds?.[0].title, "✅ Request Completed"); + assert.equal(p.embeds?.[0].description, "**Model:** `m`"); + assert.equal(p.embeds?.[0].color, 0x22c55e); + const proxy = discord.buildDiscordPayload("proxy.set_aside", {}); + assert.equal( + proxy.embeds?.[0].description, + eventDescriptions.EVENT_DESCRIPTIONS["proxy.set_aside"].description + ); + // proxy.* events have no dedicated colour → default indigo. + assert.equal(proxy.embeds?.[0].color, 0x6366f1); +}); + +test("buildTelegramPayload/Url: Markdown-escaped lines; malformed bot token throws", () => { + const p = telegram.buildTelegramPayload( + "request.failed", + { model: "gpt_4*x", latencyMs: 12, fallbackCount: 2, combo: "c" }, + "-100" + ); + assert.equal(p.chat_id, "-100"); + assert.equal(p.parse_mode, "Markdown"); + assert.deepEqual(p.text.split("\n").slice(0, 5), [ + "🚨 *Request Failed*", + "Model: `gpt\\_4\\*x`", + "Combo: `c`", + "Latency: `12ms`", + "Fallbacks: `2`", + ]); + const token = `123456:${"A".repeat(35)}`; + assert.equal( + telegram.buildTelegramUrl(token), + `https://api.telegram.org/bot${token}/sendMessage` + ); + assert.throws(() => telegram.buildTelegramUrl("bad"), { + message: "Invalid Telegram bot token format (expected :)", + }); +}); + +test("characterization: payload builders currently throw a TypeError on an unknown event", () => { + // The event is only typed, never validated at runtime: EVENT_DESCRIPTIONS[event] is undefined. + const unknown = "nope.event" as "test.ping"; + assert.throws(() => slack.buildSlackPayload(unknown, {}), TypeError); + assert.throws(() => discord.buildDiscordPayload(unknown, {}), TypeError); + assert.throws(() => telegram.buildTelegramPayload(unknown, {}, "1"), TypeError); +});