From 1c1b7e89a3af33717d8b57f1dd12fc5c65767273 Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Mon, 15 Sep 2025 15:45:38 +0100 Subject: [PATCH 1/9] test: split up e2e tests --- apps/gateway/src/api-individual.e2e.ts | 32 +- apps/gateway/src/api.e2e.ts | 1497 ----------------- apps/gateway/src/chat-api.e2e.ts | 833 +++++++++ apps/gateway/src/chat-full.e2e.ts | 165 ++ apps/gateway/src/chat-reasoning.e2e.ts | 94 ++ apps/gateway/src/chat-rs.e2e.ts | 147 ++ apps/gateway/src/chat-streaming.e2e.ts | 120 ++ apps/gateway/src/chat-toolcalls-result.e2e.ts | 138 ++ apps/gateway/src/chat-toolcalls.e2e.ts | 126 ++ 9 files changed, 1629 insertions(+), 1523 deletions(-) delete mode 100644 apps/gateway/src/api.e2e.ts create mode 100644 apps/gateway/src/chat-api.e2e.ts create mode 100644 apps/gateway/src/chat-full.e2e.ts create mode 100644 apps/gateway/src/chat-reasoning.e2e.ts create mode 100644 apps/gateway/src/chat-rs.e2e.ts create mode 100644 apps/gateway/src/chat-streaming.e2e.ts create mode 100644 apps/gateway/src/chat-toolcalls-result.e2e.ts create mode 100644 apps/gateway/src/chat-toolcalls.e2e.ts diff --git a/apps/gateway/src/api-individual.e2e.ts b/apps/gateway/src/api-individual.e2e.ts index a2ffe0b3b0..1f9f5bd6a7 100644 --- a/apps/gateway/src/api-individual.e2e.ts +++ b/apps/gateway/src/api-individual.e2e.ts @@ -1,6 +1,12 @@ import "dotenv/config"; import { beforeEach, describe, expect, test } from "vitest"; +import { + generateTestRequestId, + logMode, + validateLogByRequestId, +} from "@/chat-api.e2e"; + import { db, tables, eq } from "@llmgateway/db"; import { models, providers } from "@llmgateway/models"; @@ -12,13 +18,6 @@ import { readAll, } from "./test-utils/test-helpers"; -// Helper function to generate unique request IDs for tests -function generateTestRequestId(): string { - return `test-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; -} - -const logMode = process.env.LOG_MODE; - describe("e2e individual tests", () => { // Helper to create unique test data for each test to avoid conflicts async function createTestData(testId: string) { @@ -72,7 +71,6 @@ describe("e2e individual tests", () => { return { userId, orgId, projectId, userOrgId, token }; } - // Only clear cache before each test - avoid clearing logs as concurrent tests may be waiting for them beforeEach(async () => { await clearCache(); }); @@ -92,24 +90,6 @@ describe("e2e individual tests", () => { }); } - async function validateLogByRequestId(requestId: string) { - const log = await waitForLogByRequestId(requestId); - - if (logMode) { - console.log("log", JSON.stringify(log, null, 2)); - } - - expect(log.usedProvider).toBeTruthy(); - expect(log.errorDetails).toBeNull(); - expect(log.finishReason).not.toBeNull(); - expect(log.unifiedFinishReason).not.toBeNull(); - expect(log.unifiedFinishReason).toBeTruthy(); - expect(log.usedModel).toBeTruthy(); - expect(log.requestedModel).toBeTruthy(); - - return log; - } - function validateResponse(json: any) { expect(json).toHaveProperty("choices.[0].message.content"); diff --git a/apps/gateway/src/api.e2e.ts b/apps/gateway/src/api.e2e.ts deleted file mode 100644 index bb3ba55598..0000000000 --- a/apps/gateway/src/api.e2e.ts +++ /dev/null @@ -1,1497 +0,0 @@ -import "dotenv/config"; -import { - beforeAll, - beforeEach, - describe, - expect, - test, - type TestOptions, -} from "vitest"; - -import { db, tables } from "@llmgateway/db"; -import { - type ModelDefinition, - models, - type ProviderModelMapping, - providers, -} from "@llmgateway/models"; - -import { app } from "."; -import { - clearCache, - waitForLogByRequestId, - getProviderEnvVar, - readAll, -} from "./test-utils/test-helpers"; - -// Helper function to generate unique request IDs for tests -function generateTestRequestId(): string { - return `test-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; -} - -// Helper function to get test options with retry for CI environment -function getTestOptions(): TestOptions { - return process.env.CI ? { retry: 3 } : {}; -} - -console.log("running with test options:", getTestOptions()); - -const fullMode = process.env.FULL_MODE; -const logMode = process.env.LOG_MODE; - -// Parse TEST_MODELS environment variable -const testModelsEnv = process.env.TEST_MODELS; -const specifiedModels = testModelsEnv - ? testModelsEnv.split(",").map((m) => m.trim()) - : null; - -if (specifiedModels) { - console.log(`TEST_MODELS specified: ${specifiedModels.join(", ")}`); -} - -// Filter models based on test skip/only property -const hasOnlyModels = models.some((model) => - model.providers.some( - (provider: ProviderModelMapping) => provider.test === "only", - ), -); - -// Log if we're using "only" mode -if (hasOnlyModels) { - if (process.env.CI) { - throw new Error( - "Cannot use 'only' in test configuration when running in CI. Please remove 'only' from the test configuration and try again.", - ); - } - console.log( - "Running in 'only' mode - only testing models marked with test: 'only'", - ); -} - -const filteredModels = models - // Filter out auto/custom models - .filter((model) => !["custom", "auto"].includes(model.id)) - // Filter out deactivated models - .filter((model) => !model.deactivatedAt || new Date() <= model.deactivatedAt) - // Filter out unstable models if not in full mode, unless they have test: "only" or are in TEST_MODELS - .filter((model) => { - // Check if model or any of its providers are marked as unstable - const modelStability = (model as ModelDefinition).stability; - const hasUnstableProviders = model.providers.some( - (provider: ProviderModelMapping) => provider.stability === "unstable", - ); - const isUnstable = modelStability === "unstable" || hasUnstableProviders; - - if (!isUnstable) { - return true; - } // Non-unstable models are always included - if (fullMode) { - return true; - } // In full mode, all models are included - - // For unstable models in non-full mode, include if: - // 1. Any provider has test: "only" - if ( - model.providers.some( - (provider: ProviderModelMapping) => provider.test === "only", - ) - ) { - return true; - } - - // 2. Model is specified in TEST_MODELS - if (specifiedModels) { - const modelInTestModels = model.providers.some( - (provider: ProviderModelMapping) => { - const providerModelId = `${provider.providerId}/${model.id}`; - return specifiedModels.includes(providerModelId); - }, - ); - if (modelInTestModels) { - return true; - } - } - - return false; // Otherwise, exclude unstable models in non-full mode - }) - // Filter out free models if not in full mode, unless they have test: "only" or are in TEST_MODELS - .filter((model) => { - const isFreeModel = (model as ModelDefinition).free; - if (!isFreeModel) { - return true; - } // Non-free models are always included - if (fullMode) { - return true; - } // In full mode, all models are included - - // For free models in non-full mode, include if: - // 1. Any provider has test: "only" - if ( - model.providers.some( - (provider: ProviderModelMapping) => provider.test === "only", - ) - ) { - return true; - } - - // 2. Model is specified in TEST_MODELS - if (specifiedModels) { - const modelInTestModels = model.providers.some( - (provider: ProviderModelMapping) => { - const providerModelId = `${provider.providerId}/${model.id}`; - return specifiedModels.includes(providerModelId); - }, - ); - if (modelInTestModels) { - return true; - } - } - - return false; // Otherwise, exclude free models in non-full mode - }) - // Filter by TEST_MODELS if specified - .filter((model) => { - if (!specifiedModels) { - return true; - } - // Check if any provider/model combination from this model matches TEST_MODELS - return model.providers.some((provider: ProviderModelMapping) => { - const providerModelId = `${provider.providerId}/${model.id}`; - return specifiedModels.includes(providerModelId); - }); - }); - -const testModels = filteredModels - // If any model has test: "only", only include those models - .filter((model) => { - if (hasOnlyModels) { - return model.providers.some( - (provider: ProviderModelMapping) => provider.test === "only", - ); - } - return true; - }) - .flatMap((model) => { - const testCases = []; - - if (process.env.TEST_ALL_VARIATIONS) { - // test root model without a specific provider - testCases.push({ - model: model.id, - providers: model.providers.filter( - (provider: ProviderModelMapping) => provider.test !== "skip", - ), - }); - } - - // Create entries for provider-specific requests using provider/model format - for (const provider of model.providers as ProviderModelMapping[]) { - // Skip providers marked with test: "skip" - if (provider.test === "skip") { - continue; - } - - // Skip unstable providers if not in full mode, unless they have test: "only" or are in TEST_MODELS - if (provider.stability === "unstable" && !fullMode) { - // Allow if provider has test: "only" - if (provider.test !== "only") { - // Allow if model is specified in TEST_MODELS - if (!specifiedModels) { - continue; - } - const providerModelId = `${provider.providerId}/${model.id}`; - if (!specifiedModels.includes(providerModelId)) { - continue; - } - } - } - - // If we have any "only" providers, skip those not marked as "only" - if (hasOnlyModels && provider.test !== "only") { - continue; - } - - testCases.push({ - model: `${provider.providerId}/${model.id}`, - providers: [provider], - originalModel: model.id, // Keep track of the original model for reference - }); - } - - return testCases; - }); - -const providerModels = filteredModels - // If any model has test: "only", only include those models - .filter((model) => { - if (hasOnlyModels) { - return model.providers.some( - (provider: ProviderModelMapping) => provider.test === "only", - ); - } - return true; - }) - .flatMap((model) => { - const testCases = []; - - for (const provider of model.providers as ProviderModelMapping[]) { - // Skip providers marked with test: "skip" - if (provider.test === "skip") { - continue; - } - - // Skip unstable providers if not in full mode, unless they have test: "only" or are in TEST_MODELS - if (provider.stability === "unstable" && !fullMode) { - // Allow if provider has test: "only" - if (provider.test !== "only") { - // Allow if model is specified in TEST_MODELS - if (!specifiedModels) { - continue; - } - const providerModelId = `${provider.providerId}/${model.id}`; - if (!specifiedModels.includes(providerModelId)) { - continue; - } - } - } - - // If we have any "only" providers, skip those not marked as "only" - if (hasOnlyModels && provider.test !== "only") { - continue; - } - - testCases.push({ - model: `${provider.providerId}/${model.id}`, - provider, - originalModel: model.id, // Keep track of the original model for reference - }); - } - - return testCases; - }); - -// Log the number of test models after filtering -console.log(`Testing ${testModels.length} model configurations`); -console.log(`Testing ${providerModels.length} provider model configurations`); - -const streamingModels = testModels.filter((m) => - m.providers.some((p: ProviderModelMapping) => { - // Check model-level streaming first, then fall back to provider-level - if (p.streaming !== undefined) { - return p.streaming; - } - const provider = providers.find((pr) => pr.id === p.providerId); - return provider?.streaming; - }), -); - -const reasoningModels = testModels.filter((m) => - m.providers.some((p: ProviderModelMapping) => p.reasoning === true), -); - -const streamingReasoningModels = reasoningModels.filter((m) => - m.providers.some((p: ProviderModelMapping) => { - // Check model-level streaming first, then fall back to provider-level - if (p.streaming !== undefined) { - return p.streaming; - } - const provider = providers.find((pr) => pr.id === p.providerId); - return provider?.streaming; - }), -); - -const toolCallModels = testModels.filter((m) => - m.providers.some((p: ProviderModelMapping) => p.tools === true), -); - -const imageModels = testModels.filter((m) => { - const model = models.find((mo) => m.originalModel === mo.id); - return (model as ModelDefinition).output?.includes("image"); -}); - -const streamingImageModels = imageModels.filter((m) => - m.providers.some((p: ProviderModelMapping) => { - // Check model-level streaming first, then fall back to provider-level - if (p.streaming !== undefined) { - return p.streaming; - } - const provider = providers.find((pr) => pr.id === p.providerId); - return provider?.streaming; - }), -); - -describe("e2e", { concurrent: true }, () => { - // Set up database once before all tests - beforeAll(async () => { - await clearCache(); - - // Clean up any existing data - await Promise.all([ - db.delete(tables.log), - db.delete(tables.apiKey), - db.delete(tables.providerKey), - ]); - - await Promise.all([ - db.delete(tables.userOrganization), - db.delete(tables.project), - ]); - - await Promise.all([ - db.delete(tables.organization), - db.delete(tables.user), - db.delete(tables.account), - db.delete(tables.session), - db.delete(tables.verification), - ]); - - // Set up shared test data that all tests can use - await db.insert(tables.user).values({ - id: "user-id", - name: "user", - email: "user", - }); - - await db.insert(tables.organization).values({ - id: "org-id", - name: "Test Organization", - plan: "pro", - }); - - await db.insert(tables.userOrganization).values({ - id: "user-org-id", - userId: "user-id", - organizationId: "org-id", - }); - - await db.insert(tables.project).values({ - id: "project-id", - name: "Test Project", - organizationId: "org-id", - mode: "api-keys", - }); - - await db.insert(tables.apiKey).values({ - id: "token-id", - token: "real-token", - projectId: "project-id", - description: "Test API Key", - }); - - // Set up provider keys for all providers - for (const provider of providers) { - const envVarName = getProviderEnvVar(provider.id); - const envVarValue = envVarName ? process.env[envVarName] : undefined; - if (envVarValue) { - await createProviderKey(provider.id, envVarValue, "api-keys"); - await createProviderKey(provider.id, envVarValue, "credits"); - } - } - }); - - // Only clear cache before each test - avoid clearing logs as concurrent tests may be waiting for them - beforeEach(async () => { - await clearCache(); - }); - - async function createProviderKey( - provider: string, - token: string, - keyType: "api-keys" | "credits" = "api-keys", - ) { - const keyId = - keyType === "credits" ? `env-${provider}` : `provider-key-${provider}`; - await db.insert(tables.providerKey).values({ - id: keyId, - token, - provider: provider.replace("env-", ""), // Remove env- prefix for the provider field - organizationId: "org-id", - }); - } - - function validateResponse(json: any) { - expect(json).toHaveProperty("choices.[0].message.content"); - - expect(json).toHaveProperty("usage.prompt_tokens"); - expect(json).toHaveProperty("usage.completion_tokens"); - expect(json).toHaveProperty("usage.total_tokens"); - } - - async function validateLogByRequestId(requestId: string) { - const log = await waitForLogByRequestId(requestId); - - if (logMode) { - console.log("log", JSON.stringify(log, null, 2)); - } - - expect(log.usedProvider).toBeTruthy(); - expect(log.errorDetails).toBeNull(); - expect(log.finishReason).not.toBeNull(); - expect(log.unifiedFinishReason).not.toBeNull(); - expect(log.unifiedFinishReason).toBeTruthy(); - expect(log.usedModel).toBeTruthy(); - expect(log.requestedModel).toBeTruthy(); - - return log; - } - - test.each(testModels)( - "completions $model", - getTestOptions(), - async ({ model }) => { - const requestId = generateTestRequestId(); - const res = await app.request("/v1/chat/completions", { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-request-id": requestId, - Authorization: `Bearer real-token`, - }, - body: JSON.stringify({ - model: model, - messages: [ - { - role: "system", - content: "You are a helpful assistant.", - }, - { - role: "user", - content: "Hello, just reply 'OK'!", - }, - ], - }), - }); - - const json = await res.json(); - if (logMode) { - console.log("response:", JSON.stringify(json, null, 2)); - } - - expect(res.status).toBe(200); - validateResponse(json); - - const log = await validateLogByRequestId(requestId); - expect(log.streamed).toBe(false); - - expect(json).toHaveProperty("usage"); - expect(json.usage).toHaveProperty("prompt_tokens"); - expect(json.usage).toHaveProperty("completion_tokens"); - expect(json.usage).toHaveProperty("total_tokens"); - expect(typeof json.usage.prompt_tokens).toBe("number"); - expect(typeof json.usage.completion_tokens).toBe("number"); - expect(typeof json.usage.total_tokens).toBe("number"); - expect(json.usage.prompt_tokens).toBeGreaterThan(0); - expect(json.usage.completion_tokens).toBeGreaterThan(0); - expect(json.usage.total_tokens).toBeGreaterThan(0); - - // expect(log.inputCost).not.toBeNull(); - // expect(log.outputCost).not.toBeNull(); - // expect(log.cost).not.toBeNull(); - }, - ); - - test.each(streamingModels)( - "/v1/chat/completions streaming with $model", - getTestOptions(), - async ({ model }) => { - const requestId = generateTestRequestId(); - const res = await app.request("/v1/chat/completions", { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-request-id": requestId, - Authorization: `Bearer real-token`, - }, - body: JSON.stringify({ - model: model, - messages: [ - { - role: "system", - content: "You are a helpful assistant.", - }, - { - role: "user", - content: "Hello! This is a streaming e2e test.", - }, - ], - stream: true, - }), - }); - - if (res.status !== 200) { - console.log("response:", await res.text()); - throw new Error(`Request failed with status ${res.status}`); - } - - expect(res.status).toBe(200); - expect(res.headers.get("content-type")).toContain("text/event-stream"); - - const streamResult = await readAll(res.body); - if (logMode) { - console.log("streamResult", JSON.stringify(streamResult, null, 2)); - } - - expect(streamResult.hasValidSSE).toBe(true); - expect(streamResult.eventCount).toBeGreaterThan(0); - expect(streamResult.hasContent).toBe(true); - - // Verify that all streaming responses are transformed to OpenAI format - expect(streamResult.hasOpenAIFormat).toBe(true); - - // Verify that chunks have the correct OpenAI streaming format - const contentChunks = streamResult.chunks.filter( - (chunk) => chunk.choices?.[0]?.delta?.content, - ); - expect(contentChunks.length).toBeGreaterThan(0); - - // Verify each content chunk has proper OpenAI format - for (const chunk of contentChunks) { - expect(chunk).toHaveProperty("id"); - expect(chunk).toHaveProperty("object", "chat.completion.chunk"); - expect(chunk).toHaveProperty("created"); - expect(chunk).toHaveProperty("model"); - expect(chunk).toHaveProperty("choices"); - expect(chunk.choices).toHaveLength(1); - expect(chunk.choices[0]).toHaveProperty("index", 0); - expect(chunk.choices[0]).toHaveProperty("delta"); - expect(chunk.choices[0]).toHaveProperty("delta.role", "assistant"); - expect(chunk.choices[0].delta).toHaveProperty("content"); - expect(typeof chunk.choices[0].delta.content).toBe("string"); - } - - // Verify that usage object is returned in streaming mode - const usageChunks = streamResult.chunks.filter( - (chunk) => - chunk.usage && - (chunk.usage.prompt_tokens !== null || - chunk.usage.completion_tokens !== null || - chunk.usage.total_tokens !== null), - ); - expect(usageChunks.length).toBeGreaterThan(0); - - // Verify the usage chunk has proper format - const usageChunk = usageChunks[usageChunks.length - 1]; // Get the last usage chunk - expect(usageChunk).toHaveProperty("usage"); - expect(usageChunk.usage).toHaveProperty("prompt_tokens"); - expect(usageChunk.usage).toHaveProperty("completion_tokens"); - expect(usageChunk.usage).toHaveProperty("total_tokens"); - expect(typeof usageChunk.usage.prompt_tokens).toBe("number"); - expect(typeof usageChunk.usage.completion_tokens).toBe("number"); - expect(typeof usageChunk.usage.total_tokens).toBe("number"); - expect(usageChunk.usage.prompt_tokens).toBeGreaterThan(0); - expect(usageChunk.usage.completion_tokens).toBeGreaterThan(0); - expect(usageChunk.usage.total_tokens).toBeGreaterThan(0); - - const log = await validateLogByRequestId(requestId); - expect(log.streamed).toBe(true); - - // expect(log.cost).not.toBeNull(); - // expect(log.cost).toBeGreaterThanOrEqual(0); - }, - ); - - test.each(reasoningModels)( - "reasoning $model", - getTestOptions(), - async ({ model, providers }) => { - const requestId = generateTestRequestId(); - const res = await app.request("/v1/chat/completions", { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-request-id": requestId, - Authorization: `Bearer real-token`, - }, - body: JSON.stringify({ - model: model, - messages: [ - { - role: "system", - content: "You are a helpful assistant.", - }, - { - role: "user", - content: "What is 2/3 + 1/4 + 5/6?", - }, - ], - reasoning_effort: "medium", - }), - }); - - const json = await res.json(); - if (logMode) { - console.log("reasoning response:", JSON.stringify(json, null, 2)); - } - - expect(res.status).toBe(200); - validateResponse(json); - - const log = await validateLogByRequestId(requestId); - expect(log.streamed).toBe(false); - - expect(json).toHaveProperty("usage"); - expect(json.usage).toHaveProperty("prompt_tokens"); - expect(json.usage).toHaveProperty("completion_tokens"); - expect(json.usage).toHaveProperty("total_tokens"); - expect(typeof json.usage.prompt_tokens).toBe("number"); - expect(typeof json.usage.completion_tokens).toBe("number"); - expect(typeof json.usage.total_tokens).toBe("number"); - expect(json.usage.prompt_tokens).toBeGreaterThan(0); - expect(json.usage.completion_tokens).toBeGreaterThan(0); - expect(json.usage.total_tokens).toBeGreaterThan(0); - - // Check for reasoning tokens if available - if (json.usage.reasoning_tokens !== undefined) { - expect(typeof json.usage.reasoning_tokens).toBe("number"); - expect(json.usage.reasoning_tokens).toBeGreaterThanOrEqual(0); - } - - // check for reasoning response - only if the provider expects reasoning output - const reasoningProvider = providers?.find( - (p: ProviderModelMapping) => p.reasoning === true, - ) as ProviderModelMapping; - const useResponsesApi = process.env.USE_RESPONSES_API === "true"; - const isOpenAI = reasoningProvider?.providerId === "openai"; - // only enforce reasoning_content checks for where reasoningOutput is not "omit" and for openai, only if the responses api is used - if ( - reasoningProvider?.reasoningOutput !== "omit" && - (!isOpenAI || useResponsesApi) - ) { - expect(json.choices[0].message).toHaveProperty("reasoning_content"); - } - }, - ); - - test.each(streamingReasoningModels)( - "reasoning + streaming $model", - getTestOptions(), - async ({ model, providers }) => { - const requestId = generateTestRequestId(); - const res = await app.request("/v1/chat/completions", { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-request-id": requestId, - Authorization: `Bearer real-token`, - }, - body: JSON.stringify({ - model: model, - messages: [ - { - role: "system", - content: - "You are a helpful assistant. Think step by step and show your reasoning.", - }, - { - role: "user", - content: - "Solve this logic puzzle: If all roses are flowers, and some flowers are red, what can we conclude about roses? Think through this step by step.", - }, - ], - reasoning_effort: "medium", - stream: true, - }), - }); - - if (res.status !== 200) { - console.log("response:", await res.text()); - throw new Error(`Request failed with status ${res.status}`); - } - - expect(res.status).toBe(200); - expect(res.headers.get("content-type")).toContain("text/event-stream"); - - const streamResult = await readAll(res.body); - if (logMode) { - console.log( - "reasoning streaming response:", - JSON.stringify(streamResult.chunks, null, 2), - ); - } - - expect(streamResult.hasValidSSE).toBe(true); - expect(streamResult.eventCount).toBeGreaterThan(0); - expect(streamResult.hasContent).toBe(true); - - // Verify that all streaming responses are transformed to OpenAI format - expect(streamResult.hasOpenAIFormat).toBe(true); - - // Verify that chunks have the correct OpenAI streaming format - const contentChunks = streamResult.chunks.filter( - (chunk: any) => chunk.choices?.[0]?.delta?.content, - ); - expect(contentChunks.length).toBeGreaterThan(0); - - // Verify each content chunk has proper OpenAI format - for (const chunk of contentChunks) { - expect(chunk).toHaveProperty("id"); - expect(chunk).toHaveProperty("object", "chat.completion.chunk"); - expect(chunk).toHaveProperty("created"); - expect(chunk).toHaveProperty("model"); - expect(chunk).toHaveProperty("choices"); - expect(chunk.choices).toHaveLength(1); - expect(chunk.choices[0]).toHaveProperty("delta"); - expect(typeof chunk.choices[0].delta.content).toBe("string"); - } - - // Verify that usage object is returned in streaming mode - const usageChunks = streamResult.chunks.filter( - (chunk: any) => - chunk.usage && - (chunk.usage.prompt_tokens !== null || - chunk.usage.completion_tokens !== null || - chunk.usage.total_tokens !== null), - ); - expect(usageChunks.length).toBeGreaterThan(0); - - // Verify the usage chunk has proper format - const usageChunk = usageChunks[usageChunks.length - 1]; // Get the last usage chunk - expect(usageChunk).toHaveProperty("usage"); - expect(usageChunk.usage).toHaveProperty("prompt_tokens"); - expect(usageChunk.usage).toHaveProperty("completion_tokens"); - expect(usageChunk.usage).toHaveProperty("total_tokens"); - expect(typeof usageChunk.usage.prompt_tokens).toBe("number"); - expect(typeof usageChunk.usage.completion_tokens).toBe("number"); - expect(typeof usageChunk.usage.total_tokens).toBe("number"); - expect(usageChunk.usage.prompt_tokens).toBeGreaterThan(0); - expect(usageChunk.usage.completion_tokens).toBeGreaterThan(0); - expect(usageChunk.usage.total_tokens).toBeGreaterThan(0); - - // Check for reasoning tokens if available - if (usageChunk.usage.reasoning_tokens !== undefined) { - expect(typeof usageChunk.usage.reasoning_tokens).toBe("number"); - expect(usageChunk.usage.reasoning_tokens).toBeGreaterThanOrEqual(0); - } - - // Verify reasoning content is present in unified reasoning_content field - only if the provider expects reasoning output - const reasoningProvider = providers?.find( - (p: ProviderModelMapping) => p.reasoning === true, - ) as ProviderModelMapping; - const useResponsesApi = process.env.USE_RESPONSES_API === "true"; - const isOpenAI = reasoningProvider?.providerId === "openai"; - // When using the Responses API, only enforce reasoning_content checks for OpenAI. - if ( - reasoningProvider?.reasoningOutput !== "omit" && - (!isOpenAI || useResponsesApi) - ) { - const reasoningChunks = streamResult.chunks.filter( - (chunk: any) => - chunk.choices?.[0]?.delta?.reasoning_content && - chunk.choices[0].delta.reasoning_content.length > 0, - ); - expect(reasoningChunks.length).toBeGreaterThan(0); - } - - const log = await validateLogByRequestId(requestId); - expect(log.streamed).toBe(true); - }, - ); - - if (fullMode) { - const reasoningToolCallModels = testModels.filter((m) => - m.providers.some( - (p: ProviderModelMapping) => p.reasoning === true && p.tools === true, - ), - ); - - test.each(reasoningToolCallModels)( - "reasoning + tool calls $model", - getTestOptions(), - async ({ model, providers }) => { - const requestId = generateTestRequestId(); - const res = await app.request("/v1/chat/completions", { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-request-id": requestId, - Authorization: `Bearer real-token`, - }, - body: JSON.stringify({ - model: model, - messages: [ - { - role: "system", - content: - "You are a weather assistant that can get weather information for cities. Think step by step and use tools when needed.", - }, - { - role: "user", - content: - "What's the weather like in San Francisco? Consider all the exact details. Use the weather tool and explain your reasoning.", - }, - ], - tools: [ - { - type: "function", - function: { - name: "get_weather", - description: "Get the current weather for a given city", - parameters: { - type: "object", - properties: { - city: { - type: "string", - description: "The city name to get weather for", - }, - unit: { - type: "string", - enum: ["celsius", "fahrenheit"], - description: "Temperature unit", - default: "fahrenheit", - }, - }, - required: ["city"], - }, - }, - }, - ], - tool_choice: "auto", - reasoning_effort: "medium", - }), - }); - - const json = await res.json(); - if (logMode) { - console.log( - "reasoning + tool calls response:", - JSON.stringify(json, null, 2), - ); - } - - expect(res.status).toBe(200); - expect(json).toHaveProperty("choices"); - expect(json.choices).toHaveLength(1); - expect(json.choices[0]).toHaveProperty("message"); - - const message = json.choices[0].message; - expect(message).toHaveProperty("role", "assistant"); - - // Should have tool calls since we're asking about weather - expect(message).toHaveProperty("tool_calls"); - expect(Array.isArray(message.tool_calls)).toBe(true); - expect(message.tool_calls.length).toBeGreaterThan(0); - - // Validate tool call structure - const toolCall = message.tool_calls[0]; - expect(toolCall).toHaveProperty("id"); - expect(toolCall).toHaveProperty("type", "function"); - expect(toolCall).toHaveProperty("function"); - expect(toolCall.function).toHaveProperty("name", "get_weather"); - expect(toolCall.function).toHaveProperty("arguments"); - - // Parse and validate arguments - const args = JSON.parse(toolCall.function.arguments); - expect(args).toHaveProperty("city"); - expect(typeof args.city).toBe("string"); - expect(args.city.toLowerCase()).toContain("san francisco"); - - // Check finish reason - expect(json.choices[0]).toHaveProperty("finish_reason", "tool_calls"); - - // Check for reasoning content - only if the provider expects reasoning output - const reasoningProvider = providers?.find( - (p: ProviderModelMapping) => p.reasoning === true, - ); - if ( - (reasoningProvider as ProviderModelMapping)?.reasoningOutput !== - "omit" - ) { - expect(json.choices[0].message).toHaveProperty("reasoning_content"); - expect(typeof json.choices[0].message.reasoning_content).toBe( - "string", - ); - expect( - json.choices[0].message.reasoning_content.length, - ).toBeGreaterThan(0); - } - - // Validate logs - const log = await validateLogByRequestId(requestId); - expect(log.streamed).toBe(false); - - // Validate usage - expect(json).toHaveProperty("usage"); - expect(json.usage).toHaveProperty("prompt_tokens"); - expect(json.usage).toHaveProperty("completion_tokens"); - expect(json.usage).toHaveProperty("total_tokens"); - expect(typeof json.usage.prompt_tokens).toBe("number"); - expect(typeof json.usage.completion_tokens).toBe("number"); - expect(typeof json.usage.total_tokens).toBe("number"); - expect(json.usage.prompt_tokens).toBeGreaterThan(0); - expect(json.usage.completion_tokens).toBeGreaterThan(0); - expect(json.usage.total_tokens).toBeGreaterThan(0); - - // Check for reasoning tokens if available - if (json.usage.reasoning_tokens !== undefined) { - expect(typeof json.usage.reasoning_tokens).toBe("number"); - expect(json.usage.reasoning_tokens).toBeGreaterThanOrEqual(0); - } - }, - ); - } - - test.each(toolCallModels)( - "tool calls $model", - getTestOptions(), - async ({ model }) => { - const requestId = generateTestRequestId(); - const res = await app.request("/v1/chat/completions", { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-request-id": requestId, - Authorization: `Bearer real-token`, - }, - body: JSON.stringify({ - model: model, - messages: [ - { - role: "system", - content: - "You are a weather assistant that can get weather information for cities.", - }, - { - role: "user", - content: "What's the weather like in San Francisco?", - }, - ], - tools: [ - { - type: "function", - function: { - name: "get_weather", - description: "Get the current weather for a given city", - parameters: { - type: "object", - properties: { - city: { - type: "string", - description: "The city name to get weather for", - }, - unit: { - type: "string", - enum: ["celsius", "fahrenheit"], - description: "Temperature unit", - default: "fahrenheit", - }, - }, - required: ["city"], - }, - }, - }, - ], - tool_choice: "auto", - }), - }); - - const json = await res.json(); - if (logMode) { - console.log("tool calls response:", JSON.stringify(json, null, 2)); - } - - expect(res.status).toBe(200); - expect(json).toHaveProperty("choices"); - expect(json.choices).toHaveLength(1); - expect(json.choices[0]).toHaveProperty("message"); - - const message = json.choices[0].message; - expect(message).toHaveProperty("role", "assistant"); - - // Should have tool calls since we're asking about weather - expect(message).toHaveProperty("tool_calls"); - expect(Array.isArray(message.tool_calls)).toBe(true); - expect(message.tool_calls.length).toBeGreaterThan(0); - - // Validate tool call structure - const toolCall = message.tool_calls[0]; - expect(toolCall).toHaveProperty("id"); - expect(toolCall).toHaveProperty("type", "function"); - expect(toolCall).toHaveProperty("function"); - expect(toolCall.function).toHaveProperty("name", "get_weather"); - expect(toolCall.function).toHaveProperty("arguments"); - - // Parse and validate arguments - const args = JSON.parse(toolCall.function.arguments); - expect(args).toHaveProperty("city"); - expect(typeof args.city).toBe("string"); - expect(args.city.toLowerCase()).toContain("san francisco"); - - // Check finish reason - expect(json.choices[0]).toHaveProperty("finish_reason", "tool_calls"); - - // Validate logs - const log = await validateLogByRequestId(requestId); - expect(log.streamed).toBe(false); - - // Validate usage - expect(json).toHaveProperty("usage"); - expect(json.usage).toHaveProperty("prompt_tokens"); - expect(json.usage).toHaveProperty("completion_tokens"); - expect(json.usage).toHaveProperty("total_tokens"); - expect(typeof json.usage.prompt_tokens).toBe("number"); - expect(typeof json.usage.completion_tokens).toBe("number"); - expect(typeof json.usage.total_tokens).toBe("number"); - expect(json.usage.prompt_tokens).toBeGreaterThan(0); - expect(json.usage.completion_tokens).toBeGreaterThan(0); - expect(json.usage.total_tokens).toBeGreaterThan(0); - }, - ); - - test.each(toolCallModels)( - "tool calls with result $model", - getTestOptions(), - async ({ model }) => { - const requestId = generateTestRequestId(); - const res = await app.request("/v1/chat/completions", { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-request-id": requestId, - Authorization: `Bearer real-token`, - }, - body: JSON.stringify({ - model: model, - messages: [ - { - role: "system", - content: - "You are Noemi, a thoughtful and clear assistant. Your tone is calm, minimal, and human. You write with intention—never too much, never too little. You avoid clichés, speak simply, and offer helpful, grounded answers. When needed, you ask good questions. You don't try to impress—you aim to clarify. You may use metaphors if they bring clarity, but you stay sharp and sincere. You're here to help the user think clearly and move forward, not to overwhelm or overperform.", - }, - { - role: "user", - content: "web search for the best ai notetaker apps!!!!", - }, - { - role: "assistant", - content: "", - tool_calls: [ - { - id: "toolu_015dgN1nk5Ay12iN8e16XPbs", - type: "function", - function: { - name: "webSearch", - arguments: '{"query":"best AI notetaker apps 2024"}', - }, - }, - ], - }, - { - role: "tool", - content: - '{"type":"webSearch","query":"best AI notetaker apps 2024","results":[{"title":"My Deep Dive into 25+ AI Note-Taking Apps (The Brutally ... - Reddit","href":"https://www.reddit.com/r/Zoom/comments/1jtbxkf/my_deep_dive_into_25_ai_notetaking_apps_the/","description":"The Good: Think Obsidian meets Miro. Whiteboard-style interface for connecting notes visually. AI assistant can generate summaries and do ..."},{"title":"The 9 best AI meeting assistants in 2025 - Zapier","href":"https://zapier.com/blog/best-ai-meeting-assistant/","description":"Granola automatically transcribes, summarizes, and analyzes your meetings. It also acts as a live notepad, allowing you to manually jot down ..."},{"title":"The Best AI Tools for Taking Notes in 2025 - PCMag","href":"https://www.pcmag.com/picks/best-ai-tools-taking-notes","description":"The popular note-taking app Notion now has AI tools. Notion AI excels at answering questions about your existing data, generating text from a prompt you give it ..."},{"title":"Top 5 BEST AI Note-Taking Apps (Better than Notion?) - YouTube","href":"https://www.youtube.com/watch?v=wGLd43TkCGc","description":"Voicenotes is a voice‑to‑text powerhouse that transcribes and extracts action items in one tap. · Saner is A distraction‑free workspace built for ..."},{"title":"9 Best AI Note-Taking Apps Built For Your Meetings - Quil\'s AI","href":"https://quil.ai/2024/09/12/9-best-ai-note-taking-apps-built-for-your-meetings/","description":"Quil.ai: The AI Note-taker Built for Recruiting Firms. 2. Notion: Write, Plan, Organize. 3. Jamie AI: The Bot-Free AI Note-taker."}],"timestamp":"2025-08-29T01:20:29.553Z"}', - tool_call_id: "toolu_015dgN1nk5Ay12iN8e16XPbs", - }, - ], - tools: [ - { - type: "function", - function: { - name: "webSearch", - description: "Search the web for information", - parameters: { - type: "object", - properties: { - query: { - type: "string", - description: "Search query", - }, - }, - required: ["query"], - }, - }, - }, - ], - tool_choice: "auto", - }), - }); - - const json = await res.json(); - if (logMode) { - console.log( - "tool calls with empty content response:", - JSON.stringify(json, null, 2), - ); - } - - // Log error response if status is not 200 - if (res.status !== 200) { - console.log( - `Error ${res.status} - tool calls with result response:`, - JSON.stringify(json, null, 2), - ); - } - - expect(res.status).toBe(200); - expect(json).toHaveProperty("choices"); - expect(json.choices).toHaveLength(1); - expect(json.choices[0]).toHaveProperty("message"); - - const message = json.choices[0].message; - expect(message).toHaveProperty("role", "assistant"); - - // Should have proper content (not empty) as a response to the tool call - expect(message).toHaveProperty("content"); - // verify either content is string or tool_calls is present - expect(message.content || message.tool_calls).toBeTruthy(); - - // Should have finish reason as stop (not tool_calls since this is a response) - // TODO THIS IS FAILING ON SOME MODELS - // expect(json.choices[0]).toHaveProperty("finish_reason", "stop"); - - // Validate logs - const log = await validateLogByRequestId(requestId); - expect(log.streamed).toBe(false); - - // Validate usage - expect(json).toHaveProperty("usage"); - expect(json.usage).toHaveProperty("prompt_tokens"); - expect(json.usage).toHaveProperty("completion_tokens"); - expect(json.usage).toHaveProperty("total_tokens"); - expect(typeof json.usage.prompt_tokens).toBe("number"); - expect(typeof json.usage.completion_tokens).toBe("number"); - expect(typeof json.usage.total_tokens).toBe("number"); - expect(json.usage.prompt_tokens).toBeGreaterThan(0); - expect(json.usage.completion_tokens).toBeGreaterThan(0); - expect(json.usage.total_tokens).toBeGreaterThan(0); - }, - ); - - test.each( - testModels.filter((m) => { - const modelDef = models.find((def) => def.id === m.model); - return (modelDef as ModelDefinition)?.jsonOutput === true; - }), - )("JSON output $model", getTestOptions(), async ({ model }) => { - const res = await app.request("/v1/chat/completions", { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer real-token`, - }, - body: JSON.stringify({ - model: model, - messages: [ - { - role: "system", - content: - "You are a helpful assistant. Always respond with valid JSON.", - }, - { - role: "user", - content: 'Return a JSON object with "message": "Hello World"', - }, - ], - response_format: { type: "json_object" }, - }), - }); - - const json = await res.json(); - if (logMode) { - console.log("json", JSON.stringify(json, null, 2)); - } - expect(res.status).toBe(200); - expect(json).toHaveProperty("choices.[0].message.content"); - - const content = json.choices[0].message.content; - expect(() => JSON.parse(content)).not.toThrow(); - - const parsedContent = JSON.parse(content); - expect(parsedContent).toHaveProperty("message"); - }); - - if (fullMode) { - test.each(imageModels)( - "image output $model", - getTestOptions(), - async ({ model }) => { - const requestId = generateTestRequestId(); - const res = await app.request("/v1/chat/completions", { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-request-id": requestId, - Authorization: `Bearer real-token`, - }, - body: JSON.stringify({ - model: model, - messages: [ - { - role: "user", - content: "Generate an image of a cute dog", - }, - ], - }), - }); - - const json = await res.json(); - if (logMode) { - console.log("image output response:", JSON.stringify(json, null, 2)); - } - - expect(res.status).toBe(200); - expect(json).toHaveProperty("choices"); - expect(json.choices).toHaveLength(1); - expect(json.choices[0]).toHaveProperty("message"); - - const message = json.choices[0].message; - expect(message).toHaveProperty("role", "assistant"); - - // Check that the response contains text content - expect(message.content).toBeTruthy(); - expect(typeof message.content).toBe("string"); - - // Check for images array in OpenAI format - expect(message).toHaveProperty("images"); - expect(Array.isArray(message.images)).toBe(true); - expect(message.images.length).toBeGreaterThan(0); - - // Validate each image object - for (const image of message.images) { - expect(image).toHaveProperty("type", "image_url"); - expect(image).toHaveProperty("image_url"); - expect(image.image_url).toHaveProperty("url"); - expect(typeof image.image_url.url).toBe("string"); - // Check if it's a base64 data URL - expect(image.image_url.url).toMatch( - /^data:image\/(png|jpeg|jpg|webp);base64,/, - ); - } - - // Validate logs - const log = await validateLogByRequestId(requestId); - expect(log.streamed).toBe(false); - - // Validate usage - expect(json).toHaveProperty("usage"); - expect(json.usage).toHaveProperty("prompt_tokens"); - expect(json.usage).toHaveProperty("completion_tokens"); - expect(json.usage).toHaveProperty("total_tokens"); - }, - ); - - test.each(streamingImageModels)( - "streaming image output $model", - getTestOptions(), - async ({ model }) => { - const requestId = generateTestRequestId(); - const res = await app.request("/v1/chat/completions", { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-request-id": requestId, - Authorization: `Bearer real-token`, - }, - body: JSON.stringify({ - model: model, - messages: [ - { - role: "user", - content: "Generate an image of a cute dog", - }, - ], - stream: true, - }), - }); - - if (res.status !== 200) { - console.log("response:", await res.text()); - throw new Error(`Request failed with status ${res.status}`); - } - - expect(res.status).toBe(200); - expect(res.headers.get("content-type")).toContain("text/event-stream"); - - const streamResult = await readAll(res.body); - if (logMode) { - console.log( - "streaming image result:", - JSON.stringify(streamResult, null, 2), - ); - } - - expect(streamResult.hasValidSSE).toBe(true); - expect(streamResult.eventCount).toBeGreaterThan(0); - expect(streamResult.hasContent).toBe(true); - - // Verify that all streaming responses are transformed to OpenAI format - expect(streamResult.hasOpenAIFormat).toBe(true); - - // Look for chunks containing images - const imageChunks = streamResult.chunks.filter( - (chunk) => chunk.choices?.[0]?.delta?.images, - ); - expect(imageChunks.length).toBeGreaterThan(0); - - // Validate image chunks - for (const chunk of imageChunks) { - expect(chunk).toHaveProperty("id"); - expect(chunk).toHaveProperty("object", "chat.completion.chunk"); - expect(chunk).toHaveProperty("created"); - expect(chunk).toHaveProperty("model"); - expect(chunk).toHaveProperty("choices"); - expect(chunk.choices).toHaveLength(1); - expect(chunk.choices[0]).toHaveProperty("index", 0); - expect(chunk.choices[0]).toHaveProperty("delta"); - - const delta = chunk.choices[0].delta; - if (delta.images) { - expect(Array.isArray(delta.images)).toBe(true); - for (const image of delta.images) { - expect(image).toHaveProperty("type", "image_url"); - expect(image).toHaveProperty("image_url"); - expect(image.image_url).toHaveProperty("url"); - expect(typeof image.image_url.url).toBe("string"); - // Check if it's a base64 data URL - expect(image.image_url.url).toMatch( - /^data:image\/(png|jpeg|jpg|webp);base64,/, - ); - } - } - } - - // Verify that usage object is returned in streaming mode - const usageChunks = streamResult.chunks.filter( - (chunk) => - chunk.usage && - (chunk.usage.prompt_tokens !== null || - chunk.usage.completion_tokens !== null || - chunk.usage.total_tokens !== null), - ); - expect(usageChunks.length).toBeGreaterThan(0); - - const log = await validateLogByRequestId(requestId); - expect(log.streamed).toBe(true); - }, - ); - } - - if (process.env.EXPERIMENTAL) { - test.each(providerModels)( - "complex $model", - getTestOptions(), - async ({ model, provider }) => { - const requestId = generateTestRequestId(); - const res = await app.request("/v1/chat/completions", { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-request-id": requestId, - Authorization: `Bearer real-token`, - }, - body: JSON.stringify({ - model: model, - messages: [ - { - role: "user", - content: [ - { - type: "text", - text: "\ndescribe this image\n", - }, - { - type: "text", - text: "", // empty text – note this may need special handling - }, - // provide image url if vision is supported - ...(provider.vision - ? [ - { - type: "image_url", - image_url: { - url: "https://t1.gstatic.com/faviconV2?client=SOCIAL&type=FAVICON&fallback_opts=TYPE,SIZE,URL&url=https://google.com&size=128", - }, - }, - ] - : []), - ], - }, - ], - }), - }); - - const json = await res.json(); - if (logMode) { - console.log("response:", JSON.stringify(json, null, 2)); - } - - expect(res.status).toBe(200); - validateResponse(json); - - const log = await validateLogByRequestId(requestId); - expect(log.streamed).toBe(false); - - expect(json).toHaveProperty("usage"); - expect(json.usage).toHaveProperty("prompt_tokens"); - expect(json.usage).toHaveProperty("completion_tokens"); - expect(json.usage).toHaveProperty("total_tokens"); - expect(typeof json.usage.prompt_tokens).toBe("number"); - expect(typeof json.usage.completion_tokens).toBe("number"); - expect(typeof json.usage.total_tokens).toBe("number"); - if (provider.providerId !== "zai") { - // zai may have weird prompt tokens - expect(json.usage.prompt_tokens).toBeGreaterThan(0); - } - expect(json.usage.completion_tokens).toBeGreaterThan(0); - expect(json.usage.total_tokens).toBeGreaterThan(0); - expect(json.usage.total_tokens).toEqual( - json.usage.prompt_tokens + - json.usage.completion_tokens + - (json.usage.reasoning_tokens || 0), - ); - }, - ); - - test.each(testModels)( - "parameters $model", - getTestOptions(), - async ({ model }) => { - const requestId = generateTestRequestId(); - const res = await app.request("/v1/chat/completions", { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-request-id": requestId, - Authorization: `Bearer real-token`, - }, - body: JSON.stringify({ - model: model, - messages: [ - { - role: "system", - content: "You are a helpful assistant.", - }, - { - role: "user", - content: "Hello, just reply 'OK'!", - }, - ], - max_tokens: 200, - temperature: 0.7, - }), - }); - - const json = await res.json(); - if (logMode) { - console.log("parameters response:", JSON.stringify(json, null, 2)); - } - - expect(res.status).toBe(200); - validateResponse(json); - - const log = await validateLogByRequestId(requestId); - expect(log.streamed).toBe(false); - - expect(json).toHaveProperty("usage"); - expect(json.usage).toHaveProperty("prompt_tokens"); - expect(json.usage).toHaveProperty("completion_tokens"); - expect(json.usage).toHaveProperty("total_tokens"); - expect(typeof json.usage.prompt_tokens).toBe("number"); - expect(typeof json.usage.completion_tokens).toBe("number"); - expect(typeof json.usage.total_tokens).toBe("number"); - expect(json.usage.prompt_tokens).toBeGreaterThan(0); - expect(json.usage.completion_tokens).toBeGreaterThan(0); - expect(json.usage.total_tokens).toBeGreaterThan(0); - }, - ); - } -}); diff --git a/apps/gateway/src/chat-api.e2e.ts b/apps/gateway/src/chat-api.e2e.ts new file mode 100644 index 0000000000..9d12b98ea5 --- /dev/null +++ b/apps/gateway/src/chat-api.e2e.ts @@ -0,0 +1,833 @@ +import "dotenv/config"; +import { + beforeAll, + beforeEach, + describe, + expect, + test, + type TestOptions, +} from "vitest"; + +import { db, tables } from "@llmgateway/db"; +import { + type ModelDefinition, + models, + type ProviderModelMapping, + providers, +} from "@llmgateway/models"; + +import { app } from "."; +import { + clearCache, + waitForLogByRequestId, + getProviderEnvVar, + readAll, +} from "./test-utils/test-helpers"; + +// Helper function to generate unique request IDs for tests +export function generateTestRequestId(): string { + return `test-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; +} + +// Helper function to get test options with retry for CI environment +export function getTestOptions(): TestOptions { + return process.env.CI ? { retry: 3 } : {}; +} + +console.log("running with test options:", getTestOptions()); + +export const fullMode = process.env.FULL_MODE; +export const logMode = process.env.LOG_MODE; + +// Parse TEST_MODELS environment variable +export const testModelsEnv = process.env.TEST_MODELS; +export const specifiedModels = testModelsEnv + ? testModelsEnv.split(",").map((m) => m.trim()) + : null; + +if (specifiedModels) { + console.log(`TEST_MODELS specified: ${specifiedModels.join(", ")}`); +} + +// Filter models based on test skip/only property +export const hasOnlyModels = models.some((model) => + model.providers.some( + (provider: ProviderModelMapping) => provider.test === "only", + ), +); + +// Log if we're using "only" mode +if (hasOnlyModels) { + if (process.env.CI) { + throw new Error( + "Cannot use 'only' in test configuration when running in CI. Please remove 'only' from the test configuration and try again.", + ); + } + console.log( + "Running in 'only' mode - only testing models marked with test: 'only'", + ); +} + +export const filteredModels = models + // Filter out auto/custom models + .filter((model) => !["custom", "auto"].includes(model.id)) + // Filter out deactivated models + .filter((model) => !model.deactivatedAt || new Date() <= model.deactivatedAt) + // Filter out unstable models if not in full mode, unless they have test: "only" or are in TEST_MODELS + .filter((model) => { + // Check if model or any of its providers are marked as unstable + const modelStability = (model as ModelDefinition).stability; + const hasUnstableProviders = model.providers.some( + (provider: ProviderModelMapping) => provider.stability === "unstable", + ); + const isUnstable = modelStability === "unstable" || hasUnstableProviders; + + if (!isUnstable) { + return true; + } // Non-unstable models are always included + if (fullMode) { + return true; + } // In full mode, all models are included + + // For unstable models in non-full mode, include if: + // 1. Any provider has test: "only" + if ( + model.providers.some( + (provider: ProviderModelMapping) => provider.test === "only", + ) + ) { + return true; + } + + // 2. Model is specified in TEST_MODELS + if (specifiedModels) { + const modelInTestModels = model.providers.some( + (provider: ProviderModelMapping) => { + const providerModelId = `${provider.providerId}/${model.id}`; + return specifiedModels.includes(providerModelId); + }, + ); + if (modelInTestModels) { + return true; + } + } + + return false; // Otherwise, exclude unstable models in non-full mode + }) + // Filter out free models if not in full mode, unless they have test: "only" or are in TEST_MODELS + .filter((model) => { + const isFreeModel = (model as ModelDefinition).free; + if (!isFreeModel) { + return true; + } // Non-free models are always included + if (fullMode) { + return true; + } // In full mode, all models are included + + // For free models in non-full mode, include if: + // 1. Any provider has test: "only" + if ( + model.providers.some( + (provider: ProviderModelMapping) => provider.test === "only", + ) + ) { + return true; + } + + // 2. Model is specified in TEST_MODELS + if (specifiedModels) { + const modelInTestModels = model.providers.some( + (provider: ProviderModelMapping) => { + const providerModelId = `${provider.providerId}/${model.id}`; + return specifiedModels.includes(providerModelId); + }, + ); + if (modelInTestModels) { + return true; + } + } + + return false; // Otherwise, exclude free models in non-full mode + }) + // Filter by TEST_MODELS if specified + .filter((model) => { + if (!specifiedModels) { + return true; + } + // Check if any provider/model combination from this model matches TEST_MODELS + return model.providers.some((provider: ProviderModelMapping) => { + const providerModelId = `${provider.providerId}/${model.id}`; + return specifiedModels.includes(providerModelId); + }); + }); + +export const testModels = filteredModels + // If any model has test: "only", only include those models + .filter((model) => { + if (hasOnlyModels) { + return model.providers.some( + (provider: ProviderModelMapping) => provider.test === "only", + ); + } + return true; + }) + .flatMap((model) => { + const testCases = []; + + if (process.env.TEST_ALL_VARIATIONS) { + // test root model without a specific provider + testCases.push({ + model: model.id, + providers: model.providers.filter( + (provider: ProviderModelMapping) => provider.test !== "skip", + ), + }); + } + + // Create entries for provider-specific requests using provider/model format + for (const provider of model.providers as ProviderModelMapping[]) { + // Skip providers marked with test: "skip" + if (provider.test === "skip") { + continue; + } + + // Skip unstable providers if not in full mode, unless they have test: "only" or are in TEST_MODELS + if (provider.stability === "unstable" && !fullMode) { + // Allow if provider has test: "only" + if (provider.test !== "only") { + // Allow if model is specified in TEST_MODELS + if (!specifiedModels) { + continue; + } + const providerModelId = `${provider.providerId}/${model.id}`; + if (!specifiedModels.includes(providerModelId)) { + continue; + } + } + } + + // If we have any "only" providers, skip those not marked as "only" + if (hasOnlyModels && provider.test !== "only") { + continue; + } + + testCases.push({ + model: `${provider.providerId}/${model.id}`, + providers: [provider], + originalModel: model.id, // Keep track of the original model for reference + }); + } + + return testCases; + }); + +export const providerModels = filteredModels + // If any model has test: "only", only include those models + .filter((model) => { + if (hasOnlyModels) { + return model.providers.some( + (provider: ProviderModelMapping) => provider.test === "only", + ); + } + return true; + }) + .flatMap((model) => { + const testCases = []; + + for (const provider of model.providers as ProviderModelMapping[]) { + // Skip providers marked with test: "skip" + if (provider.test === "skip") { + continue; + } + + // Skip unstable providers if not in full mode, unless they have test: "only" or are in TEST_MODELS + if (provider.stability === "unstable" && !fullMode) { + // Allow if provider has test: "only" + if (provider.test !== "only") { + // Allow if model is specified in TEST_MODELS + if (!specifiedModels) { + continue; + } + const providerModelId = `${provider.providerId}/${model.id}`; + if (!specifiedModels.includes(providerModelId)) { + continue; + } + } + } + + // If we have any "only" providers, skip those not marked as "only" + if (hasOnlyModels && provider.test !== "only") { + continue; + } + + testCases.push({ + model: `${provider.providerId}/${model.id}`, + provider, + originalModel: model.id, // Keep track of the original model for reference + }); + } + + return testCases; + }); + +// Log the number of test models after filtering +console.log(`Testing ${testModels.length} model configurations`); +console.log(`Testing ${providerModels.length} provider model configurations`); + +export const streamingModels = testModels.filter((m) => + m.providers.some((p: ProviderModelMapping) => { + // Check model-level streaming first, then fall back to provider-level + if (p.streaming !== undefined) { + return p.streaming; + } + const provider = providers.find((pr) => pr.id === p.providerId); + return provider?.streaming; + }), +); + +export const reasoningModels = testModels.filter((m) => + m.providers.some((p: ProviderModelMapping) => p.reasoning === true), +); + +export const streamingReasoningModels = reasoningModels.filter((m) => + m.providers.some((p: ProviderModelMapping) => { + // Check model-level streaming first, then fall back to provider-level + if (p.streaming !== undefined) { + return p.streaming; + } + const provider = providers.find((pr) => pr.id === p.providerId); + return provider?.streaming; + }), +); + +export const toolCallModels = testModels.filter((m) => + m.providers.some((p: ProviderModelMapping) => p.tools === true), +); + +export const imageModels = testModels.filter((m) => { + const model = models.find((mo) => m.originalModel === mo.id); + return (model as ModelDefinition).output?.includes("image"); +}); + +export const streamingImageModels = imageModels.filter((m) => + m.providers.some((p: ProviderModelMapping) => { + // Check model-level streaming first, then fall back to provider-level + if (p.streaming !== undefined) { + return p.streaming; + } + const provider = providers.find((pr) => pr.id === p.providerId); + return provider?.streaming; + }), +); + +export async function createProviderKey( + provider: string, + token: string, + keyType: "api-keys" | "credits" = "api-keys", +) { + const keyId = + keyType === "credits" ? `env-${provider}` : `provider-key-${provider}`; + await db.insert(tables.providerKey).values({ + id: keyId, + token, + provider: provider.replace("env-", ""), // Remove env- prefix for the provider field + organizationId: "org-id", + }); +} + +export function validateResponse(json: any) { + expect(json).toHaveProperty("choices.[0].message.content"); + + expect(json).toHaveProperty("usage.prompt_tokens"); + expect(json).toHaveProperty("usage.completion_tokens"); + expect(json).toHaveProperty("usage.total_tokens"); +} + +export async function validateLogByRequestId(requestId: string) { + const log = await waitForLogByRequestId(requestId); + + if (logMode) { + console.log("log", JSON.stringify(log, null, 2)); + } + + expect(log.usedProvider).toBeTruthy(); + expect(log.errorDetails).toBeNull(); + expect(log.finishReason).not.toBeNull(); + expect(log.unifiedFinishReason).not.toBeNull(); + expect(log.unifiedFinishReason).toBeTruthy(); + expect(log.usedModel).toBeTruthy(); + expect(log.requestedModel).toBeTruthy(); + + return log; +} + +export async function beforeAllHook() { + await clearCache(); + + // Clean up any existing data + await Promise.all([ + db.delete(tables.log), + db.delete(tables.apiKey), + db.delete(tables.providerKey), + ]); + + await Promise.all([ + db.delete(tables.userOrganization), + db.delete(tables.project), + ]); + + await Promise.all([ + db.delete(tables.organization), + db.delete(tables.user), + db.delete(tables.account), + db.delete(tables.session), + db.delete(tables.verification), + ]); + + // Set up shared test data that all tests can use + await db.insert(tables.user).values({ + id: "user-id", + name: "user", + email: "user", + }); + + await db.insert(tables.organization).values({ + id: "org-id", + name: "Test Organization", + plan: "pro", + }); + + await db.insert(tables.userOrganization).values({ + id: "user-org-id", + userId: "user-id", + organizationId: "org-id", + }); + + await db.insert(tables.project).values({ + id: "project-id", + name: "Test Project", + organizationId: "org-id", + mode: "api-keys", + }); + + await db.insert(tables.apiKey).values({ + id: "token-id", + token: "real-token", + projectId: "project-id", + description: "Test API Key", + }); + + // Set up provider keys for all providers + for (const provider of providers) { + const envVarName = getProviderEnvVar(provider.id); + const envVarValue = envVarName ? process.env[envVarName] : undefined; + if (envVarValue) { + await createProviderKey(provider.id, envVarValue, "api-keys"); + await createProviderKey(provider.id, envVarValue, "credits"); + } + } +} + +export async function beforeEachHook() { + await clearCache(); +} + +describe("e2e", { concurrent: true }, () => { + beforeAll(beforeAllHook); + + beforeEach(beforeEachHook); + + test.each(testModels)( + "completions $model", + getTestOptions(), + async ({ model }) => { + const requestId = generateTestRequestId(); + const res = await app.request("/v1/chat/completions", { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-request-id": requestId, + Authorization: `Bearer real-token`, + }, + body: JSON.stringify({ + model: model, + messages: [ + { + role: "system", + content: "You are a helpful assistant.", + }, + { + role: "user", + content: "Hello, just reply 'OK'!", + }, + ], + }), + }); + + const json = await res.json(); + if (logMode) { + console.log("response:", JSON.stringify(json, null, 2)); + } + + expect(res.status).toBe(200); + validateResponse(json); + + const log = await validateLogByRequestId(requestId); + expect(log.streamed).toBe(false); + + expect(json).toHaveProperty("usage"); + expect(json.usage).toHaveProperty("prompt_tokens"); + expect(json.usage).toHaveProperty("completion_tokens"); + expect(json.usage).toHaveProperty("total_tokens"); + expect(typeof json.usage.prompt_tokens).toBe("number"); + expect(typeof json.usage.completion_tokens).toBe("number"); + expect(typeof json.usage.total_tokens).toBe("number"); + expect(json.usage.prompt_tokens).toBeGreaterThan(0); + expect(json.usage.completion_tokens).toBeGreaterThan(0); + expect(json.usage.total_tokens).toBeGreaterThan(0); + + // expect(log.inputCost).not.toBeNull(); + // expect(log.outputCost).not.toBeNull(); + // expect(log.cost).not.toBeNull(); + }, + ); + + test.each( + testModels.filter((m) => { + const modelDef = models.find((def) => def.id === m.model); + return (modelDef as ModelDefinition)?.jsonOutput === true; + }), + )("JSON output $model", getTestOptions(), async ({ model }) => { + const res = await app.request("/v1/chat/completions", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer real-token`, + }, + body: JSON.stringify({ + model: model, + messages: [ + { + role: "system", + content: + "You are a helpful assistant. Always respond with valid JSON.", + }, + { + role: "user", + content: 'Return a JSON object with "message": "Hello World"', + }, + ], + response_format: { type: "json_object" }, + }), + }); + + const json = await res.json(); + if (logMode) { + console.log("json", JSON.stringify(json, null, 2)); + } + expect(res.status).toBe(200); + expect(json).toHaveProperty("choices.[0].message.content"); + + const content = json.choices[0].message.content; + expect(() => JSON.parse(content)).not.toThrow(); + + const parsedContent = JSON.parse(content); + expect(parsedContent).toHaveProperty("message"); + }); + + if (fullMode) { + test.each(imageModels)( + "image output $model", + getTestOptions(), + async ({ model }) => { + const requestId = generateTestRequestId(); + const res = await app.request("/v1/chat/completions", { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-request-id": requestId, + Authorization: `Bearer real-token`, + }, + body: JSON.stringify({ + model: model, + messages: [ + { + role: "user", + content: "Generate an image of a cute dog", + }, + ], + }), + }); + + const json = await res.json(); + if (logMode) { + console.log("image output response:", JSON.stringify(json, null, 2)); + } + + expect(res.status).toBe(200); + expect(json).toHaveProperty("choices"); + expect(json.choices).toHaveLength(1); + expect(json.choices[0]).toHaveProperty("message"); + + const message = json.choices[0].message; + expect(message).toHaveProperty("role", "assistant"); + + // Check that the response contains text content + expect(message.content).toBeTruthy(); + expect(typeof message.content).toBe("string"); + + // Check for images array in OpenAI format + expect(message).toHaveProperty("images"); + expect(Array.isArray(message.images)).toBe(true); + expect(message.images.length).toBeGreaterThan(0); + + // Validate each image object + for (const image of message.images) { + expect(image).toHaveProperty("type", "image_url"); + expect(image).toHaveProperty("image_url"); + expect(image.image_url).toHaveProperty("url"); + expect(typeof image.image_url.url).toBe("string"); + // Check if it's a base64 data URL + expect(image.image_url.url).toMatch( + /^data:image\/(png|jpeg|jpg|webp);base64,/, + ); + } + + // Validate logs + const log = await validateLogByRequestId(requestId); + expect(log.streamed).toBe(false); + + // Validate usage + expect(json).toHaveProperty("usage"); + expect(json.usage).toHaveProperty("prompt_tokens"); + expect(json.usage).toHaveProperty("completion_tokens"); + expect(json.usage).toHaveProperty("total_tokens"); + }, + ); + + test.each(streamingImageModels)( + "streaming image output $model", + getTestOptions(), + async ({ model }) => { + const requestId = generateTestRequestId(); + const res = await app.request("/v1/chat/completions", { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-request-id": requestId, + Authorization: `Bearer real-token`, + }, + body: JSON.stringify({ + model: model, + messages: [ + { + role: "user", + content: "Generate an image of a cute dog", + }, + ], + stream: true, + }), + }); + + if (res.status !== 200) { + console.log("response:", await res.text()); + throw new Error(`Request failed with status ${res.status}`); + } + + expect(res.status).toBe(200); + expect(res.headers.get("content-type")).toContain("text/event-stream"); + + const streamResult = await readAll(res.body); + if (logMode) { + console.log( + "streaming image result:", + JSON.stringify(streamResult, null, 2), + ); + } + + expect(streamResult.hasValidSSE).toBe(true); + expect(streamResult.eventCount).toBeGreaterThan(0); + expect(streamResult.hasContent).toBe(true); + + // Verify that all streaming responses are transformed to OpenAI format + expect(streamResult.hasOpenAIFormat).toBe(true); + + // Look for chunks containing images + const imageChunks = streamResult.chunks.filter( + (chunk) => chunk.choices?.[0]?.delta?.images, + ); + expect(imageChunks.length).toBeGreaterThan(0); + + // Validate image chunks + for (const chunk of imageChunks) { + expect(chunk).toHaveProperty("id"); + expect(chunk).toHaveProperty("object", "chat.completion.chunk"); + expect(chunk).toHaveProperty("created"); + expect(chunk).toHaveProperty("model"); + expect(chunk).toHaveProperty("choices"); + expect(chunk.choices).toHaveLength(1); + expect(chunk.choices[0]).toHaveProperty("index", 0); + expect(chunk.choices[0]).toHaveProperty("delta"); + + const delta = chunk.choices[0].delta; + if (delta.images) { + expect(Array.isArray(delta.images)).toBe(true); + for (const image of delta.images) { + expect(image).toHaveProperty("type", "image_url"); + expect(image).toHaveProperty("image_url"); + expect(image.image_url).toHaveProperty("url"); + expect(typeof image.image_url.url).toBe("string"); + // Check if it's a base64 data URL + expect(image.image_url.url).toMatch( + /^data:image\/(png|jpeg|jpg|webp);base64,/, + ); + } + } + } + + // Verify that usage object is returned in streaming mode + const usageChunks = streamResult.chunks.filter( + (chunk) => + chunk.usage && + (chunk.usage.prompt_tokens !== null || + chunk.usage.completion_tokens !== null || + chunk.usage.total_tokens !== null), + ); + expect(usageChunks.length).toBeGreaterThan(0); + + const log = await validateLogByRequestId(requestId); + expect(log.streamed).toBe(true); + }, + ); + } + + if (process.env.EXPERIMENTAL) { + test.each(providerModels)( + "complex $model", + getTestOptions(), + async ({ model, provider }) => { + const requestId = generateTestRequestId(); + const res = await app.request("/v1/chat/completions", { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-request-id": requestId, + Authorization: `Bearer real-token`, + }, + body: JSON.stringify({ + model: model, + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: "\ndescribe this image\n", + }, + { + type: "text", + text: "", // empty text – note this may need special handling + }, + // provide image url if vision is supported + ...(provider.vision + ? [ + { + type: "image_url", + image_url: { + url: "https://t1.gstatic.com/faviconV2?client=SOCIAL&type=FAVICON&fallback_opts=TYPE,SIZE,URL&url=https://google.com&size=128", + }, + }, + ] + : []), + ], + }, + ], + }), + }); + + const json = await res.json(); + if (logMode) { + console.log("response:", JSON.stringify(json, null, 2)); + } + + expect(res.status).toBe(200); + validateResponse(json); + + const log = await validateLogByRequestId(requestId); + expect(log.streamed).toBe(false); + + expect(json).toHaveProperty("usage"); + expect(json.usage).toHaveProperty("prompt_tokens"); + expect(json.usage).toHaveProperty("completion_tokens"); + expect(json.usage).toHaveProperty("total_tokens"); + expect(typeof json.usage.prompt_tokens).toBe("number"); + expect(typeof json.usage.completion_tokens).toBe("number"); + expect(typeof json.usage.total_tokens).toBe("number"); + if (provider.providerId !== "zai") { + // zai may have weird prompt tokens + expect(json.usage.prompt_tokens).toBeGreaterThan(0); + } + expect(json.usage.completion_tokens).toBeGreaterThan(0); + expect(json.usage.total_tokens).toBeGreaterThan(0); + expect(json.usage.total_tokens).toEqual( + json.usage.prompt_tokens + + json.usage.completion_tokens + + (json.usage.reasoning_tokens || 0), + ); + }, + ); + + test.each(testModels)( + "parameters $model", + getTestOptions(), + async ({ model }) => { + const requestId = generateTestRequestId(); + const res = await app.request("/v1/chat/completions", { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-request-id": requestId, + Authorization: `Bearer real-token`, + }, + body: JSON.stringify({ + model: model, + messages: [ + { + role: "system", + content: "You are a helpful assistant.", + }, + { + role: "user", + content: "Hello, just reply 'OK'!", + }, + ], + max_tokens: 200, + temperature: 0.7, + }), + }); + + const json = await res.json(); + if (logMode) { + console.log("parameters response:", JSON.stringify(json, null, 2)); + } + + expect(res.status).toBe(200); + validateResponse(json); + + const log = await validateLogByRequestId(requestId); + expect(log.streamed).toBe(false); + + expect(json).toHaveProperty("usage"); + expect(json.usage).toHaveProperty("prompt_tokens"); + expect(json.usage).toHaveProperty("completion_tokens"); + expect(json.usage).toHaveProperty("total_tokens"); + expect(typeof json.usage.prompt_tokens).toBe("number"); + expect(typeof json.usage.completion_tokens).toBe("number"); + expect(typeof json.usage.total_tokens).toBe("number"); + expect(json.usage.prompt_tokens).toBeGreaterThan(0); + expect(json.usage.completion_tokens).toBeGreaterThan(0); + expect(json.usage.total_tokens).toBeGreaterThan(0); + }, + ); + } +}); diff --git a/apps/gateway/src/chat-full.e2e.ts b/apps/gateway/src/chat-full.e2e.ts new file mode 100644 index 0000000000..8acd7332c3 --- /dev/null +++ b/apps/gateway/src/chat-full.e2e.ts @@ -0,0 +1,165 @@ +import "dotenv/config"; +import { beforeAll, beforeEach, describe, expect, test } from "vitest"; + +import { + beforeAllHook, + beforeEachHook, + fullMode, + generateTestRequestId, + getTestOptions, + logMode, + testModels, + validateLogByRequestId, +} from "@/chat-api.e2e"; +import { app } from "@/index"; + +import type { ProviderModelMapping } from "@llmgateway/models"; + +describe("e2e", { concurrent: true }, () => { + beforeAll(beforeAllHook); + + beforeEach(beforeEachHook); + + if (fullMode) { + const reasoningToolCallModels = testModels.filter((m) => + m.providers.some( + (p: ProviderModelMapping) => p.reasoning === true && p.tools === true, + ), + ); + + test.each(reasoningToolCallModels)( + "reasoning + tool calls $model", + getTestOptions(), + async ({ model, providers }) => { + const requestId = generateTestRequestId(); + const res = await app.request("/v1/chat/completions", { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-request-id": requestId, + Authorization: `Bearer real-token`, + }, + body: JSON.stringify({ + model: model, + messages: [ + { + role: "system", + content: + "You are a weather assistant that can get weather information for cities. Think step by step and use tools when needed.", + }, + { + role: "user", + content: + "What's the weather like in San Francisco? Consider all the exact details. Use the weather tool and explain your reasoning.", + }, + ], + tools: [ + { + type: "function", + function: { + name: "get_weather", + description: "Get the current weather for a given city", + parameters: { + type: "object", + properties: { + city: { + type: "string", + description: "The city name to get weather for", + }, + unit: { + type: "string", + enum: ["celsius", "fahrenheit"], + description: "Temperature unit", + default: "fahrenheit", + }, + }, + required: ["city"], + }, + }, + }, + ], + tool_choice: "auto", + reasoning_effort: "medium", + }), + }); + + const json = await res.json(); + if (logMode) { + console.log( + "reasoning + tool calls response:", + JSON.stringify(json, null, 2), + ); + } + + expect(res.status).toBe(200); + expect(json).toHaveProperty("choices"); + expect(json.choices).toHaveLength(1); + expect(json.choices[0]).toHaveProperty("message"); + + const message = json.choices[0].message; + expect(message).toHaveProperty("role", "assistant"); + + // Should have tool calls since we're asking about weather + expect(message).toHaveProperty("tool_calls"); + expect(Array.isArray(message.tool_calls)).toBe(true); + expect(message.tool_calls.length).toBeGreaterThan(0); + + // Validate tool call structure + const toolCall = message.tool_calls[0]; + expect(toolCall).toHaveProperty("id"); + expect(toolCall).toHaveProperty("type", "function"); + expect(toolCall).toHaveProperty("function"); + expect(toolCall.function).toHaveProperty("name", "get_weather"); + expect(toolCall.function).toHaveProperty("arguments"); + + // Parse and validate arguments + const args = JSON.parse(toolCall.function.arguments); + expect(args).toHaveProperty("city"); + expect(typeof args.city).toBe("string"); + expect(args.city.toLowerCase()).toContain("san francisco"); + + // Check finish reason + expect(json.choices[0]).toHaveProperty("finish_reason", "tool_calls"); + + // Check for reasoning content - only if the provider expects reasoning output + const reasoningProvider = providers?.find( + (p: ProviderModelMapping) => p.reasoning === true, + ); + if ( + (reasoningProvider as ProviderModelMapping)?.reasoningOutput !== + "omit" + ) { + expect(json.choices[0].message).toHaveProperty("reasoning_content"); + expect(typeof json.choices[0].message.reasoning_content).toBe( + "string", + ); + expect( + json.choices[0].message.reasoning_content.length, + ).toBeGreaterThan(0); + } + + // Validate logs + const log = await validateLogByRequestId(requestId); + expect(log.streamed).toBe(false); + + // Validate usage + expect(json).toHaveProperty("usage"); + expect(json.usage).toHaveProperty("prompt_tokens"); + expect(json.usage).toHaveProperty("completion_tokens"); + expect(json.usage).toHaveProperty("total_tokens"); + expect(typeof json.usage.prompt_tokens).toBe("number"); + expect(typeof json.usage.completion_tokens).toBe("number"); + expect(typeof json.usage.total_tokens).toBe("number"); + expect(json.usage.prompt_tokens).toBeGreaterThan(0); + expect(json.usage.completion_tokens).toBeGreaterThan(0); + expect(json.usage.total_tokens).toBeGreaterThan(0); + + // Check for reasoning tokens if available + if (json.usage.reasoning_tokens !== undefined) { + expect(typeof json.usage.reasoning_tokens).toBe("number"); + expect(json.usage.reasoning_tokens).toBeGreaterThanOrEqual(0); + } + }, + ); + } +}); diff --git a/apps/gateway/src/chat-reasoning.e2e.ts b/apps/gateway/src/chat-reasoning.e2e.ts new file mode 100644 index 0000000000..d673a64973 --- /dev/null +++ b/apps/gateway/src/chat-reasoning.e2e.ts @@ -0,0 +1,94 @@ +import "dotenv/config"; +import { beforeAll, beforeEach, describe, expect, test } from "vitest"; + +import { + beforeAllHook, + beforeEachHook, + generateTestRequestId, + getTestOptions, + logMode, + reasoningModels, + validateLogByRequestId, + validateResponse, +} from "@/chat-api.e2e"; +import { app } from "@/index"; + +import type { ProviderModelMapping } from "@llmgateway/models"; + +describe("e2e", { concurrent: true }, () => { + beforeAll(beforeAllHook); + + beforeEach(beforeEachHook); + + test.each(reasoningModels)( + "reasoning $model", + getTestOptions(), + async ({ model, providers }) => { + const requestId = generateTestRequestId(); + const res = await app.request("/v1/chat/completions", { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-request-id": requestId, + Authorization: `Bearer real-token`, + }, + body: JSON.stringify({ + model: model, + messages: [ + { + role: "system", + content: "You are a helpful assistant.", + }, + { + role: "user", + content: "What is 2/3 + 1/4 + 5/6?", + }, + ], + reasoning_effort: "medium", + }), + }); + + const json = await res.json(); + if (logMode) { + console.log("reasoning response:", JSON.stringify(json, null, 2)); + } + + expect(res.status).toBe(200); + validateResponse(json); + + const log = await validateLogByRequestId(requestId); + expect(log.streamed).toBe(false); + + expect(json).toHaveProperty("usage"); + expect(json.usage).toHaveProperty("prompt_tokens"); + expect(json.usage).toHaveProperty("completion_tokens"); + expect(json.usage).toHaveProperty("total_tokens"); + expect(typeof json.usage.prompt_tokens).toBe("number"); + expect(typeof json.usage.completion_tokens).toBe("number"); + expect(typeof json.usage.total_tokens).toBe("number"); + expect(json.usage.prompt_tokens).toBeGreaterThan(0); + expect(json.usage.completion_tokens).toBeGreaterThan(0); + expect(json.usage.total_tokens).toBeGreaterThan(0); + + // Check for reasoning tokens if available + if (json.usage.reasoning_tokens !== undefined) { + expect(typeof json.usage.reasoning_tokens).toBe("number"); + expect(json.usage.reasoning_tokens).toBeGreaterThanOrEqual(0); + } + + // check for reasoning response - only if the provider expects reasoning output + const reasoningProvider = providers?.find( + (p: ProviderModelMapping) => p.reasoning === true, + ) as ProviderModelMapping; + const useResponsesApi = process.env.USE_RESPONSES_API === "true"; + const isOpenAI = reasoningProvider?.providerId === "openai"; + // only enforce reasoning_content checks for where reasoningOutput is not "omit" and for openai, only if the responses api is used + if ( + reasoningProvider?.reasoningOutput !== "omit" && + (!isOpenAI || useResponsesApi) + ) { + expect(json.choices[0].message).toHaveProperty("reasoning_content"); + } + }, + ); +}); diff --git a/apps/gateway/src/chat-rs.e2e.ts b/apps/gateway/src/chat-rs.e2e.ts new file mode 100644 index 0000000000..021ab6084d --- /dev/null +++ b/apps/gateway/src/chat-rs.e2e.ts @@ -0,0 +1,147 @@ +import "dotenv/config"; +import { beforeAll, beforeEach, describe, expect, test } from "vitest"; + +import { + beforeAllHook, + beforeEachHook, + generateTestRequestId, + getTestOptions, + logMode, + streamingReasoningModels, + validateLogByRequestId, +} from "@/chat-api.e2e"; +import { app } from "@/index"; +import { readAll } from "@/test-utils/test-helpers"; + +import type { ProviderModelMapping } from "@llmgateway/models"; + +describe("e2e", { concurrent: true }, () => { + beforeAll(beforeAllHook); + + beforeEach(beforeEachHook); + + test.each(streamingReasoningModels)( + "reasoning + streaming $model", + getTestOptions(), + async ({ model, providers }) => { + const requestId = generateTestRequestId(); + const res = await app.request("/v1/chat/completions", { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-request-id": requestId, + Authorization: `Bearer real-token`, + }, + body: JSON.stringify({ + model: model, + messages: [ + { + role: "system", + content: + "You are a helpful assistant. Think step by step and show your reasoning.", + }, + { + role: "user", + content: + "Solve this logic puzzle: If all roses are flowers, and some flowers are red, what can we conclude about roses? Think through this step by step.", + }, + ], + reasoning_effort: "medium", + stream: true, + }), + }); + + if (res.status !== 200) { + console.log("response:", await res.text()); + throw new Error(`Request failed with status ${res.status}`); + } + + expect(res.status).toBe(200); + expect(res.headers.get("content-type")).toContain("text/event-stream"); + + const streamResult = await readAll(res.body); + if (logMode) { + console.log( + "reasoning streaming response:", + JSON.stringify(streamResult.chunks, null, 2), + ); + } + + expect(streamResult.hasValidSSE).toBe(true); + expect(streamResult.eventCount).toBeGreaterThan(0); + expect(streamResult.hasContent).toBe(true); + + // Verify that all streaming responses are transformed to OpenAI format + expect(streamResult.hasOpenAIFormat).toBe(true); + + // Verify that chunks have the correct OpenAI streaming format + const contentChunks = streamResult.chunks.filter( + (chunk: any) => chunk.choices?.[0]?.delta?.content, + ); + expect(contentChunks.length).toBeGreaterThan(0); + + // Verify each content chunk has proper OpenAI format + for (const chunk of contentChunks) { + expect(chunk).toHaveProperty("id"); + expect(chunk).toHaveProperty("object", "chat.completion.chunk"); + expect(chunk).toHaveProperty("created"); + expect(chunk).toHaveProperty("model"); + expect(chunk).toHaveProperty("choices"); + expect(chunk.choices).toHaveLength(1); + expect(chunk.choices[0]).toHaveProperty("delta"); + expect(typeof chunk.choices[0].delta.content).toBe("string"); + } + + // Verify that usage object is returned in streaming mode + const usageChunks = streamResult.chunks.filter( + (chunk: any) => + chunk.usage && + (chunk.usage.prompt_tokens !== null || + chunk.usage.completion_tokens !== null || + chunk.usage.total_tokens !== null), + ); + expect(usageChunks.length).toBeGreaterThan(0); + + // Verify the usage chunk has proper format + const usageChunk = usageChunks[usageChunks.length - 1]; // Get the last usage chunk + expect(usageChunk).toHaveProperty("usage"); + expect(usageChunk.usage).toHaveProperty("prompt_tokens"); + expect(usageChunk.usage).toHaveProperty("completion_tokens"); + expect(usageChunk.usage).toHaveProperty("total_tokens"); + expect(typeof usageChunk.usage.prompt_tokens).toBe("number"); + expect(typeof usageChunk.usage.completion_tokens).toBe("number"); + expect(typeof usageChunk.usage.total_tokens).toBe("number"); + expect(usageChunk.usage.prompt_tokens).toBeGreaterThan(0); + expect(usageChunk.usage.completion_tokens).toBeGreaterThan(0); + expect(usageChunk.usage.total_tokens).toBeGreaterThan(0); + + // Check for reasoning tokens if available + if (usageChunk.usage.reasoning_tokens !== undefined) { + expect(typeof usageChunk.usage.reasoning_tokens).toBe("number"); + expect(usageChunk.usage.reasoning_tokens).toBeGreaterThanOrEqual(0); + } + + // Verify reasoning content is present in unified reasoning_content field - only if the provider expects reasoning output + const reasoningProvider = providers?.find( + (p: ProviderModelMapping) => p.reasoning === true, + ) as ProviderModelMapping; + const useResponsesApi = process.env.USE_RESPONSES_API === "true"; + const isOpenAI = reasoningProvider?.providerId === "openai"; + // When using the Responses API, only enforce reasoning_content checks for OpenAI. + if ( + reasoningProvider?.reasoningOutput !== "omit" && + (!isOpenAI || useResponsesApi) + ) { + const reasoningChunks = streamResult.chunks.filter( + (chunk: any) => + chunk.choices?.[0]?.delta?.reasoning_content && + chunk.choices[0].delta.reasoning_content.length > 0, + ); + expect(reasoningChunks.length).toBeGreaterThan(0); + } + + const log = await validateLogByRequestId(requestId); + expect(log.streamed).toBe(true); + }, + ); +}); diff --git a/apps/gateway/src/chat-streaming.e2e.ts b/apps/gateway/src/chat-streaming.e2e.ts new file mode 100644 index 0000000000..ea9cba9b56 --- /dev/null +++ b/apps/gateway/src/chat-streaming.e2e.ts @@ -0,0 +1,120 @@ +import "dotenv/config"; +import { beforeAll, beforeEach, describe, expect, test } from "vitest"; + +import { + beforeAllHook, + beforeEachHook, + generateTestRequestId, + getTestOptions, + logMode, + streamingModels, + validateLogByRequestId, +} from "@/chat-api.e2e"; +import { app } from "@/index"; +import { readAll } from "@/test-utils/test-helpers"; + +describe("e2e", { concurrent: true }, () => { + beforeAll(beforeAllHook); + + beforeEach(beforeEachHook); + + test.each(streamingModels)( + "/v1/chat/completions streaming with $model", + getTestOptions(), + async ({ model }) => { + const requestId = generateTestRequestId(); + const res = await app.request("/v1/chat/completions", { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-request-id": requestId, + Authorization: `Bearer real-token`, + }, + body: JSON.stringify({ + model: model, + messages: [ + { + role: "system", + content: "You are a helpful assistant.", + }, + { + role: "user", + content: "Hello! This is a streaming e2e test.", + }, + ], + stream: true, + }), + }); + + if (res.status !== 200) { + console.log("response:", await res.text()); + throw new Error(`Request failed with status ${res.status}`); + } + + expect(res.status).toBe(200); + expect(res.headers.get("content-type")).toContain("text/event-stream"); + + const streamResult = await readAll(res.body); + if (logMode) { + console.log("streamResult", JSON.stringify(streamResult, null, 2)); + } + + expect(streamResult.hasValidSSE).toBe(true); + expect(streamResult.eventCount).toBeGreaterThan(0); + expect(streamResult.hasContent).toBe(true); + + // Verify that all streaming responses are transformed to OpenAI format + expect(streamResult.hasOpenAIFormat).toBe(true); + + // Verify that chunks have the correct OpenAI streaming format + const contentChunks = streamResult.chunks.filter( + (chunk) => chunk.choices?.[0]?.delta?.content, + ); + expect(contentChunks.length).toBeGreaterThan(0); + + // Verify each content chunk has proper OpenAI format + for (const chunk of contentChunks) { + expect(chunk).toHaveProperty("id"); + expect(chunk).toHaveProperty("object", "chat.completion.chunk"); + expect(chunk).toHaveProperty("created"); + expect(chunk).toHaveProperty("model"); + expect(chunk).toHaveProperty("choices"); + expect(chunk.choices).toHaveLength(1); + expect(chunk.choices[0]).toHaveProperty("index", 0); + expect(chunk.choices[0]).toHaveProperty("delta"); + expect(chunk.choices[0]).toHaveProperty("delta.role", "assistant"); + expect(chunk.choices[0].delta).toHaveProperty("content"); + expect(typeof chunk.choices[0].delta.content).toBe("string"); + } + + // Verify that usage object is returned in streaming mode + const usageChunks = streamResult.chunks.filter( + (chunk) => + chunk.usage && + (chunk.usage.prompt_tokens !== null || + chunk.usage.completion_tokens !== null || + chunk.usage.total_tokens !== null), + ); + expect(usageChunks.length).toBeGreaterThan(0); + + // Verify the usage chunk has proper format + const usageChunk = usageChunks[usageChunks.length - 1]; // Get the last usage chunk + expect(usageChunk).toHaveProperty("usage"); + expect(usageChunk.usage).toHaveProperty("prompt_tokens"); + expect(usageChunk.usage).toHaveProperty("completion_tokens"); + expect(usageChunk.usage).toHaveProperty("total_tokens"); + expect(typeof usageChunk.usage.prompt_tokens).toBe("number"); + expect(typeof usageChunk.usage.completion_tokens).toBe("number"); + expect(typeof usageChunk.usage.total_tokens).toBe("number"); + expect(usageChunk.usage.prompt_tokens).toBeGreaterThan(0); + expect(usageChunk.usage.completion_tokens).toBeGreaterThan(0); + expect(usageChunk.usage.total_tokens).toBeGreaterThan(0); + + const log = await validateLogByRequestId(requestId); + expect(log.streamed).toBe(true); + + // expect(log.cost).not.toBeNull(); + // expect(log.cost).toBeGreaterThanOrEqual(0); + }, + ); +}); diff --git a/apps/gateway/src/chat-toolcalls-result.e2e.ts b/apps/gateway/src/chat-toolcalls-result.e2e.ts new file mode 100644 index 0000000000..440b7b734b --- /dev/null +++ b/apps/gateway/src/chat-toolcalls-result.e2e.ts @@ -0,0 +1,138 @@ +import "dotenv/config"; +import { beforeAll, beforeEach, describe, expect, test } from "vitest"; + +import { + beforeAllHook, + beforeEachHook, + generateTestRequestId, + getTestOptions, + logMode, + toolCallModels, + validateLogByRequestId, +} from "@/chat-api.e2e"; +import { app } from "@/index"; + +describe("e2e", { concurrent: true }, () => { + beforeAll(beforeAllHook); + + beforeEach(beforeEachHook); + + test.each(toolCallModels)( + "tool calls with result $model", + getTestOptions(), + async ({ model }) => { + const requestId = generateTestRequestId(); + const res = await app.request("/v1/chat/completions", { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-request-id": requestId, + Authorization: `Bearer real-token`, + }, + body: JSON.stringify({ + model: model, + messages: [ + { + role: "system", + content: + "You are Noemi, a thoughtful and clear assistant. Your tone is calm, minimal, and human. You write with intention—never too much, never too little. You avoid clichés, speak simply, and offer helpful, grounded answers. When needed, you ask good questions. You don't try to impress—you aim to clarify. You may use metaphors if they bring clarity, but you stay sharp and sincere. You're here to help the user think clearly and move forward, not to overwhelm or overperform.", + }, + { + role: "user", + content: "web search for the best ai notetaker apps!!!!", + }, + { + role: "assistant", + content: "", + tool_calls: [ + { + id: "toolu_015dgN1nk5Ay12iN8e16XPbs", + type: "function", + function: { + name: "webSearch", + arguments: '{"query":"best AI notetaker apps 2024"}', + }, + }, + ], + }, + { + role: "tool", + content: + '{"type":"webSearch","query":"best AI notetaker apps 2024","results":[{"title":"My Deep Dive into 25+ AI Note-Taking Apps (The Brutally ... - Reddit","href":"https://www.reddit.com/r/Zoom/comments/1jtbxkf/my_deep_dive_into_25_ai_notetaking_apps_the/","description":"The Good: Think Obsidian meets Miro. Whiteboard-style interface for connecting notes visually. AI assistant can generate summaries and do ..."},{"title":"The 9 best AI meeting assistants in 2025 - Zapier","href":"https://zapier.com/blog/best-ai-meeting-assistant/","description":"Granola automatically transcribes, summarizes, and analyzes your meetings. It also acts as a live notepad, allowing you to manually jot down ..."},{"title":"The Best AI Tools for Taking Notes in 2025 - PCMag","href":"https://www.pcmag.com/picks/best-ai-tools-taking-notes","description":"The popular note-taking app Notion now has AI tools. Notion AI excels at answering questions about your existing data, generating text from a prompt you give it ..."},{"title":"Top 5 BEST AI Note-Taking Apps (Better than Notion?) - YouTube","href":"https://www.youtube.com/watch?v=wGLd43TkCGc","description":"Voicenotes is a voice‑to‑text powerhouse that transcribes and extracts action items in one tap. · Saner is A distraction‑free workspace built for ..."},{"title":"9 Best AI Note-Taking Apps Built For Your Meetings - Quil\'s AI","href":"https://quil.ai/2024/09/12/9-best-ai-note-taking-apps-built-for-your-meetings/","description":"Quil.ai: The AI Note-taker Built for Recruiting Firms. 2. Notion: Write, Plan, Organize. 3. Jamie AI: The Bot-Free AI Note-taker."}],"timestamp":"2025-08-29T01:20:29.553Z"}', + tool_call_id: "toolu_015dgN1nk5Ay12iN8e16XPbs", + }, + ], + tools: [ + { + type: "function", + function: { + name: "webSearch", + description: "Search the web for information", + parameters: { + type: "object", + properties: { + query: { + type: "string", + description: "Search query", + }, + }, + required: ["query"], + }, + }, + }, + ], + tool_choice: "auto", + }), + }); + + const json = await res.json(); + if (logMode) { + console.log( + "tool calls with empty content response:", + JSON.stringify(json, null, 2), + ); + } + + // Log error response if status is not 200 + if (res.status !== 200) { + console.log( + `Error ${res.status} - tool calls with result response:`, + JSON.stringify(json, null, 2), + ); + } + + expect(res.status).toBe(200); + expect(json).toHaveProperty("choices"); + expect(json.choices).toHaveLength(1); + expect(json.choices[0]).toHaveProperty("message"); + + const message = json.choices[0].message; + expect(message).toHaveProperty("role", "assistant"); + + // Should have proper content (not empty) as a response to the tool call + expect(message).toHaveProperty("content"); + // verify either content is string or tool_calls is present + expect(message.content || message.tool_calls).toBeTruthy(); + + // Should have finish reason as stop (not tool_calls since this is a response) + // TODO THIS IS FAILING ON SOME MODELS + // expect(json.choices[0]).toHaveProperty("finish_reason", "stop"); + + // Validate logs + const log = await validateLogByRequestId(requestId); + expect(log.streamed).toBe(false); + + // Validate usage + expect(json).toHaveProperty("usage"); + expect(json.usage).toHaveProperty("prompt_tokens"); + expect(json.usage).toHaveProperty("completion_tokens"); + expect(json.usage).toHaveProperty("total_tokens"); + expect(typeof json.usage.prompt_tokens).toBe("number"); + expect(typeof json.usage.completion_tokens).toBe("number"); + expect(typeof json.usage.total_tokens).toBe("number"); + expect(json.usage.prompt_tokens).toBeGreaterThan(0); + expect(json.usage.completion_tokens).toBeGreaterThan(0); + expect(json.usage.total_tokens).toBeGreaterThan(0); + }, + ); +}); diff --git a/apps/gateway/src/chat-toolcalls.e2e.ts b/apps/gateway/src/chat-toolcalls.e2e.ts new file mode 100644 index 0000000000..a3538f327c --- /dev/null +++ b/apps/gateway/src/chat-toolcalls.e2e.ts @@ -0,0 +1,126 @@ +import "dotenv/config"; +import { beforeAll, beforeEach, describe, expect, test } from "vitest"; + +import { + beforeAllHook, + beforeEachHook, + generateTestRequestId, + getTestOptions, + logMode, + toolCallModels, + validateLogByRequestId, +} from "@/chat-api.e2e"; +import { app } from "@/index"; + +describe("e2e", { concurrent: true }, () => { + beforeAll(beforeAllHook); + + beforeEach(beforeEachHook); + + test.each(toolCallModels)( + "tool calls $model", + getTestOptions(), + async ({ model }) => { + const requestId = generateTestRequestId(); + const res = await app.request("/v1/chat/completions", { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-request-id": requestId, + Authorization: `Bearer real-token`, + }, + body: JSON.stringify({ + model: model, + messages: [ + { + role: "system", + content: + "You are a weather assistant that can get weather information for cities.", + }, + { + role: "user", + content: "What's the weather like in San Francisco?", + }, + ], + tools: [ + { + type: "function", + function: { + name: "get_weather", + description: "Get the current weather for a given city", + parameters: { + type: "object", + properties: { + city: { + type: "string", + description: "The city name to get weather for", + }, + unit: { + type: "string", + enum: ["celsius", "fahrenheit"], + description: "Temperature unit", + default: "fahrenheit", + }, + }, + required: ["city"], + }, + }, + }, + ], + tool_choice: "auto", + }), + }); + + const json = await res.json(); + if (logMode) { + console.log("tool calls response:", JSON.stringify(json, null, 2)); + } + + expect(res.status).toBe(200); + expect(json).toHaveProperty("choices"); + expect(json.choices).toHaveLength(1); + expect(json.choices[0]).toHaveProperty("message"); + + const message = json.choices[0].message; + expect(message).toHaveProperty("role", "assistant"); + + // Should have tool calls since we're asking about weather + expect(message).toHaveProperty("tool_calls"); + expect(Array.isArray(message.tool_calls)).toBe(true); + expect(message.tool_calls.length).toBeGreaterThan(0); + + // Validate tool call structure + const toolCall = message.tool_calls[0]; + expect(toolCall).toHaveProperty("id"); + expect(toolCall).toHaveProperty("type", "function"); + expect(toolCall).toHaveProperty("function"); + expect(toolCall.function).toHaveProperty("name", "get_weather"); + expect(toolCall.function).toHaveProperty("arguments"); + + // Parse and validate arguments + const args = JSON.parse(toolCall.function.arguments); + expect(args).toHaveProperty("city"); + expect(typeof args.city).toBe("string"); + expect(args.city.toLowerCase()).toContain("san francisco"); + + // Check finish reason + expect(json.choices[0]).toHaveProperty("finish_reason", "tool_calls"); + + // Validate logs + const log = await validateLogByRequestId(requestId); + expect(log.streamed).toBe(false); + + // Validate usage + expect(json).toHaveProperty("usage"); + expect(json.usage).toHaveProperty("prompt_tokens"); + expect(json.usage).toHaveProperty("completion_tokens"); + expect(json.usage).toHaveProperty("total_tokens"); + expect(typeof json.usage.prompt_tokens).toBe("number"); + expect(typeof json.usage.completion_tokens).toBe("number"); + expect(typeof json.usage.total_tokens).toBe("number"); + expect(json.usage.prompt_tokens).toBeGreaterThan(0); + expect(json.usage.completion_tokens).toBeGreaterThan(0); + expect(json.usage.total_tokens).toBeGreaterThan(0); + }, + ); +}); From 9fa8b393038781f9b9c3dd8ee31d2c31becd8cb1 Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Mon, 15 Sep 2025 16:48:19 +0100 Subject: [PATCH 2/9] test: add setup completion flag to e2e hooks Introduce a `setupComplete` flag to optimize `beforeAllHook` by skipping redundant setup steps after the initial run. --- apps/gateway/src/chat-api.e2e.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/apps/gateway/src/chat-api.e2e.ts b/apps/gateway/src/chat-api.e2e.ts index 9d12b98ea5..a958d3077e 100644 --- a/apps/gateway/src/chat-api.e2e.ts +++ b/apps/gateway/src/chat-api.e2e.ts @@ -361,7 +361,14 @@ export async function validateLogByRequestId(requestId: string) { return log; } +let setupComplete = false; + export async function beforeAllHook() { + if (setupComplete) { + await clearCache(); + return; + } + await clearCache(); // Clean up any existing data @@ -426,6 +433,8 @@ export async function beforeAllHook() { await createProviderKey(provider.id, envVarValue, "credits"); } } + + setupComplete = true; } export async function beforeEachHook() { From 4b6fc74127efb9653c9af8bb74e1f597e2c81c07 Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Mon, 15 Sep 2025 17:00:45 +0100 Subject: [PATCH 3/9] test: remove setup completion flag and handle conflicts Removed the `setupComplete` flag from `beforeAllHook`, utilizing `onConflictDoNothing` to prevent duplicate data issues during setup. --- apps/gateway/src/chat-api.e2e.ts | 103 +++++++++++++++++-------------- 1 file changed, 56 insertions(+), 47 deletions(-) diff --git a/apps/gateway/src/chat-api.e2e.ts b/apps/gateway/src/chat-api.e2e.ts index a958d3077e..a94d370abc 100644 --- a/apps/gateway/src/chat-api.e2e.ts +++ b/apps/gateway/src/chat-api.e2e.ts @@ -327,12 +327,15 @@ export async function createProviderKey( ) { const keyId = keyType === "credits" ? `env-${provider}` : `provider-key-${provider}`; - await db.insert(tables.providerKey).values({ - id: keyId, - token, - provider: provider.replace("env-", ""), // Remove env- prefix for the provider field - organizationId: "org-id", - }); + await db + .insert(tables.providerKey) + .values({ + id: keyId, + token, + provider: provider.replace("env-", ""), // Remove env- prefix for the provider field + organizationId: "org-id", + }) + .onConflictDoNothing(); } export function validateResponse(json: any) { @@ -361,14 +364,7 @@ export async function validateLogByRequestId(requestId: string) { return log; } -let setupComplete = false; - export async function beforeAllHook() { - if (setupComplete) { - await clearCache(); - return; - } - await clearCache(); // Clean up any existing data @@ -391,38 +387,53 @@ export async function beforeAllHook() { db.delete(tables.verification), ]); - // Set up shared test data that all tests can use - await db.insert(tables.user).values({ - id: "user-id", - name: "user", - email: "user", - }); - - await db.insert(tables.organization).values({ - id: "org-id", - name: "Test Organization", - plan: "pro", - }); - - await db.insert(tables.userOrganization).values({ - id: "user-org-id", - userId: "user-id", - organizationId: "org-id", - }); - - await db.insert(tables.project).values({ - id: "project-id", - name: "Test Project", - organizationId: "org-id", - mode: "api-keys", - }); - - await db.insert(tables.apiKey).values({ - id: "token-id", - token: "real-token", - projectId: "project-id", - description: "Test API Key", - }); + // Set up shared test data that all tests can use - use ON CONFLICT DO NOTHING to avoid duplicate key errors + await db + .insert(tables.user) + .values({ + id: "user-id", + name: "user", + email: "user", + }) + .onConflictDoNothing(); + + await db + .insert(tables.organization) + .values({ + id: "org-id", + name: "Test Organization", + plan: "pro", + }) + .onConflictDoNothing(); + + await db + .insert(tables.userOrganization) + .values({ + id: "user-org-id", + userId: "user-id", + organizationId: "org-id", + }) + .onConflictDoNothing(); + + await db + .insert(tables.project) + .values({ + id: "project-id", + name: "Test Project", + organizationId: "org-id", + mode: "api-keys", + }) + .onConflictDoNothing(); + + await db + .insert(tables.apiKey) + .values({ + id: "token-id", + token: "real-token", + projectId: "project-id", + description: "Test API Key", + }) + .onConflictDoNothing(); // Set up provider keys for all providers for (const provider of providers) { @@ -433,8 +444,6 @@ export async function beforeAllHook() { await createProviderKey(provider.id, envVarValue, "credits"); } } - - setupComplete = true; } export async function beforeEachHook() { From 151553c21c089b62968d286baf8eb9144d9b2db3 Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Mon, 15 Sep 2025 17:26:20 +0100 Subject: [PATCH 4/9] test(chat-full): add placeholder test case Introduced an empty test case to ensure the `chat-full` file runs without errors. --- apps/gateway/src/chat-full.e2e.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/gateway/src/chat-full.e2e.ts b/apps/gateway/src/chat-full.e2e.ts index 8acd7332c3..0a219fa241 100644 --- a/apps/gateway/src/chat-full.e2e.ts +++ b/apps/gateway/src/chat-full.e2e.ts @@ -20,6 +20,10 @@ describe("e2e", { concurrent: true }, () => { beforeEach(beforeEachHook); + test("empty", () => { + expect(true).toBe(true); + }); + if (fullMode) { const reasoningToolCallModels = testModels.filter((m) => m.providers.some( From 9ab96537eec2a919af79ff88e788ba0bf5055cdd Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Mon, 15 Sep 2025 17:35:16 +0100 Subject: [PATCH 5/9] test(package): remove no-file-parallelism flag from e2e script Updated the `test:e2e` script in `package.json` to remove the `--no-file-parallelism` flag for streamlined test execution. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index be2eb40479..c04f7a6494 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "seed": "pnpm --filter db seed", "setup": "docker compose down -v && docker compose up -d && sleep 5 && pnpm push-test && pnpm push-dev && pnpm seed", "sync": "pnpm push-dev; pnpm push-test", - "test:e2e": "cross-env-shell \"DATABASE_URL=${DATABASE_URL:-postgres://postgres:pw@localhost:5432/test}\" E2E_TEST=true vitest run -c vitest/vitest.e2e.config.mts --no-file-parallelism", + "test:e2e": "cross-env-shell \"DATABASE_URL=${DATABASE_URL:-postgres://postgres:pw@localhost:5432/test}\" E2E_TEST=true vitest run -c vitest/vitest.e2e.config.mts", "test:unit": "cross-env-shell \"DATABASE_URL=${DATABASE_URL:-postgres://postgres:pw@localhost:5432/test}\" vitest run --no-file-parallelism" }, "commitlint": { From 7663775158a4db7ca9badd167088e4a5660b1c3e Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Mon, 15 Sep 2025 18:02:34 +0100 Subject: [PATCH 6/9] test(chat-helpers): extract shared test utilities Moved shared helper functions and constants to a new `chat-helpers.e2e.ts` file for better reusability and organization. Updated imports across relevant test files. Removed redundant cleanup logic from `beforeAllHook` in `chat-api.e2e.ts`. --- apps/gateway/src/api-individual.e2e.ts | 2 +- apps/gateway/src/chat-api.e2e.ts | 20 - apps/gateway/src/chat-full.e2e.ts | 2 +- apps/gateway/src/chat-helpers.e2e.ts | 422 ++++++++++++++++++ apps/gateway/src/chat-reasoning.e2e.ts | 2 +- apps/gateway/src/chat-rs.e2e.ts | 2 +- apps/gateway/src/chat-streaming.e2e.ts | 2 +- apps/gateway/src/chat-toolcalls-result.e2e.ts | 2 +- apps/gateway/src/chat-toolcalls.e2e.ts | 2 +- 9 files changed, 429 insertions(+), 27 deletions(-) create mode 100644 apps/gateway/src/chat-helpers.e2e.ts diff --git a/apps/gateway/src/api-individual.e2e.ts b/apps/gateway/src/api-individual.e2e.ts index 1f9f5bd6a7..de07277331 100644 --- a/apps/gateway/src/api-individual.e2e.ts +++ b/apps/gateway/src/api-individual.e2e.ts @@ -5,7 +5,7 @@ import { generateTestRequestId, logMode, validateLogByRequestId, -} from "@/chat-api.e2e"; +} from "@/chat-helpers.e2e"; import { db, tables, eq } from "@llmgateway/db"; import { models, providers } from "@llmgateway/models"; diff --git a/apps/gateway/src/chat-api.e2e.ts b/apps/gateway/src/chat-api.e2e.ts index a94d370abc..be5758eb91 100644 --- a/apps/gateway/src/chat-api.e2e.ts +++ b/apps/gateway/src/chat-api.e2e.ts @@ -367,26 +367,6 @@ export async function validateLogByRequestId(requestId: string) { export async function beforeAllHook() { await clearCache(); - // Clean up any existing data - await Promise.all([ - db.delete(tables.log), - db.delete(tables.apiKey), - db.delete(tables.providerKey), - ]); - - await Promise.all([ - db.delete(tables.userOrganization), - db.delete(tables.project), - ]); - - await Promise.all([ - db.delete(tables.organization), - db.delete(tables.user), - db.delete(tables.account), - db.delete(tables.session), - db.delete(tables.verification), - ]); - // Set up shared test data that all tests can use - use ON CONFLICT DO NOTHING to avoid duplicate key errors await db .insert(tables.user) diff --git a/apps/gateway/src/chat-full.e2e.ts b/apps/gateway/src/chat-full.e2e.ts index 0a219fa241..9ee3eca33a 100644 --- a/apps/gateway/src/chat-full.e2e.ts +++ b/apps/gateway/src/chat-full.e2e.ts @@ -10,7 +10,7 @@ import { logMode, testModels, validateLogByRequestId, -} from "@/chat-api.e2e"; +} from "@/chat-helpers.e2e"; import { app } from "@/index"; import type { ProviderModelMapping } from "@llmgateway/models"; diff --git a/apps/gateway/src/chat-helpers.e2e.ts b/apps/gateway/src/chat-helpers.e2e.ts new file mode 100644 index 0000000000..aa9d9840f5 --- /dev/null +++ b/apps/gateway/src/chat-helpers.e2e.ts @@ -0,0 +1,422 @@ +import "dotenv/config"; +import { expect, type TestOptions } from "vitest"; + +import { db, tables } from "@llmgateway/db"; +import { + type ModelDefinition, + models, + type ProviderModelMapping, + providers, +} from "@llmgateway/models"; + +import { + clearCache, + waitForLogByRequestId, + getProviderEnvVar, +} from "./test-utils/test-helpers"; + +// Helper function to generate unique request IDs for tests +export function generateTestRequestId(): string { + return `test-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; +} + +// Helper function to get test options with retry for CI environment +export function getTestOptions(): TestOptions { + return process.env.CI ? { retry: 3 } : {}; +} + +console.log("running with test options:", getTestOptions()); + +export const fullMode = process.env.FULL_MODE; +export const logMode = process.env.LOG_MODE; + +// Parse TEST_MODELS environment variable +export const testModelsEnv = process.env.TEST_MODELS; +export const specifiedModels = testModelsEnv + ? testModelsEnv.split(",").map((m) => m.trim()) + : null; + +if (specifiedModels) { + console.log(`TEST_MODELS specified: ${specifiedModels.join(", ")}`); +} + +// Filter models based on test skip/only property +export const hasOnlyModels = models.some((model) => + model.providers.some( + (provider: ProviderModelMapping) => provider.test === "only", + ), +); + +// Log if we're using "only" mode +if (hasOnlyModels) { + if (process.env.CI) { + throw new Error( + "Cannot use 'only' in test configuration when running in CI. Please remove 'only' from the test configuration and try again.", + ); + } + console.log( + "Running in 'only' mode - only testing models marked with test: 'only'", + ); +} + +export const filteredModels = models + // Filter out auto/custom models + .filter((model) => !["custom", "auto"].includes(model.id)) + // Filter out deactivated models + .filter((model) => !model.deactivatedAt || new Date() <= model.deactivatedAt) + // Filter out unstable models if not in full mode, unless they have test: "only" or are in TEST_MODELS + .filter((model) => { + // Check if model or any of its providers are marked as unstable + const modelStability = (model as ModelDefinition).stability; + const hasUnstableProviders = model.providers.some( + (provider: ProviderModelMapping) => provider.stability === "unstable", + ); + const isUnstable = modelStability === "unstable" || hasUnstableProviders; + + if (!isUnstable) { + return true; + } // Non-unstable models are always included + if (fullMode) { + return true; + } // In full mode, all models are included + + // For unstable models in non-full mode, include if: + // 1. Any provider has test: "only" + if ( + model.providers.some( + (provider: ProviderModelMapping) => provider.test === "only", + ) + ) { + return true; + } + + // 2. Model is specified in TEST_MODELS + if (specifiedModels) { + const modelInTestModels = model.providers.some( + (provider: ProviderModelMapping) => { + const providerModelId = `${provider.providerId}/${model.id}`; + return specifiedModels.includes(providerModelId); + }, + ); + if (modelInTestModels) { + return true; + } + } + + return false; // Otherwise, exclude unstable models in non-full mode + }) + // Filter out free models if not in full mode, unless they have test: "only" or are in TEST_MODELS + .filter((model) => { + const isFreeModel = (model as ModelDefinition).free; + if (!isFreeModel) { + return true; + } // Non-free models are always included + if (fullMode) { + return true; + } // In full mode, all models are included + + // For free models in non-full mode, include if: + // 1. Any provider has test: "only" + if ( + model.providers.some( + (provider: ProviderModelMapping) => provider.test === "only", + ) + ) { + return true; + } + + // 2. Model is specified in TEST_MODELS + if (specifiedModels) { + const modelInTestModels = model.providers.some( + (provider: ProviderModelMapping) => { + const providerModelId = `${provider.providerId}/${model.id}`; + return specifiedModels.includes(providerModelId); + }, + ); + if (modelInTestModels) { + return true; + } + } + + return false; // Otherwise, exclude free models in non-full mode + }) + // Filter by TEST_MODELS if specified + .filter((model) => { + if (!specifiedModels) { + return true; + } + // Check if any provider/model combination from this model matches TEST_MODELS + return model.providers.some((provider: ProviderModelMapping) => { + const providerModelId = `${provider.providerId}/${model.id}`; + return specifiedModels.includes(providerModelId); + }); + }); + +export const testModels = filteredModels + // If any model has test: "only", only include those models + .filter((model) => { + if (hasOnlyModels) { + return model.providers.some( + (provider: ProviderModelMapping) => provider.test === "only", + ); + } + return true; + }) + .flatMap((model) => { + const testCases = []; + + if (process.env.TEST_ALL_VARIATIONS) { + // test root model without a specific provider + testCases.push({ + model: model.id, + providers: model.providers.filter( + (provider: ProviderModelMapping) => provider.test !== "skip", + ), + }); + } + + // Create entries for provider-specific requests using provider/model format + for (const provider of model.providers as ProviderModelMapping[]) { + // Skip providers marked with test: "skip" + if (provider.test === "skip") { + continue; + } + + // Skip unstable providers if not in full mode, unless they have test: "only" or are in TEST_MODELS + if (provider.stability === "unstable" && !fullMode) { + // Allow if provider has test: "only" + if (provider.test !== "only") { + // Allow if model is specified in TEST_MODELS + if (!specifiedModels) { + continue; + } + const providerModelId = `${provider.providerId}/${model.id}`; + if (!specifiedModels.includes(providerModelId)) { + continue; + } + } + } + + // If we have any "only" providers, skip those not marked as "only" + if (hasOnlyModels && provider.test !== "only") { + continue; + } + + testCases.push({ + model: `${provider.providerId}/${model.id}`, + providers: [provider], + originalModel: model.id, // Keep track of the original model for reference + }); + } + + return testCases; + }); + +export const providerModels = filteredModels + // If any model has test: "only", only include those models + .filter((model) => { + if (hasOnlyModels) { + return model.providers.some( + (provider: ProviderModelMapping) => provider.test === "only", + ); + } + return true; + }) + .flatMap((model) => { + const testCases = []; + + for (const provider of model.providers as ProviderModelMapping[]) { + // Skip providers marked with test: "skip" + if (provider.test === "skip") { + continue; + } + + // Skip unstable providers if not in full mode, unless they have test: "only" or are in TEST_MODELS + if (provider.stability === "unstable" && !fullMode) { + // Allow if provider has test: "only" + if (provider.test !== "only") { + // Allow if model is specified in TEST_MODELS + if (!specifiedModels) { + continue; + } + const providerModelId = `${provider.providerId}/${model.id}`; + if (!specifiedModels.includes(providerModelId)) { + continue; + } + } + } + + // If we have any "only" providers, skip those not marked as "only" + if (hasOnlyModels && provider.test !== "only") { + continue; + } + + testCases.push({ + model: `${provider.providerId}/${model.id}`, + provider, + originalModel: model.id, // Keep track of the original model for reference + }); + } + + return testCases; + }); + +// Log the number of test models after filtering +console.log(`Testing ${testModels.length} model configurations`); +console.log(`Testing ${providerModels.length} provider model configurations`); + +export const streamingModels = testModels.filter((m) => + m.providers.some((p: ProviderModelMapping) => { + // Check model-level streaming first, then fall back to provider-level + if (p.streaming !== undefined) { + return p.streaming; + } + const provider = providers.find((pr) => pr.id === p.providerId); + return provider?.streaming; + }), +); + +export const reasoningModels = testModels.filter((m) => + m.providers.some((p: ProviderModelMapping) => p.reasoning === true), +); + +export const streamingReasoningModels = reasoningModels.filter((m) => + m.providers.some((p: ProviderModelMapping) => { + // Check model-level streaming first, then fall back to provider-level + if (p.streaming !== undefined) { + return p.streaming; + } + const provider = providers.find((pr) => pr.id === p.providerId); + return provider?.streaming; + }), +); + +export const toolCallModels = testModels.filter((m) => + m.providers.some((p: ProviderModelMapping) => p.tools === true), +); + +export const imageModels = testModels.filter((m) => { + const model = models.find((mo) => m.originalModel === mo.id); + return (model as ModelDefinition).output?.includes("image"); +}); + +export const streamingImageModels = imageModels.filter((m) => + m.providers.some((p: ProviderModelMapping) => { + // Check model-level streaming first, then fall back to provider-level + if (p.streaming !== undefined) { + return p.streaming; + } + const provider = providers.find((pr) => pr.id === p.providerId); + return provider?.streaming; + }), +); + +export async function createProviderKey( + provider: string, + token: string, + keyType: "api-keys" | "credits" = "api-keys", +) { + const keyId = + keyType === "credits" ? `env-${provider}` : `provider-key-${provider}`; + await db + .insert(tables.providerKey) + .values({ + id: keyId, + token, + provider: provider.replace("env-", ""), // Remove env- prefix for the provider field + organizationId: "org-id", + }) + .onConflictDoNothing(); +} + +export function validateResponse(json: any) { + expect(json).toHaveProperty("choices.[0].message.content"); + + expect(json).toHaveProperty("usage.prompt_tokens"); + expect(json).toHaveProperty("usage.completion_tokens"); + expect(json).toHaveProperty("usage.total_tokens"); +} + +export async function validateLogByRequestId(requestId: string) { + const log = await waitForLogByRequestId(requestId); + + if (logMode) { + console.log("log", JSON.stringify(log, null, 2)); + } + + expect(log.usedProvider).toBeTruthy(); + expect(log.errorDetails).toBeNull(); + expect(log.finishReason).not.toBeNull(); + expect(log.unifiedFinishReason).not.toBeNull(); + expect(log.unifiedFinishReason).toBeTruthy(); + expect(log.usedModel).toBeTruthy(); + expect(log.requestedModel).toBeTruthy(); + + return log; +} + +export async function beforeAllHook() { + await clearCache(); + + // Set up shared test data that all tests can use - use ON CONFLICT DO NOTHING to avoid duplicate key errors + await db + .insert(tables.user) + .values({ + id: "user-id", + name: "user", + email: "user", + }) + .onConflictDoNothing(); + + await db + .insert(tables.organization) + .values({ + id: "org-id", + name: "Test Organization", + plan: "pro", + }) + .onConflictDoNothing(); + + await db + .insert(tables.userOrganization) + .values({ + id: "user-org-id", + userId: "user-id", + organizationId: "org-id", + }) + .onConflictDoNothing(); + + await db + .insert(tables.project) + .values({ + id: "project-id", + name: "Test Project", + organizationId: "org-id", + mode: "api-keys", + }) + .onConflictDoNothing(); + + await db + .insert(tables.apiKey) + .values({ + id: "token-id", + token: "real-token", + projectId: "project-id", + description: "Test API Key", + }) + .onConflictDoNothing(); + + // Set up provider keys for all providers + for (const provider of providers) { + const envVarName = getProviderEnvVar(provider.id); + const envVarValue = envVarName ? process.env[envVarName] : undefined; + if (envVarValue) { + await createProviderKey(provider.id, envVarValue, "api-keys"); + await createProviderKey(provider.id, envVarValue, "credits"); + } + } +} + +export async function beforeEachHook() { + await clearCache(); +} diff --git a/apps/gateway/src/chat-reasoning.e2e.ts b/apps/gateway/src/chat-reasoning.e2e.ts index d673a64973..a4e226742b 100644 --- a/apps/gateway/src/chat-reasoning.e2e.ts +++ b/apps/gateway/src/chat-reasoning.e2e.ts @@ -10,7 +10,7 @@ import { reasoningModels, validateLogByRequestId, validateResponse, -} from "@/chat-api.e2e"; +} from "@/chat-helpers.e2e"; import { app } from "@/index"; import type { ProviderModelMapping } from "@llmgateway/models"; diff --git a/apps/gateway/src/chat-rs.e2e.ts b/apps/gateway/src/chat-rs.e2e.ts index 021ab6084d..1653bdb5e7 100644 --- a/apps/gateway/src/chat-rs.e2e.ts +++ b/apps/gateway/src/chat-rs.e2e.ts @@ -9,7 +9,7 @@ import { logMode, streamingReasoningModels, validateLogByRequestId, -} from "@/chat-api.e2e"; +} from "@/chat-helpers.e2e"; import { app } from "@/index"; import { readAll } from "@/test-utils/test-helpers"; diff --git a/apps/gateway/src/chat-streaming.e2e.ts b/apps/gateway/src/chat-streaming.e2e.ts index ea9cba9b56..3783cd5894 100644 --- a/apps/gateway/src/chat-streaming.e2e.ts +++ b/apps/gateway/src/chat-streaming.e2e.ts @@ -9,7 +9,7 @@ import { logMode, streamingModels, validateLogByRequestId, -} from "@/chat-api.e2e"; +} from "@/chat-helpers.e2e"; import { app } from "@/index"; import { readAll } from "@/test-utils/test-helpers"; diff --git a/apps/gateway/src/chat-toolcalls-result.e2e.ts b/apps/gateway/src/chat-toolcalls-result.e2e.ts index 440b7b734b..a23b09dcac 100644 --- a/apps/gateway/src/chat-toolcalls-result.e2e.ts +++ b/apps/gateway/src/chat-toolcalls-result.e2e.ts @@ -9,7 +9,7 @@ import { logMode, toolCallModels, validateLogByRequestId, -} from "@/chat-api.e2e"; +} from "@/chat-helpers.e2e"; import { app } from "@/index"; describe("e2e", { concurrent: true }, () => { diff --git a/apps/gateway/src/chat-toolcalls.e2e.ts b/apps/gateway/src/chat-toolcalls.e2e.ts index a3538f327c..5898880ce7 100644 --- a/apps/gateway/src/chat-toolcalls.e2e.ts +++ b/apps/gateway/src/chat-toolcalls.e2e.ts @@ -9,7 +9,7 @@ import { logMode, toolCallModels, validateLogByRequestId, -} from "@/chat-api.e2e"; +} from "@/chat-helpers.e2e"; import { app } from "@/index"; describe("e2e", { concurrent: true }, () => { From 1a0e1e2d453c0962991b6f18dedc9e13006e7f83 Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Mon, 15 Sep 2025 18:17:57 +0100 Subject: [PATCH 7/9] chore: wip --- apps/gateway/src/chat-helpers.e2e.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/gateway/src/chat-helpers.e2e.ts b/apps/gateway/src/chat-helpers.e2e.ts index aa9d9840f5..94ab10d524 100644 --- a/apps/gateway/src/chat-helpers.e2e.ts +++ b/apps/gateway/src/chat-helpers.e2e.ts @@ -1,5 +1,5 @@ import "dotenv/config"; -import { expect, type TestOptions } from "vitest"; +import { describe, expect, it, type TestOptions } from "vitest"; import { db, tables } from "@llmgateway/db"; import { @@ -420,3 +420,9 @@ export async function beforeAllHook() { export async function beforeEachHook() { await clearCache(); } + +describe("e2e", { concurrent: true }, () => { + it("empty", () => { + expect(true).toBe(true); + }); +}); From 2f8d52e7aa619331daaf8690d50dd24c765fb3e2 Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Mon, 15 Sep 2025 19:07:30 +0100 Subject: [PATCH 8/9] test(e2e): refactor hooks and enhance log validation Replaced repetitive setup logic in `beforeAll` and `beforeEach` hooks with shared utilities. Updated log validation tests to use request-specific IDs for precise querying. Modified `test:e2e` script to enable `--no-file-parallelism`. --- apps/api/src/routes/keys-provider.e2e.ts | 84 +++++++++--------- apps/gateway/src/chat-api.e2e.ts | 75 +--------------- apps/gateway/src/log-queue.e2e.ts | 101 ++++++---------------- apps/ui/src/app/blog/[slug]/page.tsx | 3 +- apps/ui/src/app/changelog/[slug]/page.tsx | 3 +- package.json | 2 +- 6 files changed, 76 insertions(+), 192 deletions(-) diff --git a/apps/api/src/routes/keys-provider.e2e.ts b/apps/api/src/routes/keys-provider.e2e.ts index 98e4d9e355..265008239b 100644 --- a/apps/api/src/routes/keys-provider.e2e.ts +++ b/apps/api/src/routes/keys-provider.e2e.ts @@ -169,47 +169,47 @@ describe( }, ); - test.skip("POST /keys/provider with custom baseUrl", async () => { - if (!process.env.OPENAI_API_KEY) { - console.log("Skipping custom baseUrl test - no API key provided"); - return; - } - - const { token, orgId } = await setupTestData(); - const customBaseUrl = "https://api.custom-openai.example.com"; - const res = await app.request("/keys/provider", { - method: "POST", - headers: { - "Content-Type": "application/json", - Cookie: token, - }, - body: JSON.stringify({ - provider: "openai", - token: process.env.OPENAI_API_KEY, - baseUrl: customBaseUrl, - organizationId: orgId, - }), - }); - - expect(res.status).toBe(200); - const json = await res.json(); - expect(json).toHaveProperty("providerKey"); - expect(json.providerKey.provider).toBe("openai"); - expect(json.providerKey.baseUrl).toBe(customBaseUrl); - - const providerKey = await db.query.providerKey.findFirst({ - where: { - provider: { - eq: "openai", - }, - organizationId: { - eq: orgId, - }, - }, - }); - expect(providerKey).not.toBeNull(); - expect(providerKey?.provider).toBe("openai"); - expect(providerKey?.baseUrl).toBe(customBaseUrl); - }); + // test.skip("POST /keys/provider with custom baseUrl", async () => { + // if (!process.env.OPENAI_API_KEY) { + // console.log("Skipping custom baseUrl test - no API key provided"); + // return; + // } + // + // const { token, orgId } = await setupTestData(); + // const customBaseUrl = "https://api.custom-openai.example.com"; + // const res = await app.request("/keys/provider", { + // method: "POST", + // headers: { + // "Content-Type": "application/json", + // Cookie: token, + // }, + // body: JSON.stringify({ + // provider: "openai", + // token: process.env.OPENAI_API_KEY, + // baseUrl: customBaseUrl, + // organizationId: orgId, + // }), + // }); + // + // expect(res.status).toBe(200); + // const json = await res.json(); + // expect(json).toHaveProperty("providerKey"); + // expect(json.providerKey.provider).toBe("openai"); + // expect(json.providerKey.baseUrl).toBe(customBaseUrl); + // + // const providerKey = await db.query.providerKey.findFirst({ + // where: { + // provider: { + // eq: "openai", + // }, + // organizationId: { + // eq: orgId, + // }, + // }, + // }); + // expect(providerKey).not.toBeNull(); + // expect(providerKey?.provider).toBe("openai"); + // expect(providerKey?.baseUrl).toBe(customBaseUrl); + // }); }, ); diff --git a/apps/gateway/src/chat-api.e2e.ts b/apps/gateway/src/chat-api.e2e.ts index be5758eb91..0b10b0bb46 100644 --- a/apps/gateway/src/chat-api.e2e.ts +++ b/apps/gateway/src/chat-api.e2e.ts @@ -8,6 +8,8 @@ import { type TestOptions, } from "vitest"; +import { beforeAllHook, beforeEachHook } from "@/chat-helpers.e2e"; + import { db, tables } from "@llmgateway/db"; import { type ModelDefinition, @@ -17,12 +19,7 @@ import { } from "@llmgateway/models"; import { app } from "."; -import { - clearCache, - waitForLogByRequestId, - getProviderEnvVar, - readAll, -} from "./test-utils/test-helpers"; +import { waitForLogByRequestId, readAll } from "./test-utils/test-helpers"; // Helper function to generate unique request IDs for tests export function generateTestRequestId(): string { @@ -364,72 +361,6 @@ export async function validateLogByRequestId(requestId: string) { return log; } -export async function beforeAllHook() { - await clearCache(); - - // Set up shared test data that all tests can use - use ON CONFLICT DO NOTHING to avoid duplicate key errors - await db - .insert(tables.user) - .values({ - id: "user-id", - name: "user", - email: "user", - }) - .onConflictDoNothing(); - - await db - .insert(tables.organization) - .values({ - id: "org-id", - name: "Test Organization", - plan: "pro", - }) - .onConflictDoNothing(); - - await db - .insert(tables.userOrganization) - .values({ - id: "user-org-id", - userId: "user-id", - organizationId: "org-id", - }) - .onConflictDoNothing(); - - await db - .insert(tables.project) - .values({ - id: "project-id", - name: "Test Project", - organizationId: "org-id", - mode: "api-keys", - }) - .onConflictDoNothing(); - - await db - .insert(tables.apiKey) - .values({ - id: "token-id", - token: "real-token", - projectId: "project-id", - description: "Test API Key", - }) - .onConflictDoNothing(); - - // Set up provider keys for all providers - for (const provider of providers) { - const envVarName = getProviderEnvVar(provider.id); - const envVarValue = envVarName ? process.env[envVarName] : undefined; - if (envVarValue) { - await createProviderKey(provider.id, envVarValue, "api-keys"); - await createProviderKey(provider.id, envVarValue, "credits"); - } - } -} - -export async function beforeEachHook() { - await clearCache(); -} - describe("e2e", { concurrent: true }, () => { beforeAll(beforeAllHook); diff --git a/apps/gateway/src/log-queue.e2e.ts b/apps/gateway/src/log-queue.e2e.ts index b82e285d8f..b190b2fb8a 100644 --- a/apps/gateway/src/log-queue.e2e.ts +++ b/apps/gateway/src/log-queue.e2e.ts @@ -1,12 +1,22 @@ import "dotenv/config"; -import { beforeEach, describe, expect, test, type TestOptions } from "vitest"; +import { + beforeAll, + beforeEach, + describe, + expect, + test, + type TestOptions, +} from "vitest"; -import { db, tables } from "@llmgateway/db"; +import { + beforeAllHook, + beforeEachHook, + generateTestRequestId, +} from "@/chat-helpers.e2e"; import { app } from "."; import { - clearCache, - waitForLogs, + waitForLogByRequestId, getProviderEnvVar, } from "./test-utils/test-helpers"; @@ -16,72 +26,8 @@ function getTestOptions(): TestOptions { } describe("Log Queue Processing E2E", () => { - beforeEach(async () => { - await clearCache(); - - await Promise.all([ - db.delete(tables.log), - db.delete(tables.apiKey), - db.delete(tables.providerKey), - ]); - - await Promise.all([ - db.delete(tables.userOrganization), - db.delete(tables.project), - ]); - - await Promise.all([ - db.delete(tables.organization), - db.delete(tables.user), - db.delete(tables.account), - db.delete(tables.session), - db.delete(tables.verification), - ]); - - await db.insert(tables.user).values({ - id: "user-id", - name: "user", - email: "user", - }); - - await db.insert(tables.organization).values({ - id: "org-id", - name: "Test Organization", - plan: "pro", - }); - - await db.insert(tables.userOrganization).values({ - id: "user-org-id", - userId: "user-id", - organizationId: "org-id", - }); - - await db.insert(tables.project).values({ - id: "project-id", - name: "Test Project", - organizationId: "org-id", - mode: "api-keys", - }); - - await db.insert(tables.apiKey).values({ - id: "token-id", - token: "real-token", - projectId: "project-id", - description: "Test API Key", - }); - - // Set up a simple provider key for testing - const envVarName = getProviderEnvVar("openai"); - const envVarValue = envVarName ? process.env[envVarName] : undefined; - if (envVarValue) { - await db.insert(tables.providerKey).values({ - id: "provider-key-openai", - token: envVarValue, - provider: "openai", - organizationId: "org-id", - }); - } - }); + beforeAll(beforeAllHook); + beforeEach(beforeEachHook); test( "process log queue processes logs correctly", @@ -95,11 +41,13 @@ describe("Log Queue Processing E2E", () => { } // Make a request that should generate a log entry + const requestId = generateTestRequestId(); const res = await app.request("/v1/chat/completions", { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer real-token`, + "x-request-id": requestId, }, body: JSON.stringify({ model: "openai/gpt-4o-mini", @@ -117,10 +65,7 @@ describe("Log Queue Processing E2E", () => { expect(json).toHaveProperty("choices.[0].message.content"); // Test that the log queue processing works as expected - const logs = await waitForLogs(1); - expect(logs.length).toBe(1); - - const log = logs[0]; + const log = await waitForLogByRequestId(requestId); expect(log.usedProvider).toBeTruthy(); expect(log.errorDetails).toBeNull(); expect(log.finishReason).not.toBeNull(); @@ -145,13 +90,17 @@ describe("Log Queue Processing E2E", () => { // Make multiple requests that should generate multiple log entries const promises = []; + const requestIds = []; for (let i = 0; i < 3; i++) { + const requestId = generateTestRequestId(); + requestIds.push(requestId); promises.push( app.request("/v1/chat/completions", { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer real-token`, + "x-request-id": requestId, }, body: JSON.stringify({ model: "openai/gpt-4o-mini", @@ -172,7 +121,9 @@ describe("Log Queue Processing E2E", () => { } // Test that the log queue processing handles multiple logs - const logs = await waitForLogs(3); + const logs = await Promise.all( + requestIds.map((requestId) => waitForLogByRequestId(requestId)), + ); expect(logs.length).toBe(3); for (const log of logs) { diff --git a/apps/ui/src/app/blog/[slug]/page.tsx b/apps/ui/src/app/blog/[slug]/page.tsx index 91d96060aa..4550422324 100644 --- a/apps/ui/src/app/blog/[slug]/page.tsx +++ b/apps/ui/src/app/blog/[slug]/page.tsx @@ -1,4 +1,3 @@ -import { allBlogs } from "content-collections"; import { ArrowLeftIcon } from "lucide-react"; import Markdown from "markdown-to-jsx"; import Image from "next/image"; @@ -9,6 +8,8 @@ import Footer from "@/components/landing/footer"; import { HeroRSC } from "@/components/landing/hero-rsc"; import { getMarkdownOptions } from "@/lib/utils/markdown"; +import { allBlogs } from "content-collections"; + import type { Blog } from "content-collections"; interface BlogEntryPageProps { diff --git a/apps/ui/src/app/changelog/[slug]/page.tsx b/apps/ui/src/app/changelog/[slug]/page.tsx index 6e7a26092d..18e3f2c9aa 100644 --- a/apps/ui/src/app/changelog/[slug]/page.tsx +++ b/apps/ui/src/app/changelog/[slug]/page.tsx @@ -1,4 +1,3 @@ -import { allChangelogs } from "content-collections"; import { ArrowLeftIcon } from "lucide-react"; import Markdown from "markdown-to-jsx"; import Image from "next/image"; @@ -9,6 +8,8 @@ import Footer from "@/components/landing/footer"; import { HeroRSC } from "@/components/landing/hero-rsc"; import { getMarkdownOptions } from "@/lib/utils/markdown"; +import { allChangelogs } from "content-collections"; + import type { Changelog } from "content-collections"; interface ChangelogEntryPageProps { diff --git a/package.json b/package.json index c04f7a6494..be2eb40479 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "seed": "pnpm --filter db seed", "setup": "docker compose down -v && docker compose up -d && sleep 5 && pnpm push-test && pnpm push-dev && pnpm seed", "sync": "pnpm push-dev; pnpm push-test", - "test:e2e": "cross-env-shell \"DATABASE_URL=${DATABASE_URL:-postgres://postgres:pw@localhost:5432/test}\" E2E_TEST=true vitest run -c vitest/vitest.e2e.config.mts", + "test:e2e": "cross-env-shell \"DATABASE_URL=${DATABASE_URL:-postgres://postgres:pw@localhost:5432/test}\" E2E_TEST=true vitest run -c vitest/vitest.e2e.config.mts --no-file-parallelism", "test:unit": "cross-env-shell \"DATABASE_URL=${DATABASE_URL:-postgres://postgres:pw@localhost:5432/test}\" vitest run --no-file-parallelism" }, "commitlint": { From 126c4ddc9c8ad5d81a315ecef3b5f35f60a9c04a Mon Sep 17 00:00:00 2001 From: "Luca Steeb (bot)" Date: Mon, 15 Sep 2025 18:10:33 +0000 Subject: [PATCH 9/9] chore(autofix): apply diff --- apps/ui/src/app/blog/[slug]/page.tsx | 3 +-- apps/ui/src/app/changelog/[slug]/page.tsx | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/apps/ui/src/app/blog/[slug]/page.tsx b/apps/ui/src/app/blog/[slug]/page.tsx index 4550422324..91d96060aa 100644 --- a/apps/ui/src/app/blog/[slug]/page.tsx +++ b/apps/ui/src/app/blog/[slug]/page.tsx @@ -1,3 +1,4 @@ +import { allBlogs } from "content-collections"; import { ArrowLeftIcon } from "lucide-react"; import Markdown from "markdown-to-jsx"; import Image from "next/image"; @@ -8,8 +9,6 @@ import Footer from "@/components/landing/footer"; import { HeroRSC } from "@/components/landing/hero-rsc"; import { getMarkdownOptions } from "@/lib/utils/markdown"; -import { allBlogs } from "content-collections"; - import type { Blog } from "content-collections"; interface BlogEntryPageProps { diff --git a/apps/ui/src/app/changelog/[slug]/page.tsx b/apps/ui/src/app/changelog/[slug]/page.tsx index 18e3f2c9aa..6e7a26092d 100644 --- a/apps/ui/src/app/changelog/[slug]/page.tsx +++ b/apps/ui/src/app/changelog/[slug]/page.tsx @@ -1,3 +1,4 @@ +import { allChangelogs } from "content-collections"; import { ArrowLeftIcon } from "lucide-react"; import Markdown from "markdown-to-jsx"; import Image from "next/image"; @@ -8,8 +9,6 @@ import Footer from "@/components/landing/footer"; import { HeroRSC } from "@/components/landing/hero-rsc"; import { getMarkdownOptions } from "@/lib/utils/markdown"; -import { allChangelogs } from "content-collections"; - import type { Changelog } from "content-collections"; interface ChangelogEntryPageProps {