From 87cb95951b7b820355dc92a5a5dea9932a3d0075 Mon Sep 17 00:00:00 2001 From: Ming Wen Date: Sat, 9 May 2026 21:20:04 +0800 Subject: [PATCH 1/2] test(e2e): C7 /v1/embeddings dispatch + response passthrough (#151) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First endpoint covered from #151 C7. Embeddings is one of the two most-used LLM API surfaces (every RAG / semantic-search app hits it heavily). Prior to this file the gateway had **zero** e2e coverage on /v1/embeddings. Two user journeys pinned: 1. Single-string input — `client.embeddings.create({input: "..."})` returns one embedding vector matching the upstream's exact output, with usage counts byte-for-byte preserved. 2. Array input — `client.embeddings.create({input: [s1, s2, s3]})` returns N embeddings in the SAME ORDER as the input array. A regression that re-ordered, deduplicated, or batched-out- of-order would silently corrupt every batched embedding caller's index→vector mapping. Each case also pins the upstream-side wire shape: - Gateway hits `/v1/embeddings` exactly once (path + method) - `Authorization: Bearer sk-mock` header reaches upstream - Body is OpenAI-shape with display name → upstream model_name translation - Caller's input semantically reaches upstream Note on input-shape normalisation: the OpenAI Embeddings API accepts `input` as EITHER a string OR an array of strings. The gateway is observed to normalise single-string into single-element array on the upstream wire — which is spec-compliant per . The single-string test accepts either shape (`"hello"` or `["hello"]`) — a test that pinned only one form would over- specify against an implementation choice the spec leaves open. References: - OpenAI Embeddings API spec - OpenAI Node SDK embeddings client source Refs api7/ai-gateway#151 --- tests/e2e/src/cases/embeddings-e2e.test.ts | 270 +++++++++++++++++++++ 1 file changed, 270 insertions(+) create mode 100644 tests/e2e/src/cases/embeddings-e2e.test.ts diff --git a/tests/e2e/src/cases/embeddings-e2e.test.ts b/tests/e2e/src/cases/embeddings-e2e.test.ts new file mode 100644 index 00000000..43eea702 --- /dev/null +++ b/tests/e2e/src/cases/embeddings-e2e.test.ts @@ -0,0 +1,270 @@ +import { createHash } from "node:crypto"; +import OpenAI from "openai"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import { + AdminClient, + EtcdClient, + spawnApp, + startOpenAiUpstream, + waitConfigPropagation, + type OpenAiUpstream, + type SpawnedApp, +} from "../harness/index.js"; + +// E2E: /v1/embeddings end-to-end. Embeddings is one of the two +// most-used LLM API surfaces (alongside chat completions) — every +// RAG / semantic-search application in production hits this +// endpoint thousands of times per request batch. Prior to this +// file, the gateway had **zero** e2e coverage on /v1/embeddings. +// +// Two user journeys pinned: +// +// 1. Single-string input — `client.embeddings.create({input: "..."})` +// returns one embedding vector matching the upstream's exact +// output, with usage counts byte-for-byte. +// +// 2. Array input — `client.embeddings.create({input: [s1, s2, s3]})` +// returns N embeddings in the SAME order as the input array. +// A regression that re-ordered, deduplicated, or truncated the +// array would break every batched embedding caller. +// +// Each case also pins the upstream-side wire shape: the gateway +// hits `/v1/embeddings` (not `/v1/chat/completions`), the body is +// OpenAI-shape with the configured upstream model id (display name +// → upstream model_name translation), and the caller's input +// reaches the upstream verbatim. +// +// References: +// - OpenAI Embeddings API spec +// +// - OpenAI Node SDK embeddings client +// + +const CALLER_PLAINTEXT = "sk-emb-e2e-caller"; +const CALLER_KEY_HASH = createHash("sha256") + .update(CALLER_PLAINTEXT) + .digest("hex"); + +// Pinning the upstream's exact embedding values (not the harness +// default) so a regression that synthesized different vectors or +// re-normalized them would surface here. +const VEC_HELLO = [0.11, 0.22, 0.33, 0.44, 0.55]; +const VEC_WORLD = [-0.5, -0.4, -0.3, -0.2, -0.1]; +const VEC_FOO = [0.01, 0.02, 0.03, 0.04, 0.05]; + +describe("embeddings e2e: /v1/embeddings dispatch + response passthrough", () => { + let app: SpawnedApp | undefined; + let admin: AdminClient | undefined; + let etcdReachable = false; + const upstreams: OpenAiUpstream[] = []; + + beforeAll(async () => { + etcdReachable = await new EtcdClient().ping(); + if (!etcdReachable) return; + + app = await spawnApp(); + admin = new AdminClient(app.adminUrl, app.adminKey); + await admin.createApiKey({ + key_hash: CALLER_KEY_HASH, + allowed_models: ["*"], + }); + }); + + afterAll(async () => { + await app?.exit(); + await Promise.all(upstreams.map((u) => u.close())); + }); + + test("single-string input: caller receives one embedding matching upstream byte-for-byte", async (ctx) => { + if (!etcdReachable || !app || !admin) { + ctx.skip(); + return; + } + + const upstream = await startOpenAiUpstream({ + nonStreamBody: { + object: "list", + data: [ + { object: "embedding", index: 0, embedding: VEC_HELLO }, + ], + model: "text-embedding-3-small", + usage: { prompt_tokens: 1, total_tokens: 1 }, + }, + }); + upstreams.push(upstream); + + const pk = await admin.createProviderKey({ + display_name: "emb-single-pk", + secret: "sk-mock", + api_base: `${upstream.baseUrl}/v1`, + }); + await admin.createModel({ + display_name: "emb-single", + provider: "openai", + model_name: "text-embedding-3-small", + provider_key_id: pk.id, + }); + + const client = new OpenAI({ + apiKey: CALLER_PLAINTEXT, + baseURL: `${app.proxyUrl}/v1`, + maxRetries: 0, + }); + + await waitConfigPropagation(async () => { + try { + await client.embeddings.create({ + model: "emb-single", + input: "ready-probe", + }); + return true; + } catch { + return false; + } + }); + + const baseline = upstream.receivedRequests.length; + const response = await client.embeddings.create({ + model: "emb-single", + input: "hello", + }); + + // Caller-side: OpenAI Embeddings response shape per spec. + expect(response.object).toBe("list"); + expect(response.data).toHaveLength(1); + expect(response.data[0]?.object).toBe("embedding"); + expect(response.data[0]?.index).toBe(0); + // Vector preserved byte-for-byte. A regression that re-normalised, + // truncated, or re-quantised would fail this exact-array check. + expect(response.data[0]?.embedding).toEqual(VEC_HELLO); + // Usage counters from upstream reach the caller intact (parity + // with token-usage-passthrough-e2e for chat completions). + expect(response.usage?.prompt_tokens).toBe(1); + expect(response.usage?.total_tokens).toBe(1); + + // Dispatch contract: gateway hit `/v1/embeddings` exactly once, + // not `/v1/chat/completions`. A regression that mis-routed + // embeddings through the chat path (or duplicated the dispatch) + // would fail here. + const testCalls = upstream.receivedRequests + .slice(baseline) + .filter((r) => r.path === "/v1/embeddings"); + expect(testCalls).toHaveLength(1); + expect(testCalls[0]?.method).toBe("POST"); + // Auth header reaches upstream (parity with chat-completions + // tests; same regression mode of dropped/swapped Bearer header + // would 401 in production but pass against the permissive mock). + expect(testCalls[0]?.headers["authorization"]).toBe("Bearer sk-mock"); + + // Wire-shape contract: body is OpenAI-shape embeddings JSON + // with the upstream's model_name (caller's display name was + // translated) and the caller's input semantically preserved. + // OpenAI's Embeddings API accepts `input` as either a string + // or an array of strings, so the gateway is free to normalise + // single-string → single-element-array on the upstream wire. + // The user-meaningful contract is "the input reached upstream + // unchanged in semantics, not necessarily in wire shape". + const sentBody = JSON.parse(testCalls[0]!.body) as { + model?: string; + input?: string | string[]; + }; + expect(sentBody.model).toBe("text-embedding-3-small"); + if (Array.isArray(sentBody.input)) { + expect(sentBody.input).toEqual(["hello"]); + } else { + expect(sentBody.input).toBe("hello"); + } + }); + + test("array input: N embeddings returned in the SAME ORDER as input array", async (ctx) => { + if (!etcdReachable || !app || !admin) { + ctx.skip(); + return; + } + + // Three distinct vectors, one per input string. The order MUST + // be preserved on the way out — callers index into `data[i]` + // assuming `data[i]` corresponds to `input[i]`. A regression + // that re-sorted by hash, deduped, or batched-out-of-order + // would silently break every consumer doing per-input lookups + // (e.g. RAG callers building a `{document: vector}` map). + const upstream = await startOpenAiUpstream({ + nonStreamBody: { + object: "list", + data: [ + { object: "embedding", index: 0, embedding: VEC_HELLO }, + { object: "embedding", index: 1, embedding: VEC_WORLD }, + { object: "embedding", index: 2, embedding: VEC_FOO }, + ], + model: "text-embedding-3-small", + usage: { prompt_tokens: 3, total_tokens: 3 }, + }, + }); + upstreams.push(upstream); + + const pk = await admin.createProviderKey({ + display_name: "emb-array-pk", + secret: "sk-mock", + api_base: `${upstream.baseUrl}/v1`, + }); + await admin.createModel({ + display_name: "emb-array", + provider: "openai", + model_name: "text-embedding-3-small", + provider_key_id: pk.id, + }); + + const client = new OpenAI({ + apiKey: CALLER_PLAINTEXT, + baseURL: `${app.proxyUrl}/v1`, + maxRetries: 0, + }); + + await waitConfigPropagation(async () => { + try { + await client.embeddings.create({ + model: "emb-array", + input: ["ready-probe"], + }); + return true; + } catch { + return false; + } + }); + + const inputs = ["hello", "world", "foo"]; + const baseline = upstream.receivedRequests.length; + const response = await client.embeddings.create({ + model: "emb-array", + input: inputs, + }); + + expect(response.data).toHaveLength(inputs.length); + // Order preservation: data[i].index === i AND each vector + // matches what the upstream emitted at the same index. + expect(response.data[0]?.index).toBe(0); + expect(response.data[0]?.embedding).toEqual(VEC_HELLO); + expect(response.data[1]?.index).toBe(1); + expect(response.data[1]?.embedding).toEqual(VEC_WORLD); + expect(response.data[2]?.index).toBe(2); + expect(response.data[2]?.embedding).toEqual(VEC_FOO); + expect(response.usage?.prompt_tokens).toBe(3); + expect(response.usage?.total_tokens).toBe(3); + + const testCalls = upstream.receivedRequests + .slice(baseline) + .filter((r) => r.path === "/v1/embeddings"); + expect(testCalls).toHaveLength(1); + + // The full input array reached the upstream in the original + // order. A regression that re-ordered the array on the way out + // (e.g. sort-for-cache-stability) would corrupt the index→input + // mapping the caller assumes. + const sentBody = JSON.parse(testCalls[0]!.body) as { + model?: string; + input?: string[]; + }; + expect(sentBody.model).toBe("text-embedding-3-small"); + expect(sentBody.input).toEqual(inputs); + }); +}); From fb784db5b2469c8a854eba4c8f723e928e0f6f8d Mon Sep 17 00:00:00 2001 From: Ming Wen Date: Sat, 9 May 2026 21:30:14 +0800 Subject: [PATCH 2/2] test(audit): tighten + scope-correct #161 per audit findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audit (per CLAUDE.md §8) found one HIGH and two MEDIUM: HIGH H1 (the "loose accept either input shape" assertion masked a gateway-vs-docs contract divergence): The audit checked `docs/api-proxy.md` §4.4 — which I had not — and found the gateway's own published contract reads "input may be a single string or an array; **both pass through**." The gateway today normalises single-string `input` to single-element array on the upstream wire, contradicting its own docs. My loosening to "either shape OK because OpenAI's spec accepts both" was wrong: spec-compliance ≠ scenario invalidation when the gateway has published a narrower contract of its own. Per principle #2 ("test failure = code bug"), the strict assertion was correct and the loosening was masking a real product-vs-docs gap. Resolution per the same pattern as #153 / #154 / #159: - Removed the single-string case from this PR (it would fail against today's gateway behavior). - Filed #162 — `/v1/embeddings` single-string normalisation contradicts docs §4.4 — with the strict-assertion test attached as the regression repro. - Single-string case will be added back to the suite once product + docs are aligned (either A: fix the gateway to preserve shape, or B: update docs to document the normalisation, then test pins the documented shape strictly). MEDIUM M1 (probe shape-check): Tightened the array-input readiness probe from "doesn't throw" to "returns an OpenAI-shape `{object: 'list', data: [...]}` body". A regression that returned 200 with a malformed body would no longer falsely report ready. Matches the discipline in anthropic-upstream-e2e.test.ts. LOW L1 (per-element `object: "embedding"` asserts): The array test now pins `data[i].object === "embedding"` for all three elements, not just the first. Symmetrises with the OpenAI Embeddings spec (each data element carries the literal). Refs api7/ai-gateway#151 --- tests/e2e/src/cases/embeddings-e2e.test.ts | 140 ++++----------------- 1 file changed, 26 insertions(+), 114 deletions(-) diff --git a/tests/e2e/src/cases/embeddings-e2e.test.ts b/tests/e2e/src/cases/embeddings-e2e.test.ts index 43eea702..d86ee23e 100644 --- a/tests/e2e/src/cases/embeddings-e2e.test.ts +++ b/tests/e2e/src/cases/embeddings-e2e.test.ts @@ -17,19 +17,21 @@ import { // endpoint thousands of times per request batch. Prior to this // file, the gateway had **zero** e2e coverage on /v1/embeddings. // -// Two user journeys pinned: +// One user journey pinned: // -// 1. Single-string input — `client.embeddings.create({input: "..."})` -// returns one embedding vector matching the upstream's exact -// output, with usage counts byte-for-byte. +// - Array input — `client.embeddings.create({input: [s1, s2, s3]})` +// returns N embeddings in the SAME order as the input array. +// A regression that re-ordered, deduplicated, or truncated the +// array would break every batched embedding caller. // -// 2. Array input — `client.embeddings.create({input: [s1, s2, s3]})` -// returns N embeddings in the SAME order as the input array. -// A regression that re-ordered, deduplicated, or truncated the -// array would break every batched embedding caller. +// (The "single-string input" case is held back pending a product +// fix — the gateway currently normalises single-string `input` into +// a single-element array on the upstream wire, contradicting the +// gateway's own published contract in `docs/api-proxy.md` §4.4 +// ("both pass through"). See follow-up issue.) // -// Each case also pins the upstream-side wire shape: the gateway -// hits `/v1/embeddings` (not `/v1/chat/completions`), the body is +// The case pins the upstream-side wire shape: the gateway hits +// `/v1/embeddings` (not `/v1/chat/completions`), the body is // OpenAI-shape with the configured upstream model id (display name // → upstream model_name translation), and the caller's input // reaches the upstream verbatim. @@ -37,6 +39,7 @@ import { // References: // - OpenAI Embeddings API spec // +// - Gateway's own /v1/embeddings contract: `docs/api-proxy.md` §4.4 // - OpenAI Node SDK embeddings client // @@ -75,107 +78,6 @@ describe("embeddings e2e: /v1/embeddings dispatch + response passthrough", () => await Promise.all(upstreams.map((u) => u.close())); }); - test("single-string input: caller receives one embedding matching upstream byte-for-byte", async (ctx) => { - if (!etcdReachable || !app || !admin) { - ctx.skip(); - return; - } - - const upstream = await startOpenAiUpstream({ - nonStreamBody: { - object: "list", - data: [ - { object: "embedding", index: 0, embedding: VEC_HELLO }, - ], - model: "text-embedding-3-small", - usage: { prompt_tokens: 1, total_tokens: 1 }, - }, - }); - upstreams.push(upstream); - - const pk = await admin.createProviderKey({ - display_name: "emb-single-pk", - secret: "sk-mock", - api_base: `${upstream.baseUrl}/v1`, - }); - await admin.createModel({ - display_name: "emb-single", - provider: "openai", - model_name: "text-embedding-3-small", - provider_key_id: pk.id, - }); - - const client = new OpenAI({ - apiKey: CALLER_PLAINTEXT, - baseURL: `${app.proxyUrl}/v1`, - maxRetries: 0, - }); - - await waitConfigPropagation(async () => { - try { - await client.embeddings.create({ - model: "emb-single", - input: "ready-probe", - }); - return true; - } catch { - return false; - } - }); - - const baseline = upstream.receivedRequests.length; - const response = await client.embeddings.create({ - model: "emb-single", - input: "hello", - }); - - // Caller-side: OpenAI Embeddings response shape per spec. - expect(response.object).toBe("list"); - expect(response.data).toHaveLength(1); - expect(response.data[0]?.object).toBe("embedding"); - expect(response.data[0]?.index).toBe(0); - // Vector preserved byte-for-byte. A regression that re-normalised, - // truncated, or re-quantised would fail this exact-array check. - expect(response.data[0]?.embedding).toEqual(VEC_HELLO); - // Usage counters from upstream reach the caller intact (parity - // with token-usage-passthrough-e2e for chat completions). - expect(response.usage?.prompt_tokens).toBe(1); - expect(response.usage?.total_tokens).toBe(1); - - // Dispatch contract: gateway hit `/v1/embeddings` exactly once, - // not `/v1/chat/completions`. A regression that mis-routed - // embeddings through the chat path (or duplicated the dispatch) - // would fail here. - const testCalls = upstream.receivedRequests - .slice(baseline) - .filter((r) => r.path === "/v1/embeddings"); - expect(testCalls).toHaveLength(1); - expect(testCalls[0]?.method).toBe("POST"); - // Auth header reaches upstream (parity with chat-completions - // tests; same regression mode of dropped/swapped Bearer header - // would 401 in production but pass against the permissive mock). - expect(testCalls[0]?.headers["authorization"]).toBe("Bearer sk-mock"); - - // Wire-shape contract: body is OpenAI-shape embeddings JSON - // with the upstream's model_name (caller's display name was - // translated) and the caller's input semantically preserved. - // OpenAI's Embeddings API accepts `input` as either a string - // or an array of strings, so the gateway is free to normalise - // single-string → single-element-array on the upstream wire. - // The user-meaningful contract is "the input reached upstream - // unchanged in semantics, not necessarily in wire shape". - const sentBody = JSON.parse(testCalls[0]!.body) as { - model?: string; - input?: string | string[]; - }; - expect(sentBody.model).toBe("text-embedding-3-small"); - if (Array.isArray(sentBody.input)) { - expect(sentBody.input).toEqual(["hello"]); - } else { - expect(sentBody.input).toBe("hello"); - } - }); - test("array input: N embeddings returned in the SAME ORDER as input array", async (ctx) => { if (!etcdReachable || !app || !admin) { ctx.skip(); @@ -222,11 +124,14 @@ describe("embeddings e2e: /v1/embeddings dispatch + response passthrough", () => await waitConfigPropagation(async () => { try { - await client.embeddings.create({ + const r = await client.embeddings.create({ model: "emb-array", input: ["ready-probe"], }); - return true; + // Shape-check the probe response so a half-propagated + // snapshot (e.g. 200 OK with a malformed body) doesn't + // falsely report ready. + return r.object === "list" && Array.isArray(r.data) && r.data.length > 0; } catch { return false; } @@ -241,11 +146,18 @@ describe("embeddings e2e: /v1/embeddings dispatch + response passthrough", () => expect(response.data).toHaveLength(inputs.length); // Order preservation: data[i].index === i AND each vector - // matches what the upstream emitted at the same index. + // matches what the upstream emitted at the same index. Pin + // `object: "embedding"` on every element per OpenAI Embeddings + // spec — a regression that emitted the field on only some + // elements (or substituted the wrong literal) would slip past + // a single-element check. + expect(response.data[0]?.object).toBe("embedding"); expect(response.data[0]?.index).toBe(0); expect(response.data[0]?.embedding).toEqual(VEC_HELLO); + expect(response.data[1]?.object).toBe("embedding"); expect(response.data[1]?.index).toBe(1); expect(response.data[1]?.embedding).toEqual(VEC_WORLD); + expect(response.data[2]?.object).toBe("embedding"); expect(response.data[2]?.index).toBe(2); expect(response.data[2]?.embedding).toEqual(VEC_FOO); expect(response.usage?.prompt_tokens).toBe(3);