From 84bbd8ed89a3ce604a614722fb83296c5fb168bb Mon Sep 17 00:00:00 2001 From: Ming Wen Date: Sat, 9 May 2026 22:17:27 +0800 Subject: [PATCH] test(e2e): C7 /v1/rerank + /v1/images/generations forward + model translation (#151) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more endpoints from #151's C7 row, both following the same "verbatim-forward with model rewrite" pattern per docs: - `/v1/rerank` (docs §4.7): "Cohere-style rerank. Routed to `{base}/v1/rerank`. The Model's provider supplies the API key; the request body is forwarded verbatim after rewriting the `model` field." - `/v1/images/generations` (docs §4.9): "OpenAI Images API. Forwarded with the `model` field rewritten." Prior to this PR the gateway had **zero** e2e coverage on either endpoint. Both are real production surfaces: - Rerank is the standard relevance-scoring step in modern RAG pipelines. A regression that mangled scores or reordered results would silently corrupt RAG ranking quality across every customer using rerank. - Image generation (DALL-E / GPT-Image) is increasingly common in agent-style workflows. A regression that dropped the image url or revised_prompt fields would leave callers with no signal about what was generated. Two cases pinned, one per endpoint: Rerank: caller POSTs Cohere-shape `{model, query, documents, top_n}` to /v1/rerank. Asserts: - Caller-side response: `id`, results array (3 items), each item's `index` + `relevance_score` byte-for-byte from upstream, `meta` present - Upstream-side: hit `/v1/rerank` (NOT chat-completions), `Bearer sk-mock` auth, `model` rewritten to upstream model_name, query + documents + top_n forwarded verbatim Images: caller POSTs OpenAI-shape `{model, prompt, n, size, response_format}` to /v1/images/generations. Asserts: - Caller-side response: `created` numeric, `data[0].url` and `data[0].revised_prompt` byte-for-byte from upstream - Upstream-side: hit `/v1/images/generations` (NOT another route), `Bearer sk-mock` auth, `model` rewritten, prompt + n + size + response_format forwarded verbatim References: - Gateway's own /v1/rerank contract: `docs/api-proxy.md` §4.7 - Gateway's own /v1/images/generations contract: §4.9 - Cohere Rerank API spec: - OpenAI Images API spec: Refs api7/ai-gateway#151 --- .../src/cases/images-generations-e2e.test.ts | 176 ++++++++++++++++ tests/e2e/src/cases/rerank-e2e.test.ts | 194 ++++++++++++++++++ 2 files changed, 370 insertions(+) create mode 100644 tests/e2e/src/cases/images-generations-e2e.test.ts create mode 100644 tests/e2e/src/cases/rerank-e2e.test.ts diff --git a/tests/e2e/src/cases/images-generations-e2e.test.ts b/tests/e2e/src/cases/images-generations-e2e.test.ts new file mode 100644 index 00000000..19a3cbb4 --- /dev/null +++ b/tests/e2e/src/cases/images-generations-e2e.test.ts @@ -0,0 +1,176 @@ +import { createHash } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import { + AdminClient, + EtcdClient, + spawnApp, + startOpenAiUpstream, + waitConfigPropagation, + type OpenAiUpstream, + type SpawnedApp, +} from "../harness/index.js"; + +// E2E: /v1/images/generations end-to-end. Per gateway docs +// `docs/api-proxy.md` §4.9: +// +// > OpenAI Images API. Forwarded with the `model` field rewritten. +// +// Prior to this file, the gateway had **zero** e2e coverage on +// /v1/images/generations. +// +// One user journey pinned: +// +// - Caller POSTs OpenAI-shape image-generation request to +// /v1/images/generations. Gateway forwards to upstream's +// /v1/images/generations with only the `model` field rewritten. +// Caller receives upstream's response back unchanged. +// +// References: +// - Gateway's own /v1/images/generations contract: +// `docs/api-proxy.md` §4.9 +// - OpenAI Images API spec: +// + +const CALLER_PLAINTEXT = "sk-img-e2e-caller"; +const CALLER_KEY_HASH = createHash("sha256") + .update(CALLER_PLAINTEXT) + .digest("hex"); + +describe("images generations e2e: /v1/images/generations verbatim forward + model translation", () => { + let app: SpawnedApp | undefined; + let upstream: OpenAiUpstream | undefined; + let admin: AdminClient | undefined; + let etcdReachable = false; + + beforeAll(async () => { + etcdReachable = await new EtcdClient().ping(); + if (!etcdReachable) return; + + upstream = await startOpenAiUpstream({ + nonStreamBody: { + // OpenAI Images API response shape per + // . + created: Math.floor(Date.now() / 1000), + data: [ + { + url: "https://mock.example.com/img-1.png", + revised_prompt: "A cat sitting in a sunbeam (refined).", + }, + ], + }, + }); + app = await spawnApp(); + admin = new AdminClient(app.adminUrl, app.adminKey); + + const pk = await admin.createProviderKey({ + display_name: "img-pk", + secret: "sk-mock", + api_base: `${upstream.baseUrl}/v1`, + }); + await admin.createModel({ + display_name: "img-e2e", + provider: "openai", + model_name: "gpt-image-1", + provider_key_id: pk.id, + }); + await admin.createApiKey({ + key_hash: CALLER_KEY_HASH, + allowed_models: ["img-e2e"], + }); + }); + + afterAll(async () => { + await app?.exit(); + await upstream?.close(); + }); + + test("OpenAI-shape images.generations: caller body verbatim + model translated, response byte-for-byte", async (ctx) => { + if (!etcdReachable || !app || !upstream) { + ctx.skip(); + return; + } + + const headers = { + authorization: `Bearer ${CALLER_PLAINTEXT}`, + "content-type": "application/json", + }; + + await waitConfigPropagation(async () => { + try { + const r = await fetch(`${app!.proxyUrl}/v1/images/generations`, { + method: "POST", + headers, + body: JSON.stringify({ + model: "img-e2e", + prompt: "ready-probe", + }), + }); + if (r.status !== 200) { + await r.text(); + return false; + } + const j = (await r.json()) as { data?: unknown }; + return Array.isArray(j.data) && (j.data as unknown[]).length > 0; + } catch { + return false; + } + }); + + const baseline = upstream.receivedRequests.length; + const requestPayload = { + model: "img-e2e", + prompt: "A cat sitting in a sunbeam", + n: 1, + size: "1024x1024", + response_format: "url", + }; + const res = await fetch(`${app.proxyUrl}/v1/images/generations`, { + method: "POST", + headers, + body: JSON.stringify(requestPayload), + }); + + expect(res.status).toBe(200); + const body = (await res.json()) as { + created?: unknown; + data?: Array<{ url?: unknown; revised_prompt?: unknown }>; + }; + // Caller-side: response byte-for-byte from upstream. The image + // url and any revised_prompt the upstream produced must reach + // the caller intact — they're the only signal the caller has + // about what was generated. + expect(typeof body.created).toBe("number"); + expect(body.data).toHaveLength(1); + expect(body.data?.[0]?.url).toBe("https://mock.example.com/img-1.png"); + expect(body.data?.[0]?.revised_prompt).toBe( + "A cat sitting in a sunbeam (refined).", + ); + + // Dispatch contract: gateway hit `/v1/images/generations` (not + // /v1/chat/completions or any other route). + const testCalls = upstream.receivedRequests + .slice(baseline) + .filter((r) => r.path === "/v1/images/generations"); + expect(testCalls).toHaveLength(1); + expect(testCalls[0]?.method).toBe("POST"); + expect(testCalls[0]?.headers["authorization"]).toBe("Bearer sk-mock"); + + // Body contract per docs §4.9: forwarded verbatim with the + // `model` field rewritten. Verify: + // - `model` rewritten to upstream model_name + // - everything else byte-for-byte (prompt, n, size, + // response_format) + const sentBody = JSON.parse(testCalls[0]!.body) as { + model?: string; + prompt?: string; + n?: number; + size?: string; + response_format?: string; + }; + expect(sentBody.model).toBe("gpt-image-1"); + expect(sentBody.prompt).toBe(requestPayload.prompt); + expect(sentBody.n).toBe(requestPayload.n); + expect(sentBody.size).toBe(requestPayload.size); + expect(sentBody.response_format).toBe(requestPayload.response_format); + }); +}); diff --git a/tests/e2e/src/cases/rerank-e2e.test.ts b/tests/e2e/src/cases/rerank-e2e.test.ts new file mode 100644 index 00000000..2c453a88 --- /dev/null +++ b/tests/e2e/src/cases/rerank-e2e.test.ts @@ -0,0 +1,194 @@ +import { createHash } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import { + AdminClient, + EtcdClient, + spawnApp, + startOpenAiUpstream, + waitConfigPropagation, + type OpenAiUpstream, + type SpawnedApp, +} from "../harness/index.js"; + +// E2E: /v1/rerank end-to-end. Per gateway docs `docs/api-proxy.md` +// §4.7: +// +// > Cohere-style rerank. Routed to `{base}/v1/rerank`. The Model's +// > provider supplies the API key; the request body is forwarded +// > verbatim after rewriting the `model` field. +// +// Prior to this file, the gateway had **zero** e2e coverage on +// /v1/rerank. Rerank is the standard relevance-scoring step in +// modern RAG pipelines: a retriever returns N candidate documents, +// rerank scores them against the query, the top-K go to the LLM. +// Without e2e coverage, regressions on this path would silently +// corrupt RAG quality. +// +// One user journey pinned: +// +// - Caller POSTs Cohere-shape rerank request to /v1/rerank. +// Gateway forwards verbatim to upstream's /v1/rerank with only +// the `model` field rewritten to the upstream model_name. +// Caller receives upstream's response back unchanged. +// +// References: +// - Gateway's own /v1/rerank contract: `docs/api-proxy.md` §4.7 +// - Cohere Rerank API spec: + +const CALLER_PLAINTEXT = "sk-rerank-e2e-caller"; +const CALLER_KEY_HASH = createHash("sha256") + .update(CALLER_PLAINTEXT) + .digest("hex"); + +describe("rerank e2e: /v1/rerank verbatim forward + model translation", () => { + let app: SpawnedApp | undefined; + let upstream: OpenAiUpstream | undefined; + let admin: AdminClient | undefined; + let etcdReachable = false; + + beforeAll(async () => { + etcdReachable = await new EtcdClient().ping(); + if (!etcdReachable) return; + + // Mock upstream returns a canned Cohere-shape rerank response + // so a regression that synthesised different scores or shuffled + // results would surface here. + upstream = await startOpenAiUpstream({ + nonStreamBody: { + id: "rerank-resp-01", + results: [ + { index: 2, relevance_score: 0.92 }, + { index: 0, relevance_score: 0.78 }, + { index: 1, relevance_score: 0.31 }, + ], + meta: { + api_version: { version: "1" }, + billed_units: { search_units: 1 }, + }, + }, + }); + app = await spawnApp(); + admin = new AdminClient(app.adminUrl, app.adminKey); + + const pk = await admin.createProviderKey({ + display_name: "rerank-pk", + secret: "sk-mock", + api_base: `${upstream.baseUrl}/v1`, + }); + await admin.createModel({ + display_name: "rerank-e2e", + provider: "openai", + model_name: "rerank-english-v3.0", + provider_key_id: pk.id, + }); + await admin.createApiKey({ + key_hash: CALLER_KEY_HASH, + allowed_models: ["rerank-e2e"], + }); + }); + + afterAll(async () => { + await app?.exit(); + await upstream?.close(); + }); + + test("Cohere-shape rerank: caller's body verbatim + model translated, response byte-for-byte", async (ctx) => { + if (!etcdReachable || !app || !upstream) { + ctx.skip(); + return; + } + + const headers = { + authorization: `Bearer ${CALLER_PLAINTEXT}`, + "content-type": "application/json", + }; + + // Readiness gate: poll /v1/rerank until 200 with the canned + // body. A 200 with a different shape would be the gateway + // interfering (which §4.7 says it must not — "forwarded + // verbatim"). + await waitConfigPropagation(async () => { + try { + const r = await fetch(`${app!.proxyUrl}/v1/rerank`, { + method: "POST", + headers, + body: JSON.stringify({ + model: "rerank-e2e", + query: "ready-probe", + documents: ["doc"], + }), + }); + if (r.status !== 200) { + await r.text(); + return false; + } + const j = (await r.json()) as { id?: unknown }; + return j.id === "rerank-resp-01"; + } catch { + return false; + } + }); + + const baseline = upstream.receivedRequests.length; + const requestPayload = { + model: "rerank-e2e", + query: "What is the capital of France?", + documents: [ + "Berlin is the capital of Germany.", + "London is the capital of the United Kingdom.", + "Paris is the capital of France.", + ], + top_n: 3, + }; + const res = await fetch(`${app.proxyUrl}/v1/rerank`, { + method: "POST", + headers, + body: JSON.stringify(requestPayload), + }); + + expect(res.status).toBe(200); + const body = (await res.json()) as { + id?: unknown; + results?: Array<{ index?: unknown; relevance_score?: unknown }>; + meta?: unknown; + }; + // Caller-side: response byte-for-byte from upstream. Per docs + // §4.7 the gateway is a pass-through for the response body; + // any normalisation here would silently change relevance + // scores and break RAG ranking. + expect(body.id).toBe("rerank-resp-01"); + expect(body.results).toHaveLength(3); + expect(body.results?.[0]?.index).toBe(2); + expect(body.results?.[0]?.relevance_score).toBe(0.92); + expect(body.results?.[1]?.index).toBe(0); + expect(body.results?.[1]?.relevance_score).toBe(0.78); + expect(body.results?.[2]?.index).toBe(1); + expect(body.results?.[2]?.relevance_score).toBe(0.31); + expect(body.meta).toBeDefined(); + + // Dispatch contract: gateway hit `/v1/rerank` exactly once, + // not /v1/chat/completions or /v1/embeddings. A regression + // that mis-routed would change the path here. + const testCalls = upstream.receivedRequests + .slice(baseline) + .filter((r) => r.path === "/v1/rerank"); + expect(testCalls).toHaveLength(1); + expect(testCalls[0]?.method).toBe("POST"); + expect(testCalls[0]?.headers["authorization"]).toBe("Bearer sk-mock"); + + // Body contract per docs §4.7: forwarded verbatim after + // rewriting the `model` field. Verify: + // - `model` rewritten to upstream model_name + // - everything else byte-for-byte (query, documents, top_n) + const sentBody = JSON.parse(testCalls[0]!.body) as { + model?: string; + query?: string; + documents?: string[]; + top_n?: number; + }; + expect(sentBody.model).toBe("rerank-english-v3.0"); + expect(sentBody.query).toBe(requestPayload.query); + expect(sentBody.documents).toEqual(requestPayload.documents); + expect(sentBody.top_n).toBe(requestPayload.top_n); + }); +});