diff --git a/tests/e2e/src/cases/seed-vs-admin-characterization-e2e.test.ts b/tests/e2e/src/cases/seed-vs-admin-characterization-e2e.test.ts new file mode 100644 index 00000000..6a6ffb04 --- /dev/null +++ b/tests/e2e/src/cases/seed-vs-admin-characterization-e2e.test.ts @@ -0,0 +1,268 @@ +import { createHash } from "node:crypto"; +import OpenAI, { APIError } from "openai"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import { + AdminClient, + EtcdClient, + SeedClient, + spawnApp, + startOpenAiUpstream, + waitConfigPropagation, + type OpenAiUpstream, + type SpawnedApp, +} from "../harness/index.js"; + +// Characterization: a resource document seeded straight to etcd +// (`SeedClient`) must be indistinguishable from the same resource +// created through the Admin API — both after the store's serde +// round-trip (read back via admin GET) and behaviorally on the proxy +// (chat succeeds through seeded resources; allowed_models authz reads +// a seeded caller key exactly like an admin-created one). +// +// This pins the contract the e2e seed migration relies on: the Admin +// API's create handlers add nothing to the stored document beyond the +// id in the key, so tests may write canonical documents directly — +// the same front door the control plane uses in managed mode. +// Transitional: this pin de-risks migrating case seeding to +// direct document writes; once that migration completes it can be +// folded into the general suite or retired. + +const ADMIN_CALLER = "sk-char-admin-caller"; +const SEED_CALLER = "sk-char-seed-caller"; +const sha256 = (s: string) => createHash("sha256").update(s).digest("hex"); + +/** Strip the fields a pair intentionally differs by (identity and + * cross-references), leaving defaults and everything the handler might + * have added — which is exactly what the comparison is about. */ +function normalize( + value: Record, + varied: string[], +): Record { + const copy: Record = { ...value }; + for (const f of varied) delete copy[f]; + return copy; +} + +type Entry = { id: string; value: Record }; + +describe("seed-vs-admin characterization: direct etcd writes ≡ Admin API writes", () => { + let app: SpawnedApp | undefined; + let upstream: OpenAiUpstream | undefined; + let admin: AdminClient | undefined; + let seed: SeedClient | undefined; + let etcdClient: EtcdClient | undefined; + let etcdReachable = false; + + let adminPk: Entry, seedPk: Entry; + let adminModel: Entry, seedModel: Entry; + let adminKey: Entry, seedKey: Entry; + let adminExporter: Entry, seedExporter: Entry; + + beforeAll(async () => { + const etcd = new EtcdClient(); + etcdClient = etcd; + etcdReachable = await etcd.ping(); + if (!etcdReachable) return; + + upstream = await startOpenAiUpstream(); + app = await spawnApp(); + admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); + + // One resource of each kind through each front door. Bodies are + // identical except identity fields (display_name/name) and + // cross-references (provider_key_id, allowed_models, key_hash). + adminPk = await admin.createProviderKey({ + display_name: "char-admin-pk", + secret: "sk-mock", + api_base: `${upstream.baseUrl}/v1`, + }); + seedPk = await seed.createProviderKey({ + display_name: "char-seed-pk", + secret: "sk-mock", + api_base: `${upstream.baseUrl}/v1`, + }); + + adminModel = await admin.createModel({ + display_name: "char-admin-model", + provider: "openai", + model_name: "gpt-4o-mini", + provider_key_id: adminPk.id, + }); + seedModel = await seed.createModel({ + display_name: "char-seed-model", + provider: "openai", + model_name: "gpt-4o-mini", + provider_key_id: seedPk.id, + }); + + // Carry optional fields too so the projection comparison below has + // non-empty residue after the varied fields are stripped. + adminKey = await admin.createApiKey({ + key_hash: sha256(ADMIN_CALLER), + allowed_models: ["char-admin-model"], + rate_limit: { rpm: 1000 }, + expires_at: "2030-01-01T00:00:00Z", + }); + seedKey = await seed.createApiKey({ + key_hash: sha256(SEED_CALLER), + allowed_models: ["char-seed-model"], + rate_limit: { rpm: 1000 }, + expires_at: "2030-01-01T00:00:00Z", + }); + + // Exporters only need to load — point at a closed port, disabled. + adminExporter = await admin.createObservabilityExporter({ + name: "char-admin-exporter", + kind: "otlp_http", + endpoint: "http://127.0.0.1:9/otlp", + enabled: false, + }); + seedExporter = await seed.createObservabilityExporter({ + name: "char-seed-exporter", + kind: "otlp_http", + endpoint: "http://127.0.0.1:9/otlp", + enabled: false, + }); + }); + + afterAll(async () => { + await app?.exit(); + await upstream?.close(); + }); + + test("seeded resources serve traffic exactly like admin-created ones", async (ctx) => { + if (!etcdReachable || !app || !upstream) { + ctx.skip(); + return; + } + + const seedClient = new OpenAI({ + apiKey: SEED_CALLER, + baseURL: `${app.proxyUrl}/v1`, + maxRetries: 0, + }); + const adminClient = new OpenAI({ + apiKey: ADMIN_CALLER, + baseURL: `${app.proxyUrl}/v1`, + maxRetries: 0, + }); + + // Positive probes double as propagation gates: a 200 through each + // caller proves its ProviderKey + Model + ApiKey all propagated, + // whichever front door wrote them. + await waitConfigPropagation(async () => { + try { + const [a, s] = await Promise.all([ + adminClient.chat.completions.create({ + model: "char-admin-model", + messages: [{ role: "user", content: "ready-probe" }], + }), + seedClient.chat.completions.create({ + model: "char-seed-model", + messages: [{ role: "user", content: "ready-probe" }], + }), + ]); + return ( + a.choices[0]?.message.role === "assistant" && + s.choices[0]?.message.role === "assistant" + ); + } catch { + return false; + } + }); + + // authz reads the seeded caller key exactly like an admin-created + // one: the seeded key is NOT allowed the admin model → 403. + await expect( + seedClient.chat.completions.create({ + model: "char-admin-model", + messages: [{ role: "user", content: "must-403" }], + }), + ).rejects.toSatisfy((err: unknown) => err instanceof APIError && err.status === 403); + }); + + test("seeded documents read back identical to admin-created ones after the store round-trip", async (ctx) => { + if (!etcdReachable || !admin || !app || !etcdClient) { + ctx.skip(); + return; + } + + // Admin GET deserializes every etcd value through the same serde + // models the loader uses, then re-serializes — a serde-malformed + // seeded document would be skipped by the store and fail the entry + // lookup below. (The proxy loader additionally applies JSON-Schema + // validation, which this lens does not; loader acceptance of the + // traffic-bearing kinds is covered by the 200-probes in the + // behavioral test above. api_keys read back through a public + // projection that drops attribution fields, hence the raw-bytes + // check below.) + const find = (entries: Entry[], id: string, label: string): Entry => { + const e = entries.find((x) => x.id === id); + if (!e) throw new Error(`${label} (${id}) missing from admin GET — seeded document rejected by the store's serde round-trip?`); + return e; + }; + + const pks = await admin.json("GET", "/admin/v1/provider_keys"); + expect( + normalize(find(pks, seedPk.id, "seed provider_key").value, ["display_name"]), + ).toEqual( + normalize(find(pks, adminPk.id, "admin provider_key").value, ["display_name"]), + ); + + const models = await admin.json("GET", "/admin/v1/models"); + // The identity fields themselves must round-trip byte-exact — a + // handler-side canonicalization (trim/case) would otherwise hide + // behind normalize(). + expect(find(models, seedModel.id, "seed model").value.display_name).toBe( + "char-seed-model", + ); + expect(find(models, adminModel.id, "admin model").value.display_name).toBe( + "char-admin-model", + ); + expect( + normalize(find(models, seedModel.id, "seed model").value, [ + "display_name", + "provider_key_id", + ]), + ).toEqual( + normalize(find(models, adminModel.id, "admin model").value, [ + "display_name", + "provider_key_id", + ]), + ); + + const keys = await admin.json("GET", "/admin/v1/apikeys"); + expect( + normalize(find(keys, seedKey.id, "seed api_key").value, [ + "key_hash", + "allowed_models", + ]), + ).toEqual( + normalize(find(keys, adminKey.id, "admin api_key").value, [ + "key_hash", + "allowed_models", + ]), + ); + + // The apikeys GET is a public projection that omits attribution + // fields — pin the raw stored bytes too, so handler-side enrichment + // of the stored document cannot hide behind the projection. + const rawSeedKey = JSON.parse( + (await etcdClient.get(`${app.etcdPrefix}/api_keys/${seedKey.id}`))!, + ) as Record; + const rawAdminKey = JSON.parse( + (await etcdClient.get(`${app.etcdPrefix}/api_keys/${adminKey.id}`))!, + ) as Record; + expect(normalize(rawSeedKey, ["key_hash", "allowed_models"])).toEqual( + normalize(rawAdminKey, ["key_hash", "allowed_models"]), + ); + + const exporters = await admin.json("GET", "/admin/v1/observability_exporters"); + expect( + normalize(find(exporters, seedExporter.id, "seed exporter").value, ["name"]), + ).toEqual( + normalize(find(exporters, adminExporter.id, "admin exporter").value, ["name"]), + ); + }); +}); diff --git a/tests/e2e/src/harness/etcd.ts b/tests/e2e/src/harness/etcd.ts index c16ef76b..b2af856c 100644 --- a/tests/e2e/src/harness/etcd.ts +++ b/tests/e2e/src/harness/etcd.ts @@ -65,6 +65,24 @@ export class EtcdClient { } } + /** Read one key's value (etcd v3 `/v3/kv/range`); undefined when absent. */ + async get(key: string): Promise { + const res = await harnessRequest(`${this.endpoint}/v3/kv/range`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ key: Buffer.from(key, "utf8").toString("base64") }), + }); + if (res.statusCode >= 300) { + const body = await res.body.text(); + throw new Error(`etcd range failed (${res.statusCode}): ${body}`); + } + const parsed = JSON.parse(await res.body.text()) as { + kvs?: Array<{ value: string }>; + }; + const v = parsed.kvs?.[0]?.value; + return v === undefined ? undefined : Buffer.from(v, "base64").toString("utf8"); + } + /** Delete every key under `prefix` (range delete in etcd v3 semantics). */ async deletePrefix(prefix: string): Promise { const key = Buffer.from(prefix, "utf8").toString("base64"); diff --git a/tests/e2e/src/harness/index.ts b/tests/e2e/src/harness/index.ts index fd3104bb..16303b30 100644 --- a/tests/e2e/src/harness/index.ts +++ b/tests/e2e/src/harness/index.ts @@ -2,6 +2,7 @@ export { spawnApp, type SpawnedApp, type AppOverrides } from "./app.js"; export { AdminClient, waitConfigPropagation } from "./admin.js"; export { ProxyClient } from "./proxy.js"; export { EtcdClient } from "./etcd.js"; +export { SeedClient } from "./seed.js"; export { startOpenAiUpstream, type OpenAiUpstream, type ReceivedRequest } from "./upstream-openai.js"; export { pickFreePort, pickFreePorts } from "./ports.js"; export { diff --git a/tests/e2e/src/harness/seed.ts b/tests/e2e/src/harness/seed.ts new file mode 100644 index 00000000..3983b916 --- /dev/null +++ b/tests/e2e/src/harness/seed.ts @@ -0,0 +1,68 @@ +import { randomUUID } from "node:crypto"; + +import { EtcdClient } from "./etcd.js"; + +/** + * Seeds resources by writing canonical resource documents straight to + * etcd — the same front door the control plane uses in managed mode, + * where the Admin API is not in the write path. The + * interface mirrors `AdminClient`'s create methods (same body shapes, + * same `{id, value}` return with a generated id), so call sites migrate + * mechanically: `admin.createModel({...})` → `seed.createModel({...})`. + * + * The document written is exactly the caller-supplied body — the + * canonical resource shape from `schemas/resources/`. The loader fills + * serde defaults on load, so a sparse document loads identically to an + * Admin-API-written one; that equivalence is pinned by + * `cases/seed-vs-admin-characterization-e2e.test.ts`. + * + * Unlike the Admin API there is no synchronous validation: a malformed + * document is silently skipped by the loader and the test then times + * out in `waitConfigPropagation`. Keep seed bodies aligned with the + * schemas, and probe propagation with a positive condition. + */ +export class SeedClient { + constructor( + private readonly etcd: EtcdClient, + private readonly prefix: string, + ) {} + + async createModel( + model: Record, + ): Promise<{ id: string; value: Record }> { + return this.put("models", model); + } + + async createApiKey( + key: Record, + ): Promise<{ id: string; value: Record }> { + return this.put("api_keys", key); + } + + async createProviderKey( + pk: Record, + ): Promise<{ id: string; value: Record }> { + // Same defaulting as AdminClient.createProviderKey: cp-api always + // writes `provider` + `adapter`, so the seeded document carries the + // OpenAI-compatible pair unless a test overrides them. + return this.put("provider_keys", { provider: "openai", adapter: "openai", ...pk }); + } + + async createObservabilityExporter( + exporter: Record, + ): Promise<{ id: string; value: Record }> { + return this.put("observability_exporters", exporter); + } + + private async put( + kind: string, + value: Record, + ): Promise<{ id: string; value: Record }> { + // The Admin API generates a UUID server-side; here the harness is + // the writer, so it generates one — the id lives in the key + // (`//`), not in the document. + const id = randomUUID(); + await this.etcd.put(`${this.prefix}/${kind}/${id}`, JSON.stringify(value)); + return { id, value }; + } +}