test(e2e): add SeedClient — seed resources by writing canonical documents to etcd

tests/e2e/src/cases/seed-vs-admin-characterization-e2e.test.ts

@@ -0,0 +1,268 @@
+import { createHash } from "node:crypto";
+import OpenAI, { APIError } from "openai";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import {
+  AdminClient,
+  EtcdClient,
+  SeedClient,
+  spawnApp,
+  startOpenAiUpstream,
+  waitConfigPropagation,
+  type OpenAiUpstream,
+  type SpawnedApp,
+} from "../harness/index.js";
+
+// Characterization: a resource document seeded straight to etcd
+// (`SeedClient`) must be indistinguishable from the same resource
+// created through the Admin API — both after the store's serde
+// round-trip (read back via admin GET) and behaviorally on the proxy
+// (chat succeeds through seeded resources; allowed_models authz reads
+// a seeded caller key exactly like an admin-created one).
+//
+// This pins the contract the e2e seed migration relies on: the Admin
+// API's create handlers add nothing to the stored document beyond the
+// id in the key, so tests may write canonical documents directly —
+// the same front door the control plane uses in managed mode.
+// Transitional: this pin de-risks migrating case seeding to
+// direct document writes; once that migration completes it can be
+// folded into the general suite or retired.
+
+const ADMIN_CALLER = "sk-char-admin-caller";
+const SEED_CALLER = "sk-char-seed-caller";
+const sha256 = (s: string) => createHash("sha256").update(s).digest("hex");
+
+/** Strip the fields a pair intentionally differs by (identity and
+ * cross-references), leaving defaults and everything the handler might
+ * have added — which is exactly what the comparison is about. */
+function normalize(
+  value: Record<string, unknown>,
+  varied: string[],
+): Record<string, unknown> {
+  const copy: Record<string, unknown> = { ...value };
+  for (const f of varied) delete copy[f];
+  return copy;
+}
+
+type Entry = { id: string; value: Record<string, unknown> };
+
+describe("seed-vs-admin characterization: direct etcd writes ≡ Admin API writes", () => {
+  let app: SpawnedApp | undefined;
+  let upstream: OpenAiUpstream | undefined;
+  let admin: AdminClient | undefined;
+  let seed: SeedClient | undefined;
+  let etcdClient: EtcdClient | undefined;
+  let etcdReachable = false;
+
+  let adminPk: Entry, seedPk: Entry;
+  let adminModel: Entry, seedModel: Entry;
+  let adminKey: Entry, seedKey: Entry;
+  let adminExporter: Entry, seedExporter: Entry;
+
+  beforeAll(async () => {
+    const etcd = new EtcdClient();
+    etcdClient = etcd;
+    etcdReachable = await etcd.ping();
+    if (!etcdReachable) return;
+
+    upstream = await startOpenAiUpstream();
+    app = await spawnApp();
+    admin = new AdminClient(app.adminUrl, app.adminKey);
+    seed = new SeedClient(etcd, app.etcdPrefix);
+
+    // One resource of each kind through each front door. Bodies are
+    // identical except identity fields (display_name/name) and
+    // cross-references (provider_key_id, allowed_models, key_hash).
+    adminPk = await admin.createProviderKey({
+      display_name: "char-admin-pk",
+      secret: "sk-mock",
+      api_base: `${upstream.baseUrl}/v1`,
+    });
+    seedPk = await seed.createProviderKey({
+      display_name: "char-seed-pk",
+      secret: "sk-mock",
+      api_base: `${upstream.baseUrl}/v1`,
+    });
+
+    adminModel = await admin.createModel({
+      display_name: "char-admin-model",
+      provider: "openai",
+      model_name: "gpt-4o-mini",
+      provider_key_id: adminPk.id,
+    });
+    seedModel = await seed.createModel({
+      display_name: "char-seed-model",
+      provider: "openai",
+      model_name: "gpt-4o-mini",
+      provider_key_id: seedPk.id,
+    });
+
+    // Carry optional fields too so the projection comparison below has
+    // non-empty residue after the varied fields are stripped.
+    adminKey = await admin.createApiKey({
+      key_hash: sha256(ADMIN_CALLER),
+      allowed_models: ["char-admin-model"],
+      rate_limit: { rpm: 1000 },
+      expires_at: "2030-01-01T00:00:00Z",
+    });
+    seedKey = await seed.createApiKey({
+      key_hash: sha256(SEED_CALLER),
+      allowed_models: ["char-seed-model"],
+      rate_limit: { rpm: 1000 },
+      expires_at: "2030-01-01T00:00:00Z",
+    });
+
+    // Exporters only need to load — point at a closed port, disabled.
+    adminExporter = await admin.createObservabilityExporter({
+      name: "char-admin-exporter",
+      kind: "otlp_http",
+      endpoint: "http://127.0.0.1:9/otlp",
+      enabled: false,
+    });
+    seedExporter = await seed.createObservabilityExporter({
+      name: "char-seed-exporter",
+      kind: "otlp_http",
+      endpoint: "http://127.0.0.1:9/otlp",
+      enabled: false,
+    });
+  });
+
+  afterAll(async () => {
+    await app?.exit();
+    await upstream?.close();
+  });
+
+  test("seeded resources serve traffic exactly like admin-created ones", async (ctx) => {
+    if (!etcdReachable || !app || !upstream) {
+      ctx.skip();
+      return;
+    }
+
+    const seedClient = new OpenAI({
+      apiKey: SEED_CALLER,
+      baseURL: `${app.proxyUrl}/v1`,
+      maxRetries: 0,
+    });
+    const adminClient = new OpenAI({
+      apiKey: ADMIN_CALLER,
+      baseURL: `${app.proxyUrl}/v1`,
+      maxRetries: 0,
+    });
+
+    // Positive probes double as propagation gates: a 200 through each
+    // caller proves its ProviderKey + Model + ApiKey all propagated,
+    // whichever front door wrote them.
+    await waitConfigPropagation(async () => {
+      try {
+        const [a, s] = await Promise.all([
+          adminClient.chat.completions.create({
+            model: "char-admin-model",
+            messages: [{ role: "user", content: "ready-probe" }],
+          }),
+          seedClient.chat.completions.create({
+            model: "char-seed-model",
+            messages: [{ role: "user", content: "ready-probe" }],
+          }),
+        ]);
+        return (
+          a.choices[0]?.message.role === "assistant" &&
+          s.choices[0]?.message.role === "assistant"
+        );
+      } catch {
+        return false;
+      }
+    });
+
+    // authz reads the seeded caller key exactly like an admin-created
+    // one: the seeded key is NOT allowed the admin model → 403.
+    await expect(
+      seedClient.chat.completions.create({
+        model: "char-admin-model",
+        messages: [{ role: "user", content: "must-403" }],
+      }),
+    ).rejects.toSatisfy((err: unknown) => err instanceof APIError && err.status === 403);
+  });
+
+  test("seeded documents read back identical to admin-created ones after the store round-trip", async (ctx) => {
+    if (!etcdReachable || !admin || !app || !etcdClient) {
+      ctx.skip();
+      return;
+    }
+
+    // Admin GET deserializes every etcd value through the same serde
+    // models the loader uses, then re-serializes — a serde-malformed
+    // seeded document would be skipped by the store and fail the entry
+    // lookup below. (The proxy loader additionally applies JSON-Schema
+    // validation, which this lens does not; loader acceptance of the
+    // traffic-bearing kinds is covered by the 200-probes in the
+    // behavioral test above. api_keys read back through a public
+    // projection that drops attribution fields, hence the raw-bytes
+    // check below.)
+    const find = (entries: Entry[], id: string, label: string): Entry => {
+      const e = entries.find((x) => x.id === id);
+      if (!e) throw new Error(`${label} (${id}) missing from admin GET — seeded document rejected by the store's serde round-trip?`);
+      return e;
+    };
+
+    const pks = await admin.json<Entry[]>("GET", "/admin/v1/provider_keys");
+    expect(
+      normalize(find(pks, seedPk.id, "seed provider_key").value, ["display_name"]),
+    ).toEqual(
+      normalize(find(pks, adminPk.id, "admin provider_key").value, ["display_name"]),
+    );
+
+    const models = await admin.json<Entry[]>("GET", "/admin/v1/models");
+    // The identity fields themselves must round-trip byte-exact — a
+    // handler-side canonicalization (trim/case) would otherwise hide
+    // behind normalize().
+    expect(find(models, seedModel.id, "seed model").value.display_name).toBe(
+      "char-seed-model",
+    );
+    expect(find(models, adminModel.id, "admin model").value.display_name).toBe(
+      "char-admin-model",
+    );
+    expect(
+      normalize(find(models, seedModel.id, "seed model").value, [
+        "display_name",
+        "provider_key_id",
+      ]),
+    ).toEqual(
+      normalize(find(models, adminModel.id, "admin model").value, [
+        "display_name",
+        "provider_key_id",
+      ]),
+    );
+
+    const keys = await admin.json<Entry[]>("GET", "/admin/v1/apikeys");
+    expect(
+      normalize(find(keys, seedKey.id, "seed api_key").value, [
+        "key_hash",
+        "allowed_models",
+      ]),
+    ).toEqual(
+      normalize(find(keys, adminKey.id, "admin api_key").value, [
+        "key_hash",
+        "allowed_models",
+      ]),
+    );
+
+    // The apikeys GET is a public projection that omits attribution
+    // fields — pin the raw stored bytes too, so handler-side enrichment
+    // of the stored document cannot hide behind the projection.
+    const rawSeedKey = JSON.parse(
+      (await etcdClient.get(`${app.etcdPrefix}/api_keys/${seedKey.id}`))!,
+    ) as Record<string, unknown>;
+    const rawAdminKey = JSON.parse(
+      (await etcdClient.get(`${app.etcdPrefix}/api_keys/${adminKey.id}`))!,
+    ) as Record<string, unknown>;
+    expect(normalize(rawSeedKey, ["key_hash", "allowed_models"])).toEqual(
+      normalize(rawAdminKey, ["key_hash", "allowed_models"]),
+    );
+
+    const exporters = await admin.json<Entry[]>("GET", "/admin/v1/observability_exporters");
+    expect(
+      normalize(find(exporters, seedExporter.id, "seed exporter").value, ["name"]),
+    ).toEqual(
+      normalize(find(exporters, adminExporter.id, "admin exporter").value, ["name"]),
+    );
+  });
+});

tests/e2e/src/harness/etcd.ts

@@ -65,6 +65,24 @@ export class EtcdClient {
     }
   }
 
+  /** Read one key's value (etcd v3 `/v3/kv/range`); undefined when absent. */
+  async get(key: string): Promise<string | undefined> {
+    const res = await harnessRequest(`${this.endpoint}/v3/kv/range`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ key: Buffer.from(key, "utf8").toString("base64") }),
+    });
+    if (res.statusCode >= 300) {
+      const body = await res.body.text();
+      throw new Error(`etcd range failed (${res.statusCode}): ${body}`);
+    }
+    const parsed = JSON.parse(await res.body.text()) as {
+      kvs?: Array<{ value: string }>;
+    };
+    const v = parsed.kvs?.[0]?.value;
+    return v === undefined ? undefined : Buffer.from(v, "base64").toString("utf8");
+  }
+
   /** Delete every key under `prefix` (range delete in etcd v3 semantics). */
   async deletePrefix(prefix: string): Promise<void> {
     const key = Buffer.from(prefix, "utf8").toString("base64");

tests/e2e/src/harness/index.ts

@@ -2,6 +2,7 @@ export { spawnApp, type SpawnedApp, type AppOverrides } from "./app.js";
 export { AdminClient, waitConfigPropagation } from "./admin.js";
 export { ProxyClient } from "./proxy.js";
 export { EtcdClient } from "./etcd.js";
+export { SeedClient } from "./seed.js";
 export { startOpenAiUpstream, type OpenAiUpstream, type ReceivedRequest } from "./upstream-openai.js";
 export { pickFreePort, pickFreePorts } from "./ports.js";
 export {

tests/e2e/src/harness/seed.ts

@@ -0,0 +1,68 @@
+import { randomUUID } from "node:crypto";
+
+import { EtcdClient } from "./etcd.js";
+
+/**
+ * Seeds resources by writing canonical resource documents straight to
+ * etcd — the same front door the control plane uses in managed mode,
+ * where the Admin API is not in the write path. The
+ * interface mirrors `AdminClient`'s create methods (same body shapes,
+ * same `{id, value}` return with a generated id), so call sites migrate
+ * mechanically: `admin.createModel({...})` → `seed.createModel({...})`.
+ *
+ * The document written is exactly the caller-supplied body — the
+ * canonical resource shape from `schemas/resources/`. The loader fills
+ * serde defaults on load, so a sparse document loads identically to an
+ * Admin-API-written one; that equivalence is pinned by
+ * `cases/seed-vs-admin-characterization-e2e.test.ts`.
+ *
+ * Unlike the Admin API there is no synchronous validation: a malformed
+ * document is silently skipped by the loader and the test then times
+ * out in `waitConfigPropagation`. Keep seed bodies aligned with the
+ * schemas, and probe propagation with a positive condition.
+ */
+export class SeedClient {
+  constructor(
+    private readonly etcd: EtcdClient,
+    private readonly prefix: string,
+  ) {}
+
+  async createModel(
+    model: Record<string, unknown>,
+  ): Promise<{ id: string; value: Record<string, unknown> }> {
+    return this.put("models", model);
+  }
+
+  async createApiKey(
+    key: Record<string, unknown>,
+  ): Promise<{ id: string; value: Record<string, unknown> }> {
+    return this.put("api_keys", key);
+  }
+
+  async createProviderKey(
+    pk: Record<string, unknown>,
+  ): Promise<{ id: string; value: Record<string, unknown> }> {
+    // Same defaulting as AdminClient.createProviderKey: cp-api always
+    // writes `provider` + `adapter`, so the seeded document carries the
+    // OpenAI-compatible pair unless a test overrides them.
+    return this.put("provider_keys", { provider: "openai", adapter: "openai", ...pk });
+  }
+
+  async createObservabilityExporter(
+    exporter: Record<string, unknown>,
+  ): Promise<{ id: string; value: Record<string, unknown> }> {
+    return this.put("observability_exporters", exporter);
+  }
+
+  private async put(
+    kind: string,
+    value: Record<string, unknown>,
+  ): Promise<{ id: string; value: Record<string, unknown> }> {
+    // The Admin API generates a UUID server-side; here the harness is
+    // the writer, so it generates one — the id lives in the key
+    // (`<prefix>/<kind>/<id>`), not in the document.
+    const id = randomUUID();
+    await this.etcd.put(`${this.prefix}/${kind}/${id}`, JSON.stringify(value));
+    return { id, value };
+  }
+}