From a05098e274c3d1892b4bd007f01e2ada2ccec5ac Mon Sep 17 00:00:00 2001 From: Ming Wen Date: Sun, 12 Jul 2026 20:09:20 +0800 Subject: [PATCH] test(e2e): seed cases via canonical documents instead of the Admin API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sweep the e2e suite's SEEDING onto SeedClient (direct canonical-document writes to etcd — the same front door the control plane uses in managed mode), leaving the Admin API to the cases that exercise it as a subject or read through it. Scope: - SeedClient grows createGuardrail / createCachePolicy / createRateLimitPolicy (kind segments pinned from the Rust source) plus generic update(kind, id, value) and delete(kind, id); EtcdClient grows single-key delete (deleterange without range_end). - The seed-vs-admin characterization gains an update/delete leg: a seed.update'd model serves the NEW upstream model name, and a seed.delete'd api key stops authenticating (fail-closed) while the admin-created control caller keeps serving. - 132 case files migrated: create/PUT seeding moved to seed.*; admin.json guardrail/cache-policy POSTs to seed.createGuardrail / seed.createCachePolicy; PUT update flows to seed.update. - Direct-write seeding is faster than the admin round trip, which exposed latent propagation races that admin-write latency had been masking. Those gates are repaired, never weakened: proxy-visible /v1/models probes, revision-order canary keys for virtual routers (which /v1/models does not list), env-scoped seed roots for custom-prefix apps, and 401-tolerant probe conditions. Assertions are untouched throughout. Held back on the Admin API deliberately: - seed-vs-admin-characterization-e2e (admin is half the comparison) - smoke.test.ts (its subject IS the admin-write -> proxy-read path) - apikey-budget-e2e (its subject is admin-side validation: the unknown-field 400 cannot be observed via direct writes) - openai-sdk-compat.test.ts + datadog-exporter-e2e (deprecation-window coverage pair, marked with a comment) - read paths stay on admin everywhere: listModels/listModelStatuses, revision GETs, the guardrail store-existence gate, and the apikey rotate RPC (server-generated secret, not a document write). Verified: full suite green twice (138 files / 288 tests per run); tsc --noEmit unchanged (the 5 known TS18048 warnings, no new ones). --- .../aliyun-sls-content-capture-e2e.test.ts | 26 ++-- .../src/cases/aliyun-sls-exporter-e2e.test.ts | 16 +-- .../e2e/src/cases/allowed-models-e2e.test.ts | 17 +-- .../anthropic-cache-tpm-commit-e2e.test.ts | 15 +-- ...-content-blocks-cross-provider-e2e.test.ts | 31 +++-- .../cases/anthropic-count-tokens-e2e.test.ts | 15 +-- .../anthropic-streaming-usage-e2e.test.ts | 15 +-- ...anthropic-tools-cross-provider-e2e.test.ts | 28 +++-- .../src/cases/anthropic-upstream-e2e.test.ts | 41 ++++--- .../src/cases/api-base-tolerance-e2e.test.ts | 25 ++-- .../src/cases/apikey-lifecycle-e2e.test.ts | 18 +-- .../src/cases/audio-images-usage-e2e.test.ts | 18 +-- tests/e2e/src/cases/audio-timeout-e2e.test.ts | 19 +-- tests/e2e/src/cases/auth-baseline-e2e.test.ts | 15 +-- .../src/cases/background-health-e2e.test.ts | 28 +++-- .../cases/batch-files-finetuning-e2e.test.ts | 32 +++-- .../cases/bedrock-anonymize-mask-e2e.test.ts | 23 ++-- .../bedrock-credential-isolation-e2e.test.ts | 29 ++--- tests/e2e/src/cases/body-edges-e2e.test.ts | 15 +-- ...cache-edges-model-and-disabled-e2e.test.ts | 25 ++-- .../src/cases/cache-env-namespace-e2e.test.ts | 22 ++-- .../cache-fingerprint-collision-e2e.test.ts | 17 +-- .../cache-hit-output-guardrail-e2e.test.ts | 19 +-- .../cases/cache-policy-backend-e2e.test.ts | 20 +-- tests/e2e/src/cases/cache-policy-e2e.test.ts | 22 ++-- .../e2e/src/cases/cache-scenarios-e2e.test.ts | 19 +-- .../cache-streaming-not-cached-e2e.test.ts | 21 ++-- .../src/cases/cache-ttl-eviction-e2e.test.ts | 20 +-- .../caller-credential-isolation-e2e.test.ts | 15 +-- .../e2e/src/cases/canary-routing-e2e.test.ts | 40 +++--- ...-anthropic-stream-input-tokens-e2e.test.ts | 13 +- .../src/cases/completions-legacy-e2e.test.ts | 15 +-- .../completions-output-guardrail-e2e.test.ts | 17 +-- .../cases/completions-tpm-commit-e2e.test.ts | 15 +-- tests/e2e/src/cases/concurrency-e2e.test.ts | 21 ++-- .../src/cases/cooldown-contract-e2e.test.ts | 116 +++++++++++------- .../src/cases/cost-aware-routing-e2e.test.ts | 91 ++++++++------ .../cases/cross-provider-matrix-e2e.test.ts | 23 ++-- .../src/cases/datadog-exporter-e2e.test.ts | 1 + .../deepseek-reasoning-content-e2e.test.ts | 15 +-- tests/e2e/src/cases/embeddings-e2e.test.ts | 23 ++-- ...beddings-missing-prompt-tokens-e2e.test.ts | 15 +-- .../embeddings-native-providers-e2e.test.ts | 46 +++++-- .../error-envelope-normalization-e2e.test.ts | 17 +-- tests/e2e/src/cases/fallback-e2e.test.ts | 21 ++-- .../e2e/src/cases/fallback-edges-e2e.test.ts | 35 +++--- .../cases/fallback-on-statuses-e2e.test.ts | 27 ++-- .../src/cases/guardrail-aliyun-e2e.test.ts | 25 ++-- .../guardrail-disabled-bypass-e2e.test.ts | 14 ++- .../src/cases/guardrail-keyword-e2e.test.ts | 17 +-- ...rail-keyword-message-locations-e2e.test.ts | 17 +-- .../src/cases/guardrail-lakera-e2e.test.ts | 17 +-- .../cases/guardrail-model-scope-e2e.test.ts | 32 +++-- .../cases/guardrail-monitor-mode-e2e.test.ts | 21 ++-- ...drail-monitor-provider-failure-e2e.test.ts | 33 ++--- .../guardrail-monitor-telemetry-e2e.test.ts | 18 +-- .../guardrail-openai-moderation-e2e.test.ts | 45 ++++--- .../src/cases/guardrail-output-e2e.test.ts | 31 ++--- .../cases/guardrail-pii-redaction-e2e.test.ts | 52 ++++---- .../src/cases/guardrail-presidio-e2e.test.ts | 47 ++++--- .../src/cases/images-generations-e2e.test.ts | 23 ++-- .../invalid-upstream-config-400-e2e.test.ts | 15 +-- ...valid-upstream-credentials-401-e2e.test.ts | 15 +-- tests/e2e/src/cases/json-mode-e2e.test.ts | 15 +-- .../cases/latency-aware-routing-e2e.test.ts | 21 ++-- .../src/cases/latency-histograms-e2e.test.ts | 21 ++-- ...ast-busy-passthrough-endpoints-e2e.test.ts | 25 ++-- .../src/cases/least-busy-routing-e2e.test.ts | 21 ++-- ...messages-cross-provider-extras-e2e.test.ts | 28 +++-- ...es-cross-provider-stream-usage-e2e.test.ts | 19 +-- .../messages-input-guardrail-e2e.test.ts | 27 ++-- .../cases/messages-model-group-e2e.test.ts | 63 +++++----- .../messages-output-guardrail-e2e.test.ts | 17 +-- ...ssages-stream-delta-only-usage-e2e.test.ts | 15 +-- ...ges-streaming-output-guardrail-e2e.test.ts | 17 +-- ...s-streaming-toolcall-guardrail-e2e.test.ts | 17 +-- .../cases/messages-system-role-e2e.test.ts | 15 +-- ...metric-cardinality-model-label-e2e.test.ts | 15 +-- .../cases/metrics-dimensions-890-e2e.test.ts | 21 ++-- .../cases/model-ip-restriction-e2e.test.ts | 15 +-- .../openai-embeddings-base64-e2e.test.ts | 15 +-- tests/e2e/src/cases/openai-sdk-compat.test.ts | 1 + .../cases/openai-tools-roundtrip-e2e.test.ts | 15 +-- .../openrouter-stream-reasoning-e2e.test.ts | 15 +-- tests/e2e/src/cases/otlp-knobs-e2e.test.ts | 30 ++--- tests/e2e/src/cases/passthrough-e2e.test.ts | 23 ++-- .../cases/passthrough-guardrail-e2e.test.ts | 19 +-- .../cases/passthrough-model-acl-e2e.test.ts | 15 +-- .../cases/per-attempt-telemetry-e2e.test.ts | 44 ++++--- .../src/cases/prometheus-metrics-e2e.test.ts | 23 ++-- .../cases/provider-key-rotation-e2e.test.ts | 14 ++- .../src/cases/ratelimit-cluster-e2e.test.ts | 23 ++-- tests/e2e/src/cases/ratelimit-e2e.test.ts | 15 +-- tests/e2e/src/cases/realtime-ws-e2e.test.ts | 28 +++-- .../src/cases/request-id-header-e2e.test.ts | 13 +- .../e2e/src/cases/request-id-uuid-e2e.test.ts | 23 ++-- tests/e2e/src/cases/rerank-e2e.test.ts | 39 +++--- .../responses-cross-provider-e2e.test.ts | 19 +-- .../src/cases/responses-endpoint-e2e.test.ts | 23 ++-- .../responses-streaming-usage-e2e.test.ts | 17 +-- tests/e2e/src/cases/retry-backoff-e2e.test.ts | 17 +-- ...ry-on-429-vs-background-ignore-e2e.test.ts | 18 +-- .../src/cases/routing-strategies-e2e.test.ts | 75 ++++++----- .../cases/runtime-mixed-filtering-e2e.test.ts | 22 ++-- .../e2e/src/cases/runtime-status-e2e.test.ts | 18 +-- ...seed-vs-admin-characterization-e2e.test.ts | 84 +++++++++++++ .../src/cases/semantic-routing-e2e.test.ts | 37 +++--- .../cases/server-header-identity-e2e.test.ts | 13 +- .../sls-content-capture-masked-e2e.test.ts | 20 +-- .../sls-content-capture-nontext-e2e.test.ts | 14 +-- ...-capture-responses-completions-e2e.test.ts | 17 +-- .../src/cases/sls-failure-content-e2e.test.ts | 33 ++--- .../sls-structured-truncation-e2e.test.ts | 15 +-- .../e2e/src/cases/streaming-edges-e2e.test.ts | 23 ++-- .../cases/streaming-tpm-commit-e2e.test.ts | 21 ++-- .../streaming-usage-accumulation-e2e.test.ts | 19 +-- tests/e2e/src/cases/tag-routing-e2e.test.ts | 46 ++++--- .../cases/team-member-ratelimit-e2e.test.ts | 12 +- .../src/cases/timeout-fallback-e2e.test.ts | 29 ++--- .../cases/token-usage-passthrough-e2e.test.ts | 15 +-- .../tokens-by-client-total-1002-e2e.test.ts | 15 +-- .../cases/tools-cross-provider-e2e.test.ts | 15 +-- .../cases/upstream-401-error-code-e2e.test.ts | 41 +++---- ...stream-retry-after-passthrough-e2e.test.ts | 15 +-- .../cases/usage-cache-counters-e2e.test.ts | 35 +++--- .../src/cases/usage-client-source-e2e.test.ts | 22 ++-- tests/e2e/src/cases/vision-input-e2e.test.ts | 15 +-- .../e2e/src/cases/vision-messages-e2e.test.ts | 15 +-- .../cases/vision-messages-edges-e2e.test.ts | 15 +-- .../weighted-routing-distribution-e2e.test.ts | 21 ++-- .../cases/weighted-routing-edit-e2e.test.ts | 34 ++--- .../src/cases/wildcard-routing-e2e.test.ts | 66 +++++----- tests/e2e/src/harness/etcd.ts | 17 +++ tests/e2e/src/harness/seed.ts | 37 ++++++ 134 files changed, 1853 insertions(+), 1443 deletions(-) diff --git a/tests/e2e/src/cases/aliyun-sls-content-capture-e2e.test.ts b/tests/e2e/src/cases/aliyun-sls-content-capture-e2e.test.ts index 7f8eee18..8a8b9072 100644 --- a/tests/e2e/src/cases/aliyun-sls-content-capture-e2e.test.ts +++ b/tests/e2e/src/cases/aliyun-sls-content-capture-e2e.test.ts @@ -1,9 +1,9 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, decodedTextFor, EtcdClient, + SeedClient, spawnApp, startMockSls, startOpenAiUpstream, @@ -49,45 +49,45 @@ const ANTHROPIC_PROMPT_TOKEN = "anthropic-prompt-tok-5d1f9c"; const ANTHROPIC_RESPONSE_TOKEN = "anthropic-response-tok-a4e823"; async function seedRouting( - admin: AdminClient, + seed: SeedClient, upstream: OpenAiUpstream, streamUpstream: OpenAiUpstream, messagesUpstream: OpenAiUpstream, ) { - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "content-capture-pk", secret: PROVIDER_SECRET, api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "content-capture-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - const streamPk = await admin.createProviderKey({ + const streamPk = await seed.createProviderKey({ display_name: "content-capture-stream-pk", secret: PROVIDER_SECRET, api_base: `${streamUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "content-capture-stream-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: streamPk.id, }); - const msgPk = await admin.createProviderKey({ + const msgPk = await seed.createProviderKey({ display_name: "content-capture-messages-pk", secret: PROVIDER_SECRET, api_base: `${messagesUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "content-capture-messages-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: msgPk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: [ "content-capture-model", @@ -251,8 +251,8 @@ describe("aliyun_sls content capture e2e (#687): full vs metadata_only", () => { }, }); apps.push(app); - const admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createObservabilityExporter({ + const seed = new SeedClient(new EtcdClient(), app.etcdPrefix); + await seed.createObservabilityExporter({ name: "sls-full", enabled: true, kind: "aliyun_sls", @@ -262,7 +262,7 @@ describe("aliyun_sls content capture e2e (#687): full vs metadata_only", () => { credential_ref: CREDENTIAL_REF, content_mode: "full", }); - await admin.createObservabilityExporter({ + await seed.createObservabilityExporter({ name: "sls-meta", enabled: true, kind: "aliyun_sls", @@ -272,7 +272,7 @@ describe("aliyun_sls content capture e2e (#687): full vs metadata_only", () => { credential_ref: CREDENTIAL_REF, content_mode: "metadata_only", }); - await seedRouting(admin, upstream, streamUpstream, messagesUpstream); + await seedRouting(seed, upstream, streamUpstream, messagesUpstream); await waitConfigPropagation(async () => { try { diff --git a/tests/e2e/src/cases/aliyun-sls-exporter-e2e.test.ts b/tests/e2e/src/cases/aliyun-sls-exporter-e2e.test.ts index 8d7a2fd8..f4d984e8 100644 --- a/tests/e2e/src/cases/aliyun-sls-exporter-e2e.test.ts +++ b/tests/e2e/src/cases/aliyun-sls-exporter-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import { createServer, type IncomingMessage, type Server } from "node:http"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, pickFreePort, spawnApp, startOpenAiUpstream, @@ -89,19 +89,19 @@ async function startMockSls(): Promise { }; } -async function seedRouting(admin: AdminClient, upstream: OpenAiUpstream) { - const pk = await admin.createProviderKey({ +async function seedRouting(seed: SeedClient, upstream: OpenAiUpstream) { + const pk = await seed.createProviderKey({ display_name: "sls-exporter-pk", secret: PROVIDER_SECRET, api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "sls-exporter-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["sls-exporter-model"], }); @@ -176,8 +176,8 @@ describe("aliyun_sls exporter e2e (#687): DP delivers a signed PutLogs to SLS", }, }); apps.push(app); - const admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createObservabilityExporter({ + const seed = new SeedClient(new EtcdClient(), app.etcdPrefix); + await seed.createObservabilityExporter({ name: "mock-sls", enabled: true, kind: "aliyun_sls", @@ -186,7 +186,7 @@ describe("aliyun_sls exporter e2e (#687): DP delivers a signed PutLogs to SLS", logstore: SLS_LOGSTORE, credential_ref: CREDENTIAL_REF, }); - await seedRouting(admin, upstream); + await seedRouting(seed, upstream); await waitConfigPropagation(async () => { try { diff --git a/tests/e2e/src/cases/allowed-models-e2e.test.ts b/tests/e2e/src/cases/allowed-models-e2e.test.ts index 68ca12ac..7c9f9536 100644 --- a/tests/e2e/src/cases/allowed-models-e2e.test.ts +++ b/tests/e2e/src/cases/allowed-models-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -30,32 +30,33 @@ const CALLER_KEY_HASH = createHash("sha256") describe("allowed_models e2e: 403 on disallowed model, upstream untouched", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); // Single ProviderKey + two Models. They share the upstream so // upstream.receivedRequests is a single source of truth — if the // forbidden call leaks through, it will register here. - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "am-e2e-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "am-allowed", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createModel({ + await seed.createModel({ display_name: "am-forbidden", provider: "openai", model_name: "gpt-4o-mini", @@ -64,7 +65,7 @@ describe("allowed_models e2e: 403 on disallowed model, upstream untouched", () = // Caller permitted ONLY for am-allowed. am-forbidden exists in // the snapshot — the 403 must come from authz, not from "model // not found". - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["am-allowed"], }); diff --git a/tests/e2e/src/cases/anthropic-cache-tpm-commit-e2e.test.ts b/tests/e2e/src/cases/anthropic-cache-tpm-commit-e2e.test.ts index c45fbb89..60875506 100644 --- a/tests/e2e/src/cases/anthropic-cache-tpm-commit-e2e.test.ts +++ b/tests/e2e/src/cases/anthropic-cache-tpm-commit-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, ProxyClient, spawnApp, startOpenAiUpstream, @@ -109,11 +109,12 @@ describe("anthropic cache tokens count toward TPM (AISIX-Cloud#995)", () => { let upstreamMsg: OpenAiUpstream | undefined; let upstreamMsgStream: OpenAiUpstream | undefined; let upstreamResp: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstreamMsg = await startOpenAiUpstream({ @@ -127,7 +128,7 @@ describe("anthropic cache tokens count toward TPM (AISIX-Cloud#995)", () => { nonStreamBody: anthropicMessageBody(NONSTREAM_USAGE), }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); // The Anthropic adapter appends `/v1/messages` to the api_base, so the // provider key points at the bare mock host. @@ -136,21 +137,21 @@ describe("anthropic cache tokens count toward TPM (AISIX-Cloud#995)", () => { upstream: OpenAiUpstream, caller: string, ) => { - const pk = await admin!.createProviderKey({ + const pk = await seed!.createProviderKey({ display_name: `${name}-pk`, provider: "anthropic", adapter: "anthropic", secret: "sk-ant-mock", api_base: upstream.baseUrl, }); - await admin!.createModel({ + await seed!.createModel({ display_name: name, provider: "anthropic", model_name: "claude-3-5-haiku-20241022", provider_key_id: pk.id, }); // Separate caller keys so each scenario's TPM counter is independent. - await admin!.createApiKey({ + await seed!.createApiKey({ key_hash: hash(caller), allowed_models: [name], rate_limit: { tpm: TPM }, diff --git a/tests/e2e/src/cases/anthropic-content-blocks-cross-provider-e2e.test.ts b/tests/e2e/src/cases/anthropic-content-blocks-cross-provider-e2e.test.ts index 6ce0df56..2fd5edde 100644 --- a/tests/e2e/src/cases/anthropic-content-blocks-cross-provider-e2e.test.ts +++ b/tests/e2e/src/cases/anthropic-content-blocks-cross-provider-e2e.test.ts @@ -1,10 +1,11 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, + waitConfigPropagation, type OpenAiUpstream, type SpawnedApp, } from "../harness/index.js"; @@ -33,17 +34,18 @@ const CALLER_KEY_HASH = createHash("sha256") describe("anthropic content blocks → OpenAI upstream (#722)", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; const upstreams: OpenAiUpstream[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createApiKey({ + seed = new SeedClient(etcd, app.etcdPrefix); + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); @@ -71,22 +73,33 @@ describe("anthropic content blocks → OpenAI upstream (#722)", () => { }, }); upstreams.push(upstream); - const pk = await admin!.createProviderKey({ + const pk = await seed!.createProviderKey({ display_name: `${name}-pk`, secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin!.createModel({ + await seed!.createModel({ display_name: name, provider: "openai", model_name: "gpt-4o", provider_key_id: pk.id, }); + // Gate on the DP snapshot, not the store: /v1/models only lists the + // model once the snapshot has it, and only authenticates once the + // caller key has propagated too. Touches no upstream. + await waitConfigPropagation(async () => { + const res = await fetch(`${app!.proxyUrl}/v1/models`, { + headers: { authorization: `Bearer ${CALLER_PLAINTEXT}` }, + }); + if (res.status !== 200) return false; + const body = (await res.json()) as { data?: Array<{ id?: string }> }; + return (body.data ?? []).some((m) => m.id === name); + }); return upstream; } test("multi-turn tool loop history reaches the OpenAI upstream intact", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -182,7 +195,7 @@ describe("anthropic content blocks → OpenAI upstream (#722)", () => { }); test("vision: base64 image block reaches the upstream as an image_url data URL", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } diff --git a/tests/e2e/src/cases/anthropic-count-tokens-e2e.test.ts b/tests/e2e/src/cases/anthropic-count-tokens-e2e.test.ts index 2b833729..dad51fd2 100644 --- a/tests/e2e/src/cases/anthropic-count-tokens-e2e.test.ts +++ b/tests/e2e/src/cases/anthropic-count-tokens-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -49,11 +49,12 @@ const MODEL_ALIAS = "ct-e2e"; describe("anthropic count_tokens e2e: /v1/messages/count_tokens through the DP (#418)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ @@ -61,21 +62,21 @@ describe("anthropic count_tokens e2e: /v1/messages/count_tokens through the DP ( nonStreamBody: { input_tokens: 42 }, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); // Anthropic bridge appends the path to the bare host (no `/v1`). - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "ct-e2e-pk", secret: "sk-ant-mock", api_base: upstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: MODEL_ALIAS, provider: "anthropic", model_name: UPSTREAM_MODEL_ID, provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: [MODEL_ALIAS], }); diff --git a/tests/e2e/src/cases/anthropic-streaming-usage-e2e.test.ts b/tests/e2e/src/cases/anthropic-streaming-usage-e2e.test.ts index 9fe42f76..ba9783eb 100644 --- a/tests/e2e/src/cases/anthropic-streaming-usage-e2e.test.ts +++ b/tests/e2e/src/cases/anthropic-streaming-usage-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -84,11 +84,12 @@ const STREAM_EVENTS = [ describe("anthropic /v1/messages streaming usage telemetry (#245)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; // Path-agnostic mock: the DP forwards /v1/messages to @@ -100,22 +101,22 @@ describe("anthropic /v1/messages streaming usage telemetry (#245)", () => { eventDelayMs: 2, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); // Anthropic-provider model → exercises the byte-passthrough // streaming branch (the one #245 fixes), not the translated path. - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "anth-stream-usage-pk", secret: "sk-anth-mock", api_base: upstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: "anth-stream-usage", provider: "anthropic", model_name: "claude-3-5-haiku-20241022", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["anth-stream-usage"], }); diff --git a/tests/e2e/src/cases/anthropic-tools-cross-provider-e2e.test.ts b/tests/e2e/src/cases/anthropic-tools-cross-provider-e2e.test.ts index e2d644b9..2cfce0e2 100644 --- a/tests/e2e/src/cases/anthropic-tools-cross-provider-e2e.test.ts +++ b/tests/e2e/src/cases/anthropic-tools-cross-provider-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -37,11 +37,12 @@ const CALLER_KEY_HASH = createHash("sha256") describe("Anthropic Messages client → OpenAI upstream: tools translation (#236)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; // Mock OpenAI upstream returns a tool_calls response. @@ -76,20 +77,20 @@ describe("Anthropic Messages client → OpenAI upstream: tools translation (#236 }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "anth-tools-xprov-pk", secret: "sk-openai-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "anth-tools-xprov", provider: "openai", model_name: "gpt-4o", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["anth-tools-xprov"], }); @@ -226,11 +227,12 @@ const STREAM_CALLER_KEY_HASH = createHash("sha256") describe("Anthropic Messages client → OpenAI upstream: streaming tool_calls (#236)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; // Mock OpenAI upstream returns streaming tool_calls chunks. @@ -315,20 +317,20 @@ describe("Anthropic Messages client → OpenAI upstream: streaming tool_calls (# upstream = await startOpenAiUpstream({ streamEvents }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "anth-tools-stream-xprov-pk", secret: "sk-openai-stream-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "anth-tools-stream-xprov", provider: "openai", model_name: "gpt-4o", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: STREAM_CALLER_KEY_HASH, allowed_models: ["anth-tools-stream-xprov"], }); diff --git a/tests/e2e/src/cases/anthropic-upstream-e2e.test.ts b/tests/e2e/src/cases/anthropic-upstream-e2e.test.ts index fe41d206..893d79d5 100644 --- a/tests/e2e/src/cases/anthropic-upstream-e2e.test.ts +++ b/tests/e2e/src/cases/anthropic-upstream-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -40,11 +40,12 @@ const CALLER_KEY_HASH = createHash("sha256") describe("anthropic upstream e2e: OpenAI in, Anthropic out, OpenAI back to caller", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ @@ -59,26 +60,26 @@ describe("anthropic upstream e2e: OpenAI in, Anthropic out, OpenAI back to calle }, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); // The Anthropic bridge appends `/v1/messages` to the api_base, so // we point it at the bare host (no `/v1` suffix) — the OpenAI // bridge convention is the opposite (host + `/v1`), and getting // these mixed up was the lesson from the unit-level matrix tests. - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "an-e2e-pk", provider: "anthropic", adapter: "anthropic", secret: "sk-ant-mock", api_base: upstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: "an-e2e", provider: "anthropic", model_name: "claude-3-5-haiku-20241022", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["an-e2e"], }); @@ -203,11 +204,12 @@ const TOOL_CALLER_KEY_HASH = createHash("sha256") describe("anthropic upstream e2e: tool_use-only response surfaces content:null (#395)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ @@ -230,22 +232,22 @@ describe("anthropic upstream e2e: tool_use-only response surfaces content:null ( }, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "an-e2e-toolnull-pk", provider: "anthropic", adapter: "anthropic", secret: "sk-ant-mock", api_base: upstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: "an-e2e-toolnull", provider: "anthropic", model_name: "claude-3-5-haiku-20241022", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: TOOL_CALLER_KEY_HASH, allowed_models: ["an-e2e-toolnull"], }); @@ -308,11 +310,12 @@ const CACHE_CALLER_KEY_HASH = createHash("sha256") describe("anthropic upstream e2e: cache tokens fold into total_tokens (#906)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ @@ -332,22 +335,22 @@ describe("anthropic upstream e2e: cache tokens fold into total_tokens (#906)", ( }, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "an-e2e-cachetotal-pk", provider: "anthropic", adapter: "anthropic", secret: "sk-ant-mock", api_base: upstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: "an-e2e-cachetotal", provider: "anthropic", model_name: "claude-3-5-haiku-20241022", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CACHE_CALLER_KEY_HASH, allowed_models: ["an-e2e-cachetotal"], }); diff --git a/tests/e2e/src/cases/api-base-tolerance-e2e.test.ts b/tests/e2e/src/cases/api-base-tolerance-e2e.test.ts index e7e6521b..9dbfb719 100644 --- a/tests/e2e/src/cases/api-base-tolerance-e2e.test.ts +++ b/tests/e2e/src/cases/api-base-tolerance-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -44,22 +44,23 @@ const ANTHROPIC_CALLER_KEY_HASH = createHash("sha256") describe("api_base tolerance e2e: endpoint suffix is stripped before dispatch", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; const upstreams: OpenAiUpstream[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: OPENAI_CALLER_KEY_HASH, allowed_models: ["openai-suffix-tolerance"], }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: ANTHROPIC_CALLER_KEY_HASH, allowed_models: ["anthropic-suffix-tolerance"], }); @@ -71,7 +72,7 @@ describe("api_base tolerance e2e: endpoint suffix is stripped before dispatch", }); test("OpenAI bridge strips a `/chat/completions` suffix from api_base", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -102,12 +103,12 @@ describe("api_base tolerance e2e: endpoint suffix is stripped before dispatch", // suffix stripping, the bridge would dispatch to // `${baseUrl}/chat/completions/chat/completions` // and `upstream.receivedRequests[*].path` would echo that. - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "openai-suffix-tolerance-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/chat/completions`, }); - await admin.createModel({ + await seed.createModel({ display_name: "openai-suffix-tolerance", provider: "openai", model_name: "gpt-4o-mini", @@ -159,7 +160,7 @@ describe("api_base tolerance e2e: endpoint suffix is stripped before dispatch", }); test("Anthropic bridge strips a `/v1/messages` suffix from api_base", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -180,14 +181,14 @@ describe("api_base tolerance e2e: endpoint suffix is stripped before dispatch", // Operator pastes the full Anthropic upstream URL. Without the // strip, the bridge would dispatch to // `${baseUrl}/v1/messages/v1/messages`. - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "anthropic-suffix-tolerance-pk", provider: "anthropic", adapter: "anthropic", secret: "sk-ant-mock", api_base: `${upstream.baseUrl}/v1/messages`, }); - await admin.createModel({ + await seed.createModel({ display_name: "anthropic-suffix-tolerance", provider: "anthropic", model_name: "claude-3-5-haiku-20241022", diff --git a/tests/e2e/src/cases/apikey-lifecycle-e2e.test.ts b/tests/e2e/src/cases/apikey-lifecycle-e2e.test.ts index 57114aa4..edf44e63 100644 --- a/tests/e2e/src/cases/apikey-lifecycle-e2e.test.ts +++ b/tests/e2e/src/cases/apikey-lifecycle-e2e.test.ts @@ -3,6 +3,7 @@ import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -44,22 +45,25 @@ describe("api key lifecycle e2e: expired/disabled keys fail closed, rotate swaps let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); app = await spawnApp(); admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "apikey-lifecycle-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: MODEL, provider: "openai", model_name: "gpt-4o-mini", @@ -95,7 +99,7 @@ describe("api key lifecycle e2e: expired/disabled keys fail closed, rotate swaps extra: Record, propagated: (res: Response) => Promise, ): Promise<{ id: string }> { - const created = await admin!.createApiKey({ + const created = await seed!.createApiKey({ key_hash: sha256(plaintext), allowed_models: [MODEL], ...extra, @@ -177,7 +181,7 @@ describe("api key lifecycle e2e: expired/disabled keys fail closed, rotate swaps }); test("disable then re-enable an in-use key without re-issuing it", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -188,7 +192,7 @@ describe("api key lifecycle e2e: expired/disabled keys fail closed, rotate swaps // Disable: same key_hash, flipped flag — the credential the // caller holds is unchanged, only its status moves. - await admin.json("PUT", `/admin/v1/apikeys/${id}`, { + await seed.update("api_keys", id, { key_hash: keyHash, allowed_models: [MODEL], disabled: true, @@ -199,7 +203,7 @@ describe("api key lifecycle e2e: expired/disabled keys fail closed, rotate swaps // Re-enable: the original plaintext works again — proving disable // is reversible and did not rotate or delete the credential. - await admin.json("PUT", `/admin/v1/apikeys/${id}`, { + await seed.update("api_keys", id, { key_hash: keyHash, allowed_models: [MODEL], }); diff --git a/tests/e2e/src/cases/audio-images-usage-e2e.test.ts b/tests/e2e/src/cases/audio-images-usage-e2e.test.ts index e396c875..c4943dea 100644 --- a/tests/e2e/src/cases/audio-images-usage-e2e.test.ts +++ b/tests/e2e/src/cases/audio-images-usage-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -55,19 +55,19 @@ describe("audio + images UsageEvent emission (#406/#407)", () => { }); const app: SpawnedApp = await spawnApp(); try { - const admin = new AdminClient(app.adminUrl, app.adminKey); - const pk = await admin.createProviderKey({ + const seed = new SeedClient(new EtcdClient(), app.etcdPrefix); + const pk = await seed.createProviderKey({ display_name: "img-usage-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "img-usage", provider: "openai", model_name: "gpt-image-1", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["img-usage"], }); @@ -114,19 +114,19 @@ describe("audio + images UsageEvent emission (#406/#407)", () => { }); const app: SpawnedApp = await spawnApp(); try { - const admin = new AdminClient(app.adminUrl, app.adminKey); - const pk = await admin.createProviderKey({ + const seed = new SeedClient(new EtcdClient(), app.etcdPrefix); + const pk = await seed.createProviderKey({ display_name: "audio-usage-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "audio-usage", provider: "openai", model_name: "gpt-4o-transcribe", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["audio-usage"], }); diff --git a/tests/e2e/src/cases/audio-timeout-e2e.test.ts b/tests/e2e/src/cases/audio-timeout-e2e.test.ts index 81c60eb8..55429de2 100644 --- a/tests/e2e/src/cases/audio-timeout-e2e.test.ts +++ b/tests/e2e/src/cases/audio-timeout-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -45,13 +45,14 @@ function chatReply(content: string): unknown { describe("audio request timeout (#911 [22])", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let slow: OpenAiUpstream | undefined; let fast: OpenAiUpstream | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; // Slow upstream (delays status + headers) behind the audio model, and a @@ -63,24 +64,24 @@ describe("audio request timeout (#911 [22])", () => { fast = await startOpenAiUpstream({ nonStreamBody: chatReply("ready") }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); const slowPk = ( - await admin.createProviderKey({ + await seed.createProviderKey({ display_name: "audio-slow-pk", secret: "sk-mock", api_base: `${slow.baseUrl}/v1`, }) ).id; const fastPk = ( - await admin.createProviderKey({ + await seed.createProviderKey({ display_name: "audio-gate-pk", secret: "sk-mock", api_base: `${fast.baseUrl}/v1`, }) ).id; - await admin.createModel({ + await seed.createModel({ display_name: "audio-slow", provider: "openai", model_name: "whisper-1", @@ -90,14 +91,14 @@ describe("audio request timeout (#911 [22])", () => { // between the propagation probe and the test call. cooldown: { enabled: false }, }); - await admin.createModel({ + await seed.createModel({ display_name: "gate-fast", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: fastPk, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["audio-slow", "gate-fast"], }); diff --git a/tests/e2e/src/cases/auth-baseline-e2e.test.ts b/tests/e2e/src/cases/auth-baseline-e2e.test.ts index 9ea804d4..54cbc300 100644 --- a/tests/e2e/src/cases/auth-baseline-e2e.test.ts +++ b/tests/e2e/src/cases/auth-baseline-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -41,33 +41,34 @@ const UNKNOWN_PLAINTEXT = "sk-auth-baseline-unregistered"; describe("auth baseline e2e: missing/malformed/unknown bearer all fail closed", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); // A single Model + ProviderKey + ApiKey configured. The valid // ApiKey is what makes the negative cases meaningful: failing // auth must happen even though a valid key exists; failing auth // must NOT degrade to "any-key-works". - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "auth-baseline-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "auth-baseline", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: VALID_KEY_HASH, allowed_models: ["auth-baseline"], }); diff --git a/tests/e2e/src/cases/background-health-e2e.test.ts b/tests/e2e/src/cases/background-health-e2e.test.ts index b00f9c9c..ebda549a 100644 --- a/tests/e2e/src/cases/background-health-e2e.test.ts +++ b/tests/e2e/src/cases/background-health-e2e.test.ts @@ -4,6 +4,7 @@ import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -19,6 +20,7 @@ const CALLER_KEY_HASH = createHash("sha256") describe("background health e2e", () => { let app: SpawnedApp | undefined; let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; let unhealthyUpstream: OpenAiUpstream | undefined; let ignoredUpstream: OpenAiUpstream | undefined; @@ -28,7 +30,8 @@ describe("background health e2e", () => { let stableModelID = ""; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; unhealthyUpstream = await startOpenAiUpstream({ @@ -58,25 +61,26 @@ describe("background health e2e", () => { app = await spawnApp(); admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const unhealthyPk = await admin.createProviderKey({ + const unhealthyPk = await seed.createProviderKey({ display_name: "bg-unhealthy-pk", secret: "sk-mock", api_base: `${unhealthyUpstream.baseUrl}/v1`, }); - const ignoredPk = await admin.createProviderKey({ + const ignoredPk = await seed.createProviderKey({ display_name: "bg-ignored-pk", secret: "sk-mock", api_base: `${ignoredUpstream.baseUrl}/v1`, }); - const stablePk = await admin.createProviderKey({ + const stablePk = await seed.createProviderKey({ display_name: "bg-stable-pk", secret: "sk-mock", api_base: `${stableUpstream.baseUrl}/v1`, }); unhealthyModelID = ( - await admin.createModel({ + await seed.createModel({ display_name: "bg-unhealthy", provider: "openai", model_name: "gpt-4o-mini", @@ -94,7 +98,7 @@ describe("background health e2e", () => { ).id; ignoredModelID = ( - await admin.createModel({ + await seed.createModel({ display_name: "bg-ignored", provider: "openai", model_name: "gpt-4o-mini", @@ -112,7 +116,7 @@ describe("background health e2e", () => { ).id; stableModelID = ( - await admin.createModel({ + await seed.createModel({ display_name: "bg-stable", provider: "openai", model_name: "gpt-4o-mini", @@ -120,7 +124,7 @@ describe("background health e2e", () => { }) ).id; - await admin.createModel({ + await seed.createModel({ display_name: "bg-router", routing: { strategy: "failover", @@ -132,7 +136,7 @@ describe("background health e2e", () => { }, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["bg-router", "bg-stable"], }); @@ -211,17 +215,17 @@ describe("background health e2e", () => { }); test("active background checks keep unhealthy state fresh even with a short stale window", async (ctx) => { - if (!etcdReachable || !app || !admin || !unhealthyUpstream) { + if (!etcdReachable || !app || !admin || !seed || !unhealthyUpstream) { ctx.skip(); return; } - const shortStalePk = await admin.createProviderKey({ + const shortStalePk = await seed.createProviderKey({ display_name: "bg-stale-short-pk", secret: "sk-mock", api_base: `${unhealthyUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "bg-stale-short", provider: "openai", model_name: "gpt-4o-mini", diff --git a/tests/e2e/src/cases/batch-files-finetuning-e2e.test.ts b/tests/e2e/src/cases/batch-files-finetuning-e2e.test.ts index cfb6af39..5efe698f 100644 --- a/tests/e2e/src/cases/batch-files-finetuning-e2e.test.ts +++ b/tests/e2e/src/cases/batch-files-finetuning-e2e.test.ts @@ -3,9 +3,10 @@ import { createServer, type Server } from "node:http"; import OpenAI, { toFile } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, + waitConfigPropagation, type SpawnedApp, } from "../harness/index.js"; @@ -171,35 +172,48 @@ async function startJobsUpstream(): Promise { describe("jobs e2e: /v1/files + /v1/batches + /v1/fine_tuning/jobs (#720)", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let upstream: JobsUpstream | undefined; let etcdReachable = false; let client: OpenAI; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); upstream = await startJobsUpstream(); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "jobs-e2e-pk", secret: "sk-upstream-jobs", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "jobs-e2e-model", provider: "openai", model_name: "gpt-4o", provider_key_id: pk.id, }); + // Gate on the DP snapshot, not the store: /v1/models only lists the + // model once the snapshot has it, and only authenticates once the + // caller key has propagated too. Touches no upstream. + await waitConfigPropagation(async () => { + const res = await fetch(`${app!.proxyUrl}/v1/models`, { + headers: { authorization: `Bearer ${CALLER_PLAINTEXT}` }, + }); + if (res.status !== 200) return false; + const body = (await res.json()) as { data?: Array<{ id?: string }> }; + return (body.data ?? []).some((m) => m.id === "jobs-e2e-model"); + }); + client = new OpenAI({ apiKey: CALLER_PLAINTEXT, baseURL: `${app.proxyUrl}/v1`, @@ -212,7 +226,7 @@ describe("jobs e2e: /v1/files + /v1/batches + /v1/fine_tuning/jobs (#720)", () = }); test("full batch lifecycle through the OpenAI SDK: upload → create → retrieve → content → cancel", async (ctx) => { - if (!etcdReachable || !app || !admin || !upstream) { + if (!etcdReachable || !app || !seed || !upstream) { ctx.skip(); return; } @@ -287,7 +301,7 @@ describe("jobs e2e: /v1/files + /v1/batches + /v1/fine_tuning/jobs (#720)", () = }); test("fine-tuning: encoded training_file routes the job, provider base model forwards verbatim", async (ctx) => { - if (!etcdReachable || !app || !admin || !upstream) { + if (!etcdReachable || !app || !seed || !upstream) { ctx.skip(); return; } diff --git a/tests/e2e/src/cases/bedrock-anonymize-mask-e2e.test.ts b/tests/e2e/src/cases/bedrock-anonymize-mask-e2e.test.ts index 704286bd..1ca183d9 100644 --- a/tests/e2e/src/cases/bedrock-anonymize-mask-e2e.test.ts +++ b/tests/e2e/src/cases/bedrock-anonymize-mask-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import { createServer, type Server } from "node:http"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, pickFreePort, spawnApp, startOpenAiUpstream, @@ -139,10 +139,11 @@ describe("bedrock guardrail ANONYMIZE write-back (#932 follow-up)", () => { let streamUpstream: OpenAiUpstream | undefined; let bedrock: MockBedrock | undefined; let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; // Non-streaming upstream: the canned reply CONTAINS an email so the @@ -183,41 +184,41 @@ describe("bedrock guardrail ANONYMIZE write-back (#932 follow-up)", () => { bedrock = await startMockBedrock(); app = await spawnApp({ extra: { bedrock_endpoint_url: bedrock.url } }); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "bmask-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "bmask-e2e", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: hash(CALLER), allowed_models: ["bmask-e2e"], }); - const streamPk = await admin.createProviderKey({ + const streamPk = await seed.createProviderKey({ display_name: "bmask-stream-pk", secret: "sk-mock", api_base: `${streamUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "bmask-stream-e2e", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: streamPk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: hash(STREAM_CALLER), allowed_models: ["bmask-stream-e2e"], }); - await admin.json("POST", "/admin/v1/guardrails", { + await seed.createGuardrail({ name: "gr-bedrock-mask", enabled: true, hook_point: "both", diff --git a/tests/e2e/src/cases/bedrock-credential-isolation-e2e.test.ts b/tests/e2e/src/cases/bedrock-credential-isolation-e2e.test.ts index 6c44af68..921150da 100644 --- a/tests/e2e/src/cases/bedrock-credential-isolation-e2e.test.ts +++ b/tests/e2e/src/cases/bedrock-credential-isolation-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import { createServer, type Server } from "node:http"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, pickFreePort, spawnApp, startOpenAiUpstream, @@ -117,19 +117,19 @@ function bedrockGuardrail( }; } -async function seedRouting(admin: AdminClient, upstream: OpenAiUpstream) { - const pk = await admin.createProviderKey({ +async function seedRouting(seed: SeedClient, upstream: OpenAiUpstream) { + const pk = await seed.createProviderKey({ display_name: "bedrock-iso-pk", secret: PROVIDER_SECRET, api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "bedrock-iso-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["bedrock-iso-model"], }); @@ -157,11 +157,12 @@ describe("bedrock guardrail credential isolation (#519 UV.3)", () => { let upstream: OpenAiUpstream | undefined; let bedrock: MockBedrock | undefined; let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let guardrailAId: string | undefined; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); bedrock = await startMockBedrock(); @@ -176,13 +177,13 @@ describe("bedrock guardrail credential isolation (#519 UV.3)", () => { AWS_REGION: "us-west-2", }, }); - admin = new AdminClient(app.adminUrl, app.adminKey); - await seedRouting(admin, upstream); - const created = (await admin.json("POST", "/admin/v1/guardrails", { + seed = new SeedClient(etcd, app.etcdPrefix); + await seedRouting(seed, upstream); + const created = (await seed.createGuardrail({ ...bedrockGuardrail("gr-bedrock-a", GUARD_A, KEY_A), })) as { id: string }; guardrailAId = created.id; - await admin.json("POST", "/admin/v1/guardrails", { + await seed.createGuardrail({ ...bedrockGuardrail("gr-bedrock-b", GUARD_B, KEY_B), }); @@ -242,9 +243,9 @@ describe("bedrock guardrail credential isolation (#519 UV.3)", () => { ctx.skip(); return; } - await admin!.json( - "PUT", - `/admin/v1/guardrails/${guardrailAId}`, + await seed!.update( + "guardrails", + guardrailAId, bedrockGuardrail("gr-bedrock-a", GUARD_A, KEY_A2), ); diff --git a/tests/e2e/src/cases/body-edges-e2e.test.ts b/tests/e2e/src/cases/body-edges-e2e.test.ts index 7076cdb4..9f1e5804 100644 --- a/tests/e2e/src/cases/body-edges-e2e.test.ts +++ b/tests/e2e/src/cases/body-edges-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -49,29 +49,30 @@ const CALLER_KEY_HASH = createHash("sha256") describe("body edges e2e: multi-turn, oversize body, empty messages", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "body-edges-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "body-edges", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["body-edges"], }); diff --git a/tests/e2e/src/cases/cache-edges-model-and-disabled-e2e.test.ts b/tests/e2e/src/cases/cache-edges-model-and-disabled-e2e.test.ts index fdc86553..a588477e 100644 --- a/tests/e2e/src/cases/cache-edges-model-and-disabled-e2e.test.ts +++ b/tests/e2e/src/cases/cache-edges-model-and-disabled-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -48,34 +48,35 @@ const SHARED_PROMPT = "cache-edges-shared-prompt"; describe("cache edges: model in fingerprint + enabled:false bypass", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); // Two distinct Models pointing at the SAME mock upstream — only // the gateway-side `display_name` differs (and the upstream // `model_name` value the proxy rewrites to). If the cache // fingerprint correctly includes the model identifier, requests // against the two Models must be cached independently. - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "cache-edges-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "cache-edges-A", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createModel({ + await seed.createModel({ display_name: "cache-edges-B", provider: "openai", model_name: "gpt-4o", @@ -83,13 +84,13 @@ describe("cache edges: model in fingerprint + enabled:false bypass", () => { }); // A third Model used only for the "policy disabled" probe so it // doesn't share the cache scope with A/B above. - await admin.createModel({ + await seed.createModel({ display_name: "cache-edges-disabled", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: [ "cache-edges-A", @@ -103,17 +104,17 @@ describe("cache edges: model in fingerprint + enabled:false bypass", () => { // enabled policies must NOT use `applies_to:"all"` — if they // did, the third Model would also match an enabled rule and // test (2) would observe caching despite the disabled policy. - await admin.json("POST", "/admin/v1/cache_policies", { + await seed.createCachePolicy({ name: "cache-edges-policy-a", enabled: true, applies_to: "model:cache-edges-A", }); - await admin.json("POST", "/admin/v1/cache_policies", { + await seed.createCachePolicy({ name: "cache-edges-policy-b", enabled: true, applies_to: "model:cache-edges-B", }); - await admin.json("POST", "/admin/v1/cache_policies", { + await seed.createCachePolicy({ name: "cache-edges-disabled-policy", enabled: false, applies_to: "model:cache-edges-disabled", diff --git a/tests/e2e/src/cases/cache-env-namespace-e2e.test.ts b/tests/e2e/src/cases/cache-env-namespace-e2e.test.ts index 11fd5be7..5179a2fd 100644 --- a/tests/e2e/src/cases/cache-env-namespace-e2e.test.ts +++ b/tests/e2e/src/cases/cache-env-namespace-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash, randomUUID } from "node:crypto"; import { connect } from "node:net"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, ProxyClient, spawnApp, startOpenAiUpstream, @@ -105,25 +105,27 @@ function chatRequest(proxyUrl: string): Promise { } /** Seed one model (→ this env's own upstream), a permissive ApiKey, and a - * redis-backed `applies_to:all` cache policy via this app's admin API. */ -async function seed(app: SpawnedApp, upstreamBase: string) { - const admin = new AdminClient(app.adminUrl, app.adminKey); - const pk = await admin.createProviderKey({ + * redis-backed `applies_to:all` cache policy, written to this env's own + * etcd scope (`/` — the root the DP reads when `env_id` + * is set). */ +async function seed(etcdRoot: string, upstreamBase: string) { + const seed = new SeedClient(new EtcdClient(), etcdRoot); + const pk = await seed.createProviderKey({ display_name: `${MODEL}-pk`, secret: "sk-mock", api_base: `${upstreamBase}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: MODEL, provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: [MODEL], }); - await admin.json("POST", "/admin/v1/cache_policies", { + await seed.createCachePolicy({ name: `${MODEL}-redis-policy`, enabled: true, backend: "redis", @@ -170,8 +172,8 @@ describe("response cache is isolated per env on a shared Redis (#788 P1-1)", () appA = await spawnApp({ extra: { etcd: envEtcd(prefix, envA), cache } }); appB = await spawnApp({ extra: { etcd: envEtcd(prefix, envB), cache } }); - await seed(appA, upstreamA.baseUrl); - await seed(appB, upstreamB.baseUrl); + await seed(`${prefix}/${envA}`, upstreamA.baseUrl); + await seed(`${prefix}/${envB}`, upstreamB.baseUrl); await waitModelLive(appA.proxyUrl); await waitModelLive(appB.proxyUrl); }); diff --git a/tests/e2e/src/cases/cache-fingerprint-collision-e2e.test.ts b/tests/e2e/src/cases/cache-fingerprint-collision-e2e.test.ts index 4a7c7cec..400e6631 100644 --- a/tests/e2e/src/cases/cache-fingerprint-collision-e2e.test.ts +++ b/tests/e2e/src/cases/cache-fingerprint-collision-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -64,36 +64,37 @@ const TOOL_DEF = { describe("cache fingerprint collision e2e: extras change → distinct cache key", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "cache-fp-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "cache-fp-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["cache-fp-model"], }); // CachePolicy with no TTL override (uses backend default, plenty // long for the test) and applies_to:all so every request runs // through the cache layer. - await admin.json("POST", "/admin/v1/cache_policies", { + await seed.createCachePolicy({ name: "cache-fp-policy", enabled: true, applies_to: "all", diff --git a/tests/e2e/src/cases/cache-hit-output-guardrail-e2e.test.ts b/tests/e2e/src/cases/cache-hit-output-guardrail-e2e.test.ts index fd44b6a0..4ea3de0f 100644 --- a/tests/e2e/src/cases/cache-hit-output-guardrail-e2e.test.ts +++ b/tests/e2e/src/cases/cache-hit-output-guardrail-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -27,28 +27,29 @@ const CACHED_PROMPT = "cache-and-guard-me"; describe("cache hit runs output guardrails (#448)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); - const pk = await admin.createProviderKey({ + seed = new SeedClient(etcd, app.etcdPrefix); + const pk = await seed.createProviderKey({ display_name: "cache-gr-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "cache-gr", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ key_hash: HASH, allowed_models: ["cache-gr"] }); - await admin.json("POST", "/admin/v1/cache_policies", { + await seed.createApiKey({ key_hash: HASH, allowed_models: ["cache-gr"] }); + await seed.createCachePolicy({ name: "cache-gr-policy", enabled: true, applies_to: "all", @@ -82,7 +83,7 @@ describe("cache hit runs output guardrails (#448)", () => { expect(first.headers.get("x-aisix-cache")).toBe("miss"); // 2) Attach an output guardrail blocking the canned reply text. - await admin!.json("POST", "/admin/v1/guardrails", { + await seed!.createGuardrail({ name: "cache-gr-output-keyword", enabled: true, hook_point: "output", diff --git a/tests/e2e/src/cases/cache-policy-backend-e2e.test.ts b/tests/e2e/src/cases/cache-policy-backend-e2e.test.ts index 17740bd0..be3d49e5 100644 --- a/tests/e2e/src/cases/cache-policy-backend-e2e.test.ts +++ b/tests/e2e/src/cases/cache-policy-backend-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash, randomUUID } from "node:crypto"; import { connect } from "node:net"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -53,7 +53,6 @@ async function redisPing(url: string): Promise { interface SeededApp { app: SpawnedApp; - admin: AdminClient; } /** @@ -75,38 +74,38 @@ async function seedApp( modelAlias: string, canaryAlias: string, ): Promise { - const admin = new AdminClient(app.adminUrl, app.adminKey); - const pk = await admin.createProviderKey({ + const seed = new SeedClient(new EtcdClient(), app.etcdPrefix); + const pk = await seed.createProviderKey({ display_name: `${modelAlias}-pk`, secret: "sk-mock", api_base: `${upstreamBase}/v1`, }); for (const alias of [modelAlias, canaryAlias]) { - await admin.createModel({ + await seed.createModel({ display_name: alias, provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); } - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: [modelAlias, canaryAlias], }); // Order matters: redis policy FIRST, canary memory policy SECOND. - await admin.json("POST", "/admin/v1/cache_policies", { + await seed.createCachePolicy({ name: `${modelAlias}-redis-policy`, enabled: true, backend: "redis", applies_to: `model:${modelAlias}`, }); - await admin.json("POST", "/admin/v1/cache_policies", { + await seed.createCachePolicy({ name: `${canaryAlias}-canary-policy`, enabled: true, backend: "memory", applies_to: `model:${canaryAlias}`, }); - return { app, admin }; + return { app }; } function chatRequest( @@ -150,7 +149,8 @@ describe("cache policy backend=redis on a memory-only DP disables caching", () = let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); diff --git a/tests/e2e/src/cases/cache-policy-e2e.test.ts b/tests/e2e/src/cases/cache-policy-e2e.test.ts index 9f4a3e08..ccf9646d 100644 --- a/tests/e2e/src/cases/cache-policy-e2e.test.ts +++ b/tests/e2e/src/cases/cache-policy-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -35,36 +35,36 @@ const CALLER_KEY_HASH = createHash("sha256") describe("cache policy e2e: identical request hits cache", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "cache-e2e-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "cache-e2e", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["cache-e2e"], }); - // CachePolicy is not exposed on AdminClient yet — call the JSON - // helper directly. The admin API requires `name`, `enabled`, and - // `applies_to` to enable a policy. - await admin.json("POST", "/admin/v1/cache_policies", { + // A policy requires `name`, `enabled`, and `applies_to` to take + // effect. + await seed.createCachePolicy({ name: "cache-e2e-policy", enabled: true, applies_to: "all", diff --git a/tests/e2e/src/cases/cache-scenarios-e2e.test.ts b/tests/e2e/src/cases/cache-scenarios-e2e.test.ts index daca701e..9d3e79fc 100644 --- a/tests/e2e/src/cases/cache-scenarios-e2e.test.ts +++ b/tests/e2e/src/cases/cache-scenarios-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -36,20 +36,21 @@ const CALLER_KEY_HASH = createHash("sha256") describe("cache scenarios e2e: different prompt → miss", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; // Each test owns its own upstream + Model so request-count assertions // are isolated across cases. const upstreams: OpenAiUpstream[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); @@ -61,7 +62,7 @@ describe("cache scenarios e2e: different prompt → miss", () => { }); test("different prompt → MISS (cache fingerprint reflects the request)", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -69,18 +70,18 @@ describe("cache scenarios e2e: different prompt → miss", () => { const upstream = await startOpenAiUpstream(); upstreams.push(upstream); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "cache-diff-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "cache-diff", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.json("POST", "/admin/v1/cache_policies", { + await seed.createCachePolicy({ name: "cache-diff-policy", enabled: true, applies_to: "all", diff --git a/tests/e2e/src/cases/cache-streaming-not-cached-e2e.test.ts b/tests/e2e/src/cases/cache-streaming-not-cached-e2e.test.ts index 86bb2a91..80566b8d 100644 --- a/tests/e2e/src/cases/cache-streaming-not-cached-e2e.test.ts +++ b/tests/e2e/src/cases/cache-streaming-not-cached-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -60,11 +60,12 @@ const SSE_EVENTS = [ describe("cache streaming bypass e2e: streaming responses are never cached", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; // Mock dispatches by config: when streamEvents is configured, @@ -82,33 +83,33 @@ describe("cache streaming bypass e2e: streaming responses are never cached", () nonStreamMock; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pkStream = await admin.createProviderKey({ + const pkStream = await seed.createProviderKey({ display_name: "stream-nocache-stream-pk", secret: "sk-mock", api_base: `${streamMock.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "stream-nocache-stream-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pkStream.id, }); - const pkNon = await admin.createProviderKey({ + const pkNon = await seed.createProviderKey({ display_name: "stream-nocache-non-pk", secret: "sk-mock", api_base: `${nonStreamMock.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "stream-nocache-non-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pkNon.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: [ "stream-nocache-stream-model", @@ -118,7 +119,7 @@ describe("cache streaming bypass e2e: streaming responses are never cached", () // Single enabled policy applies to all — both Models are in // its scope. The streaming Model should bypass it anyway; the // non-streaming Model uses it normally. - await admin.json("POST", "/admin/v1/cache_policies", { + await seed.createCachePolicy({ name: "stream-nocache-policy", enabled: true, applies_to: "all", diff --git a/tests/e2e/src/cases/cache-ttl-eviction-e2e.test.ts b/tests/e2e/src/cases/cache-ttl-eviction-e2e.test.ts index c318a5c2..cdec82c1 100644 --- a/tests/e2e/src/cases/cache-ttl-eviction-e2e.test.ts +++ b/tests/e2e/src/cases/cache-ttl-eviction-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -48,36 +48,36 @@ const POST_EXPIRY_WAIT_MS = (TTL_SECONDS + 2) * 1000; describe("cache TTL eviction e2e: entry expires after ttl_seconds", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "cache-ttl-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "cache-ttl-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["cache-ttl-model"], }); // CachePolicy with a short TTL so the test can wait it out - // synchronously instead of mocking the clock. CachePolicy is not - // exposed on the typed AdminClient yet — use the JSON helper. - await admin.json("POST", "/admin/v1/cache_policies", { + // synchronously instead of mocking the clock. + await seed.createCachePolicy({ name: "cache-ttl-policy", enabled: true, applies_to: "all", diff --git a/tests/e2e/src/cases/caller-credential-isolation-e2e.test.ts b/tests/e2e/src/cases/caller-credential-isolation-e2e.test.ts index d910fba5..923cc553 100644 --- a/tests/e2e/src/cases/caller-credential-isolation-e2e.test.ts +++ b/tests/e2e/src/cases/caller-credential-isolation-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -56,29 +56,30 @@ const PROVIDER_SECRET = "sk-mock-provider-secret"; describe("caller credential isolation e2e: caller key never reaches upstream", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "cred-isolation-pk", secret: PROVIDER_SECRET, api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "cred-isolation-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["cred-isolation-model"], }); diff --git a/tests/e2e/src/cases/canary-routing-e2e.test.ts b/tests/e2e/src/cases/canary-routing-e2e.test.ts index 01c880f6..31e471b1 100644 --- a/tests/e2e/src/cases/canary-routing-e2e.test.ts +++ b/tests/e2e/src/cases/canary-routing-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -35,17 +35,18 @@ function okBody(content: string) { describe("sticky (A/B / canary) weighted routing e2e", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; const upstreams: OpenAiUpstream[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createApiKey({ + seed = new SeedClient(etcd, app.etcdPrefix); + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); @@ -60,13 +61,13 @@ describe("sticky (A/B / canary) weighted routing e2e", () => { displayName: string, upstream: OpenAiUpstream, ): Promise { - if (!admin) throw new Error("admin client not initialized"); - const providerKey = await admin.createProviderKey({ + if (!seed) throw new Error("seed client not initialized"); + const providerKey = await seed.createProviderKey({ display_name: `${displayName}-pk`, secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: displayName, provider: "openai", model_name: "gpt-4o-mini", @@ -91,7 +92,7 @@ describe("sticky (A/B / canary) weighted routing e2e", () => { } test("pins a stability key to one target while splitting across keys", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -99,9 +100,10 @@ describe("sticky (A/B / canary) weighted routing e2e", () => { const stable = await startOpenAiUpstream({ nonStreamBody: okBody("stable-served") }); const canary = await startOpenAiUpstream({ nonStreamBody: okBody("canary-served") }); upstreams.push(stable, canary); - await createOpenAiModel("canary-stable", stable); - await createOpenAiModel("canary-new", canary); - await admin.createModel({ + // Router BEFORE its targets: watch events apply in revision order, so + // once /v1/models lists both targets the router is in the snapshot too + // (virtual models don't appear in /v1/models themselves). + await seed.createModel({ display_name: "canary-router", routing: { strategy: "weighted", @@ -112,10 +114,20 @@ describe("sticky (A/B / canary) weighted routing e2e", () => { ], }, }); + await createOpenAiModel("canary-stable", stable); + await createOpenAiModel("canary-new", canary); + // Gate on the DP snapshot via /v1/models — authenticates only once the + // caller key has propagated, lists the targets only once the snapshot + // has them, and dispatches to no target (which would skew the + // per-target counts below). await waitConfigPropagation(async () => { - const models = await admin!.listModels(); - return models.some((m) => m.display_name === "canary-router"); + const res = await fetch(`${app!.proxyUrl}/v1/models`, { + headers: { authorization: `Bearer ${CALLER_PLAINTEXT}` }, + }); + if (res.status !== 200) return false; + const ids = ((await res.json()) as { data?: Array<{ id?: string }> }).data?.map((m) => m.id) ?? []; + return ids.includes("canary-stable") && ids.includes("canary-new"); }); // Same key → same target on every request (sticky). diff --git a/tests/e2e/src/cases/chat-anthropic-stream-input-tokens-e2e.test.ts b/tests/e2e/src/cases/chat-anthropic-stream-input-tokens-e2e.test.ts index 71825f00..b05df1b2 100644 --- a/tests/e2e/src/cases/chat-anthropic-stream-input-tokens-e2e.test.ts +++ b/tests/e2e/src/cases/chat-anthropic-stream-input-tokens-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -53,25 +53,26 @@ describe("/v1/chat/completions anthropic streaming input tokens (#450)", () => { let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ streamEvents: STREAM_EVENTS, eventDelayMs: 2 }); app = await spawnApp(); - const admin = new AdminClient(app.adminUrl, app.adminKey); - const pk = await admin.createProviderKey({ + const seed = new SeedClient(etcd, app.etcdPrefix); + const pk = await seed.createProviderKey({ display_name: "chat-anth-stream-pk", provider: "anthropic", adapter: "anthropic", secret: "sk-anth-mock", api_base: upstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: "chat-anth-stream", provider: "anthropic", model_name: "claude-3-5-haiku-20241022", provider_key_id: pk.id, }); - await admin.createApiKey({ key_hash: CALLER_HASH, allowed_models: ["chat-anth-stream"] }); + await seed.createApiKey({ key_hash: CALLER_HASH, allowed_models: ["chat-anth-stream"] }); }); afterAll(async () => { diff --git a/tests/e2e/src/cases/completions-legacy-e2e.test.ts b/tests/e2e/src/cases/completions-legacy-e2e.test.ts index a1a36ad7..a5b990b2 100644 --- a/tests/e2e/src/cases/completions-legacy-e2e.test.ts +++ b/tests/e2e/src/cases/completions-legacy-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -72,31 +72,32 @@ const NON_STREAM_BODY = { describe("legacy /v1/completions e2e: text-in / text-out passthrough", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ nonStreamBody: NON_STREAM_BODY, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "completions-legacy-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "completions-legacy-model", provider: "openai", model_name: "gpt-3.5-turbo-instruct", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["completions-legacy-model"], }); diff --git a/tests/e2e/src/cases/completions-output-guardrail-e2e.test.ts b/tests/e2e/src/cases/completions-output-guardrail-e2e.test.ts index 9d1983e3..cce71523 100644 --- a/tests/e2e/src/cases/completions-output-guardrail-e2e.test.ts +++ b/tests/e2e/src/cases/completions-output-guardrail-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -30,11 +30,12 @@ const GUARDRAIL_NAME = "cmpl-out-gr-keyword"; describe("completions output guardrail (#911 [23])", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; // Legacy /v1/completions response shape, carrying the forbidden word in @@ -59,26 +60,26 @@ describe("completions output guardrail (#911 [23])", () => { }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "cmpl-out-gr-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "cmpl-out-gr", provider: "openai", model_name: "gpt-3.5-turbo-instruct", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["cmpl-out-gr"], }); // Output keyword guardrail (env-wide) — runs against the completion text // after the upstream call returns, before relay to the caller. - await admin.json("POST", "/admin/v1/guardrails", { + await seed.createGuardrail({ name: GUARDRAIL_NAME, enabled: true, hook_point: "output", diff --git a/tests/e2e/src/cases/completions-tpm-commit-e2e.test.ts b/tests/e2e/src/cases/completions-tpm-commit-e2e.test.ts index 523a0368..af0043d0 100644 --- a/tests/e2e/src/cases/completions-tpm-commit-e2e.test.ts +++ b/tests/e2e/src/cases/completions-tpm-commit-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, ProxyClient, spawnApp, startOpenAiUpstream, @@ -39,23 +39,24 @@ const COMPLETION_BODY = { describe("completions TPM commit (#911 [21])", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ nonStreamBody: COMPLETION_BODY }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "tpm-commit-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "tpm-commit", provider: "openai", model_name: "gpt-3.5-turbo-instruct", @@ -63,7 +64,7 @@ describe("completions TPM commit (#911 [21])", () => { }); // TPM=10 on the caller's key. The first /v1/completions call commits 16 // tokens (> 10), so the second must be rejected on the token counter. - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["tpm-commit"], rate_limit: { tpm: TPM }, diff --git a/tests/e2e/src/cases/concurrency-e2e.test.ts b/tests/e2e/src/cases/concurrency-e2e.test.ts index ea123b0f..2957aa82 100644 --- a/tests/e2e/src/cases/concurrency-e2e.test.ts +++ b/tests/e2e/src/cases/concurrency-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, ProxyClient, spawnApp, startOpenAiUpstream, @@ -60,16 +60,17 @@ const CALLER_C_KEY_HASH = createHash("sha256") describe("concurrency e2e: rate-limit isolation across callers", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; const upstreams: OpenAiUpstream[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); }); afterAll(async () => { @@ -78,7 +79,7 @@ describe("concurrency e2e: rate-limit isolation across callers", () => { }); test("inter-caller rate-limit isolation: caller A's RPM exhaustion does NOT affect caller B", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -86,12 +87,12 @@ describe("concurrency e2e: rate-limit isolation across callers", () => { const upstream = await startOpenAiUpstream(); upstreams.push(upstream); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "conc-iso-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "conc-iso", provider: "openai", model_name: "gpt-4o-mini", @@ -99,12 +100,12 @@ describe("concurrency e2e: rate-limit isolation across callers", () => { }); // Both callers have RPM=1; the cap is per-ApiKey, so caller // A burning their slot must NOT consume caller B's slot. - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_A_KEY_HASH, allowed_models: ["conc-iso"], rate_limit: { rpm: 1 }, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_B_KEY_HASH, allowed_models: ["conc-iso"], rate_limit: { rpm: 1 }, @@ -209,7 +210,7 @@ describe("concurrency e2e: rate-limit isolation across callers", () => { // A and B are both currently rate-limited. This is the // load-bearing isolation assertion: a fresh caller's quota is // genuinely independent of other callers'. - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_C_KEY_HASH, allowed_models: ["conc-iso"], rate_limit: { rpm: 1 }, diff --git a/tests/e2e/src/cases/cooldown-contract-e2e.test.ts b/tests/e2e/src/cases/cooldown-contract-e2e.test.ts index c6e550ae..342351a5 100644 --- a/tests/e2e/src/cases/cooldown-contract-e2e.test.ts +++ b/tests/e2e/src/cases/cooldown-contract-e2e.test.ts @@ -4,6 +4,7 @@ import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -39,13 +40,15 @@ const CALLER_KEY_HASH = createHash("sha256") describe("cooldown contract (H1) — 401 cools down despite being non-retryable", () => { let app: SpawnedApp | undefined; let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; let authFailUpstream: OpenAiUpstream | undefined; let stableUpstream: OpenAiUpstream | undefined; let authFailModelID = ""; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; authFailUpstream = await startOpenAiUpstream({ @@ -73,33 +76,34 @@ describe("cooldown contract (H1) — 401 cools down despite being non-retryable" app = await spawnApp(); admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const failPk = await admin.createProviderKey({ + const failPk = await seed.createProviderKey({ display_name: "h1-401-pk", secret: "sk-mock", api_base: `${authFailUpstream.baseUrl}/v1`, }); - const stablePk = await admin.createProviderKey({ + const stablePk = await seed.createProviderKey({ display_name: "h1-stable-pk", secret: "sk-mock", api_base: `${stableUpstream.baseUrl}/v1`, }); authFailModelID = ( - await admin.createModel({ + await seed.createModel({ display_name: "h1-401-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: failPk.id, }) ).id; - await admin.createModel({ + await seed.createModel({ display_name: "h1-stable-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: stablePk.id, }); - await admin.createModel({ + await seed.createModel({ display_name: "h1-router", routing: { strategy: "failover", @@ -107,7 +111,7 @@ describe("cooldown contract (H1) — 401 cools down despite being non-retryable" max_fallbacks: 1, }, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["h1-router", "h1-stable-model"], }); @@ -187,13 +191,15 @@ describe("cooldown contract (H1) — 401 cools down despite being non-retryable" describe("cooldown contract (M1) — 429 cools down even when retry_on_429=false", () => { let app: SpawnedApp | undefined; let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; let rateLimitedUpstream: OpenAiUpstream | undefined; let stableUpstream: OpenAiUpstream | undefined; let rateLimitedModelID = ""; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; rateLimitedUpstream = await startOpenAiUpstream({ @@ -221,27 +227,28 @@ describe("cooldown contract (M1) — 429 cools down even when retry_on_429=false app = await spawnApp(); admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const rateLimitedPk = await admin.createProviderKey({ + const rateLimitedPk = await seed.createProviderKey({ display_name: "m1-429-pk", secret: "sk-mock", api_base: `${rateLimitedUpstream.baseUrl}/v1`, }); - const stablePk = await admin.createProviderKey({ + const stablePk = await seed.createProviderKey({ display_name: "m1-stable-pk", secret: "sk-mock", api_base: `${stableUpstream.baseUrl}/v1`, }); rateLimitedModelID = ( - await admin.createModel({ + await seed.createModel({ display_name: "m1-429-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: rateLimitedPk.id, }) ).id; - await admin.createModel({ + await seed.createModel({ display_name: "m1-stable-model", provider: "openai", model_name: "gpt-4o-mini", @@ -251,7 +258,7 @@ describe("cooldown contract (M1) — 429 cools down even when retry_on_429=false // behavior would NOT cooldown a 429 in this configuration; the // M1 fix decouples cooldown from retry so the 429 must still // mark the target for backoff. - await admin.createModel({ + await seed.createModel({ display_name: "m1-router", routing: { strategy: "failover", @@ -259,7 +266,7 @@ describe("cooldown contract (M1) — 429 cools down even when retry_on_429=false max_fallbacks: 1, }, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["m1-router", "m1-stable-model"], }); @@ -335,13 +342,15 @@ describe("cooldown contract (M1) — 429 cools down even when retry_on_429=false describe("cooldown contract (H2) — Retry-After header from upstream drives TTL", () => { let app: SpawnedApp | undefined; let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; let upstream: OpenAiUpstream | undefined; let stableUpstream: OpenAiUpstream | undefined; let rateLimitedModelID = ""; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ @@ -371,33 +380,34 @@ describe("cooldown contract (H2) — Retry-After header from upstream drives TTL app = await spawnApp(); admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const upstreamPk = await admin.createProviderKey({ + const upstreamPk = await seed.createProviderKey({ display_name: "h2-upstream-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - const stablePk = await admin.createProviderKey({ + const stablePk = await seed.createProviderKey({ display_name: "h2-stable-pk", secret: "sk-mock", api_base: `${stableUpstream.baseUrl}/v1`, }); rateLimitedModelID = ( - await admin.createModel({ + await seed.createModel({ display_name: "h2-429-with-retry-after", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: upstreamPk.id, }) ).id; - await admin.createModel({ + await seed.createModel({ display_name: "h2-stable-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: stablePk.id, }); - await admin.createModel({ + await seed.createModel({ display_name: "h2-router", routing: { strategy: "failover", @@ -408,7 +418,7 @@ describe("cooldown contract (H2) — Retry-After header from upstream drives TTL max_fallbacks: 1, }, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["h2-router", "h2-stable-model"], }); @@ -487,12 +497,14 @@ describe("cooldown contract (H2) — Retry-After header from upstream drives TTL describe("filter contract (H3) — all candidates unhealthy returns 503", () => { let app: SpawnedApp | undefined; let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; let downAUpstream: OpenAiUpstream | undefined; let downBUpstream: OpenAiUpstream | undefined; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; downAUpstream = await startOpenAiUpstream({ @@ -506,19 +518,20 @@ describe("filter contract (H3) — all candidates unhealthy returns 503", () => app = await spawnApp(); admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const aPk = await admin.createProviderKey({ + const aPk = await seed.createProviderKey({ display_name: "h3-down-a-pk", secret: "sk-mock", api_base: `${downAUpstream.baseUrl}/v1`, }); - const bPk = await admin.createProviderKey({ + const bPk = await seed.createProviderKey({ display_name: "h3-down-b-pk", secret: "sk-mock", api_base: `${downBUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "h3-down-a", provider: "openai", model_name: "gpt-4o-mini", @@ -533,7 +546,7 @@ describe("filter contract (H3) — all candidates unhealthy returns 503", () => stale_after_seconds: 120, }, }); - await admin.createModel({ + await seed.createModel({ display_name: "h3-down-b", provider: "openai", model_name: "gpt-4o-mini", @@ -550,7 +563,7 @@ describe("filter contract (H3) — all candidates unhealthy returns 503", () => }); // Default when_all_unavailable policy is "fail" — explicit here for // clarity even though it's the default. - await admin.createModel({ + await seed.createModel({ display_name: "h3-router-fail", routing: { strategy: "failover", @@ -559,7 +572,7 @@ describe("filter contract (H3) — all candidates unhealthy returns 503", () => when_all_unavailable: "fail", }, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["h3-router-fail"], }); @@ -619,11 +632,13 @@ describe("filter contract (H3) — all candidates unhealthy returns 503", () => describe("filter contract (H3 escape hatch) — try_anyway sends to known-bad", () => { let app: SpawnedApp | undefined; let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; let downUpstream: OpenAiUpstream | undefined; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; downUpstream = await startOpenAiUpstream({ @@ -633,14 +648,15 @@ describe("filter contract (H3 escape hatch) — try_anyway sends to known-bad", app = await spawnApp(); admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "h3-escape-pk", secret: "sk-mock", api_base: `${downUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "h3-escape-down", provider: "openai", model_name: "gpt-4o-mini", @@ -655,7 +671,7 @@ describe("filter contract (H3 escape hatch) — try_anyway sends to known-bad", stale_after_seconds: 120, }, }); - await admin.createModel({ + await seed.createModel({ display_name: "h3-router-escape", routing: { strategy: "failover", @@ -664,7 +680,7 @@ describe("filter contract (H3 escape hatch) — try_anyway sends to known-bad", when_all_unavailable: "try_anyway", }, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["h3-router-escape"], }); @@ -715,13 +731,15 @@ describe("filter contract (H3 escape hatch) — try_anyway sends to known-bad", describe("cooldown observability — a cooldown transition emits aisix_deployment_* metrics", () => { let app: SpawnedApp | undefined; let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; let failUpstream: OpenAiUpstream | undefined; let stableUpstream: OpenAiUpstream | undefined; let failModelID = ""; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; // 500 is a default cooldown trigger status → the failing target @@ -749,33 +767,34 @@ describe("cooldown observability — a cooldown transition emits aisix_deploymen app = await spawnApp(); admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const failPk = await admin.createProviderKey({ + const failPk = await seed.createProviderKey({ display_name: "cd-metrics-fail-pk", secret: "sk-mock", api_base: `${failUpstream.baseUrl}/v1`, }); - const stablePk = await admin.createProviderKey({ + const stablePk = await seed.createProviderKey({ display_name: "cd-metrics-stable-pk", secret: "sk-mock", api_base: `${stableUpstream.baseUrl}/v1`, }); failModelID = ( - await admin.createModel({ + await seed.createModel({ display_name: "cd-metrics-500-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: failPk.id, }) ).id; - await admin.createModel({ + await seed.createModel({ display_name: "cd-metrics-stable-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: stablePk.id, }); - await admin.createModel({ + await seed.createModel({ display_name: "cd-metrics-router", routing: { strategy: "failover", @@ -786,7 +805,7 @@ describe("cooldown observability — a cooldown transition emits aisix_deploymen max_fallbacks: 1, }, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["cd-metrics-router", "cd-metrics-stable-model"], }); @@ -884,6 +903,7 @@ async function scrapeDeploymentState( describe("cooldown observability — the state gauge follows a target back into rotation", () => { let app: SpawnedApp | undefined; let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; let flakyUpstream: OpenAiUpstream | undefined; let stableUpstream: OpenAiUpstream | undefined; @@ -892,7 +912,8 @@ describe("cooldown observability — the state gauge follows a target back into const RECOVERED_REPLY = "recovered after cooldown"; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; // First request 500s (trips the cooldown); every later request @@ -939,20 +960,21 @@ describe("cooldown observability — the state gauge follows a target back into app = await spawnApp(); admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const flakyPk = await admin.createProviderKey({ + const flakyPk = await seed.createProviderKey({ display_name: "cd-recover-flaky-pk", secret: "sk-mock", api_base: `${flakyUpstream.baseUrl}/v1`, }); - const stablePk = await admin.createProviderKey({ + const stablePk = await seed.createProviderKey({ display_name: "cd-recover-stable-pk", secret: "sk-mock", api_base: `${stableUpstream.baseUrl}/v1`, }); flakyModelID = ( - await admin.createModel({ + await seed.createModel({ display_name: "cd-recover-flaky-model", provider: "openai", model_name: "gpt-4o-mini", @@ -962,13 +984,13 @@ describe("cooldown observability — the state gauge follows a target back into cooldown: { default_seconds: 1 }, }) ).id; - await admin.createModel({ + await seed.createModel({ display_name: "cd-recover-stable-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: stablePk.id, }); - await admin.createModel({ + await seed.createModel({ display_name: "cd-recover-router", routing: { strategy: "failover", @@ -979,7 +1001,7 @@ describe("cooldown observability — the state gauge follows a target back into max_fallbacks: 1, }, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: [ "cd-recover-router", diff --git a/tests/e2e/src/cases/cost-aware-routing-e2e.test.ts b/tests/e2e/src/cases/cost-aware-routing-e2e.test.ts index d3cffd0f..796807c0 100644 --- a/tests/e2e/src/cases/cost-aware-routing-e2e.test.ts +++ b/tests/e2e/src/cases/cost-aware-routing-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -35,17 +35,18 @@ function okBody(content: string) { describe("cost-aware (least_cost) routing e2e", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; const upstreams: OpenAiUpstream[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createApiKey({ + seed = new SeedClient(etcd, app.etcdPrefix); + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); @@ -61,13 +62,13 @@ describe("cost-aware (least_cost) routing e2e", () => { upstream: OpenAiUpstream, extra: Record = {}, ): Promise { - if (!admin) throw new Error("admin client not initialized"); - const providerKey = await admin.createProviderKey({ + if (!seed) throw new Error("seed client not initialized"); + const providerKey = await seed.createProviderKey({ display_name: `${displayName}-pk`, secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: displayName, provider: "openai", model_name: "gpt-4o-mini", @@ -85,7 +86,7 @@ describe("cost-aware (least_cost) routing e2e", () => { } test("ranks the cheapest target first regardless of declaration order", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -94,31 +95,37 @@ describe("cost-aware (least_cost) routing e2e", () => { const pricey = await startOpenAiUpstream({ nonStreamBody: okBody("pricey-served") }); upstreams.push(cheap, pricey); - // Cheap total unit price = 0.2/1K; pricey = 20/1K. - await createOpenAiModel("cost-cheap", cheap, { - cost: { input_per_1k: 0.1, output_per_1k: 0.1 }, - }); - await createOpenAiModel("cost-pricey", pricey, { - cost: { input_per_1k: 10, output_per_1k: 10 }, - }); // Declare the expensive target FIRST — least_cost must reorder by price, - // not honor declaration order. - await admin.createModel({ + // not honor declaration order. The router document is written BEFORE its + // targets: watch events apply in revision order, so once both targets are + // visible the router is in the snapshot too (virtual models don't appear + // in /v1/models themselves). + await seed.createModel({ display_name: "cost-virtual", routing: { strategy: "least_cost", targets: [{ model: "cost-pricey" }, { model: "cost-cheap" }], }, }); + // Cheap total unit price = 0.2/1K; pricey = 20/1K. + await createOpenAiModel("cost-cheap", cheap, { + cost: { input_per_1k: 0.1, output_per_1k: 0.1 }, + }); + await createOpenAiModel("cost-pricey", pricey, { + cost: { input_per_1k: 10, output_per_1k: 10 }, + }); - // Gate on the routing model reaching the DP snapshot (probe would be - // fine here since both targets are healthy, but listModels avoids any - // per-target request skew before baselines are taken). listModels reads - // etcd directly and shouldn't fail during propagation — let a genuine - // admin failure surface instead of masking it as a 30s timeout. + // Gate on the DP snapshot via /v1/models — it only authenticates once + // the caller key has propagated and only lists the targets once the + // snapshot has them, without dispatching to a target (which would skew + // the per-target counts below). await waitConfigPropagation(async () => { - const models = await admin!.listModels(); - return models.some((m) => m.display_name === "cost-virtual"); + const res = await fetch(`${app!.proxyUrl}/v1/models`, { + headers: { authorization: `Bearer ${CALLER_PLAINTEXT}` }, + }); + if (res.status !== 200) return false; + const ids = ((await res.json()) as { data?: Array<{ id?: string }> }).data?.map((m) => m.id) ?? []; + return ids.includes("cost-cheap") && ids.includes("cost-pricey"); }); const cheapBaseline = cheap.receivedRequests.length; @@ -135,7 +142,7 @@ describe("cost-aware (least_cost) routing e2e", () => { }); test("falls forward to the next-cheapest when the cheapest fails", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -148,6 +155,18 @@ describe("cost-aware (least_cost) routing e2e", () => { const pricey = await startOpenAiUpstream({ nonStreamBody: okBody("pricey-served") }); upstreams.push(cheap, mid, pricey); + // Router BEFORE targets (revision-order gate, as above). + await seed.createModel({ + display_name: "cost-ff-virtual", + routing: { + strategy: "least_cost", + targets: [ + { model: "cost-ff-pricey" }, + { model: "cost-ff-mid" }, + { model: "cost-ff-cheap" }, + ], + }, + }); // Keep the failing cheapest in rotation so the assertion sees it attempted // (cooldown would take it out after the first 503). await createOpenAiModel("cost-ff-cheap", cheap, { @@ -160,21 +179,15 @@ describe("cost-aware (least_cost) routing e2e", () => { await createOpenAiModel("cost-ff-pricey", pricey, { cost: { input_per_1k: 10, output_per_1k: 10 }, }); - await admin.createModel({ - display_name: "cost-ff-virtual", - routing: { - strategy: "least_cost", - targets: [ - { model: "cost-ff-pricey" }, - { model: "cost-ff-mid" }, - { model: "cost-ff-cheap" }, - ], - }, - }); + // Same DP-snapshot gate as above — upstream-neutral readiness probe. await waitConfigPropagation(async () => { - const models = await admin!.listModels(); - return models.some((m) => m.display_name === "cost-ff-virtual"); + const res = await fetch(`${app!.proxyUrl}/v1/models`, { + headers: { authorization: `Bearer ${CALLER_PLAINTEXT}` }, + }); + if (res.status !== 200) return false; + const ids = ((await res.json()) as { data?: Array<{ id?: string }> }).data?.map((m) => m.id) ?? []; + return ["cost-ff-cheap", "cost-ff-mid", "cost-ff-pricey"].every((id) => ids.includes(id)); }); const cheapBaseline = cheap.receivedRequests.length; diff --git a/tests/e2e/src/cases/cross-provider-matrix-e2e.test.ts b/tests/e2e/src/cases/cross-provider-matrix-e2e.test.ts index 8f84ab1d..cf00b66c 100644 --- a/tests/e2e/src/cases/cross-provider-matrix-e2e.test.ts +++ b/tests/e2e/src/cases/cross-provider-matrix-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -67,23 +67,24 @@ const CASES: ReadonlyArray = [ describe("cross-provider matrix: OpenAI-compat upstreams", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; // Each test creates its own upstream mock (per provider × streaming // variant). Track them so `afterAll` can close every one. const upstreams: OpenAiUpstream[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); // Wildcard `allowed_models: ["*"]` lets the same caller key reach // every model the per-test setup creates, without re-creating the // ApiKey per case. - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); @@ -96,7 +97,7 @@ describe("cross-provider matrix: OpenAI-compat upstreams", () => { for (const tc of CASES) { test(`${tc.provider} upstream — non-streaming`, async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -126,7 +127,7 @@ describe("cross-provider matrix: OpenAI-compat upstreams", () => { }); upstreams.push(upstream); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: `${tc.displayPrefix}-pk-non-stream`, secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, @@ -141,7 +142,7 @@ describe("cross-provider matrix: OpenAI-compat upstreams", () => { adapter: "openai", }); const modelName = `${tc.displayPrefix}-non-stream`; - await admin.createModel({ + await seed.createModel({ display_name: modelName, provider: tc.provider, model_name: tc.upstreamModelId, @@ -211,7 +212,7 @@ describe("cross-provider matrix: OpenAI-compat upstreams", () => { }); test(`${tc.provider} upstream — streaming`, async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -228,7 +229,7 @@ describe("cross-provider matrix: OpenAI-compat upstreams", () => { const upstream = await startOpenAiUpstream({ streamEvents: sseEvents }); upstreams.push(upstream); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: `${tc.displayPrefix}-pk-stream`, secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, @@ -238,7 +239,7 @@ describe("cross-provider matrix: OpenAI-compat upstreams", () => { adapter: "openai", }); const modelName = `${tc.displayPrefix}-stream`; - await admin.createModel({ + await seed.createModel({ display_name: modelName, provider: tc.provider, model_name: tc.upstreamModelId, diff --git a/tests/e2e/src/cases/datadog-exporter-e2e.test.ts b/tests/e2e/src/cases/datadog-exporter-e2e.test.ts index 1d4fba55..16647d4a 100644 --- a/tests/e2e/src/cases/datadog-exporter-e2e.test.ts +++ b/tests/e2e/src/cases/datadog-exporter-e2e.test.ts @@ -232,6 +232,7 @@ describe("datadog exporter e2e (#688): DP delivers a gzip JSON intake to Datadog }, }); apps.push(app); + // Deliberately seeds via the Admin API: deprecation-window coverage. const admin = new AdminClient(app.adminUrl, app.adminKey); await admin.createObservabilityExporter({ name: "mock-datadog", diff --git a/tests/e2e/src/cases/deepseek-reasoning-content-e2e.test.ts b/tests/e2e/src/cases/deepseek-reasoning-content-e2e.test.ts index 459f0a8e..455ebf2d 100644 --- a/tests/e2e/src/cases/deepseek-reasoning-content-e2e.test.ts +++ b/tests/e2e/src/cases/deepseek-reasoning-content-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -40,11 +40,12 @@ const FINAL_ANSWER = "The answer is 42."; describe("deepseek reasoning_content passthrough on /v1/chat/completions (#466)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; // DeepSeek-reasoner non-streaming response: message carries both @@ -71,23 +72,23 @@ describe("deepseek reasoning_content passthrough on /v1/chat/completions (#466)" }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); // DeepSeek dispatches through the OpenAI-compat family bridge. - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "ds-reasoning-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, provider: "deepseek", adapter: "openai", }); - await admin.createModel({ + await seed.createModel({ display_name: "ds-reasoner", provider: "deepseek", model_name: "deepseek-reasoner", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["ds-reasoner"], }); diff --git a/tests/e2e/src/cases/embeddings-e2e.test.ts b/tests/e2e/src/cases/embeddings-e2e.test.ts index 89ac213c..0d81039c 100644 --- a/tests/e2e/src/cases/embeddings-e2e.test.ts +++ b/tests/e2e/src/cases/embeddings-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -60,17 +60,18 @@ const VEC_FOO = [0.01, 0.02, 0.03, 0.04, 0.05]; describe("embeddings e2e: /v1/embeddings dispatch + response passthrough", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; const upstreams: OpenAiUpstream[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createApiKey({ + seed = new SeedClient(etcd, app.etcdPrefix); + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); @@ -82,7 +83,7 @@ describe("embeddings e2e: /v1/embeddings dispatch + response passthrough", () => }); test("array input: N embeddings returned in the SAME ORDER as input array", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -107,12 +108,12 @@ describe("embeddings e2e: /v1/embeddings dispatch + response passthrough", () => }); upstreams.push(upstream); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "emb-array-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "emb-array", provider: "openai", model_name: "text-embedding-3-small", @@ -184,7 +185,7 @@ describe("embeddings e2e: /v1/embeddings dispatch + response passthrough", () => }); test("single-string input reaches upstream as a string (#162: docs §4.4 'both pass through')", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -199,12 +200,12 @@ describe("embeddings e2e: /v1/embeddings dispatch + response passthrough", () => }); upstreams.push(upstream); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "emb-single-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "emb-single", provider: "openai", model_name: "text-embedding-3-small", diff --git a/tests/e2e/src/cases/embeddings-missing-prompt-tokens-e2e.test.ts b/tests/e2e/src/cases/embeddings-missing-prompt-tokens-e2e.test.ts index 201b46fc..da37152b 100644 --- a/tests/e2e/src/cases/embeddings-missing-prompt-tokens-e2e.test.ts +++ b/tests/e2e/src/cases/embeddings-missing-prompt-tokens-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -41,11 +41,12 @@ const JINA_SHAPE_RESPONSE = { describe("OpenAI /v1/embeddings tolerates missing usage.prompt_tokens (#474)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ @@ -73,20 +74,20 @@ describe("OpenAI /v1/embeddings tolerates missing usage.prompt_tokens (#474)", ( nonStreamBody: JINA_SHAPE_RESPONSE, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "jina-embed-pk", secret: "sk-jina-mock", api_base: upstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: "jina-embed", provider: "openai", model_name: "jina-embeddings-v5-text-small", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["jina-embed"], }); diff --git a/tests/e2e/src/cases/embeddings-native-providers-e2e.test.ts b/tests/e2e/src/cases/embeddings-native-providers-e2e.test.ts index 994725d5..be141cdf 100644 --- a/tests/e2e/src/cases/embeddings-native-providers-e2e.test.ts +++ b/tests/e2e/src/cases/embeddings-native-providers-e2e.test.ts @@ -3,9 +3,10 @@ import { createServer, type Server } from "node:http"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, + waitConfigPropagation, type SpawnedApp, } from "../harness/index.js"; @@ -74,18 +75,19 @@ async function startJsonUpstream( describe("native-provider embeddings e2e (#723)", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; const upstreams: RecordingUpstream[] = []; let client: OpenAI; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createApiKey({ + seed = new SeedClient(etcd, app.etcdPrefix); + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); @@ -101,7 +103,7 @@ describe("native-provider embeddings e2e (#723)", () => { }); test("vertex/gemini model embeds via :predict with OAuth bearer + summed usage", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -114,7 +116,7 @@ describe("native-provider embeddings e2e (#723)", () => { })); upstreams.push(upstream); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "vertex-embed-pk", provider: "google", adapter: "vertex", @@ -125,13 +127,25 @@ describe("native-provider embeddings e2e (#723)", () => { }), api_base: upstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: "vertex-embed-model", provider: "google", model_name: "text-embedding-005", provider_key_id: pk.id, }); + // Gate on the DP snapshot via /v1/models — authenticates only once the + // caller key has propagated, lists the model only once the snapshot has + // it, and touches no upstream. + await waitConfigPropagation(async () => { + const res = await fetch(`${app!.proxyUrl}/v1/models`, { + headers: { authorization: `Bearer ${CALLER_PLAINTEXT}` }, + }); + if (res.status !== 200) return false; + const body = (await res.json()) as { data?: Array<{ id?: string }> }; + return (body.data ?? []).some((m) => m.id === "vertex-embed-model"); + }); + const resp = await client.embeddings.create({ model: "vertex-embed-model", input: ["hello", "world"], @@ -151,7 +165,7 @@ describe("native-provider embeddings e2e (#723)", () => { }); test("bedrock titan model embeds via one SigV4 InvokeModel per input", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -163,7 +177,7 @@ describe("native-provider embeddings e2e (#723)", () => { }); upstreams.push(upstream); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "bedrock-embed-pk", provider: "bedrock", adapter: "bedrock", @@ -174,13 +188,23 @@ describe("native-provider embeddings e2e (#723)", () => { }), api_base: upstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: "bedrock-embed-model", provider: "bedrock", model_name: "amazon.titan-embed-text-v1", provider_key_id: pk.id, }); + // Same DP-snapshot gate as the vertex case above. + await waitConfigPropagation(async () => { + const res = await fetch(`${app!.proxyUrl}/v1/models`, { + headers: { authorization: `Bearer ${CALLER_PLAINTEXT}` }, + }); + if (res.status !== 200) return false; + const body = (await res.json()) as { data?: Array<{ id?: string }> }; + return (body.data ?? []).some((m) => m.id === "bedrock-embed-model"); + }); + const resp = await client.embeddings.create({ model: "bedrock-embed-model", input: ["a", "b"], diff --git a/tests/e2e/src/cases/error-envelope-normalization-e2e.test.ts b/tests/e2e/src/cases/error-envelope-normalization-e2e.test.ts index 40892c3d..8f08c96e 100644 --- a/tests/e2e/src/cases/error-envelope-normalization-e2e.test.ts +++ b/tests/e2e/src/cases/error-envelope-normalization-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -143,17 +143,18 @@ const CASES: ReadonlyArray = [ describe("error envelope normalization e2e: provider-native 4xx → OpenAI-shape envelope", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; const upstreams: OpenAiUpstream[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createApiKey({ + seed = new SeedClient(etcd, app.etcdPrefix); + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); @@ -166,7 +167,7 @@ describe("error envelope normalization e2e: provider-native 4xx → OpenAI-shape for (const tc of CASES) { test(`${tc.provider} upstream 400 → caller sees OpenAI-shape error envelope with preserved status + reason`, async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -182,7 +183,7 @@ describe("error envelope normalization e2e: provider-native 4xx → OpenAI-shape }); upstreams.push(upstream); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: `${tc.displayName}-pk`, secret: "sk-mock", api_base: `${upstream.baseUrl}${tc.apiBaseSuffix}`, @@ -194,7 +195,7 @@ describe("error envelope normalization e2e: provider-native 4xx → OpenAI-shape provider: tc.provider, adapter: tc.adapter, }); - await admin.createModel({ + await seed.createModel({ display_name: tc.displayName, provider: tc.provider, model_name: tc.upstreamModelId, diff --git a/tests/e2e/src/cases/fallback-e2e.test.ts b/tests/e2e/src/cases/fallback-e2e.test.ts index a0407149..8554ec25 100644 --- a/tests/e2e/src/cases/fallback-e2e.test.ts +++ b/tests/e2e/src/cases/fallback-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -30,11 +30,12 @@ describe("fallback e2e: virtual routing fails over from 5xx to next target", () let app: SpawnedApp | undefined; let badUpstream: OpenAiUpstream | undefined; let goodUpstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; badUpstream = await startOpenAiUpstream({ @@ -59,19 +60,19 @@ describe("fallback e2e: virtual routing fails over from 5xx to next target", () }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const badPk = await admin.createProviderKey({ + const badPk = await seed.createProviderKey({ display_name: "fb-bad-pk", secret: "sk-mock", api_base: `${badUpstream.baseUrl}/v1`, }); - const goodPk = await admin.createProviderKey({ + const goodPk = await seed.createProviderKey({ display_name: "fb-good-pk", secret: "sk-mock", api_base: `${goodUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "fb-bad", provider: "openai", model_name: "gpt-4o-mini", @@ -84,7 +85,7 @@ describe("fallback e2e: virtual routing fails over from 5xx to next target", () // so the test exercises only its target behavior. cooldown: { enabled: false }, }); - await admin.createModel({ + await seed.createModel({ display_name: "fb-good", provider: "openai", model_name: "gpt-4o-mini", @@ -95,7 +96,7 @@ describe("fallback e2e: virtual routing fails over from 5xx to next target", () // provider/model_name/provider_key_id) or carries those three // (direct upstream — no routing). `failover` is the default // strategy; making it explicit keeps the test self-documenting. - await admin.createModel({ + await seed.createModel({ display_name: "fb-virtual", routing: { strategy: "failover", @@ -106,7 +107,7 @@ describe("fallback e2e: virtual routing fails over from 5xx to next target", () // hit a single-target Model directly, instead of probing through // `fb-virtual` (which fires the bad→good fallback on every retry // and risks off-by-one when an iteration partially succeeds). - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["fb-virtual", "fb-good"], }); diff --git a/tests/e2e/src/cases/fallback-edges-e2e.test.ts b/tests/e2e/src/cases/fallback-edges-e2e.test.ts index 0e39326f..86aed6d3 100644 --- a/tests/e2e/src/cases/fallback-edges-e2e.test.ts +++ b/tests/e2e/src/cases/fallback-edges-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -37,22 +37,23 @@ const CALLER_KEY_HASH = createHash("sha256") describe("fallback edge cases e2e", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; // Each test sets up its own pair/triple of mock upstreams so the // request-count assertions stay isolated across cases. const upstreams: OpenAiUpstream[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); // Wildcard ApiKey lets the same caller reach every per-test // virtual model without re-issuing keys. - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); @@ -64,7 +65,7 @@ describe("fallback edge cases e2e", () => { }); test("non-retryable 4xx (400) does NOT trigger fallback — caller sees the 4xx", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -104,29 +105,29 @@ describe("fallback edge cases e2e", () => { }); upstreams.push(badRequestUpstream, goodUpstream); - const badPk = await admin.createProviderKey({ + const badPk = await seed.createProviderKey({ display_name: "fb-edges-400-pk", secret: "sk-mock", api_base: `${badRequestUpstream.baseUrl}/v1`, }); - const goodPk = await admin.createProviderKey({ + const goodPk = await seed.createProviderKey({ display_name: "fb-edges-400-good-pk", secret: "sk-mock", api_base: `${goodUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "fb-edges-400-bad", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: badPk.id, }); - await admin.createModel({ + await seed.createModel({ display_name: "fb-edges-400-good", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: goodPk.id, }); - await admin.createModel({ + await seed.createModel({ display_name: "fb-edges-400-virtual", routing: { strategy: "failover", @@ -213,7 +214,7 @@ describe("fallback edge cases e2e", () => { }); test("all targets fail → caller sees a clean error envelope, not hang", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -232,29 +233,29 @@ describe("fallback edge cases e2e", () => { }); upstreams.push(bad1, bad2); - const pk1 = await admin.createProviderKey({ + const pk1 = await seed.createProviderKey({ display_name: "fb-edges-allfail-1-pk", secret: "sk-mock", api_base: `${bad1.baseUrl}/v1`, }); - const pk2 = await admin.createProviderKey({ + const pk2 = await seed.createProviderKey({ display_name: "fb-edges-allfail-2-pk", secret: "sk-mock", api_base: `${bad2.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "fb-edges-allfail-1", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk1.id, }); - await admin.createModel({ + await seed.createModel({ display_name: "fb-edges-allfail-2", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk2.id, }); - await admin.createModel({ + await seed.createModel({ display_name: "fb-edges-allfail-virtual", routing: { strategy: "failover", diff --git a/tests/e2e/src/cases/fallback-on-statuses-e2e.test.ts b/tests/e2e/src/cases/fallback-on-statuses-e2e.test.ts index 00ebdd03..83fd39ae 100644 --- a/tests/e2e/src/cases/fallback-on-statuses-e2e.test.ts +++ b/tests/e2e/src/cases/fallback-on-statuses-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -35,7 +35,8 @@ describe("fallback_on_statuses e2e: opt-in 4xx failover (#1012)", () => { let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; // "Overloaded" provider: answers every call with 422 (the shape some @@ -67,38 +68,38 @@ describe("fallback_on_statuses e2e: opt-in 4xx failover (#1012)", () => { }); app = await spawnApp(); - const admin = new AdminClient(app.adminUrl, app.adminKey); + const seed = new SeedClient(etcd, app.etcdPrefix); - const overloadedPk = await admin.createProviderKey({ + const overloadedPk = await seed.createProviderKey({ display_name: "fos-overloaded-pk", secret: "sk-mock", api_base: `${overloaded.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "fos-overloaded", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: overloadedPk.id, cooldown: { enabled: false }, }); - const badPk = await admin.createProviderKey({ + const badPk = await seed.createProviderKey({ display_name: "fos-bad-pk", secret: "sk-mock", api_base: `${badRequest.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "fos-bad", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: badPk.id, cooldown: { enabled: false }, }); - const healthyPk = await admin.createProviderKey({ + const healthyPk = await seed.createProviderKey({ display_name: "fos-healthy-pk", secret: "sk-mock", api_base: `${healthy.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "fos-healthy", provider: "openai", model_name: "gpt-4o-mini", @@ -106,14 +107,14 @@ describe("fallback_on_statuses e2e: opt-in 4xx failover (#1012)", () => { }); // Same two targets, with and without the opt-in. - await admin.createModel({ + await seed.createModel({ display_name: "fos-default-group", routing: { strategy: "failover", targets: [{ model: "fos-overloaded" }, { model: "fos-healthy" }], }, }); - await admin.createModel({ + await seed.createModel({ display_name: "fos-configured-group", routing: { strategy: "failover", @@ -123,7 +124,7 @@ describe("fallback_on_statuses e2e: opt-in 4xx failover (#1012)", () => { }); // Configured group whose FIRST target 400s — 400 is not in the list, // so it must still relay. - await admin.createModel({ + await seed.createModel({ display_name: "fos-unlisted-group", routing: { strategy: "failover", @@ -132,7 +133,7 @@ describe("fallback_on_statuses e2e: opt-in 4xx failover (#1012)", () => { }, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: [ "fos-default-group", diff --git a/tests/e2e/src/cases/guardrail-aliyun-e2e.test.ts b/tests/e2e/src/cases/guardrail-aliyun-e2e.test.ts index 58aa4956..affcb44e 100644 --- a/tests/e2e/src/cases/guardrail-aliyun-e2e.test.ts +++ b/tests/e2e/src/cases/guardrail-aliyun-e2e.test.ts @@ -3,8 +3,8 @@ import { createHash } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, pickFreePort, spawnApp, startOpenAiUpstream, @@ -106,11 +106,12 @@ describe("aliyun guardrail e2e: TextModerationPlus blocks risky input/output", ( let riskyOutputUpstream: OpenAiUpstream | undefined; let streamUpstream: OpenAiUpstream | undefined; let aliyun: AliyunMock | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; aliyun = await startAliyunMock(); @@ -164,45 +165,45 @@ describe("aliyun guardrail e2e: TextModerationPlus blocks risky input/output", ( }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const benignPk = await admin.createProviderKey({ + const benignPk = await seed.createProviderKey({ display_name: "aliyun-e2e-pk", secret: "sk-mock", api_base: `${benignUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "aliyun-e2e", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: benignPk.id, }); - const riskyOutPk = await admin.createProviderKey({ + const riskyOutPk = await seed.createProviderKey({ display_name: "aliyun-out-e2e-pk", secret: "sk-mock", api_base: `${riskyOutputUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "aliyun-out-e2e", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: riskyOutPk.id, }); - const streamPk = await admin.createProviderKey({ + const streamPk = await seed.createProviderKey({ display_name: "aliyun-stream-e2e-pk", secret: "sk-mock", api_base: `${streamUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "aliyun-stream-e2e", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: streamPk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["aliyun-e2e", "aliyun-out-e2e", "aliyun-stream-e2e"], }); @@ -210,7 +211,7 @@ describe("aliyun guardrail e2e: TextModerationPlus blocks risky input/output", ( // One env-wide guardrail covering input + output. Small window so the // streaming case triggers a windowed output call (and reuses the // stream's sessionId across windows). `endpoint` points at the mock. - await admin.json("POST", "/admin/v1/guardrails", { + await seed.createGuardrail({ name: "aliyun-e2e-guard", enabled: true, hook_point: "both", diff --git a/tests/e2e/src/cases/guardrail-disabled-bypass-e2e.test.ts b/tests/e2e/src/cases/guardrail-disabled-bypass-e2e.test.ts index 7b6190c3..9a7da138 100644 --- a/tests/e2e/src/cases/guardrail-disabled-bypass-e2e.test.ts +++ b/tests/e2e/src/cases/guardrail-disabled-bypass-e2e.test.ts @@ -4,6 +4,7 @@ import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -49,28 +50,31 @@ describe("guardrail disabled-bypass e2e: enabled:false → no block", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); app = await spawnApp(); admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "gr-disabled-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "gr-disabled-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["gr-disabled-model"], }); @@ -78,7 +82,7 @@ describe("guardrail disabled-bypass e2e: enabled:false → no block", () => { // hook_point:"input", literal pattern. ONLY difference: // `enabled:false`. If the operator switch works, this rule // is inert. - await admin.json("POST", "/admin/v1/guardrails", { + await seed.createGuardrail({ name: "gr-disabled-keyword", enabled: false, hook_point: "input", diff --git a/tests/e2e/src/cases/guardrail-keyword-e2e.test.ts b/tests/e2e/src/cases/guardrail-keyword-e2e.test.ts index aea221c3..baf18407 100644 --- a/tests/e2e/src/cases/guardrail-keyword-e2e.test.ts +++ b/tests/e2e/src/cases/guardrail-keyword-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -32,36 +32,37 @@ const FORBIDDEN_WORD = "supersecret"; describe("guardrail e2e: keyword block returns 422 and skips upstream", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "gr-e2e-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "gr-e2e", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["gr-e2e"], }); // In-process keyword blocklist. `hook_point: "input"` runs the // check on the request payload before bridge dispatch — a match // short-circuits with 422 and the upstream is never called. - await admin.json("POST", "/admin/v1/guardrails", { + await seed.createGuardrail({ name: "gr-e2e-keyword", enabled: true, hook_point: "input", diff --git a/tests/e2e/src/cases/guardrail-keyword-message-locations-e2e.test.ts b/tests/e2e/src/cases/guardrail-keyword-message-locations-e2e.test.ts index 3d4b6e83..a6a82d8e 100644 --- a/tests/e2e/src/cases/guardrail-keyword-message-locations-e2e.test.ts +++ b/tests/e2e/src/cases/guardrail-keyword-message-locations-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -45,35 +45,36 @@ const FORBIDDEN_WORD = "supersecret"; describe("guardrail keyword e2e: blocks forbidden literal in any message role", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "gr-loc-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "gr-loc-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["gr-loc-model"], }); // Same shape as guardrail-keyword-e2e: input hook, literal // pattern, ENABLED. - await admin.json("POST", "/admin/v1/guardrails", { + await seed.createGuardrail({ name: "gr-loc-keyword", enabled: true, hook_point: "input", diff --git a/tests/e2e/src/cases/guardrail-lakera-e2e.test.ts b/tests/e2e/src/cases/guardrail-lakera-e2e.test.ts index 92e4cf47..ae966a67 100644 --- a/tests/e2e/src/cases/guardrail-lakera-e2e.test.ts +++ b/tests/e2e/src/cases/guardrail-lakera-e2e.test.ts @@ -3,8 +3,8 @@ import { createHash } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, pickFreePort, spawnApp, startOpenAiUpstream, @@ -126,11 +126,12 @@ describe("lakera guardrail e2e: injection blocks, PII-only masks, 5xx fail-open" let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; let lakera: LakeraMock | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; lakera = await startLakeraMock(); @@ -153,20 +154,20 @@ describe("lakera guardrail e2e: injection blocks, PII-only masks, 5xx fail-open" }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "lakera-e2e-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "lakera-e2e", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: hash(CALLER), allowed_models: ["lakera-e2e"], }); @@ -174,7 +175,7 @@ describe("lakera guardrail e2e: injection blocks, PII-only masks, 5xx fail-open" // One env-wide guardrail on the input hook. fail_open=true so the // 5xx case exercises the bypass path (fail-closed is pinned by the // dispatcher's wiremock unit tests). - await admin.json("POST", "/admin/v1/guardrails", { + await seed.createGuardrail({ name: "lakera-e2e-guard", enabled: true, hook_point: "input", diff --git a/tests/e2e/src/cases/guardrail-model-scope-e2e.test.ts b/tests/e2e/src/cases/guardrail-model-scope-e2e.test.ts index 54167858..a663dcf6 100644 --- a/tests/e2e/src/cases/guardrail-model-scope-e2e.test.ts +++ b/tests/e2e/src/cases/guardrail-model-scope-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash, randomUUID } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -42,7 +42,7 @@ const UNSCOPED_MODEL = "unscoped-model"; describe("guardrail e2e: model-scope attachment runs only for the scoped model (#850)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcd: EtcdClient | undefined; let etcdReachable = false; @@ -53,43 +53,39 @@ describe("guardrail e2e: model-scope attachment runs only for the scoped model ( upstream = await startOpenAiUpstream(); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "gr-modelscope-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - const scoped = await admin.createModel({ + const scoped = await seed.createModel({ display_name: SCOPED_MODEL, provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createModel({ + await seed.createModel({ display_name: UNSCOPED_MODEL, provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: [SCOPED_MODEL, UNSCOPED_MODEL], }); // Keyword blocklist guardrail. `hook_point: "input"` → a match // short-circuits with 422 before dispatch. - const guardrail = await admin.json<{ id: string }>( - "POST", - "/admin/v1/guardrails", - { - name: "gr-modelscope-keyword", - enabled: true, - hook_point: "input", - kind: "keyword", - patterns: [{ kind: "literal", value: FORBIDDEN_WORD }], - }, - ); + const guardrail = await seed.createGuardrail({ + name: "gr-modelscope-keyword", + enabled: true, + hook_point: "input", + kind: "keyword", + patterns: [{ kind: "literal", value: FORBIDDEN_WORD }], + }); // Attach the guardrail to the SCOPED model only. Wire shape mirrors // cp-api's `marshalGuardrailAttachmentKV` (P0c): snake_case fields, diff --git a/tests/e2e/src/cases/guardrail-monitor-mode-e2e.test.ts b/tests/e2e/src/cases/guardrail-monitor-mode-e2e.test.ts index 4456786b..03117bac 100644 --- a/tests/e2e/src/cases/guardrail-monitor-mode-e2e.test.ts +++ b/tests/e2e/src/cases/guardrail-monitor-mode-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -41,36 +41,37 @@ function guardrailBody(enforcementMode: "block" | "monitor") { describe("guardrail e2e: enforcement_mode monitor observes without blocking", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; let guardrailId: string | undefined; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "gr-monitor-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "gr-monitor-e2e", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["gr-monitor-e2e"], }); // Start in BLOCK mode so we can prove the rule is loaded + active // before flipping it to monitor. - const g = await admin.json("POST", "/admin/v1/guardrails", guardrailBody("block")); + const g = await seed.createGuardrail(guardrailBody("block")); guardrailId = g.id as string; }); @@ -80,7 +81,7 @@ describe("guardrail e2e: enforcement_mode monitor observes without blocking", () }); test("block mode 422s; flipping to monitor lets the same request reach upstream", async (ctx) => { - if (!etcdReachable || !app || !upstream || !admin || !guardrailId) { + if (!etcdReachable || !app || !upstream || !seed || !guardrailId) { ctx.skip(); return; } @@ -105,7 +106,7 @@ describe("guardrail e2e: enforcement_mode monitor observes without blocking", () }); // 2. Flip the SAME rule to monitor mode (full-resource PUT). - await admin.json("PUT", `/admin/v1/guardrails/${guardrailId}`, guardrailBody("monitor")); + await seed.update("guardrails", guardrailId, guardrailBody("monitor")); // 3. Wait until monitor takes effect: the forbidden word no longer 422s. await waitConfigPropagation(async () => { diff --git a/tests/e2e/src/cases/guardrail-monitor-provider-failure-e2e.test.ts b/tests/e2e/src/cases/guardrail-monitor-provider-failure-e2e.test.ts index 939ecf4d..553b35e3 100644 --- a/tests/e2e/src/cases/guardrail-monitor-provider-failure-e2e.test.ts +++ b/tests/e2e/src/cases/guardrail-monitor-provider-failure-e2e.test.ts @@ -3,8 +3,8 @@ import { createHash } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, pickFreePort, spawnApp, startOpenAiUpstream, @@ -113,12 +113,13 @@ describe("guardrail e2e: monitor mode never blocks, even on provider failure (#1 let upstream: OpenAiUpstream | undefined; let broken: ModerationMock | undefined; let flagAll: ModerationMock | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let guardrailId = ""; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; broken = await startModerationMock("http500"); @@ -142,29 +143,25 @@ describe("guardrail e2e: monitor mode never blocks, even on provider failure (#1 }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "gr-monitor-failure-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "gr-monitor-failure-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["gr-monitor-failure-model"], }); - const g = (await admin.json( - "POST", - "/admin/v1/guardrails", - guardrailBody("block", broken.baseUrl), - )) as { id: string }; + const g = await seed.createGuardrail(guardrailBody("block", broken.baseUrl)); guardrailId = g.id; }); @@ -233,11 +230,7 @@ describe("guardrail e2e: monitor mode never blocks, even on provider failure (#1 ctx.skip(); return; } - await admin!.json( - "PUT", - `/admin/v1/guardrails/${guardrailId}`, - guardrailBody("monitor", broken.baseUrl), - ); + await seed!.update("guardrails", guardrailId, guardrailBody("monitor", broken.baseUrl)); // The flip from 422 to 200 can only mean monitor mode took effect — // the moderation endpoint is still hard-down. @@ -267,11 +260,7 @@ describe("guardrail e2e: monitor mode never blocks, even on provider failure (#1 ctx.skip(); return; } - await admin!.json( - "PUT", - `/admin/v1/guardrails/${guardrailId}`, - guardrailBody("monitor", flagAll.baseUrl), - ); + await seed!.update("guardrails", guardrailId, guardrailBody("monitor", flagAll.baseUrl)); // Gate on the endpoint switch: the flag-all mock starts receiving // the moderation calls. diff --git a/tests/e2e/src/cases/guardrail-monitor-telemetry-e2e.test.ts b/tests/e2e/src/cases/guardrail-monitor-telemetry-e2e.test.ts index ca22dd6b..fb4ff1bb 100644 --- a/tests/e2e/src/cases/guardrail-monitor-telemetry-e2e.test.ts +++ b/tests/e2e/src/cases/guardrail-monitor-telemetry-e2e.test.ts @@ -1,9 +1,9 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, decodedTextFor, EtcdClient, + SeedClient, spawnApp, startMockSls, startOpenAiUpstream, @@ -46,7 +46,8 @@ describe("guardrail monitor-hit telemetry: would_block/would_mask on usage event let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; sls = await startMockSls(); @@ -73,9 +74,9 @@ describe("guardrail monitor-hit telemetry: would_block/would_mask on usage event [`SLS_CRED_${CREDENTIAL_REF.toUpperCase()}_AK_SECRET`]: "mock-secret", }, }); - const admin = new AdminClient(app.adminUrl, app.adminKey); + const seed = new SeedClient(etcd, app.etcdPrefix); - await admin.createObservabilityExporter({ + await seed.createObservabilityExporter({ name: "sls-monitor-telemetry", enabled: true, kind: "aliyun_sls", @@ -86,23 +87,23 @@ describe("guardrail monitor-hit telemetry: would_block/would_mask on usage event content_mode: "metadata_only", }); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "monitor-telemetry-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "monitor-telemetry-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["monitor-telemetry-model"], }); - await admin.json("POST", "/admin/v1/guardrails", { + await seed.createGuardrail({ name: GUARD_NAME, enabled: true, hook_point: "input", @@ -147,6 +148,7 @@ describe("guardrail monitor-hit telemetry: would_block/would_mask on usage event // exported event carries a monitor hit. await waitConfigPropagation(async () => { const res = await chat(`probe mail ${EMAIL} ok`); + if (res.status === 401) return false; // caller key still propagating expect(res.status).toBe(200); return decodedTextFor(sls!, META_LOGSTORE).includes("guardrail_monitor_hits"); }); diff --git a/tests/e2e/src/cases/guardrail-openai-moderation-e2e.test.ts b/tests/e2e/src/cases/guardrail-openai-moderation-e2e.test.ts index 6cd50d45..8548c0dc 100644 --- a/tests/e2e/src/cases/guardrail-openai-moderation-e2e.test.ts +++ b/tests/e2e/src/cases/guardrail-openai-moderation-e2e.test.ts @@ -3,8 +3,8 @@ import { createHash } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, pickFreePort, spawnApp, startOpenAiUpstream, @@ -104,12 +104,13 @@ describe("openai moderation guardrail e2e: flagged blocks, monitor allows, thres let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; let moderation: ModerationMock | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let guardrailId: string | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; moderation = await startModerationMock(); @@ -132,39 +133,35 @@ describe("openai moderation guardrail e2e: flagged blocks, monitor allows, thres }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "moderation-e2e-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "moderation-e2e", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: hash(CALLER), allowed_models: ["moderation-e2e"], }); // One env-wide guardrail on the input hook, block mode. fail_open=true // so the 5xx case exercises the bypass path. - const created = await admin!.json<{ id: string }>( - "POST", - "/admin/v1/guardrails", - { - name: "moderation-e2e-guard", - enabled: true, - hook_point: "input", - fail_open: true, - kind: "openai_moderation", - api_key: "sk-moderation-key", - endpoint: moderation.baseUrl, - }, - ); + const created = await seed!.createGuardrail({ + name: "moderation-e2e-guard", + enabled: true, + hook_point: "input", + fail_open: true, + kind: "openai_moderation", + api_key: "sk-moderation-key", + endpoint: moderation.baseUrl, + }); guardrailId = created.id; }); @@ -266,11 +263,11 @@ describe("openai moderation guardrail e2e: flagged blocks, monitor allows, thres }); test("enforcement_mode=monitor → flagged content passes through", async (ctx) => { - if (!etcdReachable || !app || !upstream || !admin || !guardrailId) { + if (!etcdReachable || !app || !upstream || !seed || !guardrailId) { ctx.skip(); return; } - await admin.json("PUT", `/admin/v1/guardrails/${guardrailId}`, { + await seed.update("guardrails", guardrailId, { name: "moderation-e2e-guard", enabled: true, hook_point: "input", @@ -296,13 +293,13 @@ describe("openai moderation guardrail e2e: flagged blocks, monitor allows, thres }); test("category threshold mode enforces the configured category only", async (ctx) => { - if (!etcdReachable || !app || !upstream || !admin || !guardrailId) { + if (!etcdReachable || !app || !upstream || !seed || !guardrailId) { ctx.skip(); return; } // violence>=0.3 blocks even though the mock does NOT set flagged for // MILD_MARKER (score 0.4) — the threshold overrides the API boolean. - await admin.json("PUT", `/admin/v1/guardrails/${guardrailId}`, { + await seed.update("guardrails", guardrailId, { name: "moderation-e2e-guard", enabled: true, hook_point: "input", diff --git a/tests/e2e/src/cases/guardrail-output-e2e.test.ts b/tests/e2e/src/cases/guardrail-output-e2e.test.ts index 5843c1ba..ca355084 100644 --- a/tests/e2e/src/cases/guardrail-output-e2e.test.ts +++ b/tests/e2e/src/cases/guardrail-output-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -38,11 +38,12 @@ describe("output guardrail e2e: model-emitted forbidden text is blocked before r let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; let streamUpstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; // Mock upstream emits an OpenAI-shape completion that CONTAINS @@ -70,26 +71,26 @@ describe("output guardrail e2e: model-emitted forbidden text is blocked before r }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "gr-out-e2e-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "gr-out-e2e", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["gr-out-e2e"], }); // Output guardrail: runs against the assistant's response after // the upstream call returns, before relay to the caller. - await admin.json("POST", "/admin/v1/guardrails", { + await seed.createGuardrail({ name: "gr-out-e2e-keyword", enabled: true, hook_point: "output", @@ -114,19 +115,19 @@ describe("output guardrail e2e: model-emitted forbidden text is blocked before r ], eventDelayMs: 50, }); - const streamPk = await admin.createProviderKey({ + const streamPk = await seed.createProviderKey({ display_name: "gr-out-stream-e2e-pk", secret: "sk-mock", api_base: `${streamUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "gr-out-stream-e2e", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: streamPk.id, }); // Add the streaming Model alias to the existing caller's allow-list. - await admin.createApiKey({ + await seed.createApiKey({ key_hash: createHash("sha256") .update(`${CALLER_PLAINTEXT}-stream`) .digest("hex"), @@ -324,7 +325,7 @@ describe("output guardrail e2e: model-emitted forbidden text is blocked before r }); test("streaming Allow path: clean content streams to caller with [DONE], no error event (#204 audit M3)", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -347,19 +348,19 @@ describe("output guardrail e2e: model-emitted forbidden text is blocked before r ], eventDelayMs: 50, }); - const cleanPk = await admin.createProviderKey({ + const cleanPk = await seed.createProviderKey({ display_name: "gr-out-stream-clean-pk", secret: "sk-mock", api_base: `${cleanUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "gr-out-stream-clean-e2e", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: cleanPk.id, }); const cleanCallerPlaintext = `${CALLER_PLAINTEXT}-stream-clean`; - await admin.createApiKey({ + await seed.createApiKey({ key_hash: createHash("sha256") .update(cleanCallerPlaintext) .digest("hex"), diff --git a/tests/e2e/src/cases/guardrail-pii-redaction-e2e.test.ts b/tests/e2e/src/cases/guardrail-pii-redaction-e2e.test.ts index 2f0fa8a8..d6bc12f1 100644 --- a/tests/e2e/src/cases/guardrail-pii-redaction-e2e.test.ts +++ b/tests/e2e/src/cases/guardrail-pii-redaction-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash, randomUUID } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -38,7 +38,7 @@ describe("pii guardrail e2e: mask + block on request and response", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; let streamUpstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcd: EtcdClient | undefined; let etcdReachable = false; @@ -84,43 +84,43 @@ describe("pii guardrail e2e: mask + block on request and response", () => { }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "pii-e2e-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "pii-e2e", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: hash(CALLER), allowed_models: ["pii-e2e"], }); - const streamPk = await admin.createProviderKey({ + const streamPk = await seed.createProviderKey({ display_name: "pii-stream-e2e-pk", secret: "sk-mock", api_base: `${streamUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "pii-stream-e2e", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: streamPk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: hash(`${CALLER}-stream`), allowed_models: ["pii-stream-e2e"], }); // One env-wide pii guardrail: email masks (redact-and-continue), // china_id_card blocks (reject). - await admin.json("POST", "/admin/v1/guardrails", { + await seed.createGuardrail({ name: "pii-e2e-guard", enabled: true, hook_point: "both", @@ -154,11 +154,15 @@ describe("pii guardrail e2e: mask + block on request and response", () => { // Propagation probe: once the guardrail is live, the (email-bearing) // mock reply comes back masked. await waitConfigPropagation(async () => { - const r = await client().chat.completions.create({ - model: "pii-e2e", - messages: [{ role: "user", content: "probe" }], - }); - return (r.choices[0]?.message?.content ?? "").includes("[EMAIL_REDACTED]"); + try { + const r = await client().chat.completions.create({ + model: "pii-e2e", + messages: [{ role: "user", content: "probe" }], + }); + return (r.choices[0]?.message?.content ?? "").includes("[EMAIL_REDACTED]"); + } catch { + return false; // caller key / model still propagating + } }); const res = await client().chat.completions.create({ @@ -302,24 +306,24 @@ describe("pii guardrail e2e: mask + block on request and response", () => { }, }); const monApp = await spawnApp(); - const monAdmin = new AdminClient(monApp.adminUrl, monApp.adminKey); - const monPk = await monAdmin.createProviderKey({ + const monSeed = new SeedClient(new EtcdClient(), monApp.etcdPrefix); + const monPk = await monSeed.createProviderKey({ display_name: "pii-mon-pk", secret: "sk-mock", api_base: `${monUpstream.baseUrl}/v1`, }); - await monAdmin.createModel({ + await monSeed.createModel({ display_name: "pii-mon-e2e", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: monPk.id, }); const monCaller = `${CALLER}-mon`; - await monAdmin.createApiKey({ + await monSeed.createApiKey({ key_hash: hash(monCaller), allowed_models: ["pii-mon-e2e"], }); - await monAdmin.json("POST", "/admin/v1/guardrails", { + await monSeed.createGuardrail({ name: "pii-mon-guard", enabled: true, hook_point: "both", @@ -360,7 +364,7 @@ describe("pii guardrail e2e: mask + block on request and response", () => { }); test("/v1/messages passthrough: request masked before the Anthropic upstream", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -392,21 +396,21 @@ describe("pii guardrail e2e: mask + block on request and response", () => { await new Promise((resolve) => anthUpstream.listen(0, resolve)); const anthPort = (anthUpstream.address() as { port: number }).port; - const anthPk = await admin.createProviderKey({ + const anthPk = await seed.createProviderKey({ display_name: "pii-anth-pk", secret: "sk-ant-mock", api_base: `http://127.0.0.1:${anthPort}`, provider: "anthropic", adapter: "anthropic", }); - await admin.createModel({ + await seed.createModel({ display_name: "pii-anth-e2e", provider: "anthropic", model_name: "claude-3-5-haiku-20241022", provider_key_id: anthPk.id, }); const anthCaller = `${CALLER}-anth`; - await admin.createApiKey({ + await seed.createApiKey({ key_hash: hash(anthCaller), allowed_models: ["pii-anth-e2e"], }); diff --git a/tests/e2e/src/cases/guardrail-presidio-e2e.test.ts b/tests/e2e/src/cases/guardrail-presidio-e2e.test.ts index d85ee73c..c96a0afe 100644 --- a/tests/e2e/src/cases/guardrail-presidio-e2e.test.ts +++ b/tests/e2e/src/cases/guardrail-presidio-e2e.test.ts @@ -3,8 +3,8 @@ import { createHash } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, pickFreePort, spawnApp, startOpenAiUpstream, @@ -166,7 +166,7 @@ describe("presidio guardrail e2e: block, input/output redaction, operators", () let upstream: OpenAiUpstream | undefined; let streamUpstream: OpenAiUpstream | undefined; let presidio: PresidioMock | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let guardrailId: string | undefined; let etcdReachable = false; @@ -189,7 +189,8 @@ describe("presidio guardrail e2e: block, input/output redaction, operators", () }); beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; presidio = await startPresidioMock(); @@ -230,40 +231,36 @@ describe("presidio guardrail e2e: block, input/output redaction, operators", () }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "presidio-e2e-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "presidio-e2e", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - const streamPk = await admin.createProviderKey({ + const streamPk = await seed.createProviderKey({ display_name: "presidio-stream-e2e-pk", secret: "sk-mock", api_base: `${streamUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "presidio-stream-e2e", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: streamPk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: hash(CALLER), allowed_models: ["presidio-e2e", "presidio-stream-e2e"], }); - const created = await admin.json<{ id: string }>( - "POST", - "/admin/v1/guardrails", - guardrailBody("replace"), - ); + const created = await seed.createGuardrail(guardrailBody("replace")); guardrailId = created.id; }); @@ -288,11 +285,15 @@ describe("presidio guardrail e2e: block, input/output redaction, operators", () // baseline with a full PUT + its own propagation probe. const ensureGuardrailLive = () => waitConfigPropagation(async () => { - const r = await client().chat.completions.create({ - model: "presidio-e2e", - messages: [{ role: "user", content: "probe" }], - }); - return (r.choices[0]?.message?.content ?? "").includes(""); + try { + const r = await client().chat.completions.create({ + model: "presidio-e2e", + messages: [{ role: "user", content: "probe" }], + }); + return (r.choices[0]?.message?.content ?? "").includes(""); + } catch { + return false; // caller key / model still propagating + } }); test("redact: request PII anonymized before the upstream, response PII before the caller", async (ctx) => { @@ -392,15 +393,11 @@ describe("presidio guardrail e2e: block, input/output redaction, operators", () }); test("hash operator: anonymizer is driven with hash config and its output is honored", async (ctx) => { - if (!etcdReachable || !app || !upstream || !admin || !guardrailId || !presidio) { + if (!etcdReachable || !app || !upstream || !seed || !guardrailId || !presidio) { ctx.skip(); return; } - await admin.json( - "PUT", - `/admin/v1/guardrails/${guardrailId}`, - guardrailBody("hash"), - ); + await seed.update("guardrails", guardrailId, guardrailBody("hash")); // Propagation probe: the anonymized reply flips from // to the mock's fixed hash output. diff --git a/tests/e2e/src/cases/images-generations-e2e.test.ts b/tests/e2e/src/cases/images-generations-e2e.test.ts index a1c1fb16..7e9c2037 100644 --- a/tests/e2e/src/cases/images-generations-e2e.test.ts +++ b/tests/e2e/src/cases/images-generations-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -39,11 +39,12 @@ const CALLER_KEY_HASH = createHash("sha256") describe("images generations e2e: /v1/images/generations verbatim forward + model translation", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ @@ -60,20 +61,20 @@ describe("images generations e2e: /v1/images/generations verbatim forward + mode }, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "img-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "img-e2e", provider: "openai", model_name: "gpt-image-1", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["img-e2e"], }); @@ -175,7 +176,7 @@ describe("images generations e2e: /v1/images/generations verbatim forward + mode }); test("non-OpenAI provider returns 400 invalid_request_error, upstream untouched (#212 / docs §4.9)", async (ctx) => { - if (!etcdReachable || !app || !admin || !upstream) { + if (!etcdReachable || !app || !seed || !upstream) { ctx.skip(); return; } @@ -187,19 +188,19 @@ describe("images generations e2e: /v1/images/generations verbatim forward + mode // API; Gemini's image generation uses a different URL + body // shape; DeepSeek doesn't expose image generation). #212 covers // the e2e gap on this contract. - const nonOaPk = await admin.createProviderKey({ + const nonOaPk = await seed.createProviderKey({ display_name: "img-anthropic-pk", secret: "sk-ant-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "img-anthropic", provider: "anthropic", model_name: "claude-3-5-haiku-20241022", provider_key_id: nonOaPk.id, }); const nonOaCaller = `${CALLER_PLAINTEXT}-non-oa`; - await admin.createApiKey({ + await seed.createApiKey({ key_hash: createHash("sha256").update(nonOaCaller).digest("hex"), allowed_models: ["img-anthropic"], }); diff --git a/tests/e2e/src/cases/invalid-upstream-config-400-e2e.test.ts b/tests/e2e/src/cases/invalid-upstream-config-400-e2e.test.ts index 4ccb8d59..3fd996f6 100644 --- a/tests/e2e/src/cases/invalid-upstream-config-400-e2e.test.ts +++ b/tests/e2e/src/cases/invalid-upstream-config-400-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, ProxyClient, spawnApp, startOpenAiUpstream, @@ -28,33 +28,34 @@ const CALLER_KEY_HASH = createHash("sha256") describe("invalid upstream config maps to 400 e2e", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); // openrouter resolves via the openai adapter family. With no // api_base the family bridge refuses to fall back to // api.openai.com — a customer-fixable misconfig. (No api_base set.) - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "invalid-config-pk", provider: "openrouter", adapter: "openai", secret: "sk-mock", }); - await admin.createModel({ + await seed.createModel({ display_name: "invalid-config-model", provider: "openrouter", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["invalid-config-model"], }); diff --git a/tests/e2e/src/cases/invalid-upstream-credentials-401-e2e.test.ts b/tests/e2e/src/cases/invalid-upstream-credentials-401-e2e.test.ts index bd35cbcd..ff198b42 100644 --- a/tests/e2e/src/cases/invalid-upstream-credentials-401-e2e.test.ts +++ b/tests/e2e/src/cases/invalid-upstream-credentials-401-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, ProxyClient, spawnApp, startOpenAiUpstream, @@ -33,33 +33,34 @@ const CALLER_KEY_HASH = createHash("sha256") describe("invalid upstream credentials maps to 401 e2e", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); // Valid api_base (routing shape is fine) but a secret that can't be // an Authorization header value (embedded newline) — the openai // bridge's api_key() guard rejects this before dispatch as a // credential problem. - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "invalid-cred-pk", secret: "sk-live\n-injected", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "invalid-cred-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["invalid-cred-model"], }); diff --git a/tests/e2e/src/cases/json-mode-e2e.test.ts b/tests/e2e/src/cases/json-mode-e2e.test.ts index e809c44d..b4f555f0 100644 --- a/tests/e2e/src/cases/json-mode-e2e.test.ts +++ b/tests/e2e/src/cases/json-mode-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -64,11 +64,12 @@ const UPSTREAM_JSON_REPLY = JSON.stringify({ describe("json mode e2e: response_format passthrough on /v1/chat/completions", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ @@ -91,20 +92,20 @@ describe("json mode e2e: response_format passthrough on /v1/chat/completions", ( }, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "jsonmode-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "jsonmode-e2e", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["jsonmode-e2e"], }); diff --git a/tests/e2e/src/cases/latency-aware-routing-e2e.test.ts b/tests/e2e/src/cases/latency-aware-routing-e2e.test.ts index cc5540b3..a88d7952 100644 --- a/tests/e2e/src/cases/latency-aware-routing-e2e.test.ts +++ b/tests/e2e/src/cases/latency-aware-routing-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -35,17 +35,18 @@ function okBody(content: string) { describe("latency-aware (least_latency) routing e2e", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; const upstreams: OpenAiUpstream[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createApiKey({ + seed = new SeedClient(etcd, app.etcdPrefix); + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); @@ -61,13 +62,13 @@ describe("latency-aware (least_latency) routing e2e", () => { upstream: OpenAiUpstream, extra: Record = {}, ): Promise { - if (!admin) throw new Error("admin client not initialized"); - const providerKey = await admin.createProviderKey({ + if (!seed) throw new Error("seed client not initialized"); + const providerKey = await seed.createProviderKey({ display_name: `${displayName}-pk`, secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: displayName, provider: "openai", model_name: "gpt-4o-mini", @@ -85,7 +86,7 @@ describe("latency-aware (least_latency) routing e2e", () => { } test("routes to the fastest target once latencies are learned", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -101,7 +102,7 @@ describe("latency-aware (least_latency) routing e2e", () => { await createOpenAiModel("lat-fast", fast); // Declare the slow target FIRST — least_latency must reorder by observed // latency, not honor declaration order. - await admin.createModel({ + await seed.createModel({ display_name: "lat-virtual", routing: { strategy: "least_latency", diff --git a/tests/e2e/src/cases/latency-histograms-e2e.test.ts b/tests/e2e/src/cases/latency-histograms-e2e.test.ts index f2de157a..c87c4ce2 100644 --- a/tests/e2e/src/cases/latency-histograms-e2e.test.ts +++ b/tests/e2e/src/cases/latency-histograms-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -38,7 +38,8 @@ describe("latency histograms e2e: bucketed TTFT + e2e latency (#1011)", () => { let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ @@ -74,42 +75,42 @@ describe("latency histograms e2e: bucketed TTFT + e2e latency (#1011)", () => { }); app = await spawnApp(); - const admin = new AdminClient(app.adminUrl, app.adminKey); + const seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "histo-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "histo-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - const streamPk = await admin.createProviderKey({ + const streamPk = await seed.createProviderKey({ display_name: "histo-stream-pk", secret: "sk-mock", api_base: `${streamUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "histo-stream-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: streamPk.id, }); - const failPk = await admin.createProviderKey({ + const failPk = await seed.createProviderKey({ display_name: "histo-fail-pk", secret: "sk-mock", api_base: `${failUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "histo-fail-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: failPk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["histo-model", "histo-stream-model", "histo-fail-model"], }); diff --git a/tests/e2e/src/cases/least-busy-passthrough-endpoints-e2e.test.ts b/tests/e2e/src/cases/least-busy-passthrough-endpoints-e2e.test.ts index f1db182f..bdb39d6c 100644 --- a/tests/e2e/src/cases/least-busy-passthrough-endpoints-e2e.test.ts +++ b/tests/e2e/src/cases/least-busy-passthrough-endpoints-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -63,17 +63,18 @@ function responsesShapeBody(text: string) { describe("least-busy routing via passthrough endpoints e2e", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; const upstreams: OpenAiUpstream[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createApiKey({ + seed = new SeedClient(etcd, app.etcdPrefix); + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); @@ -88,13 +89,13 @@ describe("least-busy routing via passthrough endpoints e2e", () => { displayName: string, upstream: OpenAiUpstream, ): Promise { - if (!admin) throw new Error("admin client not initialized"); - const providerKey = await admin.createProviderKey({ + if (!seed) throw new Error("seed client not initialized"); + const providerKey = await seed.createProviderKey({ display_name: `${displayName}-pk`, secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: displayName, provider: "openai", model_name: "gpt-4o-mini", @@ -135,7 +136,7 @@ describe("least-busy routing via passthrough endpoints e2e", () => { } test("/v1/messages routes away from an in-flight target", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -153,7 +154,7 @@ describe("least-busy routing via passthrough endpoints e2e", () => { await createOpenAiModel("msg-busy-a", slow); await createOpenAiModel("msg-busy-b", fast); - await admin.createModel({ + await seed.createModel({ display_name: "msg-busy-virtual", routing: { strategy: "least_busy", @@ -196,7 +197,7 @@ describe("least-busy routing via passthrough endpoints e2e", () => { }); test("/v1/responses routes away from an in-flight target", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -213,7 +214,7 @@ describe("least-busy routing via passthrough endpoints e2e", () => { await createOpenAiModel("resp-busy-a", slow); await createOpenAiModel("resp-busy-b", fast); - await admin.createModel({ + await seed.createModel({ display_name: "resp-busy-virtual", routing: { strategy: "least_busy", diff --git a/tests/e2e/src/cases/least-busy-routing-e2e.test.ts b/tests/e2e/src/cases/least-busy-routing-e2e.test.ts index 5f5fb1a9..eafde719 100644 --- a/tests/e2e/src/cases/least-busy-routing-e2e.test.ts +++ b/tests/e2e/src/cases/least-busy-routing-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -35,17 +35,18 @@ function okBody(content: string) { describe("least-busy (least_busy) routing e2e", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; const upstreams: OpenAiUpstream[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createApiKey({ + seed = new SeedClient(etcd, app.etcdPrefix); + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); @@ -61,13 +62,13 @@ describe("least-busy (least_busy) routing e2e", () => { upstream: OpenAiUpstream, extra: Record = {}, ): Promise { - if (!admin) throw new Error("admin client not initialized"); - const providerKey = await admin.createProviderKey({ + if (!seed) throw new Error("seed client not initialized"); + const providerKey = await seed.createProviderKey({ display_name: `${displayName}-pk`, secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: displayName, provider: "openai", model_name: "gpt-4o-mini", @@ -85,7 +86,7 @@ describe("least-busy (least_busy) routing e2e", () => { } test("routes away from an in-flight target to the idle one", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -100,7 +101,7 @@ describe("least-busy (least_busy) routing e2e", () => { await createOpenAiModel("busy-a", slow); await createOpenAiModel("busy-b", fast); - await admin.createModel({ + await seed.createModel({ display_name: "busy-virtual", routing: { strategy: "least_busy", diff --git a/tests/e2e/src/cases/messages-cross-provider-extras-e2e.test.ts b/tests/e2e/src/cases/messages-cross-provider-extras-e2e.test.ts index 6fb11368..868f6c6b 100644 --- a/tests/e2e/src/cases/messages-cross-provider-extras-e2e.test.ts +++ b/tests/e2e/src/cases/messages-cross-provider-extras-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -53,11 +53,12 @@ const CALLER_KEY_HASH = createHash("sha256") describe("Anthropic Messages → OpenAI upstream: Anthropic-only extras are not forwarded (#953)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ @@ -78,20 +79,20 @@ describe("Anthropic Messages → OpenAI upstream: Anthropic-only extras are not }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "anth-extras-xprov-pk", secret: "sk-openai-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "anth-extras-xprov", provider: "openai", model_name: "gpt-4o", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["anth-extras-xprov"], }); @@ -196,11 +197,12 @@ const STREAM_CALLER_KEY_HASH = createHash("sha256") describe("Anthropic Messages → OpenAI upstream: streaming extras are not forwarded (#953)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ @@ -229,20 +231,20 @@ describe("Anthropic Messages → OpenAI upstream: streaming extras are not forwa }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "anth-extras-stream-xprov-pk", secret: "sk-openai-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "anth-extras-stream-xprov", provider: "openai", model_name: "gpt-4o", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: STREAM_CALLER_KEY_HASH, allowed_models: ["anth-extras-stream-xprov"], }); diff --git a/tests/e2e/src/cases/messages-cross-provider-stream-usage-e2e.test.ts b/tests/e2e/src/cases/messages-cross-provider-stream-usage-e2e.test.ts index c7e27fe0..d9893624 100644 --- a/tests/e2e/src/cases/messages-cross-provider-stream-usage-e2e.test.ts +++ b/tests/e2e/src/cases/messages-cross-provider-stream-usage-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -105,11 +105,12 @@ describe("anthropic→openai streaming usage (#790)", () => { let app: SpawnedApp | undefined; let upstreamUsage: OpenAiUpstream | undefined; let upstreamNoUsage: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstreamUsage = await startOpenAiUpstream({ @@ -121,33 +122,33 @@ describe("anthropic→openai streaming usage (#790)", () => { eventDelayMs: 2, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pkUsage = await admin.createProviderKey({ + const pkUsage = await seed.createProviderKey({ display_name: "issue790-pk", secret: "sk-openai-mock", api_base: `${upstreamUsage.baseUrl}/v1`, }); - const pkNoUsage = await admin.createProviderKey({ + const pkNoUsage = await seed.createProviderKey({ display_name: "issue790-pk-nousage", secret: "sk-openai-mock", api_base: `${upstreamNoUsage.baseUrl}/v1`, }); // Mirrors the customer config from #790: alias gpt-5.5 → upstream // model mog-6 on an OpenAI-protocol provider. - await admin.createModel({ + await seed.createModel({ display_name: "gpt-5.5", provider: "openai", model_name: "mog-6", provider_key_id: pkUsage.id, }); - await admin.createModel({ + await seed.createModel({ display_name: "gpt-5.5-nousage", provider: "openai", model_name: "mog-6", provider_key_id: pkNoUsage.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["gpt-5.5", "gpt-5.5-nousage"], }); diff --git a/tests/e2e/src/cases/messages-input-guardrail-e2e.test.ts b/tests/e2e/src/cases/messages-input-guardrail-e2e.test.ts index bcd33b0c..a027eca1 100644 --- a/tests/e2e/src/cases/messages-input-guardrail-e2e.test.ts +++ b/tests/e2e/src/cases/messages-input-guardrail-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -23,28 +23,29 @@ const FORBIDDEN = "forbiddenmsgword"; describe("/v1/messages input guardrail (#448)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); - const pk = await admin.createProviderKey({ + seed = new SeedClient(etcd, app.etcdPrefix); + const pk = await seed.createProviderKey({ display_name: "msg-gr-pk", secret: "sk-anth-mock", api_base: upstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: "msg-gr", provider: "anthropic", model_name: "claude-3-5-haiku-20241022", provider_key_id: pk.id, }); - await admin.createApiKey({ key_hash: HASH, allowed_models: ["msg-gr"] }); - await admin.json("POST", "/admin/v1/guardrails", { + await seed.createApiKey({ key_hash: HASH, allowed_models: ["msg-gr"] }); + await seed.createGuardrail({ name: "msg-gr-input-keyword", enabled: true, hook_point: "input", @@ -74,8 +75,14 @@ describe("/v1/messages input guardrail (#448)", () => { ctx.skip(); return; } - // Gate on guardrail propagation: the forbidden prompt must be rejected. - await waitConfigPropagation(async () => (await messages(`probe ${FORBIDDEN}`)).status >= 400); + // Gate on guardrail propagation. A bare ">= 400 on forbidden" would + // also match the transitional 401 while the caller key itself is + // still propagating — require a benign prompt to pass first, so the + // rejection can only come from the guardrail. + await waitConfigPropagation(async () => { + if ((await messages("propagation probe")).status >= 400) return false; + return (await messages(`probe ${FORBIDDEN}`)).status >= 400; + }); const hitsBefore = upstream.receivedRequests.length; const blocked = await messages(`please do ${FORBIDDEN} now`); diff --git a/tests/e2e/src/cases/messages-model-group-e2e.test.ts b/tests/e2e/src/cases/messages-model-group-e2e.test.ts index 6dba6397..06c4025f 100644 --- a/tests/e2e/src/cases/messages-model-group-e2e.test.ts +++ b/tests/e2e/src/cases/messages-model-group-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -73,17 +73,18 @@ type MessagesResult = { describe("model group via passthrough endpoints e2e (#471)", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; const upstreams: OpenAiUpstream[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createApiKey({ + seed = new SeedClient(etcd, app.etcdPrefix); + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); @@ -98,14 +99,14 @@ describe("model group via passthrough endpoints e2e (#471)", () => { displayName: string, upstream: OpenAiUpstream, ): Promise { - if (!admin) throw new Error("admin client not initialized"); - const pk = await admin.createProviderKey({ + if (!seed) throw new Error("seed client not initialized"); + const pk = await seed.createProviderKey({ display_name: `${displayName}-pk`, secret: "sk-openai-mock", // OpenAI bridge convention: host + /v1. api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: displayName, provider: "openai", model_name: "gpt-4o-mini", @@ -117,8 +118,8 @@ describe("model group via passthrough endpoints e2e (#471)", () => { displayName: string, upstream: OpenAiUpstream, ): Promise { - if (!admin) throw new Error("admin client not initialized"); - const pk = await admin.createProviderKey({ + if (!seed) throw new Error("seed client not initialized"); + const pk = await seed.createProviderKey({ display_name: `${displayName}-pk`, provider: "anthropic", adapter: "anthropic", @@ -126,7 +127,7 @@ describe("model group via passthrough endpoints e2e (#471)", () => { // Anthropic bridge appends /v1/messages, so point at the bare host. api_base: upstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: displayName, provider: "anthropic", model_name: "claude-3-5-haiku-20241022", @@ -203,19 +204,25 @@ describe("model group via passthrough endpoints e2e (#471)", () => { async function waitUntilGroupRoutable(virtual: string): Promise { // Gate on the virtual record reaching the DP snapshot without - // sending traffic through it, so failover hit-counts below start clean. + // sending traffic through it, so failover hit-counts below start + // clean: seed a throwaway canary key AFTER the group document — + // watch events apply in revision order, so once the canary bearer + // authenticates against /v1/models the group is in the snapshot too. + const canary = `sk-canary-${virtual}-${Date.now()}`; + await seed!.createApiKey({ + key_hash: createHash("sha256").update(canary).digest("hex"), + allowed_models: ["*"], + }); await waitConfigPropagation(async () => { - try { - const models = await admin!.listModels(); - return models.some((m) => m.display_name === virtual); - } catch { - return false; - } + const res = await fetch(`${app!.proxyUrl}/v1/models`, { + headers: { authorization: `Bearer ${canary}` }, + }); + return res.status === 200; }); } test("Anthropic target in a group is reachable via /v1/messages", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -232,7 +239,7 @@ describe("model group via passthrough endpoints e2e (#471)", () => { await createOpenAiModel("mg-an-openai", openai); await waitUntilModelResponds("mg-an-anthropic"); await waitUntilModelResponds("mg-an-openai"); - await admin.createModel({ + await seed.createModel({ display_name: "mg-anthropic-first", routing: { strategy: "failover", @@ -257,7 +264,7 @@ describe("model group via passthrough endpoints e2e (#471)", () => { }); test("OpenAI target in a group is reachable via /v1/messages (cross-provider)", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -269,7 +276,7 @@ describe("model group via passthrough endpoints e2e (#471)", () => { await createOpenAiModel("mg-op-openai", openai); await waitUntilModelResponds("mg-op-openai"); - await admin.createModel({ + await seed.createModel({ display_name: "mg-openai-first", routing: { strategy: "failover", @@ -291,7 +298,7 @@ describe("model group via passthrough endpoints e2e (#471)", () => { }); test("cross-protocol failover: OpenAI target down falls over to Anthropic target", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -311,7 +318,7 @@ describe("model group via passthrough endpoints e2e (#471)", () => { // left unprobed so its cooldown doesn't exclude it from the attempt // list before the measured call below. await waitUntilModelResponds("mg-fo-anthropic"); - await admin.createModel({ + await seed.createModel({ display_name: "mg-failover", routing: { strategy: "failover", @@ -339,7 +346,7 @@ describe("model group via passthrough endpoints e2e (#471)", () => { }); test("Anthropic group is reachable via /v1/messages/count_tokens", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -350,7 +357,7 @@ describe("model group via passthrough endpoints e2e (#471)", () => { upstreams.push(anthropic); await createAnthropicModel("mg-ct-anthropic", anthropic); - await admin.createModel({ + await seed.createModel({ display_name: "mg-count-tokens", routing: { strategy: "failover", @@ -372,7 +379,7 @@ describe("model group via passthrough endpoints e2e (#471)", () => { }); test("OpenAI group is reachable via /v1/responses", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -397,7 +404,7 @@ describe("model group via passthrough endpoints e2e (#471)", () => { upstreams.push(openai); await createOpenAiModel("mg-resp-openai", openai); - await admin.createModel({ + await seed.createModel({ display_name: "mg-responses", routing: { strategy: "failover", diff --git a/tests/e2e/src/cases/messages-output-guardrail-e2e.test.ts b/tests/e2e/src/cases/messages-output-guardrail-e2e.test.ts index 3418d9fa..87aaee83 100644 --- a/tests/e2e/src/cases/messages-output-guardrail-e2e.test.ts +++ b/tests/e2e/src/cases/messages-output-guardrail-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -23,30 +23,31 @@ const HASH = createHash("sha256").update(CALLER).digest("hex"); describe("/v1/messages output guardrail (#448)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); - const pk = await admin.createProviderKey({ + seed = new SeedClient(etcd, app.etcdPrefix); + const pk = await seed.createProviderKey({ display_name: "msgout-gr-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); // OpenAI-provider model → /v1/messages takes the cross-provider path // (Anthropic protocol translated to the OpenAI bridge). - await admin.createModel({ + await seed.createModel({ display_name: "msgout-gr", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ key_hash: HASH, allowed_models: ["msgout-gr"] }); - await admin.json("POST", "/admin/v1/guardrails", { + await seed.createApiKey({ key_hash: HASH, allowed_models: ["msgout-gr"] }); + await seed.createGuardrail({ name: "msgout-gr-output-keyword", enabled: true, hook_point: "output", diff --git a/tests/e2e/src/cases/messages-stream-delta-only-usage-e2e.test.ts b/tests/e2e/src/cases/messages-stream-delta-only-usage-e2e.test.ts index 9d69ba68..4ad545e4 100644 --- a/tests/e2e/src/cases/messages-stream-delta-only-usage-e2e.test.ts +++ b/tests/e2e/src/cases/messages-stream-delta-only-usage-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, type OpenAiUpstream, @@ -61,11 +61,12 @@ const STREAM_EVENTS = [ describe("anthropic /v1/messages stream with usage only on message_delta (#952)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ @@ -73,20 +74,20 @@ describe("anthropic /v1/messages stream with usage only on message_delta (#952)" eventDelayMs: 2, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "anth-952-delta-pk", secret: "sk-anth-mock", api_base: upstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: "anth-952-delta", provider: "anthropic", model_name: "mco-5", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["anth-952-delta"], }); diff --git a/tests/e2e/src/cases/messages-streaming-output-guardrail-e2e.test.ts b/tests/e2e/src/cases/messages-streaming-output-guardrail-e2e.test.ts index 5cc7faf5..9a2cc84b 100644 --- a/tests/e2e/src/cases/messages-streaming-output-guardrail-e2e.test.ts +++ b/tests/e2e/src/cases/messages-streaming-output-guardrail-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -35,28 +35,29 @@ const STREAM_EVENTS = [ describe("streaming /v1/messages output guardrail (#448)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ streamEvents: STREAM_EVENTS, eventDelayMs: 2 }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); - const pk = await admin.createProviderKey({ + seed = new SeedClient(etcd, app.etcdPrefix); + const pk = await seed.createProviderKey({ display_name: "msgstream-gr-pk", secret: "sk-anth-mock", api_base: upstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: "msgstream-gr", provider: "anthropic", model_name: "claude-3-5-haiku-20241022", provider_key_id: pk.id, }); - await admin.createApiKey({ key_hash: HASH, allowed_models: ["msgstream-gr"] }); - await admin.json("POST", "/admin/v1/guardrails", { + await seed.createApiKey({ key_hash: HASH, allowed_models: ["msgstream-gr"] }); + await seed.createGuardrail({ name: "msgstream-gr-output-keyword", enabled: true, hook_point: "output", diff --git a/tests/e2e/src/cases/messages-streaming-toolcall-guardrail-e2e.test.ts b/tests/e2e/src/cases/messages-streaming-toolcall-guardrail-e2e.test.ts index c94871d5..7be0b286 100644 --- a/tests/e2e/src/cases/messages-streaming-toolcall-guardrail-e2e.test.ts +++ b/tests/e2e/src/cases/messages-streaming-toolcall-guardrail-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -35,28 +35,29 @@ const STREAM_EVENTS = [ describe("streaming /v1/messages tool_use output guardrail (#448)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ streamEvents: STREAM_EVENTS, eventDelayMs: 2 }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); - const pk = await admin.createProviderKey({ + seed = new SeedClient(etcd, app.etcdPrefix); + const pk = await seed.createProviderKey({ display_name: "msgtool-gr-pk", secret: "sk-anth-mock", api_base: upstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: "msgtool-gr", provider: "anthropic", model_name: "claude-3-5-haiku-20241022", provider_key_id: pk.id, }); - await admin.createApiKey({ key_hash: HASH, allowed_models: ["msgtool-gr"] }); - await admin.json("POST", "/admin/v1/guardrails", { + await seed.createApiKey({ key_hash: HASH, allowed_models: ["msgtool-gr"] }); + await seed.createGuardrail({ name: "msgtool-gr-output-keyword", enabled: true, hook_point: "output", diff --git a/tests/e2e/src/cases/messages-system-role-e2e.test.ts b/tests/e2e/src/cases/messages-system-role-e2e.test.ts index d71b8fcd..7d0a0176 100644 --- a/tests/e2e/src/cases/messages-system-role-e2e.test.ts +++ b/tests/e2e/src/cases/messages-system-role-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -25,11 +25,12 @@ const CALLER_KEY_HASH = createHash("sha256") describe("anthropic /v1/messages with system role in messages[] (#597)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ @@ -49,22 +50,22 @@ describe("anthropic /v1/messages with system role in messages[] (#597)", () => { }, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "issue597-pk", secret: "sk-deepseek-mock", api_base: `${upstream.baseUrl}/v1`, }); // Mirrors the customer setup from #597: Claude CLI talking to a // DeepSeek/OpenAI-protocol model through the Anthropic endpoint. - await admin.createModel({ + await seed.createModel({ display_name: "issue597-model", provider: "deepseek", model_name: "deepseek-chat", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["issue597-model"], }); diff --git a/tests/e2e/src/cases/metric-cardinality-model-label-e2e.test.ts b/tests/e2e/src/cases/metric-cardinality-model-label-e2e.test.ts index 928baa0e..3806d3a0 100644 --- a/tests/e2e/src/cases/metric-cardinality-model-label-e2e.test.ts +++ b/tests/e2e/src/cases/metric-cardinality-model-label-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -43,20 +43,21 @@ function chatReply(content: string): unknown { describe("metric label cardinality for unresolved model (#911 [27])", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let upstream: OpenAiUpstream | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ nonStreamBody: chatReply("ready") }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); const pk = ( - await admin.createProviderKey({ + await seed.createProviderKey({ display_name: "card-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, @@ -64,7 +65,7 @@ describe("metric label cardinality for unresolved model (#911 [27])", () => { ).id; // One real model, used only to gate on config propagation. - await admin.createModel({ + await seed.createModel({ display_name: "card-gate", provider: "openai", model_name: "gpt-4o-mini", @@ -74,7 +75,7 @@ describe("metric label cardinality for unresolved model (#911 [27])", () => { // Wildcard so ANY model name passes the allowed_models authz check and // reaches model resolution — where the unknown names fail (model-not- // found) and hit the metric-recording error path under test. - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); diff --git a/tests/e2e/src/cases/metrics-dimensions-890-e2e.test.ts b/tests/e2e/src/cases/metrics-dimensions-890-e2e.test.ts index b8d19e49..81a8cf54 100644 --- a/tests/e2e/src/cases/metrics-dimensions-890-e2e.test.ts +++ b/tests/e2e/src/cases/metrics-dimensions-890-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash, randomUUID } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, ProxyClient, spawnApp, startOpenAiUpstream, @@ -34,11 +34,12 @@ describe("metrics dimensions #890 e2e", () => { let nonStreamUpstream: OpenAiUpstream | undefined; let streamUpstream: OpenAiUpstream | undefined; let errorUpstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; nonStreamUpstream = await startOpenAiUpstream({ nonStreamBody: responseBody() }); @@ -54,38 +55,38 @@ describe("metrics dimensions #890 e2e", () => { }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const nsPk = await admin.createProviderKey({ + const nsPk = await seed.createProviderKey({ display_name: PK_NAME, secret: "sk-mock", api_base: `${nonStreamUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: NONSTREAM_MODEL, provider: "openai", model_name: "gpt-4o-mini", provider_key_id: nsPk.id, }); - const stPk = await admin.createProviderKey({ + const stPk = await seed.createProviderKey({ display_name: `${PK_NAME}-stream`, secret: "sk-mock", api_base: `${streamUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: STREAM_MODEL, provider: "openai", model_name: "gpt-4o-mini", provider_key_id: stPk.id, }); - const errPk = await admin.createProviderKey({ + const errPk = await seed.createProviderKey({ display_name: `${PK_NAME}-error`, secret: "sk-mock", api_base: `${errorUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: ERROR_MODEL, provider: "openai", model_name: "gpt-4o-mini", diff --git a/tests/e2e/src/cases/model-ip-restriction-e2e.test.ts b/tests/e2e/src/cases/model-ip-restriction-e2e.test.ts index 0184c6f9..9e14ad04 100644 --- a/tests/e2e/src/cases/model-ip-restriction-e2e.test.ts +++ b/tests/e2e/src/cases/model-ip-restriction-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -58,7 +58,8 @@ describe("model IP restriction e2e (#557): allowed_cidrs gate before upstream", let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); @@ -68,15 +69,15 @@ describe("model IP restriction e2e (#557): allowed_cidrs gate before upstream", app = await spawnApp({ realIp: { trusted_proxies: ["127.0.0.1/32"], recursive: true }, }); - const admin = new AdminClient(app.adminUrl, app.adminKey); + const seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "model-ip-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); // Restricted model: only the 10.0.0.0/8 range may call it. - await admin.createModel({ + await seed.createModel({ display_name: RESTRICTED_MODEL, provider: "openai", model_name: "gpt-4o-mini", @@ -84,13 +85,13 @@ describe("model IP restriction e2e (#557): allowed_cidrs gate before upstream", allowed_cidrs: ["10.0.0.0/8"], }); // Open model: no restriction, same upstream. - await admin.createModel({ + await seed.createModel({ display_name: OPEN_MODEL, provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: [RESTRICTED_MODEL, OPEN_MODEL], }); diff --git a/tests/e2e/src/cases/openai-embeddings-base64-e2e.test.ts b/tests/e2e/src/cases/openai-embeddings-base64-e2e.test.ts index a65f699c..27c65b71 100644 --- a/tests/e2e/src/cases/openai-embeddings-base64-e2e.test.ts +++ b/tests/e2e/src/cases/openai-embeddings-base64-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -95,11 +95,12 @@ const BASE64_RESPONSE = { describe("OpenAI /v1/embeddings encoding_format (#393)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; // Scripted upstream — readiness probe (chat) consumes step 0 @@ -135,20 +136,20 @@ describe("OpenAI /v1/embeddings encoding_format (#393)", () => { nonStreamBody: FLOAT_RESPONSE, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "openai-embed-b64-pk", secret: "sk-openai-mock", api_base: upstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: "openai-embed-b64", provider: "openai", model_name: "text-embedding-3-small", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["openai-embed-b64"], }); diff --git a/tests/e2e/src/cases/openai-sdk-compat.test.ts b/tests/e2e/src/cases/openai-sdk-compat.test.ts index 42761201..d3f6f420 100644 --- a/tests/e2e/src/cases/openai-sdk-compat.test.ts +++ b/tests/e2e/src/cases/openai-sdk-compat.test.ts @@ -52,6 +52,7 @@ describe("openai SDK compat: drive gateway through real client", () => { }); app = await spawnApp(); + // Deliberately seeds via the Admin API: deprecation-window coverage. admin = new AdminClient(app.adminUrl, app.adminKey); // Two ProviderKeys → two Models — keeps `receivedRequests` on each diff --git a/tests/e2e/src/cases/openai-tools-roundtrip-e2e.test.ts b/tests/e2e/src/cases/openai-tools-roundtrip-e2e.test.ts index d4cd6313..ed247cad 100644 --- a/tests/e2e/src/cases/openai-tools-roundtrip-e2e.test.ts +++ b/tests/e2e/src/cases/openai-tools-roundtrip-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -202,11 +202,12 @@ const TOOL_CALLS_STREAM_EVENTS = [ describe("OpenAI tools round-trip (#220 + #202)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; // Scripted upstream — each test gets its own scripted step. The @@ -227,20 +228,20 @@ describe("OpenAI tools round-trip (#220 + #202)", () => { nonStreamBody: READY_PROBE_BODY, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "openai-tools-rt-pk", secret: "sk-openai-mock", api_base: upstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: "openai-tools-rt", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["openai-tools-rt"], }); diff --git a/tests/e2e/src/cases/openrouter-stream-reasoning-e2e.test.ts b/tests/e2e/src/cases/openrouter-stream-reasoning-e2e.test.ts index eddb930c..69d16f68 100644 --- a/tests/e2e/src/cases/openrouter-stream-reasoning-e2e.test.ts +++ b/tests/e2e/src/cases/openrouter-stream-reasoning-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -47,11 +47,12 @@ function chunk(delta: Record, finish: string | null = null) { describe("openrouter delta.reasoning normalization on streaming /v1/chat/completions (#502)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; // OpenRouter-style streaming: reasoning text arrives on `delta.reasoning`, @@ -68,23 +69,23 @@ describe("openrouter delta.reasoning normalization on streaming /v1/chat/complet }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); // OpenRouter dispatches through the OpenAI-compat family bridge. - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "or-stream-reasoning-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, provider: "openrouter", adapter: "openai", }); - await admin.createModel({ + await seed.createModel({ display_name: "or-reasoner", provider: "openrouter", model_name: "some-reasoner", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["or-reasoner"], }); diff --git a/tests/e2e/src/cases/otlp-knobs-e2e.test.ts b/tests/e2e/src/cases/otlp-knobs-e2e.test.ts index a29dfd43..8c356d11 100644 --- a/tests/e2e/src/cases/otlp-knobs-e2e.test.ts +++ b/tests/e2e/src/cases/otlp-knobs-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import { createServer, type Server } from "node:http"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, pickFreePort, spawnApp, startOpenAiUpstream, @@ -76,19 +76,19 @@ async function startOtlpReceiver(): Promise { }; } -async function seedRouting(admin: AdminClient, upstream: OpenAiUpstream) { - const pk = await admin.createProviderKey({ +async function seedRouting(seed: SeedClient, upstream: OpenAiUpstream) { + const pk = await seed.createProviderKey({ display_name: "otlp-knobs-pk", secret: PROVIDER_SECRET, api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "otlp-knobs-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["otlp-knobs-model"], }); @@ -155,14 +155,14 @@ describe("otlp exporter knobs e2e (#519 B.2): sample_rate + content capture", () receivers.push(otlp); const app = await spawnApp(); apps.push(app); - const admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createObservabilityExporter({ + const seed = new SeedClient(new EtcdClient(), app.etcdPrefix); + await seed.createObservabilityExporter({ name: "otlp-defaults", enabled: true, kind: "otlp_http", endpoint: otlp.url, }); - await seedRouting(admin, upstream); + await seedRouting(seed, upstream); await waitConfigPropagation(async () => { try { const r = await chat(app, "default-probe"); @@ -191,8 +191,8 @@ describe("otlp exporter knobs e2e (#519 B.2): sample_rate + content capture", () receivers.push(otlp); const app = await spawnApp(); apps.push(app); - const admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createObservabilityExporter({ + const seed = new SeedClient(new EtcdClient(), app.etcdPrefix); + await seed.createObservabilityExporter({ name: "otlp-full", enabled: true, kind: "otlp_http", @@ -203,7 +203,7 @@ describe("otlp exporter knobs e2e (#519 B.2): sample_rate + content capture", () // must leave room for the probe to stay visible untruncated. content_max_bytes: 200, }); - await seedRouting(admin, upstream); + await seedRouting(seed, upstream); await waitConfigPropagation(async () => { try { const r = await chat(app, "content-probe"); @@ -250,22 +250,22 @@ describe("otlp exporter knobs e2e (#519 B.2): sample_rate + content capture", () receivers.push(otlpZero, otlpAll); const app = await spawnApp(); apps.push(app); - const admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createObservabilityExporter({ + const seed = new SeedClient(new EtcdClient(), app.etcdPrefix); + await seed.createObservabilityExporter({ name: "otlp-sample-zero", enabled: true, kind: "otlp_http", endpoint: otlpZero.url, sample_rate: 0, }); - await admin.createObservabilityExporter({ + await seed.createObservabilityExporter({ name: "otlp-sample-all", enabled: true, kind: "otlp_http", endpoint: otlpAll.url, sample_rate: 1.0, }); - await seedRouting(admin, upstream); + await seedRouting(seed, upstream); await waitConfigPropagation(async () => { try { const r = await chat(app, "sampling-probe"); diff --git a/tests/e2e/src/cases/passthrough-e2e.test.ts b/tests/e2e/src/cases/passthrough-e2e.test.ts index c8a87961..5bd33d94 100644 --- a/tests/e2e/src/cases/passthrough-e2e.test.ts +++ b/tests/e2e/src/cases/passthrough-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -64,17 +64,18 @@ const CALLER_KEY_HASH = createHash("sha256") describe("passthrough e2e: /passthrough/{provider}/*rest verbatim forwarding with auth injection + double-/v1 dedup", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; const upstreams: OpenAiUpstream[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createApiKey({ + seed = new SeedClient(etcd, app.etcdPrefix); + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); @@ -86,7 +87,7 @@ describe("passthrough e2e: /passthrough/{provider}/*rest verbatim forwarding wit }); test("OpenAI passthrough: gateway dedups doubled /v1, forwards verbatim, injects Bearer auth", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -113,14 +114,14 @@ describe("passthrough e2e: /passthrough/{provider}/*rest verbatim forwarding wit // Configure exactly per docs/api-admin.md §4.3 example: // `api_base: "https://api.openai.com/v1"` (with /v1). - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "pt-openai-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); // The Model's provider field is what the passthrough route // matches on (per docs §4.10 "first Model with that prefix"). - await admin.createModel({ + await seed.createModel({ display_name: "pt-openai-model", provider: "openai", model_name: "gpt-4o-mini", @@ -215,7 +216,7 @@ describe("passthrough e2e: /passthrough/{provider}/*rest verbatim forwarding wit }); test("Anthropic passthrough: gateway uses x-api-key + anthropic-version, NOT Bearer", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -238,12 +239,12 @@ describe("passthrough e2e: /passthrough/{provider}/*rest verbatim forwarding wit // Anthropic api_base is the bare host; bridge composes the // rest of the path. For passthrough, the gateway appends // `/*rest` directly, so we pass the bare host. - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "pt-anthropic-pk", secret: "sk-ant-mock", api_base: upstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: "pt-anthropic-model", provider: "anthropic", model_name: "claude-3-5-haiku-20241022", diff --git a/tests/e2e/src/cases/passthrough-guardrail-e2e.test.ts b/tests/e2e/src/cases/passthrough-guardrail-e2e.test.ts index b74e8bad..3656973f 100644 --- a/tests/e2e/src/cases/passthrough-guardrail-e2e.test.ts +++ b/tests/e2e/src/cases/passthrough-guardrail-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -34,11 +34,12 @@ const OUTPUT_GUARDRAIL = "pt-gr-output-keyword"; describe("passthrough guardrail (#911 [6])", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; // Upstream reply carries the forbidden OUTPUT word; the caller's request @@ -61,31 +62,31 @@ describe("passthrough guardrail (#911 [6])", () => { }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "pt-gr-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "pt-gr", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["pt-gr"], }); - await admin.json("POST", "/admin/v1/guardrails", { + await seed.createGuardrail({ name: INPUT_GUARDRAIL, enabled: true, hook_point: "input", kind: "keyword", patterns: [{ kind: "literal", value: FORBIDDEN_INPUT }], }); - await admin.json("POST", "/admin/v1/guardrails", { + await seed.createGuardrail({ name: OUTPUT_GUARDRAIL, enabled: true, hook_point: "output", diff --git a/tests/e2e/src/cases/passthrough-model-acl-e2e.test.ts b/tests/e2e/src/cases/passthrough-model-acl-e2e.test.ts index 9c23b3f0..40989108 100644 --- a/tests/e2e/src/cases/passthrough-model-acl-e2e.test.ts +++ b/tests/e2e/src/cases/passthrough-model-acl-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -28,17 +28,18 @@ describe("passthrough model ACL (#449)", () => { let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({}); app = await spawnApp(); - const admin = new AdminClient(app.adminUrl, app.adminKey); - const pk = await admin.createProviderKey({ + const seed = new SeedClient(etcd, app.etcdPrefix); + const pk = await seed.createProviderKey({ display_name: "pt-acl-pk", secret: "sk-openai-mock", api_base: upstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: "pt-acl-model", provider: "openai", model_name: "gpt-x", @@ -46,8 +47,8 @@ describe("passthrough model ACL (#449)", () => { }); // ALLOWED key may use the openai model; DENIED key may only use an // unrelated model name (no openai model in its ACL). - await admin.createApiKey({ key_hash: sha(ALLOWED), allowed_models: ["pt-acl-model"] }); - await admin.createApiKey({ key_hash: sha(DENIED), allowed_models: ["unrelated-model"] }); + await seed.createApiKey({ key_hash: sha(ALLOWED), allowed_models: ["pt-acl-model"] }); + await seed.createApiKey({ key_hash: sha(DENIED), allowed_models: ["unrelated-model"] }); }); afterAll(async () => { diff --git a/tests/e2e/src/cases/per-attempt-telemetry-e2e.test.ts b/tests/e2e/src/cases/per-attempt-telemetry-e2e.test.ts index 72e98f50..5ab0b986 100644 --- a/tests/e2e/src/cases/per-attempt-telemetry-e2e.test.ts +++ b/tests/e2e/src/cases/per-attempt-telemetry-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import { createServer, type Server } from "node:http"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, pickFreePort, spawnApp, startOpenAiUpstream, @@ -102,23 +102,24 @@ async function waitForAttempts( describe("per-attempt telemetry e2e (#655): one UsageEvent per upstream attempt", () => { let etcdReachable = false; let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let otlp: OtlpReceiver | undefined; const upstreams: OpenAiUpstream[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); otlp = await startOtlpReceiver(); - await admin.createObservabilityExporter({ + await seed.createObservabilityExporter({ name: "per-attempt-otlp", enabled: true, kind: "otlp_http", endpoint: otlp.url, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); @@ -134,13 +135,13 @@ describe("per-attempt telemetry e2e (#655): one UsageEvent per upstream attempt" displayName: string, upstream: OpenAiUpstream, ): Promise { - if (!admin) throw new Error("admin client not initialized"); - const providerKey = await admin.createProviderKey({ + if (!seed) throw new Error("seed client not initialized"); + const providerKey = await seed.createProviderKey({ display_name: `${displayName}-pk`, secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: displayName, provider: "openai", model_name: "gpt-4o-mini", @@ -149,7 +150,7 @@ describe("per-attempt telemetry e2e (#655): one UsageEvent per upstream attempt" } test("a failover request emits a failed initial + successful fallback attempt sharing request_id", async (ctx) => { - if (!etcdReachable || !app || !admin || !otlp) { + if (!etcdReachable || !app || !seed || !otlp) { ctx.skip(); return; } @@ -178,7 +179,7 @@ describe("per-attempt telemetry e2e (#655): one UsageEvent per upstream attempt" await createOpenAiModel("attempt-primary", primary); await createOpenAiModel("attempt-secondary", secondary); - await admin.createModel({ + await seed.createModel({ display_name: "attempt-virtual", routing: { strategy: "failover", @@ -188,17 +189,22 @@ describe("per-attempt telemetry e2e (#655): one UsageEvent per upstream attempt" }, }); - // Gate on admin-snapshot presence rather than probing the virtual — + // Gate on DP-snapshot presence rather than probing the virtual — // probing would warm the primary's cooldown (every retryable upstream // failure cools the failing direct target) and the measured request - // would then skip the primary entirely. + // would then skip the primary entirely. A throwaway canary key seeded + // AFTER the virtual authenticates only once the snapshot has caught + // up past it (watch events apply in revision order). + const canary = `sk-canary-attempt-${Date.now()}`; + await seed.createApiKey({ + key_hash: createHash("sha256").update(canary).digest("hex"), + allowed_models: ["*"], + }); await waitConfigPropagation(async () => { - try { - const models = await admin!.listModels(); - return models.some((m) => m.display_name === "attempt-virtual"); - } catch { - return false; - } + const res = await fetch(`${app!.proxyUrl}/v1/models`, { + headers: { authorization: `Bearer ${canary}` }, + }); + return res.status === 200; }); const res = await fetch(`${app.proxyUrl}/v1/chat/completions`, { diff --git a/tests/e2e/src/cases/prometheus-metrics-e2e.test.ts b/tests/e2e/src/cases/prometheus-metrics-e2e.test.ts index 1ed89835..c633e86a 100644 --- a/tests/e2e/src/cases/prometheus-metrics-e2e.test.ts +++ b/tests/e2e/src/cases/prometheus-metrics-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, ProxyClient, spawnApp, startOpenAiUpstream, @@ -19,20 +19,21 @@ const CALLER_KEY_HASH = createHash("sha256") describe("prometheus metrics e2e", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ nonStreamBody: responseBody(), }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - await configureOpenAi(admin, upstream, "prometheus-gpt"); + await configureOpenAi(seed, upstream, "prometheus-gpt"); }); afterAll(async () => { @@ -93,8 +94,8 @@ describe("prometheus metrics e2e", () => { }); const customApp = await spawnApp({ prometheusPath: "/custom-metrics" }); try { - const customAdmin = new AdminClient(customApp.adminUrl, customApp.adminKey); - await configureOpenAi(customAdmin, customUpstream, "prometheus-custom-gpt"); + const customSeed = new SeedClient(new EtcdClient(), customApp.etcdPrefix); + await configureOpenAi(customSeed, customUpstream, "prometheus-custom-gpt"); const proxy = new ProxyClient(customApp.proxyUrl, CALLER_PLAINTEXT); await waitConfigPropagation(async () => { const probe = await proxy.chat({ @@ -281,22 +282,22 @@ function parseUsageEmittedCount( } async function configureOpenAi( - admin: AdminClient, + seed: SeedClient, upstream: OpenAiUpstream, modelName: string, ) { - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: `${modelName}-pk`, secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: modelName, provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: [modelName], }); diff --git a/tests/e2e/src/cases/provider-key-rotation-e2e.test.ts b/tests/e2e/src/cases/provider-key-rotation-e2e.test.ts index a3e5bbab..c4882c32 100644 --- a/tests/e2e/src/cases/provider-key-rotation-e2e.test.ts +++ b/tests/e2e/src/cases/provider-key-rotation-e2e.test.ts @@ -4,6 +4,7 @@ import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -66,11 +67,13 @@ describe("provider_key rotation: zero in-flight disruption (#196 L3 / #271)", () let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let pkId = ""; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ @@ -88,20 +91,21 @@ describe("provider_key rotation: zero in-flight disruption (#196 L3 / #271)", () app = await spawnApp(); admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "pkrot-pk", secret: "sk-mock-v1", api_base: `${upstream.baseUrl}/v1`, }); pkId = pk.id; - await admin.createModel({ + await seed.createModel({ display_name: "rot-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["rot-model"], }); @@ -159,7 +163,7 @@ describe("provider_key rotation: zero in-flight disruption (#196 L3 / #271)", () if (i === ROTATE_AT && !rotated) { rotated = true; // Rotate the credential in place: same id + api_base, new secret. - await admin!.json("PUT", `/admin/v1/provider_keys/${pkId}`, { + await seed!.update("provider_keys", pkId, { provider: "openai", adapter: "openai", display_name: "pkrot-pk", diff --git a/tests/e2e/src/cases/ratelimit-cluster-e2e.test.ts b/tests/e2e/src/cases/ratelimit-cluster-e2e.test.ts index 68225c0b..0d995380 100644 --- a/tests/e2e/src/cases/ratelimit-cluster-e2e.test.ts +++ b/tests/e2e/src/cases/ratelimit-cluster-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash, randomUUID } from "node:crypto"; import { connect } from "node:net"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, ProxyClient, spawnApp, startOpenAiUpstream, @@ -79,22 +79,22 @@ function chatRequest(proxyUrl: string, model: string): Promise { }); } -/** Seed one model + an RPM=1 ApiKey via this app's admin API. The peer - * replica picks the same config up over the shared etcd watch. */ -async function seed(app: SpawnedApp, upstreamBase: string, model: string) { - const admin = new AdminClient(app.adminUrl, app.adminKey); - const pk = await admin.createProviderKey({ +/** Seed one model + an RPM=1 ApiKey into the SHARED config namespace — + * both replicas pick it up over the same etcd watch. */ +async function seed(etcdRoot: string, upstreamBase: string, model: string) { + const seed = new SeedClient(new EtcdClient(), etcdRoot); + const pk = await seed.createProviderKey({ display_name: `${model}-pk`, secret: "sk-mock", api_base: `${upstreamBase}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: model, provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: [model], rate_limit: { rpm: 1 }, @@ -132,7 +132,7 @@ describe("rate limit is shared across replicas with backend=redis (#798)", () => }; appA = await spawnApp({ extra }); appB = await spawnApp({ extra }); - await seed(appA, upstream.baseUrl, model); + await seed(prefix, upstream.baseUrl, model); await waitModelLive(appA.proxyUrl, model); await waitModelLive(appB.proxyUrl, model); }); @@ -175,7 +175,8 @@ describe("rate limit is NOT shared with backend=memory (per-replica, the #798 bu const model = "rl-cluster-mem"; beforeAll(async () => { - etcdReady = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReady = await etcd.ping(); if (!etcdReady) return; upstream = await startOpenAiUpstream(); @@ -184,7 +185,7 @@ describe("rate limit is NOT shared with backend=memory (per-replica, the #798 bu const extra = { etcd: sharedEtcd(prefix) }; appA = await spawnApp({ extra }); appB = await spawnApp({ extra }); - await seed(appA, upstream.baseUrl, model); + await seed(prefix, upstream.baseUrl, model); await waitModelLive(appA.proxyUrl, model); await waitModelLive(appB.proxyUrl, model); }); diff --git a/tests/e2e/src/cases/ratelimit-e2e.test.ts b/tests/e2e/src/cases/ratelimit-e2e.test.ts index bd73fd0c..19e86dcd 100644 --- a/tests/e2e/src/cases/ratelimit-e2e.test.ts +++ b/tests/e2e/src/cases/ratelimit-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, ProxyClient, spawnApp, startOpenAiUpstream, @@ -30,23 +30,24 @@ const CALLER_KEY_HASH = createHash("sha256") describe("rate limit e2e: RPM=1 second call gets 429", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "rl-e2e-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "rl-e2e", provider: "openai", model_name: "gpt-4o-mini", @@ -55,7 +56,7 @@ describe("rate limit e2e: RPM=1 second call gets 429", () => { // Rate limit is per-ApiKey here (matching the unit-level // `seed_snapshot_with_limits` pattern). RPM=1 means the first // call inside a 60s window succeeds; the second is rejected. - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["rl-e2e"], rate_limit: { rpm: 1 }, diff --git a/tests/e2e/src/cases/realtime-ws-e2e.test.ts b/tests/e2e/src/cases/realtime-ws-e2e.test.ts index 5d68b43e..4a5cdcfd 100644 --- a/tests/e2e/src/cases/realtime-ws-e2e.test.ts +++ b/tests/e2e/src/cases/realtime-ws-e2e.test.ts @@ -3,9 +3,10 @@ import { WebSocket } from "undici"; import { WebSocketServer, type WebSocket as WsSocket } from "ws"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, + waitConfigPropagation, type SpawnedApp, } from "../harness/index.js"; @@ -81,33 +82,46 @@ async function startRealtimeUpstream(): Promise { describe("realtime e2e: /v1/realtime WebSocket relay (#721)", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let upstream: RealtimeUpstream | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); upstream = await startRealtimeUpstream(); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "realtime-e2e-pk", secret: "sk-upstream-realtime", api_base: `http://127.0.0.1:${upstream.port}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "realtime-e2e-model", provider: "openai", model_name: "gpt-realtime-mock", provider_key_id: pk.id, }); + + // Gate on the DP snapshot via /v1/models — the WS upgrade below + // authenticates against the same snapshot, and a handshake fired + // before the caller key propagates is rejected outright. + await waitConfigPropagation(async () => { + const res = await fetch(`${app!.proxyUrl}/v1/models`, { + headers: { authorization: `Bearer ${CALLER_PLAINTEXT}` }, + }); + if (res.status !== 200) return false; + const body = (await res.json()) as { data?: Array<{ id?: string }> }; + return (body.data ?? []).some((m) => m.id === "realtime-e2e-model"); + }); }); afterAll(async () => { diff --git a/tests/e2e/src/cases/request-id-header-e2e.test.ts b/tests/e2e/src/cases/request-id-header-e2e.test.ts index 1bccac8c..7bf3de57 100644 --- a/tests/e2e/src/cases/request-id-header-e2e.test.ts +++ b/tests/e2e/src/cases/request-id-header-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -34,25 +34,26 @@ describe("data plane stamps x-aisix-request-id on every response", () => { let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); app = await spawnApp(); - const admin = new AdminClient(app.adminUrl, app.adminKey); + const seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "request-id-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "request-id-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: VALID_KEY_HASH, allowed_models: ["request-id-model"], }); diff --git a/tests/e2e/src/cases/request-id-uuid-e2e.test.ts b/tests/e2e/src/cases/request-id-uuid-e2e.test.ts index 8807ae98..ee72f1a4 100644 --- a/tests/e2e/src/cases/request-id-uuid-e2e.test.ts +++ b/tests/e2e/src/cases/request-id-uuid-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -21,14 +21,15 @@ const UUID_RE = describe("request id e2e: gateway-generated request IDs are UUIDs", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let chatUpstream: OpenAiUpstream | undefined; let embeddingsUpstream: OpenAiUpstream | undefined; let messagesUpstream: OpenAiUpstream | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; chatUpstream = await startOpenAiUpstream(); @@ -53,45 +54,45 @@ describe("request id e2e: gateway-generated request IDs are UUIDs", () => { }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const chatPk = await admin.createProviderKey({ + const chatPk = await seed.createProviderKey({ display_name: "request-id-chat-pk", secret: "sk-mock", api_base: `${chatUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "request-id-chat", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: chatPk.id, }); - const embeddingsPk = await admin.createProviderKey({ + const embeddingsPk = await seed.createProviderKey({ display_name: "request-id-embeddings-pk", secret: "sk-mock", api_base: `${embeddingsUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "request-id-embeddings", provider: "openai", model_name: "text-embedding-3-small", provider_key_id: embeddingsPk.id, }); - const messagesPk = await admin.createProviderKey({ + const messagesPk = await seed.createProviderKey({ display_name: "request-id-messages-pk", secret: "sk-ant-mock", api_base: messagesUpstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: "request-id-messages", provider: "anthropic", model_name: "claude-3-5-haiku-20241022", provider_key_id: messagesPk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); diff --git a/tests/e2e/src/cases/rerank-e2e.test.ts b/tests/e2e/src/cases/rerank-e2e.test.ts index 1b953ddd..286e42e8 100644 --- a/tests/e2e/src/cases/rerank-e2e.test.ts +++ b/tests/e2e/src/cases/rerank-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -56,11 +56,12 @@ const CALLER_KEY_HASH = createHash("sha256") describe("rerank e2e: /v1/rerank verbatim forward + model translation", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; // Mock upstream returns a canned Cohere-shape rerank response @@ -81,20 +82,20 @@ describe("rerank e2e: /v1/rerank verbatim forward + model translation", () => { }, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "rerank-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "rerank-e2e", provider: "openai", model_name: "rerank-english-v3.0", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["rerank-e2e"], }); @@ -206,7 +207,7 @@ describe("rerank e2e: /v1/rerank verbatim forward + model translation", () => { }); test("Cohere provider: gateway dispatches with Bearer auth + rewritten model (#213 Phase 1)", async (ctx) => { - if (!etcdReachable || !app || !admin || !upstream) { + if (!etcdReachable || !app || !seed || !upstream) { ctx.skip(); return; } @@ -218,7 +219,7 @@ describe("rerank e2e: /v1/rerank verbatim forward + model translation", () => { // Bearer ` auth, so the gateway forwards verbatim with the // `model` field rewritten — no transform required. const cohereSecret = "sk-cohere-mock-e2e"; - const coherePk = await admin.createProviderKey({ + const coherePk = await seed.createProviderKey({ display_name: "rerank-cohere-pk", secret: cohereSecret, // Operator can also set this to `https://api.cohere.com`; the @@ -226,14 +227,14 @@ describe("rerank e2e: /v1/rerank verbatim forward + model translation", () => { // form. api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "rerank-cohere", provider: "cohere", model_name: "rerank-english-v3.0", provider_key_id: coherePk.id, }); const cohereCallerPlaintext = `${CALLER_PLAINTEXT}-cohere`; - await admin.createApiKey({ + await seed.createApiKey({ key_hash: createHash("sha256") .update(cohereCallerPlaintext) .digest("hex"), @@ -330,7 +331,7 @@ describe("rerank e2e: /v1/rerank verbatim forward + model translation", () => { }); test("Jina provider: gateway dispatches with Bearer auth + rewritten model (#213 Phase 2)", async (ctx) => { - if (!etcdReachable || !app || !admin || !upstream) { + if (!etcdReachable || !app || !seed || !upstream) { ctx.skip(); return; } @@ -343,19 +344,19 @@ describe("rerank e2e: /v1/rerank verbatim forward + model translation", () => { // OpenAI-compat. The gateway forwards verbatim with only the // `model` field rewritten. const jinaSecret = "jina_mock_secret_e2e"; - const jinaPk = await admin.createProviderKey({ + const jinaPk = await seed.createProviderKey({ display_name: "rerank-jina-pk", secret: jinaSecret, api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "rerank-jina", provider: "jina", model_name: "jina-reranker-v2-base-multilingual", provider_key_id: jinaPk.id, }); const jinaCallerPlaintext = `${CALLER_PLAINTEXT}-jina`; - await admin.createApiKey({ + await seed.createApiKey({ key_hash: createHash("sha256") .update(jinaCallerPlaintext) .digest("hex"), @@ -441,7 +442,7 @@ describe("rerank e2e: /v1/rerank verbatim forward + model translation", () => { }); test("non-OpenAI provider returns 400 invalid_request_error, upstream untouched (#212 / docs §4.7)", async (ctx) => { - if (!etcdReachable || !app || !admin || !upstream) { + if (!etcdReachable || !app || !seed || !upstream) { ctx.skip(); return; } @@ -452,19 +453,19 @@ describe("rerank e2e: /v1/rerank verbatim forward + model translation", () => { // and the upstream returned 404 (Anthropic / DeepSeek have no // rerank API; Gemini's OpenAI-compat surface doesn't expose // /v1/rerank). #212 covers the e2e gap on this contract. - const nonOaPk = await admin.createProviderKey({ + const nonOaPk = await seed.createProviderKey({ display_name: "rerank-anthropic-pk", secret: "sk-ant-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "rerank-anthropic", provider: "anthropic", model_name: "claude-3-5-haiku-20241022", provider_key_id: nonOaPk.id, }); const nonOaCaller = `${CALLER_PLAINTEXT}-non-oa`; - await admin.createApiKey({ + await seed.createApiKey({ key_hash: createHash("sha256").update(nonOaCaller).digest("hex"), allowed_models: ["rerank-anthropic"], }); diff --git a/tests/e2e/src/cases/responses-cross-provider-e2e.test.ts b/tests/e2e/src/cases/responses-cross-provider-e2e.test.ts index 145ccda4..03a3f555 100644 --- a/tests/e2e/src/cases/responses-cross-provider-e2e.test.ts +++ b/tests/e2e/src/cases/responses-cross-provider-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import { createServer, type Server } from "node:http"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, pickFreePort, spawnApp, startOpenAiUpstream, @@ -166,7 +166,8 @@ describe("/v1/responses cross-provider → Anthropic (#825)", () => { let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; nsUpstream = await startOpenAiUpstream({ nonStreamBody: ANTHROPIC_NONSTREAM }); @@ -175,9 +176,9 @@ describe("/v1/responses cross-provider → Anthropic (#825)", () => { eventDelayMs: 2, }); app = await spawnApp(); - const admin = new AdminClient(app.adminUrl, app.adminKey); + const seed = new SeedClient(etcd, app.etcdPrefix); otlp = await startOtlpReceiver(); - await admin.createObservabilityExporter({ + await seed.createObservabilityExporter({ name: "issue825-otlp", enabled: true, kind: "otlp_http", @@ -186,33 +187,33 @@ describe("/v1/responses cross-provider → Anthropic (#825)", () => { // Two Anthropic-backed models, one per mock (non-streaming vs streaming); // api_base is the bare host — the bridge composes `/v1/messages`. - const nsPk = await admin.createProviderKey({ + const nsPk = await seed.createProviderKey({ display_name: "issue825-ns-pk", provider: "anthropic", adapter: "anthropic", secret: "sk-ant-mock", api_base: nsUpstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: "opus-4.7", provider: "anthropic", model_name: "claude-3-haiku-20240307", provider_key_id: nsPk.id, }); - const streamPk = await admin.createProviderKey({ + const streamPk = await seed.createProviderKey({ display_name: "issue825-stream-pk", provider: "anthropic", adapter: "anthropic", secret: "sk-ant-mock", api_base: streamUpstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: "opus-4.7-stream", provider: "anthropic", model_name: "claude-3-haiku-20240307", provider_key_id: streamPk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["opus-4.7", "opus-4.7-stream"], }); diff --git a/tests/e2e/src/cases/responses-endpoint-e2e.test.ts b/tests/e2e/src/cases/responses-endpoint-e2e.test.ts index 87b72679..5db5350b 100644 --- a/tests/e2e/src/cases/responses-endpoint-e2e.test.ts +++ b/tests/e2e/src/cases/responses-endpoint-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -49,17 +49,18 @@ const UPSTREAM_REPLY_TEXT = "Hello from /v1/responses!"; describe("responses endpoint e2e: /v1/responses dispatch + provider mismatch", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; const upstreams: OpenAiUpstream[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createApiKey({ + seed = new SeedClient(etcd, app.etcdPrefix); + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); @@ -71,7 +72,7 @@ describe("responses endpoint e2e: /v1/responses dispatch + provider mismatch", ( }); test("OpenAI provider: caller receives upstream Responses body byte-for-byte", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -104,12 +105,12 @@ describe("responses endpoint e2e: /v1/responses dispatch + provider mismatch", ( }); upstreams.push(upstream); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "resp-openai-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "resp-openai", provider: "openai", model_name: "gpt-4o-mini", @@ -224,7 +225,7 @@ describe("responses endpoint e2e: /v1/responses dispatch + provider mismatch", ( // reply back into the Responses shape. (Richer Anthropic streaming/tool // coverage lives in responses-cross-provider-e2e.) test("non-OpenAI provider (deepseek): /v1/responses is bridged to a Responses-shape 200", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -247,14 +248,14 @@ describe("responses endpoint e2e: /v1/responses dispatch + provider mismatch", ( }); upstreams.push(upstream); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "resp-deepseek-pk", provider: "deepseek", adapter: "openai", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "resp-deepseek", provider: "deepseek", model_name: "deepseek-chat", diff --git a/tests/e2e/src/cases/responses-streaming-usage-e2e.test.ts b/tests/e2e/src/cases/responses-streaming-usage-e2e.test.ts index fa2e8934..6372d3e1 100644 --- a/tests/e2e/src/cases/responses-streaming-usage-e2e.test.ts +++ b/tests/e2e/src/cases/responses-streaming-usage-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import { createServer, type Server } from "node:http"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, pickFreePort, spawnApp, startOpenAiUpstream, @@ -143,12 +143,13 @@ async function collectUsageSpans( describe("responses streaming usage emission (#808)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let otlp: OtlpReceiver | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ @@ -156,27 +157,27 @@ describe("responses streaming usage emission (#808)", () => { eventDelayMs: 2, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); otlp = await startOtlpReceiver(); - await admin.createObservabilityExporter({ + await seed.createObservabilityExporter({ name: "issue808-otlp", enabled: true, kind: "otlp_http", endpoint: otlp.url, }); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "issue808-pk", secret: "sk-openai-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "gpt-5-codex", provider: "openai", model_name: "gpt-5-codex", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["gpt-5-codex"], }); diff --git a/tests/e2e/src/cases/retry-backoff-e2e.test.ts b/tests/e2e/src/cases/retry-backoff-e2e.test.ts index 07d3dae1..005ce093 100644 --- a/tests/e2e/src/cases/retry-backoff-e2e.test.ts +++ b/tests/e2e/src/cases/retry-backoff-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -34,11 +34,12 @@ const MIN_EXPECTED_MS = 600; describe("retry backoff e2e: same-target retries wait before re-hitting upstream", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; // Every request to this upstream returns a retryable 503. @@ -47,14 +48,14 @@ describe("retry backoff e2e: same-target retries wait before re-hitting upstream errorBody: { error: { message: "always down", type: "server_error" } }, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "retry-backoff-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "retry-backoff-target", provider: "openai", model_name: "gpt-4o-mini", @@ -62,7 +63,7 @@ describe("retry backoff e2e: same-target retries wait before re-hitting upstream }); // Single-target router with retries=2: the same target is attempted // three times, so the two inter-retry backoffs are the only delay. - await admin.createModel({ + await seed.createModel({ display_name: "retry-backoff-router", routing: { strategy: "failover", @@ -70,7 +71,7 @@ describe("retry backoff e2e: same-target retries wait before re-hitting upstream retries: 2, }, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["retry-backoff-router"], }); diff --git a/tests/e2e/src/cases/retry-on-429-vs-background-ignore-e2e.test.ts b/tests/e2e/src/cases/retry-on-429-vs-background-ignore-e2e.test.ts index 03affbe3..bb2a7a38 100644 --- a/tests/e2e/src/cases/retry-on-429-vs-background-ignore-e2e.test.ts +++ b/tests/e2e/src/cases/retry-on-429-vs-background-ignore-e2e.test.ts @@ -4,6 +4,7 @@ import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -19,12 +20,14 @@ const CALLER_KEY_HASH = createHash("sha256") describe("retry_on_429 vs background ignore e2e", () => { let app: SpawnedApp | undefined; let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; let rateLimitedUpstream: OpenAiUpstream | undefined; let fallbackUpstream: OpenAiUpstream | undefined; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; rateLimitedUpstream = await startOpenAiUpstream({ @@ -62,19 +65,20 @@ describe("retry_on_429 vs background ignore e2e", () => { app = await spawnApp(); admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const primaryPk = await admin.createProviderKey({ + const primaryPk = await seed.createProviderKey({ display_name: "retry-429-primary-pk", secret: "sk-mock", api_base: `${rateLimitedUpstream.baseUrl}/v1`, }); - const fallbackPk = await admin.createProviderKey({ + const fallbackPk = await seed.createProviderKey({ display_name: "retry-429-fallback-pk", secret: "sk-mock", api_base: `${fallbackUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "retry-429-primary", provider: "openai", model_name: "gpt-4o-mini", @@ -89,13 +93,13 @@ describe("retry_on_429 vs background ignore e2e", () => { stale_after_seconds: 90, }, }); - await admin.createModel({ + await seed.createModel({ display_name: "retry-429-fallback", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: fallbackPk.id, }); - await admin.createModel({ + await seed.createModel({ display_name: "retry-429-router", routing: { strategy: "failover", @@ -108,7 +112,7 @@ describe("retry_on_429 vs background ignore e2e", () => { retry_on_429: true, }, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["retry-429-router", "retry-429-fallback"], }); diff --git a/tests/e2e/src/cases/routing-strategies-e2e.test.ts b/tests/e2e/src/cases/routing-strategies-e2e.test.ts index 24cf092f..615ca3a7 100644 --- a/tests/e2e/src/cases/routing-strategies-e2e.test.ts +++ b/tests/e2e/src/cases/routing-strategies-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -18,17 +18,18 @@ const CALLER_KEY_HASH = createHash("sha256") describe("routing strategies and retry behavior e2e", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; const upstreams: OpenAiUpstream[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createApiKey({ + seed = new SeedClient(etcd, app.etcdPrefix); + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); @@ -44,13 +45,13 @@ describe("routing strategies and retry behavior e2e", () => { upstream: OpenAiUpstream, extra: Record = {}, ): Promise { - if (!admin) throw new Error("admin client not initialized"); - const providerKey = await admin.createProviderKey({ + if (!seed) throw new Error("seed client not initialized"); + const providerKey = await seed.createProviderKey({ display_name: `${displayName}-pk`, secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: displayName, provider: "openai", model_name: "gpt-4o-mini", @@ -81,8 +82,28 @@ describe("routing strategies and retry behavior e2e", () => { }); } + + // Seed a throwaway canary key AFTER the resources under test — watch + // events apply in revision order, so once this bearer authenticates + // against /v1/models everything written before it is in the snapshot + // too. Probing the routed models directly would warm cooldowns and + // skew the per-target hit counts the assertions rely on. + async function waitSeedApplied(label: string): Promise { + const canary = `sk-canary-${label}-${Date.now()}`; + await seed!.createApiKey({ + key_hash: createHash("sha256").update(canary).digest("hex"), + allowed_models: ["*"], + }); + await waitConfigPropagation(async () => { + const res = await fetch(`${app!.proxyUrl}/v1/models`, { + headers: { authorization: `Bearer ${canary}` }, + }); + return res.status === 200; + }); + } + test("failover retries the current target before moving to the next target", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -117,7 +138,7 @@ describe("routing strategies and retry behavior e2e", () => { }); await createOpenAiModel("routing-retry-secondary", secondary); await waitUntilModelResponds("routing-retry-secondary", "after retries"); - await admin.createModel({ + await seed.createModel({ display_name: "routing-retry-virtual", routing: { strategy: "failover", @@ -170,7 +191,7 @@ describe("routing strategies and retry behavior e2e", () => { }); test("retry_on_429 lets 429 participate in retry and failover", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -200,7 +221,7 @@ describe("routing strategies and retry behavior e2e", () => { await createOpenAiModel("routing-429-primary", primary); await createOpenAiModel("routing-429-secondary", secondary); await waitUntilModelResponds("routing-429-secondary", "429 fallback worked"); - await admin.createModel({ + await seed.createModel({ display_name: "routing-429-virtual", routing: { strategy: "failover", @@ -220,18 +241,11 @@ describe("routing strategies and retry behavior e2e", () => { maxRetries: 0, }); - // Gate on admin-snapshot presence rather than probing the + // Gate on DP-snapshot presence rather than probing the // virtual — probe would warm the primary's 429 cooldown and // zero out per-target counts (post-PR #268: 429 cools down // regardless of retry_on_429). - await waitConfigPropagation(async () => { - try { - const models = await admin!.listModels(); - return models.some((m) => m.display_name === "routing-429-virtual"); - } catch { - return false; - } - }); + await waitSeedApplied("routing-429"); const primaryBaseline = primary.receivedRequests.length; const secondaryBaseline = secondary.receivedRequests.length; @@ -247,7 +261,7 @@ describe("routing strategies and retry behavior e2e", () => { }); test("round_robin rotates the starting target between requests", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -290,7 +304,7 @@ describe("routing strategies and retry behavior e2e", () => { await createOpenAiModel("routing-rr-b", second); await waitUntilModelResponds("routing-rr-a", "rr-a"); await waitUntilModelResponds("routing-rr-b", "rr-b"); - await admin.createModel({ + await seed.createModel({ display_name: "routing-rr-virtual", routing: { strategy: "round_robin", @@ -337,7 +351,7 @@ describe("routing strategies and retry behavior e2e", () => { }); test("weighted picks the positive-weight target first and falls forward from there", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -385,7 +399,7 @@ describe("routing strategies and retry behavior e2e", () => { await createOpenAiModel("routing-weighted-fallback", forwardFallback); await waitUntilModelResponds("routing-weighted-before", "should-not-run"); await waitUntilModelResponds("routing-weighted-fallback", "weighted fallback worked"); - await admin.createModel({ + await seed.createModel({ display_name: "routing-weighted-virtual", routing: { strategy: "weighted", @@ -404,19 +418,12 @@ describe("routing strategies and retry behavior e2e", () => { maxRetries: 0, }); - // Gate on admin-snapshot presence rather than probing the + // Gate on DP-snapshot presence rather than probing the // virtual — probe would warm the weighted primary's 502 cooldown // and skew per-target hit counts. Both direct models' readiness // was already established above; this confirms the routing record // has reached the DP snapshot. - await waitConfigPropagation(async () => { - try { - const models = await admin!.listModels(); - return models.some((m) => m.display_name === "routing-weighted-virtual"); - } catch { - return false; - } - }); + await waitSeedApplied("routing-weighted"); const beforeBaseline = zeroWeightBefore.receivedRequests.length; const primaryBaseline = weightedPrimary.receivedRequests.length; diff --git a/tests/e2e/src/cases/runtime-mixed-filtering-e2e.test.ts b/tests/e2e/src/cases/runtime-mixed-filtering-e2e.test.ts index e62d372a..da8da13d 100644 --- a/tests/e2e/src/cases/runtime-mixed-filtering-e2e.test.ts +++ b/tests/e2e/src/cases/runtime-mixed-filtering-e2e.test.ts @@ -4,6 +4,7 @@ import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -19,13 +20,15 @@ const CALLER_KEY_HASH = createHash("sha256") describe("runtime mixed filtering e2e", () => { let app: SpawnedApp | undefined; let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; let unhealthyUpstream: OpenAiUpstream | undefined; let cooldownUpstream: OpenAiUpstream | undefined; let healthyUpstream: OpenAiUpstream | undefined; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; unhealthyUpstream = await startOpenAiUpstream({ @@ -75,24 +78,25 @@ describe("runtime mixed filtering e2e", () => { app = await spawnApp(); admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const unhealthyPk = await admin.createProviderKey({ + const unhealthyPk = await seed.createProviderKey({ display_name: "mixed-unhealthy-pk", secret: "sk-mock", api_base: `${unhealthyUpstream.baseUrl}/v1`, }); - const cooldownPk = await admin.createProviderKey({ + const cooldownPk = await seed.createProviderKey({ display_name: "mixed-cooldown-pk", secret: "sk-mock", api_base: `${cooldownUpstream.baseUrl}/v1`, }); - const healthyPk = await admin.createProviderKey({ + const healthyPk = await seed.createProviderKey({ display_name: "mixed-healthy-pk", secret: "sk-mock", api_base: `${healthyUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "mixed-unhealthy", provider: "openai", model_name: "gpt-4o-mini", @@ -107,20 +111,20 @@ describe("runtime mixed filtering e2e", () => { stale_after_seconds: 90, }, }); - await admin.createModel({ + await seed.createModel({ display_name: "mixed-cooldown", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: cooldownPk.id, }); - await admin.createModel({ + await seed.createModel({ display_name: "mixed-healthy", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: healthyPk.id, }); - await admin.createModel({ + await seed.createModel({ display_name: "mixed-router", routing: { strategy: "failover", @@ -133,7 +137,7 @@ describe("runtime mixed filtering e2e", () => { }, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["mixed-router", "mixed-cooldown", "mixed-healthy"], }); diff --git a/tests/e2e/src/cases/runtime-status-e2e.test.ts b/tests/e2e/src/cases/runtime-status-e2e.test.ts index 4524b947..21a56f5c 100644 --- a/tests/e2e/src/cases/runtime-status-e2e.test.ts +++ b/tests/e2e/src/cases/runtime-status-e2e.test.ts @@ -4,6 +4,7 @@ import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -19,6 +20,7 @@ const CALLER_KEY_HASH = createHash("sha256") describe("runtime status e2e", () => { let app: SpawnedApp | undefined; let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; let flakyUpstream: OpenAiUpstream | undefined; let stableUpstream: OpenAiUpstream | undefined; @@ -27,7 +29,8 @@ describe("runtime status e2e", () => { let routerModelID = ""; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; flakyUpstream = await startOpenAiUpstream({ @@ -73,32 +76,33 @@ describe("runtime status e2e", () => { app = await spawnApp(); admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const flakyPk = await admin.createProviderKey({ + const flakyPk = await seed.createProviderKey({ display_name: "runtime-flaky-pk", secret: "sk-mock", api_base: `${flakyUpstream.baseUrl}/v1`, }); - const stablePk = await admin.createProviderKey({ + const stablePk = await seed.createProviderKey({ display_name: "runtime-stable-pk", secret: "sk-mock", api_base: `${stableUpstream.baseUrl}/v1`, }); - const flakyModel = await admin.createModel({ + const flakyModel = await seed.createModel({ display_name: "runtime-flaky", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: flakyPk.id, }); flakyModelID = flakyModel.id; - const stableModel = await admin.createModel({ + const stableModel = await seed.createModel({ display_name: "runtime-stable", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: stablePk.id, }); stableModelID = stableModel.id; - const routerModel = await admin.createModel({ + const routerModel = await seed.createModel({ display_name: "runtime-router", routing: { strategy: "failover", @@ -108,7 +112,7 @@ describe("runtime status e2e", () => { }, }); routerModelID = routerModel.id; - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["runtime-router", "runtime-stable", "runtime-flaky"], }); diff --git a/tests/e2e/src/cases/seed-vs-admin-characterization-e2e.test.ts b/tests/e2e/src/cases/seed-vs-admin-characterization-e2e.test.ts index 6a6ffb04..a26090fe 100644 --- a/tests/e2e/src/cases/seed-vs-admin-characterization-e2e.test.ts +++ b/tests/e2e/src/cases/seed-vs-admin-characterization-e2e.test.ts @@ -265,4 +265,88 @@ describe("seed-vs-admin characterization: direct etcd writes ≡ Admin API write normalize(find(exporters, adminExporter.id, "admin exporter").value, ["name"]), ); }); + + // Declared last on purpose: it mutates then removes seed-created + // resources the earlier tests rely on. + test("seed update rewrites the stored document; seed delete revokes it", async (ctx) => { + if (!etcdReachable || !admin || !app || !upstream || !seed) { + ctx.skip(); + return; + } + + // -- update: overwrite the seeded model's document with a new + // upstream model_name. Admin GET reads etcd directly, so the store + // must reflect the write immediately; the proxy snapshot follows + // after propagation. + const updated = { ...seedModel.value, model_name: "gpt-4o-mini-updated" }; + await seed.update("models", seedModel.id, updated); + + const models = await admin.json("GET", "/admin/v1/models"); + expect(models.find((m) => m.id === seedModel.id)?.value.model_name).toBe( + "gpt-4o-mini-updated", + ); + + const seedCaller = new OpenAI({ + apiKey: SEED_CALLER, + baseURL: `${app.proxyUrl}/v1`, + maxRetries: 0, + }); + // Propagated when a chat through the alias forwards the NEW + // model_name to the upstream — the strongest observable effect of + // the update (a 200 alone would pass on the stale snapshot too). + await waitConfigPropagation(async () => { + try { + const r = await seedCaller.chat.completions.create({ + model: "char-seed-model", + messages: [{ role: "user", content: "update-probe" }], + }); + if (r.choices[0]?.message.role !== "assistant") return false; + const last = upstream!.receivedRequests.at(-1); + return ( + last !== undefined && + (JSON.parse(last.body) as { model?: string }).model === + "gpt-4o-mini-updated" + ); + } catch { + return false; + } + }); + + // -- delete: remove the seeded caller key. The store no longer + // returns it, and after propagation its bearer stops + // authenticating (fail-closed), while the admin-created caller is + // untouched. + await seed.delete("api_keys", seedKey.id); + expect( + await etcdClient!.get(`${app.etcdPrefix}/api_keys/${seedKey.id}`), + ).toBeUndefined(); + + await waitConfigPropagation(async () => { + try { + await seedCaller.chat.completions.create({ + model: "char-seed-model", + messages: [{ role: "user", content: "revoked-probe" }], + }); + return false; // still authenticating — not propagated yet + } catch (err) { + return ( + err instanceof APIError && (err.status === 401 || err.status === 403) + ); + } + }); + + // Control probe: the delete revoked only that key — the + // admin-created caller still serves (rules out "gateway fell over" + // as the reason the seed caller went dark). + const adminCaller = new OpenAI({ + apiKey: ADMIN_CALLER, + baseURL: `${app.proxyUrl}/v1`, + maxRetries: 0, + }); + const control = await adminCaller.chat.completions.create({ + model: "char-admin-model", + messages: [{ role: "user", content: "control-probe" }], + }); + expect(control.choices[0]?.message.role).toBe("assistant"); + }); }); diff --git a/tests/e2e/src/cases/semantic-routing-e2e.test.ts b/tests/e2e/src/cases/semantic-routing-e2e.test.ts index 5d7e4c2d..f89057ec 100644 --- a/tests/e2e/src/cases/semantic-routing-e2e.test.ts +++ b/tests/e2e/src/cases/semantic-routing-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import { createServer, type Server } from "node:http"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -134,18 +134,19 @@ function chatUpstreamReplying(content: string): Promise { describe("semantic routing e2e", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; const upstreams: OpenAiUpstream[] = []; const embedMocks: EmbeddingMock[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createApiKey({ + seed = new SeedClient(etcd, app.etcdPrefix); + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); @@ -161,7 +162,7 @@ describe("semantic routing e2e", () => { await createEmbeddingModel("bge-mock", embed); await createDirectModel("legal-model", legal); await createDirectModel("default-model", fallback); - await admin.createModel({ + await seed.createModel({ display_name: "prod-chat", semantic: { embedding_model: "bge-mock", @@ -198,13 +199,13 @@ describe("semantic routing e2e", () => { displayName: string, upstream: OpenAiUpstream, ): Promise { - if (!admin) throw new Error("admin not ready"); - const pk = await admin.createProviderKey({ + if (!seed) throw new Error("seed not ready"); + const pk = await seed.createProviderKey({ display_name: `${displayName}-pk`, secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: displayName, provider: "openai", model_name: "gpt-4o-mini", @@ -216,13 +217,13 @@ describe("semantic routing e2e", () => { displayName: string, mock: EmbeddingMock, ): Promise { - if (!admin) throw new Error("admin not ready"); - const pk = await admin.createProviderKey({ + if (!seed) throw new Error("seed not ready"); + const pk = await seed.createProviderKey({ display_name: `${displayName}-pk`, secret: "sk-mock", api_base: `${mock.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: displayName, provider: "openai", model_name: "embed-mock", @@ -268,7 +269,7 @@ describe("semantic routing e2e", () => { } test("routes a matching prompt to its route target and sets x-aisix-route", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -282,7 +283,7 @@ describe("semantic routing e2e", () => { }); test("falls through to default when no route clears its threshold (no x-aisix-route)", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -296,7 +297,7 @@ describe("semantic routing e2e", () => { }); test("on_embedding_failure: default routes to the default model when embedding errors", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -307,7 +308,7 @@ describe("semantic routing e2e", () => { await createEmbeddingModel("broken-embed", brokenEmbed); await createDirectModel("safe-default-model", fallback); - await admin.createModel({ + await seed.createModel({ display_name: "degrading-router", semantic: { embedding_model: "broken-embed", @@ -342,7 +343,7 @@ describe("semantic routing e2e", () => { }); test("on_embedding_failure: fail returns 503 when embedding errors", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -353,7 +354,7 @@ describe("semantic routing e2e", () => { await createEmbeddingModel("broken-embed-2", brokenEmbed); await createDirectModel("unreached-model", target); - await admin.createModel({ + await seed.createModel({ display_name: "strict-router", semantic: { embedding_model: "broken-embed-2", diff --git a/tests/e2e/src/cases/server-header-identity-e2e.test.ts b/tests/e2e/src/cases/server-header-identity-e2e.test.ts index 5073e9d9..b39345df 100644 --- a/tests/e2e/src/cases/server-header-identity-e2e.test.ts +++ b/tests/e2e/src/cases/server-header-identity-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -64,25 +64,26 @@ describe("data plane identifies itself via Server header on every response", () let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); app = await spawnApp(); - const admin = new AdminClient(app.adminUrl, app.adminKey); + const seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "server-header-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "server-header-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: VALID_KEY_HASH, allowed_models: ["server-header-model"], }); diff --git a/tests/e2e/src/cases/sls-content-capture-masked-e2e.test.ts b/tests/e2e/src/cases/sls-content-capture-masked-e2e.test.ts index 7b2da753..d3438c26 100644 --- a/tests/e2e/src/cases/sls-content-capture-masked-e2e.test.ts +++ b/tests/e2e/src/cases/sls-content-capture-masked-e2e.test.ts @@ -1,9 +1,9 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, decodedTextFor, EtcdClient, + SeedClient, spawnApp, startMockSls, startOpenAiUpstream, @@ -115,8 +115,8 @@ describe("sls content capture e2e: streaming capture is post-mask (#932 × AISIX }, }); apps.push(app); - const admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createObservabilityExporter({ + const seed = new SeedClient(new EtcdClient(), app.etcdPrefix); + await seed.createObservabilityExporter({ name: "sls-full-masked", enabled: true, kind: "aliyun_sls", @@ -126,7 +126,7 @@ describe("sls content capture e2e: streaming capture is post-mask (#932 × AISIX credential_ref: CREDENTIAL_REF, content_mode: "full", }); - await admin.createObservabilityExporter({ + await seed.createObservabilityExporter({ name: "sls-meta-masked", enabled: true, kind: "aliyun_sls", @@ -136,38 +136,38 @@ describe("sls content capture e2e: streaming capture is post-mask (#932 × AISIX credential_ref: CREDENTIAL_REF, content_mode: "metadata_only", }); - const chatPk = await admin.createProviderKey({ + const chatPk = await seed.createProviderKey({ display_name: "masked-chat-pk", secret: PROVIDER_SECRET, api_base: `${chatUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "masked-chat-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: chatPk.id, }); - const bridgePk = await admin.createProviderKey({ + const bridgePk = await seed.createProviderKey({ display_name: "masked-bridge-pk", secret: PROVIDER_SECRET, api_base: `${bridgeUpstream.baseUrl}/v1`, provider: "deepseek", adapter: "openai", }); - await admin.createModel({ + await seed.createModel({ display_name: "masked-bridge-model", provider: "deepseek", model_name: "gpt-4o-mini", provider_key_id: bridgePk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["masked-chat-model", "masked-bridge-model"], }); // Env-wide output-hook PII guardrail: email → mask. Its presence also // forces the streaming hold-back (BufferFull) path — the path where // the capture/wire divergence lived. - await admin.json("POST", "/admin/v1/guardrails", { + await seed.createGuardrail({ name: "masked-capture-guard", enabled: true, hook_point: "output", diff --git a/tests/e2e/src/cases/sls-content-capture-nontext-e2e.test.ts b/tests/e2e/src/cases/sls-content-capture-nontext-e2e.test.ts index 05ff1a35..57b58110 100644 --- a/tests/e2e/src/cases/sls-content-capture-nontext-e2e.test.ts +++ b/tests/e2e/src/cases/sls-content-capture-nontext-e2e.test.ts @@ -1,9 +1,9 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, decodedTextFor, EtcdClient, + SeedClient, spawnApp, startMockSls, startOpenAiUpstream, @@ -130,8 +130,8 @@ describe("sls content capture e2e (#700): embeddings / rerank / images / audio", }, }); apps.push(app); - const admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createObservabilityExporter({ + const seedClient = new SeedClient(new EtcdClient(), app.etcdPrefix); + await seedClient.createObservabilityExporter({ name: "sls-full-nontext", enabled: true, kind: "aliyun_sls", @@ -141,7 +141,7 @@ describe("sls content capture e2e (#700): embeddings / rerank / images / audio", credential_ref: CREDENTIAL_REF, content_mode: "full", }); - await admin.createObservabilityExporter({ + await seedClient.createObservabilityExporter({ name: "sls-meta-nontext", enabled: true, kind: "aliyun_sls", @@ -152,12 +152,12 @@ describe("sls content capture e2e (#700): embeddings / rerank / images / audio", content_mode: "metadata_only", }); const seed = async (name: string, modelName: string, upstream: OpenAiUpstream) => { - const pk = await admin.createProviderKey({ + const pk = await seedClient.createProviderKey({ display_name: `${name}-pk`, secret: PROVIDER_SECRET, api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seedClient.createModel({ display_name: name, provider: "openai", model_name: modelName, @@ -169,7 +169,7 @@ describe("sls content capture e2e (#700): embeddings / rerank / images / audio", await seed("cc700-images", "dall-e-3", imagesUpstream); await seed("cc700-speech", "tts-1", speechUpstream); await seed("cc700-transcribe", "whisper-1", transcribeUpstream); - await admin.createApiKey({ + await seedClient.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: [ "cc700-embed", diff --git a/tests/e2e/src/cases/sls-content-capture-responses-completions-e2e.test.ts b/tests/e2e/src/cases/sls-content-capture-responses-completions-e2e.test.ts index cce3e9cf..6b79ef3e 100644 --- a/tests/e2e/src/cases/sls-content-capture-responses-completions-e2e.test.ts +++ b/tests/e2e/src/cases/sls-content-capture-responses-completions-e2e.test.ts @@ -1,9 +1,9 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, decodedTextFor, EtcdClient, + SeedClient, spawnApp, startMockSls, startOpenAiUpstream, @@ -135,7 +135,8 @@ describe("sls content capture e2e (AISIX-Cloud#947): /v1/responses + /v1/complet const apps: SpawnedApp[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; responsesUpstream = await startOpenAiUpstream({ nonStreamBody: responsesBody(`sure, ${RESP_RESPONSE_TOKEN} noted`), @@ -212,8 +213,8 @@ describe("sls content capture e2e (AISIX-Cloud#947): /v1/responses + /v1/complet }, }); apps.push(app); - const admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createObservabilityExporter({ + const seed = new SeedClient(new EtcdClient(), app.etcdPrefix); + await seed.createObservabilityExporter({ name: "sls-full-947", enabled: true, kind: "aliyun_sls", @@ -223,7 +224,7 @@ describe("sls content capture e2e (AISIX-Cloud#947): /v1/responses + /v1/complet credential_ref: CREDENTIAL_REF, content_mode: "full", }); - await admin.createObservabilityExporter({ + await seed.createObservabilityExporter({ name: "sls-meta-947", enabled: true, kind: "aliyun_sls", @@ -235,14 +236,14 @@ describe("sls content capture e2e (AISIX-Cloud#947): /v1/responses + /v1/complet }); const seedModel = async (name: string, provider: string, upstream: OpenAiUpstream) => { - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: `${name}-pk`, secret: PROVIDER_SECRET, api_base: `${upstream.baseUrl}/v1`, provider, adapter: "openai", }); - await admin.createModel({ + await seed.createModel({ display_name: name, provider, model_name: "gpt-4o-mini", @@ -254,7 +255,7 @@ describe("sls content capture e2e (AISIX-Cloud#947): /v1/responses + /v1/complet await seedModel("cc947-bridge", "deepseek", bridgeUpstream); await seedModel("cc947-bridge-stream", "deepseek", bridgeStreamUpstream); await seedModel("cc947-completions", "openai", completionsUpstream); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: [ "cc947-responses", diff --git a/tests/e2e/src/cases/sls-failure-content-e2e.test.ts b/tests/e2e/src/cases/sls-failure-content-e2e.test.ts index 51b516b6..83ac2b5d 100644 --- a/tests/e2e/src/cases/sls-failure-content-e2e.test.ts +++ b/tests/e2e/src/cases/sls-failure-content-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, lz4DecompressBlock, spawnApp, startMockSls, @@ -170,7 +170,8 @@ describe("sls e2e: failed requests record the request body (#1013)", () => { let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; sls = await startMockSls(); @@ -203,9 +204,9 @@ describe("sls e2e: failed requests record the request body (#1013)", () => { [`SLS_CRED_${CREDENTIAL_REF.toUpperCase()}_AK_SECRET`]: MOCK_AK_SECRET, }, }); - const admin = new AdminClient(app.adminUrl, app.adminKey); + const seed = new SeedClient(etcd, app.etcdPrefix); - await admin.createObservabilityExporter({ + await seed.createObservabilityExporter({ name: "sls-failure-full", enabled: true, kind: "aliyun_sls", @@ -215,7 +216,7 @@ describe("sls e2e: failed requests record the request body (#1013)", () => { credential_ref: CREDENTIAL_REF, content_mode: "full", }); - await admin.createObservabilityExporter({ + await seed.createObservabilityExporter({ name: "sls-failure-meta", enabled: true, kind: "aliyun_sls", @@ -226,23 +227,23 @@ describe("sls e2e: failed requests record the request body (#1013)", () => { content_mode: "metadata_only", }); - const okPk = await admin.createProviderKey({ + const okPk = await seed.createProviderKey({ display_name: "failure-content-ok-pk", secret: "sk-mock", api_base: `${okUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "failure-content-ok", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: okPk.id, }); - const failPk = await admin.createProviderKey({ + const failPk = await seed.createProviderKey({ display_name: "failure-content-fail-pk", secret: "sk-mock", api_base: `${failUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "failure-content-fail", provider: "openai", model_name: "gpt-4o-mini", @@ -252,7 +253,7 @@ describe("sls e2e: failed requests record the request body (#1013)", () => { cooldown: { enabled: false }, }); // A model the caller is NOT allowed to use (403 case). - await admin.createModel({ + await seed.createModel({ display_name: "failure-content-offlimits", provider: "openai", model_name: "gpt-4o-mini", @@ -260,14 +261,14 @@ describe("sls e2e: failed requests record the request body (#1013)", () => { }); // Second hard-failing target so a routing group can fail on BOTH // targets (content must ride the LAST attempt only). - await admin.createModel({ + await seed.createModel({ display_name: "failure-content-fail-b", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: failPk.id, cooldown: { enabled: false }, }); - await admin.createModel({ + await seed.createModel({ display_name: "failure-content-group", routing: { strategy: "failover", @@ -279,7 +280,7 @@ describe("sls e2e: failed requests record the request body (#1013)", () => { }); // Fail-then-recover group: the failed attempt must stay content-less // while the winner's success event carries the prompt. - await admin.createModel({ + await seed.createModel({ display_name: "failure-content-recover", routing: { strategy: "failover", @@ -290,7 +291,7 @@ describe("sls e2e: failed requests record the request body (#1013)", () => { }, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: [ "failure-content-ok", @@ -301,7 +302,7 @@ describe("sls e2e: failed requests record the request body (#1013)", () => { }); // Input-side keyword guardrail (block mode) for the 422 case. - await admin.json("POST", "/admin/v1/guardrails", { + await seed.createGuardrail({ name: "failure-content-guard", enabled: true, hook_point: "input", @@ -311,7 +312,7 @@ describe("sls e2e: failed requests record the request body (#1013)", () => { // PII guardrail: email masks, china_id_card blocks. A blocked request // carrying BOTH must capture the post-mask body (the email placeholder, // never the raw address). - await admin.json("POST", "/admin/v1/guardrails", { + await seed.createGuardrail({ name: "failure-content-pii", enabled: true, hook_point: "input", diff --git a/tests/e2e/src/cases/sls-structured-truncation-e2e.test.ts b/tests/e2e/src/cases/sls-structured-truncation-e2e.test.ts index 81bda81b..a943108d 100644 --- a/tests/e2e/src/cases/sls-structured-truncation-e2e.test.ts +++ b/tests/e2e/src/cases/sls-structured-truncation-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, lz4DecompressBlock, spawnApp, startMockSls, @@ -140,7 +140,8 @@ describe("sls e2e: oversized captured content is truncated structurally (#1014)" let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; sls = await startMockSls(); @@ -167,9 +168,9 @@ describe("sls e2e: oversized captured content is truncated structurally (#1014)" [`SLS_CRED_${CREDENTIAL_REF.toUpperCase()}_AK_SECRET`]: MOCK_AK_SECRET, }, }); - const admin = new AdminClient(app.adminUrl, app.adminKey); + const seed = new SeedClient(etcd, app.etcdPrefix); - await admin.createObservabilityExporter({ + await seed.createObservabilityExporter({ name: "sls-structured-trunc", enabled: true, kind: "aliyun_sls", @@ -181,18 +182,18 @@ describe("sls e2e: oversized captured content is truncated structurally (#1014)" content_max_bytes: CONTENT_MAX_BYTES, }); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "sls-trunc-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "sls-trunc-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["sls-trunc-model"], }); diff --git a/tests/e2e/src/cases/streaming-edges-e2e.test.ts b/tests/e2e/src/cases/streaming-edges-e2e.test.ts index 6e4a6621..f3ea8300 100644 --- a/tests/e2e/src/cases/streaming-edges-e2e.test.ts +++ b/tests/e2e/src/cases/streaming-edges-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, ProxyClient, spawnApp, startOpenAiUpstream, @@ -51,17 +51,18 @@ const CALLER_KEY_HASH = createHash("sha256") describe("streaming edges e2e: client abort mid-stream", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; const upstreams: OpenAiUpstream[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createApiKey({ + seed = new SeedClient(etcd, app.etcdPrefix); + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); @@ -73,7 +74,7 @@ describe("streaming edges e2e: client abort mid-stream", () => { }); test("client aborts mid-stream: gateway stays healthy, subsequent request succeeds", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -94,12 +95,12 @@ describe("streaming edges e2e: client abort mid-stream", () => { }); upstreams.push(upstream); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "stream-abort-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "stream-abort", provider: "openai", model_name: "gpt-4o-mini", @@ -184,7 +185,7 @@ describe("streaming edges e2e: client abort mid-stream", () => { }); test("upstream disconnects mid-stream: partial chunks reach caller, iterator throws, no [DONE]", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -214,12 +215,12 @@ describe("streaming edges e2e: client abort mid-stream", () => { }); upstreams.push(upstream); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "stream-disc-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "stream-disc", provider: "openai", model_name: "gpt-4o-mini", diff --git a/tests/e2e/src/cases/streaming-tpm-commit-e2e.test.ts b/tests/e2e/src/cases/streaming-tpm-commit-e2e.test.ts index ada17863..668ad643 100644 --- a/tests/e2e/src/cases/streaming-tpm-commit-e2e.test.ts +++ b/tests/e2e/src/cases/streaming-tpm-commit-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, ProxyClient, spawnApp, startOpenAiUpstream, @@ -101,24 +101,25 @@ describe("streaming TPM commit (#688)", () => { let app: SpawnedApp | undefined; let upstreamMsg: OpenAiUpstream | undefined; let upstreamResp: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstreamMsg = await startOpenAiUpstream({ streamEvents: MSG_STREAM, eventDelayMs: 2 }); upstreamResp = await startOpenAiUpstream({ streamEvents: RESP_STREAM, eventDelayMs: 2 }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pkMsg = await admin.createProviderKey({ + const pkMsg = await seed.createProviderKey({ display_name: "688-msg-pk", secret: "sk-mock", api_base: `${upstreamMsg.baseUrl}/v1`, }); - const pkResp = await admin.createProviderKey({ + const pkResp = await seed.createProviderKey({ display_name: "688-resp-pk", secret: "sk-mock", api_base: `${upstreamResp.baseUrl}/v1`, @@ -126,25 +127,25 @@ describe("streaming TPM commit (#688)", () => { // Both models are OpenAI-protocol: /v1/messages bridges to it // (cross_provider_dispatch), /v1/responses forwards verbatim // (responses_to_target). - await admin.createModel({ + await seed.createModel({ display_name: "msg-tpm", provider: "openai", model_name: "up-688", provider_key_id: pkMsg.id, }); - await admin.createModel({ + await seed.createModel({ display_name: "resp-tpm", provider: "openai", model_name: "up-688", provider_key_id: pkResp.id, }); // Separate caller keys so each endpoint's TPM counter is independent. - await admin.createApiKey({ + await seed.createApiKey({ key_hash: hash(MSG_CALLER), allowed_models: ["msg-tpm"], rate_limit: { tpm: TPM }, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: hash(RESP_CALLER), allowed_models: ["resp-tpm"], rate_limit: { tpm: TPM }, diff --git a/tests/e2e/src/cases/streaming-usage-accumulation-e2e.test.ts b/tests/e2e/src/cases/streaming-usage-accumulation-e2e.test.ts index 14ff7c22..b9692f9e 100644 --- a/tests/e2e/src/cases/streaming-usage-accumulation-e2e.test.ts +++ b/tests/e2e/src/cases/streaming-usage-accumulation-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -78,11 +78,12 @@ describe("streaming usage accumulation e2e: stream final usage == non-stream usa let app: SpawnedApp | undefined; let nonStreamUpstream: OpenAiUpstream | undefined; let streamUpstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; nonStreamUpstream = await startOpenAiUpstream({ @@ -93,35 +94,35 @@ describe("streaming usage accumulation e2e: stream final usage == non-stream usa }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); // Two ProviderKeys → two Models so receivedRequests counts on // each mock are unambiguous, and so the streaming and non- // streaming calls cannot accidentally cross-pollinate request // bodies in the assertion below. - const pkNon = await admin.createProviderKey({ + const pkNon = await seed.createProviderKey({ display_name: "stream-usage-non-pk", secret: "sk-mock", api_base: `${nonStreamUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "stream-usage-non", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pkNon.id, }); - const pkStream = await admin.createProviderKey({ + const pkStream = await seed.createProviderKey({ display_name: "stream-usage-stream-pk", secret: "sk-mock", api_base: `${streamUpstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "stream-usage-stream", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pkStream.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["stream-usage-non", "stream-usage-stream"], }); diff --git a/tests/e2e/src/cases/tag-routing-e2e.test.ts b/tests/e2e/src/cases/tag-routing-e2e.test.ts index 98d8dac9..0f57d8c1 100644 --- a/tests/e2e/src/cases/tag-routing-e2e.test.ts +++ b/tests/e2e/src/cases/tag-routing-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -35,17 +35,18 @@ function okBody(content: string) { describe("tag/metadata conditional routing e2e", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; const upstreams: OpenAiUpstream[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createApiKey({ + seed = new SeedClient(etcd, app.etcdPrefix); + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); @@ -60,13 +61,13 @@ describe("tag/metadata conditional routing e2e", () => { displayName: string, upstream: OpenAiUpstream, ): Promise { - if (!admin) throw new Error("admin client not initialized"); - const providerKey = await admin.createProviderKey({ + if (!seed) throw new Error("seed client not initialized"); + const providerKey = await seed.createProviderKey({ display_name: `${displayName}-pk`, secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: displayName, provider: "openai", model_name: "gpt-4o-mini", @@ -91,8 +92,28 @@ describe("tag/metadata conditional routing e2e", () => { return completion.choices[0]?.message.content ?? null; } + + // Seed a throwaway canary key AFTER the resources under test — watch + // events apply in revision order, so once this bearer authenticates + // against /v1/models everything written before it is in the snapshot + // too. Probing the routed models directly would warm cooldowns and + // skew the per-target hit counts the assertions rely on. + async function waitSeedApplied(label: string): Promise { + const canary = `sk-canary-${label}-${Date.now()}`; + await seed!.createApiKey({ + key_hash: createHash("sha256").update(canary).digest("hex"), + allowed_models: ["*"], + }); + await waitConfigPropagation(async () => { + const res = await fetch(`${app!.proxyUrl}/v1/models`, { + headers: { authorization: `Bearer ${canary}` }, + }); + return res.status === 200; + }); + } + test("selects the tagged target, defaulting when unmatched or untagged", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -106,7 +127,7 @@ describe("tag/metadata conditional routing e2e", () => { await createOpenAiModel("tag-default", def); // failover keeps target selection deterministic: whatever the tag filter // leaves, the first survivor is attempted. - await admin.createModel({ + await seed.createModel({ display_name: "tag-router", routing: { strategy: "failover", @@ -118,10 +139,7 @@ describe("tag/metadata conditional routing e2e", () => { }, }); - await waitConfigPropagation(async () => { - const models = await admin!.listModels(); - return models.some((m) => m.display_name === "tag-router"); - }); + await waitSeedApplied("tag-router"); // Matching tags route to their target. expect(await askWithTags("eu")).toBe("eu-served"); diff --git a/tests/e2e/src/cases/team-member-ratelimit-e2e.test.ts b/tests/e2e/src/cases/team-member-ratelimit-e2e.test.ts index 41ddf345..94f89447 100644 --- a/tests/e2e/src/cases/team-member-ratelimit-e2e.test.ts +++ b/tests/e2e/src/cases/team-member-ratelimit-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, ProxyClient, spawnApp, startOpenAiUpstream, @@ -39,13 +39,13 @@ describe("rate limit e2e: team_member per-member default buckets", () => { let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream(); app = await spawnApp(); - const admin = new AdminClient(app.adminUrl, app.adminKey); - const etcd = new EtcdClient(); + const seed = new SeedClient(etcd, app.etcdPrefix); // Seed the policy FIRST (lower etcd revision) so that once the // probe sees the model (created below, higher revision) the policy @@ -61,12 +61,12 @@ describe("rate limit e2e: team_member per-member default buckets", () => { }), ); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "tm-e2e-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "tm-e2e", provider: "openai", model_name: "gpt-4o-mini", diff --git a/tests/e2e/src/cases/timeout-fallback-e2e.test.ts b/tests/e2e/src/cases/timeout-fallback-e2e.test.ts index ca9371bb..8b48334e 100644 --- a/tests/e2e/src/cases/timeout-fallback-e2e.test.ts +++ b/tests/e2e/src/cases/timeout-fallback-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -75,7 +75,7 @@ function streamFor(content: string): string[] { describe("timeout fallback e2e (#554)", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; // Non-streaming upstreams. @@ -89,7 +89,8 @@ describe("timeout fallback e2e (#554)", () => { let stBackup: OpenAiUpstream | undefined; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; nsSlow = await startOpenAiUpstream({ @@ -116,11 +117,11 @@ describe("timeout fallback e2e (#554)", () => { }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); const pk = async (name: string, u: OpenAiUpstream) => ( - await admin!.createProviderKey({ + await seed!.createProviderKey({ display_name: name, secret: "sk-mock", api_base: `${u.baseUrl}/v1`, @@ -138,7 +139,7 @@ describe("timeout fallback e2e (#554)", () => { // Direct targets. Cooldown is disabled on the slow/stall primaries so a // timeout doesn't take them out of rotation between probe and test call // (mirrors fallback-e2e). - await admin.createModel({ + await seed.createModel({ display_name: "t-m-ns-slow", provider: "openai", model_name: "gpt-4o-mini", @@ -146,20 +147,20 @@ describe("timeout fallback e2e (#554)", () => { timeout: TIMEOUT_MS, cooldown: { enabled: false }, }); - await admin.createModel({ + await seed.createModel({ display_name: "t-m-ns-fast", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: nsFastPk, timeout: GENEROUS_MS, }); - await admin.createModel({ + await seed.createModel({ display_name: "t-m-ns-backup", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: nsBackupPk, }); - await admin.createModel({ + await seed.createModel({ display_name: "t-m-st-slow", provider: "openai", model_name: "gpt-4o-mini", @@ -167,14 +168,14 @@ describe("timeout fallback e2e (#554)", () => { stream_timeout: TIMEOUT_MS, cooldown: { enabled: false }, }); - await admin.createModel({ + await seed.createModel({ display_name: "t-m-st-fast", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: stFastPk, stream_timeout: GENEROUS_MS, }); - await admin.createModel({ + await seed.createModel({ display_name: "t-m-st-stall", provider: "openai", model_name: "gpt-4o-mini", @@ -182,7 +183,7 @@ describe("timeout fallback e2e (#554)", () => { stream_timeout: TIMEOUT_MS, cooldown: { enabled: false }, }); - await admin.createModel({ + await seed.createModel({ display_name: "t-m-st-backup", provider: "openai", model_name: "gpt-4o-mini", @@ -190,7 +191,7 @@ describe("timeout fallback e2e (#554)", () => { }); const router = async (name: string, targets: string[]) => - admin!.createModel({ + seed!.createModel({ display_name: name, routing: { strategy: "failover", targets: targets.map((model) => ({ model })) }, }); @@ -200,7 +201,7 @@ describe("timeout fallback e2e (#554)", () => { await router("t-r-st-fast", ["t-m-st-fast", "t-m-st-backup"]); await router("t-r-st-stall", ["t-m-st-stall", "t-m-st-backup"]); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: [ "t-r-ns-timeout", diff --git a/tests/e2e/src/cases/token-usage-passthrough-e2e.test.ts b/tests/e2e/src/cases/token-usage-passthrough-e2e.test.ts index 10d4a9e9..b95a8c85 100644 --- a/tests/e2e/src/cases/token-usage-passthrough-e2e.test.ts +++ b/tests/e2e/src/cases/token-usage-passthrough-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -42,11 +42,12 @@ const EXPECTED_TOTAL_TOKENS = 8; describe("token usage e2e: upstream usage block reaches client byte-for-byte", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ @@ -70,20 +71,20 @@ describe("token usage e2e: upstream usage block reaches client byte-for-byte", ( }, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "tu-e2e-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "tu-e2e", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["tu-e2e"], }); diff --git a/tests/e2e/src/cases/tokens-by-client-total-1002-e2e.test.ts b/tests/e2e/src/cases/tokens-by-client-total-1002-e2e.test.ts index 2e2ac16b..9d41c11f 100644 --- a/tests/e2e/src/cases/tokens-by-client-total-1002-e2e.test.ts +++ b/tests/e2e/src/cases/tokens-by-client-total-1002-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, ProxyClient, spawnApp, startOpenAiUpstream, @@ -76,33 +76,34 @@ function seriesValue(text: string, tokenType: string): number | undefined { describe("aisix_llm_tokens_by_client_total token_type=total is cache-inclusive (AISIX-Cloud#1002)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ nonStreamBody: anthropicMessageBody(USAGE), }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: `${MODEL}-pk`, provider: "anthropic", adapter: "anthropic", secret: "sk-ant-mock", api_base: upstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: MODEL, provider: "anthropic", model_name: "claude-3-5-haiku-20241022", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: hash(CALLER), allowed_models: [MODEL], }); diff --git a/tests/e2e/src/cases/tools-cross-provider-e2e.test.ts b/tests/e2e/src/cases/tools-cross-provider-e2e.test.ts index 1dee4b2c..7b7d10ae 100644 --- a/tests/e2e/src/cases/tools-cross-provider-e2e.test.ts +++ b/tests/e2e/src/cases/tools-cross-provider-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -57,11 +57,12 @@ const CALLER_KEY_HASH = createHash("sha256") describe("tools cross-provider e2e: OpenAI tools → Anthropic upstream tool_use → OpenAI tool_calls back", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; // Mock Anthropic upstream returns a Messages-shape response @@ -87,24 +88,24 @@ describe("tools cross-provider e2e: OpenAI tools → Anthropic upstream tool_use }, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); // Anthropic api_base = bare host; bridge composes /v1/messages // (mirrors anthropic-upstream-e2e convention). - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "tools-xprov-pk", provider: "anthropic", adapter: "anthropic", secret: "sk-ant-mock", api_base: upstream.baseUrl, }); - await admin.createModel({ + await seed.createModel({ display_name: "tools-xprov", provider: "anthropic", model_name: "claude-3-5-sonnet-20241022", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["tools-xprov"], }); diff --git a/tests/e2e/src/cases/upstream-401-error-code-e2e.test.ts b/tests/e2e/src/cases/upstream-401-error-code-e2e.test.ts index 8034ff70..2b269c12 100644 --- a/tests/e2e/src/cases/upstream-401-error-code-e2e.test.ts +++ b/tests/e2e/src/cases/upstream-401-error-code-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -49,11 +49,12 @@ const OPENAI_401_BODY = { describe("upstream 401 error.code preserved across Content-Type (#543)", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; // Upstream returns the canonical OpenAI 401 JSON body but labels it @@ -64,20 +65,20 @@ describe("upstream 401 error.code preserved across Content-Type (#543)", () => { errorContentType: "text/plain", }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "err401-pk", secret: "sk-mock-bad", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "err401-gpt", provider: "openai", model_name: "gpt-4o", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["err401-gpt"], }); @@ -94,23 +95,21 @@ describe("upstream 401 error.code preserved across Content-Type (#543)", () => { return; } - // Propagation probe: the model must be live (any non-5xx-from-DP - // response means config propagated; the upstream always 401s). + // Propagation probe via /v1/models: a bare "chat returns 401" gate + // would also match the gateway's own unknown-key 401 while the caller + // key is still propagating — which carries no upstream error.code and + // is exactly what this case must NOT confuse with the upstream's 401. await waitConfigPropagation(async () => { try { - const r = await fetch(`${app!.proxyUrl}/v1/chat/completions`, { - method: "POST", - headers: { - "content-type": "application/json", - authorization: `Bearer ${CALLER_PLAINTEXT}`, - }, - body: JSON.stringify({ - model: "err401-gpt", - messages: [{ role: "user", content: "probe" }], - }), + const models = await fetch(`${app!.proxyUrl}/v1/models`, { + headers: { authorization: `Bearer ${CALLER_PLAINTEXT}` }, }); - // 401 propagated correctly is what we want; 404 = model not yet live. - return r.status === 401; + if (models.status !== 200) return false; + const ids = + ((await models.json()) as { data?: Array<{ id?: string }> }).data?.map( + (m) => m.id, + ) ?? []; + return ids.includes("err401-gpt"); } catch { return false; } diff --git a/tests/e2e/src/cases/upstream-retry-after-passthrough-e2e.test.ts b/tests/e2e/src/cases/upstream-retry-after-passthrough-e2e.test.ts index b5bcf8ed..950e3b96 100644 --- a/tests/e2e/src/cases/upstream-retry-after-passthrough-e2e.test.ts +++ b/tests/e2e/src/cases/upstream-retry-after-passthrough-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI, { APIError } from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, ProxyClient, spawnApp, startOpenAiUpstream, @@ -26,11 +26,12 @@ const CALLER_KEY_HASH = createHash("sha256") describe("upstream 429 Retry-After passthrough e2e", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; // Every upstream reply is a 429 carrying `Retry-After: 30`. The @@ -42,20 +43,20 @@ describe("upstream 429 Retry-After passthrough e2e", () => { errorBody: { error: { message: "slow down", type: "rate_limit_error" } }, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "retry-after-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "retry-after-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["retry-after-model"], }); diff --git a/tests/e2e/src/cases/usage-cache-counters-e2e.test.ts b/tests/e2e/src/cases/usage-cache-counters-e2e.test.ts index 6a3b6105..38e99268 100644 --- a/tests/e2e/src/cases/usage-cache-counters-e2e.test.ts +++ b/tests/e2e/src/cases/usage-cache-counters-e2e.test.ts @@ -1,8 +1,8 @@ import { createHash } from "node:crypto"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -63,11 +63,12 @@ describe("usage cache-counter passthrough on /v1/chat/completions (#542)", () => let app: SpawnedApp | undefined; let openaiUpstream: OpenAiUpstream | undefined; let deepseekUpstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; // OpenAI-shape upstream: cache hit nested under prompt_tokens_details. @@ -110,41 +111,41 @@ describe("usage cache-counter passthrough on /v1/chat/completions (#542)", () => }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); // Post-#302 Phase A: cp-api writes `provider` + `adapter` on every // PK row; the snapshot's two-tier dispatch needs both. DeepSeek // dispatches through the OpenAI-compat family bridge (`adapter: // "openai"`) — the same bridge whose usage parser this PR fixes. - const oaiPk = await admin.createProviderKey({ + const oaiPk = await seed.createProviderKey({ display_name: "cache-oai-pk", secret: "sk-mock", api_base: `${openaiUpstream.baseUrl}/v1`, provider: "openai", adapter: "openai", }); - await admin.createModel({ + await seed.createModel({ display_name: "cache-oai", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: oaiPk.id, }); - const dsPk = await admin.createProviderKey({ + const dsPk = await seed.createProviderKey({ display_name: "cache-ds-pk", secret: "sk-mock", api_base: `${deepseekUpstream.baseUrl}/v1`, provider: "deepseek", adapter: "openai", }); - await admin.createModel({ + await seed.createModel({ display_name: "cache-ds", provider: "deepseek", model_name: "deepseek-chat", provider_key_id: dsPk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["cache-oai", "cache-ds"], }); @@ -232,21 +233,21 @@ describe("usage cache-counter passthrough on /v1/chat/completions (#542)", () => }); const plainApp = await spawnApp(); try { - const plainAdmin = new AdminClient(plainApp.adminUrl, plainApp.adminKey); - const pk = await plainAdmin.createProviderKey({ + const plainSeed = new SeedClient(new EtcdClient(), plainApp.etcdPrefix); + const pk = await plainSeed.createProviderKey({ display_name: "cache-plain-pk", secret: "sk-mock", api_base: `${plainUpstream.baseUrl}/v1`, provider: "openai", adapter: "openai", }); - await plainAdmin.createModel({ + await plainSeed.createModel({ display_name: "cache-plain", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await plainAdmin.createApiKey({ + await plainSeed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["cache-plain"], }); @@ -318,21 +319,21 @@ describe("usage cache-counter passthrough on /v1/chat/completions (#542)", () => }); const streamApp = await spawnApp(); try { - const streamAdmin = new AdminClient(streamApp.adminUrl, streamApp.adminKey); - const pk = await streamAdmin.createProviderKey({ + const streamSeed = new SeedClient(new EtcdClient(), streamApp.etcdPrefix); + const pk = await streamSeed.createProviderKey({ display_name: "cache-ds-stream-pk", secret: "sk-mock", api_base: `${streamUpstream.baseUrl}/v1`, provider: "deepseek", adapter: "openai", }); - await streamAdmin.createModel({ + await streamSeed.createModel({ display_name: "cache-ds-stream", provider: "deepseek", model_name: "deepseek-chat", provider_key_id: pk.id, }); - await streamAdmin.createApiKey({ + await streamSeed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["cache-ds-stream"], }); diff --git a/tests/e2e/src/cases/usage-client-source-e2e.test.ts b/tests/e2e/src/cases/usage-client-source-e2e.test.ts index 73e34694..15a2795f 100644 --- a/tests/e2e/src/cases/usage-client-source-e2e.test.ts +++ b/tests/e2e/src/cases/usage-client-source-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import { createServer, type Server } from "node:http"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, pickFreePort, spawnApp, startOpenAiUpstream, @@ -84,19 +84,19 @@ async function startOtlpReceiver(): Promise { }; } -async function seedRouting(admin: AdminClient, upstream: OpenAiUpstream) { - const pk = await admin.createProviderKey({ +async function seedRouting(seed: SeedClient, upstream: OpenAiUpstream) { + const pk = await seed.createProviderKey({ display_name: "client-source-pk", secret: PROVIDER_SECRET, api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "client-source-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["client-source-model"], }); @@ -161,14 +161,14 @@ describe("usage client source e2e (#492): source IP + User-Agent on usage events realIp: { trusted_proxies: ["127.0.0.1/32", "10.0.0.0/8"], recursive: true }, }); apps.push(app); - const admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createObservabilityExporter({ + const seed = new SeedClient(new EtcdClient(), app.etcdPrefix); + await seed.createObservabilityExporter({ name: "mock-otlp", enabled: true, kind: "otlp_http", endpoint: otlp.url, }); - await seedRouting(admin, upstream); + await seedRouting(seed, upstream); await waitConfigPropagation(async () => { try { @@ -208,14 +208,14 @@ describe("usage client source e2e (#492): source IP + User-Agent on usage events } const app = await spawnApp(); // no real_ip → trust nothing apps.push(app); - const admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createObservabilityExporter({ + const seed = new SeedClient(new EtcdClient(), app.etcdPrefix); + await seed.createObservabilityExporter({ name: "mock-otlp-2", enabled: true, kind: "otlp_http", endpoint: otlp.url, }); - await seedRouting(admin, upstream); + await seedRouting(seed, upstream); const marker = "aisix-e2e-peer-probe/9.9"; await waitConfigPropagation(async () => { diff --git a/tests/e2e/src/cases/vision-input-e2e.test.ts b/tests/e2e/src/cases/vision-input-e2e.test.ts index 6cf246a0..f39a60e8 100644 --- a/tests/e2e/src/cases/vision-input-e2e.test.ts +++ b/tests/e2e/src/cases/vision-input-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -59,11 +59,12 @@ const REMOTE_IMAGE_URL = "https://example.com/test-image.jpg"; describe("vision input e2e: OpenAI-native image content blocks pass through to upstream", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ @@ -86,20 +87,20 @@ describe("vision input e2e: OpenAI-native image content blocks pass through to u }, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "vision-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "vision-e2e", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["vision-e2e"], }); diff --git a/tests/e2e/src/cases/vision-messages-e2e.test.ts b/tests/e2e/src/cases/vision-messages-e2e.test.ts index 7375a0b2..b4a361d6 100644 --- a/tests/e2e/src/cases/vision-messages-e2e.test.ts +++ b/tests/e2e/src/cases/vision-messages-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -81,31 +81,32 @@ const NON_STREAM_BODY = { describe("vision messages e2e: image_url content array forwarded byte-for-byte", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ nonStreamBody: NON_STREAM_BODY, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "vision-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "vision-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["vision-model"], }); diff --git a/tests/e2e/src/cases/vision-messages-edges-e2e.test.ts b/tests/e2e/src/cases/vision-messages-edges-e2e.test.ts index 4827b10c..2f078c42 100644 --- a/tests/e2e/src/cases/vision-messages-edges-e2e.test.ts +++ b/tests/e2e/src/cases/vision-messages-edges-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -97,31 +97,32 @@ const NON_STREAM_BODY = { describe("vision messages edges e2e: multi-image, detail param, mixed-source", () => { let app: SpawnedApp | undefined; let upstream: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstream = await startOpenAiUpstream({ nonStreamBody: NON_STREAM_BODY, }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pk = await admin.createProviderKey({ + const pk = await seed.createProviderKey({ display_name: "vision-edges-pk", secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "vision-edges-model", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pk.id, }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["vision-edges-model"], }); diff --git a/tests/e2e/src/cases/weighted-routing-distribution-e2e.test.ts b/tests/e2e/src/cases/weighted-routing-distribution-e2e.test.ts index 02813c0c..f9809545 100644 --- a/tests/e2e/src/cases/weighted-routing-distribution-e2e.test.ts +++ b/tests/e2e/src/cases/weighted-routing-distribution-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -60,11 +60,12 @@ describe("weighted routing distribution e2e: 70/30 split lands inside [55,85] / let app: SpawnedApp | undefined; let upstreamA: OpenAiUpstream | undefined; let upstreamB: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstreamA = await startOpenAiUpstream({ @@ -101,28 +102,28 @@ describe("weighted routing distribution e2e: 70/30 split lands inside [55,85] / }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); // Two distinct ProviderKeys so each Model points at its own // upstream — necessary for receivedRequests counts to be // attributable per side. - const pkA = await admin.createProviderKey({ + const pkA = await seed.createProviderKey({ display_name: "wr-a-pk", secret: "sk-mock", api_base: `${upstreamA.baseUrl}/v1`, }); - const pkB = await admin.createProviderKey({ + const pkB = await seed.createProviderKey({ display_name: "wr-b-pk", secret: "sk-mock", api_base: `${upstreamB.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "wr-a", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pkA.id, }); - await admin.createModel({ + await seed.createModel({ display_name: "wr-b", provider: "openai", model_name: "gpt-4o-mini", @@ -131,7 +132,7 @@ describe("weighted routing distribution e2e: 70/30 split lands inside [55,85] / // Virtual Model: routing-only, weighted strategy. Per the schema // enum the gateway publishes (round_robin / weighted / failover), // `weighted` should honour each target's `weight` integer. - await admin.createModel({ + await seed.createModel({ display_name: "wr-virtual", routing: { strategy: "weighted", @@ -143,7 +144,7 @@ describe("weighted routing distribution e2e: 70/30 split lands inside [55,85] / }); // Caller is allowed all three Models so the readiness probes can // hit the leaves directly without firing the weighted dispatcher. - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["wr-virtual", "wr-a", "wr-b"], }); diff --git a/tests/e2e/src/cases/weighted-routing-edit-e2e.test.ts b/tests/e2e/src/cases/weighted-routing-edit-e2e.test.ts index 666774c7..4fb16d44 100644 --- a/tests/e2e/src/cases/weighted-routing-edit-e2e.test.ts +++ b/tests/e2e/src/cases/weighted-routing-edit-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -16,7 +16,7 @@ import { // // The sibling weighted-routing-distribution-e2e pins that the INITIAL // weights are honored. The gap this closes: after the model is live -// and serving, an operator PATCHes the weights via the admin API, and +// and serving, an operator rewrites the weights on the stored model, and // the change must propagate through the etcd watch and the weighted // scheduler must REBUILD — a scheduler that cached its weight wheel on // first dispatch and never rebuilt on config update would keep serving @@ -25,14 +25,13 @@ import { // Design is deterministic (no statistics): weight 0 = excluded (see // routing-strategies-e2e "weighted picks the positive-weight target"). // - Start [wr-edit-a: 100, wr-edit-b: 0] → every dispatch hits A. -// - PATCH to [wr-edit-a: 0, wr-edit-b: 100] → every dispatch hits B. +// - Edit to [wr-edit-a: 0, wr-edit-b: 100] → every dispatch hits B. // The propagation signal is unambiguous: a probe through the virtual // model returning "served by B" is IMPOSSIBLE under the old [100,0] // config, so it proves the edit is live before we count. // // Reference: OpenAI Chat Completions shape the caller sees -// (https://platform.openai.com/docs/api-reference/chat); admin model -// update is PUT /admin/v1/models/:id (docs api-admin.md). +// (https://platform.openai.com/docs/api-reference/chat). const CALLER_PLAINTEXT = "sk-wre-e2e-caller"; const CALLER_KEY_HASH = createHash("sha256") @@ -58,12 +57,13 @@ describe("weighted routing live-edit: changing weights shifts real traffic (#196 let app: SpawnedApp | undefined; let upstreamA: OpenAiUpstream | undefined; let upstreamB: OpenAiUpstream | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let virtualId = ""; let etcdReachable = false; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; upstreamA = await startOpenAiUpstream({ @@ -74,25 +74,25 @@ describe("weighted routing live-edit: changing weights shifts real traffic (#196 }); app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); + seed = new SeedClient(etcd, app.etcdPrefix); - const pkA = await admin.createProviderKey({ + const pkA = await seed.createProviderKey({ display_name: "wre-a-pk", secret: "sk-mock", api_base: `${upstreamA.baseUrl}/v1`, }); - const pkB = await admin.createProviderKey({ + const pkB = await seed.createProviderKey({ display_name: "wre-b-pk", secret: "sk-mock", api_base: `${upstreamB.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: "wr-edit-a", provider: "openai", model_name: "gpt-4o-mini", provider_key_id: pkA.id, }); - await admin.createModel({ + await seed.createModel({ display_name: "wr-edit-b", provider: "openai", model_name: "gpt-4o-mini", @@ -100,7 +100,7 @@ describe("weighted routing live-edit: changing weights shifts real traffic (#196 }); // Virtual model: weighted, ALL traffic to A initially (B excluded // via weight 0). Capture the generated id so we can PUT it below. - const virtual = await admin.createModel({ + const virtual = await seed.createModel({ display_name: "wr-edit-virtual", routing: { strategy: "weighted", @@ -112,7 +112,7 @@ describe("weighted routing live-edit: changing weights shifts real traffic (#196 }); virtualId = (virtual as { id: string }).id; - await admin.createApiKey({ + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["wr-edit-virtual", "wr-edit-a", "wr-edit-b"], }); @@ -125,7 +125,7 @@ describe("weighted routing live-edit: changing weights shifts real traffic (#196 }); test("editing weights [100,0] → [0,100] flips the served upstream", async (ctx) => { - if (!etcdReachable || !app || !upstreamA || !upstreamB || !admin || !virtualId) { + if (!etcdReachable || !app || !upstreamA || !upstreamB || !seed || !virtualId) { ctx.skip(); return; } @@ -183,8 +183,8 @@ describe("weighted routing live-edit: changing weights shifts real traffic (#196 expect(upstreamA.receivedRequests.length - aBase1).toBe(BATCH); expect(upstreamB.receivedRequests.length - bBase1).toBe(0); - // --- Edit: invert the weights to [0,100] via PUT /admin/v1/models/:id. --- - await admin.json("PUT", `/admin/v1/models/${virtualId}`, { + // --- Edit: invert the weights to [0,100] by rewriting the document. --- + await seed.update("models", virtualId, { display_name: "wr-edit-virtual", routing: { strategy: "weighted", diff --git a/tests/e2e/src/cases/wildcard-routing-e2e.test.ts b/tests/e2e/src/cases/wildcard-routing-e2e.test.ts index fb02d904..7dedcdc9 100644 --- a/tests/e2e/src/cases/wildcard-routing-e2e.test.ts +++ b/tests/e2e/src/cases/wildcard-routing-e2e.test.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import OpenAI from "openai"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { - AdminClient, EtcdClient, + SeedClient, spawnApp, startOpenAiUpstream, waitConfigPropagation, @@ -47,21 +47,22 @@ function sentModel(upstream: OpenAiUpstream): string { describe("wildcard (provider/*) model routing e2e", () => { let app: SpawnedApp | undefined; - let admin: AdminClient | undefined; + let seed: SeedClient | undefined; let etcdReachable = false; const upstreams: OpenAiUpstream[] = []; beforeAll(async () => { - etcdReachable = await new EtcdClient().ping(); + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); if (!etcdReachable) return; app = await spawnApp(); - admin = new AdminClient(app.adminUrl, app.adminKey); - await admin.createApiKey({ + seed = new SeedClient(etcd, app.etcdPrefix); + await seed.createApiKey({ key_hash: CALLER_KEY_HASH, allowed_models: ["*"], }); - await admin.createApiKey({ + await seed.createApiKey({ key_hash: SCOPED_KEY_HASH, allowed_models: ["vendor-a/*"], }); @@ -79,13 +80,13 @@ describe("wildcard (provider/*) model routing e2e", () => { modelName: string, upstream: OpenAiUpstream, ): Promise { - if (!admin) throw new Error("admin client not initialized"); - const providerKey = await admin.createProviderKey({ + if (!seed) throw new Error("seed client not initialized"); + const providerKey = await seed.createProviderKey({ display_name: `${displayName.replace(/[^a-z0-9]/gi, "-")}-pk`, secret: "sk-mock", api_base: `${upstream.baseUrl}/v1`, }); - await admin.createModel({ + await seed.createModel({ display_name: displayName, provider: "openai", model_name: modelName, @@ -101,8 +102,28 @@ describe("wildcard (provider/*) model routing e2e", () => { }); } + + // Seed a throwaway canary key AFTER the resources under test — watch + // events apply in revision order, so once this bearer authenticates + // against /v1/models everything written before it is in the snapshot + // too. Probing the routed models directly would warm cooldowns and + // skew the per-target hit counts the assertions rely on. + async function waitSeedApplied(label: string): Promise { + const canary = `sk-canary-${label}-${Date.now()}`; + await seed!.createApiKey({ + key_hash: createHash("sha256").update(canary).digest("hex"), + allowed_models: ["*"], + }); + await waitConfigPropagation(async () => { + const res = await fetch(`${app!.proxyUrl}/v1/models`, { + headers: { authorization: `Bearer ${canary}` }, + }); + return res.status === 200; + }); + } + test("resolves provider/* and sends the captured model upstream", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -111,10 +132,7 @@ describe("wildcard (provider/*) model routing e2e", () => { upstreams.push(up); await createDirectModel("openai/*", "*", up); - await waitConfigPropagation(async () => { - const models = await admin!.listModels(); - return models.some((m) => m.display_name === "openai/*"); - }); + await waitSeedApplied("wild-openai"); const baseline = up.receivedRequests.length; const completion = await client().chat.completions.create({ @@ -129,7 +147,7 @@ describe("wildcard (provider/*) model routing e2e", () => { }); test("an exact model name wins over a matching wildcard", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -140,13 +158,7 @@ describe("wildcard (provider/*) model routing e2e", () => { await createDirectModel("azure/*", "*", wild); await createDirectModel("azure/gpt-4o", "gpt-4o-2024-08-06", exact); - await waitConfigPropagation(async () => { - const models = await admin!.listModels(); - return ( - models.some((m) => m.display_name === "azure/*") && - models.some((m) => m.display_name === "azure/gpt-4o") - ); - }); + await waitSeedApplied("wild-azure"); // Exact name → the concrete model, not the wildcard. const exactBaseline = exact.receivedRequests.length; @@ -171,7 +183,7 @@ describe("wildcard (provider/*) model routing e2e", () => { }); test("wildcard allowed_models scopes access to matching names", async (ctx) => { - if (!etcdReachable || !app || !admin) { + if (!etcdReachable || !app || !seed) { ctx.skip(); return; } @@ -184,13 +196,7 @@ describe("wildcard (provider/*) model routing e2e", () => { // rejection is authz (403), not not-found (404). await createDirectModel("vendor-b/thing", "thing", denied); - await waitConfigPropagation(async () => { - const models = await admin!.listModels(); - return ( - models.some((m) => m.display_name === "vendor-a/*") && - models.some((m) => m.display_name === "vendor-b/thing") - ); - }); + await waitSeedApplied("wild-scoped"); // In-scope wildcard name → allowed. const deniedBaseline = denied.receivedRequests.length; diff --git a/tests/e2e/src/harness/etcd.ts b/tests/e2e/src/harness/etcd.ts index b2af856c..af1586b1 100644 --- a/tests/e2e/src/harness/etcd.ts +++ b/tests/e2e/src/harness/etcd.ts @@ -83,6 +83,23 @@ export class EtcdClient { return v === undefined ? undefined : Buffer.from(v, "base64").toString("utf8"); } + /** + * Delete exactly one key (etcd v3 `/v3/kv/deleterange`). With + * `range_end` omitted the DeleteRange request applies to only `key` + * — the single-key form of the same RPC `deletePrefix` uses. + */ + async delete(key: string): Promise { + const res = await harnessRequest(`${this.endpoint}/v3/kv/deleterange`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ key: Buffer.from(key, "utf8").toString("base64") }), + }); + if (res.statusCode >= 300) { + const body = await res.body.text(); + throw new Error(`etcd deleterange failed (${res.statusCode}): ${body}`); + } + } + /** Delete every key under `prefix` (range delete in etcd v3 semantics). */ async deletePrefix(prefix: string): Promise { const key = Buffer.from(prefix, "utf8").toString("base64"); diff --git a/tests/e2e/src/harness/seed.ts b/tests/e2e/src/harness/seed.ts index 3983b916..3f50973f 100644 --- a/tests/e2e/src/harness/seed.ts +++ b/tests/e2e/src/harness/seed.ts @@ -54,6 +54,43 @@ export class SeedClient { return this.put("observability_exporters", exporter); } + async createGuardrail( + guardrail: Record, + ): Promise<{ id: string; value: Record }> { + return this.put("guardrails", guardrail); + } + + async createCachePolicy( + policy: Record, + ): Promise<{ id: string; value: Record }> { + return this.put("cache_policies", policy); + } + + async createRateLimitPolicy( + policy: Record, + ): Promise<{ id: string; value: Record }> { + return this.put("rate_limit_policies", policy); + } + + /** + * Overwrite the document at `//` — the seed-side + * equivalent of an Admin API PUT. Propagation is asynchronous; probe + * it with the case's `waitConfigPropagation` condition as with + * creates. + */ + async update( + kind: string, + id: string, + value: Record, + ): Promise { + await this.etcd.put(`${this.prefix}/${kind}/${id}`, JSON.stringify(value)); + } + + /** Remove `//` so the loader drops the resource. */ + async delete(kind: string, id: string): Promise { + await this.etcd.delete(`${this.prefix}/${kind}/${id}`); + } + private async put( kind: string, value: Record,