diff --git a/crates/aisix-gateway/src/bridge.rs b/crates/aisix-gateway/src/bridge.rs index 047cf1a5..8f06eeec 100644 --- a/crates/aisix-gateway/src/bridge.rs +++ b/crates/aisix-gateway/src/bridge.rs @@ -586,6 +586,38 @@ impl BridgeError { } } + /// Whether the request actually left for the upstream before this + /// error was raised. + /// + /// Gates the `aisix_deployment_*` families, which read as **upstream + /// health** for one deployment target. The three config/credential + /// variants are raised while the request is still being assembled — an + /// empty or unusable `api_key`, a missing `model_name`/`api_base`, a + /// body that would not serialize, a `split_system` shape the provider + /// cannot express — so no provider was ever contacted, and counting + /// them against a deployment reports our own misconfiguration as + /// provider degradation. + /// + /// `Timeout` and `Transport` stay `true` on purpose: a connect timeout + /// or a refused connection means we did try to reach the upstream, and + /// "unreachable" is exactly the kind of health this family exists to + /// show. Kept exhaustive (like [`http_status`](Self::http_status) and + /// `routing_error_class`) so a new variant has to declare which side of + /// the network boundary it sits on instead of inheriting a default. + pub fn reached_upstream(&self) -> bool { + match self { + BridgeError::Timeout { .. } + | BridgeError::UpstreamStatus { .. } + | BridgeError::UpstreamDecode(_) + | BridgeError::UpstreamInBand { .. } + | BridgeError::Transport(_) + | BridgeError::StreamAborted => true, + BridgeError::Config(_) + | BridgeError::InvalidUpstreamConfig(_) + | BridgeError::InvalidUpstreamCredentials(_) => false, + } + } + /// Stable error-type token for the error envelope's `type` field. pub fn error_type(&self) -> &'static str { match self { diff --git a/crates/aisix-gateway/src/lib.rs b/crates/aisix-gateway/src/lib.rs index 23a01354..1fc860ca 100644 --- a/crates/aisix-gateway/src/lib.rs +++ b/crates/aisix-gateway/src/lib.rs @@ -51,6 +51,6 @@ pub use upstream_headers::{ RESERVED_UPSTREAM_HEADERS, }; pub use upstream_http::{ - client_builder, error_with_causes, transport_error_message, UpstreamHttpConfig, + client_builder, error_with_causes, send_error, transport_error_message, UpstreamHttpConfig, }; pub use upstream_tls::TlsSettings; diff --git a/crates/aisix-gateway/src/upstream_http.rs b/crates/aisix-gateway/src/upstream_http.rs index 3fcc1286..52e6fd57 100644 --- a/crates/aisix-gateway/src/upstream_http.rs +++ b/crates/aisix-gateway/src/upstream_http.rs @@ -18,6 +18,7 @@ use std::sync::OnceLock; use std::time::Duration; +use crate::bridge::BridgeError; use crate::upstream_tls::TlsSettings; /// Suffixes marking a query parameter whose value is a credential and must @@ -194,6 +195,33 @@ pub fn transport_error_message(err: &reqwest::Error) -> String { msg } +/// Classify a `reqwest` **send** failure into its [`BridgeError`]. +/// +/// reqwest reports a *builder* error when the request could not even be +/// constructed. In practice that is an `api_base` that does not parse as a +/// URL: [`crate::url_cache::EndpointUrl::Unparsed`] deliberately hands the +/// raw string to the request builder so the message stays exactly what it +/// always was, and the parse failure then surfaces here at `send()` with +/// `is_builder()` set and no URL attached. +/// +/// Nothing was sent, so this is customer-fixable upstream config — the same +/// class as a *missing* `api_base`, which already maps to +/// [`BridgeError::InvalidUpstreamConfig`] — rather than a transport failure. +/// Calling it `Transport` would report a 502 for an operator's typo, retry +/// a URL that can never parse, and (via +/// [`BridgeError::reached_upstream`]) count it against the target's +/// `aisix_deployment_*` health even though no provider was contacted. +/// +/// Use at `send()` sites only. A failure reading an already-open response +/// body or stream is never a builder error and stays [`BridgeError::Transport`]. +pub fn send_error(err: reqwest::Error) -> BridgeError { + if err.is_builder() { + BridgeError::InvalidUpstreamConfig(transport_error_message(&err)) + } else { + BridgeError::Transport(transport_error_message(&err)) + } +} + /// Same as [`transport_error_message`] for error types that aren't /// `reqwest::Error` (websocket handshakes, SDK dispatch errors) — no URL /// is available to redact, so only the cause chain is appended. @@ -635,4 +663,35 @@ mod tests { "causes must add information" ); } + + /// The distinction `send_error` exists to make. `EndpointUrl::Unparsed` + /// hands a malformed `api_base` to the request builder verbatim, and + /// reqwest reports the parse failure only here, at `send()`, as a + /// builder error with no URL attached. Classifying it as `Transport` + /// would 502 an operator's typo, retry a URL that can never parse, and + /// count it against the target's upstream health. + #[tokio::test] + async fn builder_errors_are_upstream_config_not_transport() { + let client = reqwest::Client::new(); + let builder_err = crate::url_cache::EndpointUrl::Unparsed("ht tp://not a url".to_string()) + .post_on(&client) + .send() + .await + .expect_err("a malformed api_base cannot produce a response"); + assert!(builder_err.is_builder()); + assert!(matches!( + send_error(builder_err), + BridgeError::InvalidUpstreamConfig(_) + )); + + // A real connection attempt to a closed port stays transport: we + // did try to reach the provider, and that is upstream health. + let io_err = client + .post("http://127.0.0.1:1/v1/chat/completions") + .send() + .await + .expect_err("nothing listens on port 1"); + assert!(!io_err.is_builder()); + assert!(matches!(send_error(io_err), BridgeError::Transport(_))); + } } diff --git a/crates/aisix-provider-anthropic/src/bridge.rs b/crates/aisix-provider-anthropic/src/bridge.rs index b8d71762..dfe5a7e6 100644 --- a/crates/aisix-provider-anthropic/src/bridge.rs +++ b/crates/aisix-provider-anthropic/src/bridge.rs @@ -303,7 +303,7 @@ impl Bridge for AnthropicBridge { .json(&body) .send() .await - .map_err(|e| BridgeError::Transport(aisix_gateway::transport_error_message(&e)))?; + .map_err(aisix_gateway::send_error)?; let status = resp.status(); if !status.is_success() { @@ -355,7 +355,7 @@ impl Bridge for AnthropicBridge { .json(&body) .send() .await - .map_err(|e| BridgeError::Transport(aisix_gateway::transport_error_message(&e))) + .map_err(aisix_gateway::send_error) }) .await?; diff --git a/crates/aisix-provider-azure-openai/src/bridge.rs b/crates/aisix-provider-azure-openai/src/bridge.rs index e1cece7f..744303c4 100644 --- a/crates/aisix-provider-azure-openai/src/bridge.rs +++ b/crates/aisix-provider-azure-openai/src/bridge.rs @@ -729,7 +729,7 @@ impl Bridge for AzureOpenAiBridge { .json(&body) .send() .await - .map_err(|e| BridgeError::Transport(aisix_gateway::transport_error_message(&e)))?; + .map_err(aisix_gateway::send_error)?; let status = resp.status(); if !status.is_success() { @@ -780,7 +780,7 @@ impl Bridge for AzureOpenAiBridge { .json(&body) .send() .await - .map_err(|e| BridgeError::Transport(aisix_gateway::transport_error_message(&e))) + .map_err(aisix_gateway::send_error) }) .await?; diff --git a/crates/aisix-provider-openai/src/bridge.rs b/crates/aisix-provider-openai/src/bridge.rs index cc613c82..46770750 100644 --- a/crates/aisix-provider-openai/src/bridge.rs +++ b/crates/aisix-provider-openai/src/bridge.rs @@ -418,7 +418,7 @@ impl Bridge for OpenAiBridge { .json(&body) .send() .await - .map_err(|e| BridgeError::Transport(aisix_gateway::transport_error_message(&e)))?; + .map_err(aisix_gateway::send_error)?; let status = resp.status(); if !status.is_success() { @@ -470,7 +470,7 @@ impl Bridge for OpenAiBridge { .json(&body) .send() .await - .map_err(|e| BridgeError::Transport(aisix_gateway::transport_error_message(&e)))?; + .map_err(aisix_gateway::send_error)?; let status = resp.status(); if !status.is_success() { @@ -529,7 +529,7 @@ impl Bridge for OpenAiBridge { .json(&outbound) .send() .await - .map_err(|e| BridgeError::Transport(aisix_gateway::transport_error_message(&e)))?; + .map_err(aisix_gateway::send_error)?; let status = resp.status(); if !status.is_success() { @@ -586,7 +586,7 @@ impl Bridge for OpenAiBridge { .json(&outbound) .send() .await - .map_err(|e| BridgeError::Transport(aisix_gateway::transport_error_message(&e)))?; + .map_err(aisix_gateway::send_error)?; let status = resp.status(); if !status.is_success() { @@ -634,7 +634,7 @@ impl Bridge for OpenAiBridge { .json(&body) .send() .await - .map_err(|e| BridgeError::Transport(aisix_gateway::transport_error_message(&e))) + .map_err(aisix_gateway::send_error) }) .await?; diff --git a/crates/aisix-provider-vertex/src/bridge.rs b/crates/aisix-provider-vertex/src/bridge.rs index afe7d2f4..38b9e954 100644 --- a/crates/aisix-provider-vertex/src/bridge.rs +++ b/crates/aisix-provider-vertex/src/bridge.rs @@ -775,7 +775,7 @@ impl Bridge for VertexBridge { .json(&body) .send() .await - .map_err(|e| BridgeError::Transport(aisix_gateway::transport_error_message(&e)))?; + .map_err(aisix_gateway::send_error)?; let status = resp.status(); if !status.is_success() { return Err(map_http_error(status, resp).await); @@ -912,7 +912,7 @@ impl VertexBridge { .json(&body) .send() .await - .map_err(|e| BridgeError::Transport(aisix_gateway::transport_error_message(&e)))?; + .map_err(aisix_gateway::send_error)?; let status = resp.status(); if !status.is_success() { @@ -1017,7 +1017,7 @@ impl VertexBridge { .json(&body_value) .send() .await - .map_err(|e| BridgeError::Transport(aisix_gateway::transport_error_message(&e)))?; + .map_err(aisix_gateway::send_error)?; let status = resp.status(); if !status.is_success() { @@ -1113,7 +1113,7 @@ impl VertexBridge { .json(&body_value) .send() .await - .map_err(|e| BridgeError::Transport(aisix_gateway::transport_error_message(&e))) + .map_err(aisix_gateway::send_error) }) .await?; @@ -1233,7 +1233,7 @@ impl VertexBridge { .json(&body) .send() .await - .map_err(|e| BridgeError::Transport(aisix_gateway::transport_error_message(&e)))?; + .map_err(aisix_gateway::send_error)?; let status = resp.status(); if !status.is_success() { return Err(map_http_error(status, resp).await); @@ -1294,7 +1294,7 @@ impl VertexBridge { .json(&body) .send() .await - .map_err(|e| BridgeError::Transport(aisix_gateway::transport_error_message(&e))) + .map_err(aisix_gateway::send_error) }) .await?; @@ -1437,7 +1437,7 @@ impl VertexBridge { .json(&body) .send() .await - .map_err(|e| BridgeError::Transport(aisix_gateway::transport_error_message(&e)))?; + .map_err(aisix_gateway::send_error)?; let status = resp.status(); if !status.is_success() { return Err(map_http_error(status, resp).await); @@ -1515,7 +1515,7 @@ impl VertexBridge { .json(&body) .send() .await - .map_err(|e| BridgeError::Transport(aisix_gateway::transport_error_message(&e))) + .map_err(aisix_gateway::send_error) }) .await?; @@ -1635,7 +1635,7 @@ impl VertexBridge { .json(&body) .send() .await - .map_err(|e| BridgeError::Transport(aisix_gateway::transport_error_message(&e))) + .map_err(aisix_gateway::send_error) }) .await?; diff --git a/crates/aisix-proxy/src/attempt.rs b/crates/aisix-proxy/src/attempt.rs index 276b0266..beb5bb45 100644 --- a/crates/aisix-proxy/src/attempt.rs +++ b/crates/aisix-proxy/src/attempt.rs @@ -62,11 +62,17 @@ pub(crate) struct AttemptRecord { /// Whether this attempt actually reached the upstream. /// /// False for an attempt the target's own rate-limit layers refused - /// before dispatch: it produced no upstream response, so it stays out - /// of the `aisix_deployment_*_responses_total` families an operator - /// reads as upstream health. It is still a real attempt everywhere - /// else — the per-attempt usage event, and the initial/retry/fallback - /// classification the next attempt is measured against. + /// before dispatch, and equally for one the bridge rejected while still + /// assembling the request — an unusable `api_key`, a missing + /// `model_name`/`api_base`, a body that would not serialize (see + /// [`BridgeError::reached_upstream`] and `attempt_reached_upstream`, + /// which decide this per error variant). Neither produced an upstream + /// response, so both stay out of the `aisix_deployment_*_responses_total` + /// families an operator reads as upstream health; counting them there + /// reports our own misconfiguration as provider degradation. Such an + /// attempt is still real everywhere else — the per-attempt usage event, + /// and the initial/retry/fallback classification the next attempt is + /// measured against. pub dispatched: bool, } @@ -265,6 +271,49 @@ pub(crate) fn attempt_error_from_proxy(err: &ProxyError) -> (String, String) { } } +/// Whether a failed attempt actually reached its upstream — the +/// `AttemptRecord::dispatched` value for the `ProxyError`-typed dispatch +/// loops (`/v1/messages`, `/v1/responses`), mirroring +/// [`BridgeError::reached_upstream`] for the `BridgeError`-typed one. +/// +/// `ContentFiltered` is `true` because only the **output** hook can fire +/// inside a dispatch call — the input hook runs once, before the loop — so +/// the provider had already answered when we blocked it. (That attempt is +/// therefore counted as a deployment *failure* by +/// `RequestOutcome::from_status`, even though the upstream was healthy; +/// that is a defect in the outcome mapping, not in this predicate, and +/// fixing it needs the error to carry which hook fired.) Every remaining +/// variant is a gateway-side decision — auth, ACL, budget, rate limit, +/// unknown model — taken without contacting any provider. Exhaustive so a +/// new variant has to declare its side of the network boundary. +pub(crate) fn attempt_reached_upstream(err: &ProxyError) -> bool { + match err { + ProxyError::Bridge(be) => be.reached_upstream(), + ProxyError::ContentFiltered(_) => true, + ProxyError::MissingAuth + | ProxyError::InvalidApiKey + | ProxyError::ApiKeyExpired + | ProxyError::ApiKeyDisabled + | ProxyError::JwtInvalid + | ProxyError::JwtExpired + | ProxyError::JwtClaimsRejected + | ProxyError::JwtIdentityUnmapped + | ProxyError::JwksUnavailable + | ProxyError::ModelNotFound(_) + | ProxyError::VideoNotFound(_) + | ProxyError::ModelForbidden(_) + | ProxyError::ModelIpRestricted(_) + | ProxyError::InvalidRequest(_) + | ProxyError::WebSocketUpgradeRequired { .. } + | ProxyError::ProviderUnavailable + | ProxyError::AllCandidatesUnavailable { .. } + | ProxyError::BudgetExceeded(_) + | ProxyError::RequestTooLarge { .. } + | ProxyError::RateLimit(_) + | ProxyError::PolicyRateLimit { .. } => false, + } +} + /// Milliseconds elapsed since `started`, saturating at `u32::MAX`. pub(crate) fn ms_since(started: Instant) -> u32 { started.elapsed().as_millis().min(u32::MAX as u128) as u32 @@ -275,6 +324,62 @@ mod tests { use super::*; use aisix_gateway::{UpstreamWire, MAX_UPSTREAM_ERROR_MESSAGE_BYTES}; + /// The `aisix_deployment_*` families read as upstream health, so an + /// error raised while the request was still being assembled has to stay + /// out of them. Both matches are exhaustive, so a NEW variant is already + /// a compile error; this pins the classification of the existing ones, + /// which is what a well-meaning refactor would silently flip. + #[test] + fn only_errors_that_reached_the_provider_count_as_upstream_attempts() { + for err in [ + BridgeError::Config("serialize request body: eof".into()), + BridgeError::InvalidUpstreamConfig("model.model_name missing".into()), + BridgeError::InvalidUpstreamCredentials("provider_key.api_key is empty".into()), + ] { + assert!( + !err.reached_upstream(), + "{err} is raised before the request is sent" + ); + assert!(!attempt_reached_upstream(&ProxyError::Bridge(err))); + } + + // A timeout or a refused connection IS upstream health: we tried to + // reach the provider and could not. Excluding these would hide the + // outage the family exists to show. + for err in [ + BridgeError::Timeout { + elapsed_ms: 7167, + cause: String::new(), + }, + BridgeError::Transport("connection refused".into()), + BridgeError::upstream_status(502, "bad gateway"), + BridgeError::UpstreamDecode("unparseable body".into()), + BridgeError::UpstreamInBand { + status: Some(500), + message: "overloaded".into(), + parsed: None, + wire: UpstreamWire::Unknown, + }, + BridgeError::StreamAborted, + ] { + assert!( + err.reached_upstream(), + "{err} means the provider was contacted" + ); + assert!(attempt_reached_upstream(&ProxyError::Bridge(err))); + } + + // Only the output hook can fire inside a dispatch call, so the + // provider had already answered — the attempt did reach it. + assert!(attempt_reached_upstream(&ProxyError::ContentFiltered( + "blocked by response guardrail".into() + ))); + // Gateway-side refusals never contacted anyone. + assert!(!attempt_reached_upstream(&ProxyError::ModelNotFound( + "nope".into() + ))); + } + /// AISIX-Cloud#1093: the access log is the one line an operator gets /// per request, so EVERY failure has to name itself there — including /// the variants `attempt_error_from_proxy` deliberately leaves diff --git a/crates/aisix-proxy/src/chat.rs b/crates/aisix-proxy/src/chat.rs index 11fe4f36..596c0da5 100644 --- a/crates/aisix-proxy/src/chat.rs +++ b/crates/aisix-proxy/src/chat.rs @@ -1717,7 +1717,7 @@ async fn dispatch( error_class: routing_error_class(&err).to_string(), error_message: attempt_error_message(&err), latency_ms, - dispatched: true, + dispatched: err.reached_upstream(), }, ); let retryable = is_retryable(&err, retry_on_429, fallback_statuses); @@ -2788,7 +2788,7 @@ async fn dispatch( error_class: routing_error_class(&err).to_string(), error_message: attempt_error_message(&err), latency_ms: attempt_latency_ms, - dispatched: true, + dispatched: err.reached_upstream(), }, ); let retryable = is_retryable(&err, retry_on_429, fallback_statuses); diff --git a/crates/aisix-proxy/src/messages.rs b/crates/aisix-proxy/src/messages.rs index eaef1def..9565fdd8 100644 --- a/crates/aisix-proxy/src/messages.rs +++ b/crates/aisix-proxy/src/messages.rs @@ -54,7 +54,8 @@ use std::time::{Duration, Instant}; use uuid::Uuid; use crate::attempt::{ - attempt_error_from_proxy, ms_since, AttemptInfo, AttemptRecord, RoutingTelemetry, + attempt_error_from_proxy, attempt_reached_upstream, ms_since, AttemptInfo, AttemptRecord, + RoutingTelemetry, }; use crate::auth::AuthenticatedKey; use crate::chat::sanitize_tag; @@ -866,7 +867,7 @@ async fn dispatch( error_class, error_message, latency_ms: ms_since(attempt_started), - dispatched: true, + dispatched: attempt_reached_upstream(&e), }, ); // See `RetryBudget::covers`: a default budget skips diff --git a/crates/aisix-proxy/src/responses.rs b/crates/aisix-proxy/src/responses.rs index 8e5e118b..1c6341cb 100644 --- a/crates/aisix-proxy/src/responses.rs +++ b/crates/aisix-proxy/src/responses.rs @@ -25,7 +25,8 @@ use std::time::{Duration, Instant}; use uuid::Uuid; use crate::attempt::{ - attempt_error_from_proxy, ms_since, AttemptInfo, AttemptRecord, RoutingTelemetry, + attempt_error_from_proxy, attempt_reached_upstream, ms_since, AttemptInfo, AttemptRecord, + RoutingTelemetry, }; use crate::auth::AuthenticatedKey; use crate::chat::sanitize_tag; @@ -857,7 +858,7 @@ async fn dispatch( error_class, error_message, latency_ms: ms_since(attempt_started), - dispatched: true, + dispatched: attempt_reached_upstream(&e), }, ); // See `RetryBudget::covers`: a default budget skips diff --git a/tests/e2e/src/cases/dispatched-provenance-e2e.test.ts b/tests/e2e/src/cases/dispatched-provenance-e2e.test.ts new file mode 100644 index 00000000..1ae7039d --- /dev/null +++ b/tests/e2e/src/cases/dispatched-provenance-e2e.test.ts @@ -0,0 +1,285 @@ +import { createHash } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import { + EtcdClient, + ProxyClient, + SeedClient, + metricDelta, + scrapeMetrics, + spawnApp, + startOpenAiUpstream, + waitConfigPropagation, + type OpenAiUpstream, + type SpawnedApp, +} from "../harness/index.js"; +import { harnessRequest } from "../harness/http.js"; + +// E2E: the `aisix_deployment_*` families read as UPSTREAM health for one +// deployment target, so an attempt that never left the gateway must not +// appear in them. Two kinds of attempt never leave: one the target's own +// rate-limit layers refuse, and — the kind this file pins — one the bridge +// rejects while still assembling the request. +// +// Counting those as `aisix_deployment_failure_responses_total` reports our +// own misconfiguration as provider degradation: an operator watching a +// target's failure rate sees the provider "failing" while the provider was +// never asked anything. The upstream request count is the ground truth +// here — it stays at zero across every measured call. +// +// Two rejection points, because they are raised in different places and a +// fix for one does not imply the other: +// +// - an unusable secret, caught by the bridge's own `api_key()` guard +// before any request is built; +// - an `api_base` that does not parse, which reaches reqwest as a raw +// string and fails at `send()` as a *builder* error, with no socket +// ever opened. +// +// Every changed dispatch branch is exercised, not just one: the +// classification is applied at four separate call sites (streaming and +// non-streaming chat, `/v1/messages`, `/v1/responses`) and any of them can +// regress to `dispatched: true` on its own while the others stay correct. +// +// The mirror assertion matters just as much: the attempt is NOT dropped. +// It stays a real attempt in the per-attempt usage events (the rows the +// dashboard log counts), exactly as a rate-limit-refused attempt does. The +// deployment families are the only place it is excluded from. + +const CALLER_PLAINTEXT = "sk-dispatch-provenance"; +const CALLER_KEY_HASH = createHash("sha256") + .update(CALLER_PLAINTEXT) + .digest("hex"); + +describe("dispatched provenance e2e: a pre-dispatch failure is not upstream health", () => { + let app: SpawnedApp | undefined; + let upstream: OpenAiUpstream | undefined; + let seed: SeedClient | undefined; + let etcdReachable = false; + + beforeAll(async () => { + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); + if (!etcdReachable) return; + + // A real, healthy upstream. It exists so "the upstream received + // nothing" is a measurement rather than an artifact of there being + // nowhere to send to. + upstream = await startOpenAiUpstream(); + app = await spawnApp(); + seed = new SeedClient(etcd, app.etcdPrefix); + + // Valid api_base — routing shape is fine — but a secret carrying an + // embedded newline, which cannot be an Authorization header value. + // The openai bridge's api_key() guard rejects it before dispatch. + // (An empty secret is the same error class but never gets this far: + // the admin schema's min-length check rejects it at admission.) + const badCredPk = await seed.createProviderKey({ + display_name: "dp-badcred-pk", + secret: "sk-live\n-injected", + api_base: `${upstream.baseUrl}/v1`, + }); + // Usable secret, but an api_base that is not a URL. The schema types + // api_base as a plain string, so this is admitted; the parse failure + // surfaces only when reqwest builds the request. + const badUrlPk = await seed.createProviderKey({ + display_name: "dp-badurl-pk", + secret: "sk-mock", + api_base: "ht tp://not a url/v1", + }); + + for (const [name, pkId] of [ + ["dp-badcred", badCredPk.id], + ["dp-badurl", badUrlPk.id], + ] as const) { + await seed.createModel({ + display_name: name, + provider: "openai", + model_name: "gpt-4o-mini", + provider_key_id: pkId, + // Every call in this file fails this target by design. With + // cooldown on, the target would be marked after the first one and + // later calls would be refused before an attempt is ever built — + // removing the very attempt these tests are about. + cooldown: { enabled: false }, + }); + } + + // Routing models, because the `aisix_deployment_*` families are emitted + // from the Model-Group dispatch loops: a direct model builds no + // AttemptRecord at all and would not exercise the code under test. + // One target each keeps the assertions about this attempt rather than + // about failover ordering. + await seed.createModel({ + display_name: "dp-cred-group", + routing: { strategy: "failover", targets: [{ model: "dp-badcred" }] }, + }); + await seed.createModel({ + display_name: "dp-url-group", + routing: { strategy: "failover", targets: [{ model: "dp-badurl" }] }, + }); + // Seeded last: once this key authenticates, the whole seed set is in + // the snapshot. + await seed.createApiKey({ + key_hash: CALLER_KEY_HASH, + allowed_models: ["dp-cred-group", "dp-url-group"], + }); + }); + + afterAll(async () => { + await app?.exit(); + await upstream?.close(); + }); + + /** POST `path` as the seeded caller and return the raw status + body. */ + const post = async ( + path: string, + body: unknown, + ): Promise<{ status: number; text: string }> => { + const res = await harnessRequest(`${app!.proxyUrl}${path}`, { + method: "POST", + headers: { + authorization: `Bearer ${CALLER_PLAINTEXT}`, + "content-type": "application/json", + }, + body: JSON.stringify(body), + }); + return { status: res.statusCode, text: await res.body.text() }; + }; + + /** + * Run `call` and assert it was refused without any upstream request and + * without moving `model`'s deployment counters — while still landing in + * the per-attempt usage events. + */ + const expectNotAttributedToUpstream = async ( + model: string, + wantStatus: number, + call: () => Promise<{ status: number; text: string }>, + ) => { + const upstreamBaseline = upstream!.receivedRequests.length; + const before = await scrapeMetrics(app!.metricsUrl); + + const res = await call(); + expect(res.status, `body: ${res.text}`).toBe(wantStatus); + + const after = await scrapeMetrics(app!.metricsUrl); + const delta = (name: string, want?: Record) => + metricDelta(before, after, name, want); + + // Ground truth: the provider was never asked anything. + expect(upstream!.receivedRequests.length - upstreamBaseline).toBe(0); + + // Therefore it owes the deployment families nothing. Before this + // change, `requests_total` and `failure_responses_total` each moved by + // 1 — the failure counter is what made a healthy provider look like it + // was failing. + for (const family of [ + "aisix_deployment_requests_total", + "aisix_deployment_failure_responses_total", + "aisix_deployment_success_responses_total", + ]) { + expect(delta(family, { model })).toBe(0); + } + + // …but the attempt itself is not lost. It is still a per-attempt usage + // event, the same way a rate-limit-refused attempt is. A regression + // that "fixes" the counters by dropping the attempt fails here. The + // count is a lower bound rather than exactly 1 because a retryable + // classification (see `/v1/responses` below) legitimately produces + // several attempts for one call. + expect(delta("aisix_usage_events_emitted_total")).toBeGreaterThanOrEqual(1); + }; + + const ready = async () => { + const probe = new ProxyClient(app!.proxyUrl, CALLER_PLAINTEXT); + await waitConfigPropagation(async () => { + const res = await probe.listModels(); + if (res.status !== 200) return false; + const data = (res.body as { data?: Array<{ id?: string }> }).data ?? []; + return ( + data.some((m) => m.id === "dp-cred-group") && + data.some((m) => m.id === "dp-url-group") + ); + }); + }; + + // One case per changed call site. `dispatched` is decided independently + // in each dispatch loop, so covering only one branch would let the other + // three regress silently. + const branches: Array< + [string, number, () => Promise<{ status: number; text: string }>] + > = [ + [ + "/v1/chat/completions", + 401, + () => + post("/v1/chat/completions", { + model: "dp-cred-group", + messages: [{ role: "user", content: "hi" }], + }), + ], + [ + "/v1/chat/completions (streaming)", + 401, + () => + post("/v1/chat/completions", { + model: "dp-cred-group", + messages: [{ role: "user", content: "hi" }], + stream: true, + }), + ], + [ + "/v1/messages", + 401, + () => + post("/v1/messages", { + model: "dp-cred-group", + max_tokens: 16, + messages: [{ role: "user", content: "hi" }], + }), + ], + // `/v1/responses` builds the auth header on its own path and reports + // the same unusable secret as `BridgeError::Config` (500 config_error) + // rather than the 401 authentication_error the other three give. That + // divergence is pinned here as current behavior, not endorsed — it is a + // status-taxonomy question of its own. It does not affect what this + // file is about: `Config` is a pre-dispatch variant too, so the + // deployment counters stay untouched either way. Being retryable, it + // also produces more than one attempt per call. + [ + "/v1/responses", + 500, + () => post("/v1/responses", { model: "dp-cred-group", input: "hi" }), + ], + ]; + + for (const [label, wantStatus, call] of branches) { + test(`${label}: an unusable credential counts as no upstream request`, async (ctx) => { + if (!etcdReachable || !app || !upstream) { + ctx.skip(); + return; + } + await ready(); + await expectNotAttributedToUpstream("dp-badcred", wantStatus, call); + }); + } + + // The second rejection point. An api_base that does not parse reaches + // reqwest as a raw string and fails at send() as a builder error, with no + // socket opened — so it is upstream *config* (400, like a missing + // api_base), not a transport failure, and owes the deployment families + // nothing either. + test("a malformed api_base is upstream config, not a transport failure", async (ctx) => { + if (!etcdReachable || !app || !upstream) { + ctx.skip(); + return; + } + await ready(); + await expectNotAttributedToUpstream("dp-badurl", 400, () => + post("/v1/chat/completions", { + model: "dp-url-group", + messages: [{ role: "user", content: "hi" }], + }), + ); + }); +});