From 7889c1f3ffea7eeca15e1d9e49cdf930e76b5282 Mon Sep 17 00:00:00 2001 From: Jarvis Date: Fri, 28 Aug 2026 09:56:41 +0000 Subject: [PATCH 1/3] feat(telemetry): attribute usage events and their counters to the org member MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Logs question "show me Alice's 429s in the last 24h" had no answer. A usage event named the credential it arrived on (`api_key_id`, plus a JWT subject) but never the person behind it, so answering meant enumerating every key a member owns, querying each, and merging by hand — and a member who calls through both an API key and OIDC was still split across two identities that no single filter could join. Add `user_id` to the UsageEvent wire contract, stamped from the resolved `ApiKey.user_id` at request time. A snapshot rather than something cp-api resolves from `api_key_id` at query time: rebinding a key would otherwise re-attribute its whole history to the new owner, and deleting one would erase the attribution of everything it ever sent. The JWT path needs no separate handling — a token runs AS the key it resolves to, so one field covers both halves of the auth surface. `apply_jwt_identity` becomes `apply_caller_identity` and takes the member alongside the JWT identity. The rename is the point: this family is one call per handler and a new one is easy to forget, so the compiler now refuses any site that has not been considered. On the metrics side, `aisix_usage_events_emitted_total` gains: - `user_id`, via `UsageEventLabels` — attribution, so it lands on `aisix_usage_event_drops_total` too and "whose usage records were lost" stays answerable. `UsageSink::try_emit` fills it from the event's own field, so the counter and the row cp-api persists cannot name different members. The readable `user_name` is deliberately not duplicated here: it is 1:1 with `user_id` on the request family and joinable from there, and reaching it would mean widening `CallerIdentity` across the gateway crate for a display convenience. - `http_status_code`, the raw code. `status_code` keeps its `2xx` / `4xx` / `5xx` bucketing, so dashboards and alerts written against it keep working; the raw code determines the family, so carrying both adds no series over carrying the raw code alone. The drop counter deliberately gains no status dimension. `handler` / `status_code` / `http_status_code` / `inbound_protocol` are the emit counter's own arguments rather than attribution, and `emitted == delivered + dropped` is an invariant over the attribution dimensions — which is exactly why `user_id` does appear on both. OTLP spans carry the member as `aisix.user_id`, beside `aisix.api_key_id`. E2E over a real binary and a real SLS export: an owned key stamps its member; a JWT resolving to that member's *other* key stamps the same member (the case a query-time join on `api_key_id` gets wrong); an unowned key stamps nobody, so a member filter cannot sweep up traffic that was never theirs; and an upstream 429 is addressable as `{user_id, http_status_code="429"}` with the `4xx` family still present. Ref api7/AISIX-Cloud#1389 --- crates/aisix-obs/src/metrics.rs | 60 +++- crates/aisix-obs/src/otlp_http_sink.rs | 6 + crates/aisix-obs/src/usage.rs | 40 ++- crates/aisix-proxy/src/a2a.rs | 6 +- crates/aisix-proxy/src/audio.rs | 6 +- crates/aisix-proxy/src/chat.rs | 6 +- crates/aisix-proxy/src/completions.rs | 6 +- crates/aisix-proxy/src/embeddings.rs | 6 +- crates/aisix-proxy/src/images.rs | 6 +- crates/aisix-proxy/src/jobs.rs | 11 +- crates/aisix-proxy/src/mcp.rs | 6 +- crates/aisix-proxy/src/messages.rs | 6 +- crates/aisix-proxy/src/passthrough_route.rs | 11 +- crates/aisix-proxy/src/realtime.rs | 6 +- crates/aisix-proxy/src/rerank.rs | 6 +- crates/aisix-proxy/src/responses.rs | 12 +- crates/aisix-proxy/src/usage_attr.rs | 31 +- crates/aisix-proxy/src/videos.rs | 6 +- .../usage-member-attribution-e2e.test.ts | 292 ++++++++++++++++++ 19 files changed, 495 insertions(+), 34 deletions(-) create mode 100644 tests/e2e/src/cases/usage-member-attribution-e2e.test.ts diff --git a/crates/aisix-obs/src/metrics.rs b/crates/aisix-obs/src/metrics.rs index 47a49efec..c9edfeec9 100644 --- a/crates/aisix-obs/src/metrics.rs +++ b/crates/aisix-obs/src/metrics.rs @@ -259,8 +259,14 @@ pub const M_GUARDRAIL_LATENCY_SECONDS: &str = "aisix_guardrail_latency_seconds"; /// - `handler`: which OpenAI-shape handler emitted (chat / /// embeddings / responses / completions / rerank / audio / /// images / messages). Fixed enumeration, low cardinality. -/// - `status_code`: bucketed as `2xx` / `4xx` / `5xx` (avoid the -/// 1000-value cardinality blowup of raw u16 codes). +/// - `status_code`: bucketed as `2xx` / `4xx` / `5xx`. +/// - `http_status_code`: the raw code (`429`, `502`, ...), so a query can +/// name one failure mode instead of a whole family (AISIX-Cloud#1389). +/// It adds no series over `status_code` alone — the raw code determines +/// the family — and `status_code` is kept so existing dashboards and +/// alerts written against `status_code="4xx"` keep working unchanged. +/// - `user_id`: the org member behind the request, from +/// [`UsageEventLabels`]. /// - `inbound_protocol`: `openai` / `anthropic`. Matches the /// wire-level field on UsageEvent. /// - `upstream_protocol`: the wire protocol of the ProviderKey the event @@ -2013,8 +2019,11 @@ impl Metrics { /// that invariant exists only on dimensions BOTH sides carry. The /// protocol is a function of the ProviderKey row `provider_key_id` /// already names, so like `provider_key_name` it adds no series here. - /// (`handler` / `status_code` / `inbound_protocol` are the emit - /// counter's own arguments, not attribution, and stay one-sided.) + /// (`handler` / `status_code` / `http_status_code` / `inbound_protocol` + /// are the emit counter's own arguments, not attribution, and stay + /// one-sided — `emitted == delivered + dropped` is an invariant over + /// the attribution dimensions, which is why `user_id` DOES appear on + /// both.) pub fn record_usage_event_drop(&self, reason: &str, labels: UsageEventLabels<'_>) { self.cached_counter( M_USAGE_EVENT_DROPS_TOTAL, @@ -2024,6 +2033,7 @@ impl Metrics { k.label(labels.model); k.label(labels.provider_key_id); k.label(labels.provider_key_name); + k.label(labels.user_id); k.label(labels.upstream_protocol); }, || { @@ -2033,6 +2043,7 @@ impl Metrics { "model" => labels.model.to_string(), "provider_key_id" => labels.provider_key_id.to_string(), "provider_key_name" => labels.provider_key_name.to_string(), + "user_id" => labels.user_id.to_string(), "upstream_protocol" => labels.upstream_protocol.to_string(), ) }, @@ -2068,16 +2079,19 @@ impl Metrics { labels: UsageEventLabels<'_>, ) { let status_class = status_bucket(status_code); + let http_status_code = status_code.to_string(); self.cached_counter( M_USAGE_EVENT_EMITS_TOTAL, 1, |k| { k.label(handler); k.label(status_class); + k.label(&http_status_code); k.label(inbound_protocol); k.label(labels.model); k.label(labels.provider_key_id); k.label(labels.provider_key_name); + k.label(labels.user_id); k.label(labels.upstream_protocol); }, || { @@ -2085,11 +2099,13 @@ impl Metrics { M_USAGE_EVENT_EMITS_TOTAL, "handler" => handler, "status_code" => status_class, + "http_status_code" => http_status_code.clone(), "inbound_protocol" => inbound_protocol, "upstream_protocol" => labels.upstream_protocol.to_string(), "model" => labels.model.to_string(), "provider_key_id" => labels.provider_key_id.to_string(), "provider_key_name" => labels.provider_key_name.to_string(), + "user_id" => labels.user_id.to_string(), ) }, ); @@ -2683,6 +2699,19 @@ pub struct UsageEventLabels<'a> { pub model: &'a str, pub provider_key_id: &'a str, pub provider_key_name: &'a str, + /// Org member the authenticating key belongs to (AISIX-Cloud#1389) — + /// the same `user_id` `aisix_proxy_requests_total` carries, so a + /// member's request rate and their usage-event rate slice alike, and + /// the readable `user_name` is joinable from that family rather than + /// duplicated here. `unknown` when the key is bound to no member, or + /// when auth never resolved a key. + /// + /// This is attribution, so it sits on BOTH counters: "which member's + /// usage records were lost" is exactly the question the shared label + /// set exists to keep answerable. `UsageSink::try_emit` fills it from + /// the event's own `user_id`, so the counter and the row cp-api + /// persists can never name different people. + pub user_id: &'a str, /// The upstream wire protocol of the key named by `provider_key_id` /// (AISIX-Cloud#1403) — the same value, resolved the same way, that /// the request and usage families carry, so an operator can align @@ -2703,6 +2732,7 @@ impl Default for UsageEventLabels<'_> { model: "unknown", provider_key_id: "unknown", provider_key_name: "unknown", + user_id: "unknown", upstream_protocol: "unknown", } } @@ -4281,6 +4311,7 @@ mod tests { model: "gpt-4o", provider_key_id: "pk-1", provider_key_name: "openai-prod", + user_id: "member-1", upstream_protocol: "openai", }; @@ -4349,14 +4380,14 @@ mod tests { assert!( rendered.contains( - "handler=\"messages\",status_code=\"2xx\",\ + "handler=\"messages\",status_code=\"2xx\",http_status_code=\"200\",\ inbound_protocol=\"anthropic\",upstream_protocol=\"openai\"" ), "cross-protocol sample must report the upstream's protocol:\n{rendered}" ); assert!( rendered.contains( - "handler=\"chat\",status_code=\"4xx\",\ + "handler=\"chat\",status_code=\"4xx\",http_status_code=\"401\",\ inbound_protocol=\"openai\",upstream_protocol=\"unknown\"" ), "an unresolved upstream must read `unknown`, never borrow the \ @@ -4414,16 +4445,23 @@ mod tests { }; // The emit counter's own arguments — a surface of the emitting // handler, not of the event's attribution. - let emit_only: BTreeSet = ["handler", "status_code", "inbound_protocol"] - .into_iter() - .map(str::to_string) - .collect(); + let emit_only: BTreeSet = [ + "handler", + "status_code", + "http_status_code", + "inbound_protocol", + ] + .into_iter() + .map(str::to_string) + .collect(); let attribution: BTreeSet = labels_of(M_USAGE_EVENT_EMITS_TOTAL) .difference(&emit_only) .cloned() .collect(); assert!( - attribution.contains("model") && attribution.contains("provider_key_id"), + attribution.contains("model") + && attribution.contains("provider_key_id") + && attribution.contains("user_id"), "the attribution set looks wrong: {attribution:?}" ); diff --git a/crates/aisix-obs/src/otlp_http_sink.rs b/crates/aisix-obs/src/otlp_http_sink.rs index ea7f87b8e..3148e9b6d 100644 --- a/crates/aisix-obs/src/otlp_http_sink.rs +++ b/crates/aisix-obs/src/otlp_http_sink.rs @@ -822,6 +822,12 @@ fn event_attributes(record: &SinkRecord, exporter_name: &str) -> Vec { // spans back to the AISIX api_key dashboard. attributes.push(attr_string("aisix.api_key_id", &event.api_key_id)); } + // The org member behind the credential (AISIX-Cloud#1389). Exported + // beside the key rather than derived from it downstream: a member can + // hold several keys, and a key can be rebound to someone else. + if !event.user_id.is_empty() { + attributes.push(attr_string("aisix.user_id", &event.user_id)); + } if !event.model_id.is_empty() { attributes.push(attr_string("aisix.model_id", &event.model_id)); } diff --git a/crates/aisix-obs/src/usage.rs b/crates/aisix-obs/src/usage.rs index 4e2ec60bb..683d541bc 100644 --- a/crates/aisix-obs/src/usage.rs +++ b/crates/aisix-obs/src/usage.rs @@ -76,6 +76,17 @@ pub struct UsageEvent { #[serde(default)] pub api_key_id: String, + /// UUID of the org member the authenticating ApiKey is owned by + /// (`ApiKey.user_id`), snapshotted at request time so the Logs + /// member filter keeps naming who actually made the call + /// (AISIX-Cloud#1389). Resolving it from `api_key_id` at query time + /// instead would re-attribute a key's whole history the moment an + /// operator rebinds it, and lose the attribution entirely once the + /// key is deleted. Empty when the key is bound to no member, or + /// when auth failed before resolution; cp-api stores empty as NULL. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub user_id: String, + /// The model alias exactly as the client sent it in the request /// body (`model` field) — a Model-Group name for routed requests, /// a direct model's display name otherwise. `model_id` records the @@ -775,8 +786,25 @@ impl UsageSink { /// records did we lose" answerable. The event itself cannot supply /// them: its `requested_model` is caller-controlled text (#451) and it /// carries no ProviderKey id at all. + /// + /// The one attribution dimension that DOES come off the event is + /// `user_id` (AISIX-Cloud#1389): it is a resolved ApiKey field, not + /// caller text, and taking it here rather than from each handler's + /// label builder is what makes the counter and the row cp-api persists + /// structurally incapable of naming different members. pub fn try_emit(&self, handler: &'static str, event: UsageEvent, labels: UsageEventLabels<'_>) { log_provider_call(handler, &event); + // Owned because `event` is moved into the channel below while the + // drop counter still needs the label. + let user_id = event.user_id.clone(); + let labels = UsageEventLabels { + user_id: if user_id.is_empty() { + "unknown" + } else { + user_id.as_str() + }, + ..labels + }; // Normalise inbound_protocol to a fixed `&'static str` set at // the boundary (audit MEDIUM-3). This both kills the heap // alloc per call AND pins prometheus cardinality at the type @@ -1175,14 +1203,18 @@ mod tests { sink.try_emit( "chat", UsageEvent { - status_code: 200, + status_code: 429, inbound_protocol: "openai".into(), + // Attribution the sink reads off the event itself, not off + // the label set the handler built. + user_id: "member-1".into(), ..Default::default() }, UsageEventLabels { model: "customer-chat", provider_key_id: "pk-1", provider_key_name: "openai-prod", + user_id: "unknown", upstream_protocol: "openai", }, ); @@ -1196,6 +1228,11 @@ mod tests { ("model", "customer-chat"), ("provider_key_id", "pk-1"), ("provider_key_name", "openai-prod"), + ("user_id", "member-1"), + // The raw code sits beside the family, so a query can name + // one failure mode without giving up the family rollup. + ("status_code", "4xx"), + ("http_status_code", "429"), ], ); let dropped = parse_counter_value( @@ -1206,6 +1243,7 @@ mod tests { ("model", "customer-chat"), ("provider_key_id", "pk-1"), ("provider_key_name", "openai-prod"), + ("user_id", "member-1"), ], ); assert_eq!( diff --git a/crates/aisix-proxy/src/a2a.rs b/crates/aisix-proxy/src/a2a.rs index 7b7e09238..6a64bd1b8 100644 --- a/crates/aisix-proxy/src/a2a.rs +++ b/crates/aisix-proxy/src/a2a.rs @@ -889,7 +889,11 @@ fn emit_a2a_usage( guardrail_blocked, ..Default::default() }; - crate::usage_attr::apply_jwt_identity(&mut event, auth.jwt.as_ref()); + crate::usage_attr::apply_caller_identity( + &mut event, + auth.jwt.as_ref(), + auth.key().user_id.as_deref(), + ); // The client-perceived duration of the call. Nothing else records it for // `/a2a`: the handler returns the moment a stream's response head is out, // so `aisix_proxy_request_duration_seconds` times only how long a stream diff --git a/crates/aisix-proxy/src/audio.rs b/crates/aisix-proxy/src/audio.rs index f6a2ee12e..de0cdd245 100644 --- a/crates/aisix-proxy/src/audio.rs +++ b/crates/aisix-proxy/src/audio.rs @@ -2104,7 +2104,11 @@ fn emit_usage_event( // responses (AISIX-Cloud#867 parity). crate::usage_attr::apply_pk_telemetry(&mut event, pk); // Handler label "audio" — bucketed prometheus counter (#408). - crate::usage_attr::apply_jwt_identity(&mut event, client.jwt.as_ref()); + crate::usage_attr::apply_caller_identity( + &mut event, + client.jwt.as_ref(), + client.caller.user_id.as_deref(), + ); let usage_model = crate::usage_attr::usage_event_model_label(snap, &event.requested_model).into_owned(); crate::usage_attr::emit_usage( diff --git a/crates/aisix-proxy/src/chat.rs b/crates/aisix-proxy/src/chat.rs index 416a463db..51e845002 100644 --- a/crates/aisix-proxy/src/chat.rs +++ b/crates/aisix-proxy/src/chat.rs @@ -4409,7 +4409,11 @@ fn emit_usage_event( // MCP attribution does not apply to the chat path. ..Default::default() }; - crate::usage_attr::apply_jwt_identity(&mut event, client.jwt.as_ref()); + crate::usage_attr::apply_caller_identity( + &mut event, + client.jwt.as_ref(), + client.caller.user_id.as_deref(), + ); // Guardrail outcome counters (#379). Recorded here — the one place every // chat path (success / error / streaming / cache-hit) funnels through — // from the same guardrail fields the UsageEvent carries. diff --git a/crates/aisix-proxy/src/completions.rs b/crates/aisix-proxy/src/completions.rs index add774fcd..84b10af67 100644 --- a/crates/aisix-proxy/src/completions.rs +++ b/crates/aisix-proxy/src/completions.rs @@ -766,7 +766,11 @@ fn emit_usage_event( ..Default::default() }; crate::usage_attr::apply_pk_telemetry(&mut event, pk); - crate::usage_attr::apply_jwt_identity(&mut event, client.jwt.as_ref()); + crate::usage_attr::apply_caller_identity( + &mut event, + client.jwt.as_ref(), + client.caller.user_id.as_deref(), + ); let usage_model = crate::usage_attr::usage_event_model_label(snap, &event.requested_model).into_owned(); crate::usage_attr::emit_usage( diff --git a/crates/aisix-proxy/src/embeddings.rs b/crates/aisix-proxy/src/embeddings.rs index e89f29fd5..266fa276a 100644 --- a/crates/aisix-proxy/src/embeddings.rs +++ b/crates/aisix-proxy/src/embeddings.rs @@ -715,7 +715,11 @@ fn emit_usage_event( }; crate::usage_attr::apply_pk_telemetry(&mut event, pk); // Handler label "embeddings" — bucketed prometheus counter (#408). - crate::usage_attr::apply_jwt_identity(&mut event, client.jwt.as_ref()); + crate::usage_attr::apply_caller_identity( + &mut event, + client.jwt.as_ref(), + client.caller.user_id.as_deref(), + ); let usage_model = crate::usage_attr::usage_event_model_label(snap, &event.requested_model).into_owned(); crate::usage_attr::emit_usage( diff --git a/crates/aisix-proxy/src/images.rs b/crates/aisix-proxy/src/images.rs index 9b25ee51a..c5b26eba6 100644 --- a/crates/aisix-proxy/src/images.rs +++ b/crates/aisix-proxy/src/images.rs @@ -551,7 +551,11 @@ pub(crate) fn emit_usage_event( }; crate::usage_attr::apply_pk_telemetry(&mut event, pk); // Handler label "images" — bucketed prometheus counter (#408). - crate::usage_attr::apply_jwt_identity(&mut event, client.jwt.as_ref()); + crate::usage_attr::apply_caller_identity( + &mut event, + client.jwt.as_ref(), + client.caller.user_id.as_deref(), + ); let usage_model = crate::usage_attr::usage_event_model_label(snap, &event.requested_model).into_owned(); crate::usage_attr::emit_usage( diff --git a/crates/aisix-proxy/src/jobs.rs b/crates/aisix-proxy/src/jobs.rs index c248f9d4c..4250fe482 100644 --- a/crates/aisix-proxy/src/jobs.rs +++ b/crates/aisix-proxy/src/jobs.rs @@ -623,7 +623,11 @@ fn emit_job_usage_event( }; let pk = crate::usage_attr::ResolvedPk::resolve(snap, &target.pk_entry.id); crate::usage_attr::apply_pk_telemetry(&mut event, &pk); - crate::usage_attr::apply_jwt_identity(&mut event, auth.jwt.as_ref()); + crate::usage_attr::apply_caller_identity( + &mut event, + auth.jwt.as_ref(), + auth.key().user_id.as_deref(), + ); let usage_model = crate::usage_attr::usage_event_model_label(snap, &event.requested_model).into_owned(); crate::usage_attr::emit_usage( @@ -1802,6 +1806,7 @@ fn maybe_attribute_batch( let state = state.clone(); let api_key_id = auth.entry.id.clone(); + let user_id = auth.entry.value.user_id.clone(); let jwt = auth.jwt.clone(); let model_id = target.model_entry.id.clone(); let display_name = target.display_name().to_string(); @@ -1818,6 +1823,7 @@ fn maybe_attribute_batch( &state, &api_key_id, jwt.as_ref(), + user_id.as_deref(), &model_id, &display_name, cost.as_ref(), @@ -1850,6 +1856,7 @@ async fn attribute_batch_usage( state: &ProxyState, api_key_id: &str, jwt: Option<&std::sync::Arc>, + user_id: Option<&str>, model_id: &str, display_name: &str, cost: Option<&aisix_core::models::model::ModelCost>, @@ -1962,7 +1969,7 @@ async fn attribute_batch_usage( crate::usage_attr::apply_pk_telemetry(&mut event, &pk); // Attribution names the identity that observed completion — the // same caller the event's api_key_id already reflects. - crate::usage_attr::apply_jwt_identity(&mut event, jwt); + crate::usage_attr::apply_caller_identity(&mut event, jwt, user_id); let usage_model = crate::usage_attr::usage_event_model_label(&snap, &event.requested_model).into_owned(); state.usage_sink.try_emit( diff --git a/crates/aisix-proxy/src/mcp.rs b/crates/aisix-proxy/src/mcp.rs index a13df5fcb..661c48ae6 100644 --- a/crates/aisix-proxy/src/mcp.rs +++ b/crates/aisix-proxy/src/mcp.rs @@ -1188,7 +1188,11 @@ fn emit_tool_call_usage( .unwrap_or_default(), ..Default::default() }; - crate::usage_attr::apply_jwt_identity(&mut event, auth.jwt.as_ref()); + crate::usage_attr::apply_caller_identity( + &mut event, + auth.jwt.as_ref(), + auth.key().user_id.as_deref(), + ); crate::usage_attr::apply_auth_type(&mut event, auth); // A tool call resolves neither a model nor a ProviderKey, so the // attribution labels are the placeholder — present so this family has diff --git a/crates/aisix-proxy/src/messages.rs b/crates/aisix-proxy/src/messages.rs index 421100f2e..b3ead3330 100644 --- a/crates/aisix-proxy/src/messages.rs +++ b/crates/aisix-proxy/src/messages.rs @@ -2996,7 +2996,11 @@ fn emit_anthropic_usage_event( }; // Handler label "messages" — Anthropic /v1/messages inbound // path. Bucketed prometheus counter (#408). - crate::usage_attr::apply_jwt_identity(&mut event, client.jwt.as_ref()); + crate::usage_attr::apply_caller_identity( + &mut event, + client.jwt.as_ref(), + client.caller.user_id.as_deref(), + ); let usage_model = crate::usage_attr::usage_event_model_label(snap, &event.requested_model); // The metric code below still reads `event`, so the chokepoint gets // its own copy (it stamps `trace_id` on the emitted one). diff --git a/crates/aisix-proxy/src/passthrough_route.rs b/crates/aisix-proxy/src/passthrough_route.rs index 67a132581..088c0315b 100644 --- a/crates/aisix-proxy/src/passthrough_route.rs +++ b/crates/aisix-proxy/src/passthrough_route.rs @@ -795,6 +795,7 @@ async fn dispatch( path: path.clone(), request_id: client.request_id.clone(), api_key_id: auth.entry.id.clone(), + user_id: auth.entry.value.user_id.clone(), jwt: auth.jwt.clone(), anonymous: auth.anonymous, client_identity, @@ -1967,6 +1968,10 @@ struct RouteTelemetry { path: String, request_id: String, api_key_id: String, + /// Org member the authenticating key belongs to (AISIX-Cloud#1389). + /// `None` for a key bound to no member — including the anonymous + /// route key, which belongs to the route rather than to a person. + user_id: Option, jwt: Option>, /// Whether the caller reached this route through `auth_mode: /// anonymous` rather than a credential of its own. Stamped onto the @@ -2133,7 +2138,11 @@ impl RouteTelemetry { ..Default::default() }; crate::usage_attr::apply_pk_telemetry(&mut event, &pk); - crate::usage_attr::apply_jwt_identity(&mut event, self.jwt.as_ref()); + crate::usage_attr::apply_caller_identity( + &mut event, + self.jwt.as_ref(), + self.user_id.as_deref(), + ); if self.anonymous { event.auth_type = "anonymous".to_string(); } diff --git a/crates/aisix-proxy/src/realtime.rs b/crates/aisix-proxy/src/realtime.rs index c42b173c6..26f37b4ac 100644 --- a/crates/aisix-proxy/src/realtime.rs +++ b/crates/aisix-proxy/src/realtime.rs @@ -849,7 +849,11 @@ async fn run_session( ..Default::default() }; crate::usage_attr::apply_pk_telemetry(&mut event, &pk); - crate::usage_attr::apply_jwt_identity(&mut event, auth.jwt.as_ref()); + crate::usage_attr::apply_caller_identity( + &mut event, + auth.jwt.as_ref(), + auth.key().user_id.as_deref(), + ); let usage_model = crate::usage_attr::usage_event_model_label(&snap, &event.requested_model).into_owned(); crate::usage_attr::emit_usage( diff --git a/crates/aisix-proxy/src/rerank.rs b/crates/aisix-proxy/src/rerank.rs index a09971ef5..5a3b09c8c 100644 --- a/crates/aisix-proxy/src/rerank.rs +++ b/crates/aisix-proxy/src/rerank.rs @@ -725,7 +725,11 @@ fn emit_usage_event( // branded_provider / pk_label / byo_label) ARE populated — same lookup as // chat / messages / responses / embeddings (AISIX-Cloud#867 parity). crate::usage_attr::apply_pk_telemetry(&mut event, pk); - crate::usage_attr::apply_jwt_identity(&mut event, client.jwt.as_ref()); + crate::usage_attr::apply_caller_identity( + &mut event, + client.jwt.as_ref(), + client.caller.user_id.as_deref(), + ); let usage_model = crate::usage_attr::usage_event_model_label(snap, &event.requested_model).into_owned(); crate::usage_attr::emit_usage( diff --git a/crates/aisix-proxy/src/responses.rs b/crates/aisix-proxy/src/responses.rs index 849c4ca6c..13644bdae 100644 --- a/crates/aisix-proxy/src/responses.rs +++ b/crates/aisix-proxy/src/responses.rs @@ -3035,7 +3035,11 @@ fn emit_usage_event( guardrail_enforced_hits: crate::usage_attr::terminal_enforced_hits(terminal, audit), ..Default::default() }; - crate::usage_attr::apply_jwt_identity(&mut event, client.jwt.as_ref()); + crate::usage_attr::apply_caller_identity( + &mut event, + client.jwt.as_ref(), + client.caller.user_id.as_deref(), + ); let usage_model = crate::usage_attr::usage_event_model_label(snap, &event.requested_model); crate::usage_attr::emit_usage( state, @@ -3160,7 +3164,11 @@ fn emit_zero_token_event( guardrail_blocked: terminal && guardrail_blocked, ..Default::default() }; - crate::usage_attr::apply_jwt_identity(&mut event, client.jwt.as_ref()); + crate::usage_attr::apply_caller_identity( + &mut event, + client.jwt.as_ref(), + client.caller.user_id.as_deref(), + ); let usage_model = crate::usage_attr::usage_event_model_label(snap, &event.requested_model); crate::usage_attr::emit_usage( state, diff --git a/crates/aisix-proxy/src/usage_attr.rs b/crates/aisix-proxy/src/usage_attr.rs index 974e27db4..9239e054d 100644 --- a/crates/aisix-proxy/src/usage_attr.rs +++ b/crates/aisix-proxy/src/usage_attr.rs @@ -392,18 +392,29 @@ pub(crate) fn apply_pk_telemetry(event: &mut UsageEvent, pk: &ResolvedPk<'_>) { event.byo_label = sanitize_tag(tags.byo_label.unwrap_or_default()); } -/// Stamp the JWT identity attribution fields onto an in-progress -/// UsageEvent (AISIX-Cloud#564). A `None` identity (the API-key path) -/// leaves the fields empty, which skip-serialize to wire NULL. One -/// source of truth for the mapping so the handler family can't drift — +/// Stamp the caller-identity attribution fields onto an in-progress +/// UsageEvent: the JWT identity (AISIX-Cloud#564) and the org member the +/// authenticating key belongs to (AISIX-Cloud#1389). A `None` in either +/// argument leaves its fields empty, which skip-serialize to wire NULL. +/// One source of truth for the mapping so the handler family can't drift — /// same rationale as [`apply_pk_telemetry`]. The values are sanitised /// like every other externally-influenced tag: the subject is a claim /// from a verified token, but the identity provider is still not a /// trusted emitter of control characters or unbounded strings. -pub(crate) fn apply_jwt_identity( +/// +/// `user_id` takes both halves of the auth surface: on the API-key path it +/// is the key's own `user_id`, and on the JWT path it is the `user_id` of +/// the key the token resolved to — a JWT request runs AS a key, so one +/// argument covers both and the member filter sees every credential a +/// member calls through. +pub(crate) fn apply_caller_identity( event: &mut UsageEvent, jwt: Option<&std::sync::Arc>, + user_id: Option<&str>, ) { + if let Some(user_id) = user_id { + event.user_id = sanitize_tag(user_id.to_string()); + } let Some(jwt) = jwt else { return; }; @@ -516,7 +527,11 @@ pub(crate) fn build_error_usage_event( guardrail_enforced_hits: enforced, ..Default::default() }; - apply_jwt_identity(&mut event, client.jwt.as_ref()); + apply_caller_identity( + &mut event, + client.jwt.as_ref(), + client.caller.user_id.as_deref(), + ); event } @@ -560,6 +575,10 @@ pub(crate) fn usage_event_labels<'a>( labels.id() }, provider_key_name: labels.name(), + // Overwritten by `UsageSink::try_emit` from the event's own + // `user_id` (AISIX-Cloud#1389) — one place, so no handler in this + // family can build a label set that disagrees with the row. + user_id: "unknown", // Same `PkLabels` the request families read (AISIX-Cloud#1403), so // `aisix_usage_events_emitted_total` joins with them on // `upstream_protocol` instead of dropping out of the aggregation. diff --git a/crates/aisix-proxy/src/videos.rs b/crates/aisix-proxy/src/videos.rs index e46a2ee6e..eabde68b4 100644 --- a/crates/aisix-proxy/src/videos.rs +++ b/crates/aisix-proxy/src/videos.rs @@ -2025,7 +2025,11 @@ fn emit_submit_usage_event( }; let pk = crate::usage_attr::ResolvedPk::resolve(snap, provider_key_id); crate::usage_attr::apply_pk_telemetry(&mut event, &pk); - crate::usage_attr::apply_jwt_identity(&mut event, client.jwt.as_ref()); + crate::usage_attr::apply_caller_identity( + &mut event, + client.jwt.as_ref(), + client.caller.user_id.as_deref(), + ); let usage_model = crate::usage_attr::usage_event_model_label(snap, &event.requested_model).into_owned(); crate::usage_attr::emit_usage( diff --git a/tests/e2e/src/cases/usage-member-attribution-e2e.test.ts b/tests/e2e/src/cases/usage-member-attribution-e2e.test.ts new file mode 100644 index 000000000..2e34806fe --- /dev/null +++ b/tests/e2e/src/cases/usage-member-attribution-e2e.test.ts @@ -0,0 +1,292 @@ +import { createHash } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import { + agentClaims, + EtcdClient, + metricDelta, + ProxyClient, + scrapeMetrics, + SeedClient, + spawnApp, + startMockIdp, + startMockSls, + startOpenAiUpstream, + waitConfigPropagation, + waitForSlsLog, + type MetricSample, + type MockIdp, + type MockSls, + type OpenAiUpstream, + type SpawnedApp, +} from "../harness/index.js"; + +// E2E for AISIX-Cloud#1389: the usage event must name the org MEMBER behind +// the request, not just the credential it arrived on. +// +// The ask is "show me Alice's 429s in the last 24h". Before this, the only +// caller identity on a usage row was `api_key_id` (plus a JWT subject), so +// answering it meant enumerating every key Alice owns, querying each, and +// merging by hand — and a member calling through both an API key and OIDC +// would still be split across two identities. `user_id` is snapshotted onto +// the event at request time rather than resolved from `api_key_id` at query +// time, so rebinding or deleting a key cannot re-attribute or erase the +// history it already produced. +// +// Read back off a real Aliyun-SLS export from a real `aisix` binary, so what +// is asserted is the row a consumer actually receives. +// +// The three probes are the claim's real content: +// - the API-key path stamps the owning member; +// - a JWT resolving to a DIFFERENT key owned by the SAME member stamps the +// same member — this is the "one member, several credentials" half of +// the report, and it is the one a join on api_key_id gets wrong; +// - an unowned key stamps nobody, so a member filter cannot sweep up +// traffic that was never theirs. + +const OWNED_PLAINTEXT = "sk-usage-member-owned"; +const UNOWNED_PLAINTEXT = "sk-usage-member-unowned"; +const hash = (s: string) => createHash("sha256").update(s).digest("hex"); + +const CREDENTIAL_REF = "mock"; +const SLS_PROJECT = "aisix-e2e-obs"; +const LOGSTORE = "usage-member-attribution"; + +const MODEL = "uma-model"; +// A model whose upstream always throttles — the issue's headline scenario +// ("show me Alice's 429s") needs a 429 that actually reaches an upstream and +// therefore produces a per-attempt usage row. +const THROTTLED_MODEL = "uma-throttled"; +// The member every owned credential in this case belongs to. A uuid-shaped +// value because that is what cp-api projects (`api_keys.user_id`). +const ALICE = "3f1b7c62-9d4e-4a51-8f0b-2c6d5e4a1b90"; + +describe("usage member attribution e2e: a usage row names the org member (#1389)", () => { + let upstream: OpenAiUpstream | undefined; + let throttled: OpenAiUpstream | undefined; + let sls: MockSls | undefined; + let idp: MockIdp | undefined; + let app: SpawnedApp | undefined; + let etcdReachable = false; + + async function chat( + credential: string, + marker: string, + model = MODEL, + ): Promise { + const res = await fetch(`${app!.proxyUrl}/v1/chat/completions`, { + method: "POST", + headers: { + authorization: `Bearer ${credential}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + model, + messages: [{ role: "user", content: marker }], + }), + }); + await res.text(); + return res; + } + + beforeAll(async () => { + const etcd = new EtcdClient(); + etcdReachable = await etcd.ping(); + if (!etcdReachable) return; + + sls = await startMockSls(); + idp = await startMockIdp(); + upstream = await startOpenAiUpstream(); + throttled = await startOpenAiUpstream({ + status: 429, + errorBody: { error: { message: "slow down", type: "rate_limit_error" } }, + }); + + app = await spawnApp({ + extraEnv: { + [`SLS_CRED_${CREDENTIAL_REF.toUpperCase()}_AK_ID`]: "LTAI_mock_ak", + [`SLS_CRED_${CREDENTIAL_REF.toUpperCase()}_AK_SECRET`]: "mock_ak_secret", + }, + }); + const seed = new SeedClient(etcd, app.etcdPrefix); + + await seed.createObservabilityExporter({ + name: "uma-sls", + enabled: true, + kind: "aliyun_sls", + endpoint: sls.url, + project: SLS_PROJECT, + logstore: LOGSTORE, + credential_ref: CREDENTIAL_REF, + content_mode: "full", + }); + + const pk = await seed.createProviderKey({ + display_name: "uma-pk", + secret: "sk-mock", + api_base: `${upstream.baseUrl}/v1`, + }); + await seed.createModel({ + display_name: MODEL, + provider: "openai", + model_name: "gpt-4o-mini", + provider_key_id: pk.id, + }); + + const throttledPk = await seed.createProviderKey({ + display_name: "uma-throttled-pk", + secret: "sk-mock", + api_base: `${throttled.baseUrl}/v1`, + }); + await seed.createModel({ + display_name: THROTTLED_MODEL, + provider: "openai", + model_name: "gpt-4o-mini", + provider_key_id: throttledPk.id, + }); + + await seed.createOidcProvider({ + name: "uma-idp", + issuer: idp.url, + audiences: ["aisix-gateway"], + jwks_uri: idp.jwksUrl, + }); + + // Alice's SECOND credential, reached over OIDC rather than a bearer + // key. A different api_key row, the same person. + await seed.createApiKey({ + key_hash: hash("sk-usage-member-jwt-bound"), + allowed_models: [MODEL, THROTTLED_MODEL], + user_id: ALICE, + jwt_subject: "alice", + jwt_provider: "uma-idp", + }); + + // A key owned by nobody — the shape every key created without an + // explicit owner takes. + await seed.createApiKey({ + key_hash: hash(UNOWNED_PLAINTEXT), + allowed_models: [MODEL], + }); + + // Written LAST so the readiness gate below implies everything above is + // already in the snapshot (one watch, applied in revision order). + await seed.createApiKey({ + key_hash: hash(OWNED_PLAINTEXT), + allowed_models: [MODEL, THROTTLED_MODEL], + user_id: ALICE, + }); + + const proxy = new ProxyClient(app.proxyUrl, OWNED_PLAINTEXT); + await waitConfigPropagation(async () => (await proxy.listModels()).status === 200); + }); + + afterAll(async () => { + await app?.exit(); + await upstream?.close(); + await throttled?.close(); + await sls?.close(); + await idp?.close(); + }); + + test("an API key bound to a member stamps that member on its usage row", async (ctx) => { + if (!etcdReachable || !app || !sls) { + ctx.skip(); + return; + } + const marker = "uma-apikey-4f21"; + expect((await chat(OWNED_PLAINTEXT, marker)).status).toBe(200); + + const log = await waitForSlsLog( + sls, + LOGSTORE, + (l) => (l.get("prompt") ?? "").includes(marker), + "api-key usage row", + ); + expect(log.get("user_id")).toBe(ALICE); + }); + + test("a JWT resolving to the member's OTHER key stamps the same member", async (ctx) => { + if (!etcdReachable || !app || !sls || !idp) { + ctx.skip(); + return; + } + const marker = "uma-jwt-8c07"; + const token = idp.sign(agentClaims(idp.url, { sub: "alice" })); + expect((await chat(token, marker)).status).toBe(200); + + const log = await waitForSlsLog( + sls, + LOGSTORE, + (l) => (l.get("prompt") ?? "").includes(marker), + "jwt usage row", + ); + // Same person, different credential: this is what a query-time join on + // api_key_id cannot express in one filter. + expect(log.get("user_id")).toBe(ALICE); + expect(log.get("jwt_subject")).toBe("alice"); + }); + + test("a key owned by nobody stamps no member", async (ctx) => { + if (!etcdReachable || !app || !sls) { + ctx.skip(); + return; + } + const marker = "uma-unowned-1d93"; + expect((await chat(UNOWNED_PLAINTEXT, marker)).status).toBe(200); + + const log = await waitForSlsLog( + sls, + LOGSTORE, + (l) => (l.get("prompt") ?? "").includes(marker), + "unowned-key usage row", + ); + // Absent, not a placeholder: a member filter must not sweep up traffic + // that belongs to no member. + expect(log.get("user_id")).toBeUndefined(); + }); + + test("a member's 429 is addressable by member and by raw status code", async (ctx) => { + if (!etcdReachable || !app || !sls) { + ctx.skip(); + return; + } + const before: MetricSample[] = await scrapeMetrics(app.metricsUrl); + const marker = "uma-throttled-2b55"; + expect((await chat(OWNED_PLAINTEXT, marker, THROTTLED_MODEL)).status).toBe(429); + const after: MetricSample[] = await scrapeMetrics(app.metricsUrl); + + // The row an operator finds under "member = Alice, status = 429". + const log = await waitForSlsLog( + sls, + LOGSTORE, + (l) => (l.get("prompt") ?? "").includes(marker), + "throttled usage row", + ); + expect(log.get("user_id")).toBe(ALICE); + expect(log.get("status_code")).toBe("429"); + + // …and the same question answered on the counter, which before this + // could only say "4xx" and could not say "whose". + expect( + metricDelta(before, after, "aisix_usage_events_emitted_total", { + user_id: ALICE, + http_status_code: "429", + }), + ).toBeGreaterThanOrEqual(1); + // The status family survives beside the raw code, so a dashboard + // written against `status_code="4xx"` keeps working unchanged. + expect( + metricDelta(before, after, "aisix_usage_events_emitted_total", { + user_id: ALICE, + status_code: "4xx", + }), + ).toBeGreaterThanOrEqual(1); + // Traffic that resolved no member must not land on the member's series. + expect( + metricDelta(before, after, "aisix_usage_events_emitted_total", { + user_id: "unknown", + http_status_code: "429", + }), + ).toBe(0); + }); +}); From 20de40d745199cd58a14cf300aa667cd92260320 Mon Sep 17 00:00:00 2001 From: Jarvis Date: Fri, 28 Aug 2026 10:05:22 +0000 Subject: [PATCH 2/3] perf(telemetry): key the emit counter on the raw status without an allocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The worker-cache key is built on every usage event; rendering the status code into a String ahead of the lookup put an allocation on the cache-HIT path, which is the common one. WorkerKey::label_u16 exists for exactly this (record_request already uses it) — its hand-rolled digits avoid the fmt::write machinery — so the String render moves into the register closure that only runs on a miss. --- crates/aisix-obs/src/metrics.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/aisix-obs/src/metrics.rs b/crates/aisix-obs/src/metrics.rs index c9edfeec9..c848a7b6b 100644 --- a/crates/aisix-obs/src/metrics.rs +++ b/crates/aisix-obs/src/metrics.rs @@ -2079,14 +2079,13 @@ impl Metrics { labels: UsageEventLabels<'_>, ) { let status_class = status_bucket(status_code); - let http_status_code = status_code.to_string(); self.cached_counter( M_USAGE_EVENT_EMITS_TOTAL, 1, |k| { k.label(handler); k.label(status_class); - k.label(&http_status_code); + k.label_u16(status_code); k.label(inbound_protocol); k.label(labels.model); k.label(labels.provider_key_id); @@ -2099,7 +2098,7 @@ impl Metrics { M_USAGE_EVENT_EMITS_TOTAL, "handler" => handler, "status_code" => status_class, - "http_status_code" => http_status_code.clone(), + "http_status_code" => status_code.to_string(), "inbound_protocol" => inbound_protocol, "upstream_protocol" => labels.upstream_protocol.to_string(), "model" => labels.model.to_string(), From 8f13e8f7860afca6d249431d23cbdadcd8bcd8ae Mon Sep 17 00:00:00 2001 From: Jarvis Date: Fri, 28 Aug 2026 10:17:50 +0000 Subject: [PATCH 3/3] fix(telemetry): name the raw status label `status`, not `http_status_code` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The e2e prometheus guard caught it: `http_status_code` contains `status_code` as a substring, so the existing assertion that the family label stays bucketed (`not.toMatch(status_code="200")`) started matching the new raw label and failed. Any operator regex or grep over `status_code="…"` would have had the same ambiguity. `status` is what `aisix_proxy_requests_total` already calls the raw code, so one spelling now works across both families — and it collides with nothing. `status_code` keeps its `2xx`/`4xx`/`5xx` bucketing, so the change stays non-breaking. The guard's comment said raw codes must never appear on this family at all; that is no longer the intent, so it now states which label is bucketed and which is raw, and asserts both. --- crates/aisix-obs/src/metrics.rs | 35 +++++++++---------- crates/aisix-obs/src/usage.rs | 2 +- .../src/cases/prometheus-metrics-e2e.test.ts | 13 +++++-- .../usage-member-attribution-e2e.test.ts | 4 +-- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/crates/aisix-obs/src/metrics.rs b/crates/aisix-obs/src/metrics.rs index c848a7b6b..8f62e580a 100644 --- a/crates/aisix-obs/src/metrics.rs +++ b/crates/aisix-obs/src/metrics.rs @@ -260,11 +260,15 @@ pub const M_GUARDRAIL_LATENCY_SECONDS: &str = "aisix_guardrail_latency_seconds"; /// embeddings / responses / completions / rerank / audio / /// images / messages). Fixed enumeration, low cardinality. /// - `status_code`: bucketed as `2xx` / `4xx` / `5xx`. -/// - `http_status_code`: the raw code (`429`, `502`, ...), so a query can -/// name one failure mode instead of a whole family (AISIX-Cloud#1389). -/// It adds no series over `status_code` alone — the raw code determines -/// the family — and `status_code` is kept so existing dashboards and -/// alerts written against `status_code="4xx"` keep working unchanged. +/// - `status`: the raw code (`429`, `502`, ...), so a query can name one +/// failure mode instead of a whole family (AISIX-Cloud#1389). Named to +/// match `aisix_proxy_requests_total`'s own raw-code label, so one +/// spelling works across both families — and deliberately NOT +/// `http_status_code`, which contains `status_code` as a substring and +/// would make every `status_code="…"` matcher ambiguous. It adds no +/// series over `status_code` alone (the raw code determines the family), +/// and `status_code` is kept so dashboards and alerts written against +/// `status_code="4xx"` keep working unchanged. /// - `user_id`: the org member behind the request, from /// [`UsageEventLabels`]. /// - `inbound_protocol`: `openai` / `anthropic`. Matches the @@ -2019,7 +2023,7 @@ impl Metrics { /// that invariant exists only on dimensions BOTH sides carry. The /// protocol is a function of the ProviderKey row `provider_key_id` /// already names, so like `provider_key_name` it adds no series here. - /// (`handler` / `status_code` / `http_status_code` / `inbound_protocol` + /// (`handler` / `status_code` / `status` / `inbound_protocol` /// are the emit counter's own arguments, not attribution, and stay /// one-sided — `emitted == delivered + dropped` is an invariant over /// the attribution dimensions, which is why `user_id` DOES appear on @@ -2098,7 +2102,7 @@ impl Metrics { M_USAGE_EVENT_EMITS_TOTAL, "handler" => handler, "status_code" => status_class, - "http_status_code" => status_code.to_string(), + "status" => status_code.to_string(), "inbound_protocol" => inbound_protocol, "upstream_protocol" => labels.upstream_protocol.to_string(), "model" => labels.model.to_string(), @@ -4379,14 +4383,14 @@ mod tests { assert!( rendered.contains( - "handler=\"messages\",status_code=\"2xx\",http_status_code=\"200\",\ + "handler=\"messages\",status_code=\"2xx\",status=\"200\",\ inbound_protocol=\"anthropic\",upstream_protocol=\"openai\"" ), "cross-protocol sample must report the upstream's protocol:\n{rendered}" ); assert!( rendered.contains( - "handler=\"chat\",status_code=\"4xx\",http_status_code=\"401\",\ + "handler=\"chat\",status_code=\"4xx\",status=\"401\",\ inbound_protocol=\"openai\",upstream_protocol=\"unknown\"" ), "an unresolved upstream must read `unknown`, never borrow the \ @@ -4444,15 +4448,10 @@ mod tests { }; // The emit counter's own arguments — a surface of the emitting // handler, not of the event's attribution. - let emit_only: BTreeSet = [ - "handler", - "status_code", - "http_status_code", - "inbound_protocol", - ] - .into_iter() - .map(str::to_string) - .collect(); + let emit_only: BTreeSet = ["handler", "status_code", "status", "inbound_protocol"] + .into_iter() + .map(str::to_string) + .collect(); let attribution: BTreeSet = labels_of(M_USAGE_EVENT_EMITS_TOTAL) .difference(&emit_only) .cloned() diff --git a/crates/aisix-obs/src/usage.rs b/crates/aisix-obs/src/usage.rs index 683d541bc..e94a2a74a 100644 --- a/crates/aisix-obs/src/usage.rs +++ b/crates/aisix-obs/src/usage.rs @@ -1232,7 +1232,7 @@ mod tests { // The raw code sits beside the family, so a query can name // one failure mode without giving up the family rollup. ("status_code", "4xx"), - ("http_status_code", "429"), + ("status", "429"), ], ); let dropped = parse_counter_value( diff --git a/tests/e2e/src/cases/prometheus-metrics-e2e.test.ts b/tests/e2e/src/cases/prometheus-metrics-e2e.test.ts index 0a0d5d8e5..8d4b67798 100644 --- a/tests/e2e/src/cases/prometheus-metrics-e2e.test.ts +++ b/tests/e2e/src/cases/prometheus-metrics-e2e.test.ts @@ -175,11 +175,20 @@ describe("prometheus metrics e2e", () => { expect(after).toMatch( /aisix_usage_events_emitted_total\{[^}]*inbound_protocol="openai"[^}]*\}/, ); - // Status codes MUST be bucketed (2xx / 4xx / 5xx) — raw "200" - // would explode cardinality at ~1000 series per handler×protocol. + // `status_code` MUST stay bucketed (2xx / 4xx / 5xx): dashboards and + // alerts are written against it, and AISIX-Cloud#1389 added the raw + // code as a SEPARATE label rather than redefining this one. expect(after).not.toMatch( /aisix_usage_events_emitted_total\{[^}]*status_code="200"/, ); + // …and that separate label is `status`, the same name the request + // family gives the raw code. The two live side by side, so a query can + // ask for one exact failure mode without giving up the family rollup + // (#1389). Deliberately not `http_status_code`: it would contain + // `status_code` as a substring and make the assertion above ambiguous. + expect(after).toMatch( + /aisix_usage_events_emitted_total\{[^}]*[,{]status="200"/, + ); const afterCount = parseUsageEmittedCount(after, "chat", "2xx", "openai"); expect(afterCount - beforeCount).toBeGreaterThanOrEqual(1); diff --git a/tests/e2e/src/cases/usage-member-attribution-e2e.test.ts b/tests/e2e/src/cases/usage-member-attribution-e2e.test.ts index 2e34806fe..8cb155783 100644 --- a/tests/e2e/src/cases/usage-member-attribution-e2e.test.ts +++ b/tests/e2e/src/cases/usage-member-attribution-e2e.test.ts @@ -270,7 +270,7 @@ describe("usage member attribution e2e: a usage row names the org member (#1389) expect( metricDelta(before, after, "aisix_usage_events_emitted_total", { user_id: ALICE, - http_status_code: "429", + status: "429", }), ).toBeGreaterThanOrEqual(1); // The status family survives beside the raw code, so a dashboard @@ -285,7 +285,7 @@ describe("usage member attribution e2e: a usage row names the org member (#1389) expect( metricDelta(before, after, "aisix_usage_events_emitted_total", { user_id: "unknown", - http_status_code: "429", + status: "429", }), ).toBe(0); });