diff --git a/crates/aisix-core/src/models/apikey.rs b/crates/aisix-core/src/models/apikey.rs index 6f6832f5..f456451f 100644 --- a/crates/aisix-core/src/models/apikey.rs +++ b/crates/aisix-core/src/models/apikey.rs @@ -30,6 +30,24 @@ pub struct ApiKey { #[serde(default, skip_serializing_if = "Option::is_none")] pub rate_limit: Option, + /// Team this API key belongs to. Used as a limiter bucket key + /// (`team:`) for team-level rate limiting. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub team_id: Option, + + /// Rate limit inherited from the owning team. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub team_rate_limit: Option, + + /// Org member who owns this key. Used as a limiter bucket key + /// (`member:`) for member-level rate limiting. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub owner_id: Option, + + /// Rate limit inherited from the owning member. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub owner_rate_limit: Option, + /// etcd-key uuid; filled by the loader, never in the JSON payload. #[serde(skip)] pub(crate) runtime_id: String, @@ -144,6 +162,10 @@ mod tests { key_hash: "abc".into(), allowed_models: vec![], rate_limit: None, + team_id: None, + team_rate_limit: None, + owner_id: None, + owner_rate_limit: None, runtime_id: String::new(), }; assert!(!k.can_access("my-gpt4")); @@ -207,4 +229,32 @@ mod tests { // Resource::name now returns key_hash, not plaintext. assert_eq!(k.name(), SAMPLE_HASH); } + + #[test] + fn deserialises_with_team_and_owner_fields() { + let k: ApiKey = serde_json::from_str(&format!( + r#"{{ + "key_hash": "{SAMPLE_HASH}", + "allowed_models": ["gpt-4o"], + "team_id": "team-uuid-1", + "team_rate_limit": {{"rpm": 600}}, + "owner_id": "member-uuid-1", + "owner_rate_limit": {{"tpm": 200000}} + }}"# + )) + .unwrap(); + assert_eq!(k.team_id.as_deref(), Some("team-uuid-1")); + assert_eq!(k.team_rate_limit.as_ref().unwrap().rpm, Some(600)); + assert_eq!(k.owner_id.as_deref(), Some("member-uuid-1")); + assert_eq!(k.owner_rate_limit.as_ref().unwrap().tpm, Some(200000)); + } + + #[test] + fn absent_team_owner_fields_default_to_none() { + let k = sample(); + assert!(k.team_id.is_none()); + assert!(k.team_rate_limit.is_none()); + assert!(k.owner_id.is_none()); + assert!(k.owner_rate_limit.is_none()); + } } diff --git a/crates/aisix-proxy/src/audio.rs b/crates/aisix-proxy/src/audio.rs index 9c0ea0d3..df92a569 100644 --- a/crates/aisix-proxy/src/audio.rs +++ b/crates/aisix-proxy/src/audio.rs @@ -287,9 +287,8 @@ async fn multipart_dispatch( return Err(ProxyError::ModelForbidden(model_name.clone())); } - // Budget + rate-limit gate (issue #107). Audio transcriptions / - // translations bypassed both — Whisper customers ran unmetered. - let _reservation = crate::quota::enforce(state, auth).await?; + let model_rl = crate::quota::ModelRateLimit::from_model(&model_name, &model_entry.value); + let _reservation = crate::quota::enforce(state, auth, model_rl).await?; let model = &model_entry.value; let provider = crate::dispatch::require_provider(model)?; @@ -388,9 +387,8 @@ async fn speech_dispatch( return Err(ProxyError::ModelForbidden(model_name.clone())); } - // Budget + rate-limit gate (issue #107). TTS/speech bypassed - // both pre-fix. - let _reservation = crate::quota::enforce(state, auth).await?; + let model_rl = crate::quota::ModelRateLimit::from_model(&model_name, &model_entry.value); + let _reservation = crate::quota::enforce(state, auth, model_rl).await?; let model = &model_entry.value; let provider = crate::dispatch::require_provider(model)?; diff --git a/crates/aisix-proxy/src/chat.rs b/crates/aisix-proxy/src/chat.rs index d05faaf3..014afd3f 100644 --- a/crates/aisix-proxy/src/chat.rs +++ b/crates/aisix-proxy/src/chat.rs @@ -460,12 +460,10 @@ async fn dispatch( } } - let rl_key = auth.entry.id.clone(); - let rl_limits = auth.key().rate_limit.clone().unwrap_or_default(); - let reservation = state - .limiter - .pre_commit(&rl_key, &rl_limits) - .map_err(|e| with_model(ProxyError::from(e)))?; + // Multi-layer rate-limit reservation (api_key + model + team + member). + let model_rl = crate::quota::ModelRateLimit::from_model(&req.model, &virtual_entry.value); + let reservation = + crate::quota::enforce_rate_limit(state, auth, model_rl).map_err(&with_model)?; let now = created_ts(); @@ -488,13 +486,10 @@ async fn dispatch( .chat_stream(req, &ctx) .await .map_err(|e| with_model(ProxyError::Bridge(e)))?; - // Drop the reservation now: concurrency releases (the SSE - // stream that follows is driven by the client, not by the - // proxy holding open an upstream-bound future), and RPM was - // already counted by pre_commit. TPM is updated retroactively - // on stream-end by `add_tokens_post_stream` — see issue #108. - // Pre-fix this path called commit_tokens(0) and never came - // back, leaving TPM caps blind for all streaming traffic. + // Drop the reservation now: concurrency releases on all layers. + // RPM was already counted by pre_commit. TPM is updated + // retroactively on stream-end by `add_tokens_post_stream`. + let post_stream_keys = reservation.keys(); drop(reservation); // Capture everything the stream-completion callback needs so // it can fire `emit_usage_event` once the terminal SSE chunk @@ -503,7 +498,6 @@ async fn dispatch( // populate `usage` on the last chunk; emitting at handler // return (the non-streaming path's spot) would record zeros. let limiter = Arc::clone(&state.limiter); - let post_stream_key = rl_key.clone(); let state_for_telem = state.clone(); let request_id_for_telem = request_id.to_string(); let model_id_for_telem = model_id.clone(); @@ -534,8 +528,10 @@ async fn dispatch( now, stream_guardrail, move |comp: StreamCompletion| { - // Existing: rate-limit accounting (TPM cap) — see #108. - limiter.add_tokens_post_stream(&post_stream_key, comp.total_tokens); + // Rate-limit accounting (TPM cap) for all layers. + for key in &post_stream_keys { + limiter.add_tokens_post_stream(key, comp.total_tokens); + } // Telemetry: emit with the actual upstream-reported counts. // cost_usd stays 0.0; cp-api recomputes server-side from // its model_pricing catalog (same pattern as the non- diff --git a/crates/aisix-proxy/src/completions.rs b/crates/aisix-proxy/src/completions.rs index c7d0e835..b2fd0239 100644 --- a/crates/aisix-proxy/src/completions.rs +++ b/crates/aisix-proxy/src/completions.rs @@ -108,11 +108,8 @@ async fn dispatch( return Err(ProxyError::ModelForbidden(model_name.to_string())); } - // Budget + rate-limit gate (issue #107). Reservation drops at - // end of dispatch — RPM counts on pre_commit, concurrency - // releases on drop, TPM is left at 0 (this handler doesn't - // surface a uniform token count today). - let _reservation = crate::quota::enforce(state, auth).await?; + let model_rl = crate::quota::ModelRateLimit::from_model(model_name, &model_entry.value); + let _reservation = crate::quota::enforce(state, auth, model_rl).await?; let model = &model_entry.value; let provider = crate::dispatch::require_provider(model)?; diff --git a/crates/aisix-proxy/src/embeddings.rs b/crates/aisix-proxy/src/embeddings.rs index a55e809c..30f3dd0c 100644 --- a/crates/aisix-proxy/src/embeddings.rs +++ b/crates/aisix-proxy/src/embeddings.rs @@ -141,13 +141,8 @@ async fn dispatch( .get(provider) .ok_or(ProxyError::ProviderUnavailable)?; - // Budget + rate-limit gate (issue #107). Pre-fix this endpoint - // bypassed both. The reservation is held until commit_tokens at - // the end of dispatch — embeddings don't surface a stable token - // count across providers, so we commit 0 for now (RPM counts, - // TPM doesn't). Plumbing per-provider token totals through is a - // follow-up. - let reservation = crate::quota::enforce(state, auth).await?; + let model_rl = crate::quota::ModelRateLimit::from_model(&body.model, &model_entry.value); + let reservation = crate::quota::enforce(state, auth, model_rl).await?; let upstream_model_id = crate::dispatch::require_upstream_model(model)?.to_string(); diff --git a/crates/aisix-proxy/src/images.rs b/crates/aisix-proxy/src/images.rs index 0deb08f1..176e70f6 100644 --- a/crates/aisix-proxy/src/images.rs +++ b/crates/aisix-proxy/src/images.rs @@ -104,8 +104,8 @@ async fn dispatch( return Err(ProxyError::ModelForbidden(model_name.to_string())); } - // Budget + rate-limit gate (issue #107). - let _reservation = crate::quota::enforce(state, auth).await?; + let model_rl = crate::quota::ModelRateLimit::from_model(model_name, &model_entry.value); + let _reservation = crate::quota::enforce(state, auth, model_rl).await?; let model = &model_entry.value; diff --git a/crates/aisix-proxy/src/messages.rs b/crates/aisix-proxy/src/messages.rs index 24e07046..7d283283 100644 --- a/crates/aisix-proxy/src/messages.rs +++ b/crates/aisix-proxy/src/messages.rs @@ -159,14 +159,8 @@ async fn dispatch( return Err(ProxyError::ModelForbidden(model_name.clone())); } - // Budget + rate-limit gate (issue #107). Pre-fix this endpoint - // bypassed both — Anthropic-API customers ran unmetered. Held - // until end of dispatch via Drop, which releases the concurrency - // permit. RPM is recorded immediately on pre_commit; TPM stays - // 0 for now (the streaming + cross-provider paths emit token - // counts in different shapes — plumbing them uniformly is a - // follow-up). - let _reservation = crate::quota::enforce(state, auth).await?; + let model_rl = crate::quota::ModelRateLimit::from_model(&model_name, &model_entry.value); + let _reservation = crate::quota::enforce(state, auth, model_rl).await?; let model = &model_entry.value; let pk_entry = crate::dispatch::resolve_provider_key(&snapshot, model)?; diff --git a/crates/aisix-proxy/src/passthrough.rs b/crates/aisix-proxy/src/passthrough.rs index e139483a..4dbcccaf 100644 --- a/crates/aisix-proxy/src/passthrough.rs +++ b/crates/aisix-proxy/src/passthrough.rs @@ -162,13 +162,7 @@ async fn dispatch( req: Request, request_id: &str, ) -> Result<(Response, String), ProxyError> { - // Budget + rate-limit gate (issue #107). The previous _auth - // binding ignored the AuthenticatedKey entirely — passthrough - // ran completely unmetered, with no per-key budget cap and no - // RPM/TPM limit. This was the most exploitable gap because the - // /passthrough/* family covers everything OpenAI ships *plus* - // every provider's own API. Held for the dispatch lifetime. - let _reservation = crate::quota::enforce(&state, auth).await?; + let _reservation = crate::quota::enforce(&state, auth, None).await?; let snapshot = state.snapshot.load(); // Find a model for this provider so we can borrow its provider_key. diff --git a/crates/aisix-proxy/src/quota.rs b/crates/aisix-proxy/src/quota.rs index 6858c277..e49c933b 100644 --- a/crates/aisix-proxy/src/quota.rs +++ b/crates/aisix-proxy/src/quota.rs @@ -1,50 +1,110 @@ //! Pre-dispatch quota gate shared by every LLM endpoint. //! -//! Before this gate landed, only `/v1/chat/completions` ran budget + -//! rate-limit checks (`chat::dispatch`). Every other LLM endpoint — -//! `/v1/embeddings`, `/v1/messages`, `/v1/audio/*`, -//! `/v1/images/generations`, `/v1/responses`, `/v1/rerank`, -//! `/v1/completions`, the `/passthrough/...` family — went straight -//! from auth into the upstream Bridge, silently bypassing both. A -//! customer running the gateway as their org's LLM proxy expected -//! RPM/TPM caps and budget cutoffs to apply uniformly across -//! endpoints; the gap was visible to anyone using `/v1/messages` -//! (Anthropic API-shape) or `/v1/embeddings`. See issue #107. +//! Applies budget + multi-layer rate limiting: +//! 1. Budget pre-check (cp-api cached decision) +//! 2. API-key rate limit (`auth.entry.id`) +//! 3. Model rate limit (`model:`) — when the resolved Model has one +//! 4. Team rate limit (`team:`) — when the ApiKey carries team info +//! 5. Member rate limit (`member:`) — when the ApiKey carries owner info //! -//! This module hosts the minimum check every non-chat handler now -//! performs: a budget pre-check via cp-api, then a rate-limit -//! reservation. Guardrails are *not* applied here — they need -//! per-handler text extraction (chat reads messages, embeddings reads -//! `input` strings, audio reads transcripts) and that wiring is a -//! larger, separate change. The chat handler still has its own -//! guardrail path; this gate runs in parallel for every other -//! endpoint. -//! -//! Returning a [`Reservation`] (not just a permit) lets the caller -//! commit token usage post-dispatch. Non-chat handlers that don't -//! track upstream tokens uniformly call -//! [`aisix_ratelimit::Reservation::commit_tokens`] with `0`, which -//! still releases the concurrency permit and counts the request -//! against RPM but skips TPM. Future work can plumb per-endpoint -//! token totals through. +//! All layers use AND logic — every layer must pass or the request gets +//! 429. The returned [`MultiReservation`] commits token usage to all +//! layers and releases all concurrency permits on drop. -use aisix_ratelimit::Reservation; +use aisix_core::RateLimit; +use aisix_ratelimit::MultiReservation; use crate::auth::AuthenticatedKey; use crate::error::ProxyError; use crate::state::ProxyState; -/// Apply budget + rate-limit checks for one request. Call this before -/// touching the Bridge in every LLM endpoint handler. The returned -/// [`Reservation`] is alive until the caller commits or drops it; on -/// commit, RPM is finalised and TPM accounted for the supplied total. +/// Optional model rate-limit info resolved by the caller before enforce. +pub(crate) struct ModelRateLimit { + pub name: String, + pub limits: RateLimit, +} + +impl ModelRateLimit { + /// Build from a resolved model entry. Returns `None` when the model + /// has no rate limit configured or has an unrestricted one (all fields + /// are `None`). + pub fn from_model(model_name: &str, model: &aisix_core::Model) -> Option { + model + .rate_limit + .as_ref() + .filter(|rl| !rl.is_unrestricted()) + .map(|rl| Self { + name: model_name.to_owned(), + limits: rl.clone(), + }) + } +} + +/// Reserve across all applicable rate-limit layers (api_key, model, team, member). +fn reserve_layers<'a>( + state: &'a ProxyState, + auth: &AuthenticatedKey, + model_rl: Option, +) -> Result, ProxyError> { + let mut reservations = Vec::with_capacity(4); + + // Layer 1: API key rate limit. + let key_limits = auth.key().rate_limit.clone().unwrap_or_default(); + if !key_limits.is_unrestricted() { + let r = state + .limiter + .pre_commit(&auth.entry.id, &key_limits) + .map_err(ProxyError::from)?; + reservations.push(r); + } + + // Layer 2: Model rate limit. + if let Some(mrl) = model_rl { + if !mrl.limits.is_unrestricted() { + let key = format!("model:{}", mrl.name); + let r = state + .limiter + .pre_commit(&key, &mrl.limits) + .map_err(ProxyError::from)?; + reservations.push(r); + } + } + + // Layer 3: Team rate limit. + if let (Some(tid), Some(trl)) = (&auth.key().team_id, &auth.key().team_rate_limit) { + if !tid.is_empty() && !trl.is_unrestricted() { + let key = format!("team:{tid}"); + let r = state + .limiter + .pre_commit(&key, trl) + .map_err(ProxyError::from)?; + reservations.push(r); + } + } + + // Layer 4: Member/owner rate limit. + if let (Some(oid), Some(orl)) = (&auth.key().owner_id, &auth.key().owner_rate_limit) { + if !oid.is_empty() && !orl.is_unrestricted() { + let key = format!("member:{oid}"); + let r = state + .limiter + .pre_commit(&key, orl) + .map_err(ProxyError::from)?; + reservations.push(r); + } + } + + Ok(MultiReservation::new(reservations)) +} + +/// Apply budget + multi-layer rate-limit checks for one request. +/// `model_rl` is the resolved Model's rate_limit (if any). Pass `None` +/// for endpoints that don't resolve a model (e.g. passthrough). pub(crate) async fn enforce<'a>( state: &'a ProxyState, auth: &AuthenticatedKey, -) -> Result, ProxyError> { - // Budget pre-check via cp-api. Mirrors chat::dispatch — the DP no - // longer owns budget state; cp-api returns a cached/live decision - // per api_key. + model_rl: Option, +) -> Result, ProxyError> { let decision = state.budgets.check(&auth.entry.id).await; if !decision.allowed { return Err(ProxyError::BudgetExceeded( @@ -52,14 +112,15 @@ pub(crate) async fn enforce<'a>( )); } - // Rate-limit reservation. The reservation holds a concurrency - // permit until it's committed (or dropped). Commit at the end of - // dispatch with whatever token count the upstream returned (0 if - // the handler doesn't track tokens — RPM still counts). - let rl_key = auth.entry.id.clone(); - let rl_limits = auth.key().rate_limit.clone().unwrap_or_default(); - state - .limiter - .pre_commit(&rl_key, &rl_limits) - .map_err(ProxyError::from) + reserve_layers(state, auth, model_rl) +} + +/// Rate-limit-only enforcement (no budget check). Used by `chat.rs` +/// which handles budget separately. +pub(crate) fn enforce_rate_limit<'a>( + state: &'a ProxyState, + auth: &AuthenticatedKey, + model_rl: Option, +) -> Result, ProxyError> { + reserve_layers(state, auth, model_rl) } diff --git a/crates/aisix-proxy/src/rerank.rs b/crates/aisix-proxy/src/rerank.rs index 002e2100..8e04420e 100644 --- a/crates/aisix-proxy/src/rerank.rs +++ b/crates/aisix-proxy/src/rerank.rs @@ -104,8 +104,8 @@ async fn dispatch( return Err(ProxyError::ModelForbidden(model_name.clone())); } - // Budget + rate-limit gate (issue #107). - let _reservation = crate::quota::enforce(state, auth).await?; + let model_rl = crate::quota::ModelRateLimit::from_model(&model_name, &model_entry.value); + let _reservation = crate::quota::enforce(state, auth, model_rl).await?; let model = &model_entry.value; diff --git a/crates/aisix-proxy/src/responses.rs b/crates/aisix-proxy/src/responses.rs index 6accf3b0..035637d5 100644 --- a/crates/aisix-proxy/src/responses.rs +++ b/crates/aisix-proxy/src/responses.rs @@ -108,9 +108,8 @@ async fn dispatch( return Err(ProxyError::ModelForbidden(model_name.clone())); } - // Budget + rate-limit gate (issue #107). RPM counts on - // pre_commit; concurrency releases on Drop at function end. - let _reservation = crate::quota::enforce(state, auth).await?; + let model_rl = crate::quota::ModelRateLimit::from_model(&model_name, &model_entry.value); + let _reservation = crate::quota::enforce(state, auth, model_rl).await?; let model = &model_entry.value; diff --git a/crates/aisix-ratelimit/src/lib.rs b/crates/aisix-ratelimit/src/lib.rs index df8f1f51..a37f20cb 100644 --- a/crates/aisix-ratelimit/src/lib.rs +++ b/crates/aisix-ratelimit/src/lib.rs @@ -20,5 +20,5 @@ mod window; pub use clock::{Clock, SystemClock, TestClock}; pub use error::RateLimitError; -pub use limiter::{Limiter, RateLimitStatus, Reservation}; +pub use limiter::{Limiter, MultiReservation, RateLimitStatus, Reservation}; pub use window::{FixedWindowCounter, WindowCheck}; diff --git a/crates/aisix-ratelimit/src/limiter.rs b/crates/aisix-ratelimit/src/limiter.rs index f7a35be6..21e231ab 100644 --- a/crates/aisix-ratelimit/src/limiter.rs +++ b/crates/aisix-ratelimit/src/limiter.rs @@ -286,6 +286,39 @@ impl<'a, C: Clock> Drop for Reservation<'a, C> { } } +/// Wraps multiple [`Reservation`]s across rate-limit layers (api_key, +/// model, team, member). Commits all with the same token count; +/// dropping releases all concurrency permits. +pub struct MultiReservation<'a, C: Clock> { + reservations: Vec>, +} + +impl<'a, C: Clock> MultiReservation<'a, C> { + pub fn new(reservations: Vec>) -> Self { + Self { reservations } + } + + /// Commit the actual token cost to every layer. + pub fn commit_tokens(self, tokens: u64) { + for r in self.reservations { + r.commit_tokens(tokens); + } + } + + /// Return owned keys for post-stream token accounting. + pub fn keys(&self) -> Vec { + self.reservations.iter().map(|r| r.key.clone()).collect() + } +} + +impl<'a, C: Clock> std::fmt::Debug for MultiReservation<'a, C> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MultiReservation") + .field("layers", &self.reservations.len()) + .finish() + } +} + #[cfg(test)] mod tests { use super::*; @@ -615,4 +648,88 @@ mod tests { "TPM cap should block the next request after streaming over-shoot; got {err:?}", ); } + + // --- MultiReservation tests ---------------------------------------- + + #[test] + fn multi_reservation_commit_tokens_updates_all_layers() { + let clock = TestClock::new(100); + let limiter = Limiter::with_clock(clock.clone()); + let l = limits(None, Some(1000), None); + + let r1 = limiter.pre_commit("api_key:k1", &l).unwrap(); + let r2 = limiter.pre_commit("model:gpt-4o", &l).unwrap(); + let multi = MultiReservation::new(vec![r1, r2]); + + multi.commit_tokens(500); + + let s1 = limiter.peek("api_key:k1", &l).unwrap(); + let s2 = limiter.peek("model:gpt-4o", &l).unwrap(); + assert_eq!(s1.tpm_used, 500); + assert_eq!(s2.tpm_used, 500); + } + + #[test] + fn multi_reservation_drop_releases_all_concurrency() { + let clock = TestClock::new(100); + let limiter = Limiter::with_clock(clock.clone()); + let l = limits(None, None, Some(1)); + + let r1 = limiter.pre_commit("k1", &l).unwrap(); + let r2 = limiter.pre_commit("k2", &l).unwrap(); + let multi = MultiReservation::new(vec![r1, r2]); + + assert!(limiter.pre_commit("k1", &l).is_err()); + assert!(limiter.pre_commit("k2", &l).is_err()); + + drop(multi); + + assert!(limiter.pre_commit("k1", &l).is_ok()); + assert!(limiter.pre_commit("k2", &l).is_ok()); + } + + #[test] + fn multi_reservation_keys_returns_all_held_keys() { + let clock = TestClock::new(100); + let limiter = Limiter::with_clock(clock.clone()); + let l = limits(Some(10), None, None); + + let r1 = limiter.pre_commit("api_key:k1", &l).unwrap(); + let r2 = limiter.pre_commit("model:m1", &l).unwrap(); + let r3 = limiter.pre_commit("team:t1", &l).unwrap(); + let multi = MultiReservation::new(vec![r1, r2, r3]); + + let keys = multi.keys(); + assert_eq!(keys, vec!["api_key:k1", "model:m1", "team:t1"]); + } + + #[test] + fn multi_reservation_partial_failure_releases_acquired_layers() { + let clock = TestClock::new(100); + let limiter = Limiter::with_clock(clock.clone()); + let l_key = limits(None, None, Some(1)); + let l_team = limits(None, None, Some(1)); + let l_model = limits(Some(1), None, None); + + // Exhaust model RPM so the third layer will fail. + let _exhaust = limiter.pre_commit("model:m1", &l_model).unwrap(); + + // Simulate multi-layer acquisition: key + team succeed, model fails. + let r_key = limiter.pre_commit("k1", &l_key).unwrap(); + let r_team = limiter.pre_commit("team:t1", &l_team).unwrap(); + let acquired = vec![r_key, r_team]; + + // Both concurrency slots are now taken. + assert!(limiter.pre_commit("k1", &l_key).is_err()); + assert!(limiter.pre_commit("team:t1", &l_team).is_err()); + + // Model layer fails — drop acquired reservations (simulates error + // path where partially-built MultiReservation is dropped). + assert!(limiter.pre_commit("model:m1", &l_model).is_err()); + drop(MultiReservation::new(acquired)); + + // Both earlier layers' concurrency is released. + assert!(limiter.pre_commit("k1", &l_key).is_ok()); + assert!(limiter.pre_commit("team:t1", &l_team).is_ok()); + } }