Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions tests/e2e/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

from __future__ import annotations

from datetime import datetime
from typing import Literal

from pydantic import BaseModel, ConfigDict, RootModel, model_validator
Expand All @@ -23,6 +24,10 @@ class BudgetWindow(BaseModel):
max_budget: float


class BudgetWindowState(BudgetWindow):
reset_at: datetime | None = None


class KeyLoggingCallbackVars(BaseModel):
langfuse_public_key: str | None = None
langfuse_secret_key: str | None = None
Expand Down Expand Up @@ -94,6 +99,7 @@ class KeyInfo(BaseModel):
budget_reset_at: str | None = None
budget_id: str | None = None
litellm_budget_table: LiteLLMBudgetTable | None = None
budget_limits: list[BudgetWindowState] | None = None


class KeyInfoResponse(BaseModel):
Expand Down
50 changes: 36 additions & 14 deletions tests/e2e/quota_management/budgets/budget_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,16 @@

import time
from dataclasses import dataclass
from datetime import datetime

from pydantic import AliasPath, BaseModel, Field, RootModel

from proxy_client import ProxyClient
from e2e_http import NoBody, Result, StreamingResponse, Success, unwrap
from proxy_client import ProxyClient
from models import (
AnthropicMessagesBody,
BudgetWindow,
BudgetWindowState,
ChatBody,
ChatMessage,
ChatMetadata,
Expand Down Expand Up @@ -130,8 +132,13 @@ class TeamInfoParams(BaseModel):
team_id: str


class TeamInfoRow(BaseModel):
budget_limits: list[BudgetWindowState] | None = None


class TeamInfoResponse(BaseModel):
team_memberships: list[TeamMembershipRow] = []
team_info: TeamInfoRow | None = None


class TagNewBody(BaseModel):
Expand Down Expand Up @@ -173,6 +180,10 @@ class BudgetInfoResponse(RootModel[list[BudgetRow]]):
pass


def window_reset_at(windows: list[BudgetWindowState], budget_duration: str) -> datetime | None:
return next((w.reset_at for w in windows if w.budget_duration == budget_duration), None)


def is_budget_block(result: StreamingResponse) -> bool:
"""True if the call was rejected for being over budget (vs a provider error)."""
return not result.ok and "budget_exceeded" in result.body
Expand Down Expand Up @@ -221,6 +232,20 @@ def generate_key(
def delete_key(self, key: str) -> None:
self.proxy.delete_key(key)

def key_budget_windows(self, key: str) -> list[BudgetWindowState]:
"""A key's budget_limits windows as /key/info stores them. Each window's
reset_at is advanced by the reset job in the same pass that zeroes the
window's spend counter, so a strictly-later value proves the wipe ran."""
return self.proxy.key_info(key).budget_limits or []

def team_budget_windows(self, team_id: str) -> list[BudgetWindowState]:
"""Team analog of key_budget_windows, read from /team/info."""
match self._team_info(team_id):
case Success(data=data) if data.team_info is not None:
return data.team_info.budget_limits or []
case _:
return []

def delete_customers(self, user_ids: list[str]) -> None:
self.proxy.delete_customers(user_ids)

Expand Down Expand Up @@ -386,15 +411,18 @@ def delete_team(self, team_id: str) -> None:
response_type=NoBody,
)

def _team_info(self, team_id: str) -> Result[TeamInfoResponse]:
return self.proxy.transport.get(
"/team/info",
headers=self.proxy.transport.master,
params=TeamInfoParams(team_id=team_id),
response_type=TeamInfoResponse,
)

def _wait_for_team(self, team_id: str) -> None:
last: Result[TeamInfoResponse] | None = None
for _ in range(_TEAM_READY_ATTEMPTS):
last = self.proxy.transport.get(
"/team/info",
headers=self.proxy.transport.master,
params=TeamInfoParams(team_id=team_id),
response_type=TeamInfoResponse,
)
last = self._team_info(team_id)
match last:
case Success():
return
Expand Down Expand Up @@ -448,13 +476,7 @@ def member_budget_reset_at(self, team_id: str, user_id: str) -> str | None:
"""The member's per-team budget_reset_at as /team/info reports it, or None if
no reset is scheduled. The reset job advances this each time the window
elapses; a job that skips the row leaves it pinned forever."""
result = self.proxy.transport.get(
"/team/info",
headers=self.proxy.transport.master,
params=TeamInfoParams(team_id=team_id),
response_type=TeamInfoResponse,
)
match result:
match self._team_info(team_id):
case Success(data=data):
return next(
(row.budget_reset_at for row in data.team_memberships if row.user_id == user_id),
Expand Down
110 changes: 87 additions & 23 deletions tests/e2e/quota_management/budgets/test_multi_window_budget_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,22 @@
PROXY_BUDGET_RESCHEDULER_* in docker-compose) - the window resets and calls flow
again. Closes the multi-window gap (enforcement + per-window reset) in
BUDGET_TEST_COVERAGE_MATRIX.md, which the unit suite covered but no live test did.

The second test is the long-window direction: one burn crosses
both caps; after the 30s window's reset_at strictly advances (read post-block since
a mint-time read races the boundary; the reset job zeroes the counter in the same
pass), the key must still be refused with "over 1d budget". That check polls because
enforcement's cached auth view lags the DB write; any 200 or non-budget error fails
immediately.
"""

import time

import pytest

from budget_client import BudgetClient, is_budget_block
from budget_client import BudgetClient, is_budget_block, window_reset_at
from e2e_http import StreamingResponse, require_successful_call
from e2e_config import CHEAP_OPENAI_MODEL, unique_marker
from e2e_http import require_successful_call
from lifecycle import ResourceManager
from models import BudgetWindow

Expand All @@ -28,38 +35,41 @@
# max_tokens must be >1: gpt-5.5 refuses completions that hit the output limit
# mid-message when capped at 1 token.
MODEL = CHEAP_OPENAI_MODEL
SHORT_WINDOW = f"{WINDOW_SECONDS}s"
LONG_WINDOW = "1d"
TINY_CAP = 1e-9
LONG_CAP = 5e-7
RESET_DEADLINE_SECONDS = 150


def _call(client: BudgetClient, key: str):
return client.chat(
key, MODEL, f"window {unique_marker()}", max_tokens=16
)
return client.chat(key, MODEL, f"window {unique_marker()}", max_tokens=16)


def _drive_to_block(client: BudgetClient, key: str) -> StreamingResponse:
for _ in range(20):
result = _call(client, key)
if is_budget_block(result):
return result
require_successful_call(result)
time.sleep(2)
pytest.fail("budget never enforced before block")


@pytest.mark.covers("quota_management.budget.key_multi_window.blocks_then_resets")
def test_short_window_blocks_then_resets(
client: BudgetClient, resources: ResourceManager
) -> None:
def test_short_window_blocks_then_resets(client: BudgetClient, resources: ResourceManager) -> None:
key = client.generate_key(
models=[MODEL],
budget_limits=[
BudgetWindow(budget_duration=f"{WINDOW_SECONDS}s", max_budget=1e-9),
BudgetWindow(budget_duration=SHORT_WINDOW, max_budget=TINY_CAP),
BudgetWindow(budget_duration="1m", max_budget=1.0), # roomy: never blocks
],
)
resources.defer(lambda: client.delete_key(key))

# 1. exhaust the tight window -> litellm returns budget_exceeded
start = time.monotonic()
blocked = False
for _ in range(20):
result = _call(client, key)
if is_budget_block(result):
blocked = True
break
require_successful_call(result)
time.sleep(2)
assert blocked, f"{WINDOW_SECONDS}s window never enforced"
_drive_to_block(client, key)

# 2. the window resets at the next wall-clock-aligned boundary (up to a window
# after start), then the reset job (~15-20s rescheduler) zeroes the spend.
Expand All @@ -71,12 +81,66 @@ def test_short_window_blocks_then_resets(
result = _call(client, key)
if result.ok:
elapsed = time.monotonic() - start
assert elapsed < WINDOW_SECONDS + 90, (
f"reset took {elapsed:.0f}s - too long for a {WINDOW_SECONDS}s window"
)
assert elapsed < WINDOW_SECONDS + 90, f"reset took {elapsed:.0f}s - too long for a {WINDOW_SECONDS}s window"
return
assert is_budget_block(result), (
f"non-budget error during reset wait: status={result.status_code} "
f"body={result.body[:200]}"
f"non-budget error during reset wait: status={result.status_code} body={result.body[:200]}"
)
pytest.fail(f"{WINDOW_SECONDS}s window never reset within 150s")


@pytest.mark.covers("quota_management.budget.key_multi_window.blocks_then_resets")
def test_long_window_blocks_after_short_window_resets(client: BudgetClient, resources: ResourceManager) -> None:
key = client.generate_key(
models=[MODEL],
budget_limits=[
BudgetWindow(budget_duration=SHORT_WINDOW, max_budget=TINY_CAP),
BudgetWindow(budget_duration=LONG_WINDOW, max_budget=LONG_CAP),
],
)
resources.defer(lambda: client.delete_key(key))

# 1. drive the key to get blocked by SHORT_WINDOW, assert it's budget error
blocked = _drive_to_block(client, key)
assert blocked.status_code == 429, f"budget block was not a 429: {blocked.status_code} {blocked.body[:200]}"

# 2. check the reset times of both budget windows after we drove to being blocked
blocked_reset_at = window_reset_at(client.key_budget_windows(key), SHORT_WINDOW)
assert blocked_reset_at is not None, "short window missing from /key/info budget_limits"
blocked_long_reset_at = window_reset_at(client.key_budget_windows(key), LONG_WINDOW)
assert blocked_long_reset_at is not None, "long window missing from /key/info budget_limits"

# 3. poll every 5s for the SHORT_WINDOW reset time until it is past it, fails if it doesnt reset
deadline = time.monotonic() + RESET_DEADLINE_SECONDS
while time.monotonic() < deadline:
time.sleep(5)
current = window_reset_at(client.key_budget_windows(key), SHORT_WINDOW)
if current is not None and current > blocked_reset_at:
break
else:
pytest.fail(
f"{SHORT_WINDOW} window's reset_at never advanced past {blocked_reset_at} within {RESET_DEADLINE_SECONDS}s"
)

# 4. short window just reset in 3, so now make a call, check that its blocked (should be blocked by LONG_WINDOW because short window reset), also make sure its budget error
deadline = time.monotonic() + RESET_DEADLINE_SECONDS
last_body = ""
while time.monotonic() < deadline:
result = _call(client, key)
if result.ok:
rolled = window_reset_at(client.key_budget_windows(key), LONG_WINDOW) != blocked_long_reset_at
pytest.fail(
f"{LONG_WINDOW} window failed to block after the {SHORT_WINDOW} window reset"
+ (f" (the {LONG_WINDOW} window itself rolled mid-test - boundary crossed; rerun)" if rolled else "")
)
assert is_budget_block(result), (
f"non-budget error while waiting for {LONG_WINDOW} attribution: "
f"status={result.status_code} body={result.body[:200]}"
)
if f"over {LONG_WINDOW} budget" in result.body:
return
last_body = result.body
time.sleep(5)
pytest.fail(
f"block never attributed to the {LONG_WINDOW} window within {RESET_DEADLINE_SECONDS}s: {last_body[:200]}"
)
Loading
Loading