From 0105287df4858185cdf4e56ee77088eee7dfd086 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Fri, 17 Jul 2026 11:32:48 -0700 Subject: [PATCH 1/2] test(e2e): cover per-user budget across keys and team-member budget isolation Gap-fill for the budgets e2e suite against two Quota Management behavior contracts that had incomplete coverage. A per-user max_budget must be enforced across all of that user's keys, not only the key that spent it down. The prior internal-user test drove a single key, so it never proved the budget is shared. The new test gives one internal user a tiny max_budget and two keys carrying no budget of their own, drives the first key to a budget_exceeded block, then shows the untouched second key is rejected the same way and the user's recorded spend has crossed the cap. Per-team-member budgets must be enforced independently between members. The prior team-member test used one member, so it never proved isolation. The new test puts two members on one team with a large team budget, caps the tight member at a tiny per-team budget and drives it past the block, then shows the roomy member still serves on the same team with its calls attributed to its own user id in the spend logs while the tight member stays blocked. Adds a /user/info read-back helper, two coverage-registry rows, and the two new assertion tokens to the naming grammar so the collector --strict passes. Resolves LIT-4548 --- tests/e2e/CLAUDE.md | 3 +- .../coverage_registry/quota_management.yaml | 2 + .../quota_management/budgets/budget_client.py | 26 ++++ .../test_team_member_budget_isolation_e2e.py | 119 ++++++++++++++++++ .../test_user_budget_across_keys_e2e.py | 61 +++++++++ 5 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/quota_management/budgets/test_team_member_budget_isolation_e2e.py create mode 100644 tests/e2e/quota_management/budgets/test_user_budget_across_keys_e2e.py diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 0e1eafb5196..3635e471faa 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -139,7 +139,8 @@ quota_management... | spend_calculate | pagination assertion : blocks_over_limit | resets_after_window | headers_report_remaining | picks_under_tpm | blocks_then_resets | resets_windows_independently | alerts_without_blocking - | isolates_per_model | routes_to_fallback | reseed_matches_db | logs_cost | zero_cost + | isolates_per_model | isolates_per_member | enforced_across_keys | routes_to_fallback + | reseed_matches_db | logs_cost | zero_cost | matches_sum_of_logs | loses_no_spend | attributes_spend | writes_own_rows | writes_failure_row | returns_cost | keeps_total e.g. quota_management.ratelimit.rpm.blocks_over_limit exercised_on=[chat_completions, messages] diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index fac266149f4..1e91a2c1c47 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -8,9 +8,11 @@ - {id: quota_management.ratelimit.priority_strict.picks_under_tpm, module: quota_management, tier: P1, behavior: ratelimit, variant: priority_strict, assertions: [picks_under_tpm], exercised_on: [chat_completions, messages], source: "dynamic_rate_limiter_v3.py:53-71", rationale: "Strict mode (>=80% sat) enforces priority fairness"} - {id: quota_management.budget.key.blocks_over_limit, module: quota_management, tier: P0, behavior: budget, variant: key, assertions: [blocks_over_limit], exercised_on: [chat_completions], source: "proxy/auth/auth_checks.py", rationale: "A key's max_budget blocks further paid calls once spend crosses it"} - {id: quota_management.budget.internal_user.blocks_over_limit, module: quota_management, tier: P1, behavior: budget, variant: internal_user, assertions: [blocks_over_limit], exercised_on: [chat_completions], source: "proxy/auth/auth_checks.py", rationale: "An internal user's max_budget governs personal keys"} +- {id: quota_management.budget.internal_user.enforced_across_keys, module: quota_management, tier: P1, behavior: budget, variant: internal_user, assertions: [enforced_across_keys], exercised_on: [chat_completions], source: "proxy/auth/auth_checks.py", rationale: "An internal user's max_budget governs every personal key it owns; a second untouched key is blocked once the shared user budget is exhausted"} - {id: quota_management.budget.end_user.blocks_over_limit, module: quota_management, tier: P1, behavior: budget, variant: end_user, assertions: [blocks_over_limit], exercised_on: [chat_completions], source: "proxy/auth/auth_checks.py", rationale: "A customer (end-user) max_budget blocks calls attributed via user="} - {id: quota_management.budget.organization.blocks_over_limit, module: quota_management, tier: P1, behavior: budget, variant: organization, assertions: [blocks_over_limit], exercised_on: [chat_completions], source: "proxy/auth/auth_checks.py", rationale: "An organization's max_budget blocks keys under its teams"} - {id: quota_management.budget.team_member.blocks_over_limit, module: quota_management, tier: P1, behavior: budget, variant: team_member, assertions: [blocks_over_limit], exercised_on: [chat_completions], source: "proxy/auth/auth_checks.py", rationale: "A member's per-team budget blocks independently of the team budget"} +- {id: quota_management.budget.team_member.isolates_per_member, module: quota_management, tier: P1, behavior: budget, variant: team_member, assertions: [isolates_per_member], exercised_on: [chat_completions], source: "proxy/auth/auth_checks.py", rationale: "One team member's exhausted per-team budget does not block a different member on the same team"} - {id: quota_management.budget.tag.blocks_over_limit, module: quota_management, tier: P1, behavior: budget, variant: tag, assertions: [blocks_over_limit], exercised_on: [chat_completions], source: "router_strategy/budget_limiter.py", rationale: "Proxy-level tag budgets block tagged requests at the cap"} - {id: quota_management.budget.model_max.isolates_per_model, module: quota_management, tier: P1, behavior: budget, variant: model_max, assertions: [isolates_per_model], exercised_on: [chat_completions], source: "proxy/hooks/model_max_budget_limiter.py", rationale: "model_max_budget caps one model without touching a sibling's budget"} - {id: quota_management.budget.soft.alerts_without_blocking, module: quota_management, tier: P1, behavior: budget, variant: soft, assertions: [alerts_without_blocking], exercised_on: [chat_completions], source: "proxy/auth/auth_checks.py", rationale: "soft_budget alerts but never blocks traffic"} diff --git a/tests/e2e/quota_management/budgets/budget_client.py b/tests/e2e/quota_management/budgets/budget_client.py index 7b37c3af98e..01e4d63c1c3 100644 --- a/tests/e2e/quota_management/budgets/budget_client.py +++ b/tests/e2e/quota_management/budgets/budget_client.py @@ -39,6 +39,19 @@ class UserNewResponse(BaseModel): user_id: str +class UserInfoParams(BaseModel): + user_id: str + + +class UserInfoRow(BaseModel): + spend: float | None = None + max_budget: float | None = None + + +class UserInfoResponse(BaseModel): + user_info: UserInfoRow | None = None + + class UserDeleteBody(BaseModel): user_ids: list[str] @@ -262,6 +275,19 @@ def delete_user(self, user_id: str) -> None: response_type=NoBody, ) + def user_info(self, user_id: str) -> UserInfoRow | None: + result = self.gateway.transport.get( + "/user/info", + headers=self.gateway.transport.master, + params=UserInfoParams(user_id=user_id), + response_type=UserInfoResponse, + ) + match result: + case Success(data=data): + return data.user_info + case _: + return None + # ---- customer / end-user ------------------------------------------- def create_customer(self, customer_id: str, *, max_budget: float) -> str: diff --git a/tests/e2e/quota_management/budgets/test_team_member_budget_isolation_e2e.py b/tests/e2e/quota_management/budgets/test_team_member_budget_isolation_e2e.py new file mode 100644 index 00000000000..87855a9a1c1 --- /dev/null +++ b/tests/e2e/quota_management/budgets/test_team_member_budget_isolation_e2e.py @@ -0,0 +1,119 @@ +"""Live e2e: per-team-member budgets are enforced independently between members. + +Two members share one team that has a large team budget. The tight member is capped +at a tiny per-team budget and spends past it; the roomy member has plenty of room. +Once the tight member is blocked with budget_exceeded, the roomy member still serves +on the same team, its calls land in the spend logs under its own user id, and the +tight member stays blocked. A shared or leaky member counter would either block the +roomy member too or let the tight member back through once its peer spent. +""" + +import time +from collections.abc import Iterator +from dataclasses import dataclass + +import pytest + +from budget_client import BudgetClient, is_budget_block +from e2e_config import unique_marker +from e2e_http import Success, require_successful_call +from lifecycle import ResourceManager +from models import ChatBody, ChatMessage + +pytestmark = pytest.mark.e2e + +MODEL = "gpt-5.5" +TEAM_BUDGET = 100.0 +TIGHT_MEMBER_BUDGET = 3e-6 +ROOMY_MEMBER_BUDGET = 100.0 +ROOMY_BURST = 3 + + +@dataclass(frozen=True, slots=True) +class _Pair: + team_id: str + tight_user_id: str + roomy_user_id: str + tight_key: str + roomy_key: str + + +@pytest.fixture(scope="class") +def pair(client: BudgetClient) -> Iterator[_Pair]: + """One team with a large budget and two members on it: a tight member capped at + a tiny per-team budget and a roomy member with headroom, each with their own key. + Shared across the class and torn down LIFO best-effort when it finishes.""" + resources = ResourceManager(client=client.gateway) + try: + marker = unique_marker() + team_id = client.create_team(alias=f"e2e-member-iso-{marker}", max_budget=TEAM_BUDGET) + resources.defer(lambda: client.delete_team(team_id)) + tight_user = client.create_user(max_budget=TEAM_BUDGET) + resources.defer(lambda: client.delete_user(tight_user)) + roomy_user = client.create_user(max_budget=TEAM_BUDGET) + resources.defer(lambda: client.delete_user(roomy_user)) + client.add_team_member(team_id, tight_user, max_budget_in_team=TIGHT_MEMBER_BUDGET) + client.add_team_member(team_id, roomy_user, max_budget_in_team=ROOMY_MEMBER_BUDGET) + tight_key = client.generate_key(team_id=team_id, user_id=tight_user) + resources.defer(lambda: client.delete_key(tight_key)) + roomy_key = client.generate_key(team_id=team_id, user_id=roomy_user) + resources.defer(lambda: client.delete_key(roomy_key)) + yield _Pair( + team_id=team_id, + tight_user_id=tight_user, + roomy_user_id=roomy_user, + tight_key=tight_key, + roomy_key=roomy_key, + ) + finally: + resources.teardown() + + +def _roomy_send(client: BudgetClient, key: str) -> str: + """One roomy-member call that must go through; returns its request id.""" + match client.gateway.chat( + key, + ChatBody( + model=MODEL, + messages=[ChatMessage(role="user", content=f"roomy {unique_marker()}")], + max_tokens=16, + ), + ): + case Success(data=response): + assert response.id is not None, "roomy member call returned no id" + return response.id + case other: + pytest.fail(f"roomy member call failed while a peer was over budget: {other}") + + +class TestTeamMemberBudgetIsolation: + @pytest.mark.covers("quota_management.budget.team_member.isolates_per_member") + def test_blocked_member_does_not_block_peer(self, client: BudgetClient, pair: _Pair) -> None: + blocked = False + for _ in range(40): + result = client.chat(pair.tight_key, MODEL, f"tight {unique_marker()}", max_tokens=16) + if is_budget_block(result): + blocked = True + break + require_successful_call(result) + time.sleep(2) + assert blocked, "tight member's per-team budget never enforced" + + sent = frozenset(_roomy_send(client, pair.roomy_key) for _ in range(ROOMY_BURST)) + + assert is_budget_block( + client.chat(pair.tight_key, MODEL, f"tight {unique_marker()}", max_tokens=16) + ), "tight member stopped being blocked once the peer spent" + + rows = client.gateway.poll_logs_for_key( + pair.roomy_key, predicate=lambda rs: bool(sent & {r.request_id for r in rs}) + ) + logged = [row for row in rows if row.request_id in sent] + assert logged, "none of the roomy member's calls reached the spend logs" + for row in logged: + assert row.user == pair.roomy_user_id, ( + f"roomy call {row.request_id} logged under user {row.user}, not {pair.roomy_user_id}" + ) + assert row.team_id == pair.team_id, ( + f"roomy call {row.request_id} logged under team {row.team_id}, not {pair.team_id}" + ) diff --git a/tests/e2e/quota_management/budgets/test_user_budget_across_keys_e2e.py b/tests/e2e/quota_management/budgets/test_user_budget_across_keys_e2e.py new file mode 100644 index 00000000000..984a100756c --- /dev/null +++ b/tests/e2e/quota_management/budgets/test_user_budget_across_keys_e2e.py @@ -0,0 +1,61 @@ +"""Live e2e: a per-user max_budget is enforced across ALL of that user's keys. + +An internal user's budget governs every personal key it owns, not only the one +that happened to spend it down. One user with a tiny max_budget owns two keys: +driving the first key to a budget_exceeded block then makes a fresh, untouched +second key of the same user (which carries no budget of its own, so nothing but the +shared user budget can block it) reject the same way, and the user's recorded spend +has crossed the cap. A key-scoped-only budget would leave the second key serving. +""" + +import time + +import pytest + +from budget_client import BudgetClient, is_budget_block +from e2e_config import unique_marker +from e2e_http import StreamingResponse, require_successful_call +from lifecycle import ResourceManager + +pytestmark = pytest.mark.e2e + +MODEL = "gpt-5.5" +TINY_CAP = 3e-6 +RECORDED_SPEND_DEADLINE_SECONDS = 90 + + +def _call(client: BudgetClient, key: str) -> StreamingResponse: + return client.chat(key, MODEL, f"across {unique_marker()}", max_tokens=16) + + +def _drive_to_block(client: BudgetClient, key: str, subject: str) -> None: + for _ in range(40): + result = _call(client, key) + if is_budget_block(result): + return + require_successful_call(result) + time.sleep(2) + pytest.fail(f"user budget never enforced on {subject} within the call budget") + + +class TestUserBudgetAcrossKeys: + @pytest.mark.covers("quota_management.budget.internal_user.enforced_across_keys") + def test_user_budget_blocks_a_second_key(self, client: BudgetClient, resources: ResourceManager) -> None: + user_id = client.create_user(max_budget=TINY_CAP) + resources.defer(lambda: client.delete_user(user_id)) + + first_key = client.generate_key(user_id=user_id) + resources.defer(lambda: client.delete_key(first_key)) + second_key = client.generate_key(user_id=user_id) + resources.defer(lambda: client.delete_key(second_key)) + + _drive_to_block(client, first_key, "the first key") + _drive_to_block(client, second_key, "the second key") + + deadline = time.monotonic() + RECORDED_SPEND_DEADLINE_SECONDS + while time.monotonic() < deadline: + info = client.user_info(user_id) + if info is not None and (info.spend or 0.0) >= TINY_CAP: + return + time.sleep(5) + pytest.fail(f"user spend never reached the {TINY_CAP} cap in the recorded state") From c27515c5b7458763a2223ea51b7daf75e8573a26 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Fri, 17 Jul 2026 11:42:14 -0700 Subject: [PATCH 2/2] test(e2e): block the second key promptly to isolate the shared user budget Drive only the first key to exhaust the shared user budget, then require the second key (which has no budget of its own) to be rejected within a small bounded number of calls rather than driving it to accumulate its own spend. The block can then only come from the shared user budget, not the second key's own usage. --- .../test_user_budget_across_keys_e2e.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/e2e/quota_management/budgets/test_user_budget_across_keys_e2e.py b/tests/e2e/quota_management/budgets/test_user_budget_across_keys_e2e.py index 984a100756c..4dc7a2df647 100644 --- a/tests/e2e/quota_management/budgets/test_user_budget_across_keys_e2e.py +++ b/tests/e2e/quota_management/budgets/test_user_budget_across_keys_e2e.py @@ -22,6 +22,7 @@ MODEL = "gpt-5.5" TINY_CAP = 3e-6 RECORDED_SPEND_DEADLINE_SECONDS = 90 +SECOND_KEY_BLOCK_ATTEMPTS = 6 def _call(client: BudgetClient, key: str) -> StreamingResponse: @@ -38,6 +39,23 @@ def _drive_to_block(client: BudgetClient, key: str, subject: str) -> None: pytest.fail(f"user budget never enforced on {subject} within the call budget") +def _expect_prompt_block(client: BudgetClient, key: str, subject: str) -> None: + """The shared user budget is already exhausted before this key makes a single + call, so a key with no budget of its own must be rejected promptly. The small + bounded retry only absorbs spend-propagation lag between the two keys; it is far + below the spend a key-scoped budget would need to accumulate to block itself, so + a block here can only come from the shared user budget.""" + for _ in range(SECOND_KEY_BLOCK_ATTEMPTS): + result = _call(client, key) + if is_budget_block(result): + return + require_successful_call(result) + time.sleep(2) + pytest.fail( + f"{subject} was not blocked by the shared user budget within {SECOND_KEY_BLOCK_ATTEMPTS} calls" + ) + + class TestUserBudgetAcrossKeys: @pytest.mark.covers("quota_management.budget.internal_user.enforced_across_keys") def test_user_budget_blocks_a_second_key(self, client: BudgetClient, resources: ResourceManager) -> None: @@ -50,7 +68,7 @@ def test_user_budget_blocks_a_second_key(self, client: BudgetClient, resources: resources.defer(lambda: client.delete_key(second_key)) _drive_to_block(client, first_key, "the first key") - _drive_to_block(client, second_key, "the second key") + _expect_prompt_block(client, second_key, "the second key") deadline = time.monotonic() + RECORDED_SPEND_DEADLINE_SECONDS while time.monotonic() < deadline: