diff --git a/tests/e2e/coverage_registry/logging.yaml b/tests/e2e/coverage_registry/logging.yaml index 0f703632805..598e7715e58 100644 --- a/tests/e2e/coverage_registry/logging.yaml +++ b/tests/e2e/coverage_registry/logging.yaml @@ -6,6 +6,7 @@ - {id: logging.datadog.stream.exports_metric, module: logging, tier: P0, event: stream, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses], source: "integrations/datadog/datadog.py", rationale: "Streaming aggregates usage after the last chunk; delivery and cost must survive that path"} - {id: logging.datadog.failure.exports_metric, module: logging, tier: P0, event: failure, assertions: [exports_metric], exercised_on: [chat_completions], source: "integrations/datadog/datadog.py", rationale: "Failure metrics for alerting/SLO"} - {id: logging.prometheus.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, embeddings], source: "integrations/prometheus.py", rationale: "Standard OSS metrics; per-key cardinality (existing e2e)"} +- {id: logging.prometheus.failure.attributes_rate_limit_source, module: logging, tier: P1, event: failure, assertions: [attributes_rate_limit_source], exercised_on: [chat_completions], source: "PR #27687 / integrations/prometheus.py", fail_before_fix: proven, rationale: "A 429 must say whether the gateway or the vendor rejected it and name the provider; operators respond differently to each and cannot tell them apart otherwise"} - {id: logging.otel.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses, embeddings], source: "integrations/otel/logger.py", rationale: "OTEL spans on every call path"} - {id: logging.otel.stream.exports_metric, module: logging, tier: P0, event: stream, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses], source: "integrations/otel/logger.py", rationale: "Streaming closes the LLM span from the stream path; historically prone to duplicate/orphaned spans"} - {id: logging.otel.stream.records_ttft, module: logging, tier: P1, event: stream, assertions: [records_ttft], exercised_on: [chat_completions, messages, responses], source: "integrations/otel/mappers/genai.py", rationale: "TTFT is the streaming latency SLI; a zero or span-length value silently corrupts dashboards"} diff --git a/tests/e2e/coverage_registry/mgmt.yaml b/tests/e2e/coverage_registry/mgmt.yaml index 68c5ef6b31d..8233d051a3d 100644 --- a/tests/e2e/coverage_registry/mgmt.yaml +++ b/tests/e2e/coverage_registry/mgmt.yaml @@ -47,6 +47,7 @@ - {id: mgmt.tag.new.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "tag_management_endpoints.py:160", rationale: "Tag for spend categorization"} - {id: mgmt.tag.list.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "tag_management_endpoints.py:315", rationale: "Tag enumeration"} - {id: mgmt.tag.delete.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "tag_management_endpoints.py:390", rationale: "Stops future tagging"} +- {id: mgmt.model.info.reports_regional_uplift, module: mgmt, tier: P1, surface: api, assertions: [reports_regional_uplift], source: "LIT-3912 / model_prices_and_context_window.json", fail_before_fix: proven, rationale: "Regional bedrock variants must resolve their own uplifted cost-map rate, not the global one; resolving global silently under-bills every regional call"} - {id: mgmt.model.update.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "model_management_endpoints.py:1358", rationale: "Pricing/concurrency persist"} - {id: mgmt.model.delete.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "model_management_endpoints.py:1045", rationale: "Removes from registry"} - {id: mgmt.model.block.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "model_management_endpoints.py", rationale: "Blocked model stays blocked"} diff --git a/tests/e2e/llm_translation/test_regional_pricing_e2e.py b/tests/e2e/llm_translation/test_regional_pricing_e2e.py new file mode 100644 index 00000000000..c0b6ed272e6 --- /dev/null +++ b/tests/e2e/llm_translation/test_regional_pricing_e2e.py @@ -0,0 +1,104 @@ +"""Live e2e: regional model variants resolve their own uplifted cost-map pricing. + +Bedrock publishes a regional variant of each Claude model (`us.`, `eu.`) priced +above the `global.` variant, and the cost map carries all three. The mappings +regressed so regional deployments resolved the global rate (LIT-3912), which +silently under-bills every regional call and makes a customer's cost reporting +disagree with their AWS invoice. + +`/model/info` is where the resolved rate is observable without spending money on +a call, so this registers the three variants of one model and pins each rate. It +asserts the exact expected uplift rather than merely `regional > global`: a +mapping that resolved regional to some other model's rate would still satisfy an +inequality while being just as wrong. + +No provider call is made, so this needs no AWS credentials. +""" + +from __future__ import annotations + +import pytest + +from e2e_config import unique_marker +from endpoints_client import EndpointsClient +from lifecycle import ResourceManager +from models import LiteLLMParamsBody, ModelInfoEntry + +pytestmark = pytest.mark.e2e + +BASE_MODEL = "anthropic.claude-opus-4-7" +GLOBAL_INPUT_COST = 5e-06 +GLOBAL_OUTPUT_COST = 2.5e-05 +REGIONAL_UPLIFT = 1.1 +RELATIVE_TOLERANCE = 1e-9 + + +def _matches(actual: float, expected: float) -> bool: + """Float-safe equality for rates; the uplift arithmetic is not exact in binary.""" + return abs(actual - expected) <= abs(expected) * RELATIVE_TOLERANCE + + +def _register( + client: EndpointsClient, resources: ResourceManager, prefix: str +) -> str: + model = f"e2e-regional-{prefix}-{unique_marker()}" + model_id = client.create_model( + model, + LiteLLMParamsBody( + model=f"bedrock/{prefix}.{BASE_MODEL}", aws_region_name="us-east-1" + ), + ) + resources.defer(lambda: client.delete_model(model_id)) + return model + + +def _entry(entries: list[ModelInfoEntry], model_name: str) -> ModelInfoEntry: + for entry in entries: + if entry.model_name == model_name: + return entry + pytest.fail(f"{model_name} absent from /model/info; the deployment did not register") + + +def _resolved_costs(client: EndpointsClient, model: str) -> tuple[float, float]: + """The rate the proxy resolved from the cost map, with no override configured.""" + resolved = _entry(client.proxy.model_info(), model).model_info + assert resolved.input_cost_per_token is not None, ( + f"{model}: /model/info resolved no input_cost_per_token, so the deployment " + f"matched no cost-map entry at all" + ) + assert resolved.output_cost_per_token is not None, ( + f"{model}: /model/info resolved no output_cost_per_token" + ) + return resolved.input_cost_per_token, resolved.output_cost_per_token + + +class TestRegionalUpliftPricing: + @pytest.mark.covers("mgmt.model.info.reports_regional_uplift") + def test_regional_variants_price_above_global( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + global_model = _register(endpoints_client, resources, "global") + global_input, global_output = _resolved_costs(endpoints_client, global_model) + + assert _matches(global_input, GLOBAL_INPUT_COST), ( + f"global variant input rate drifted from the cost map: " + f"expected {GLOBAL_INPUT_COST}, got {global_input}" + ) + assert _matches(global_output, GLOBAL_OUTPUT_COST), ( + f"global variant output rate drifted from the cost map: " + f"expected {GLOBAL_OUTPUT_COST}, got {global_output}" + ) + + for prefix in ("us", "eu"): + model = _register(endpoints_client, resources, prefix) + input_cost, output_cost = _resolved_costs(endpoints_client, model) + + assert _matches(input_cost, global_input * REGIONAL_UPLIFT), ( + f"{prefix}. variant input rate {input_cost} is not the expected " + f"{REGIONAL_UPLIFT}x uplift over the global rate {global_input}; a " + f"regional deployment billed at the global rate under-charges every call" + ) + assert _matches(output_cost, global_output * REGIONAL_UPLIFT), ( + f"{prefix}. variant output rate {output_cost} is not the expected " + f"{REGIONAL_UPLIFT}x uplift over the global rate {global_output}" + ) diff --git a/tests/e2e/logging/test_prometheus_ratelimit_failure_e2e.py b/tests/e2e/logging/test_prometheus_ratelimit_failure_e2e.py new file mode 100644 index 00000000000..9cfb271d6a3 --- /dev/null +++ b/tests/e2e/logging/test_prometheus_ratelimit_failure_e2e.py @@ -0,0 +1,102 @@ +"""Live e2e: a rate-limit rejection is attributed to the gateway, not the vendor. + +Operators page on 429s, and the response to one depends entirely on who produced +it: a vendor 429 means back off or fail over, while a gateway 429 means the +customer's own key limit is set too low. `litellm_proxy_failed_requests_metric` +has to make that difference readable, and it has to name the provider, otherwise +a multi-provider deployment cannot tell which upstream is saturating (PR #27687). + +The two are distinguished by `exception_class`: the gateway's own limiter raises +`HTTPException`, while a vendor rejection surfaces its provider error class (for +example `Openai.RateLimitError`). This drives a key over its own RPM limit, which +is unambiguously a gateway-side rejection, and requires the resulting series to +be labelled that way and to carry a provider. + +The metric is written on the failure-logging callback, so the scrape polls to a +deadline rather than sleeping once. +""" + +from __future__ import annotations + +import time + +import pytest +from prometheus_client.parser import text_string_to_metric_families + +from e2e_config import unique_marker +from lifecycle import ResourceManager +from logging_client import LoggingClient +from models import KeyGenerateBody + +pytestmark = pytest.mark.e2e + +DRIVER_MODEL = "gemini-2.5-flash" +FAILURE_METRIC = "litellm_proxy_failed_requests_metric_total" +GATEWAY_EXCEPTION_CLASS = "HTTPException" +RPM_LIMIT = 1 +CALLS = 3 + + +def _rate_limited_series(exposition: str, alias: str) -> tuple[dict[str, str], ...]: + """Every 429 sample on the failure metric belonging to our key.""" + return tuple( + sample.labels + for family in text_string_to_metric_families(exposition) + for sample in family.samples + if sample.name == FAILURE_METRIC + and sample.labels.get("api_key_alias") == alias + and sample.labels.get("exception_status") == "429" + ) + + +class TestPrometheusRateLimitAttribution: + @pytest.mark.covers("logging.prometheus.failure.attributes_rate_limit_source") + def test_gateway_rate_limit_is_labelled_with_source_and_provider( + self, client: LoggingClient, resources: ResourceManager + ) -> None: + alias = f"e2e-rl-attr-{unique_marker()}" + key = client.proxy.generate_key( + KeyGenerateBody( + key_alias=alias, + models=[DRIVER_MODEL], + user_id=f"e2e-{alias}", + rpm_limit=RPM_LIMIT, + ) + ) + resources.defer(lambda: client.delete_key(key)) + + statuses = tuple( + client.chat_raw(key, DRIVER_MODEL, f"say ok {unique_marker()}").status_code + for _ in range(CALLS) + ) + assert 429 in statuses, ( + f"driving {CALLS} calls against an rpm_limit of {RPM_LIMIT} produced no 429; " + f"statuses were {statuses}, so the limiter never rejected and there is " + f"nothing for the metric to attribute" + ) + + deadline = time.monotonic() + client.proxy.poll_timeout + series: tuple[dict[str, str], ...] = () + while time.monotonic() < deadline: + series = _rate_limited_series(client.scrape_metrics(), alias) + if series: + break + time.sleep(client.proxy.poll_interval) + + assert series, ( + f"{FAILURE_METRIC} has no 429 series for api_key_alias={alias}; a rate-limit " + f"rejection that is never counted cannot be alerted on" + ) + + classes = {labels.get("exception_class") for labels in series} + assert classes == {GATEWAY_EXCEPTION_CLASS}, ( + f"a gateway rpm rejection must be attributed to the gateway as " + f"{GATEWAY_EXCEPTION_CLASS}, got {sorted(c or '' for c in classes)}; " + f"a vendor error class here would send operators chasing the upstream" + ) + + providers = {labels.get("api_provider") for labels in series} + assert providers and all(providers), ( + f"{FAILURE_METRIC} 429 series carries an empty api_provider ({providers}); " + f"a multi-provider deployment cannot tell which upstream the rejection was for" + )