test(e2e): pin regional cost-map uplift and rate-limit attribution

tests/e2e/coverage_registry/logging.yaml

@@ -6,6 +6,7 @@
 - {id: logging.datadog.stream.exports_metric, module: logging, tier: P0, event: stream, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses], source: "integrations/datadog/datadog.py", rationale: "Streaming aggregates usage after the last chunk; delivery and cost must survive that path"}
 - {id: logging.datadog.failure.exports_metric, module: logging, tier: P0, event: failure, assertions: [exports_metric], exercised_on: [chat_completions], source: "integrations/datadog/datadog.py", rationale: "Failure metrics for alerting/SLO"}
 - {id: logging.prometheus.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, embeddings], source: "integrations/prometheus.py", rationale: "Standard OSS metrics; per-key cardinality (existing e2e)"}
+- {id: logging.prometheus.failure.attributes_rate_limit_source, module: logging, tier: P1, event: failure, assertions: [attributes_rate_limit_source], exercised_on: [chat_completions], source: "PR #27687 / integrations/prometheus.py", fail_before_fix: proven, rationale: "A 429 must say whether the gateway or the vendor rejected it and name the provider; operators respond differently to each and cannot tell them apart otherwise"}
 - {id: logging.otel.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses, embeddings], source: "integrations/otel/logger.py", rationale: "OTEL spans on every call path"}
 - {id: logging.otel.stream.exports_metric, module: logging, tier: P0, event: stream, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses], source: "integrations/otel/logger.py", rationale: "Streaming closes the LLM span from the stream path; historically prone to duplicate/orphaned spans"}
 - {id: logging.otel.stream.records_ttft, module: logging, tier: P1, event: stream, assertions: [records_ttft], exercised_on: [chat_completions, messages, responses], source: "integrations/otel/mappers/genai.py", rationale: "TTFT is the streaming latency SLI; a zero or span-length value silently corrupts dashboards"}

tests/e2e/coverage_registry/mgmt.yaml

@@ -47,6 +47,7 @@
 - {id: mgmt.tag.new.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "tag_management_endpoints.py:160", rationale: "Tag for spend categorization"}
 - {id: mgmt.tag.list.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "tag_management_endpoints.py:315", rationale: "Tag enumeration"}
 - {id: mgmt.tag.delete.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "tag_management_endpoints.py:390", rationale: "Stops future tagging"}
+- {id: mgmt.model.info.reports_regional_uplift, module: mgmt, tier: P1, surface: api, assertions: [reports_regional_uplift], source: "LIT-3912 / model_prices_and_context_window.json", fail_before_fix: proven, rationale: "Regional bedrock variants must resolve their own uplifted cost-map rate, not the global one; resolving global silently under-bills every regional call"}
 - {id: mgmt.model.update.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "model_management_endpoints.py:1358", rationale: "Pricing/concurrency persist"}
 - {id: mgmt.model.delete.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "model_management_endpoints.py:1045", rationale: "Removes from registry"}
 - {id: mgmt.model.block.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "model_management_endpoints.py", rationale: "Blocked model stays blocked"}

tests/e2e/llm_translation/test_regional_pricing_e2e.py

@@ -0,0 +1,104 @@
+"""Live e2e: regional model variants resolve their own uplifted cost-map pricing.
+
+Bedrock publishes a regional variant of each Claude model (`us.`, `eu.`) priced
+above the `global.` variant, and the cost map carries all three. The mappings
+regressed so regional deployments resolved the global rate (LIT-3912), which
+silently under-bills every regional call and makes a customer's cost reporting
+disagree with their AWS invoice.
+
+`/model/info` is where the resolved rate is observable without spending money on
+a call, so this registers the three variants of one model and pins each rate. It
+asserts the exact expected uplift rather than merely `regional > global`: a
+mapping that resolved regional to some other model's rate would still satisfy an
+inequality while being just as wrong.
+
+No provider call is made, so this needs no AWS credentials.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from e2e_config import unique_marker
+from endpoints_client import EndpointsClient
+from lifecycle import ResourceManager
+from models import LiteLLMParamsBody, ModelInfoEntry
+
+pytestmark = pytest.mark.e2e
+
+BASE_MODEL = "anthropic.claude-opus-4-7"
+GLOBAL_INPUT_COST = 5e-06
+GLOBAL_OUTPUT_COST = 2.5e-05
+REGIONAL_UPLIFT = 1.1
+RELATIVE_TOLERANCE = 1e-9
+
+
+def _matches(actual: float, expected: float) -> bool:
+    """Float-safe equality for rates; the uplift arithmetic is not exact in binary."""
+    return abs(actual - expected) <= abs(expected) * RELATIVE_TOLERANCE
+
+
+def _register(
+    client: EndpointsClient, resources: ResourceManager, prefix: str
+) -> str:
+    model = f"e2e-regional-{prefix}-{unique_marker()}"
+    model_id = client.create_model(
+        model,
+        LiteLLMParamsBody(
+            model=f"bedrock/{prefix}.{BASE_MODEL}", aws_region_name="us-east-1"
+        ),
+    )
+    resources.defer(lambda: client.delete_model(model_id))
+    return model
+
+
+def _entry(entries: list[ModelInfoEntry], model_name: str) -> ModelInfoEntry:
+    for entry in entries:
+        if entry.model_name == model_name:
+            return entry
+    pytest.fail(f"{model_name} absent from /model/info; the deployment did not register")
+
+
+def _resolved_costs(client: EndpointsClient, model: str) -> tuple[float, float]:
+    """The rate the proxy resolved from the cost map, with no override configured."""
+    resolved = _entry(client.proxy.model_info(), model).model_info
+    assert resolved.input_cost_per_token is not None, (
+        f"{model}: /model/info resolved no input_cost_per_token, so the deployment "
+        f"matched no cost-map entry at all"
+    )
+    assert resolved.output_cost_per_token is not None, (
+        f"{model}: /model/info resolved no output_cost_per_token"
+    )
+    return resolved.input_cost_per_token, resolved.output_cost_per_token
+
+
+class TestRegionalUpliftPricing:
+    @pytest.mark.covers("mgmt.model.info.reports_regional_uplift")
+    def test_regional_variants_price_above_global(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        global_model = _register(endpoints_client, resources, "global")
+        global_input, global_output = _resolved_costs(endpoints_client, global_model)
+
+        assert _matches(global_input, GLOBAL_INPUT_COST), (
+            f"global variant input rate drifted from the cost map: "
+            f"expected {GLOBAL_INPUT_COST}, got {global_input}"
+        )
+        assert _matches(global_output, GLOBAL_OUTPUT_COST), (
+            f"global variant output rate drifted from the cost map: "
+            f"expected {GLOBAL_OUTPUT_COST}, got {global_output}"
+        )
+
+        for prefix in ("us", "eu"):
+            model = _register(endpoints_client, resources, prefix)
+            input_cost, output_cost = _resolved_costs(endpoints_client, model)
+
+            assert _matches(input_cost, global_input * REGIONAL_UPLIFT), (
+                f"{prefix}. variant input rate {input_cost} is not the expected "
+                f"{REGIONAL_UPLIFT}x uplift over the global rate {global_input}; a "
+                f"regional deployment billed at the global rate under-charges every call"
+            )
+            assert _matches(output_cost, global_output * REGIONAL_UPLIFT), (
+                f"{prefix}. variant output rate {output_cost} is not the expected "
+                f"{REGIONAL_UPLIFT}x uplift over the global rate {global_output}"
+            )

tests/e2e/logging/test_prometheus_ratelimit_failure_e2e.py

@@ -0,0 +1,102 @@
+"""Live e2e: a rate-limit rejection is attributed to the gateway, not the vendor.
+
+Operators page on 429s, and the response to one depends entirely on who produced
+it: a vendor 429 means back off or fail over, while a gateway 429 means the
+customer's own key limit is set too low. `litellm_proxy_failed_requests_metric`
+has to make that difference readable, and it has to name the provider, otherwise
+a multi-provider deployment cannot tell which upstream is saturating (PR #27687).
+
+The two are distinguished by `exception_class`: the gateway's own limiter raises
+`HTTPException`, while a vendor rejection surfaces its provider error class (for
+example `Openai.RateLimitError`). This drives a key over its own RPM limit, which
+is unambiguously a gateway-side rejection, and requires the resulting series to
+be labelled that way and to carry a provider.
+
+The metric is written on the failure-logging callback, so the scrape polls to a
+deadline rather than sleeping once.
+"""
+
+from __future__ import annotations
+
+import time
+
+import pytest
+from prometheus_client.parser import text_string_to_metric_families
+
+from e2e_config import unique_marker
+from lifecycle import ResourceManager
+from logging_client import LoggingClient
+from models import KeyGenerateBody
+
+pytestmark = pytest.mark.e2e
+
+DRIVER_MODEL = "gemini-2.5-flash"
+FAILURE_METRIC = "litellm_proxy_failed_requests_metric_total"
+GATEWAY_EXCEPTION_CLASS = "HTTPException"
+RPM_LIMIT = 1
+CALLS = 3
+
+
+def _rate_limited_series(exposition: str, alias: str) -> tuple[dict[str, str], ...]:
+    """Every 429 sample on the failure metric belonging to our key."""
+    return tuple(
+        sample.labels
+        for family in text_string_to_metric_families(exposition)
+        for sample in family.samples
+        if sample.name == FAILURE_METRIC
+        and sample.labels.get("api_key_alias") == alias
+        and sample.labels.get("exception_status") == "429"
+    )
+
+
+class TestPrometheusRateLimitAttribution:
+    @pytest.mark.covers("logging.prometheus.failure.attributes_rate_limit_source")
+    def test_gateway_rate_limit_is_labelled_with_source_and_provider(
+        self, client: LoggingClient, resources: ResourceManager
+    ) -> None:
+        alias = f"e2e-rl-attr-{unique_marker()}"
+        key = client.proxy.generate_key(
+            KeyGenerateBody(
+                key_alias=alias,
+                models=[DRIVER_MODEL],
+                user_id=f"e2e-{alias}",
+                rpm_limit=RPM_LIMIT,
+            )
+        )
+        resources.defer(lambda: client.delete_key(key))
+
+        statuses = tuple(
+            client.chat_raw(key, DRIVER_MODEL, f"say ok {unique_marker()}").status_code
+            for _ in range(CALLS)
+        )
+        assert 429 in statuses, (
+            f"driving {CALLS} calls against an rpm_limit of {RPM_LIMIT} produced no 429; "
+            f"statuses were {statuses}, so the limiter never rejected and there is "
+            f"nothing for the metric to attribute"
+        )
+
+        deadline = time.monotonic() + client.proxy.poll_timeout
+        series: tuple[dict[str, str], ...] = ()
+        while time.monotonic() < deadline:
+            series = _rate_limited_series(client.scrape_metrics(), alias)
+            if series:
+                break
+            time.sleep(client.proxy.poll_interval)
+
+        assert series, (
+            f"{FAILURE_METRIC} has no 429 series for api_key_alias={alias}; a rate-limit "
+            f"rejection that is never counted cannot be alerted on"
+        )
+
+        classes = {labels.get("exception_class") for labels in series}
+        assert classes == {GATEWAY_EXCEPTION_CLASS}, (
+            f"a gateway rpm rejection must be attributed to the gateway as "
+            f"{GATEWAY_EXCEPTION_CLASS}, got {sorted(c or '' for c in classes)}; "
+            f"a vendor error class here would send operators chasing the upstream"
+        )
+
+        providers = {labels.get("api_provider") for labels in series}
+        assert providers and all(providers), (
+            f"{FAILURE_METRIC} 429 series carries an empty api_provider ({providers}); "
+            f"a multi-provider deployment cannot tell which upstream the rejection was for"
+        )