test(e2e): cover the reliability retry, cooldown, fallback, and routing-strategy cells

litellm/integrations/custom_logger.py

@@ -822,46 +822,33 @@ async def async_build_chat_completion_agentic_loop_plan(
     def truncate_standard_logging_payload_content(
         self,
         standard_logging_object: StandardLoggingPayload,
-    ):
+    ) -> StandardLoggingPayload:
         """
-        Truncate error strings and message content in logging payload
+        Return a copy of the logging payload with error_str, messages, and response truncated
 
         Some loggers like DataDog/ GCS Bucket have a limit on the size of the payload. (1MB)
 
-        This function truncates the error string and the message content if they exceed a certain length.
+        Every callback of a request shares one standard logging object, so the payload passed in is left
+        untouched and the callbacks that run later (the prompt caching router check, spend logs) still see
+        the original fields.
         """
-        MAX_STR_LENGTH: Final = 10_000
-
-        # Truncate fields that might exceed max length
-        fields_to_truncate: Final = ["error_str", "messages", "response"]
-        for field in fields_to_truncate:
-            self._truncate_field(
-                standard_logging_object=standard_logging_object,
-                field_name=field,
-                max_length=MAX_STR_LENGTH,
-            )
+        max_str_length: Final = 10_000
+        candidates: Final = {
+            field: self._truncate_field(field_value=standard_logging_object.get(field), max_length=max_str_length)
+            for field in ("error_str", "messages", "response")
+        }
+        truncated_fields: Final = {field: text for field, text in candidates.items() if text is not None}
+        return {**standard_logging_object, **truncated_fields}
 
-    def _truncate_field(
-        self,
-        standard_logging_object: StandardLoggingPayload,
-        field_name: str,
-        max_length: int,
-    ) -> None:
+    def _truncate_field(self, field_value: object, max_length: int) -> str | None:
         """
-        Helper function to truncate a field in the logging payload
-
-        This converts the field to a string and then truncates it if it exceeds the max length.
+        Return the truncated text of a field that exceeds max_length, or None when the field fits
 
-        Why convert to string ?
-        1. User was sending a poorly formatted list for `messages` field, we could not predict where they would send content
-            - Converting to string and then truncating the logged content catches this
-        2. We want to avoid modifying the original `messages`, `response`, and `error_str` in the logging payload since these are in kwargs and could be returned to the user
+        The field is measured as a string because users send poorly formatted lists for `messages`, so there is
+        no fixed place the content would be.
         """
-        field_value: Final[object] = standard_logging_object.get(field_name)
-        if field_value:
-            str_value: Final = str(field_value)
-            if len(str_value) > max_length:
-                standard_logging_object[field_name] = self._truncate_text(text=str_value, max_length=max_length)
+        text: Final = str(field_value or "")
+        return self._truncate_text(text=text, max_length=max_length) if len(text) > max_length else None
 
     def _truncate_text(self, text: str, max_length: int) -> str:
         """Truncate text if it exceeds max_length"""

litellm/integrations/datadog/datadog.py

@@ -563,11 +563,10 @@ def create_datadog_logging_payload(
         if standard_logging_object.get("status") == "failure":
             status = DataDogStatus.ERROR
 
-        # Build the initial payload
-        self.truncate_standard_logging_payload_content(standard_logging_object)
+        truncated_payload: Final = self.truncate_standard_logging_payload_content(standard_logging_object)
 
         dd_payload: Final = self._create_datadog_logging_payload_helper(
-            standard_logging_object=standard_logging_object,
+            standard_logging_object=truncated_payload,
             status=status,
         )
         return dd_payload

tests/e2e/CONTRIBUTING.md

@@ -9,7 +9,7 @@ When contributing to this directory, please first discuss the change you wish to
 
 ## Setup
 
-The suites run against a live proxy, so bring one up first by running the litellm proxy locally. Point it at a config that prewires the example models the suites use (`gpt-5.5`, `claude-haiku-4-5`, `gemini-2.5-flash`, `openai-text-embedding-3-small`) with keys from your `.env`, and enables prompt storage, a redis cache, and the fast budget rescheduler the quota suites rely on. If your test needs another model, a pricing override, or a guardrail declared up front, add it to that config and read it back in the test rather than hardcoding values
+The suites run against a live proxy, so bring one up first by running the litellm proxy locally. Point it at a config that prewires the example models the suites use (`gpt-5.5`, `claude-haiku-4-5`, `gemini-2.5-flash`, `openai-text-embedding-3-small`) with keys from your `.env`, and enables prompt storage, a redis cache, the fast budget rescheduler the quota suites rely on, and `router_settings.optional_pre_call_checks: ["prompt_caching"]`, which the router suite's prompt-cache affinity test reads back from `GET /router/settings` and fails without. If your test needs another model, a pricing override, or a guardrail declared up front, add it to that config and read it back in the test rather than hardcoding values
 
 ## Running the tests locally
 
@@ -212,7 +212,7 @@ Each suite provides its own `client` fixture (see `llm_translation/passthrough_c
 
 Request and response bodies are typed pydantic models in `models.py`; only the fields a test reads are modelled, and nothing passes raw dicts. Outcomes come back as a `Result[R]` tagged union (`Success`, `NetworkError`, `UnauthorizedError`, `RateLimitedError`, `ValidationError`, `UnknownApiError`). Handle them with `match`, or call `unwrap(...)` when a non-success should fail the test. The harness hard-fails and never skips: a test marked `e2e` fails when no proxy answers its liveness probe, and once a request reaches the proxy any wrong behavior is likewise a hard failure, so a missing proxy turns the run red instead of being mistaken for a pass
 
-Mark live tests with `@pytest.mark.e2e` (on the class or the module). Pure coverage of the harness itself carries no marker and runs regardless. Use `scoped_key` for a fresh all-models key that auto-deletes, `resources` when you need to create and tear down more than a key, and `unique_marker()` from `e2e_config` to keep prompts, tags, and customer ids from colliding across concurrent runs and the shared response cache
+Mark live tests with `@pytest.mark.e2e` (on the class or the module). Pure coverage of the harness itself carries no marker and runs regardless. A test that needs proxy configuration the default stack does not carry goes behind an opt-in marker (`managed_files`, `prompt_caching_stack`, `weekly`), each deselected unless its env var is set; `OPT_IN_MARKERS` in `conftest.py` maps marker to env var, and the coverage collector counts such a cell only where the env var is set. Use `scoped_key` for a fresh all-models key that auto-deletes, `resources` when you need to create and tear down more than a key, and `unique_marker()` from `e2e_config` to keep prompts, tags, and customer ids from colliding across concurrent runs and the shared response cache
 
 ## Pre-commit steps
 

tests/e2e/batches/conftest.py

@@ -12,14 +12,12 @@
 
 from __future__ import annotations
 
-import os
 from typing import Final, Iterator
 
 import pytest
 
 from batch_client import BatchClient, build_client
 from capabilities import PROVIDERS
-from e2e_config import MANAGED_FILES_OPT_IN_ENV
 from e2e_http import NoBody
 from lifecycle import ResourceManager
 from proxy_client import ProxyClient
@@ -32,22 +30,6 @@ def pytest_configure(config: pytest.Config) -> None:
     )
 
 
-def pytest_collection_modifyitems(
-    config: pytest.Config, items: list[pytest.Item]
-) -> None:
-    if os.environ.get(MANAGED_FILES_OPT_IN_ENV):
-        return
-    deselected = [
-        item for item in items if item.get_closest_marker("managed_files") is not None
-    ]
-    if not deselected:
-        return
-    config.hook.pytest_deselected(items=deselected)
-    items[:] = [
-        item for item in items if item.get_closest_marker("managed_files") is None
-    ]
-
-
 @pytest.fixture(scope="session")
 def client(proxy: ProxyClient) -> BatchClient:
     return build_client(proxy)

tests/e2e/batches/test_managed_files_enforcement_e2e.py

@@ -5,7 +5,7 @@
 flag would 400 every files_settings-routed upload in the rest of the suite. The
 PR gate instead reconfigures the same stack sequentially after the main run and
 executes only this file with E2E_MANAGED_FILES_STACK set; without that env every
-test here is deselected (see conftest.py, mirroring the weekly marker).
+test here is deselected (see OPT_IN_MARKERS in tests/e2e/conftest.py).
 
 Pins: an upload without target_model_names is rejected 400, an upload that also
 carries a model param is rejected 400, a raw provider file id is rejected 400 on

tests/e2e/conftest.py

@@ -17,11 +17,23 @@
 import os
 from collections.abc import Generator, Iterator
 from datetime import datetime, timezone
+from types import MappingProxyType
 from typing import Final
 
 import pytest
 import requests
-from e2e_config import CONTROL_PLANE_BASE_URL, FIXTURE_DIR, FIXTURE_MODE_RAW, PROXY_BASE_URL, unique_marker
+
+from e2e_config import (
+    CONTROL_PLANE_BASE_URL,
+    FIXTURE_DIR,
+    FIXTURE_MODE_RAW,
+    MANAGED_FILES_OPT_IN_ENV,
+    PROMPT_CACHING_OPT_IN_ENV,
+    PROXY_BASE_URL,
+    REDIS_CHAOS_OPT_IN_ENV,
+    WEEKLY_ANOMALY_OPT_IN_ENV,
+    unique_marker,
+)
 from e2e_db import RESET_OPT_IN_ENV, reset_spend_logs, run_spend_log_cleanup
 from e2e_http import unwrap
 from fixture_mode import fixture_mode_collection_error, fixture_report_lines
@@ -35,6 +47,15 @@
 _E2E_TEST_RAN = pytest.StashKey[bool]()
 _CALL_PASSED = pytest.StashKey[bool]()
 
+OPT_IN_MARKERS: Final = MappingProxyType(
+    {
+        "weekly": WEEKLY_ANOMALY_OPT_IN_ENV,
+        "managed_files": MANAGED_FILES_OPT_IN_ENV,
+        "prompt_caching_stack": PROMPT_CACHING_OPT_IN_ENV,
+        "redis_chaos": REDIS_CHAOS_OPT_IN_ENV,
+    }
+)
+
 
 @pytest.fixture(scope="session")
 def idp() -> Keycloak:
@@ -89,6 +110,11 @@ def pytest_configure(config: pytest.Config) -> None:
         "markers",
         "managed_files: needs a proxy running with require_managed_files enabled; deselected unless E2E_MANAGED_FILES_STACK is set",
     )
+    config.addinivalue_line(
+        "markers",
+        "prompt_caching_stack: needs a proxy running with router_settings.optional_pre_call_checks including "
+        "prompt_caching; deselected unless E2E_PROMPT_CACHING_STACK is set",
+    )
     config.addinivalue_line(
         "markers",
         "redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from "
@@ -111,16 +137,32 @@ def pytest_report_header(config: pytest.Config) -> list[str]:
     return fixture_report_lines(FIXTURE_MODE_RAW, FIXTURE_DIR, now=datetime.now(timezone.utc))
 
 
-def pytest_collection_modifyitems(items: list[pytest.Item]) -> None:
-    """Attach the two custom signals (suite package and covered cell ids) to every
-    test's user_properties so the standard JUnit report (`--junitxml`) records them
-    as `<property>` entries, on every outcome including skips and setup errors.
-    Downstream (Loki/Grafana) reads outcome and duration from the standard report
-    and these properties for package rollups and coverage drill-down. See
-    junit_properties.py.
+def _needs_unset_opt_in(item: pytest.Item) -> bool:
+    return any(
+        item.get_closest_marker(marker) is not None and not os.environ.get(opt_in_env)
+        for marker, opt_in_env in OPT_IN_MARKERS.items()
+    )
+
+
+def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
+    """Deselect every test behind an opt-in marker whose env var is unset (see
+    OPT_IN_MARKERS): those tests need a proxy configured differently from the
+    default stack, so the coverage collector, which runs over the same collection,
+    counts their cells only where they actually run.
+
+    Attach the two custom signals (suite package and covered cell ids) to every
+    remaining test's user_properties so the standard JUnit report (`--junitxml`)
+    records them as `<property>` entries, on every outcome including skips and
+    setup errors. Downstream (Loki/Grafana) reads outcome and duration from the
+    standard report and these properties for package rollups and coverage
+    drill-down. See junit_properties.py.
 
     Also sort `load`-marked items last so a whole-tree run drives heavy throughput
     traffic only after the latency-sensitive suites have finished."""
+    deselected = [item for item in items if _needs_unset_opt_in(item)]
+    if deselected:
+        config.hook.pytest_deselected(items=deselected)
+        items[:] = [item for item in items if not _needs_unset_opt_in(item)]
     for item in items:
         attach_result_properties(item)
     items.sort(key=lambda item: item.get_closest_marker("load") is not None)