From d03914b589f77b2c153d568ad4df877345da1d36 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 4 May 2026 19:58:31 +0000 Subject: [PATCH 01/12] test: add 24hr Redis-backed VCR cache to additional test suites Extracts the existing llm_translation VCR plumbing into a reusable helper (tests/_vcr_conftest_common.py) and wires it into the conftest.py files of the test directories listed in LIT-2787: audio_tests, batches_tests, guardrails_tests, image_gen_tests, litellm_utils_tests, local_testing, logging_callback_tests, pass_through_unit_tests, router_unit_tests, unified_google_tests The same helper is also adopted by the pre-existing llm_translation and llm_responses_api_testing conftests to remove the copy-pasted VCR setup. Each consuming conftest: - registers the Redis persister via pytest_recording_configure - auto-marks collected tests with pytest.mark.vcr (skipping respx-using files where applicable, since respx and vcrpy both patch httpx) - gates cassette writes on test success via _vcr_outcome_gate The cache is opt-in via CASSETTE_REDIS_URL; when unset, VCR is disabled and tests hit live providers as before. LITELLM_VCR_DISABLE=1 still forces a bypass for ad-hoc local runs. Test directories that run LiteLLM proxy in Docker (build_and_test, proxy_logging_guardrails_model_info_tests, proxy_store_model_in_db_tests) are intentionally not included: VCR.py patches the in-process httpx transport and cannot intercept calls made from inside a Docker container. The installing_litellm_on_python* jobs make no LLM calls and don't benefit from caching. https://linear.app/litellm-ai/issue/LIT-2787/add-24hr-caching-to-additional-test-suites --- tests/_vcr_conftest_common.py | 260 ++++++++++++++++++++ tests/audio_tests/conftest.py | 56 +++++ tests/batches_tests/conftest.py | 48 +++- tests/guardrails_tests/conftest.py | 42 ++++ tests/image_gen_tests/conftest.py | 50 +++- tests/litellm_utils_tests/conftest.py | 49 +++- tests/llm_responses_api_testing/conftest.py | 148 +---------- tests/llm_translation/Readme.md | 34 ++- tests/llm_translation/conftest.py | 166 ++----------- tests/local_testing/conftest.py | 57 +++++ tests/logging_callback_tests/conftest.py | 55 +++++ tests/pass_through_unit_tests/conftest.py | 56 +++++ tests/router_unit_tests/conftest.py | 51 +++- tests/unified_google_tests/conftest.py | 50 +++- 14 files changed, 813 insertions(+), 309 deletions(-) create mode 100644 tests/_vcr_conftest_common.py create mode 100644 tests/audio_tests/conftest.py create mode 100644 tests/pass_through_unit_tests/conftest.py diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py new file mode 100644 index 000000000000..6baf618a3167 --- /dev/null +++ b/tests/_vcr_conftest_common.py @@ -0,0 +1,260 @@ +""" +Shared VCR (Redis-backed) plumbing for test directories. + +This module is imported by per-directory ``conftest.py`` files to enable +24-hour HTTP caching against a Redis backend. The first run hits live +provider APIs and records the exchange; subsequent runs within 24h replay +from Redis without touching the network. See +``tests/llm_translation/Readme.md`` for the full design notes. + +Each consuming ``conftest.py`` should: + +1. Define a ``vcr_config`` fixture that delegates to :func:`vcr_config_dict`. +2. Define ``pytest_recording_configure(config, vcr)`` that calls + :func:`register_persister_if_enabled`. +3. Define ``pytest_runtest_makereport`` and a ``_vcr_outcome_gate`` autouse + fixture using :func:`make_outcome_gate_fixture` (or copy the snippet) so + failed tests don't poison cassettes. +4. Add ``apply_vcr_auto_marker_to_items`` inside + ``pytest_collection_modifyitems`` so non-respx tests are auto-marked + with ``pytest.mark.vcr``. +""" + +from __future__ import annotations + +import os +from typing import Iterable, Optional + +import pytest + +from tests._vcr_redis_persister import ( + filter_non_2xx_response, + format_vcr_verdict, + make_redis_persister, + mark_test_outcome_for_cassette, + patch_vcrpy_aiohttp_record_path, + vcr_verbose_enabled, +) + +FILTERED_REQUEST_HEADERS = ( + "authorization", + "x-api-key", + "anthropic-api-key", + "anthropic-version", + "openai-api-key", + "azure-api-key", + "api-key", + "cookie", + "x-amz-security-token", + "x-amz-date", + "x-amz-content-sha256", + "amz-sdk-invocation-id", + "amz-sdk-request", + "x-goog-api-key", + "x-goog-user-project", +) + +FILTERED_RESPONSE_HEADERS = ( + "set-cookie", + "x-request-id", + "request-id", + "cf-ray", + "anthropic-organization-id", + "openai-organization", + "x-amzn-requestid", + "x-amzn-trace-id", + "date", +) + + +def _scrub_response(response): + if not isinstance(response, dict): + return response + headers = response.get("headers") or {} + if isinstance(headers, dict): + for header in list(headers): + if header.lower() in FILTERED_RESPONSE_HEADERS: + headers.pop(header, None) + return response + + +def _before_record_response(response): + return filter_non_2xx_response(_scrub_response(response)) + + +def vcr_config_dict() -> dict: + """Return the VCR config dict shared across all consuming conftests.""" + return { + "filter_headers": list(FILTERED_REQUEST_HEADERS), + "decode_compressed_response": True, + "record_mode": "new_episodes", + "allow_playback_repeats": True, + "match_on": ( + "method", + "scheme", + "host", + "port", + "path", + "query", + "body", + ), + "before_record_response": _before_record_response, + } + + +def vcr_disabled() -> bool: + """VCR is disabled when explicitly turned off, or when no Redis is configured.""" + if os.environ.get("LITELLM_VCR_DISABLE") == "1": + return True + return not os.environ.get("CASSETTE_REDIS_URL") + + +def register_persister_if_enabled(vcr) -> None: + """Wire the Redis persister into vcrpy if VCR is enabled. + + Call this from ``pytest_recording_configure(config, vcr)`` in conftest. + """ + if vcr_disabled(): + return + vcr.register_persister(make_redis_persister()) + patch_vcrpy_aiohttp_record_path() + + +def apply_vcr_auto_marker_to_items( + items, + *, + skip_files: Iterable[str] = (), + skip_nodeid_suffixes: Iterable[str] = (), +) -> None: + """Auto-apply ``pytest.mark.vcr`` to collected items. + + ``skip_files`` is a set of basenames (e.g. ``test_openai.py``) that + should not be auto-marked — typically files that already use ``respx``, + since respx and vcrpy both patch the httpx transport and conflict. + + ``skip_nodeid_suffixes`` is a set of node-id suffixes (e.g. + ``"::test_prompt_caching"``) that observe live cross-call provider + state which replay can't reproduce. + """ + if vcr_disabled(): + return + skip_files = frozenset(skip_files) + skip_nodeid_suffixes = tuple(skip_nodeid_suffixes) + for item in items: + filename = os.path.basename(str(item.fspath)) + if filename in skip_files: + continue + if any(item.nodeid.endswith(suffix) for suffix in skip_nodeid_suffixes): + continue + if item.get_closest_marker("vcr") is not None: + continue + item.add_marker(pytest.mark.vcr) + + +def record_vcr_outcome(request, vcr) -> None: + """Mark the cassette with the test outcome and emit a verbose verdict. + + Call this from a ``yield``-after section of an autouse fixture in + conftest, after the test has run. + """ + cassette = vcr + rep_call = getattr(request.node, "rep_call", None) + test_passed = bool(rep_call and rep_call.passed) + cassette_path = getattr(cassette, "_path", None) if cassette is not None else None + if cassette_path: + mark_test_outcome_for_cassette(cassette_path, test_passed) + + if not vcr_verbose_enabled(): + return + verdict = format_vcr_verdict(cassette) + request.node.user_properties.append(("vcr_verdict", verdict)) + + +# --------------------------------------------------------------------------- +# Verbose-verdict reporter helpers (optional; used by conftests that want to +# print "[VCR HIT]/[VCR MISS]/..." lines next to each test in CI logs). +# --------------------------------------------------------------------------- +class VerboseReporterState: + """Container for the controller-process plugin manager / terminal reporter. + + A single instance lives in each conftest that wants verbose output. + """ + + def __init__(self) -> None: + self.pluginmanager = None + self.terminal_reporter = None + + def remember_pluginmanager(self, config) -> None: + if os.environ.get("PYTEST_XDIST_WORKER"): + return + self.pluginmanager = config.pluginmanager + + def resolve_terminal_reporter(self): + if self.terminal_reporter is not None: + return self.terminal_reporter + if self.pluginmanager is None: + return None + self.terminal_reporter = self.pluginmanager.getplugin("terminalreporter") + return self.terminal_reporter + + def maybe_emit_verdict(self, report) -> None: + if report.when != "teardown": + return + if os.environ.get("PYTEST_XDIST_WORKER"): + return + if not vcr_verbose_enabled(): + return + reporter = self.resolve_terminal_reporter() + if reporter is None: + return + verdict = next( + (v for k, v in (report.user_properties or []) if k == "vcr_verdict"), + None, + ) + if not verdict: + return + reporter.write_line(f"{verdict} :: {report.nodeid}") + + +# --------------------------------------------------------------------------- +# Drop-in conftest snippet (copy/paste guidance, not executed). +# --------------------------------------------------------------------------- +# from tests._vcr_conftest_common import ( +# VerboseReporterState, +# apply_vcr_auto_marker_to_items, +# record_vcr_outcome, +# register_persister_if_enabled, +# vcr_config_dict, +# ) +# +# _verbose_state = VerboseReporterState() +# _RESPX_CONFLICTING_FILES = frozenset({...}) +# +# @pytest.fixture(scope="module") +# def vcr_config(): +# return vcr_config_dict() +# +# def pytest_recording_configure(config, vcr): +# register_persister_if_enabled(vcr) +# +# @pytest.hookimpl(hookwrapper=True) +# def pytest_runtest_makereport(item, call): +# outcome = yield +# rep = outcome.get_result() +# setattr(item, f"rep_{rep.when}", rep) +# +# @pytest.fixture(autouse=True) +# def _vcr_outcome_gate(request, vcr): +# yield +# record_vcr_outcome(request, vcr) +# +# def pytest_configure(config): +# _verbose_state.remember_pluginmanager(config) +# +# def pytest_runtest_logreport(report): +# _verbose_state.maybe_emit_verdict(report) +# +# def pytest_collection_modifyitems(config, items): +# apply_vcr_auto_marker_to_items( +# items, skip_files=_RESPX_CONFLICTING_FILES, +# ) diff --git a/tests/audio_tests/conftest.py b/tests/audio_tests/conftest.py new file mode 100644 index 000000000000..5b36a5d434d5 --- /dev/null +++ b/tests/audio_tests/conftest.py @@ -0,0 +1,56 @@ +# conftest.py +# +# Wires audio tests into the Redis-backed VCR cache so live provider +# calls are replayed for 24h. See tests/llm_translation/Readme.md for +# the design overview. + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +from tests._vcr_conftest_common import ( # noqa: E402 + VerboseReporterState, + apply_vcr_auto_marker_to_items, + record_vcr_outcome, + register_persister_if_enabled, + vcr_config_dict, +) + +_verbose_state = VerboseReporterState() + + +@pytest.fixture(scope="module") +def vcr_config(): + return vcr_config_dict() + + +def pytest_recording_configure(config, vcr): + register_persister_if_enabled(vcr) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + rep = outcome.get_result() + setattr(item, f"rep_{rep.when}", rep) + + +@pytest.fixture(autouse=True) +def _vcr_outcome_gate(request, vcr): + yield + record_vcr_outcome(request, vcr) + + +def pytest_configure(config): + _verbose_state.remember_pluginmanager(config) + + +def pytest_runtest_logreport(report): + _verbose_state.maybe_emit_verdict(report) + + +def pytest_collection_modifyitems(config, items): + apply_vcr_auto_marker_to_items(items) diff --git a/tests/batches_tests/conftest.py b/tests/batches_tests/conftest.py index f0a282365969..e6e31546a82c 100644 --- a/tests/batches_tests/conftest.py +++ b/tests/batches_tests/conftest.py @@ -1,5 +1,6 @@ # conftest.py +import asyncio import importlib import os import sys @@ -9,8 +10,17 @@ sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path -import litellm -import asyncio +import litellm # noqa: E402,F401 + +from tests._vcr_conftest_common import ( # noqa: E402 + VerboseReporterState, + apply_vcr_auto_marker_to_items, + record_vcr_outcome, + register_persister_if_enabled, + vcr_config_dict, +) + +_verbose_state = VerboseReporterState() @pytest.fixture(scope="session") @@ -21,3 +31,37 @@ def event_loop(): loop = asyncio.new_event_loop() yield loop loop.close() + + +@pytest.fixture(scope="module") +def vcr_config(): + return vcr_config_dict() + + +def pytest_recording_configure(config, vcr): + register_persister_if_enabled(vcr) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + rep = outcome.get_result() + setattr(item, f"rep_{rep.when}", rep) + + +@pytest.fixture(autouse=True) +def _vcr_outcome_gate(request, vcr): + yield + record_vcr_outcome(request, vcr) + + +def pytest_configure(config): + _verbose_state.remember_pluginmanager(config) + + +def pytest_runtest_logreport(report): + _verbose_state.maybe_emit_verdict(report) + + +def pytest_collection_modifyitems(config, items): + apply_vcr_auto_marker_to_items(items) diff --git a/tests/guardrails_tests/conftest.py b/tests/guardrails_tests/conftest.py index c57d4ed5de78..674d5500c3c2 100644 --- a/tests/guardrails_tests/conftest.py +++ b/tests/guardrails_tests/conftest.py @@ -16,6 +16,46 @@ ) # Adds the parent directory to the system path import litellm +from tests._vcr_conftest_common import ( # noqa: E402 + VerboseReporterState, + apply_vcr_auto_marker_to_items, + record_vcr_outcome, + register_persister_if_enabled, + vcr_config_dict, +) + +_verbose_state = VerboseReporterState() + + +@pytest.fixture(scope="module") +def vcr_config(): + return vcr_config_dict() + + +def pytest_recording_configure(config, vcr): + register_persister_if_enabled(vcr) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + rep = outcome.get_result() + setattr(item, f"rep_{rep.when}", rep) + + +@pytest.fixture(autouse=True) +def _vcr_outcome_gate(request, vcr): + yield + record_vcr_outcome(request, vcr) + + +def pytest_configure(config): + _verbose_state.remember_pluginmanager(config) + + +def pytest_runtest_logreport(report): + _verbose_state.maybe_emit_verdict(report) + @pytest.fixture(scope="function", autouse=True) def isolate_litellm_state(): @@ -97,6 +137,8 @@ def setup_and_teardown(): def pytest_collection_modifyitems(config, items): + apply_vcr_auto_marker_to_items(items) + # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests custom_logger_tests = [ item for item in items if "custom_logger" in item.parent.name diff --git a/tests/image_gen_tests/conftest.py b/tests/image_gen_tests/conftest.py index c0b1a44be84b..ae67a4a9243c 100644 --- a/tests/image_gen_tests/conftest.py +++ b/tests/image_gen_tests/conftest.py @@ -1,15 +1,23 @@ -import importlib +import asyncio import os import sys -import asyncio + import pytest sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path -import litellm +import litellm # noqa: E402,F401 -import asyncio +from tests._vcr_conftest_common import ( # noqa: E402 + VerboseReporterState, + apply_vcr_auto_marker_to_items, + record_vcr_outcome, + register_persister_if_enabled, + vcr_config_dict, +) + +_verbose_state = VerboseReporterState() @pytest.fixture(scope="session") @@ -20,3 +28,37 @@ def event_loop(): loop = asyncio.new_event_loop() yield loop loop.close() + + +@pytest.fixture(scope="module") +def vcr_config(): + return vcr_config_dict() + + +def pytest_recording_configure(config, vcr): + register_persister_if_enabled(vcr) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + rep = outcome.get_result() + setattr(item, f"rep_{rep.when}", rep) + + +@pytest.fixture(autouse=True) +def _vcr_outcome_gate(request, vcr): + yield + record_vcr_outcome(request, vcr) + + +def pytest_configure(config): + _verbose_state.remember_pluginmanager(config) + + +def pytest_runtest_logreport(report): + _verbose_state.maybe_emit_verdict(report) + + +def pytest_collection_modifyitems(config, items): + apply_vcr_auto_marker_to_items(items) diff --git a/tests/litellm_utils_tests/conftest.py b/tests/litellm_utils_tests/conftest.py index eca0bc431a5a..dd21601a2152 100644 --- a/tests/litellm_utils_tests/conftest.py +++ b/tests/litellm_utils_tests/conftest.py @@ -1,5 +1,6 @@ # conftest.py +import asyncio import importlib import os import sys @@ -9,7 +10,17 @@ sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path -import litellm +import litellm # noqa: E402,F401 + +from tests._vcr_conftest_common import ( # noqa: E402 + VerboseReporterState, + apply_vcr_auto_marker_to_items, + record_vcr_outcome, + register_persister_if_enabled, + vcr_config_dict, +) + +_verbose_state = VerboseReporterState() @pytest.fixture(scope="function", autouse=True) @@ -17,21 +28,17 @@ def setup_and_teardown(): """ This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. """ - curr_dir = os.getcwd() # Get the current working directory sys.path.insert( 0, os.path.abspath("../..") ) # Adds the project directory to the system path import litellm - from litellm import Router importlib.reload(litellm) - import asyncio loop = asyncio.get_event_loop_policy().new_event_loop() asyncio.set_event_loop(loop) print(litellm) - # from litellm import Router, completion, aembedding, acompletion, embedding yield # Teardown code (executes after the yield point) @@ -39,7 +46,39 @@ def setup_and_teardown(): asyncio.set_event_loop(None) # Remove the reference to the loop +@pytest.fixture(scope="module") +def vcr_config(): + return vcr_config_dict() + + +def pytest_recording_configure(config, vcr): + register_persister_if_enabled(vcr) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + rep = outcome.get_result() + setattr(item, f"rep_{rep.when}", rep) + + +@pytest.fixture(autouse=True) +def _vcr_outcome_gate(request, vcr): + yield + record_vcr_outcome(request, vcr) + + +def pytest_configure(config): + _verbose_state.remember_pluginmanager(config) + + +def pytest_runtest_logreport(report): + _verbose_state.maybe_emit_verdict(report) + + def pytest_collection_modifyitems(config, items): + apply_vcr_auto_marker_to_items(items) + # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests custom_logger_tests = [ item for item in items if "custom_logger" in item.parent.name diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py index 80f36e159a9c..e16d3cb4a3f5 100644 --- a/tests/llm_responses_api_testing/conftest.py +++ b/tests/llm_responses_api_testing/conftest.py @@ -13,97 +13,24 @@ import litellm # noqa: E402 -from tests._vcr_redis_persister import ( # noqa: E402 - filter_non_2xx_response, - format_vcr_verdict, - make_redis_persister, - mark_test_outcome_for_cassette, - patch_vcrpy_aiohttp_record_path, - vcr_verbose_enabled, +from tests._vcr_conftest_common import ( # noqa: E402 + VerboseReporterState, + apply_vcr_auto_marker_to_items, + record_vcr_outcome, + register_persister_if_enabled, + vcr_config_dict, ) - -_controller_pluginmanager = None -_controller_terminal_reporter = None - - -_FILTERED_REQUEST_HEADERS = ( - "authorization", - "x-api-key", - "anthropic-api-key", - "anthropic-version", - "openai-api-key", - "azure-api-key", - "api-key", - "cookie", - "x-amz-security-token", - "x-amz-date", - "x-amz-content-sha256", - "amz-sdk-invocation-id", - "amz-sdk-request", - "x-goog-api-key", - "x-goog-user-project", -) - -_FILTERED_RESPONSE_HEADERS = ( - "set-cookie", - "x-request-id", - "request-id", - "cf-ray", - "anthropic-organization-id", - "openai-organization", - "x-amzn-requestid", - "x-amzn-trace-id", - "date", -) - - -def _scrub_response(response): - if not isinstance(response, dict): - return response - headers = response.get("headers") or {} - if isinstance(headers, dict): - for header in list(headers): - if header.lower() in _FILTERED_RESPONSE_HEADERS: - headers.pop(header, None) - return response - - -def _before_record_response(response): - return filter_non_2xx_response(_scrub_response(response)) +_verbose_state = VerboseReporterState() @pytest.fixture(scope="module") def vcr_config(): - return { - "filter_headers": list(_FILTERED_REQUEST_HEADERS), - "decode_compressed_response": True, - "record_mode": "new_episodes", - "allow_playback_repeats": True, - "match_on": ( - "method", - "scheme", - "host", - "port", - "path", - "query", - "body", - ), - "before_record_response": _before_record_response, - } - - -def _vcr_disabled() -> bool: - if os.environ.get("LITELLM_VCR_DISABLE") == "1": - return True - return not os.environ.get("CASSETTE_REDIS_URL") + return vcr_config_dict() def pytest_recording_configure(config, vcr): - if _vcr_disabled(): - return - vcr.register_persister(make_redis_persister()) - patch_vcrpy_aiohttp_record_path() + register_persister_if_enabled(vcr) @pytest.hookimpl(hookwrapper=True) @@ -116,55 +43,15 @@ def pytest_runtest_makereport(item, call): @pytest.fixture(autouse=True) def _vcr_outcome_gate(request, vcr): yield - cassette = vcr - rep_call = getattr(request.node, "rep_call", None) - test_passed = bool(rep_call and rep_call.passed) - cassette_path = getattr(cassette, "_path", None) if cassette is not None else None - if cassette_path: - mark_test_outcome_for_cassette(cassette_path, test_passed) - - if not vcr_verbose_enabled(): - return - verdict = format_vcr_verdict(cassette) - request.node.user_properties.append(("vcr_verdict", verdict)) + record_vcr_outcome(request, vcr) def pytest_configure(config): - global _controller_pluginmanager - if os.environ.get("PYTEST_XDIST_WORKER"): - return - _controller_pluginmanager = config.pluginmanager - - -def _resolve_terminal_reporter(): - global _controller_terminal_reporter - if _controller_terminal_reporter is not None: - return _controller_terminal_reporter - if _controller_pluginmanager is None: - return None - _controller_terminal_reporter = _controller_pluginmanager.getplugin( - "terminalreporter" - ) - return _controller_terminal_reporter + _verbose_state.remember_pluginmanager(config) def pytest_runtest_logreport(report): - if report.when != "teardown": - return - if os.environ.get("PYTEST_XDIST_WORKER"): - return - if not vcr_verbose_enabled(): - return - reporter = _resolve_terminal_reporter() - if reporter is None: - return - verdict = next( - (v for k, v in (report.user_properties or []) if k == "vcr_verdict"), - None, - ) - if not verdict: - return - reporter.write_line(f"{verdict} :: {report.nodeid}") + _verbose_state.maybe_emit_verdict(report) @pytest.fixture(scope="session") @@ -182,13 +69,11 @@ def setup_and_teardown(): """ This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. """ - curr_dir = os.getcwd() # Get the current working directory sys.path.insert( 0, os.path.abspath("../..") ) # Adds the project directory to the system path import litellm - from litellm import Router importlib.reload(litellm) @@ -200,12 +85,9 @@ def setup_and_teardown(): except Exception as e: print(f"Error reloading litellm.proxy.proxy_server: {e}") - import asyncio - loop = asyncio.get_event_loop_policy().new_event_loop() asyncio.set_event_loop(loop) print(litellm) - # from litellm import Router, completion, aembedding, acompletion, embedding yield # Teardown code (executes after the yield point) @@ -214,11 +96,7 @@ def setup_and_teardown(): def pytest_collection_modifyitems(config, items): - if not _vcr_disabled(): - for item in items: - if item.get_closest_marker("vcr") is not None: - continue - item.add_marker(pytest.mark.vcr) + apply_vcr_auto_marker_to_items(items) custom_logger_tests = [ item for item in items if "custom_logger" in item.parent.name diff --git a/tests/llm_translation/Readme.md b/tests/llm_translation/Readme.md index 958adbd97559..813c188ee7ba 100644 --- a/tests/llm_translation/Readme.md +++ b/tests/llm_translation/Readme.md @@ -16,12 +16,38 @@ The persister, header scrubbing, and 2xx-only filtering are defined in patches the same httpx transport vcrpy does) are excluded from the auto-marker — see `_RESPX_CONFLICTING_FILES` in `conftest.py`. +The same VCR cache is used by other test directories that exercise live +provider APIs. The reusable conftest plumbing lives in +`tests/_vcr_conftest_common.py` and is wired into: + +- `tests/llm_translation/` +- `tests/llm_responses_api_testing/` +- `tests/audio_tests/` +- `tests/batches_tests/` +- `tests/guardrails_tests/` +- `tests/image_gen_tests/` +- `tests/litellm_utils_tests/` +- `tests/local_testing/` (covers `local_testing_part1`, `local_testing_part2`, + `litellm_router_testing`, `litellm_assistants_api_testing`, + `langfuse_logging_unit_tests`) +- `tests/logging_callback_tests/` +- `tests/pass_through_unit_tests/` +- `tests/router_unit_tests/` +- `tests/unified_google_tests/` + +Test directories that run LiteLLM proxy in Docker (e.g. `build_and_test`, +`proxy_logging_guardrails_model_info_tests`, `proxy_store_model_in_db_tests`) +are intentionally not included: VCR.py patches the in-process httpx +transport, so it cannot intercept the LLM calls that originate inside the +Docker container. + ### Required environment -`REDIS_HOST`, `REDIS_PORT`, `REDIS_PASSWORD` — same vars CircleCI uses for -its other Redis-backed jobs. Provider credentials -(`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `AWS_*`, etc.) are needed only on -cache-miss (the daily re-record), not on replay. +`CASSETTE_REDIS_URL` — separate Redis instance from the application +Redis (`REDIS_URL`/`REDIS_HOST`) so test cassettes are not flushed by +proxy tests. Provider credentials (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, +`AWS_*`, etc.) are needed only on cache-miss (the daily re-record), not +on replay. ### Flushing the cache diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index 09da0520be07..255aca02d12e 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -18,20 +18,14 @@ import litellm # noqa: E402 -from tests._vcr_redis_persister import ( # noqa: E402 - filter_non_2xx_response, - format_vcr_verdict, - make_redis_persister, - mark_test_outcome_for_cassette, - patch_vcrpy_aiohttp_record_path, - vcr_verbose_enabled, +from tests._vcr_conftest_common import ( # noqa: E402 + VerboseReporterState, + apply_vcr_auto_marker_to_items, + record_vcr_outcome, + register_persister_if_enabled, + vcr_config_dict, ) - -_controller_pluginmanager = None -_controller_terminal_reporter = None - - # vcrpy and respx both patch the httpx transport — applying both makes one # silently win, so respx-using files opt out of the auto-marker. _RESPX_CONFLICTING_FILES = frozenset( @@ -52,96 +46,23 @@ # Tests that observe live cross-call provider state (e.g. prompt-cache # warm-up between two consecutive calls); replay can't reproduce that state. -_VCR_INCOMPATIBLE_NODEID_SUFFIXES = frozenset( - { - "::test_prompt_caching", - "TestBedrockInvokeNovaJson::test_json_response_pydantic_obj", - "::test_bedrock_converse__streaming_passthrough", - } -) - - -def _is_vcr_incompatible(nodeid: str) -> bool: - return any(nodeid.endswith(suffix) for suffix in _VCR_INCOMPATIBLE_NODEID_SUFFIXES) - - -_FILTERED_REQUEST_HEADERS = ( - "authorization", - "x-api-key", - "anthropic-api-key", - "anthropic-version", - "openai-api-key", - "azure-api-key", - "api-key", - "cookie", - "x-amz-security-token", - "x-amz-date", - "x-amz-content-sha256", - "amz-sdk-invocation-id", - "amz-sdk-request", - "x-goog-api-key", - "x-goog-user-project", -) - -_FILTERED_RESPONSE_HEADERS = ( - "set-cookie", - "x-request-id", - "request-id", - "cf-ray", - "anthropic-organization-id", - "openai-organization", - "x-amzn-requestid", - "x-amzn-trace-id", - "date", +_VCR_INCOMPATIBLE_NODEID_SUFFIXES = ( + "::test_prompt_caching", + "TestBedrockInvokeNovaJson::test_json_response_pydantic_obj", + "::test_bedrock_converse__streaming_passthrough", ) -def _scrub_response(response): - if not isinstance(response, dict): - return response - headers = response.get("headers") or {} - if isinstance(headers, dict): - for header in list(headers): - if header.lower() in _FILTERED_RESPONSE_HEADERS: - headers.pop(header, None) - return response - - -def _before_record_response(response): - return filter_non_2xx_response(_scrub_response(response)) +_verbose_state = VerboseReporterState() @pytest.fixture(scope="module") def vcr_config(): - return { - "filter_headers": list(_FILTERED_REQUEST_HEADERS), - "decode_compressed_response": True, - "record_mode": "new_episodes", - "allow_playback_repeats": True, - "match_on": ( - "method", - "scheme", - "host", - "port", - "path", - "query", - "body", - ), - "before_record_response": _before_record_response, - } - - -def _vcr_disabled() -> bool: - if os.environ.get("LITELLM_VCR_DISABLE") == "1": - return True - return not os.environ.get("CASSETTE_REDIS_URL") + return vcr_config_dict() def pytest_recording_configure(config, vcr): - if _vcr_disabled(): - return - vcr.register_persister(make_redis_persister()) - patch_vcrpy_aiohttp_record_path() + register_persister_if_enabled(vcr) @pytest.hookimpl(hookwrapper=True) @@ -154,55 +75,15 @@ def pytest_runtest_makereport(item, call): @pytest.fixture(autouse=True) def _vcr_outcome_gate(request, vcr): yield - cassette = vcr - rep_call = getattr(request.node, "rep_call", None) - test_passed = bool(rep_call and rep_call.passed) - cassette_path = getattr(cassette, "_path", None) if cassette is not None else None - if cassette_path: - mark_test_outcome_for_cassette(cassette_path, test_passed) - - if not vcr_verbose_enabled(): - return - verdict = format_vcr_verdict(cassette) - request.node.user_properties.append(("vcr_verdict", verdict)) + record_vcr_outcome(request, vcr) def pytest_configure(config): - global _controller_pluginmanager - if os.environ.get("PYTEST_XDIST_WORKER"): - return - _controller_pluginmanager = config.pluginmanager - - -def _resolve_terminal_reporter(): - global _controller_terminal_reporter - if _controller_terminal_reporter is not None: - return _controller_terminal_reporter - if _controller_pluginmanager is None: - return None - _controller_terminal_reporter = _controller_pluginmanager.getplugin( - "terminalreporter" - ) - return _controller_terminal_reporter + _verbose_state.remember_pluginmanager(config) def pytest_runtest_logreport(report): - if report.when != "teardown": - return - if os.environ.get("PYTEST_XDIST_WORKER"): - return - if not vcr_verbose_enabled(): - return - reporter = _resolve_terminal_reporter() - if reporter is None: - return - verdict = next( - (v for k, v in (report.user_properties or []) if k == "vcr_verdict"), - None, - ) - if not verdict: - return - reporter.write_line(f"{verdict} :: {report.nodeid}") + _verbose_state.maybe_emit_verdict(report) # --------------------------------------------------------------------------- @@ -283,16 +164,11 @@ def setup_and_teardown(event_loop): # Add event_loop as a dependency def pytest_collection_modifyitems(config, items): - if not _vcr_disabled(): - for item in items: - filename = os.path.basename(str(item.fspath)) - if filename in _VCR_AUTO_MARKER_SKIP_FILES: - continue - if _is_vcr_incompatible(item.nodeid): - continue - if item.get_closest_marker("vcr") is not None: - continue - item.add_marker(pytest.mark.vcr) + apply_vcr_auto_marker_to_items( + items, + skip_files=_VCR_AUTO_MARKER_SKIP_FILES, + skip_nodeid_suffixes=_VCR_INCOMPATIBLE_NODEID_SUFFIXES, + ) custom_logger_tests = [ item for item in items if "custom_logger" in item.parent.name diff --git a/tests/local_testing/conftest.py b/tests/local_testing/conftest.py index 94d4f135a56c..eff88fe73c65 100644 --- a/tests/local_testing/conftest.py +++ b/tests/local_testing/conftest.py @@ -22,6 +22,58 @@ ) # Adds the parent directory to the system path import litellm +from tests._vcr_conftest_common import ( # noqa: E402 + VerboseReporterState, + apply_vcr_auto_marker_to_items, + record_vcr_outcome, + register_persister_if_enabled, + vcr_config_dict, +) + +# vcrpy and respx both patch the httpx transport — applying both makes one +# silently win, so respx-using files opt out of the auto-marker. +_RESPX_CONFLICTING_FILES = frozenset( + { + "test_router.py", + "test_amazing_vertex_completion.py", + "test_azure_openai.py", + } +) + + +_verbose_state = VerboseReporterState() + + +@pytest.fixture(scope="module") +def vcr_config(): + return vcr_config_dict() + + +def pytest_recording_configure(config, vcr): + register_persister_if_enabled(vcr) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + rep = outcome.get_result() + setattr(item, f"rep_{rep.when}", rep) + + +@pytest.fixture(autouse=True) +def _vcr_outcome_gate(request, vcr): + yield + record_vcr_outcome(request, vcr) + + +def pytest_configure(config): + _verbose_state.remember_pluginmanager(config) + + +def pytest_runtest_logreport(report): + _verbose_state.maybe_emit_verdict(report) + + # --------------------------------------------------------------------------- # Capture TRUE defaults at conftest import time. This runs before any test # module's top-level code (e.g. `litellm.num_retries = 3`) executes, so @@ -147,6 +199,11 @@ def setup_and_teardown(): def pytest_collection_modifyitems(config, items): + apply_vcr_auto_marker_to_items( + items, + skip_files=_RESPX_CONFLICTING_FILES, + ) + # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests custom_logger_tests = [ item for item in items if "custom_logger" in item.parent.name diff --git a/tests/logging_callback_tests/conftest.py b/tests/logging_callback_tests/conftest.py index 7100c8456a97..aa63c1ee9002 100644 --- a/tests/logging_callback_tests/conftest.py +++ b/tests/logging_callback_tests/conftest.py @@ -19,6 +19,56 @@ ) # Adds the parent directory to the system path import litellm +from tests._vcr_conftest_common import ( # noqa: E402 + VerboseReporterState, + apply_vcr_auto_marker_to_items, + record_vcr_outcome, + register_persister_if_enabled, + vcr_config_dict, +) + +# vcrpy and respx both patch the httpx transport — applying both makes one +# silently win, so respx-using files opt out of the auto-marker. +_RESPX_CONFLICTING_FILES = frozenset( + { + "test_assemble_streaming_responses.py", + "test_langfuse_unit_tests.py", + } +) + + +_verbose_state = VerboseReporterState() + + +@pytest.fixture(scope="module") +def vcr_config(): + return vcr_config_dict() + + +def pytest_recording_configure(config, vcr): + register_persister_if_enabled(vcr) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + rep = outcome.get_result() + setattr(item, f"rep_{rep.when}", rep) + + +@pytest.fixture(autouse=True) +def _vcr_outcome_gate(request, vcr): + yield + record_vcr_outcome(request, vcr) + + +def pytest_configure(config): + _verbose_state.remember_pluginmanager(config) + + +def pytest_runtest_logreport(report): + _verbose_state.maybe_emit_verdict(report) + _LIST_ATTRS = ( "callbacks", @@ -137,6 +187,11 @@ def setup_and_teardown(): def pytest_collection_modifyitems(config, items): + apply_vcr_auto_marker_to_items( + items, + skip_files=_RESPX_CONFLICTING_FILES, + ) + # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests custom_logger_tests = [ item for item in items if "custom_logger" in item.parent.name diff --git a/tests/pass_through_unit_tests/conftest.py b/tests/pass_through_unit_tests/conftest.py new file mode 100644 index 000000000000..617d06517362 --- /dev/null +++ b/tests/pass_through_unit_tests/conftest.py @@ -0,0 +1,56 @@ +# conftest.py +# +# Wires pass-through unit tests into the Redis-backed VCR cache so live +# provider calls are replayed for 24h. See tests/llm_translation/Readme.md +# for the design overview. + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +from tests._vcr_conftest_common import ( # noqa: E402 + VerboseReporterState, + apply_vcr_auto_marker_to_items, + record_vcr_outcome, + register_persister_if_enabled, + vcr_config_dict, +) + +_verbose_state = VerboseReporterState() + + +@pytest.fixture(scope="module") +def vcr_config(): + return vcr_config_dict() + + +def pytest_recording_configure(config, vcr): + register_persister_if_enabled(vcr) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + rep = outcome.get_result() + setattr(item, f"rep_{rep.when}", rep) + + +@pytest.fixture(autouse=True) +def _vcr_outcome_gate(request, vcr): + yield + record_vcr_outcome(request, vcr) + + +def pytest_configure(config): + _verbose_state.remember_pluginmanager(config) + + +def pytest_runtest_logreport(report): + _verbose_state.maybe_emit_verdict(report) + + +def pytest_collection_modifyitems(config, items): + apply_vcr_auto_marker_to_items(items) diff --git a/tests/router_unit_tests/conftest.py b/tests/router_unit_tests/conftest.py index 6e331d3a4c0f..a210244b3df8 100644 --- a/tests/router_unit_tests/conftest.py +++ b/tests/router_unit_tests/conftest.py @@ -1,5 +1,6 @@ # conftest.py +import asyncio import importlib import os import sys @@ -9,9 +10,17 @@ sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path -import litellm +import litellm # noqa: E402,F401 -import asyncio +from tests._vcr_conftest_common import ( # noqa: E402 + VerboseReporterState, + apply_vcr_auto_marker_to_items, + record_vcr_outcome, + register_persister_if_enabled, + vcr_config_dict, +) + +_verbose_state = VerboseReporterState() @pytest.fixture(scope="session") @@ -29,14 +38,11 @@ def setup_and_teardown(): """ This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. """ - curr_dir = os.getcwd() # Get the current working directory sys.path.insert( 0, os.path.abspath("../..") ) # Adds the project directory to the system path import litellm - from litellm import Router - import asyncio from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER @@ -53,12 +59,9 @@ def setup_and_teardown(): except Exception as e: print(f"Error reloading litellm.proxy.proxy_server: {e}") - import asyncio - loop = asyncio.get_event_loop_policy().new_event_loop() asyncio.set_event_loop(loop) print(litellm) - # from litellm import Router, completion, aembedding, acompletion, embedding yield # Teardown code (executes after the yield point) @@ -66,7 +69,39 @@ def setup_and_teardown(): asyncio.set_event_loop(None) # Remove the reference to the loop +@pytest.fixture(scope="module") +def vcr_config(): + return vcr_config_dict() + + +def pytest_recording_configure(config, vcr): + register_persister_if_enabled(vcr) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + rep = outcome.get_result() + setattr(item, f"rep_{rep.when}", rep) + + +@pytest.fixture(autouse=True) +def _vcr_outcome_gate(request, vcr): + yield + record_vcr_outcome(request, vcr) + + +def pytest_configure(config): + _verbose_state.remember_pluginmanager(config) + + +def pytest_runtest_logreport(report): + _verbose_state.maybe_emit_verdict(report) + + def pytest_collection_modifyitems(config, items): + apply_vcr_auto_marker_to_items(items) + # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests custom_logger_tests = [ item for item in items if "custom_logger" in item.parent.name diff --git a/tests/unified_google_tests/conftest.py b/tests/unified_google_tests/conftest.py index 01d5f69974ed..bae5769ad3cf 100644 --- a/tests/unified_google_tests/conftest.py +++ b/tests/unified_google_tests/conftest.py @@ -1,5 +1,6 @@ # conftest.py +import asyncio import importlib import os import sys @@ -9,8 +10,17 @@ sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path -import litellm -import asyncio +import litellm # noqa: E402,F401 + +from tests._vcr_conftest_common import ( # noqa: E402 + VerboseReporterState, + apply_vcr_auto_marker_to_items, + record_vcr_outcome, + register_persister_if_enabled, + vcr_config_dict, +) + +_verbose_state = VerboseReporterState() @pytest.fixture(scope="session") @@ -28,21 +38,17 @@ def setup_and_teardown(): """ This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. """ - curr_dir = os.getcwd() # Get the current working directory sys.path.insert( 0, os.path.abspath("../..") ) # Adds the project directory to the system path import litellm - from litellm import Router importlib.reload(litellm) - import asyncio loop = asyncio.get_event_loop_policy().new_event_loop() asyncio.set_event_loop(loop) print(litellm) - # from litellm import Router, completion, aembedding, acompletion, embedding yield # Teardown code (executes after the yield point) @@ -50,7 +56,39 @@ def setup_and_teardown(): asyncio.set_event_loop(None) # Remove the reference to the loop +@pytest.fixture(scope="module") +def vcr_config(): + return vcr_config_dict() + + +def pytest_recording_configure(config, vcr): + register_persister_if_enabled(vcr) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + rep = outcome.get_result() + setattr(item, f"rep_{rep.when}", rep) + + +@pytest.fixture(autouse=True) +def _vcr_outcome_gate(request, vcr): + yield + record_vcr_outcome(request, vcr) + + +def pytest_configure(config): + _verbose_state.remember_pluginmanager(config) + + +def pytest_runtest_logreport(report): + _verbose_state.maybe_emit_verdict(report) + + def pytest_collection_modifyitems(config, items): + apply_vcr_auto_marker_to_items(items) + # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests custom_logger_tests = [ item for item in items if "custom_logger" in item.parent.name From dfe1dc258c02a72e7db53a943c916616d4ebe399 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 5 May 2026 03:53:46 +0000 Subject: [PATCH 02/12] test(vcr): add safe-body matcher to handle JSONL and binary request bodies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vcrpy's stock body matcher inspects Content-Type and unconditionally runs json.loads on application/json bodies. JSON Lines payloads (used by the Bedrock batch S3 PUT and other upload paths) crash that with json.JSONDecodeError: Extra data, before the matcher can return 'not a match'. This was the root cause of the batches_testing CI job failing on test_async_create_file once VCR auto-marking was applied to the batches_tests directory. Add a conservative byte-equality body matcher and use it in place of 'body' in the shared match_on tuple. The matcher is strictly more conservative than vcrpy's default — the only thing it gives up is 'different JSON key order is treated as the same body', which doesn't apply to deterministic litellm-built request payloads. It can never produce a false positive that the default would have rejected, so there is no cross-contamination risk. Co-authored-by: Mateo Wang --- tests/_vcr_conftest_common.py | 54 ++++++++- .../test_vcr_safe_body_matcher.py | 105 ++++++++++++++++++ 2 files changed, 157 insertions(+), 2 deletions(-) create mode 100644 tests/test_litellm/test_vcr_safe_body_matcher.py diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index 6baf618a3167..d6b7a74fb120 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -36,6 +36,9 @@ vcr_verbose_enabled, ) + +SAFE_BODY_MATCHER_NAME = "safe_body" + FILTERED_REQUEST_HEADERS = ( "authorization", "x-api-key", @@ -82,6 +85,51 @@ def _before_record_response(response): return filter_non_2xx_response(_scrub_response(response)) +def _safe_body_matcher(r1, r2) -> None: + """Body matcher that compares raw bytes and never raises on bad JSON. + + vcrpy's stock ``body`` matcher inspects ``Content-Type`` and runs + ``json.loads`` on bodies typed ``application/json`` so it can compare + semantically. That crashes (``json.JSONDecodeError: Extra data``) on + JSON Lines payloads — which the Bedrock batch S3 PUT and a few other + upload paths use — before the matcher even gets a chance to return + "not a match". + + This matcher avoids the JSON normalization step entirely and just + compares the request bodies as bytes, falling back to repr equality + for non-bytes/str payloads. It is strictly more conservative than + vcrpy's default — the only thing it gives up is "different JSON key + order is treated as the same body", which doesn't matter for our + deterministic litellm-built request payloads. It can never produce a + false positive that the default would have rejected. + + The trade-off is that bodies containing nondeterministic values (UUIDs, + timestamps) will produce a cache miss; the right fix for those cases + is a ``before_record_request`` scrubber, not a smarter matcher. + """ + body1 = getattr(r1, "body", None) + body2 = getattr(r2, "body", None) + if body1 == body2: + return + + def _to_bytes(b): + if b is None: + return b"" + if isinstance(b, bytes): + return b + if isinstance(b, str): + return b.encode("utf-8") + return None + + n1 = _to_bytes(body1) + n2 = _to_bytes(body2) + if n1 is not None and n2 is not None: + if n1 == n2: + return + raise AssertionError("request bodies differ") + raise AssertionError("request bodies differ") + + def vcr_config_dict() -> dict: """Return the VCR config dict shared across all consuming conftests.""" return { @@ -96,7 +144,7 @@ def vcr_config_dict() -> dict: "port", "path", "query", - "body", + SAFE_BODY_MATCHER_NAME, ), "before_record_response": _before_record_response, } @@ -110,13 +158,15 @@ def vcr_disabled() -> bool: def register_persister_if_enabled(vcr) -> None: - """Wire the Redis persister into vcrpy if VCR is enabled. + """Wire the Redis persister and custom matchers into vcrpy if VCR is + enabled. Call this from ``pytest_recording_configure(config, vcr)`` in conftest. """ if vcr_disabled(): return vcr.register_persister(make_redis_persister()) + vcr.register_matcher(SAFE_BODY_MATCHER_NAME, _safe_body_matcher) patch_vcrpy_aiohttp_record_path() diff --git a/tests/test_litellm/test_vcr_safe_body_matcher.py b/tests/test_litellm/test_vcr_safe_body_matcher.py new file mode 100644 index 000000000000..046b3c45339a --- /dev/null +++ b/tests/test_litellm/test_vcr_safe_body_matcher.py @@ -0,0 +1,105 @@ +"""Unit tests for the shared VCR helpers in ``tests/_vcr_conftest_common``. + +The most important guarantee here is that the custom ``safe_body`` matcher +gracefully handles JSON Lines (and other non-strict-JSON) request bodies +without raising ``json.JSONDecodeError`` — vcrpy's default ``body`` matcher +crashes on those because it unconditionally runs ``json.loads`` for any +``application/json`` request body. +""" + +from __future__ import annotations + +import os +import sys +from types import SimpleNamespace + +import pytest + +# Tests live in ``tests/test_litellm/`` but ``_vcr_conftest_common`` lives in +# the parent ``tests/`` package. Make sure both are importable regardless of +# how pytest is invoked. +_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if _REPO_ROOT not in sys.path: + sys.path.insert(0, _REPO_ROOT) + +from tests._vcr_conftest_common import ( # noqa: E402 + SAFE_BODY_MATCHER_NAME, + _safe_body_matcher, + vcr_config_dict, +) + + +def _req(body): + return SimpleNamespace(body=body, headers={"Content-Type": "application/json"}) + + +def test_safe_body_matcher_is_in_match_on(): + cfg = vcr_config_dict() + assert SAFE_BODY_MATCHER_NAME in cfg["match_on"] + assert "body" not in cfg["match_on"] + + +def test_safe_body_matcher_accepts_identical_bytes(): + _safe_body_matcher(_req(b"hello"), _req(b"hello")) + + +def test_safe_body_matcher_accepts_str_bytes_equivalent(): + _safe_body_matcher(_req("hello"), _req(b"hello")) + + +def test_safe_body_matcher_handles_jsonl_without_crashing(): + """vcrpy's default ``body`` matcher raises ``JSONDecodeError`` on JSONL. + + The Bedrock batch S3 PUT sends a JSON Lines body under + ``Content-Type: application/json``. The safe matcher must compare such + bodies as bytes and never invoke ``json.loads``. + """ + jsonl = ( + b'{"recordId": "request-1", "modelInput": {}}\n' + b'{"recordId": "request-2", "modelInput": {}}\n' + ) + _safe_body_matcher(_req(jsonl), _req(jsonl)) + + +def test_safe_body_matcher_rejects_different_jsonl_bodies(): + a = b'{"recordId": "request-1"}\n{"recordId": "request-2"}\n' + b = b'{"recordId": "request-1"}\n{"recordId": "request-3"}\n' + with pytest.raises(AssertionError): + _safe_body_matcher(_req(a), _req(b)) + + +def test_safe_body_matcher_rejects_different_bytes(): + with pytest.raises(AssertionError): + _safe_body_matcher(_req(b"a"), _req(b"b")) + + +def test_safe_body_matcher_treats_none_bodies_as_equal(): + _safe_body_matcher(_req(None), _req(None)) + + +def test_safe_body_matcher_does_not_normalize_json_key_order(): + """The safe matcher is strictly more conservative than vcrpy's default. + + Two semantically-equal JSON bodies with different key order are + treated as *different* requests (cache miss, never a false hit). + """ + with pytest.raises(AssertionError): + _safe_body_matcher(_req(b'{"a":1,"b":2}'), _req(b'{"b":2,"a":1}')) + + +def test_default_vcrpy_body_matcher_crashes_on_jsonl_for_documentation(): + """Document the behavior we are working around. + + vcrpy's stock body matcher raises ``json.JSONDecodeError`` (not even + a clean ``AssertionError``) when given a JSONL payload typed as + ``application/json``. This is precisely the crash that broke + ``tests/batches_tests/test_bedrock_files_and_batches.py::test_async_create_file`` + and is the reason ``safe_body`` exists. + """ + import json + + from vcr.matchers import body as vcrpy_body # type: ignore + + jsonl = b'{"recordId": "request-1"}\n{"recordId": "request-2"}\n' + with pytest.raises(json.JSONDecodeError): + vcrpy_body(_req(jsonl), _req(jsonl)) From 4de86118a10651fdb58dc193d41501aa868eab73 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 5 May 2026 03:53:59 +0000 Subject: [PATCH 03/12] test(vcr): exclude tests that VCR replay actively breaks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A few tests are incompatible with cassette replay and were failing on the latest CI run after VCR auto-marking was extended to local_testing and logging_callback_tests: - test_amazing_s3_logs.py (logging_callback_tests): the test asserts on a per-run response_id that should round-trip through a real S3 PUT/LIST. vcrpy's boto3 stub intercepts the PUT and the LIST replays stale keys, so the freshly-generated id is never found. - test_async_embedding_azure (logging_callback_tests) and test_amazing_sync_embedding (local_testing): the failure branches deliberately pass api_key='my-bad-key' to assert that the failure callback fires. We scrub auth headers from cassettes (so the bad-key request matches the prior good-key request), and vcrpy replays the recorded 200 — the failure callback never fires. - test_assistants.py (local_testing): the OpenAI Assistants polling APIs mint fresh thread/run IDs every recording session and then poll until status=='completed'. Replays of those polled GETs can never match a freshly-generated run id, so every CI run effectively re-records and the suite blows past the 15m no_output_timeout. Skip these from VCR auto-marking so they continue to hit live providers as they did before this change. The remaining tests in each directory still get cached. Co-authored-by: Mateo Wang --- tests/local_testing/conftest.py | 26 +++++++++++++++++++++++- tests/logging_callback_tests/conftest.py | 23 ++++++++++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/tests/local_testing/conftest.py b/tests/local_testing/conftest.py index eff88fe73c65..ba5ea5b03f45 100644 --- a/tests/local_testing/conftest.py +++ b/tests/local_testing/conftest.py @@ -40,6 +40,29 @@ } ) +# Files where VCR replay actively breaks the test: +# - ``test_assistants.py`` exercises the OpenAI Assistants polling APIs +# which mint fresh thread/run/message IDs every recording session and +# then poll until ``status == "completed"``. Replays of those polled +# GETs would have to match the new run id (impossible) or be played +# back in lockstep with a freshly recorded creation, neither of which +# ``record_mode="new_episodes"`` does well. The result in CI is that +# every run effectively re-records, blowing past the 15-minute step +# timeout for ``litellm_assistants_api_testing``. +_VCR_INCOMPATIBLE_FILES = frozenset( + { + "test_assistants.py", + } +) + +# Specific tests where VCR replay actively breaks the test: +# - ``test_amazing_sync_embedding`` deliberately calls the embedding API +# with ``api_key="my-bad-key"`` to assert the failure callback fires. +# We scrub auth headers from cassettes (so the bad-key request matches +# the prior good-key request), and vcrpy replays the recorded 200 — so +# the failure callback never fires and the assertion flips. +_VCR_INCOMPATIBLE_NODEID_SUFFIXES = ("::test_amazing_sync_embedding",) + _verbose_state = VerboseReporterState() @@ -201,7 +224,8 @@ def setup_and_teardown(): def pytest_collection_modifyitems(config, items): apply_vcr_auto_marker_to_items( items, - skip_files=_RESPX_CONFLICTING_FILES, + skip_files=_RESPX_CONFLICTING_FILES | _VCR_INCOMPATIBLE_FILES, + skip_nodeid_suffixes=_VCR_INCOMPATIBLE_NODEID_SUFFIXES, ) # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests diff --git a/tests/logging_callback_tests/conftest.py b/tests/logging_callback_tests/conftest.py index aa63c1ee9002..8e0fe9dd6b5e 100644 --- a/tests/logging_callback_tests/conftest.py +++ b/tests/logging_callback_tests/conftest.py @@ -36,6 +36,26 @@ } ) +# Files where VCR replay actively breaks the test: +# - ``test_amazing_s3_logs.py`` exercises the S3 success callback using +# ``mock_response`` (so there is no upstream LLM call worth caching) and +# asserts on a per-run ``response_id`` round-tripped through a real S3 +# PUT/LIST. vcrpy's boto3 stub intercepts the PUT and replays a stale LIST, +# so the freshly-generated id is never found in the cached keys. +_VCR_INCOMPATIBLE_FILES = frozenset( + { + "test_amazing_s3_logs.py", + } +) + +# Specific tests where VCR replay actively breaks the test: +# - The "failure" branches of these callback tests deliberately pass a bad +# API key to assert that the ``async_failure`` / ``failure`` callback fires. +# We scrub auth headers from cassettes (so the bad-key request matches the +# prior good-key request), and vcrpy replays the recorded 200 — so the +# failure callback never fires and the assertion flips. +_VCR_INCOMPATIBLE_NODEID_SUFFIXES = ("::test_async_embedding_azure",) + _verbose_state = VerboseReporterState() @@ -189,7 +209,8 @@ def setup_and_teardown(): def pytest_collection_modifyitems(config, items): apply_vcr_auto_marker_to_items( items, - skip_files=_RESPX_CONFLICTING_FILES, + skip_files=_RESPX_CONFLICTING_FILES | _VCR_INCOMPATIBLE_FILES, + skip_nodeid_suffixes=_VCR_INCOMPATIBLE_NODEID_SUFFIXES, ) # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests From 95c6eaaca203cd0a8bd172e3146c13f3af4ef95b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 5 May 2026 04:10:49 +0000 Subject: [PATCH 04/12] test(vcr): expand skip lists for second batch of incompatible tests Followup to the previous commit. After re-running CI on the rebuilt branch, three more tests surfaced as VCR-replay-incompatible: - litellm_utils_testing :: test_get_valid_models_from_dynamic_api_key Calls GET /v1/models with api_key='123' to assert the result is empty. We scrub auth headers, so the bad-key request matches the prior good-key cassette and replays the recorded model list. - litellm_utils_testing :: test_litellm_overhead.py Measures litellm_overhead_time_ms as a percentage of total wall-clock time. With cached responses the upstream 'network' time collapses to microseconds, blowing past the 40%% threshold the test asserts on. Skip the whole file (every parametrization is at risk). - local_testing_part1 :: test_async_custom_handler_completion and test_async_custom_handler_embedding Same bad-key failure-callback pattern as the already-skipped test_amazing_sync_embedding. - litellm_router_testing :: test_router_caching.py Asserts on litellm's own router-level response cache by comparing response1.id to response2.id across repeat upstream calls (test bypasses litellm cache via ttl=0 and expects upstream to return a *new* id). With VCR replay both upstream calls return the same cassette body, so the ids are identical. Skip the whole file. - logging_callback_tests :: test_async_chat_azure (preemptive) Same shape as already-skipped test_async_embedding_azure; was masked by upstream OpenAI rate-limit failures on baseline. Co-authored-by: Mateo Wang --- tests/litellm_utils_tests/conftest.py | 26 +++++++++++++++++++++++- tests/local_testing/conftest.py | 23 +++++++++++++++------ tests/logging_callback_tests/conftest.py | 5 ++++- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/tests/litellm_utils_tests/conftest.py b/tests/litellm_utils_tests/conftest.py index dd21601a2152..9dd69bc980a1 100644 --- a/tests/litellm_utils_tests/conftest.py +++ b/tests/litellm_utils_tests/conftest.py @@ -23,6 +23,26 @@ _verbose_state = VerboseReporterState() +# Files where VCR replay actively breaks the test: +# - ``test_litellm_overhead.py`` measures ``litellm_overhead_time_ms`` as a +# percentage of total wall-clock time. With cached responses the upstream +# "network" time collapses to microseconds, so the overhead percentage +# blows past the 40% threshold the test asserts on. +_VCR_INCOMPATIBLE_FILES = frozenset( + { + "test_litellm_overhead.py", + } +) + +# Specific tests where VCR replay actively breaks the test: +# - ``test_get_valid_models_from_dynamic_api_key`` deliberately calls +# ``GET /v1/models`` with ``api_key="123"`` to assert the result is empty. +# We scrub auth headers from cassettes (so the bad-key request matches +# the prior good-key request), and vcrpy replays the recorded list of +# models — flipping ``len(...) == 0`` to a long list. +_VCR_INCOMPATIBLE_NODEID_SUFFIXES = ("::test_get_valid_models_from_dynamic_api_key",) + + @pytest.fixture(scope="function", autouse=True) def setup_and_teardown(): """ @@ -77,7 +97,11 @@ def pytest_runtest_logreport(report): def pytest_collection_modifyitems(config, items): - apply_vcr_auto_marker_to_items(items) + apply_vcr_auto_marker_to_items( + items, + skip_files=_VCR_INCOMPATIBLE_FILES, + skip_nodeid_suffixes=_VCR_INCOMPATIBLE_NODEID_SUFFIXES, + ) # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests custom_logger_tests = [ diff --git a/tests/local_testing/conftest.py b/tests/local_testing/conftest.py index ba5ea5b03f45..3d6a5f4e1747 100644 --- a/tests/local_testing/conftest.py +++ b/tests/local_testing/conftest.py @@ -49,19 +49,30 @@ # ``record_mode="new_episodes"`` does well. The result in CI is that # every run effectively re-records, blowing past the 15-minute step # timeout for ``litellm_assistants_api_testing``. +# - ``test_router_caching.py`` asserts on litellm's own router-level +# response cache by comparing ``response1.id`` to ``response2.id`` +# across repeat upstream calls (the test bypasses litellm's cache via +# ``ttl=0`` and expects the upstream to return a *new* id each time). +# With VCR replay both upstream calls return the same cassette body, +# so the ids are identical and ``response1.id != response2.id`` flips. _VCR_INCOMPATIBLE_FILES = frozenset( { "test_assistants.py", + "test_router_caching.py", } ) # Specific tests where VCR replay actively breaks the test: -# - ``test_amazing_sync_embedding`` deliberately calls the embedding API -# with ``api_key="my-bad-key"`` to assert the failure callback fires. -# We scrub auth headers from cassettes (so the bad-key request matches -# the prior good-key request), and vcrpy replays the recorded 200 — so -# the failure callback never fires and the assertion flips. -_VCR_INCOMPATIBLE_NODEID_SUFFIXES = ("::test_amazing_sync_embedding",) +# - These tests deliberately call the LLM API with ``api_key="my-bad-key"`` +# to assert that the failure callback fires. We scrub auth headers from +# cassettes (so the bad-key request matches the prior good-key request), +# and vcrpy replays the recorded 200 — so the failure callback never +# fires and the assertion flips. +_VCR_INCOMPATIBLE_NODEID_SUFFIXES = ( + "::test_amazing_sync_embedding", + "::test_async_custom_handler_completion", + "::test_async_custom_handler_embedding", +) _verbose_state = VerboseReporterState() diff --git a/tests/logging_callback_tests/conftest.py b/tests/logging_callback_tests/conftest.py index 8e0fe9dd6b5e..8ab7673eacc0 100644 --- a/tests/logging_callback_tests/conftest.py +++ b/tests/logging_callback_tests/conftest.py @@ -54,7 +54,10 @@ # We scrub auth headers from cassettes (so the bad-key request matches the # prior good-key request), and vcrpy replays the recorded 200 — so the # failure callback never fires and the assertion flips. -_VCR_INCOMPATIBLE_NODEID_SUFFIXES = ("::test_async_embedding_azure",) +_VCR_INCOMPATIBLE_NODEID_SUFFIXES = ( + "::test_async_chat_azure", + "::test_async_embedding_azure", +) _verbose_state = VerboseReporterState() From d9754465463e3b2b9c8e22be3227219ac1c0c828 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 5 May 2026 04:52:20 +0000 Subject: [PATCH 05/12] test(vcr): use item.path and tighten matcher docstring - Replace pytest's deprecated item.fspath with item.path in apply_vcr_auto_marker_to_items so we don't emit deprecation warnings under pytest 8. - Clarify _safe_body_matcher docstring to reflect actual behavior (direct == first, then UTF-8 bytes comparison, no repr fallback). Addresses Greptile review feedback on PR #27159. --- tests/_vcr_conftest_common.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index d6b7a74fb120..f0e2f3da6aba 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -95,13 +95,16 @@ def _safe_body_matcher(r1, r2) -> None: upload paths use — before the matcher even gets a chance to return "not a match". - This matcher avoids the JSON normalization step entirely and just - compares the request bodies as bytes, falling back to repr equality - for non-bytes/str payloads. It is strictly more conservative than - vcrpy's default — the only thing it gives up is "different JSON key - order is treated as the same body", which doesn't matter for our - deterministic litellm-built request payloads. It can never produce a - false positive that the default would have rejected. + This matcher avoids the JSON normalization step entirely. It first + tries direct ``==`` equality on the original payloads (so dicts, + lists, etc. are compared structurally), then falls back to a bytes + comparison after coercing ``str`` to UTF-8. Anything that's not + bytes/str/equal-by-default is treated as a mismatch. This is + strictly more conservative than vcrpy's default — the only thing it + gives up is "different JSON key order is treated as the same body", + which doesn't matter for our deterministic litellm-built request + payloads. It can never produce a false positive that the default + would have rejected. The trade-off is that bodies containing nondeterministic values (UUIDs, timestamps) will produce a cache miss; the right fix for those cases @@ -191,7 +194,7 @@ def apply_vcr_auto_marker_to_items( skip_files = frozenset(skip_files) skip_nodeid_suffixes = tuple(skip_nodeid_suffixes) for item in items: - filename = os.path.basename(str(item.fspath)) + filename = os.path.basename(str(item.path)) if filename in skip_files: continue if any(item.nodeid.endswith(suffix) for suffix in skip_nodeid_suffixes): From f62f147ed6be05269f39b7e0d776ae152d1451a2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 5 May 2026 05:52:03 +0000 Subject: [PATCH 06/12] test(vcr): swallow all RedisError on cassette save/load MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cassette persistence is strictly best-effort: any Redis-side failure (connection blip, timeout, OutOfMemoryError when the maxmemory cap is hit, READONLY replicas, etc.) should degrade to 'test passed but cassette not cached' rather than fail the test on teardown. Previously the persister only caught ConnectionError and TimeoutError, so OutOfMemoryError — which Redis Cloud raises when the cassette cache hits its memory cap and there are no evictable keys — propagated out of vcrpy's autouse fixture and ERRORed otherwise-passing tests on teardown. This caused the litellm_utils_testing CircleCI job to fail on the latest commit's run, even though the underlying test was a unit test that used mock_response and produced no real upstream traffic (the cassette was dirtied by a background langfuse callback). The rerun only succeeded because Redis evictions happened to free enough room before the SET — i.e. it was timing-dependent flakiness. Catch redis.exceptions.RedisError (the common base of all server- and client-side Redis exceptions) on both save and load, and parametrize the regression tests across ConnectionError, TimeoutError, and OutOfMemoryError to pin the new behavior. --- tests/_vcr_redis_persister.py | 16 ++++--- .../test_vcr_redis_persister.py | 44 ++++++++++++++++--- 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index a6ed448f1cb6..4673e9fbca63 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -70,19 +70,16 @@ def make_redis_persister( redis_client = client if client is not None else _build_default_client() try: - from redis.exceptions import ConnectionError as RedisConnectionError - from redis.exceptions import TimeoutError as RedisTimeoutError - - _transient_errors: tuple = (RedisConnectionError, RedisTimeoutError) + from redis.exceptions import RedisError except ImportError: # pragma: no cover - redis is a hard test dep - _transient_errors = () + RedisError = Exception # type: ignore[assignment,misc] class _RedisPersister: @staticmethod def load_cassette(cassette_path, serializer): try: data = redis_client.get(redis_key_for(cassette_path)) - except _transient_errors as exc: + except RedisError as exc: _log.warning( "VCR redis load failed for %s; treating as cache miss: %s", cassette_path, @@ -123,7 +120,12 @@ def save_cassette(cassette_path, cassette_dict, serializer): payload = data.encode("utf-8") if isinstance(data, str) else data try: redis_client.set(key, payload, ex=ttl_seconds) - except _transient_errors as exc: + except RedisError as exc: + # Cassette persistence is strictly best-effort: connection + # blips, timeouts, OOM at the maxmemory cap, READONLY + # replicas, etc. should all degrade gracefully to "test + # passed but cassette not cached" rather than failing the + # test on teardown. _log.warning( "VCR redis save failed for %s; cassette not persisted: %s", cassette_path, diff --git a/tests/llm_translation/test_vcr_redis_persister.py b/tests/llm_translation/test_vcr_redis_persister.py index 6e62e4491cb6..1629b13d3a91 100644 --- a/tests/llm_translation/test_vcr_redis_persister.py +++ b/tests/llm_translation/test_vcr_redis_persister.py @@ -6,6 +6,8 @@ import fakeredis import pytest from redis.exceptions import ConnectionError as RedisConnectionError +from redis.exceptions import OutOfMemoryError as RedisOutOfMemoryError +from redis.exceptions import TimeoutError as RedisTimeoutError from vcr.persisters.filesystem import CassetteNotFoundError from vcr.request import Request from vcr.serializers import yamlserializer @@ -107,23 +109,42 @@ def test_redis_key_is_stable_across_working_directories(tmp_path, monkeypatch): class _FlakyRedis: - def __init__(self, inner, fail_on: str): + def __init__(self, inner, fail_on: str, exc=None): self._inner = inner self._fail_on = fail_on + self._exc = exc if exc is not None else RedisConnectionError("simulated outage") def get(self, *args, **kwargs): if self._fail_on == "get": - raise RedisConnectionError("simulated outage") + raise self._exc return self._inner.get(*args, **kwargs) def set(self, *args, **kwargs): if self._fail_on == "set": - raise RedisConnectionError("simulated outage") + raise self._exc return self._inner.set(*args, **kwargs) -def test_save_swallows_connection_errors_so_teardown_does_not_fail(): - flaky = _FlakyRedis(fakeredis.FakeStrictRedis(), fail_on="set") +@pytest.mark.parametrize( + "exc", + [ + RedisConnectionError("simulated outage"), + RedisTimeoutError("simulated timeout"), + RedisOutOfMemoryError("command not allowed when used memory > 'maxmemory'."), + ], + ids=["connection_error", "timeout", "out_of_memory"], +) +def test_save_swallows_redis_errors_so_teardown_does_not_fail(exc): + """Redis-side failures during cassette persistence must never fail + the test on teardown. + + Regression: previously the persister only swallowed + ConnectionError/TimeoutError, so OutOfMemoryError (raised by Redis + Cloud when the cassette cache hit its maxmemory cap) propagated out + of vcrpy's autouse fixture and failed otherwise-passing tests on + teardown. + """ + flaky = _FlakyRedis(fakeredis.FakeStrictRedis(), fail_on="set", exc=exc) persister = make_redis_persister(client=flaky) persister.save_cassette( @@ -229,8 +250,17 @@ def test_save_proceeds_when_outcome_unknown(): assert fake.get(key) is not None -def test_load_treats_connection_errors_as_cassette_miss(): - flaky = _FlakyRedis(fakeredis.FakeStrictRedis(), fail_on="get") +@pytest.mark.parametrize( + "exc", + [ + RedisConnectionError("simulated outage"), + RedisTimeoutError("simulated timeout"), + RedisOutOfMemoryError("command not allowed when used memory > 'maxmemory'."), + ], + ids=["connection_error", "timeout", "out_of_memory"], +) +def test_load_treats_redis_errors_as_cassette_miss(exc): + flaky = _FlakyRedis(fakeredis.FakeStrictRedis(), fail_on="get", exc=exc) persister = make_redis_persister(client=flaky) with pytest.raises(CassetteNotFoundError): From aac94c4b69c5e366aa6820bd88397a3b1d51faf5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 5 May 2026 06:00:26 +0000 Subject: [PATCH 07/12] test(vcr): surface cassette-cache failures with warnings + session banner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the persister silently swallows a Redis OOM (or any RedisError) on save/load there is otherwise no visible signal that the cache is degraded — tests pass, the cassette just isn't persisted, and the next session still hits the same Redis at the same near-cap memory. Add three layers of observability so that failure mode is loud: 1. Per-process health counters ("save_failures", "load_failures", and the last error string for each), exposed via cassette_cache_health() and reset via reset_cassette_cache_health(). The persister increments these in addition to logging. 2. VCRCassetteCacheWarning (UserWarning subclass) emitted via warnings.warn() inside the persister's except block. Pytest's built-in warnings summary at session end automatically lists every such warning, so the failure is visible in CI logs without any conftest-level wiring. 3. Session-end banner via emit_cassette_cache_session_banner() and a stderr-fallback atexit handler registered from register_persister_if_enabled(). Two states: - red "VCR CASSETTE CACHE DEGRADED" when save_failures or load_failures > 0 - yellow "VCR CASSETTE CACHE NEAR CAPACITY" (no failures, but used_memory >= 85% of maxmemory) so the next session knows the Redis is approaching OOM before any SET actually fails Capacity comes from a best-effort INFO memory probe (cassette_cache_capacity_snapshot) that returns None on any failure or when maxmemory is uncapped. The atexit handler skips xdist workers so only the controller emits. Tests: parametrize the existing save/load swallow-error tests across ConnectionError/TimeoutError/OutOfMemoryError, add direct tests for the health counters and warning emission, and a new test_vcr_conftest_common_banner.py covering banner output for every state (silent/red/yellow/disabled/xdist-worker). --- tests/_vcr_conftest_common.py | 135 ++++++++++++- tests/_vcr_redis_persister.py | 95 ++++++++- .../test_vcr_conftest_common_banner.py | 183 ++++++++++++++++++ .../test_vcr_redis_persister.py | 131 +++++++++++++ 4 files changed, 534 insertions(+), 10 deletions(-) create mode 100644 tests/llm_translation/test_vcr_conftest_common_banner.py diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index f0e2f3da6aba..feb31f771c46 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -22,12 +22,16 @@ from __future__ import annotations +import atexit import os -from typing import Iterable, Optional +import sys +from typing import Iterable import pytest from tests._vcr_redis_persister import ( + cassette_cache_capacity_snapshot, + cassette_cache_health, filter_non_2xx_response, format_vcr_verdict, make_redis_persister, @@ -36,6 +40,11 @@ vcr_verbose_enabled, ) +# Surface a warning when used memory exceeds this fraction of the cap, +# even if no SET has failed yet. Gives early notice before tests start +# failing on teardown. +CASSETTE_CACHE_HIGH_WATER_FRACTION = 0.85 + SAFE_BODY_MATCHER_NAME = "safe_body" @@ -160,6 +169,52 @@ def vcr_disabled() -> bool: return not os.environ.get("CASSETTE_REDIS_URL") +_atexit_banner_registered = False + + +def _print_atexit_banner() -> None: + """Stderr fallback banner — fires even when no consuming conftest + wires up ``pytest_terminal_summary``. Skipped on xdist workers so + the controller is the only emitter. + """ + if vcr_disabled(): + return + if os.environ.get("PYTEST_XDIST_WORKER"): + return + health = cassette_cache_health() + save_failures = int(health.get("save_failures", 0) or 0) + load_failures = int(health.get("load_failures", 0) or 0) + snapshot = cassette_cache_capacity_snapshot() + + def _emit(line: str) -> None: + sys.stderr.write(f"{line}\n") + + if save_failures or load_failures: + bar = "=" * 60 + _emit(bar) + _emit("VCR CASSETTE CACHE DEGRADED") + if save_failures: + _emit( + f" {save_failures} cassette save failure(s); last error: " + f"{health.get('save_failure_last_error', '')}" + ) + if load_failures: + _emit( + f" {load_failures} cassette load failure(s); last error: " + f"{health.get('load_failure_last_error', '')}" + ) + if snapshot: + _emit(_format_capacity_line(snapshot)) + _emit(bar) + return + if snapshot and snapshot["used_pct"] >= CASSETTE_CACHE_HIGH_WATER_FRACTION * 100: + bar = "=" * 60 + _emit(bar) + _emit("VCR CASSETTE CACHE NEAR CAPACITY") + _emit(_format_capacity_line(snapshot)) + _emit(bar) + + def register_persister_if_enabled(vcr) -> None: """Wire the Redis persister and custom matchers into vcrpy if VCR is enabled. @@ -171,6 +226,10 @@ def register_persister_if_enabled(vcr) -> None: vcr.register_persister(make_redis_persister()) vcr.register_matcher(SAFE_BODY_MATCHER_NAME, _safe_body_matcher) patch_vcrpy_aiohttp_record_path() + global _atexit_banner_registered + if not _atexit_banner_registered: + atexit.register(_print_atexit_banner) + _atexit_banner_registered = True def apply_vcr_auto_marker_to_items( @@ -223,6 +282,80 @@ def record_vcr_outcome(request, vcr) -> None: request.node.user_properties.append(("vcr_verdict", verdict)) +def _format_capacity_line(snapshot: dict) -> str: + used = int(snapshot.get("used_memory_bytes", 0) or 0) + cap = int(snapshot.get("maxmemory_bytes", 0) or 0) + pct = float(snapshot.get("used_pct", 0.0) or 0.0) + used_mb = used / (1024 * 1024) + cap_mb = cap / (1024 * 1024) + return ( + f" Cassette Redis usage: {used_mb:.1f} MiB / {cap_mb:.1f} MiB " + f"({pct:.1f}% of maxmemory)" + ) + + +def emit_cassette_cache_session_banner(terminalreporter) -> None: + """Write a session-end banner about cassette-cache health. + + - If any save/load failed during the session, prints a loud red banner + with the failure counts and last error. + - Otherwise, if Redis is at ≥ ``CASSETTE_CACHE_HIGH_WATER_FRACTION`` + of its maxmemory, prints an orange warning so the next CI run + knows the cache is close to OOM. + - No-op when VCR is disabled, no failures occurred, and capacity is + healthy. + + Call this from ``pytest_terminal_summary(terminalreporter, ...)`` in + consuming conftests. Safe to call from an xdist controller; xdist + workers will skip emitting (they'd duplicate output). + """ + if vcr_disabled(): + return + if os.environ.get("PYTEST_XDIST_WORKER"): + return + + health = cassette_cache_health() + save_failures = int(health.get("save_failures", 0) or 0) + load_failures = int(health.get("load_failures", 0) or 0) + snapshot = cassette_cache_capacity_snapshot() + + if save_failures or load_failures: + terminalreporter.write_sep( + "=", "VCR CASSETTE CACHE DEGRADED", red=True, bold=True + ) + if save_failures: + terminalreporter.write_line( + f" {save_failures} cassette save failure(s); last error: " + f"{health.get('save_failure_last_error', '')}" + ) + if load_failures: + terminalreporter.write_line( + f" {load_failures} cassette load failure(s); last error: " + f"{health.get('load_failure_last_error', '')}" + ) + terminalreporter.write_line( + " Tests still passed because cassette persistence is best-effort, " + "but the Redis cache may be degraded (e.g. at maxmemory cap, " + "unreachable, or read-only)." + ) + if snapshot: + terminalreporter.write_line(_format_capacity_line(snapshot)) + terminalreporter.write_sep("=", red=True, bold=True) + return + + if snapshot and snapshot["used_pct"] >= CASSETTE_CACHE_HIGH_WATER_FRACTION * 100: + terminalreporter.write_sep( + "=", "VCR CASSETTE CACHE NEAR CAPACITY", yellow=True, bold=True + ) + terminalreporter.write_line(_format_capacity_line(snapshot)) + terminalreporter.write_line( + " No save failures yet, but Redis is approaching maxmemory. " + "Consider running tests/_flush_vcr_cache.py or letting more " + "keys age out before the next session." + ) + terminalreporter.write_sep("=", yellow=True, bold=True) + + # --------------------------------------------------------------------------- # Verbose-verdict reporter helpers (optional; used by conftests that want to # print "[VCR HIT]/[VCR MISS]/..." lines next to each test in CI logs). diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index 4673e9fbca63..bf98905942b1 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -2,6 +2,7 @@ import logging import os +import warnings from typing import Any, Optional from vcr.persisters.filesystem import CassetteNotFoundError @@ -19,6 +20,76 @@ _passed_by_cassette_key: dict[str, bool] = {} +class VCRCassetteCacheWarning(UserWarning): + """Emitted when the cassette Redis cache fails to load or save. + + Surfaced in pytest's session-end warnings summary so failures are + visible in CI logs even when the underlying tests pass. + """ + + +# Per-process counters; surfaced via :func:`cassette_cache_health` so +# conftests can emit a session-end banner when failures occurred. +_cache_health = { + "save_failures": 0, + "save_failure_last_error": "", + "load_failures": 0, + "load_failure_last_error": "", +} + + +def _record_cache_failure(kind: str, exc: BaseException) -> None: + err = f"{type(exc).__name__}: {exc}" + if kind == "save": + _cache_health["save_failures"] = int(_cache_health["save_failures"]) + 1 + _cache_health["save_failure_last_error"] = err + elif kind == "load": + _cache_health["load_failures"] = int(_cache_health["load_failures"]) + 1 + _cache_health["load_failure_last_error"] = err + + +def cassette_cache_health() -> dict: + """Return a snapshot of cassette-cache failure counters for this process.""" + return dict(_cache_health) + + +def reset_cassette_cache_health() -> None: + """Reset cassette-cache counters. Intended for tests.""" + _cache_health["save_failures"] = 0 + _cache_health["save_failure_last_error"] = "" + _cache_health["load_failures"] = 0 + _cache_health["load_failure_last_error"] = "" + + +def cassette_cache_capacity_snapshot(client: Optional[Any] = None) -> Optional[dict]: + """Probe Redis ``INFO memory`` and return used/max bytes and percent. + + Returns ``None`` if Redis is unreachable, the server didn't report + ``maxmemory``, or ``maxmemory`` is 0 (uncapped). Best-effort: any + exception turns into ``None`` so this never breaks a test session. + """ + try: + if client is None: + client = _build_default_client() + info = client.info(section="memory") + except Exception: # pragma: no cover - best-effort probe + return None + used = info.get("used_memory") + maxmem = info.get("maxmemory") + try: + used = int(used) if used is not None else None + maxmem = int(maxmem) if maxmem is not None else None + except (TypeError, ValueError): # pragma: no cover - defensive + return None + if not used or not maxmem or maxmem <= 0: + return None + return { + "used_memory_bytes": used, + "maxmemory_bytes": maxmem, + "used_pct": (used / maxmem) * 100.0, + } + + def mark_test_outcome_for_cassette(cassette_path: str, passed: bool) -> None: _passed_by_cassette_key[redis_key_for(cassette_path)] = passed @@ -80,11 +151,13 @@ def load_cassette(cassette_path, serializer): try: data = redis_client.get(redis_key_for(cassette_path)) except RedisError as exc: - _log.warning( - "VCR redis load failed for %s; treating as cache miss: %s", - cassette_path, - exc, + _record_cache_failure("load", exc) + msg = ( + f"VCR redis load failed for {cassette_path}; treating " + f"as cache miss: {type(exc).__name__}: {exc}" ) + _log.warning(msg) + warnings.warn(msg, VCRCassetteCacheWarning, stacklevel=2) raise CassetteNotFoundError() from exc if data is None: raise CassetteNotFoundError() @@ -125,12 +198,16 @@ def save_cassette(cassette_path, cassette_dict, serializer): # blips, timeouts, OOM at the maxmemory cap, READONLY # replicas, etc. should all degrade gracefully to "test # passed but cassette not cached" rather than failing the - # test on teardown. - _log.warning( - "VCR redis save failed for %s; cassette not persisted: %s", - cassette_path, - exc, + # test on teardown. We still want a loud signal so the + # failure shows up in pytest's warnings summary at the + # end of the session and feeds the session-end banner. + _record_cache_failure("save", exc) + msg = ( + f"VCR redis save failed for {cassette_path}; cassette " + f"not persisted: {type(exc).__name__}: {exc}" ) + _log.warning(msg) + warnings.warn(msg, VCRCassetteCacheWarning, stacklevel=2) return _RedisPersister diff --git a/tests/llm_translation/test_vcr_conftest_common_banner.py b/tests/llm_translation/test_vcr_conftest_common_banner.py new file mode 100644 index 000000000000..86b79505904e --- /dev/null +++ b/tests/llm_translation/test_vcr_conftest_common_banner.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +import os +import sys +from io import StringIO + +import pytest + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) + +from tests._vcr_conftest_common import ( # noqa: E402 + emit_cassette_cache_session_banner, +) +from tests._vcr_redis_persister import ( # noqa: E402 + _cache_health, + reset_cassette_cache_health, +) + + +class _FakeTerminalReporter: + """Minimal stand-in for pytest's TerminalReporter.""" + + def __init__(self) -> None: + self.buf = StringIO() + + def write_sep(self, sep, title="", **kwargs): + if title: + self.buf.write(f"{sep * 5} {title} {sep * 5}\n") + else: + self.buf.write(f"{sep * 60}\n") + + def write_line(self, line): + self.buf.write(f"{line}\n") + + @property + def output(self) -> str: + return self.buf.getvalue() + + +@pytest.fixture +def health_reset(): + reset_cassette_cache_health() + yield + reset_cassette_cache_health() + + +@pytest.fixture +def vcr_enabled(monkeypatch): + """Make :func:`vcr_disabled` return False so the banner emits.""" + monkeypatch.setenv("CASSETTE_REDIS_URL", "redis://stub") + monkeypatch.delenv("LITELLM_VCR_DISABLE", raising=False) + monkeypatch.delenv("PYTEST_XDIST_WORKER", raising=False) + + +@pytest.fixture +def patch_capacity_snapshot(monkeypatch): + """Stub :func:`cassette_cache_capacity_snapshot` so we don't try to + open a real Redis connection. The fixture returns a setter that + each test uses to inject the snapshot it wants.""" + state = {"snapshot": None} + + def _stub(): + return state["snapshot"] + + import tests._vcr_conftest_common as common + + monkeypatch.setattr(common, "cassette_cache_capacity_snapshot", _stub) + + def _set(snapshot): + state["snapshot"] = snapshot + + return _set + + +def test_banner_silent_when_no_failures_and_capacity_healthy( + health_reset, vcr_enabled, patch_capacity_snapshot +): + patch_capacity_snapshot( + {"used_memory_bytes": 100, "maxmemory_bytes": 1000, "used_pct": 10.0} + ) + reporter = _FakeTerminalReporter() + + emit_cassette_cache_session_banner(reporter) + + assert reporter.output == "" + + +def test_banner_red_section_when_save_failures_recorded( + health_reset, vcr_enabled, patch_capacity_snapshot +): + _cache_health["save_failures"] = 3 + _cache_health["save_failure_last_error"] = ( + "OutOfMemoryError: command not allowed when used memory > 'maxmemory'." + ) + patch_capacity_snapshot( + {"used_memory_bytes": 990, "maxmemory_bytes": 1000, "used_pct": 99.0} + ) + reporter = _FakeTerminalReporter() + + emit_cassette_cache_session_banner(reporter) + + out = reporter.output + assert "VCR CASSETTE CACHE DEGRADED" in out + assert "3 cassette save failure(s)" in out + assert "OutOfMemoryError" in out + assert "99.0% of maxmemory" in out + + +def test_banner_red_section_when_load_failures_recorded( + health_reset, vcr_enabled, patch_capacity_snapshot +): + _cache_health["load_failures"] = 2 + _cache_health["load_failure_last_error"] = "ConnectionError: simulated outage" + patch_capacity_snapshot(None) + reporter = _FakeTerminalReporter() + + emit_cassette_cache_session_banner(reporter) + + out = reporter.output + assert "VCR CASSETTE CACHE DEGRADED" in out + assert "2 cassette load failure(s)" in out + assert "ConnectionError" in out + + +def test_banner_yellow_high_water_when_no_failures_but_near_capacity( + health_reset, vcr_enabled, patch_capacity_snapshot +): + patch_capacity_snapshot( + {"used_memory_bytes": 900, "maxmemory_bytes": 1000, "used_pct": 90.0} + ) + reporter = _FakeTerminalReporter() + + emit_cassette_cache_session_banner(reporter) + + out = reporter.output + assert "VCR CASSETTE CACHE NEAR CAPACITY" in out + assert "90.0% of maxmemory" in out + assert "VCR CASSETTE CACHE DEGRADED" not in out + + +def test_banner_silent_when_below_high_water_and_no_failures( + health_reset, vcr_enabled, patch_capacity_snapshot +): + patch_capacity_snapshot( + {"used_memory_bytes": 800, "maxmemory_bytes": 1000, "used_pct": 80.0} + ) + reporter = _FakeTerminalReporter() + + emit_cassette_cache_session_banner(reporter) + + assert reporter.output == "" + + +def test_banner_silent_when_vcr_disabled( + monkeypatch, health_reset, patch_capacity_snapshot +): + monkeypatch.delenv("CASSETTE_REDIS_URL", raising=False) + _cache_health["save_failures"] = 5 + _cache_health["save_failure_last_error"] = "OutOfMemoryError: foo" + patch_capacity_snapshot( + {"used_memory_bytes": 999, "maxmemory_bytes": 1000, "used_pct": 99.9} + ) + reporter = _FakeTerminalReporter() + + emit_cassette_cache_session_banner(reporter) + + assert reporter.output == "" + + +def test_banner_silent_on_xdist_worker( + monkeypatch, vcr_enabled, health_reset, patch_capacity_snapshot +): + monkeypatch.setenv("PYTEST_XDIST_WORKER", "gw3") + _cache_health["save_failures"] = 1 + _cache_health["save_failure_last_error"] = "OutOfMemoryError: bar" + patch_capacity_snapshot( + {"used_memory_bytes": 999, "maxmemory_bytes": 1000, "used_pct": 99.9} + ) + reporter = _FakeTerminalReporter() + + emit_cassette_cache_session_banner(reporter) + + assert reporter.output == "" diff --git a/tests/llm_translation/test_vcr_redis_persister.py b/tests/llm_translation/test_vcr_redis_persister.py index 1629b13d3a91..ac97a5efcaeb 100644 --- a/tests/llm_translation/test_vcr_redis_persister.py +++ b/tests/llm_translation/test_vcr_redis_persister.py @@ -17,10 +17,14 @@ from tests._vcr_redis_persister import ( # noqa: E402 CASSETTE_TTL_SECONDS, MAX_EPISODES_PER_CASSETTE, + VCRCassetteCacheWarning, + cassette_cache_capacity_snapshot, + cassette_cache_health, filter_non_2xx_response, make_redis_persister, mark_test_outcome_for_cassette, redis_key_for, + reset_cassette_cache_health, ) @@ -296,3 +300,130 @@ def test_only_2xx_responses_are_cached(status_code, expect_dropped): assert (result is None) == expect_dropped if not expect_dropped: assert result is response + + +# --------------------------------------------------------------------------- +# Cache-health observability +# --------------------------------------------------------------------------- + + +@pytest.fixture +def reset_health(): + reset_cassette_cache_health() + yield + reset_cassette_cache_health() + + +def test_save_failure_increments_health_counter_and_emits_warning(reset_health): + flaky = _FlakyRedis( + fakeredis.FakeStrictRedis(), + fail_on="set", + exc=RedisOutOfMemoryError( + "command not allowed when used memory > 'maxmemory'." + ), + ) + persister = make_redis_persister(client=flaky) + + with pytest.warns(VCRCassetteCacheWarning, match="OutOfMemoryError"): + persister.save_cassette( + "tests/llm_translation/test_x/test_save_outage", + _sample_cassette_dict(), + yamlserializer, + ) + + health = cassette_cache_health() + assert health["save_failures"] == 1 + assert "OutOfMemoryError" in health["save_failure_last_error"] + assert health["load_failures"] == 0 + + +def test_load_failure_increments_health_counter_and_emits_warning(reset_health): + flaky = _FlakyRedis( + fakeredis.FakeStrictRedis(), + fail_on="get", + exc=RedisConnectionError("simulated outage"), + ) + persister = make_redis_persister(client=flaky) + + with pytest.warns(VCRCassetteCacheWarning, match="ConnectionError"): + with pytest.raises(CassetteNotFoundError): + persister.load_cassette( + "tests/llm_translation/test_x/test_load_outage", yamlserializer + ) + + health = cassette_cache_health() + assert health["load_failures"] == 1 + assert "ConnectionError" in health["load_failure_last_error"] + assert health["save_failures"] == 0 + + +def test_health_counters_accumulate_across_failures(reset_health): + flaky = _FlakyRedis( + fakeredis.FakeStrictRedis(), + fail_on="set", + exc=RedisConnectionError("simulated outage"), + ) + persister = make_redis_persister(client=flaky) + + for i in range(3): + with pytest.warns(VCRCassetteCacheWarning): + persister.save_cassette( + f"tests/llm_translation/test_x/test_outage_{i}", + _sample_cassette_dict(), + yamlserializer, + ) + + assert cassette_cache_health()["save_failures"] == 3 + + +def test_successful_save_does_not_emit_warning_or_increment_counter(reset_health): + _, persister = _persister_with_fake_redis() + + import warnings as _warnings + + with _warnings.catch_warnings(): + _warnings.simplefilter("error", VCRCassetteCacheWarning) + persister.save_cassette( + "tests/llm_translation/test_x/test_happy", + _sample_cassette_dict(), + yamlserializer, + ) + + assert cassette_cache_health()["save_failures"] == 0 + + +class _FakeRedisWithInfo: + def __init__(self, used: int, maxmem: int): + self._used = used + self._maxmem = maxmem + + def info(self, section=None): + return {"used_memory": self._used, "maxmemory": self._maxmem} + + +def test_capacity_snapshot_returns_used_max_and_pct(): + client = _FakeRedisWithInfo(used=900, maxmem=1000) + snap = cassette_cache_capacity_snapshot(client=client) + assert snap == { + "used_memory_bytes": 900, + "maxmemory_bytes": 1000, + "used_pct": 90.0, + } + + +def test_capacity_snapshot_returns_none_when_uncapped(): + client = _FakeRedisWithInfo(used=900, maxmem=0) + assert cassette_cache_capacity_snapshot(client=client) is None + + +def test_capacity_snapshot_returns_none_when_used_unknown(): + client = _FakeRedisWithInfo(used=0, maxmem=1000) + assert cassette_cache_capacity_snapshot(client=client) is None + + +def test_capacity_snapshot_swallows_exceptions(): + class _Boom: + def info(self, section=None): + raise RuntimeError("redis offline") + + assert cassette_cache_capacity_snapshot(client=_Boom()) is None From 9a438a6caa542e196a946d090e9091928a5358db Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 5 May 2026 06:10:41 +0000 Subject: [PATCH 08/12] test(vcr): bucket cassettes by API key fingerprint, drop bad-key skips Tests that deliberately call an LLM API with a bad key (e.g. to assert that the failure callback fires, or that check_valid_key returns False) were being silently served the prior good-key cassette: we scrub the real Authorization / x-api-key header from the cassette before storing it, so a follow-up bad-key call is byte-identical to the good-key call under the existing match_on tuple. Add a 'key_fingerprint' custom matcher that distinguishes requests by the SHA-256 of their API-key headers. The fingerprint is stamped into a synthetic 'x-litellm-key-fp' header by a new before_record_request hook, which then strips the real auth headers (we have to do the scrubbing here instead of via vcrpy's filter_headers knob, because filter_headers runs *first* and would erase the value we want to hash). Bad-key requests now get a different cassette bucket than good-key requests, so vcrpy will not replay a recorded 200 in place of the expected 401. The fingerprint is a one-way hash of the secret, so cassettes never contain the key. This permanently removes the 'bad-key' category of skips: - tests/local_testing: dropped ::test_amazing_sync_embedding, ::test_async_custom_handler_completion, ::test_async_custom_handler_embedding - tests/logging_callback_tests: dropped ::test_async_chat_azure, ::test_async_embedding_azure - tests/litellm_utils_tests: dropped ::test_get_valid_models_from_dynamic_api_key Coverage: 7 new unit tests in tests/test_litellm/test_vcr_safe_body_matcher.py covering header stripping, fingerprint determinism, no-auth bucketing, good-vs-bad key discrimination, x-api-key (Anthropic/Azure) discrimination, and idempotence under replay. Co-authored-by: Mateo Wang --- tests/_vcr_conftest_common.py | 168 +++++++++++++++++- tests/litellm_utils_tests/conftest.py | 12 +- tests/local_testing/conftest.py | 18 +- tests/logging_callback_tests/conftest.py | 14 +- .../test_vcr_safe_body_matcher.py | 95 ++++++++++ 5 files changed, 277 insertions(+), 30 deletions(-) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index feb31f771c46..a2d570c29601 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -23,6 +23,7 @@ from __future__ import annotations import atexit +import hashlib import os import sys from typing import Iterable @@ -47,6 +48,32 @@ SAFE_BODY_MATCHER_NAME = "safe_body" +KEY_FINGERPRINT_MATCHER_NAME = "key_fingerprint" + +# Synthetic header name we attach to every request before it is recorded. +# The value is a one-way hash of the API-key headers on the request. This +# lets the ``key_fingerprint`` matcher distinguish requests that carry +# different API keys *after* we have scrubbed the real key value out of +# the cassette, without ever leaking the secret. See ``_before_record_request`` +# and ``_key_fingerprint_matcher`` below. +KEY_FINGERPRINT_HEADER = "x-litellm-key-fp" + +# Headers that carry an API key whose VALUE distinguishes one request from +# another (e.g. "the prior request used a real key, this one uses a bad +# key, so the cassette should not be reused"). We hash these into the +# fingerprint header. Note this is intentionally narrower than +# ``FILTERED_REQUEST_HEADERS`` — AWS SigV4 headers (``x-amz-date`` etc.) +# are *also* secrets-bearing but their values change on every call, so +# fingerprinting them would defeat caching entirely. +API_KEY_HEADERS = ( + "authorization", + "x-api-key", + "anthropic-api-key", + "openai-api-key", + "azure-api-key", + "api-key", + "x-goog-api-key", +) FILTERED_REQUEST_HEADERS = ( "authorization", @@ -142,10 +169,144 @@ def _to_bytes(b): raise AssertionError("request bodies differ") +def _iter_header_values(headers, name: str): + """Yield all values for ``name`` from a vcrpy request headers object. + + vcrpy normalizes request headers into a dict-like (case-insensitive on + most transports, case-sensitive on a few). Some transports also pass + through multi-valued headers as lists. We accept either shape so the + fingerprint stays stable across transports. + """ + if headers is None: + return + target = name.lower() + try: + items = headers.items() + except AttributeError: + return + for key, value in items: + if str(key).lower() != target: + continue + if isinstance(value, (list, tuple)): + for v in value: + yield v + else: + yield value + + +def _compute_key_fingerprint(request) -> str: + """Return a short, deterministic hash of the request's API-key headers. + + Empty/missing keys hash to a stable sentinel so requests without auth + headers all bucket together (and don't accidentally collide with any + real key). The hash is one-way so cassettes never contain the secret. + """ + headers = getattr(request, "headers", None) + parts: list[str] = [] + for header_name in API_KEY_HEADERS: + for value in _iter_header_values(headers, header_name): + if value is None: + continue + text = value if isinstance(value, str) else str(value) + text = text.strip() + if not text: + continue + parts.append(f"{header_name}={text}") + if not parts: + return "no-key" + digest = hashlib.sha256("\n".join(parts).encode("utf-8")).hexdigest() + return digest[:16] + + +def _strip_headers(headers, names: Iterable[str]) -> None: + """Remove any header in ``names`` from a vcrpy request headers dict. + + Comparison is case-insensitive; we delete *all* keys whose lowercase + name is in ``names``. We do this in-place because vcrpy already deep- + copies the request before invoking ``before_record_request``. + """ + if headers is None: + return + targets = {n.lower() for n in names} + try: + keys = list(headers.keys()) + except AttributeError: + return + for key in keys: + if str(key).lower() in targets: + try: + del headers[key] + except (KeyError, TypeError): + pass + + +def _before_record_request(request): + """Stamp + scrub auth headers before vcrpy persists the request. + + Two responsibilities, in this order: + + 1. Compute a one-way hash of the API-key headers and stash it as the + synthetic ``KEY_FINGERPRINT_HEADER``. The ``key_fingerprint`` + matcher uses this so a request made with a different API key is + not silently served from a cassette recorded with the real key. + 2. Strip every ``FILTERED_REQUEST_HEADERS`` value (auth headers, AWS + SigV4 timestamps, cookies, etc.) so the cassette never contains + the secret. We do the scrubbing here — instead of via vcrpy's + ``filter_headers`` knob — because ``filter_headers`` runs *before* + ``before_record_request`` and would erase the auth value before + step 1 could read it. + + Without step 1, scrubbing the auth header would cause a "bad-key" + request to match a "good-key" cassette (because everything else — + method, URL, body — is identical), so vcrpy would replay the recorded + 200 and any test that expects a 401 (failure callbacks, + ``check_valid_key`` returning False, etc.) would silently flip. + """ + headers = getattr(request, "headers", None) + if headers is None: + return request + fingerprint = _compute_key_fingerprint(request) + try: + headers[KEY_FINGERPRINT_HEADER] = fingerprint + except (TypeError, AttributeError): + pass + _strip_headers(headers, FILTERED_REQUEST_HEADERS) + return request + + +def _key_fingerprint_matcher(r1, r2) -> None: + """Match requests by the synthetic key-fingerprint header. + + Two requests are considered equivalent only if they were sent with the + same API key (or both with no key). Combined with the existing scrub + of the real auth header, this lets us cache responses for the real + key without poisoning the cassette for tests that deliberately use a + bad key. + """ + + def _fp(req): + for value in _iter_header_values( + getattr(req, "headers", None), KEY_FINGERPRINT_HEADER + ): + if value is None: + continue + return value if isinstance(value, str) else str(value) + return "no-key" + + if _fp(r1) != _fp(r2): + raise AssertionError("API key fingerprints differ") + + def vcr_config_dict() -> dict: - """Return the VCR config dict shared across all consuming conftests.""" + """Return the VCR config dict shared across all consuming conftests. + + Note: the auth-header scrubbing is intentionally performed inside + ``_before_record_request`` instead of via vcrpy's ``filter_headers`` + knob. ``filter_headers`` runs *before* ``before_record_request``, and + we need the raw auth header values available so we can fingerprint + them for the ``key_fingerprint`` matcher before stripping them. + """ return { - "filter_headers": list(FILTERED_REQUEST_HEADERS), "decode_compressed_response": True, "record_mode": "new_episodes", "allow_playback_repeats": True, @@ -156,8 +317,10 @@ def vcr_config_dict() -> dict: "port", "path", "query", + KEY_FINGERPRINT_MATCHER_NAME, SAFE_BODY_MATCHER_NAME, ), + "before_record_request": _before_record_request, "before_record_response": _before_record_response, } @@ -225,6 +388,7 @@ def register_persister_if_enabled(vcr) -> None: return vcr.register_persister(make_redis_persister()) vcr.register_matcher(SAFE_BODY_MATCHER_NAME, _safe_body_matcher) + vcr.register_matcher(KEY_FINGERPRINT_MATCHER_NAME, _key_fingerprint_matcher) patch_vcrpy_aiohttp_record_path() global _atexit_banner_registered if not _atexit_banner_registered: diff --git a/tests/litellm_utils_tests/conftest.py b/tests/litellm_utils_tests/conftest.py index 9dd69bc980a1..2a1a1e4454c0 100644 --- a/tests/litellm_utils_tests/conftest.py +++ b/tests/litellm_utils_tests/conftest.py @@ -34,13 +34,11 @@ } ) -# Specific tests where VCR replay actively breaks the test: -# - ``test_get_valid_models_from_dynamic_api_key`` deliberately calls -# ``GET /v1/models`` with ``api_key="123"`` to assert the result is empty. -# We scrub auth headers from cassettes (so the bad-key request matches -# the prior good-key request), and vcrpy replays the recorded list of -# models — flipping ``len(...) == 0`` to a long list. -_VCR_INCOMPATIBLE_NODEID_SUFFIXES = ("::test_get_valid_models_from_dynamic_api_key",) +# No node-id suffix skips at the moment. Tests that deliberately use a +# bad API key (e.g. ``test_get_valid_models_from_dynamic_api_key`` with +# ``api_key="123"``) are handled transparently by the ``key_fingerprint`` +# matcher in ``tests/_vcr_conftest_common.py``. +_VCR_INCOMPATIBLE_NODEID_SUFFIXES: tuple[str, ...] = () @pytest.fixture(scope="function", autouse=True) diff --git a/tests/local_testing/conftest.py b/tests/local_testing/conftest.py index 3d6a5f4e1747..06e41c74936a 100644 --- a/tests/local_testing/conftest.py +++ b/tests/local_testing/conftest.py @@ -62,17 +62,13 @@ } ) -# Specific tests where VCR replay actively breaks the test: -# - These tests deliberately call the LLM API with ``api_key="my-bad-key"`` -# to assert that the failure callback fires. We scrub auth headers from -# cassettes (so the bad-key request matches the prior good-key request), -# and vcrpy replays the recorded 200 — so the failure callback never -# fires and the assertion flips. -_VCR_INCOMPATIBLE_NODEID_SUFFIXES = ( - "::test_amazing_sync_embedding", - "::test_async_custom_handler_completion", - "::test_async_custom_handler_embedding", -) +# No node-id suffix skips at the moment. Tests that deliberately use +# ``api_key="my-bad-key"`` to assert a failure callback fires are handled +# transparently by the ``key_fingerprint`` matcher in +# ``tests/_vcr_conftest_common.py`` — bad-key requests get a different +# cassette bucket than good-key ones, so vcrpy will not replay a recorded +# 200 in place of the expected 401. +_VCR_INCOMPATIBLE_NODEID_SUFFIXES: tuple[str, ...] = () _verbose_state = VerboseReporterState() diff --git a/tests/logging_callback_tests/conftest.py b/tests/logging_callback_tests/conftest.py index 8ab7673eacc0..531b4812bf88 100644 --- a/tests/logging_callback_tests/conftest.py +++ b/tests/logging_callback_tests/conftest.py @@ -48,16 +48,10 @@ } ) -# Specific tests where VCR replay actively breaks the test: -# - The "failure" branches of these callback tests deliberately pass a bad -# API key to assert that the ``async_failure`` / ``failure`` callback fires. -# We scrub auth headers from cassettes (so the bad-key request matches the -# prior good-key request), and vcrpy replays the recorded 200 — so the -# failure callback never fires and the assertion flips. -_VCR_INCOMPATIBLE_NODEID_SUFFIXES = ( - "::test_async_chat_azure", - "::test_async_embedding_azure", -) +# No node-id suffix skips at the moment. Tests that deliberately use a +# bad API key to assert a failure callback fires are handled transparently +# by the ``key_fingerprint`` matcher in ``tests/_vcr_conftest_common.py``. +_VCR_INCOMPATIBLE_NODEID_SUFFIXES: tuple[str, ...] = () _verbose_state = VerboseReporterState() diff --git a/tests/test_litellm/test_vcr_safe_body_matcher.py b/tests/test_litellm/test_vcr_safe_body_matcher.py index 046b3c45339a..e10950582dec 100644 --- a/tests/test_litellm/test_vcr_safe_body_matcher.py +++ b/tests/test_litellm/test_vcr_safe_body_matcher.py @@ -23,7 +23,11 @@ sys.path.insert(0, _REPO_ROOT) from tests._vcr_conftest_common import ( # noqa: E402 + KEY_FINGERPRINT_HEADER, + KEY_FINGERPRINT_MATCHER_NAME, SAFE_BODY_MATCHER_NAME, + _before_record_request, + _key_fingerprint_matcher, _safe_body_matcher, vcr_config_dict, ) @@ -103,3 +107,94 @@ def test_default_vcrpy_body_matcher_crashes_on_jsonl_for_documentation(): jsonl = b'{"recordId": "request-1"}\n{"recordId": "request-2"}\n' with pytest.raises(json.JSONDecodeError): vcrpy_body(_req(jsonl), _req(jsonl)) + + +# --------------------------------------------------------------------------- +# Key-fingerprint matcher +# --------------------------------------------------------------------------- + + +def _req_with_headers(headers, body=b""): + return SimpleNamespace(headers=dict(headers), body=body) + + +def test_key_fingerprint_matcher_is_in_match_on(): + cfg = vcr_config_dict() + assert KEY_FINGERPRINT_MATCHER_NAME in cfg["match_on"] + + +def test_before_record_request_strips_auth_and_adds_fingerprint(): + """The hook must scrub the secret AND stamp a fingerprint.""" + req = _req_with_headers( + { + "Authorization": "Bearer sk-real-key-1234567890", + "Content-Type": "application/json", + "x-amz-date": "20240115T120000Z", + } + ) + out = _before_record_request(req) + assert ( + "Authorization" not in out.headers + ), "Authorization must be removed before the cassette is recorded" + assert "x-amz-date" not in out.headers, ( + "AWS SigV4 timestamp must be scrubbed (it changes every call and " + "would defeat caching)" + ) + fp = out.headers.get(KEY_FINGERPRINT_HEADER) + assert fp and isinstance(fp, str) + assert len(fp) >= 8 + assert "sk-real" not in fp, "fingerprint must not leak the secret" + + +def test_before_record_request_no_auth_yields_stable_no_key_bucket(): + a = _before_record_request(_req_with_headers({"Content-Type": "application/json"})) + b = _before_record_request(_req_with_headers({})) + assert a.headers[KEY_FINGERPRINT_HEADER] == b.headers[KEY_FINGERPRINT_HEADER] + # Two no-auth requests must match so we don't defeat caching for + # SigV4-style requests where auth lives in headers we've stripped. + _key_fingerprint_matcher(a, b) + + +def test_key_fingerprint_matcher_distinguishes_good_and_bad_keys(): + """The whole point: bad-key calls must not replay good-key cassettes.""" + good = _before_record_request( + _req_with_headers({"Authorization": "Bearer sk-real-good-key"}) + ) + bad = _before_record_request( + _req_with_headers({"Authorization": "Bearer my-bad-key"}) + ) + assert good.headers[KEY_FINGERPRINT_HEADER] != bad.headers[KEY_FINGERPRINT_HEADER] + with pytest.raises(AssertionError): + _key_fingerprint_matcher(good, bad) + + +def test_key_fingerprint_matcher_matches_repeated_good_key_calls(): + a = _before_record_request( + _req_with_headers({"Authorization": "Bearer sk-same-key"}) + ) + b = _before_record_request( + _req_with_headers({"Authorization": "Bearer sk-same-key"}) + ) + _key_fingerprint_matcher(a, b) + + +def test_key_fingerprint_matcher_distinguishes_x_api_key_callers(): + """Anthropic / Azure use ``x-api-key`` (or ``api-key``) instead of Authorization.""" + a = _before_record_request(_req_with_headers({"x-api-key": "anthropic-real"})) + b = _before_record_request(_req_with_headers({"x-api-key": "anthropic-bad"})) + with pytest.raises(AssertionError): + _key_fingerprint_matcher(a, b) + + +def test_before_record_request_is_idempotent_under_replay(): + """vcrpy runs ``before_record_request`` on both record and replay paths. + + The fingerprint must be deterministic so a request made today matches + a cassette recorded yesterday from the same key. + """ + payload = {"Authorization": "Bearer sk-deterministic"} + first = _before_record_request(_req_with_headers(payload)) + second = _before_record_request(_req_with_headers(payload)) + assert ( + first.headers[KEY_FINGERPRINT_HEADER] == second.headers[KEY_FINGERPRINT_HEADER] + ) From 797fe6c427906dfbd64fdd2a1f4af3f2d3754540 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 5 May 2026 06:19:20 +0000 Subject: [PATCH 09/12] test(vcr): drop redundant comments and docstrings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Trim narration of code that is already self-evident from function and variable names. Keep the two genuinely non-obvious bits: - ordering constraint between filter_headers and before_record_request, which would invite a maintainer to re-introduce the bug if removed - the per-directory _VCR_INCOMPATIBLE_FILES rationale, since 'why exactly is this skipped' is not knowable from the test name alone Also drop the 40-line commented-out drop-in conftest snippet at the bottom of _vcr_conftest_common.py — the consuming conftests are the canonical reference. Co-authored-by: Mateo Wang --- tests/_vcr_conftest_common.py | 241 +++--------------- tests/litellm_utils_tests/conftest.py | 12 +- tests/local_testing/conftest.py | 26 +- tests/logging_callback_tests/conftest.py | 12 +- .../test_vcr_safe_body_matcher.py | 66 +---- 5 files changed, 53 insertions(+), 304 deletions(-) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index a2d570c29601..121a493354d9 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -1,24 +1,7 @@ -""" -Shared VCR (Redis-backed) plumbing for test directories. - -This module is imported by per-directory ``conftest.py`` files to enable -24-hour HTTP caching against a Redis backend. The first run hits live -provider APIs and records the exchange; subsequent runs within 24h replay -from Redis without touching the network. See -``tests/llm_translation/Readme.md`` for the full design notes. - -Each consuming ``conftest.py`` should: - -1. Define a ``vcr_config`` fixture that delegates to :func:`vcr_config_dict`. -2. Define ``pytest_recording_configure(config, vcr)`` that calls - :func:`register_persister_if_enabled`. -3. Define ``pytest_runtest_makereport`` and a ``_vcr_outcome_gate`` autouse - fixture using :func:`make_outcome_gate_fixture` (or copy the snippet) so - failed tests don't poison cassettes. -4. Add ``apply_vcr_auto_marker_to_items`` inside - ``pytest_collection_modifyitems`` so non-respx tests are auto-marked - with ``pytest.mark.vcr``. -""" +"""Shared VCR (Redis-backed) plumbing imported by per-directory conftests. + +See ``tests/llm_translation/Readme.md`` for the full design and +``tests/llm_translation/conftest.py`` for the reference wiring.""" from __future__ import annotations @@ -41,30 +24,16 @@ vcr_verbose_enabled, ) -# Surface a warning when used memory exceeds this fraction of the cap, -# even if no SET has failed yet. Gives early notice before tests start -# failing on teardown. CASSETTE_CACHE_HIGH_WATER_FRACTION = 0.85 SAFE_BODY_MATCHER_NAME = "safe_body" KEY_FINGERPRINT_MATCHER_NAME = "key_fingerprint" - -# Synthetic header name we attach to every request before it is recorded. -# The value is a one-way hash of the API-key headers on the request. This -# lets the ``key_fingerprint`` matcher distinguish requests that carry -# different API keys *after* we have scrubbed the real key value out of -# the cassette, without ever leaking the secret. See ``_before_record_request`` -# and ``_key_fingerprint_matcher`` below. KEY_FINGERPRINT_HEADER = "x-litellm-key-fp" -# Headers that carry an API key whose VALUE distinguishes one request from -# another (e.g. "the prior request used a real key, this one uses a bad -# key, so the cassette should not be reused"). We hash these into the -# fingerprint header. Note this is intentionally narrower than -# ``FILTERED_REQUEST_HEADERS`` — AWS SigV4 headers (``x-amz-date`` etc.) -# are *also* secrets-bearing but their values change on every call, so -# fingerprinting them would defeat caching entirely. +# Intentionally narrower than ``FILTERED_REQUEST_HEADERS``: AWS SigV4 headers +# carry secrets but their values rotate on every call, so fingerprinting them +# would defeat caching. API_KEY_HEADERS = ( "authorization", "x-api-key", @@ -122,29 +91,13 @@ def _before_record_response(response): def _safe_body_matcher(r1, r2) -> None: - """Body matcher that compares raw bytes and never raises on bad JSON. - - vcrpy's stock ``body`` matcher inspects ``Content-Type`` and runs - ``json.loads`` on bodies typed ``application/json`` so it can compare - semantically. That crashes (``json.JSONDecodeError: Extra data``) on - JSON Lines payloads — which the Bedrock batch S3 PUT and a few other - upload paths use — before the matcher even gets a chance to return - "not a match". - - This matcher avoids the JSON normalization step entirely. It first - tries direct ``==`` equality on the original payloads (so dicts, - lists, etc. are compared structurally), then falls back to a bytes - comparison after coercing ``str`` to UTF-8. Anything that's not - bytes/str/equal-by-default is treated as a mismatch. This is - strictly more conservative than vcrpy's default — the only thing it - gives up is "different JSON key order is treated as the same body", - which doesn't matter for our deterministic litellm-built request - payloads. It can never produce a false positive that the default - would have rejected. - - The trade-off is that bodies containing nondeterministic values (UUIDs, - timestamps) will produce a cache miss; the right fix for those cases - is a ``before_record_request`` scrubber, not a smarter matcher. + """Compare request bodies as bytes; never invokes ``json.loads``. + + vcrpy's stock ``body`` matcher unconditionally json-decodes + ``application/json`` payloads, which raises on JSON Lines bodies + (e.g. the Bedrock batch S3 PUT) before it can return "no match". + This matcher is strictly more conservative — the only equivalence + it gives up vs. the default is "JSON key order doesn't matter". """ body1 = getattr(r1, "body", None) body2 = getattr(r2, "body", None) @@ -162,21 +115,12 @@ def _to_bytes(b): n1 = _to_bytes(body1) n2 = _to_bytes(body2) - if n1 is not None and n2 is not None: - if n1 == n2: - return - raise AssertionError("request bodies differ") + if n1 is not None and n2 is not None and n1 == n2: + return raise AssertionError("request bodies differ") def _iter_header_values(headers, name: str): - """Yield all values for ``name`` from a vcrpy request headers object. - - vcrpy normalizes request headers into a dict-like (case-insensitive on - most transports, case-sensitive on a few). Some transports also pass - through multi-valued headers as lists. We accept either shape so the - fingerprint stays stable across transports. - """ if headers is None: return target = name.lower() @@ -195,12 +139,6 @@ def _iter_header_values(headers, name: str): def _compute_key_fingerprint(request) -> str: - """Return a short, deterministic hash of the request's API-key headers. - - Empty/missing keys hash to a stable sentinel so requests without auth - headers all bucket together (and don't accidentally collide with any - real key). The hash is one-way so cassettes never contain the secret. - """ headers = getattr(request, "headers", None) parts: list[str] = [] for header_name in API_KEY_HEADERS: @@ -219,12 +157,6 @@ def _compute_key_fingerprint(request) -> str: def _strip_headers(headers, names: Iterable[str]) -> None: - """Remove any header in ``names`` from a vcrpy request headers dict. - - Comparison is case-insensitive; we delete *all* keys whose lowercase - name is in ``names``. We do this in-place because vcrpy already deep- - copies the request before invoking ``before_record_request``. - """ if headers is None: return targets = {n.lower() for n in names} @@ -241,26 +173,12 @@ def _strip_headers(headers, names: Iterable[str]) -> None: def _before_record_request(request): - """Stamp + scrub auth headers before vcrpy persists the request. - - Two responsibilities, in this order: - - 1. Compute a one-way hash of the API-key headers and stash it as the - synthetic ``KEY_FINGERPRINT_HEADER``. The ``key_fingerprint`` - matcher uses this so a request made with a different API key is - not silently served from a cassette recorded with the real key. - 2. Strip every ``FILTERED_REQUEST_HEADERS`` value (auth headers, AWS - SigV4 timestamps, cookies, etc.) so the cassette never contains - the secret. We do the scrubbing here — instead of via vcrpy's - ``filter_headers`` knob — because ``filter_headers`` runs *before* - ``before_record_request`` and would erase the auth value before - step 1 could read it. - - Without step 1, scrubbing the auth header would cause a "bad-key" - request to match a "good-key" cassette (because everything else — - method, URL, body — is identical), so vcrpy would replay the recorded - 200 and any test that expects a 401 (failure callbacks, - ``check_valid_key`` returning False, etc.) would silently flip. + """Fingerprint API keys, then scrub them. + + Order matters: vcrpy's ``filter_headers`` config option runs *before* + ``before_record_request`` and would erase the auth value before we + could hash it. Doing both steps here keeps the fingerprint available + while ensuring the secret never reaches the cassette. """ headers = getattr(request, "headers", None) if headers is None: @@ -275,15 +193,6 @@ def _before_record_request(request): def _key_fingerprint_matcher(r1, r2) -> None: - """Match requests by the synthetic key-fingerprint header. - - Two requests are considered equivalent only if they were sent with the - same API key (or both with no key). Combined with the existing scrub - of the real auth header, this lets us cache responses for the real - key without poisoning the cassette for tests that deliberately use a - bad key. - """ - def _fp(req): for value in _iter_header_values( getattr(req, "headers", None), KEY_FINGERPRINT_HEADER @@ -298,14 +207,7 @@ def _fp(req): def vcr_config_dict() -> dict: - """Return the VCR config dict shared across all consuming conftests. - - Note: the auth-header scrubbing is intentionally performed inside - ``_before_record_request`` instead of via vcrpy's ``filter_headers`` - knob. ``filter_headers`` runs *before* ``before_record_request``, and - we need the raw auth header values available so we can fingerprint - them for the ``key_fingerprint`` matcher before stripping them. - """ + """Return the VCR config dict shared across all consuming conftests.""" return { "decode_compressed_response": True, "record_mode": "new_episodes", @@ -336,10 +238,7 @@ def vcr_disabled() -> bool: def _print_atexit_banner() -> None: - """Stderr fallback banner — fires even when no consuming conftest - wires up ``pytest_terminal_summary``. Skipped on xdist workers so - the controller is the only emitter. - """ + """Fallback for conftests that don't wire up ``pytest_terminal_summary``.""" if vcr_disabled(): return if os.environ.get("PYTEST_XDIST_WORKER"): @@ -379,11 +278,7 @@ def _emit(line: str) -> None: def register_persister_if_enabled(vcr) -> None: - """Wire the Redis persister and custom matchers into vcrpy if VCR is - enabled. - - Call this from ``pytest_recording_configure(config, vcr)`` in conftest. - """ + """Call from ``pytest_recording_configure(config, vcr)`` in each conftest.""" if vcr_disabled(): return vcr.register_persister(make_redis_persister()) @@ -404,13 +299,10 @@ def apply_vcr_auto_marker_to_items( ) -> None: """Auto-apply ``pytest.mark.vcr`` to collected items. - ``skip_files`` is a set of basenames (e.g. ``test_openai.py``) that - should not be auto-marked — typically files that already use ``respx``, - since respx and vcrpy both patch the httpx transport and conflict. - - ``skip_nodeid_suffixes`` is a set of node-id suffixes (e.g. - ``"::test_prompt_caching"``) that observe live cross-call provider - state which replay can't reproduce. + ``skip_files`` are basenames to leave un-marked (e.g. respx-using + files, since respx and vcrpy both patch the httpx transport). + ``skip_nodeid_suffixes`` are node-id suffixes for individual tests + that depend on live cross-call provider state. """ if vcr_disabled(): return @@ -428,11 +320,7 @@ def apply_vcr_auto_marker_to_items( def record_vcr_outcome(request, vcr) -> None: - """Mark the cassette with the test outcome and emit a verbose verdict. - - Call this from a ``yield``-after section of an autouse fixture in - conftest, after the test has run. - """ + """Call from the post-yield section of an autouse fixture per test.""" cassette = vcr rep_call = getattr(request.node, "rep_call", None) test_passed = bool(rep_call and rep_call.passed) @@ -459,20 +347,7 @@ def _format_capacity_line(snapshot: dict) -> str: def emit_cassette_cache_session_banner(terminalreporter) -> None: - """Write a session-end banner about cassette-cache health. - - - If any save/load failed during the session, prints a loud red banner - with the failure counts and last error. - - Otherwise, if Redis is at ≥ ``CASSETTE_CACHE_HIGH_WATER_FRACTION`` - of its maxmemory, prints an orange warning so the next CI run - knows the cache is close to OOM. - - No-op when VCR is disabled, no failures occurred, and capacity is - healthy. - - Call this from ``pytest_terminal_summary(terminalreporter, ...)`` in - consuming conftests. Safe to call from an xdist controller; xdist - workers will skip emitting (they'd duplicate output). - """ + """Call from ``pytest_terminal_summary``. No-op on xdist workers.""" if vcr_disabled(): return if os.environ.get("PYTEST_XDIST_WORKER"): @@ -520,15 +395,9 @@ def emit_cassette_cache_session_banner(terminalreporter) -> None: terminalreporter.write_sep("=", yellow=True, bold=True) -# --------------------------------------------------------------------------- -# Verbose-verdict reporter helpers (optional; used by conftests that want to -# print "[VCR HIT]/[VCR MISS]/..." lines next to each test in CI logs). -# --------------------------------------------------------------------------- class VerboseReporterState: - """Container for the controller-process plugin manager / terminal reporter. - - A single instance lives in each conftest that wants verbose output. - """ + """Holds the controller's plugin manager / terminal reporter so each + consuming conftest can print ``[VCR HIT|MISS|...]`` lines next to tests.""" def __init__(self) -> None: self.pluginmanager = None @@ -564,47 +433,3 @@ def maybe_emit_verdict(self, report) -> None: if not verdict: return reporter.write_line(f"{verdict} :: {report.nodeid}") - - -# --------------------------------------------------------------------------- -# Drop-in conftest snippet (copy/paste guidance, not executed). -# --------------------------------------------------------------------------- -# from tests._vcr_conftest_common import ( -# VerboseReporterState, -# apply_vcr_auto_marker_to_items, -# record_vcr_outcome, -# register_persister_if_enabled, -# vcr_config_dict, -# ) -# -# _verbose_state = VerboseReporterState() -# _RESPX_CONFLICTING_FILES = frozenset({...}) -# -# @pytest.fixture(scope="module") -# def vcr_config(): -# return vcr_config_dict() -# -# def pytest_recording_configure(config, vcr): -# register_persister_if_enabled(vcr) -# -# @pytest.hookimpl(hookwrapper=True) -# def pytest_runtest_makereport(item, call): -# outcome = yield -# rep = outcome.get_result() -# setattr(item, f"rep_{rep.when}", rep) -# -# @pytest.fixture(autouse=True) -# def _vcr_outcome_gate(request, vcr): -# yield -# record_vcr_outcome(request, vcr) -# -# def pytest_configure(config): -# _verbose_state.remember_pluginmanager(config) -# -# def pytest_runtest_logreport(report): -# _verbose_state.maybe_emit_verdict(report) -# -# def pytest_collection_modifyitems(config, items): -# apply_vcr_auto_marker_to_items( -# items, skip_files=_RESPX_CONFLICTING_FILES, -# ) diff --git a/tests/litellm_utils_tests/conftest.py b/tests/litellm_utils_tests/conftest.py index 2a1a1e4454c0..a110128d2fff 100644 --- a/tests/litellm_utils_tests/conftest.py +++ b/tests/litellm_utils_tests/conftest.py @@ -23,21 +23,15 @@ _verbose_state = VerboseReporterState() -# Files where VCR replay actively breaks the test: -# - ``test_litellm_overhead.py`` measures ``litellm_overhead_time_ms`` as a -# percentage of total wall-clock time. With cached responses the upstream -# "network" time collapses to microseconds, so the overhead percentage -# blows past the 40% threshold the test asserts on. +# Files where VCR replay breaks the test: +# - ``test_litellm_overhead.py``: asserts overhead/total < 40%, which +# inverts when cached replay collapses the upstream time to microseconds. _VCR_INCOMPATIBLE_FILES = frozenset( { "test_litellm_overhead.py", } ) -# No node-id suffix skips at the moment. Tests that deliberately use a -# bad API key (e.g. ``test_get_valid_models_from_dynamic_api_key`` with -# ``api_key="123"``) are handled transparently by the ``key_fingerprint`` -# matcher in ``tests/_vcr_conftest_common.py``. _VCR_INCOMPATIBLE_NODEID_SUFFIXES: tuple[str, ...] = () diff --git a/tests/local_testing/conftest.py b/tests/local_testing/conftest.py index 06e41c74936a..cad27869ad2c 100644 --- a/tests/local_testing/conftest.py +++ b/tests/local_testing/conftest.py @@ -40,21 +40,11 @@ } ) -# Files where VCR replay actively breaks the test: -# - ``test_assistants.py`` exercises the OpenAI Assistants polling APIs -# which mint fresh thread/run/message IDs every recording session and -# then poll until ``status == "completed"``. Replays of those polled -# GETs would have to match the new run id (impossible) or be played -# back in lockstep with a freshly recorded creation, neither of which -# ``record_mode="new_episodes"`` does well. The result in CI is that -# every run effectively re-records, blowing past the 15-minute step -# timeout for ``litellm_assistants_api_testing``. -# - ``test_router_caching.py`` asserts on litellm's own router-level -# response cache by comparing ``response1.id`` to ``response2.id`` -# across repeat upstream calls (the test bypasses litellm's cache via -# ``ttl=0`` and expects the upstream to return a *new* id each time). -# With VCR replay both upstream calls return the same cassette body, -# so the ids are identical and ``response1.id != response2.id`` flips. +# Files where VCR replay breaks the test: +# - ``test_assistants.py``: polls fresh per-session run IDs that no cassette +# can match, so every CI run re-records and the suite times out. +# - ``test_router_caching.py``: asserts upstream returns a *new* id per call, +# which a deterministic cassette replay violates. _VCR_INCOMPATIBLE_FILES = frozenset( { "test_assistants.py", @@ -62,12 +52,6 @@ } ) -# No node-id suffix skips at the moment. Tests that deliberately use -# ``api_key="my-bad-key"`` to assert a failure callback fires are handled -# transparently by the ``key_fingerprint`` matcher in -# ``tests/_vcr_conftest_common.py`` — bad-key requests get a different -# cassette bucket than good-key ones, so vcrpy will not replay a recorded -# 200 in place of the expected 401. _VCR_INCOMPATIBLE_NODEID_SUFFIXES: tuple[str, ...] = () diff --git a/tests/logging_callback_tests/conftest.py b/tests/logging_callback_tests/conftest.py index 531b4812bf88..4f847d93a521 100644 --- a/tests/logging_callback_tests/conftest.py +++ b/tests/logging_callback_tests/conftest.py @@ -36,21 +36,15 @@ } ) -# Files where VCR replay actively breaks the test: -# - ``test_amazing_s3_logs.py`` exercises the S3 success callback using -# ``mock_response`` (so there is no upstream LLM call worth caching) and -# asserts on a per-run ``response_id`` round-tripped through a real S3 -# PUT/LIST. vcrpy's boto3 stub intercepts the PUT and replays a stale LIST, -# so the freshly-generated id is never found in the cached keys. +# Files where VCR replay breaks the test: +# - ``test_amazing_s3_logs.py``: vcrpy's boto3 stub intercepts a real S3 +# PUT/LIST round-trip the test asserts on, so the per-run id is never found. _VCR_INCOMPATIBLE_FILES = frozenset( { "test_amazing_s3_logs.py", } ) -# No node-id suffix skips at the moment. Tests that deliberately use a -# bad API key to assert a failure callback fires are handled transparently -# by the ``key_fingerprint`` matcher in ``tests/_vcr_conftest_common.py``. _VCR_INCOMPATIBLE_NODEID_SUFFIXES: tuple[str, ...] = () diff --git a/tests/test_litellm/test_vcr_safe_body_matcher.py b/tests/test_litellm/test_vcr_safe_body_matcher.py index e10950582dec..fcf240c7415f 100644 --- a/tests/test_litellm/test_vcr_safe_body_matcher.py +++ b/tests/test_litellm/test_vcr_safe_body_matcher.py @@ -1,11 +1,4 @@ -"""Unit tests for the shared VCR helpers in ``tests/_vcr_conftest_common``. - -The most important guarantee here is that the custom ``safe_body`` matcher -gracefully handles JSON Lines (and other non-strict-JSON) request bodies -without raising ``json.JSONDecodeError`` — vcrpy's default ``body`` matcher -crashes on those because it unconditionally runs ``json.loads`` for any -``application/json`` request body. -""" +"""Unit tests for the shared VCR helpers in ``tests/_vcr_conftest_common``.""" from __future__ import annotations @@ -15,9 +8,6 @@ import pytest -# Tests live in ``tests/test_litellm/`` but ``_vcr_conftest_common`` lives in -# the parent ``tests/`` package. Make sure both are importable regardless of -# how pytest is invoked. _REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if _REPO_ROOT not in sys.path: sys.path.insert(0, _REPO_ROOT) @@ -37,6 +27,10 @@ def _req(body): return SimpleNamespace(body=body, headers={"Content-Type": "application/json"}) +def _req_with_headers(headers, body=b""): + return SimpleNamespace(headers=dict(headers), body=body) + + def test_safe_body_matcher_is_in_match_on(): cfg = vcr_config_dict() assert SAFE_BODY_MATCHER_NAME in cfg["match_on"] @@ -52,12 +46,6 @@ def test_safe_body_matcher_accepts_str_bytes_equivalent(): def test_safe_body_matcher_handles_jsonl_without_crashing(): - """vcrpy's default ``body`` matcher raises ``JSONDecodeError`` on JSONL. - - The Bedrock batch S3 PUT sends a JSON Lines body under - ``Content-Type: application/json``. The safe matcher must compare such - bodies as bytes and never invoke ``json.loads``. - """ jsonl = ( b'{"recordId": "request-1", "modelInput": {}}\n' b'{"recordId": "request-2", "modelInput": {}}\n' @@ -82,24 +70,12 @@ def test_safe_body_matcher_treats_none_bodies_as_equal(): def test_safe_body_matcher_does_not_normalize_json_key_order(): - """The safe matcher is strictly more conservative than vcrpy's default. - - Two semantically-equal JSON bodies with different key order are - treated as *different* requests (cache miss, never a false hit). - """ with pytest.raises(AssertionError): _safe_body_matcher(_req(b'{"a":1,"b":2}'), _req(b'{"b":2,"a":1}')) def test_default_vcrpy_body_matcher_crashes_on_jsonl_for_documentation(): - """Document the behavior we are working around. - - vcrpy's stock body matcher raises ``json.JSONDecodeError`` (not even - a clean ``AssertionError``) when given a JSONL payload typed as - ``application/json``. This is precisely the crash that broke - ``tests/batches_tests/test_bedrock_files_and_batches.py::test_async_create_file`` - and is the reason ``safe_body`` exists. - """ + """Pin the upstream behavior our ``safe_body`` matcher exists to work around.""" import json from vcr.matchers import body as vcrpy_body # type: ignore @@ -109,22 +85,12 @@ def test_default_vcrpy_body_matcher_crashes_on_jsonl_for_documentation(): vcrpy_body(_req(jsonl), _req(jsonl)) -# --------------------------------------------------------------------------- -# Key-fingerprint matcher -# --------------------------------------------------------------------------- - - -def _req_with_headers(headers, body=b""): - return SimpleNamespace(headers=dict(headers), body=body) - - def test_key_fingerprint_matcher_is_in_match_on(): cfg = vcr_config_dict() assert KEY_FINGERPRINT_MATCHER_NAME in cfg["match_on"] def test_before_record_request_strips_auth_and_adds_fingerprint(): - """The hook must scrub the secret AND stamp a fingerprint.""" req = _req_with_headers( { "Authorization": "Bearer sk-real-key-1234567890", @@ -133,30 +99,22 @@ def test_before_record_request_strips_auth_and_adds_fingerprint(): } ) out = _before_record_request(req) - assert ( - "Authorization" not in out.headers - ), "Authorization must be removed before the cassette is recorded" - assert "x-amz-date" not in out.headers, ( - "AWS SigV4 timestamp must be scrubbed (it changes every call and " - "would defeat caching)" - ) + assert "Authorization" not in out.headers + assert "x-amz-date" not in out.headers fp = out.headers.get(KEY_FINGERPRINT_HEADER) assert fp and isinstance(fp, str) assert len(fp) >= 8 - assert "sk-real" not in fp, "fingerprint must not leak the secret" + assert "sk-real" not in fp def test_before_record_request_no_auth_yields_stable_no_key_bucket(): a = _before_record_request(_req_with_headers({"Content-Type": "application/json"})) b = _before_record_request(_req_with_headers({})) assert a.headers[KEY_FINGERPRINT_HEADER] == b.headers[KEY_FINGERPRINT_HEADER] - # Two no-auth requests must match so we don't defeat caching for - # SigV4-style requests where auth lives in headers we've stripped. _key_fingerprint_matcher(a, b) def test_key_fingerprint_matcher_distinguishes_good_and_bad_keys(): - """The whole point: bad-key calls must not replay good-key cassettes.""" good = _before_record_request( _req_with_headers({"Authorization": "Bearer sk-real-good-key"}) ) @@ -179,7 +137,6 @@ def test_key_fingerprint_matcher_matches_repeated_good_key_calls(): def test_key_fingerprint_matcher_distinguishes_x_api_key_callers(): - """Anthropic / Azure use ``x-api-key`` (or ``api-key``) instead of Authorization.""" a = _before_record_request(_req_with_headers({"x-api-key": "anthropic-real"})) b = _before_record_request(_req_with_headers({"x-api-key": "anthropic-bad"})) with pytest.raises(AssertionError): @@ -187,11 +144,6 @@ def test_key_fingerprint_matcher_distinguishes_x_api_key_callers(): def test_before_record_request_is_idempotent_under_replay(): - """vcrpy runs ``before_record_request`` on both record and replay paths. - - The fingerprint must be deterministic so a request made today matches - a cassette recorded yesterday from the same key. - """ payload = {"Authorization": "Bearer sk-deterministic"} first = _before_record_request(_req_with_headers(payload)) second = _before_record_request(_req_with_headers(payload)) From 8cd2f92157c79ea25417d045a3720a387b688d2b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 5 May 2026 06:31:26 +0000 Subject: [PATCH 10/12] test(vcr): make _before_record_request idempotent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vcrpy invokes before_record_request more than once per request: can_play_response_for calls it, then __contains__ / _responses (reached via play_response) call it again on the result. The second invocation sees a request whose auth headers we already stripped, so a naive recompute yields "no-key" and overwrites the real fingerprint stored in the header. This makes can_play_response_for and play_response disagree on matchability — the former says "yes, we have a stored response for this" (matching no-key to no-key) and the latter throws UnhandledHTTPRequestError because it computes a fresh real fingerprint that doesn't match the stored no-key. In CI this manifested as ~30 failing tests across guardrails_testing, audio_testing, batches_testing, image_gen_testing, llm_responses_api, litellm_router_unit_testing, etc. Skip the recompute when the header is already set, so re-applying the hook is a no-op. Adds a regression test that fires the hook twice on the same dict and asserts the fingerprint stays put. Co-authored-by: Mateo Wang --- tests/_vcr_conftest_common.py | 28 +++++++++++++------ .../test_vcr_safe_body_matcher.py | 20 ++++++++++++- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index 121a493354d9..28136dee7265 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -175,19 +175,29 @@ def _strip_headers(headers, names: Iterable[str]) -> None: def _before_record_request(request): """Fingerprint API keys, then scrub them. - Order matters: vcrpy's ``filter_headers`` config option runs *before* - ``before_record_request`` and would erase the auth value before we - could hash it. Doing both steps here keeps the fingerprint available - while ensuring the secret never reaches the cassette. + Order matters in two ways: + + 1. vcrpy's ``filter_headers`` config option runs *before* + ``before_record_request``, so the auth-header scrubbing has to + live here; otherwise the secret would already be gone when we + try to hash it. + 2. vcrpy invokes this hook more than once per request (e.g. + ``can_play_response_for`` calls it, then ``_responses`` calls it + again on the result). The second invocation sees a request whose + auth headers we already stripped, so re-hashing would yield + ``"no-key"`` and the stored vs. incoming fingerprints would + diverge. Skip the recompute when the header is already set so + this hook is idempotent. """ headers = getattr(request, "headers", None) if headers is None: return request - fingerprint = _compute_key_fingerprint(request) - try: - headers[KEY_FINGERPRINT_HEADER] = fingerprint - except (TypeError, AttributeError): - pass + if not any(_iter_header_values(headers, KEY_FINGERPRINT_HEADER)): + fingerprint = _compute_key_fingerprint(request) + try: + headers[KEY_FINGERPRINT_HEADER] = fingerprint + except (TypeError, AttributeError): + pass _strip_headers(headers, FILTERED_REQUEST_HEADERS) return request diff --git a/tests/test_litellm/test_vcr_safe_body_matcher.py b/tests/test_litellm/test_vcr_safe_body_matcher.py index fcf240c7415f..41771378ed91 100644 --- a/tests/test_litellm/test_vcr_safe_body_matcher.py +++ b/tests/test_litellm/test_vcr_safe_body_matcher.py @@ -143,10 +143,28 @@ def test_key_fingerprint_matcher_distinguishes_x_api_key_callers(): _key_fingerprint_matcher(a, b) -def test_before_record_request_is_idempotent_under_replay(): +def test_before_record_request_is_deterministic_across_distinct_requests(): payload = {"Authorization": "Bearer sk-deterministic"} first = _before_record_request(_req_with_headers(payload)) second = _before_record_request(_req_with_headers(payload)) assert ( first.headers[KEY_FINGERPRINT_HEADER] == second.headers[KEY_FINGERPRINT_HEADER] ) + + +def test_before_record_request_is_idempotent_on_the_same_request_object(): + """vcrpy invokes ``before_record_request`` more than once per request. + + ``can_play_response_for`` calls it, then ``__contains__`` / + ``_responses`` call it again on the result. The second call sees a + request whose auth headers are already gone, so a naive recompute + would produce ``"no-key"`` and the matcher would consider the + request distinct from anything it just stored — manifesting in CI as + ``UnhandledHTTPRequestError`` from ``play_response``. + """ + req = _req_with_headers({"Authorization": "Bearer sk-someone"}) + _before_record_request(req) + fp_after_first = req.headers[KEY_FINGERPRINT_HEADER] + _before_record_request(req) + assert req.headers[KEY_FINGERPRINT_HEADER] == fp_after_first + assert fp_after_first != "no-key" From 3f2d257538c5130241d5cf5a37ace8075a4d479e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 5 May 2026 18:23:46 +0000 Subject: [PATCH 11/12] test(vcr): drop more redundant docstrings and headers --- tests/_vcr_conftest_common.py | 2 -- tests/_vcr_redis_persister.py | 2 -- tests/audio_tests/conftest.py | 6 ------ tests/llm_translation/test_vcr_conftest_common_banner.py | 2 -- tests/llm_translation/test_vcr_redis_persister.py | 5 ----- tests/pass_through_unit_tests/conftest.py | 6 ------ tests/test_litellm/test_vcr_safe_body_matcher.py | 2 -- 7 files changed, 25 deletions(-) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index 28136dee7265..73a5635ab671 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -217,7 +217,6 @@ def _fp(req): def vcr_config_dict() -> dict: - """Return the VCR config dict shared across all consuming conftests.""" return { "decode_compressed_response": True, "record_mode": "new_episodes", @@ -238,7 +237,6 @@ def vcr_config_dict() -> dict: def vcr_disabled() -> bool: - """VCR is disabled when explicitly turned off, or when no Redis is configured.""" if os.environ.get("LITELLM_VCR_DISABLE") == "1": return True return not os.environ.get("CASSETTE_REDIS_URL") diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index bf98905942b1..7fdb7267a382 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -49,12 +49,10 @@ def _record_cache_failure(kind: str, exc: BaseException) -> None: def cassette_cache_health() -> dict: - """Return a snapshot of cassette-cache failure counters for this process.""" return dict(_cache_health) def reset_cassette_cache_health() -> None: - """Reset cassette-cache counters. Intended for tests.""" _cache_health["save_failures"] = 0 _cache_health["save_failure_last_error"] = "" _cache_health["load_failures"] = 0 diff --git a/tests/audio_tests/conftest.py b/tests/audio_tests/conftest.py index 5b36a5d434d5..d07057a4b637 100644 --- a/tests/audio_tests/conftest.py +++ b/tests/audio_tests/conftest.py @@ -1,9 +1,3 @@ -# conftest.py -# -# Wires audio tests into the Redis-backed VCR cache so live provider -# calls are replayed for 24h. See tests/llm_translation/Readme.md for -# the design overview. - import os import sys diff --git a/tests/llm_translation/test_vcr_conftest_common_banner.py b/tests/llm_translation/test_vcr_conftest_common_banner.py index 86b79505904e..70ee39abd39b 100644 --- a/tests/llm_translation/test_vcr_conftest_common_banner.py +++ b/tests/llm_translation/test_vcr_conftest_common_banner.py @@ -18,8 +18,6 @@ class _FakeTerminalReporter: - """Minimal stand-in for pytest's TerminalReporter.""" - def __init__(self) -> None: self.buf = StringIO() diff --git a/tests/llm_translation/test_vcr_redis_persister.py b/tests/llm_translation/test_vcr_redis_persister.py index ac97a5efcaeb..ec86ee735974 100644 --- a/tests/llm_translation/test_vcr_redis_persister.py +++ b/tests/llm_translation/test_vcr_redis_persister.py @@ -302,11 +302,6 @@ def test_only_2xx_responses_are_cached(status_code, expect_dropped): assert result is response -# --------------------------------------------------------------------------- -# Cache-health observability -# --------------------------------------------------------------------------- - - @pytest.fixture def reset_health(): reset_cassette_cache_health() diff --git a/tests/pass_through_unit_tests/conftest.py b/tests/pass_through_unit_tests/conftest.py index 617d06517362..d07057a4b637 100644 --- a/tests/pass_through_unit_tests/conftest.py +++ b/tests/pass_through_unit_tests/conftest.py @@ -1,9 +1,3 @@ -# conftest.py -# -# Wires pass-through unit tests into the Redis-backed VCR cache so live -# provider calls are replayed for 24h. See tests/llm_translation/Readme.md -# for the design overview. - import os import sys diff --git a/tests/test_litellm/test_vcr_safe_body_matcher.py b/tests/test_litellm/test_vcr_safe_body_matcher.py index 41771378ed91..0ed6ad69e3c2 100644 --- a/tests/test_litellm/test_vcr_safe_body_matcher.py +++ b/tests/test_litellm/test_vcr_safe_body_matcher.py @@ -1,5 +1,3 @@ -"""Unit tests for the shared VCR helpers in ``tests/_vcr_conftest_common``.""" - from __future__ import annotations import os From 23c5d383c13f3a8ffdecb68754c1c6c81235fd24 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 5 May 2026 18:51:47 +0000 Subject: [PATCH 12/12] test(vcr): enable 24hr cache for ocr_tests and search_tests These two directories were the only non-dockerized test suites in the build_and_test workflow that make live LLM/provider API calls but were not VCR-enabled by this PR. Together they account for 96 tests: - tests/ocr_tests/ (31): Mistral OCR, Azure AI OCR, Azure Document Intelligence, Vertex AI OCR. Pure-unit tests inside the same files (e.g. TestAzureDocumentIntelligencePagesParam) make no HTTP calls and become benign VCR NOOPs. - tests/search_tests/ (65): Brave, DataForSEO, DuckDuckGo, Exa, Firecrawl, Google PSE, Linkup, Parallel.ai, Perplexity, SearchAPI, Searxng, Serper, Tavily. Both directories use the canonical minimal conftest pattern from tests/audio_tests/conftest.py with no skip lists. None of the test files use respx, none assert on per-call upstream non-determinism (no response1.id != response2.id, no overhead-as-fraction-of-total, no live polling), so the default match_on tuple should cache cleanly. If a flake surfaces during the first cassette-recording CI run, we can add a targeted skip the same way we did for the other dirs. Co-authored-by: Mateo Wang --- tests/ocr_tests/conftest.py | 57 +++++++++++++++++++++++++++++++++ tests/search_tests/conftest.py | 58 ++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 tests/ocr_tests/conftest.py create mode 100644 tests/search_tests/conftest.py diff --git a/tests/ocr_tests/conftest.py b/tests/ocr_tests/conftest.py new file mode 100644 index 000000000000..db48e2db2a57 --- /dev/null +++ b/tests/ocr_tests/conftest.py @@ -0,0 +1,57 @@ +# conftest.py +# +# Wires OCR tests into the Redis-backed VCR cache so live provider +# calls (Mistral OCR, Azure AI OCR, Azure Document Intelligence, +# Vertex AI OCR) are replayed for 24h. See tests/llm_translation/Readme.md +# for the design overview. + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +from tests._vcr_conftest_common import ( # noqa: E402 + VerboseReporterState, + apply_vcr_auto_marker_to_items, + record_vcr_outcome, + register_persister_if_enabled, + vcr_config_dict, +) + +_verbose_state = VerboseReporterState() + + +@pytest.fixture(scope="module") +def vcr_config(): + return vcr_config_dict() + + +def pytest_recording_configure(config, vcr): + register_persister_if_enabled(vcr) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + rep = outcome.get_result() + setattr(item, f"rep_{rep.when}", rep) + + +@pytest.fixture(autouse=True) +def _vcr_outcome_gate(request, vcr): + yield + record_vcr_outcome(request, vcr) + + +def pytest_configure(config): + _verbose_state.remember_pluginmanager(config) + + +def pytest_runtest_logreport(report): + _verbose_state.maybe_emit_verdict(report) + + +def pytest_collection_modifyitems(config, items): + apply_vcr_auto_marker_to_items(items) diff --git a/tests/search_tests/conftest.py b/tests/search_tests/conftest.py new file mode 100644 index 000000000000..3b4623c53a5b --- /dev/null +++ b/tests/search_tests/conftest.py @@ -0,0 +1,58 @@ +# conftest.py +# +# Wires search tests into the Redis-backed VCR cache so live provider +# calls (Brave, DataForSEO, DuckDuckGo, Exa, Firecrawl, Google PSE, +# Linkup, Parallel.ai, Perplexity, SearchAPI, Searxng, Serper, Tavily) +# are replayed for 24h. See tests/llm_translation/Readme.md for the +# design overview. + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +from tests._vcr_conftest_common import ( # noqa: E402 + VerboseReporterState, + apply_vcr_auto_marker_to_items, + record_vcr_outcome, + register_persister_if_enabled, + vcr_config_dict, +) + +_verbose_state = VerboseReporterState() + + +@pytest.fixture(scope="module") +def vcr_config(): + return vcr_config_dict() + + +def pytest_recording_configure(config, vcr): + register_persister_if_enabled(vcr) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + rep = outcome.get_result() + setattr(item, f"rep_{rep.when}", rep) + + +@pytest.fixture(autouse=True) +def _vcr_outcome_gate(request, vcr): + yield + record_vcr_outcome(request, vcr) + + +def pytest_configure(config): + _verbose_state.remember_pluginmanager(config) + + +def pytest_runtest_logreport(report): + _verbose_state.maybe_emit_verdict(report) + + +def pytest_collection_modifyitems(config, items): + apply_vcr_auto_marker_to_items(items)