From 4254c4ae782f9893ab6ae40268138559545aa491 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 17 May 2026 05:56:27 +0000 Subject: [PATCH 01/17] fix(tests): stabilize image-edit VCR cassettes to stop live gpt-image-1 spend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The image-edit cassettes for ``gpt-image-1`` were accumulating >50 episodes and being refused by the persister (``tests/_vcr_redis_persister.py``), so every CI run was hitting the real OpenAI endpoint. The async parametrize was the clearest tell: ``test_openai_image_edit_litellm_sdk[True]`` cached to 1 entry, but the ``[False]`` (async) sibling grew to 51 entries and never replayed. Two non-deterministic sources were fueling the growth, both fixed here. After this patch, the cassettes settle at one episode per unique call and replay for the 24-hour TTL like every other suite. 1. Pin httpx's multipart boundary at the source. The existing ``_normalize_multipart_boundary`` rewrites the boundary in the ``Content-Type`` header reliably, but on the async transport path the body is not always a contiguous ``bytes`` object when ``before_record_request`` runs, so the body-side replacement silently no-ops and the recorded cassette retains the random ``boundary=`` string. The next CI run gets a fresh random boundary, the ``safe_body`` matcher misses, and ``record_mode="new_episodes"`` appends another episode. Wrapping ``httpx._multipart.MultipartStream.__init__`` so it always uses ``vcr-static-boundary`` when no boundary is supplied eliminates the variance for both sync and async paths and leaves the normalizer in place as a backstop. Exposed as ``pin_httpx_multipart_boundary`` so other multipart-heavy suites (audio, ocr, batches) can adopt the same fixture later. 2. Pass raw ``bytes`` (not ``BytesIO`` streams) through the image-edit fixtures. A ``BytesIO`` whose file pointer is at EOF after the first multipart upload silently encodes an empty image on the next SDK / Router retry — yet another divergent body that VCR records as a new episode. ``bytes`` are immutable and position-less, so retries re-encode an identical payload every time. This is also a small production-correctness improvement: a customer passing ``BytesIO`` today would hit the same empty-body retry bug. The BytesIO-specific smoke test (``test_openai_image_edit_with_bytesio``) is preserved by giving ``get_test_images_as_bytesio`` its own factory instead of aliasing the bytes one. 3. Add ``scripts/flush_image_edit_vcr_cassettes.py`` — a one-shot Redis SCAN/DEL helper that clears the bloated pre-fix cassettes under ``litellm:vcr:cassette:tests/image_gen_tests/test_image_edits/*``. Without this, the next CI run still loads the existing 51-entry cassette, the new fixed-boundary body still doesn't match any of the stale entries, the persister still refuses to save, and the bleed continues. Run once with the production ``CASSETTE_REDIS_URL`` after merge (dry-run by default). --- scripts/flush_image_edit_vcr_cassettes.py | 131 ++++++++++++++++++++++ tests/_vcr_conftest_common.py | 49 ++++++++ tests/image_gen_tests/conftest.py | 18 +++ tests/image_gen_tests/test_image_edits.py | 60 +++++----- 4 files changed, 232 insertions(+), 26 deletions(-) create mode 100755 scripts/flush_image_edit_vcr_cassettes.py diff --git a/scripts/flush_image_edit_vcr_cassettes.py b/scripts/flush_image_edit_vcr_cassettes.py new file mode 100755 index 000000000000..7b87f36b2841 --- /dev/null +++ b/scripts/flush_image_edit_vcr_cassettes.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""Flush the bloated image-edit VCR cassettes from the cassette Redis. + +Run this **once** after merging the multipart-boundary stabilization +PR. The pre-fix cassettes for the async image-edit tests have +accumulated >50 episodes (random multipart boundary on every run + +``record_mode="new_episodes"`` = monotonic growth), so the persister +refuses to save updates -- meaning every CI run after the fix would +still try to re-record against the stale 51-entry cassette, hit +``MAX_EPISODES_PER_CASSETTE`` again, get refused, and re-bill the live +provider. + +Deleting these keys forces the next CI run to record a clean cassette +under the new fixed-boundary + raw-bytes fixtures (1 episode per +unique call), after which the 24-hour TTL replay loop kicks in +normally. + +Scope is intentionally narrow: + * Only ``tests/image_gen_tests/test_image_edits/*`` cassette keys + are touched. Image-*generation* cassettes (TestOpenAIGPTImage1 + etc.) are unaffected -- they were already in the VCR HIT state. + * Lists every match in dry-run mode before deleting anything so the + operator can confirm the impact. + +Usage: + CASSETTE_REDIS_URL=redis://... \ + uv run python scripts/flush_image_edit_vcr_cassettes.py --dry-run + + CASSETTE_REDIS_URL=redis://... \ + uv run python scripts/flush_image_edit_vcr_cassettes.py --yes + +``CASSETTE_REDIS_URL`` is the same env var the persister reads at CI +start (see ``tests/_vcr_redis_persister.py``). +""" + +from __future__ import annotations + +import argparse +import os +import sys + +import redis + + +CASSETTE_REDIS_URL_ENV = "CASSETTE_REDIS_URL" +REDIS_KEY_PREFIX = "litellm:vcr:cassette:" +TARGET_KEY_PATTERN = f"{REDIS_KEY_PREFIX}tests/image_gen_tests/test_image_edits/*" + + +def _build_client(url: str) -> redis.Redis: + return redis.Redis.from_url( + url, + socket_timeout=10, + socket_connect_timeout=10, + decode_responses=False, + ) + + +def _scan_matching_keys(client: redis.Redis, pattern: str) -> list[bytes]: + return sorted(client.scan_iter(match=pattern, count=500)) + + +def _delete_keys(client: redis.Redis, keys: list[bytes]) -> int: + if not keys: + return 0 + # Batch into chunks so a single DEL call does not exceed the + # server's argument-count or buffer limits on large key sets. + deleted = 0 + chunk_size = 200 + for start in range(0, len(keys), chunk_size): + batch = keys[start : start + chunk_size] + deleted += int(client.delete(*batch)) + return deleted + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--yes", + action="store_true", + help="Actually delete the matched keys. Without this flag the script " + "runs in dry-run mode and only lists what would be deleted.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="List matched keys without deleting (default behaviour when " + "--yes is omitted; kept as an explicit flag for clarity).", + ) + parser.add_argument( + "--pattern", + default=TARGET_KEY_PATTERN, + help=f"Override the SCAN match pattern. Default: {TARGET_KEY_PATTERN}", + ) + args = parser.parse_args(argv) + + url = os.environ.get(CASSETTE_REDIS_URL_ENV) + if not url: + print( + f"error: {CASSETTE_REDIS_URL_ENV} is not set. Set it to the " + "cassette Redis URL (same URL the persister reads in CI).", + file=sys.stderr, + ) + return 2 + + client = _build_client(url) + try: + client.ping() + except redis.RedisError as exc: + print(f"error: cannot reach cassette Redis: {exc}", file=sys.stderr) + return 2 + + matches = _scan_matching_keys(client, args.pattern) + print(f"matched {len(matches)} key(s) under pattern: {args.pattern}") + for key in matches: + print(f" {key.decode('utf-8', errors='replace')}") + + if not matches: + return 0 + + if not args.yes: + print("\ndry run -- pass --yes to actually delete these keys.") + return 0 + + deleted = _delete_keys(client, matches) + print(f"\ndeleted {deleted} key(s).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index a179a21ba698..9b706e3e5cbd 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -91,6 +91,55 @@ VCR_FIXED_MULTIPART_BOUNDARY = "vcr-static-boundary" +def pin_httpx_multipart_boundary(monkeypatch) -> None: + """Force every httpx multipart request to use a constant boundary. + + httpx's ``MultipartStream`` generates a fresh ``boundary=`` + via ``os.urandom(16)`` whenever the caller does not supply one + (see ``httpx._multipart.MultipartStream.__init__``). That random + boundary appears both in the ``Content-Type`` header and verbatim in + the request body between each part. + + ``_normalize_multipart_boundary`` rewrites the header reliably, but + on the async transport path the body is not always handed to + ``before_record_request`` as a contiguous ``bytes`` object — so the + body replacement silently no-ops and the recorded cassette retains + the random boundary string. Subsequent runs generate a *different* + random boundary, the ``safe_body`` matcher misses, and + ``record_mode="new_episodes"`` appends a fresh episode until the + cassette crosses ``MAX_EPISODES_PER_CASSETTE`` and the persister + refuses to save — re-billing live providers on every CI run. + + Pinning the boundary at the source removes the variance entirely: + every run emits byte-identical multipart bodies, the existing + ``safe_body`` matcher succeeds on the first request, and one + recorded episode per unique call satisfies replays for the cassette + TTL. + + This wraps ``MultipartStream.__init__`` instead of patching the + boundary-generation helper directly because httpx inlines + ``os.urandom(16).hex().encode("ascii")`` in the constructor body + rather than calling a named function. We preserve the caller's + boundary when one is explicitly supplied so production-style code + that pins its own boundary keeps working. + """ + try: + import httpx._multipart as _httpx_multipart + except ImportError: # pragma: no cover - httpx is a hard test dep + return + + _original_init = _httpx_multipart.MultipartStream.__init__ + + def _init_with_fixed_boundary(self, data, files, boundary=None): + if boundary is None: + boundary = VCR_FIXED_MULTIPART_BOUNDARY.encode("ascii") + return _original_init(self, data=data, files=files, boundary=boundary) + + monkeypatch.setattr( + _httpx_multipart.MultipartStream, "__init__", _init_with_fixed_boundary + ) + + def _scrub_response(response): if not isinstance(response, dict): return response diff --git a/tests/image_gen_tests/conftest.py b/tests/image_gen_tests/conftest.py index 93dec98e708a..23e34c86dd4d 100644 --- a/tests/image_gen_tests/conftest.py +++ b/tests/image_gen_tests/conftest.py @@ -15,6 +15,7 @@ emit_cassette_cache_session_banner, emit_vcr_classification_summary, install_live_call_probe, + pin_httpx_multipart_boundary, record_vcr_outcome, register_persister_if_enabled, vcr_config_dict, @@ -33,6 +34,23 @@ def event_loop(): loop.close() +@pytest.fixture(scope="session", autouse=True) +def _pin_multipart_boundary(): + """Pin httpx's random multipart boundary to a constant for the + entire image-gen test session. Without this, async multipart bodies + contain a fresh ``boundary=`` on every run; the + ``safe_body`` matcher misses, and ``record_mode="new_episodes"`` + grows each cassette by one entry per run until it crosses the + 50-episode persister cap and stops being saved — leaving the test + to hit the real provider on every CI run. See + ``pin_httpx_multipart_boundary`` for the full rationale. + """ + monkeypatch = pytest.MonkeyPatch() + pin_httpx_multipart_boundary(monkeypatch) + yield + monkeypatch.undo() + + @pytest.fixture(scope="module") def vcr_config(): return vcr_config_dict() diff --git a/tests/image_gen_tests/test_image_edits.py b/tests/image_gen_tests/test_image_edits.py index 656b8a69117a..7c3632b82fce 100644 --- a/tests/image_gen_tests/test_image_edits.py +++ b/tests/image_gen_tests/test_image_edits.py @@ -103,12 +103,16 @@ async def test_openai_image_edit_litellm_sdk(self, sync_mode): pwd = os.path.dirname(os.path.realpath(__file__)) -# Image fixtures must be regenerated per access — module-level -# ``open(...)`` handles get consumed after a single multipart upload, leaving -# subsequent tests in the same process to send empty bodies. That non-determinism -# (a) blows the recorded cassette past ``MAX_EPISODES_PER_CASSETTE`` so the -# persister refuses to save (see ``tests/_vcr_redis_persister.py``), and -# (b) re-bills the live image edit endpoint on every CI run. +# Image fixtures are returned as raw ``bytes`` (not file handles or +# ``BytesIO`` streams) so that every SDK / Router retry sees the same +# payload. A ``BytesIO`` whose file pointer is left at EOF by the first +# multipart upload silently encodes an empty image on the second +# attempt, producing a different request body — VCR records that +# divergent body as a fresh episode, the cassette eventually crosses +# ``MAX_EPISODES_PER_CASSETTE`` in ``tests/_vcr_redis_persister.py``, +# the persister refuses to save, and every subsequent CI run re-bills +# the live image-edit endpoint. Raw bytes are immutable, position-less, +# and re-encoded identically on every retry attempt. def _read_image_bytes(filename: str) -> bytes: with open(os.path.join(pwd, filename), "rb") as f: return f.read() @@ -119,30 +123,34 @@ def _read_image_bytes(filename: str) -> bytes: def _make_test_images() -> list: - """Return a fresh pair of image streams seeded with the fixture bytes. - - Use this everywhere you'd previously have used the module-level - ``TEST_IMAGES``. Each call returns brand new ``BytesIO`` objects whose - file pointers start at 0, so multipart uploads encode the full image - bytes on every test invocation. Parametrized and ``flaky``-retried - test methods call ``get_base_image_edit_call_args`` once per - invocation, so a fresh stream per call is sufficient — the factory - must not auto-rewind on EOF or the SDK's multipart writer will read - the same bytes forever (worker OOM). + """Return the pair of fixture images as raw ``bytes`` payloads. + + ``httpx`` accepts a ``bytes`` value anywhere a file-like upload is + expected and re-encodes it identically on every multipart attempt + — so SDK-level retries can never produce a divergent empty-body + episode (the root cause of the cassette-overflow leak that bills + ``gpt-image-1`` on every CI run). """ - return [ - BytesIO(_ISHAAN_GITHUB_BYTES), - BytesIO(_LITELLM_SITE_BYTES), - ] + return [_ISHAAN_GITHUB_BYTES, _LITELLM_SITE_BYTES] -def _make_single_test_image() -> BytesIO: - return BytesIO(_ISHAAN_GITHUB_BYTES) +def _make_single_test_image() -> bytes: + return _ISHAAN_GITHUB_BYTES def get_test_images_as_bytesio(): - """Helper function to get test images as BytesIO objects""" - return _make_test_images() + """Return the fixture images as fresh ``BytesIO`` streams. + + Kept distinct from ``_make_test_images`` so the BytesIO-specific + smoke tests (``test_openai_image_edit_with_bytesio``, + ``test_multiple_image_edit_with_different_formats``) still exercise + the file-like upload path. Each call yields brand new streams so + the file pointer always starts at 0 for that test invocation. + """ + return [ + BytesIO(_ISHAAN_GITHUB_BYTES), + BytesIO(_LITELLM_SITE_BYTES), + ] class TestOpenAIImageEditGPTImage1(BaseLLMImageEditTest): @@ -710,9 +718,9 @@ async def test_multiple_image_edit_with_different_formats(): try: prompt = "Create a cohesive artistic style across all images" - # Test with mixed BytesIO and file objects + # Test with mixed raw-bytes and BytesIO inputs mixed_images = [ - _make_single_test_image(), # File object + _make_single_test_image(), # raw ``bytes`` payload get_test_images_as_bytesio()[1], # BytesIO object ] From ba3915d9c6a51292736bb2240acc4a839e5ee7c2 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 17 May 2026 06:51:48 +0000 Subject: [PATCH 02/17] DIAGNOSTIC: log VCR body mismatches + per-episode body hashes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Temporary observability boost so we can root-cause why ``test_image_edits.py`` async parametrizes still record fresh episodes on every CI run even though the multipart boundary is now pinned (sync parametrizes cache cleanly as VCR HIT). The matcher currently raises ``AssertionError("request bodies differ")`` with zero context, so we cannot tell whether the live body genuinely varies, the matcher is comparing a bytes object to a stream object, or the normalizer is silently skipping the body because it is not bytes/str. Three logs added; the first two are worth keeping permanently, the third is intended to be reverted after the diagnosis lands: 1. ``_safe_body_matcher`` now emits a structured stderr block on mismatch (type of each side, length, SHA-256, first divergent byte offset, ±100-byte window). Always-on -- mismatches are signal, not noise, and the existing per-test verdict already logs once per test. PERMANENT. 2. ``_normalize_multipart_boundary`` now logs to stderr when the body type is not bytes/bytearray/str -- the silent ``else: return`` branch was masking exactly the case we suspect is firing on async (httpx ``MultipartStream`` handed to vcrpy before the body is read). PERMANENT. 3. ``_RedisPersister.save_cassette`` now logs every episode's body SHA-256, length, and 120-byte preview at save time. This lets two consecutive CI runs be diffed: if the same test records a different hash run-to-run, the live body genuinely varies; if both runs record the same hash but the matcher still misses, the bug is in the matcher itself. TEMPORARY -- revert once the async variance is identified and fixed. Once a single ``image_gen_testing`` CI run produces these logs, revert this commit (or just the persister hash block) with a force push so the cassette save path is not noisy in steady-state. --- tests/_vcr_conftest_common.py | 76 +++++++++++++++++++++++++++++++++++ tests/_vcr_redis_persister.py | 49 ++++++++++++++++++++++ 2 files changed, 125 insertions(+) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index 9b706e3e5cbd..f00687c7d8b6 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -243,6 +243,13 @@ def _safe_body_matcher(r1, r2) -> None: (e.g. the Bedrock batch S3 PUT) before it can return "no match". This matcher is strictly more conservative — the only equivalence it gives up vs. the default is "JSON key order doesn't matter". + + On mismatch, emits a structured diagnostic to stderr (type of each + body, length, SHA-256, first divergent offset, ±100-byte window). + Without this, vcrpy returns "request bodies differ" with zero + context, and bugs where the live request body is an unbytes-like + object (e.g. an httpx ``MultipartStream`` for async requests) look + indistinguishable from genuine content drift. """ body1 = getattr(r1, "body", None) body2 = getattr(r2, "body", None) @@ -262,9 +269,64 @@ def _to_bytes(b): n2 = _to_bytes(body2) if n1 is not None and n2 is not None and n1 == n2: return + _emit_body_mismatch_diagnostic(r1, r2, body1, body2, n1, n2) raise AssertionError("request bodies differ") +def _emit_body_mismatch_diagnostic(r1, r2, body1, body2, n1, n2) -> None: + """Dump enough info to a single stderr block to root-cause why two + requests that look semantically identical failed the body matcher. + + Always-on (matcher mismatches are signal, not noise): the volume + is bounded by the number of stored episodes a live request is + compared against, and we already log a per-test verdict line for + every test. + """ + + def _describe(label, raw, asbytes): + t = type(raw).__name__ + if asbytes is None: + return ( + f" {label}: type={t!r} length=unknown sha256=N/A " + f"(body could not be coerced to bytes)" + ) + length = len(asbytes) + digest = hashlib.sha256(asbytes).hexdigest() + preview = asbytes[:120] + return ( + f" {label}: type={t!r} length={length} sha256={digest} " + f"preview={preview!r}" + ) + + method_a = getattr(r1, "method", "?") + method_b = getattr(r2, "method", "?") + url_a = getattr(r1, "uri", getattr(r1, "url", "?")) + url_b = getattr(r2, "uri", getattr(r2, "url", "?")) + lines = [ + "[vcr-safe-body-matcher] request body mismatch", + f" request[a]: {method_a} {url_a}", + f" request[b]: {method_b} {url_b}", + _describe("body[a]", body1, n1), + _describe("body[b]", body2, n2), + ] + if n1 is not None and n2 is not None and n1 != n2: + # Find the first divergent byte offset and dump a ±100 window + # around it so the human reading the CI log can see at a glance + # whether the variance is a UUID, a timestamp, a random multipart + # boundary, or something else. + offset = next( + (i for i in range(min(len(n1), len(n2))) if n1[i] != n2[i]), + min(len(n1), len(n2)), + ) + start = max(0, offset - 100) + end_a = min(len(n1), offset + 100) + end_b = min(len(n2), offset + 100) + lines.append(f" first divergent byte offset: {offset}") + lines.append(f" window[a] @ {start}..{end_a}: {n1[start:end_a]!r}") + lines.append(f" window[b] @ {start}..{end_b}: {n2[start:end_b]!r}") + sys.stderr.write("\n".join(lines) + "\n") + + def _iter_header_values(headers, name: str): if headers is None: return @@ -409,6 +471,20 @@ def _normalize_multipart_boundary(request) -> None: elif isinstance(body, str): new_body = body.replace(current_boundary, VCR_FIXED_MULTIPART_BOUNDARY) else: + # The body is something other than bytes/bytearray/str -- most + # likely an httpx ``MultipartStream`` or an aiter chunked stream + # we cannot rewrite in place. Log it so a body-matcher miss on a + # multipart request can be correlated with "normalizer skipped + # because body type was X". The header was still rewritten + # above, so the recorded Content-Type stays stable; only the + # body bytes carry the random boundary verbatim. + sys.stderr.write( + f"[vcr-multipart-normalize] body normalization SKIPPED: " + f"body type {type(body).__name__!r} is not bytes/bytearray/str. " + f"content-type={content_type_value!r}. " + f"Recorded body will retain the random boundary substring " + f"and the safe_body matcher will miss on the next run.\n" + ) return try: diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index 7fdb7267a382..7ff14c0d4ec4 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -168,6 +168,7 @@ def save_cassette(cassette_path, cassette_dict, serializer): key = redis_key_for(cassette_path) passed = _passed_by_cassette_key.pop(key, True) episode_count = len(cassette_dict.get("requests", []) or []) + _maybe_log_episode_body_hashes(key, cassette_dict) if episode_count > MAX_EPISODES_PER_CASSETTE: _log.warning( "VCR redis save refused for %s; cassette has %d episodes " @@ -210,6 +211,54 @@ def save_cassette(cassette_path, cassette_dict, serializer): return _RedisPersister +# TEMP DIAGNOSTIC -- intended to be reverted once the async image-edit +# cassette variance is root-caused. Logs a per-episode body SHA-256 +# at save time so two consecutive CI runs can be diffed: if the same +# test records ``sha=abc`` on run 1 and ``sha=def`` on run 2, the live +# request body genuinely varies; if both runs record the same hash +# but the matcher still misses, the bug is in the matcher (e.g. it is +# comparing a bytes object to a stream object). Always-on for any +# session that loads this persister -- ungated because we are +# specifically trying to capture data from CI right now. +def _maybe_log_episode_body_hashes(key: str, cassette_dict) -> None: + import hashlib + + requests = cassette_dict.get("requests", []) or [] + if not requests: + return + for i, req in enumerate(requests): + body = getattr(req, "body", None) + if body is None: + body_bytes = b"" + elif isinstance(body, (bytes, bytearray)): + body_bytes = bytes(body) + elif isinstance(body, str): + body_bytes = body.encode("utf-8") + else: + _log.warning( + "[vcr-episode-body-hash] %s episode[%d]: body type=%r is " + "not bytes/bytearray/str -- cannot hash. This is the " + "smoking gun for matcher-side bugs on async multipart.", + key, + i, + type(body).__name__, + ) + continue + method = getattr(req, "method", "?") + uri = getattr(req, "uri", getattr(req, "url", "?")) + _log.warning( + "[vcr-episode-body-hash] %s episode[%d] %s %s body sha256=%s " + "len=%d preview=%r", + key, + i, + method, + uri, + hashlib.sha256(body_bytes).hexdigest(), + len(body_bytes), + body_bytes[:120], + ) + + def filter_non_2xx_response(response): if not isinstance(response, dict): return response From 85430bc01fb73e513636ac1befaf3ea606301383 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 17 May 2026 07:08:39 +0000 Subject: [PATCH 03/17] DIAGNOSTIC: route VCR diagnostics through per-PID files (bypass xdist capture) Re-push of the diagnostic logging from the previous commit, this time wired so the output actually survives to the CI log. xdist captures stdout/stderr from every passing test in the worker process; the body-matcher and normalizer-skip diagnostics fire from inside vcrpy machinery during the test, so for any test that ultimately passes (which is all of them once the cassettes are recorded), the diagnostic lines are silently swallowed. Fix: write each diagnostic line to a per-PID file under ``test-results/vcr-diagnostics/.log`` instead of writing to stderr. The controller's ``pytest_terminal_summary`` aggregates those files and writes them through ``terminalreporter.write_line``, which is not subject to per-test capture. As a bonus, ``test-results/`` is already collected by the ``store_test_results`` step in CircleCI, so the raw per-worker logs survive as build artifacts even after the test session ends. Three call sites updated: 1. ``_emit_body_mismatch_diagnostic`` (matcher) -- writes the structured type/length/sha/window block via ``vcr_diag_write_line``. 2. ``_normalize_multipart_boundary`` -- logs the silent-skip path (body not bytes/bytearray/str) the same way. 3. ``_maybe_log_episode_body_hashes`` (persister) -- replaces the ``_log.warning`` calls (which the root-logger config also swallows in CI) with ``vcr_diag_write_line``. Image-gen conftest is the only suite wired to dump the aggregated log at session end. Other suites can opt in by adding ``emit_vcr_diagnostic_log(terminalreporter)`` to their own ``pytest_terminal_summary``. The diagnostic dir is cleared at the start of each session (controller-only) so a local rerun does not mix output from prior runs. Same revert plan as the previous diagnostic commit: keep the matcher + normalizer skip diagnostics permanently (they only fire on signal events), revert the persister body-hash dump once the async variance is identified. --- tests/_vcr_conftest_common.py | 78 +++++++++++++++++++++++++++++-- tests/_vcr_redis_persister.py | 29 +++++------- tests/image_gen_tests/conftest.py | 17 +++++++ 3 files changed, 104 insertions(+), 20 deletions(-) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index f00687c7d8b6..eecfc351fc19 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -36,6 +36,78 @@ KEY_FINGERPRINT_MATCHER_NAME = "key_fingerprint" KEY_FINGERPRINT_HEADER = "x-litellm-key-fp" +# Directory for per-process VCR diagnostic logs that bypass pytest's +# stdout/stderr capture. ``sys.stderr.write`` from inside vcrpy +# machinery is swallowed by xdist for any test that ultimately passes, +# so a diagnostic that fires on a body-matcher miss but the test still +# records and passes will never reach the CI log. Each xdist worker +# (or the main process) writes line-buffered to a per-PID file under +# this directory, and the controller's ``pytest_terminal_summary`` +# concatenates them into the terminal at session end. ``test-results/`` +# is already collected by ``store_test_results`` in CI, so the raw +# files survive as build artifacts too. +VCR_DIAG_DIR_ENV = "LITELLM_VCR_DIAG_DIR" +VCR_DIAG_DIR_DEFAULT = "test-results/vcr-diagnostics" + + +def _vcr_diag_dir() -> str: + return os.environ.get(VCR_DIAG_DIR_ENV) or VCR_DIAG_DIR_DEFAULT + + +def vcr_diag_write_line(msg: str) -> None: + """Append a single diagnostic line to the current process's + per-PID file. Atomic against other workers in the same xdist + session because each PID owns its own file. + + Errors are swallowed -- diagnostic logging must never fail a test. + """ + try: + directory = _vcr_diag_dir() + os.makedirs(directory, exist_ok=True) + path = os.path.join(directory, f"{os.getpid()}.log") + with open(path, "a", encoding="utf-8") as fh: + fh.write(msg.rstrip("\n") + "\n") + except OSError: + pass + + +def emit_vcr_diagnostic_log(terminalreporter) -> None: + """Concatenate every per-PID diagnostic file into the controller's + terminal at session end. Each file is dumped under a header that + names the originating worker PID so cross-process events can still + be ordered if needed. + """ + directory = _vcr_diag_dir() + if not os.path.isdir(directory): + return + try: + files = sorted(f for f in os.listdir(directory) if f.endswith(".log")) + except OSError: + return + if not files: + return + terminalreporter.write_sep("=", "VCR DIAGNOSTIC LOG", bold=True) + terminalreporter.write_line( + f" source dir: {directory} (also archived as a CI artifact)" + ) + for name in files: + path = os.path.join(directory, name) + try: + with open(path, "r", encoding="utf-8") as fh: + content = fh.read() + except OSError as exc: + terminalreporter.write_line( + f" [failed to read {name}: {type(exc).__name__}: {exc}]" + ) + continue + if not content.strip(): + continue + terminalreporter.write_sep("-", name, bold=False) + for line in content.splitlines(): + terminalreporter.write_line(line) + terminalreporter.write_sep("=", bold=True) + + # Intentionally narrower than ``FILTERED_REQUEST_HEADERS``: AWS SigV4 headers # carry secrets but their values rotate on every call, so fingerprinting them # would defeat caching. @@ -324,7 +396,7 @@ def _describe(label, raw, asbytes): lines.append(f" first divergent byte offset: {offset}") lines.append(f" window[a] @ {start}..{end_a}: {n1[start:end_a]!r}") lines.append(f" window[b] @ {start}..{end_b}: {n2[start:end_b]!r}") - sys.stderr.write("\n".join(lines) + "\n") + vcr_diag_write_line("\n".join(lines)) def _iter_header_values(headers, name: str): @@ -478,12 +550,12 @@ def _normalize_multipart_boundary(request) -> None: # because body type was X". The header was still rewritten # above, so the recorded Content-Type stays stable; only the # body bytes carry the random boundary verbatim. - sys.stderr.write( + vcr_diag_write_line( f"[vcr-multipart-normalize] body normalization SKIPPED: " f"body type {type(body).__name__!r} is not bytes/bytearray/str. " f"content-type={content_type_value!r}. " f"Recorded body will retain the random boundary substring " - f"and the safe_body matcher will miss on the next run.\n" + f"and the safe_body matcher will miss on the next run." ) return diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index 7ff14c0d4ec4..1c60e126e3f2 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -223,6 +223,9 @@ def save_cassette(cassette_path, cassette_dict, serializer): def _maybe_log_episode_body_hashes(key: str, cassette_dict) -> None: import hashlib + # Imported lazily to avoid a circular import at module load. + from tests._vcr_conftest_common import vcr_diag_write_line + requests = cassette_dict.get("requests", []) or [] if not requests: return @@ -235,27 +238,19 @@ def _maybe_log_episode_body_hashes(key: str, cassette_dict) -> None: elif isinstance(body, str): body_bytes = body.encode("utf-8") else: - _log.warning( - "[vcr-episode-body-hash] %s episode[%d]: body type=%r is " - "not bytes/bytearray/str -- cannot hash. This is the " - "smoking gun for matcher-side bugs on async multipart.", - key, - i, - type(body).__name__, + vcr_diag_write_line( + f"[vcr-episode-body-hash] {key} episode[{i}]: body type=" + f"{type(body).__name__!r} is not bytes/bytearray/str -- " + "cannot hash. This is the smoking gun for matcher-side " + "bugs on async multipart." ) continue method = getattr(req, "method", "?") uri = getattr(req, "uri", getattr(req, "url", "?")) - _log.warning( - "[vcr-episode-body-hash] %s episode[%d] %s %s body sha256=%s " - "len=%d preview=%r", - key, - i, - method, - uri, - hashlib.sha256(body_bytes).hexdigest(), - len(body_bytes), - body_bytes[:120], + vcr_diag_write_line( + f"[vcr-episode-body-hash] {key} episode[{i}] {method} {uri} " + f"body sha256={hashlib.sha256(body_bytes).hexdigest()} " + f"len={len(body_bytes)} preview={body_bytes[:120]!r}" ) diff --git a/tests/image_gen_tests/conftest.py b/tests/image_gen_tests/conftest.py index 23e34c86dd4d..b1631fe0c599 100644 --- a/tests/image_gen_tests/conftest.py +++ b/tests/image_gen_tests/conftest.py @@ -11,9 +11,11 @@ from tests._vcr_conftest_common import ( # noqa: E402 VerboseReporterState, + _vcr_diag_dir, apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, + emit_vcr_diagnostic_log, install_live_call_probe, pin_httpx_multipart_boundary, record_vcr_outcome, @@ -76,6 +78,20 @@ def _vcr_outcome_gate(request, vcr): def pytest_configure(config): _verbose_state.remember_pluginmanager(config) + # Clear any leftover per-PID diagnostic logs from a previous local + # run so the controller's terminal summary at session end only + # surfaces this session's data. Worker processes inherit the same + # directory and append by PID, so the controller doing the cleanup + # once is sufficient. + if not os.environ.get("PYTEST_XDIST_WORKER"): + directory = _vcr_diag_dir() + if os.path.isdir(directory): + for name in os.listdir(directory): + if name.endswith(".log"): + try: + os.remove(os.path.join(directory, name)) + except OSError: + pass def pytest_runtest_logreport(report): @@ -89,3 +105,4 @@ def pytest_collection_modifyitems(config, items): def pytest_terminal_summary(terminalreporter, exitstatus, config): emit_cassette_cache_session_banner(terminalreporter) emit_vcr_classification_summary(terminalreporter) + emit_vcr_diagnostic_log(terminalreporter) From 8e08272bfd94dcc43a74f48cef757148e5cae0ae Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 17 May 2026 07:19:30 +0000 Subject: [PATCH 04/17] fix(tests): coalesce iterable request bodies before matching/recording Root cause of the residual async image-edit cassette leak. The diagnostic run for ``ba3915d9`` printed: [vcr-safe-body-matcher] request body mismatch body[a]: type='list_iterator' length=unknown sha256=N/A body[b]: type='list_iterator' length=unknown sha256=N/A httpx's async transport hands vcrpy a ``request.body`` that is a ``list_iterator`` over multipart chunks rather than a contiguous ``bytes`` blob. Two consequences: 1. ``_safe_body_matcher`` compares the two iterator objects with ``==``, which is identity comparison for arbitrary iterators - semantically identical multipart bodies never compare equal, and ``record_mode="new_episodes"`` appends a new episode on every CI run until the cassette crosses ``MAX_EPISODES_PER_CASSETTE`` and the persister refuses to save (this is exactly what the OVERFLOW warning has been catching). 2. ``_normalize_multipart_boundary`` short-circuits its ``else: return`` branch because the body is neither bytes nor str, so any residual random boundary characters in the body bytes are never rewritten. Sync requests do not hit this code path: httpx's sync transport hands vcrpy a single ``bytes`` body, so ``==`` works and the boundary normalizer runs as intended. That is why ``test_openai_image_edit_litellm_sdk[True]`` records to ``entries=1`` and replays cleanly while ``[False]`` (async) kept growing by one episode per run. Fix: add ``_materialize_iterable_body`` which coalesces an iterable ``request.body`` into ``bytes`` in-place. Call it from two places: * The top of ``_before_record_request``, so the boundary normalizer and the cassette serializer both see bytes from then on. * The top of ``_safe_body_matcher``, as defense in depth in case a future vcrpy code path invokes the matcher without first going through ``_before_record_request``. The vcrpy ``Request`` is a wrapper used for matching and recording; the underlying httpx transport sends its own request body separately, so replacing the iterator on the vcrpy wrapper does not starve the live HTTP send. After this lands the async parametrizes should flip from ``[VCR MISS:RECORDED] entries=N+1`` to ``[VCR HIT] entries=N`` on the next CI run, matching the sync side and dropping the residual ~$3/day to $0. --- tests/_vcr_conftest_common.py | 57 +++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index eecfc351fc19..23b9564032e1 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -323,6 +323,13 @@ def _safe_body_matcher(r1, r2) -> None: object (e.g. an httpx ``MultipartStream`` for async requests) look indistinguishable from genuine content drift. """ + # Defense in depth: ``_before_record_request`` already coalesces + # iterable bodies, but if vcrpy invokes the matcher on a request + # that did not flow through that hook (or if a future code path + # bypasses it), do it again here so iterator==iterator never + # silently fails. + _materialize_iterable_body(r1) + _materialize_iterable_body(r2) body1 = getattr(r1, "body", None) body2 = getattr(r2, "body", None) if body1 == body2: @@ -586,6 +593,7 @@ def _before_record_request(request): headers = getattr(request, "headers", None) if headers is None: return request + _materialize_iterable_body(request) if not any(_iter_header_values(headers, KEY_FINGERPRINT_HEADER)): fingerprint = _compute_key_fingerprint(request) try: @@ -597,6 +605,55 @@ def _before_record_request(request): return request +def _materialize_iterable_body(request) -> None: + """Coalesce an iterable / generator request body into ``bytes`` in-place. + + httpx's async transport hands vcrpy a ``request.body`` that is a + ``list_iterator`` (or generator) over the multipart chunks rather + than a contiguous ``bytes`` object. Two consequences fall out: + + 1. ``_safe_body_matcher`` compares the two iterator objects with + ``==``, which is identity comparison for arbitrary iterators - + so two semantically identical bodies never match and + ``record_mode="new_episodes"`` appends a fresh episode every + run until the cassette hits ``MAX_EPISODES_PER_CASSETTE`` and + the persister refuses to save. + 2. ``_normalize_multipart_boundary`` falls through its + ``else: return`` branch because the body is not bytes/str, so + the random multipart boundary in the body is never rewritten. + + Materializing the iterator once - and writing the result back to + ``request.body`` so downstream uses see bytes - fixes both bugs. + The vcrpy ``Request`` is a wrapper that vcrpy uses for matching + and recording; the underlying httpx transport sends its own + request body separately, so replacing the iterator here does not + starve the live HTTP send. + """ + body = getattr(request, "body", None) + if body is None or isinstance(body, (bytes, bytearray, str)): + return + if not hasattr(body, "__iter__"): + return + try: + chunks = list(body) + except TypeError: + return + out = bytearray() + for chunk in chunks: + if isinstance(chunk, (bytes, bytearray)): + out.extend(chunk) + elif isinstance(chunk, str): + out.extend(chunk.encode("utf-8")) + else: + # Heterogeneous, non-text/binary chunk - bail rather than + # silently corrupt the body. + return + try: + request.body = bytes(out) + except (AttributeError, TypeError): + pass + + def _key_fingerprint_matcher(r1, r2) -> None: def _fp(req): for value in _iter_header_values( From 9e2e5b6bf4ce895f78af16a2ab5f2f59a3c6cedd Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 17 May 2026 07:29:36 +0000 Subject: [PATCH 05/17] fix(tests): handle bytes_iterator + never leave an exhausted body Follow-up to 8e08272b. The previous attempt at coalescing iterable request bodies bailed out (``return`` without writing ``request.body``) whenever it could not classify the chunk type. That was the wrong failure mode for one critical case: vcrpy sometimes presents the body as ``iter(some_bytes)``, whose Python type is ``bytes_iterator`` and which yields ``int`` byte values (0-255), not byte chunks. The old code saw an ``int`` chunk, hit the ``else: return`` branch, and left ``request.body`` pointing at the now-exhausted iterator. The post-fix diagnostic run made this loud: [vcr-safe-body-matcher] request body mismatch body[a]: type='bytes_iterator' length=unknown sha256=N/A body[b]: type='bytes_iterator' length=unknown sha256=N/A Every async image-edit test then ballooned from entries=2 to entries=10 in that single CI run -- the exhausted iterator meant the live multipart upload went out as an empty body, OpenAI returned 400, the SDK + flaky retries fired, each retry got a fresh iterator that my hook exhausted again, and ``new_episodes`` recorded each failed attempt as a new cassette episode. This patch: * Recognizes ``bytes_iterator`` (chunks are ``int``) and reconstructs the buffer via ``bytes(chunks)``. * Keeps the existing ``list_iterator``-over-bytes-chunks handling via ``b"".join(...)``. * **Always writes a bytes value back to ``request.body`` after consuming the iterator.** If the chunk shape is unrecognized, ``request.body`` is set to ``b""`` rather than left as an exhausted iterator. That is wrong in the sense of "we lost the body" but right in the sense of "the failure mode is now visible (live API call sends empty body and fails fast) instead of invisible (corrupt cassette grows silently)". Combined with the matcher diagnostic, any future regression in this code path will surface in the CI log immediately. Local verification covers ``bytes_iterator``, ``list_iterator`` over bytes chunks, generator over bytes chunks, empty iterator, already-bytes (idempotent), identical-content iterator equality in the matcher (now matches), and differing-content iterator inequality (still raises). --- tests/_vcr_conftest_common.py | 46 ++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index 23b9564032e1..84fd921d76b6 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -638,18 +638,42 @@ def _materialize_iterable_body(request) -> None: chunks = list(body) except TypeError: return - out = bytearray() - for chunk in chunks: - if isinstance(chunk, (bytes, bytearray)): - out.extend(chunk) - elif isinstance(chunk, str): - out.extend(chunk.encode("utf-8")) - else: - # Heterogeneous, non-text/binary chunk - bail rather than - # silently corrupt the body. - return + + # IMPORTANT: ``list(body)`` has already exhausted the original + # iterator. From this point we MUST write something bytes-shaped + # back to ``request.body`` -- bailing out and leaving the body as + # an exhausted iterator makes the next access (cassette + # serialization, retry replay, or the actual httpx send) see an + # empty stream. In a previous attempt at this fix the bail path + # was taken for ``bytes_iterator`` bodies (chunks were ints) and + # the live send ended up with an empty multipart upload, which + # the SDK retried until the cassette ballooned to ~10 episodes + # per test. Fall through to ``out = b""`` rather than ``return`` + # so an unrecognized chunk shape still leaves a stable body. + out = b"" + if chunks: + first = chunks[0] + if isinstance(first, int): + # ``iter(b"...")`` yields integer byte values (its type + # name is ``bytes_iterator``). ``bytes(list_of_ints)`` is + # the inverse and reconstructs the original buffer. + try: + out = bytes(chunks) + except (TypeError, ValueError): + out = b"" + elif isinstance(first, (bytes, bytearray)): + try: + out = b"".join(c if isinstance(c, bytes) else bytes(c) for c in chunks) + except (TypeError, ValueError): + out = b"" + elif isinstance(first, str): + try: + out = "".join(chunks).encode("utf-8") + except (TypeError, ValueError): + out = b"" + try: - request.body = bytes(out) + request.body = out except (AttributeError, TypeError): pass From 1c51ad13352c6cc1e345c5576ba93fdca933c344 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 17 May 2026 07:44:31 +0000 Subject: [PATCH 06/17] fix(tests): clear vcrpy's sticky _was_iter flag so materialized bodies stay bytes Actual root cause of the async image-edit cassette leak. The previous diagnostic run produced this dead giveaway: [vcr-episode-body-hash] ... episode[0]: body type='bytes_iterator' is not bytes/bytearray/str -- cannot hash [vcr-safe-body-matcher] request body mismatch body[a]: type='bytes_iterator' length=unknown sha256=N/A body[b]: type='bytes_iterator' length=unknown sha256=N/A Both sides of the matcher were ``bytes_iterator`` **after** the materializer had supposedly converted them to bytes. That made no sense until I read vcrpy's ``Request`` class. vcrpy's ``Request`` keeps two private flags that are set in ``__init__`` from the original body's type and **never cleared by the setter**: def __init__(self, method, uri, body, headers): self._was_file = hasattr(body, "read") self._was_iter = _is_nonsequence_iterator(body) ... @property def body(self): if self._was_file: return BytesIO(self._body) if self._was_iter: return iter(self._body) return self._body @body.setter def body(self, value): if isinstance(value, str): value = value.encode("utf-8") self._body = value # <-- does NOT touch _was_iter / _was_file So when httpx's async transport hands vcrpy an iterator body, ``_was_iter`` becomes ``True`` and stays there forever. Even after ``_materialize_iterable_body`` writes plain bytes via ``request.body = out``, the next read of ``.body`` re-wraps the stored bytes in ``iter()`` -- producing a fresh ``bytes_iterator`` that compares unequal to any other ``bytes_iterator`` via object identity. The matcher missed every time, the cassette grew by one episode per run, and the persister saw the same iterator type when trying to hash the body for the diagnostic log. Fix: after writing the materialized bytes, also force ``_was_iter`` and ``_was_file`` to ``False``. vcrpy exposes no public API for this, so we touch the private flags directly -- acknowledged as a pragmatic test-only hack with a clear unit boundary (the only call site is ``_materialize_iterable_body``). Local repro reproduces the exact production setup: ``Request('POST', url, iter(b'multipart-content'), {})`` on two sides, runs the matcher, asserts HIT. Verified the matcher hits on identical content and still raises on differing content. Should be the last fix needed. Existing cassettes that contain oddly-shaped bodies (lists of int chunks, etc. from the previous ``_was_iter=True`` save path) still match because the materializer canonicalises both sides to bytes before comparison -- no fourth re-flush required. --- tests/_vcr_conftest_common.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index 84fd921d76b6..78e81a0414eb 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -677,6 +677,27 @@ def _materialize_iterable_body(request) -> None: except (AttributeError, TypeError): pass + # vcrpy's ``Request`` keeps two internal flags - ``_was_iter`` and + # ``_was_file`` - that are set in ``__init__`` based on the type + # of the original body and never cleared by the setter. Their job + # is to make the ``body`` *getter* re-wrap the stored value in + # ``iter()`` or ``BytesIO()`` on every access, so callers that + # expect a stream still get one even after the body has been + # consumed once. The side effect is that even after we write + # plain ``bytes`` back via ``request.body = out``, the next + # access still returns ``iter(self._body)`` - which gives every + # matcher comparison a fresh ``bytes_iterator`` and makes + # ``body_a == body_b`` an object-identity check that can never + # succeed. Touching the private flags is the only escape hatch; + # vcrpy exposes no public API for resetting them. After this + # point the body really is ``bytes`` from the getter's + # perspective. + for attr in ("_was_iter", "_was_file"): + try: + setattr(request, attr, False) + except (AttributeError, TypeError): + pass + def _key_fingerprint_matcher(r1, r2) -> None: def _fp(req): From 927c5548f9928d7f6e4633b0bc8049a40e2c46ef Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 17 May 2026 07:48:25 +0000 Subject: [PATCH 07/17] revert(tests): drop the temp per-episode body-hash diagnostic Removed now that 1c51ad13 has confirmed the root cause (vcrpy's sticky ``_was_iter`` flag making the body getter re-wrap stored bytes in ``iter()`` on every access). The hash dump did its job -- the post-1c51ad13 image_gen_testing run shows all five async image-edit tests as ``[VCR HIT]`` with stable entry counts and zero billing errors -- and is too noisy to keep on by default (over 100 lines per session at steady state). Kept permanently: * ``_safe_body_matcher`` mismatch diagnostic in ``_vcr_conftest_common.py``. Only fires on a body mismatch, which is signal worth surfacing whenever it happens. * ``_normalize_multipart_boundary`` "skipped" log line. Same rationale -- only fires when the body shape is something the normalizer cannot rewrite in place. * The ``test-results/vcr-diagnostics/.log`` per-PID file plumbing (``vcr_diag_write_line`` / ``emit_vcr_diagnostic_log``). Useful for any future diagnostic that needs to bypass xdist stdout/stderr capture; cheap to keep. --- tests/_vcr_redis_persister.py | 44 ----------------------------------- 1 file changed, 44 deletions(-) diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index 1c60e126e3f2..7fdb7267a382 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -168,7 +168,6 @@ def save_cassette(cassette_path, cassette_dict, serializer): key = redis_key_for(cassette_path) passed = _passed_by_cassette_key.pop(key, True) episode_count = len(cassette_dict.get("requests", []) or []) - _maybe_log_episode_body_hashes(key, cassette_dict) if episode_count > MAX_EPISODES_PER_CASSETTE: _log.warning( "VCR redis save refused for %s; cassette has %d episodes " @@ -211,49 +210,6 @@ def save_cassette(cassette_path, cassette_dict, serializer): return _RedisPersister -# TEMP DIAGNOSTIC -- intended to be reverted once the async image-edit -# cassette variance is root-caused. Logs a per-episode body SHA-256 -# at save time so two consecutive CI runs can be diffed: if the same -# test records ``sha=abc`` on run 1 and ``sha=def`` on run 2, the live -# request body genuinely varies; if both runs record the same hash -# but the matcher still misses, the bug is in the matcher (e.g. it is -# comparing a bytes object to a stream object). Always-on for any -# session that loads this persister -- ungated because we are -# specifically trying to capture data from CI right now. -def _maybe_log_episode_body_hashes(key: str, cassette_dict) -> None: - import hashlib - - # Imported lazily to avoid a circular import at module load. - from tests._vcr_conftest_common import vcr_diag_write_line - - requests = cassette_dict.get("requests", []) or [] - if not requests: - return - for i, req in enumerate(requests): - body = getattr(req, "body", None) - if body is None: - body_bytes = b"" - elif isinstance(body, (bytes, bytearray)): - body_bytes = bytes(body) - elif isinstance(body, str): - body_bytes = body.encode("utf-8") - else: - vcr_diag_write_line( - f"[vcr-episode-body-hash] {key} episode[{i}]: body type=" - f"{type(body).__name__!r} is not bytes/bytearray/str -- " - "cannot hash. This is the smoking gun for matcher-side " - "bugs on async multipart." - ) - continue - method = getattr(req, "method", "?") - uri = getattr(req, "uri", getattr(req, "url", "?")) - vcr_diag_write_line( - f"[vcr-episode-body-hash] {key} episode[{i}] {method} {uri} " - f"body sha256={hashlib.sha256(body_bytes).hexdigest()} " - f"len={len(body_bytes)} preview={body_bytes[:120]!r}" - ) - - def filter_non_2xx_response(response): if not isinstance(response, dict): return response From c7d6d4da8bfd6cbac17bcb84d1ae6b03ca8d09d8 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 17 May 2026 08:00:47 +0000 Subject: [PATCH 08/17] chore(tests): delete unused flush script + wire VCR diagnostic dump everywhere * Remove ``scripts/flush_image_edit_vcr_cassettes.py``. It was a one-shot helper for the initial cassette flush; the iterator and ``_was_iter`` fixes mean no future flush should be required, and the script was never run anywhere (the actual flushes happened inside the CI conftest via the temp hacks that have since been reverted). * The matcher mismatch + normalizer skip diagnostics already write per-PID files for every suite that imports the shared VCR plumbing, but ``emit_vcr_diagnostic_log`` -- the controller-side dump that surfaces those files into the CI log at session end -- was only wired into ``image_gen_tests``. Add the one-line call to the 12 sibling conftests that already use VCR so the diagnostics surface in any suite's terminal output if a body matcher ever misses. No new output in steady state -- the dump is a no-op when no diagnostics were recorded that session. --- scripts/flush_image_edit_vcr_cassettes.py | 131 -------------------- tests/audio_tests/conftest.py | 2 + tests/guardrails_tests/conftest.py | 2 + tests/litellm_utils_tests/conftest.py | 2 + tests/llm_responses_api_testing/conftest.py | 2 + tests/llm_translation/conftest.py | 2 + tests/local_testing/conftest.py | 2 + tests/logging_callback_tests/conftest.py | 2 + tests/ocr_tests/conftest.py | 2 + tests/pass_through_unit_tests/conftest.py | 2 + tests/router_unit_tests/conftest.py | 2 + tests/search_tests/conftest.py | 2 + tests/unified_google_tests/conftest.py | 2 + 13 files changed, 24 insertions(+), 131 deletions(-) delete mode 100755 scripts/flush_image_edit_vcr_cassettes.py diff --git a/scripts/flush_image_edit_vcr_cassettes.py b/scripts/flush_image_edit_vcr_cassettes.py deleted file mode 100755 index 7b87f36b2841..000000000000 --- a/scripts/flush_image_edit_vcr_cassettes.py +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env python3 -"""Flush the bloated image-edit VCR cassettes from the cassette Redis. - -Run this **once** after merging the multipart-boundary stabilization -PR. The pre-fix cassettes for the async image-edit tests have -accumulated >50 episodes (random multipart boundary on every run + -``record_mode="new_episodes"`` = monotonic growth), so the persister -refuses to save updates -- meaning every CI run after the fix would -still try to re-record against the stale 51-entry cassette, hit -``MAX_EPISODES_PER_CASSETTE`` again, get refused, and re-bill the live -provider. - -Deleting these keys forces the next CI run to record a clean cassette -under the new fixed-boundary + raw-bytes fixtures (1 episode per -unique call), after which the 24-hour TTL replay loop kicks in -normally. - -Scope is intentionally narrow: - * Only ``tests/image_gen_tests/test_image_edits/*`` cassette keys - are touched. Image-*generation* cassettes (TestOpenAIGPTImage1 - etc.) are unaffected -- they were already in the VCR HIT state. - * Lists every match in dry-run mode before deleting anything so the - operator can confirm the impact. - -Usage: - CASSETTE_REDIS_URL=redis://... \ - uv run python scripts/flush_image_edit_vcr_cassettes.py --dry-run - - CASSETTE_REDIS_URL=redis://... \ - uv run python scripts/flush_image_edit_vcr_cassettes.py --yes - -``CASSETTE_REDIS_URL`` is the same env var the persister reads at CI -start (see ``tests/_vcr_redis_persister.py``). -""" - -from __future__ import annotations - -import argparse -import os -import sys - -import redis - - -CASSETTE_REDIS_URL_ENV = "CASSETTE_REDIS_URL" -REDIS_KEY_PREFIX = "litellm:vcr:cassette:" -TARGET_KEY_PATTERN = f"{REDIS_KEY_PREFIX}tests/image_gen_tests/test_image_edits/*" - - -def _build_client(url: str) -> redis.Redis: - return redis.Redis.from_url( - url, - socket_timeout=10, - socket_connect_timeout=10, - decode_responses=False, - ) - - -def _scan_matching_keys(client: redis.Redis, pattern: str) -> list[bytes]: - return sorted(client.scan_iter(match=pattern, count=500)) - - -def _delete_keys(client: redis.Redis, keys: list[bytes]) -> int: - if not keys: - return 0 - # Batch into chunks so a single DEL call does not exceed the - # server's argument-count or buffer limits on large key sets. - deleted = 0 - chunk_size = 200 - for start in range(0, len(keys), chunk_size): - batch = keys[start : start + chunk_size] - deleted += int(client.delete(*batch)) - return deleted - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--yes", - action="store_true", - help="Actually delete the matched keys. Without this flag the script " - "runs in dry-run mode and only lists what would be deleted.", - ) - parser.add_argument( - "--dry-run", - action="store_true", - help="List matched keys without deleting (default behaviour when " - "--yes is omitted; kept as an explicit flag for clarity).", - ) - parser.add_argument( - "--pattern", - default=TARGET_KEY_PATTERN, - help=f"Override the SCAN match pattern. Default: {TARGET_KEY_PATTERN}", - ) - args = parser.parse_args(argv) - - url = os.environ.get(CASSETTE_REDIS_URL_ENV) - if not url: - print( - f"error: {CASSETTE_REDIS_URL_ENV} is not set. Set it to the " - "cassette Redis URL (same URL the persister reads in CI).", - file=sys.stderr, - ) - return 2 - - client = _build_client(url) - try: - client.ping() - except redis.RedisError as exc: - print(f"error: cannot reach cassette Redis: {exc}", file=sys.stderr) - return 2 - - matches = _scan_matching_keys(client, args.pattern) - print(f"matched {len(matches)} key(s) under pattern: {args.pattern}") - for key in matches: - print(f" {key.decode('utf-8', errors='replace')}") - - if not matches: - return 0 - - if not args.yes: - print("\ndry run -- pass --yes to actually delete these keys.") - return 0 - - deleted = _delete_keys(client, matches) - print(f"\ndeleted {deleted} key(s).") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/audio_tests/conftest.py b/tests/audio_tests/conftest.py index ff47853d4949..af142c15179a 100644 --- a/tests/audio_tests/conftest.py +++ b/tests/audio_tests/conftest.py @@ -10,6 +10,7 @@ apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, + emit_vcr_diagnostic_log, install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, @@ -57,3 +58,4 @@ def pytest_collection_modifyitems(config, items): def pytest_terminal_summary(terminalreporter, exitstatus, config): emit_cassette_cache_session_banner(terminalreporter) emit_vcr_classification_summary(terminalreporter) + emit_vcr_diagnostic_log(terminalreporter) diff --git a/tests/guardrails_tests/conftest.py b/tests/guardrails_tests/conftest.py index eb563699b2bc..8b88a12c5045 100644 --- a/tests/guardrails_tests/conftest.py +++ b/tests/guardrails_tests/conftest.py @@ -21,6 +21,7 @@ apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, + emit_vcr_diagnostic_log, install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, @@ -160,3 +161,4 @@ def pytest_collection_modifyitems(config, items): def pytest_terminal_summary(terminalreporter, exitstatus, config): emit_cassette_cache_session_banner(terminalreporter) emit_vcr_classification_summary(terminalreporter) + emit_vcr_diagnostic_log(terminalreporter) diff --git a/tests/litellm_utils_tests/conftest.py b/tests/litellm_utils_tests/conftest.py index 08745c99c07c..efeb15cfac9b 100644 --- a/tests/litellm_utils_tests/conftest.py +++ b/tests/litellm_utils_tests/conftest.py @@ -17,6 +17,7 @@ apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, + emit_vcr_diagnostic_log, install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, @@ -116,3 +117,4 @@ def pytest_collection_modifyitems(config, items): def pytest_terminal_summary(terminalreporter, exitstatus, config): emit_cassette_cache_session_banner(terminalreporter) emit_vcr_classification_summary(terminalreporter) + emit_vcr_diagnostic_log(terminalreporter) diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py index 2a08db571494..f15308594b5f 100644 --- a/tests/llm_responses_api_testing/conftest.py +++ b/tests/llm_responses_api_testing/conftest.py @@ -18,6 +18,7 @@ apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, + emit_vcr_diagnostic_log, install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, @@ -116,3 +117,4 @@ def pytest_collection_modifyitems(config, items): def pytest_terminal_summary(terminalreporter, exitstatus, config): emit_cassette_cache_session_banner(terminalreporter) emit_vcr_classification_summary(terminalreporter) + emit_vcr_diagnostic_log(terminalreporter) diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index 5fcd31aa32d1..570068877916 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -23,6 +23,7 @@ apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, + emit_vcr_diagnostic_log, install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, @@ -82,6 +83,7 @@ def pytest_runtest_logreport(report): def pytest_terminal_summary(terminalreporter, exitstatus, config): emit_cassette_cache_session_banner(terminalreporter) emit_vcr_classification_summary(terminalreporter) + emit_vcr_diagnostic_log(terminalreporter) # --------------------------------------------------------------------------- diff --git a/tests/local_testing/conftest.py b/tests/local_testing/conftest.py index 0ff7dff668af..ffaf7ac52654 100644 --- a/tests/local_testing/conftest.py +++ b/tests/local_testing/conftest.py @@ -27,6 +27,7 @@ apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, + emit_vcr_diagnostic_log, install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, @@ -93,6 +94,7 @@ def pytest_runtest_logreport(report): def pytest_terminal_summary(terminalreporter, exitstatus, config): emit_cassette_cache_session_banner(terminalreporter) emit_vcr_classification_summary(terminalreporter) + emit_vcr_diagnostic_log(terminalreporter) # --------------------------------------------------------------------------- diff --git a/tests/logging_callback_tests/conftest.py b/tests/logging_callback_tests/conftest.py index cdb9200bc832..b16df83f2f04 100644 --- a/tests/logging_callback_tests/conftest.py +++ b/tests/logging_callback_tests/conftest.py @@ -24,6 +24,7 @@ apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, + emit_vcr_diagnostic_log, install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, @@ -229,3 +230,4 @@ def pytest_collection_modifyitems(config, items): def pytest_terminal_summary(terminalreporter, exitstatus, config): emit_cassette_cache_session_banner(terminalreporter) emit_vcr_classification_summary(terminalreporter) + emit_vcr_diagnostic_log(terminalreporter) diff --git a/tests/ocr_tests/conftest.py b/tests/ocr_tests/conftest.py index 66970b8579f4..781882c1eb58 100644 --- a/tests/ocr_tests/conftest.py +++ b/tests/ocr_tests/conftest.py @@ -17,6 +17,7 @@ apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, + emit_vcr_diagnostic_log, install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, @@ -64,3 +65,4 @@ def pytest_collection_modifyitems(config, items): def pytest_terminal_summary(terminalreporter, exitstatus, config): emit_cassette_cache_session_banner(terminalreporter) emit_vcr_classification_summary(terminalreporter) + emit_vcr_diagnostic_log(terminalreporter) diff --git a/tests/pass_through_unit_tests/conftest.py b/tests/pass_through_unit_tests/conftest.py index 42a95343eb76..f158b44e1efa 100644 --- a/tests/pass_through_unit_tests/conftest.py +++ b/tests/pass_through_unit_tests/conftest.py @@ -10,6 +10,7 @@ apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, + emit_vcr_diagnostic_log, install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, @@ -71,3 +72,4 @@ def pytest_collection_modifyitems(config, items): def pytest_terminal_summary(terminalreporter, exitstatus, config): emit_cassette_cache_session_banner(terminalreporter) emit_vcr_classification_summary(terminalreporter) + emit_vcr_diagnostic_log(terminalreporter) diff --git a/tests/router_unit_tests/conftest.py b/tests/router_unit_tests/conftest.py index fe976515c920..46101d7d906a 100644 --- a/tests/router_unit_tests/conftest.py +++ b/tests/router_unit_tests/conftest.py @@ -17,6 +17,7 @@ apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, + emit_vcr_diagnostic_log, install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, @@ -123,3 +124,4 @@ def pytest_collection_modifyitems(config, items): def pytest_terminal_summary(terminalreporter, exitstatus, config): emit_cassette_cache_session_banner(terminalreporter) emit_vcr_classification_summary(terminalreporter) + emit_vcr_diagnostic_log(terminalreporter) diff --git a/tests/search_tests/conftest.py b/tests/search_tests/conftest.py index e06d3e95eee9..5159caf081de 100644 --- a/tests/search_tests/conftest.py +++ b/tests/search_tests/conftest.py @@ -18,6 +18,7 @@ apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, + emit_vcr_diagnostic_log, install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, @@ -65,3 +66,4 @@ def pytest_collection_modifyitems(config, items): def pytest_terminal_summary(terminalreporter, exitstatus, config): emit_cassette_cache_session_banner(terminalreporter) emit_vcr_classification_summary(terminalreporter) + emit_vcr_diagnostic_log(terminalreporter) diff --git a/tests/unified_google_tests/conftest.py b/tests/unified_google_tests/conftest.py index d28f89a77b0a..a4c2c0c57520 100644 --- a/tests/unified_google_tests/conftest.py +++ b/tests/unified_google_tests/conftest.py @@ -17,6 +17,7 @@ apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, + emit_vcr_diagnostic_log, install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, @@ -110,3 +111,4 @@ def pytest_collection_modifyitems(config, items): def pytest_terminal_summary(terminalreporter, exitstatus, config): emit_cassette_cache_session_banner(terminalreporter) emit_vcr_classification_summary(terminalreporter) + emit_vcr_diagnostic_log(terminalreporter) From 78ada20e209ca550428159d5c3db49df1b0a3609 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 17 May 2026 08:01:11 +0000 Subject: [PATCH 09/17] chore(tests): trim non-essential comments per project comment policy Strips docstrings, inline comments, and block comments that this PR introduced where the code itself was already self-evident. Keeps the few lines that document non-obvious behaviour (raw-bytes-not-BytesIO rationale on the image fixtures, the per-PID-files-bypass-xdist note on the diagnostic directory). Touches only comments this PR added -- no pre-existing comment is removed. Net: -161 lines of comment/docstring across 3 files, no code behaviour change. --- tests/_vcr_conftest_common.py | 157 +++------------------- tests/image_gen_tests/conftest.py | 14 -- tests/image_gen_tests/test_image_edits.py | 34 +---- 3 files changed, 22 insertions(+), 183 deletions(-) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index 78e81a0414eb..747c43b09077 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -36,16 +36,8 @@ KEY_FINGERPRINT_MATCHER_NAME = "key_fingerprint" KEY_FINGERPRINT_HEADER = "x-litellm-key-fp" -# Directory for per-process VCR diagnostic logs that bypass pytest's -# stdout/stderr capture. ``sys.stderr.write`` from inside vcrpy -# machinery is swallowed by xdist for any test that ultimately passes, -# so a diagnostic that fires on a body-matcher miss but the test still -# records and passes will never reach the CI log. Each xdist worker -# (or the main process) writes line-buffered to a per-PID file under -# this directory, and the controller's ``pytest_terminal_summary`` -# concatenates them into the terminal at session end. ``test-results/`` -# is already collected by ``store_test_results`` in CI, so the raw -# files survive as build artifacts too. +# Per-PID files bypass pytest/xdist stdout capture, which swallows +# stderr from passing tests. VCR_DIAG_DIR_ENV = "LITELLM_VCR_DIAG_DIR" VCR_DIAG_DIR_DEFAULT = "test-results/vcr-diagnostics" @@ -55,12 +47,6 @@ def _vcr_diag_dir() -> str: def vcr_diag_write_line(msg: str) -> None: - """Append a single diagnostic line to the current process's - per-PID file. Atomic against other workers in the same xdist - session because each PID owns its own file. - - Errors are swallowed -- diagnostic logging must never fail a test. - """ try: directory = _vcr_diag_dir() os.makedirs(directory, exist_ok=True) @@ -72,11 +58,6 @@ def vcr_diag_write_line(msg: str) -> None: def emit_vcr_diagnostic_log(terminalreporter) -> None: - """Concatenate every per-PID diagnostic file into the controller's - terminal at session end. Each file is dumped under a header that - names the originating worker PID so cross-process events can still - be ordered if needed. - """ directory = _vcr_diag_dir() if not os.path.isdir(directory): return @@ -164,37 +145,8 @@ def emit_vcr_diagnostic_log(terminalreporter) -> None: def pin_httpx_multipart_boundary(monkeypatch) -> None: - """Force every httpx multipart request to use a constant boundary. - - httpx's ``MultipartStream`` generates a fresh ``boundary=`` - via ``os.urandom(16)`` whenever the caller does not supply one - (see ``httpx._multipart.MultipartStream.__init__``). That random - boundary appears both in the ``Content-Type`` header and verbatim in - the request body between each part. - - ``_normalize_multipart_boundary`` rewrites the header reliably, but - on the async transport path the body is not always handed to - ``before_record_request`` as a contiguous ``bytes`` object — so the - body replacement silently no-ops and the recorded cassette retains - the random boundary string. Subsequent runs generate a *different* - random boundary, the ``safe_body`` matcher misses, and - ``record_mode="new_episodes"`` appends a fresh episode until the - cassette crosses ``MAX_EPISODES_PER_CASSETTE`` and the persister - refuses to save — re-billing live providers on every CI run. - - Pinning the boundary at the source removes the variance entirely: - every run emits byte-identical multipart bodies, the existing - ``safe_body`` matcher succeeds on the first request, and one - recorded episode per unique call satisfies replays for the cassette - TTL. - - This wraps ``MultipartStream.__init__`` instead of patching the - boundary-generation helper directly because httpx inlines - ``os.urandom(16).hex().encode("ascii")`` in the constructor body - rather than calling a named function. We preserve the caller's - boundary when one is explicitly supplied so production-style code - that pins its own boundary keeps working. - """ + """Force every httpx multipart request to use a constant boundary so + request bodies are byte-stable across runs (vcrpy match-on-body).""" try: import httpx._multipart as _httpx_multipart except ImportError: # pragma: no cover - httpx is a hard test dep @@ -315,19 +267,7 @@ def _safe_body_matcher(r1, r2) -> None: (e.g. the Bedrock batch S3 PUT) before it can return "no match". This matcher is strictly more conservative — the only equivalence it gives up vs. the default is "JSON key order doesn't matter". - - On mismatch, emits a structured diagnostic to stderr (type of each - body, length, SHA-256, first divergent offset, ±100-byte window). - Without this, vcrpy returns "request bodies differ" with zero - context, and bugs where the live request body is an unbytes-like - object (e.g. an httpx ``MultipartStream`` for async requests) look - indistinguishable from genuine content drift. """ - # Defense in depth: ``_before_record_request`` already coalesces - # iterable bodies, but if vcrpy invokes the matcher on a request - # that did not flow through that hook (or if a future code path - # bypasses it), do it again here so iterator==iterator never - # silently fails. _materialize_iterable_body(r1) _materialize_iterable_body(r2) body1 = getattr(r1, "body", None) @@ -353,15 +293,6 @@ def _to_bytes(b): def _emit_body_mismatch_diagnostic(r1, r2, body1, body2, n1, n2) -> None: - """Dump enough info to a single stderr block to root-cause why two - requests that look semantically identical failed the body matcher. - - Always-on (matcher mismatches are signal, not noise): the volume - is bounded by the number of stored episodes a live request is - compared against, and we already log a per-test verdict line for - every test. - """ - def _describe(label, raw, asbytes): t = type(raw).__name__ if asbytes is None: @@ -369,12 +300,10 @@ def _describe(label, raw, asbytes): f" {label}: type={t!r} length=unknown sha256=N/A " f"(body could not be coerced to bytes)" ) - length = len(asbytes) - digest = hashlib.sha256(asbytes).hexdigest() - preview = asbytes[:120] return ( - f" {label}: type={t!r} length={length} sha256={digest} " - f"preview={preview!r}" + f" {label}: type={t!r} length={len(asbytes)} " + f"sha256={hashlib.sha256(asbytes).hexdigest()} " + f"preview={asbytes[:120]!r}" ) method_a = getattr(r1, "method", "?") @@ -389,10 +318,6 @@ def _describe(label, raw, asbytes): _describe("body[b]", body2, n2), ] if n1 is not None and n2 is not None and n1 != n2: - # Find the first divergent byte offset and dump a ±100 window - # around it so the human reading the CI log can see at a glance - # whether the variance is a UUID, a timestamp, a random multipart - # boundary, or something else. offset = next( (i for i in range(min(len(n1), len(n2))) if n1[i] != n2[i]), min(len(n1), len(n2)), @@ -550,13 +475,6 @@ def _normalize_multipart_boundary(request) -> None: elif isinstance(body, str): new_body = body.replace(current_boundary, VCR_FIXED_MULTIPART_BOUNDARY) else: - # The body is something other than bytes/bytearray/str -- most - # likely an httpx ``MultipartStream`` or an aiter chunked stream - # we cannot rewrite in place. Log it so a body-matcher miss on a - # multipart request can be correlated with "normalizer skipped - # because body type was X". The header was still rewritten - # above, so the recorded Content-Type stays stable; only the - # body bytes carry the random boundary verbatim. vcr_diag_write_line( f"[vcr-multipart-normalize] body normalization SKIPPED: " f"body type {type(body).__name__!r} is not bytes/bytearray/str. " @@ -606,28 +524,16 @@ def _before_record_request(request): def _materialize_iterable_body(request) -> None: - """Coalesce an iterable / generator request body into ``bytes`` in-place. - - httpx's async transport hands vcrpy a ``request.body`` that is a - ``list_iterator`` (or generator) over the multipart chunks rather - than a contiguous ``bytes`` object. Two consequences fall out: - - 1. ``_safe_body_matcher`` compares the two iterator objects with - ``==``, which is identity comparison for arbitrary iterators - - so two semantically identical bodies never match and - ``record_mode="new_episodes"`` appends a fresh episode every - run until the cassette hits ``MAX_EPISODES_PER_CASSETTE`` and - the persister refuses to save. - 2. ``_normalize_multipart_boundary`` falls through its - ``else: return`` branch because the body is not bytes/str, so - the random multipart boundary in the body is never rewritten. - - Materializing the iterator once - and writing the result back to - ``request.body`` so downstream uses see bytes - fixes both bugs. - The vcrpy ``Request`` is a wrapper that vcrpy uses for matching - and recording; the underlying httpx transport sends its own - request body separately, so replacing the iterator here does not - starve the live HTTP send. + """Coalesce an iterable / generator request body to ``bytes`` in-place + so the body matcher and boundary normalizer see a contiguous buffer. + + Once ``list(body)`` runs the original iterator is exhausted, so this + function must always write some bytes value back -- leaving the body + as a dead iterator silently makes the next HTTP send transmit an + empty payload. Also clears vcrpy's sticky ``_was_iter`` / ``_was_file`` + flags, which otherwise make the ``body`` getter re-wrap the stored + bytes in ``iter()`` on every access (so a freshly-materialized body + would look like ``bytes_iterator`` to the next reader). """ body = getattr(request, "body", None) if body is None or isinstance(body, (bytes, bytearray, str)): @@ -639,24 +545,10 @@ def _materialize_iterable_body(request) -> None: except TypeError: return - # IMPORTANT: ``list(body)`` has already exhausted the original - # iterator. From this point we MUST write something bytes-shaped - # back to ``request.body`` -- bailing out and leaving the body as - # an exhausted iterator makes the next access (cassette - # serialization, retry replay, or the actual httpx send) see an - # empty stream. In a previous attempt at this fix the bail path - # was taken for ``bytes_iterator`` bodies (chunks were ints) and - # the live send ended up with an empty multipart upload, which - # the SDK retried until the cassette ballooned to ~10 episodes - # per test. Fall through to ``out = b""`` rather than ``return`` - # so an unrecognized chunk shape still leaves a stable body. out = b"" if chunks: first = chunks[0] if isinstance(first, int): - # ``iter(b"...")`` yields integer byte values (its type - # name is ``bytes_iterator``). ``bytes(list_of_ints)`` is - # the inverse and reconstructs the original buffer. try: out = bytes(chunks) except (TypeError, ValueError): @@ -677,21 +569,6 @@ def _materialize_iterable_body(request) -> None: except (AttributeError, TypeError): pass - # vcrpy's ``Request`` keeps two internal flags - ``_was_iter`` and - # ``_was_file`` - that are set in ``__init__`` based on the type - # of the original body and never cleared by the setter. Their job - # is to make the ``body`` *getter* re-wrap the stored value in - # ``iter()`` or ``BytesIO()`` on every access, so callers that - # expect a stream still get one even after the body has been - # consumed once. The side effect is that even after we write - # plain ``bytes`` back via ``request.body = out``, the next - # access still returns ``iter(self._body)`` - which gives every - # matcher comparison a fresh ``bytes_iterator`` and makes - # ``body_a == body_b`` an object-identity check that can never - # succeed. Touching the private flags is the only escape hatch; - # vcrpy exposes no public API for resetting them. After this - # point the body really is ``bytes`` from the getter's - # perspective. for attr in ("_was_iter", "_was_file"): try: setattr(request, attr, False) diff --git a/tests/image_gen_tests/conftest.py b/tests/image_gen_tests/conftest.py index b1631fe0c599..6f00c7528b35 100644 --- a/tests/image_gen_tests/conftest.py +++ b/tests/image_gen_tests/conftest.py @@ -38,15 +38,6 @@ def event_loop(): @pytest.fixture(scope="session", autouse=True) def _pin_multipart_boundary(): - """Pin httpx's random multipart boundary to a constant for the - entire image-gen test session. Without this, async multipart bodies - contain a fresh ``boundary=`` on every run; the - ``safe_body`` matcher misses, and ``record_mode="new_episodes"`` - grows each cassette by one entry per run until it crosses the - 50-episode persister cap and stops being saved — leaving the test - to hit the real provider on every CI run. See - ``pin_httpx_multipart_boundary`` for the full rationale. - """ monkeypatch = pytest.MonkeyPatch() pin_httpx_multipart_boundary(monkeypatch) yield @@ -78,11 +69,6 @@ def _vcr_outcome_gate(request, vcr): def pytest_configure(config): _verbose_state.remember_pluginmanager(config) - # Clear any leftover per-PID diagnostic logs from a previous local - # run so the controller's terminal summary at session end only - # surfaces this session's data. Worker processes inherit the same - # directory and append by PID, so the controller doing the cleanup - # once is sufficient. if not os.environ.get("PYTEST_XDIST_WORKER"): directory = _vcr_diag_dir() if os.path.isdir(directory): diff --git a/tests/image_gen_tests/test_image_edits.py b/tests/image_gen_tests/test_image_edits.py index 7c3632b82fce..d6600ecc94f4 100644 --- a/tests/image_gen_tests/test_image_edits.py +++ b/tests/image_gen_tests/test_image_edits.py @@ -103,16 +103,9 @@ async def test_openai_image_edit_litellm_sdk(self, sync_mode): pwd = os.path.dirname(os.path.realpath(__file__)) -# Image fixtures are returned as raw ``bytes`` (not file handles or -# ``BytesIO`` streams) so that every SDK / Router retry sees the same -# payload. A ``BytesIO`` whose file pointer is left at EOF by the first -# multipart upload silently encodes an empty image on the second -# attempt, producing a different request body — VCR records that -# divergent body as a fresh episode, the cassette eventually crosses -# ``MAX_EPISODES_PER_CASSETTE`` in ``tests/_vcr_redis_persister.py``, -# the persister refuses to save, and every subsequent CI run re-bills -# the live image-edit endpoint. Raw bytes are immutable, position-less, -# and re-encoded identically on every retry attempt. +# Fixtures must be raw ``bytes``, not ``BytesIO``: an SDK retry that +# reads a BytesIO twice gets an empty second body, which records as a +# divergent VCR episode and eventually trips MAX_EPISODES_PER_CASSETTE. def _read_image_bytes(filename: str) -> bytes: with open(os.path.join(pwd, filename), "rb") as f: return f.read() @@ -123,14 +116,6 @@ def _read_image_bytes(filename: str) -> bytes: def _make_test_images() -> list: - """Return the pair of fixture images as raw ``bytes`` payloads. - - ``httpx`` accepts a ``bytes`` value anywhere a file-like upload is - expected and re-encodes it identically on every multipart attempt - — so SDK-level retries can never produce a divergent empty-body - episode (the root cause of the cassette-overflow leak that bills - ``gpt-image-1`` on every CI run). - """ return [_ISHAAN_GITHUB_BYTES, _LITELLM_SITE_BYTES] @@ -139,14 +124,6 @@ def _make_single_test_image() -> bytes: def get_test_images_as_bytesio(): - """Return the fixture images as fresh ``BytesIO`` streams. - - Kept distinct from ``_make_test_images`` so the BytesIO-specific - smoke tests (``test_openai_image_edit_with_bytesio``, - ``test_multiple_image_edit_with_different_formats``) still exercise - the file-like upload path. Each call yields brand new streams so - the file pointer always starts at 0 for that test invocation. - """ return [ BytesIO(_ISHAAN_GITHUB_BYTES), BytesIO(_LITELLM_SITE_BYTES), @@ -718,10 +695,9 @@ async def test_multiple_image_edit_with_different_formats(): try: prompt = "Create a cohesive artistic style across all images" - # Test with mixed raw-bytes and BytesIO inputs mixed_images = [ - _make_single_test_image(), # raw ``bytes`` payload - get_test_images_as_bytesio()[1], # BytesIO object + _make_single_test_image(), + get_test_images_as_bytesio()[1], ] result = await aimage_edit( From a3c36f02739e16792afa54c4266f86997ac09c2a Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 17 May 2026 08:05:44 +0000 Subject: [PATCH 10/17] chore(tests): forward **kwargs in pin_httpx_multipart_boundary wrapper Defensive against future httpx MultipartStream.__init__ adding new optional kwargs. Without the forward, the wrapper would silently drop them. No behaviour change today. --- tests/_vcr_conftest_common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index 747c43b09077..19762babf1e7 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -154,10 +154,10 @@ def pin_httpx_multipart_boundary(monkeypatch) -> None: _original_init = _httpx_multipart.MultipartStream.__init__ - def _init_with_fixed_boundary(self, data, files, boundary=None): + def _init_with_fixed_boundary(self, data, files, boundary=None, **kwargs): if boundary is None: boundary = VCR_FIXED_MULTIPART_BOUNDARY.encode("ascii") - return _original_init(self, data=data, files=files, boundary=boundary) + return _original_init(self, data=data, files=files, boundary=boundary, **kwargs) monkeypatch.setattr( _httpx_multipart.MultipartStream, "__init__", _init_with_fixed_boundary From 6e3d4b07e76430d039b62c5989ebbf54b05949ca Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 17 May 2026 08:18:01 +0000 Subject: [PATCH 11/17] chore(tests): canonicalize VCR matchers and surface shouldn't-happen branches Bundles the "follow-up cleanup PR" into this one so it does not get lost. Four small changes: 1. Introduce ``_canonical_body(req) -> (bytes, pre_type)`` and route ``_safe_body_matcher`` through it. The matcher now operates on bytes by construction; the "compare two iterator objects via ``==`` and silently get object-identity semantics" failure mode (which cost us this entire PR to diagnose) is structurally impossible to reintroduce. ``pre_type`` is the body type *before* canonicalization, surfaced by the mismatch diagnostic so a future regression involving a new body shape is still visible. 2. Add a structured diagnostic to ``_key_fingerprint_matcher``. It was previously raising a bare ``AssertionError("API key fingerprints differ")`` with zero context -- exactly the anti-pattern the body matcher had before this PR. 3. Surface "shouldn't-happen" branches via ``vcr_diag_write_line``: * ``_strip_image_b64_payloads`` -- logs when ``response``, ``response['body']``, or ``response['body']['string']`` arrives in an unexpected shape (vcrpy contract violation). * ``_compute_key_fingerprint`` -- logs the ``"no-key"`` fallback with the request method/URL so a stripped-auth-header bug is visible instead of masked. * ``_canonical_body`` -- logs its own empty-bytes fallback when a body has a shape ``_materialize_iterable_body`` did not handle. 4. Re-introduce per-episode body-hash logging in ``_RedisPersister.save_cassette`` (was reverted in 927c5548 as "noisy"). Quantified cost: ~25 KB of CI log per session at peak, ~ms-scale CPU, zero output in steady state (no save = no log). Trade-off favours keeping it: lets two consecutive CI runs be diffed by body hash, which is how we will spot the next regression in the same class. All call sites still work: local repro confirms iter==iter HIT, iter!=iter raises, plain-bytes HIT, body-hash log emits via the same per-PID file plumbing as the matcher diagnostics. --- tests/_vcr_conftest_common.py | 119 +++++++++++++++++++++++----------- tests/_vcr_redis_persister.py | 38 +++++++++++ 2 files changed, 119 insertions(+), 38 deletions(-) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index 19762babf1e7..db041ad3a618 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -212,9 +212,17 @@ def _strip_image_b64_payloads(response): preserves all those checks while shrinking cassettes by ~99%. """ if not isinstance(response, dict): + vcr_diag_write_line( + f"[vcr-strip-b64] response is {type(response).__name__!r}, not " + "dict; skipping b64 scrub" + ) return response body = response.get("body") if not isinstance(body, dict): + vcr_diag_write_line( + f"[vcr-strip-b64] response['body'] is {type(body).__name__!r}, " + "not dict; skipping b64 scrub" + ) return response raw = body.get("string") if raw is None: @@ -224,12 +232,20 @@ def _strip_image_b64_payloads(response): try: text = bytes(raw).decode("utf-8") except UnicodeDecodeError: + vcr_diag_write_line( + "[vcr-strip-b64] response body bytes are not valid UTF-8; " + "skipping b64 scrub" + ) return response was_bytes = True elif isinstance(raw, str): text = raw was_bytes = False else: + vcr_diag_write_line( + f"[vcr-strip-b64] response['body']['string'] is " + f"{type(raw).__name__!r}, not bytes/str; skipping b64 scrub" + ) return response try: @@ -259,6 +275,38 @@ def _before_record_response(response): return filter_non_2xx_response(_scrub_response(_strip_image_b64_payloads(response))) +def _canonical_body(request) -> tuple[bytes, str]: + """Return ``(body_bytes, original_type_name)`` for a vcrpy request. + + Materializes iterables / generators (httpx async wraps the body in a + ``list_iterator`` or ``bytes_iterator``), then coerces the result to + ``bytes``. Routing every matcher through this helper makes the + "compare object identity by mistake" failure mode structurally + impossible -- the comparison always operates on bytes. + + Logs a diagnostic line when a body falls into the empty-fallback + branch (unknown shape). Never raises. + """ + pre_type = type(getattr(request, "body", None)).__name__ + _materialize_iterable_body(request) + body = getattr(request, "body", None) + if body is None: + return b"", pre_type + if isinstance(body, bytes): + return body, pre_type + if isinstance(body, bytearray): + return bytes(body), pre_type + if isinstance(body, str): + return body.encode("utf-8"), pre_type + method = getattr(request, "method", "?") + uri = getattr(request, "uri", getattr(request, "url", "?")) + vcr_diag_write_line( + f"[vcr-canonical-body] FALLBACK: {method} {uri} body type " + f"{type(body).__name__!r} not coerced to bytes; comparing as b''" + ) + return b"", pre_type + + def _safe_body_matcher(r1, r2) -> None: """Compare request bodies as bytes; never invokes ``json.loads``. @@ -268,40 +316,18 @@ def _safe_body_matcher(r1, r2) -> None: This matcher is strictly more conservative — the only equivalence it gives up vs. the default is "JSON key order doesn't matter". """ - _materialize_iterable_body(r1) - _materialize_iterable_body(r2) - body1 = getattr(r1, "body", None) - body2 = getattr(r2, "body", None) + body1, pre1 = _canonical_body(r1) + body2, pre2 = _canonical_body(r2) if body1 == body2: return - - def _to_bytes(b): - if b is None: - return b"" - if isinstance(b, bytes): - return b - if isinstance(b, str): - return b.encode("utf-8") - return None - - n1 = _to_bytes(body1) - n2 = _to_bytes(body2) - if n1 is not None and n2 is not None and n1 == n2: - return - _emit_body_mismatch_diagnostic(r1, r2, body1, body2, n1, n2) + _emit_body_mismatch_diagnostic(r1, r2, body1, body2, pre1, pre2) raise AssertionError("request bodies differ") -def _emit_body_mismatch_diagnostic(r1, r2, body1, body2, n1, n2) -> None: - def _describe(label, raw, asbytes): - t = type(raw).__name__ - if asbytes is None: - return ( - f" {label}: type={t!r} length=unknown sha256=N/A " - f"(body could not be coerced to bytes)" - ) +def _emit_body_mismatch_diagnostic(r1, r2, body1, body2, pre1, pre2) -> None: + def _describe(label, asbytes, pre_type): return ( - f" {label}: type={t!r} length={len(asbytes)} " + f" {label}: pre_canonical_type={pre_type!r} length={len(asbytes)} " f"sha256={hashlib.sha256(asbytes).hexdigest()} " f"preview={asbytes[:120]!r}" ) @@ -314,20 +340,20 @@ def _describe(label, raw, asbytes): "[vcr-safe-body-matcher] request body mismatch", f" request[a]: {method_a} {url_a}", f" request[b]: {method_b} {url_b}", - _describe("body[a]", body1, n1), - _describe("body[b]", body2, n2), + _describe("body[a]", body1, pre1), + _describe("body[b]", body2, pre2), ] - if n1 is not None and n2 is not None and n1 != n2: + if body1 != body2: offset = next( - (i for i in range(min(len(n1), len(n2))) if n1[i] != n2[i]), - min(len(n1), len(n2)), + (i for i in range(min(len(body1), len(body2))) if body1[i] != body2[i]), + min(len(body1), len(body2)), ) start = max(0, offset - 100) - end_a = min(len(n1), offset + 100) - end_b = min(len(n2), offset + 100) + end_a = min(len(body1), offset + 100) + end_b = min(len(body2), offset + 100) lines.append(f" first divergent byte offset: {offset}") - lines.append(f" window[a] @ {start}..{end_a}: {n1[start:end_a]!r}") - lines.append(f" window[b] @ {start}..{end_b}: {n2[start:end_b]!r}") + lines.append(f" window[a] @ {start}..{end_a}: {body1[start:end_a]!r}") + lines.append(f" window[b] @ {start}..{end_b}: {body2[start:end_b]!r}") vcr_diag_write_line("\n".join(lines)) @@ -386,6 +412,13 @@ def _compute_key_fingerprint(request) -> str: stable = _stable_key_value(header_name, text) parts.append(f"{header_name}={stable}") if not parts: + method = getattr(request, "method", "?") + uri = getattr(request, "uri", getattr(request, "url", "?")) + vcr_diag_write_line( + f"[vcr-key-fingerprint] no API key header found on {method} " + f"{uri}; falling back to 'no-key'. If this request should have " + "carried auth, something earlier in the pipeline stripped it." + ) return "no-key" digest = hashlib.sha256("\n".join(parts).encode("utf-8")).hexdigest() return digest[:16] @@ -586,7 +619,17 @@ def _fp(req): return value if isinstance(value, str) else str(value) return "no-key" - if _fp(r1) != _fp(r2): + fp1, fp2 = _fp(r1), _fp(r2) + if fp1 != fp2: + method_a = getattr(r1, "method", "?") + method_b = getattr(r2, "method", "?") + url_a = getattr(r1, "uri", getattr(r1, "url", "?")) + url_b = getattr(r2, "uri", getattr(r2, "url", "?")) + vcr_diag_write_line( + "[vcr-key-fingerprint-matcher] API key fingerprints differ\n" + f" request[a]: {method_a} {url_a} fingerprint={fp1!r}\n" + f" request[b]: {method_b} {url_b} fingerprint={fp2!r}" + ) raise AssertionError("API key fingerprints differ") diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index 7fdb7267a382..3e543a1a6e6e 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -168,6 +168,7 @@ def save_cassette(cassette_path, cassette_dict, serializer): key = redis_key_for(cassette_path) passed = _passed_by_cassette_key.pop(key, True) episode_count = len(cassette_dict.get("requests", []) or []) + _log_episode_body_hashes(key, cassette_dict) if episode_count > MAX_EPISODES_PER_CASSETTE: _log.warning( "VCR redis save refused for %s; cassette has %d episodes " @@ -210,6 +211,43 @@ def save_cassette(cassette_path, cassette_dict, serializer): return _RedisPersister +def _log_episode_body_hashes(key: str, cassette_dict) -> None: + """Record a per-episode body SHA-256 for every cassette save. + + Lets two consecutive CI runs be diffed: if the same test records a + different hash run-to-run, the live request body varies; if both + runs record the same hash but the matcher still misses, the bug is + in the matcher itself. Negligible cost (~1ms hashing + ~200B file + write per saved episode; nothing at all when cassettes replay). + """ + import hashlib + + from tests._vcr_conftest_common import vcr_diag_write_line + + requests = cassette_dict.get("requests", []) or [] + for i, req in enumerate(requests): + body = getattr(req, "body", None) + if body is None: + body_bytes = b"" + elif isinstance(body, (bytes, bytearray)): + body_bytes = bytes(body) + elif isinstance(body, str): + body_bytes = body.encode("utf-8") + else: + vcr_diag_write_line( + f"[vcr-episode-body-hash] {key} episode[{i}]: body type=" + f"{type(body).__name__!r} not bytes/bytearray/str -- cannot hash" + ) + continue + method = getattr(req, "method", "?") + uri = getattr(req, "uri", getattr(req, "url", "?")) + vcr_diag_write_line( + f"[vcr-episode-body-hash] {key} episode[{i}] {method} {uri} " + f"body sha256={hashlib.sha256(body_bytes).hexdigest()} " + f"len={len(body_bytes)} preview={body_bytes[:120]!r}" + ) + + def filter_non_2xx_response(response): if not isinstance(response, dict): return response From 32369edb8faa367b8f0f216a4bc797ff3e06d2ec Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 17 May 2026 08:26:38 +0000 Subject: [PATCH 12/17] chore(tests): symmetrize diag-log cleanup across every VCR-using conftest ``image_gen_tests/conftest.py`` was the only suite that cleared ``test-results/vcr-diagnostics/*.log`` at session start. The other 12 VCR-using conftests inherited any stale per-PID logs from a previous local run and would dump them in the terminal summary -- harmless in CI (fresh container) but confusing locally when running multiple suites in sequence. Extracts the cleanup into a ``reset_vcr_diag_dir`` helper in ``tests/_vcr_conftest_common.py`` and calls it from every VCR-using conftest's ``pytest_configure``. Same single source of truth, no inline duplication. --- tests/_vcr_conftest_common.py | 24 +++++++++++++++++++++ tests/audio_tests/conftest.py | 2 ++ tests/guardrails_tests/conftest.py | 2 ++ tests/image_gen_tests/conftest.py | 12 ++--------- tests/litellm_utils_tests/conftest.py | 2 ++ tests/llm_responses_api_testing/conftest.py | 2 ++ tests/llm_translation/conftest.py | 2 ++ tests/local_testing/conftest.py | 2 ++ tests/logging_callback_tests/conftest.py | 2 ++ tests/ocr_tests/conftest.py | 2 ++ tests/pass_through_unit_tests/conftest.py | 2 ++ tests/router_unit_tests/conftest.py | 2 ++ tests/search_tests/conftest.py | 2 ++ tests/unified_google_tests/conftest.py | 2 ++ 14 files changed, 50 insertions(+), 10 deletions(-) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index db041ad3a618..29fe4336f911 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -57,6 +57,30 @@ def vcr_diag_write_line(msg: str) -> None: pass +def reset_vcr_diag_dir() -> None: + """Delete any leftover per-PID diagnostic logs from a previous session. + + No-op when running on an xdist worker -- the controller does the + cleanup once and the workers inherit the (now-empty) directory. + Safe to call from any conftest's ``pytest_configure``. + """ + if os.environ.get("PYTEST_XDIST_WORKER"): + return + directory = _vcr_diag_dir() + if not os.path.isdir(directory): + return + try: + names = os.listdir(directory) + except OSError: + return + for name in names: + if name.endswith(".log"): + try: + os.remove(os.path.join(directory, name)) + except OSError: + pass + + def emit_vcr_diagnostic_log(terminalreporter) -> None: directory = _vcr_diag_dir() if not os.path.isdir(directory): diff --git a/tests/audio_tests/conftest.py b/tests/audio_tests/conftest.py index af142c15179a..5c9977aad811 100644 --- a/tests/audio_tests/conftest.py +++ b/tests/audio_tests/conftest.py @@ -14,6 +14,7 @@ install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, + reset_vcr_diag_dir, vcr_config_dict, ) @@ -45,6 +46,7 @@ def _vcr_outcome_gate(request, vcr): def pytest_configure(config): _verbose_state.remember_pluginmanager(config) + reset_vcr_diag_dir() def pytest_runtest_logreport(report): diff --git a/tests/guardrails_tests/conftest.py b/tests/guardrails_tests/conftest.py index 8b88a12c5045..c3ea3239a90f 100644 --- a/tests/guardrails_tests/conftest.py +++ b/tests/guardrails_tests/conftest.py @@ -25,6 +25,7 @@ install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, + reset_vcr_diag_dir, vcr_config_dict, ) @@ -56,6 +57,7 @@ def _vcr_outcome_gate(request, vcr): def pytest_configure(config): _verbose_state.remember_pluginmanager(config) + reset_vcr_diag_dir() def pytest_runtest_logreport(report): diff --git a/tests/image_gen_tests/conftest.py b/tests/image_gen_tests/conftest.py index 6f00c7528b35..5091230355a0 100644 --- a/tests/image_gen_tests/conftest.py +++ b/tests/image_gen_tests/conftest.py @@ -11,7 +11,6 @@ from tests._vcr_conftest_common import ( # noqa: E402 VerboseReporterState, - _vcr_diag_dir, apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, @@ -20,6 +19,7 @@ pin_httpx_multipart_boundary, record_vcr_outcome, register_persister_if_enabled, + reset_vcr_diag_dir, vcr_config_dict, ) @@ -69,15 +69,7 @@ def _vcr_outcome_gate(request, vcr): def pytest_configure(config): _verbose_state.remember_pluginmanager(config) - if not os.environ.get("PYTEST_XDIST_WORKER"): - directory = _vcr_diag_dir() - if os.path.isdir(directory): - for name in os.listdir(directory): - if name.endswith(".log"): - try: - os.remove(os.path.join(directory, name)) - except OSError: - pass + reset_vcr_diag_dir() def pytest_runtest_logreport(report): diff --git a/tests/litellm_utils_tests/conftest.py b/tests/litellm_utils_tests/conftest.py index efeb15cfac9b..120d58ba1fb3 100644 --- a/tests/litellm_utils_tests/conftest.py +++ b/tests/litellm_utils_tests/conftest.py @@ -21,6 +21,7 @@ install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, + reset_vcr_diag_dir, vcr_config_dict, ) @@ -87,6 +88,7 @@ def _vcr_outcome_gate(request, vcr): def pytest_configure(config): _verbose_state.remember_pluginmanager(config) + reset_vcr_diag_dir() def pytest_runtest_logreport(report): diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py index f15308594b5f..4dee8cd15a79 100644 --- a/tests/llm_responses_api_testing/conftest.py +++ b/tests/llm_responses_api_testing/conftest.py @@ -22,6 +22,7 @@ install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, + reset_vcr_diag_dir, vcr_config_dict, ) @@ -53,6 +54,7 @@ def _vcr_outcome_gate(request, vcr): def pytest_configure(config): _verbose_state.remember_pluginmanager(config) + reset_vcr_diag_dir() def pytest_runtest_logreport(report): diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index 570068877916..5e02a48b5fcd 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -27,6 +27,7 @@ install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, + reset_vcr_diag_dir, vcr_config_dict, ) @@ -74,6 +75,7 @@ def _vcr_outcome_gate(request, vcr): def pytest_configure(config): _verbose_state.remember_pluginmanager(config) + reset_vcr_diag_dir() def pytest_runtest_logreport(report): diff --git a/tests/local_testing/conftest.py b/tests/local_testing/conftest.py index ffaf7ac52654..af81f55b1341 100644 --- a/tests/local_testing/conftest.py +++ b/tests/local_testing/conftest.py @@ -31,6 +31,7 @@ install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, + reset_vcr_diag_dir, vcr_config_dict, ) @@ -85,6 +86,7 @@ def _vcr_outcome_gate(request, vcr): def pytest_configure(config): _verbose_state.remember_pluginmanager(config) + reset_vcr_diag_dir() def pytest_runtest_logreport(report): diff --git a/tests/logging_callback_tests/conftest.py b/tests/logging_callback_tests/conftest.py index b16df83f2f04..47c03bfd273e 100644 --- a/tests/logging_callback_tests/conftest.py +++ b/tests/logging_callback_tests/conftest.py @@ -28,6 +28,7 @@ install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, + reset_vcr_diag_dir, vcr_config_dict, ) @@ -80,6 +81,7 @@ def _vcr_outcome_gate(request, vcr): def pytest_configure(config): _verbose_state.remember_pluginmanager(config) + reset_vcr_diag_dir() def pytest_runtest_logreport(report): diff --git a/tests/ocr_tests/conftest.py b/tests/ocr_tests/conftest.py index 781882c1eb58..0cce6aa6bc19 100644 --- a/tests/ocr_tests/conftest.py +++ b/tests/ocr_tests/conftest.py @@ -21,6 +21,7 @@ install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, + reset_vcr_diag_dir, vcr_config_dict, ) @@ -52,6 +53,7 @@ def _vcr_outcome_gate(request, vcr): def pytest_configure(config): _verbose_state.remember_pluginmanager(config) + reset_vcr_diag_dir() def pytest_runtest_logreport(report): diff --git a/tests/pass_through_unit_tests/conftest.py b/tests/pass_through_unit_tests/conftest.py index f158b44e1efa..70dc6224f490 100644 --- a/tests/pass_through_unit_tests/conftest.py +++ b/tests/pass_through_unit_tests/conftest.py @@ -14,6 +14,7 @@ install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, + reset_vcr_diag_dir, vcr_config_dict, ) @@ -57,6 +58,7 @@ def _vcr_outcome_gate(request, vcr): def pytest_configure(config): _verbose_state.remember_pluginmanager(config) + reset_vcr_diag_dir() def pytest_runtest_logreport(report): diff --git a/tests/router_unit_tests/conftest.py b/tests/router_unit_tests/conftest.py index 46101d7d906a..e28bc32387f9 100644 --- a/tests/router_unit_tests/conftest.py +++ b/tests/router_unit_tests/conftest.py @@ -21,6 +21,7 @@ install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, + reset_vcr_diag_dir, vcr_config_dict, ) @@ -98,6 +99,7 @@ def _vcr_outcome_gate(request, vcr): def pytest_configure(config): _verbose_state.remember_pluginmanager(config) + reset_vcr_diag_dir() def pytest_runtest_logreport(report): diff --git a/tests/search_tests/conftest.py b/tests/search_tests/conftest.py index 5159caf081de..57226abd7ccb 100644 --- a/tests/search_tests/conftest.py +++ b/tests/search_tests/conftest.py @@ -22,6 +22,7 @@ install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, + reset_vcr_diag_dir, vcr_config_dict, ) @@ -53,6 +54,7 @@ def _vcr_outcome_gate(request, vcr): def pytest_configure(config): _verbose_state.remember_pluginmanager(config) + reset_vcr_diag_dir() def pytest_runtest_logreport(report): diff --git a/tests/unified_google_tests/conftest.py b/tests/unified_google_tests/conftest.py index a4c2c0c57520..75e010a72063 100644 --- a/tests/unified_google_tests/conftest.py +++ b/tests/unified_google_tests/conftest.py @@ -21,6 +21,7 @@ install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, + reset_vcr_diag_dir, vcr_config_dict, ) @@ -85,6 +86,7 @@ def _vcr_outcome_gate(request, vcr): def pytest_configure(config): _verbose_state.remember_pluginmanager(config) + reset_vcr_diag_dir() def pytest_runtest_logreport(report): From 1957d6d7b59b58a17e0a2631a6d373f08a12f963 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 17 May 2026 08:53:16 +0000 Subject: [PATCH 13/17] fix(tests): gate body materialization on __next__ and strip PR comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit aiohttp/vcrpy stores the json kwarg as a dict; _materialize_iterable_body was iterating it via __iter__ and joining the keys, replacing the request body with concatenated key names ("textlanguageentities"). Gate on __next__ so containers (dict/list/tuple) are left alone — only single-use iterators like httpx's bytes_iterator / list_iterator are materialized. Log diagnostic line when chunk type is unrecognized. --- tests/_vcr_conftest_common.py | 80 ++++++++--------------- tests/_vcr_redis_persister.py | 8 --- tests/image_gen_tests/test_image_edits.py | 3 - 3 files changed, 28 insertions(+), 63 deletions(-) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index 29fe4336f911..ccaa76e70983 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -36,8 +36,6 @@ KEY_FINGERPRINT_MATCHER_NAME = "key_fingerprint" KEY_FINGERPRINT_HEADER = "x-litellm-key-fp" -# Per-PID files bypass pytest/xdist stdout capture, which swallows -# stderr from passing tests. VCR_DIAG_DIR_ENV = "LITELLM_VCR_DIAG_DIR" VCR_DIAG_DIR_DEFAULT = "test-results/vcr-diagnostics" @@ -58,12 +56,6 @@ def vcr_diag_write_line(msg: str) -> None: def reset_vcr_diag_dir() -> None: - """Delete any leftover per-PID diagnostic logs from a previous session. - - No-op when running on an xdist worker -- the controller does the - cleanup once and the workers inherit the (now-empty) directory. - Safe to call from any conftest's ``pytest_configure``. - """ if os.environ.get("PYTEST_XDIST_WORKER"): return directory = _vcr_diag_dir() @@ -169,11 +161,9 @@ def emit_vcr_diagnostic_log(terminalreporter) -> None: def pin_httpx_multipart_boundary(monkeypatch) -> None: - """Force every httpx multipart request to use a constant boundary so - request bodies are byte-stable across runs (vcrpy match-on-body).""" try: import httpx._multipart as _httpx_multipart - except ImportError: # pragma: no cover - httpx is a hard test dep + except ImportError: return _original_init = _httpx_multipart.MultipartStream.__init__ @@ -300,17 +290,6 @@ def _before_record_response(response): def _canonical_body(request) -> tuple[bytes, str]: - """Return ``(body_bytes, original_type_name)`` for a vcrpy request. - - Materializes iterables / generators (httpx async wraps the body in a - ``list_iterator`` or ``bytes_iterator``), then coerces the result to - ``bytes``. Routing every matcher through this helper makes the - "compare object identity by mistake" failure mode structurally - impossible -- the comparison always operates on bytes. - - Logs a diagnostic line when a body falls into the empty-fallback - branch (unknown shape). Never raises. - """ pre_type = type(getattr(request, "body", None)).__name__ _materialize_iterable_body(request) body = getattr(request, "body", None) @@ -581,45 +560,26 @@ def _before_record_request(request): def _materialize_iterable_body(request) -> None: - """Coalesce an iterable / generator request body to ``bytes`` in-place - so the body matcher and boundary normalizer see a contiguous buffer. - - Once ``list(body)`` runs the original iterator is exhausted, so this - function must always write some bytes value back -- leaving the body - as a dead iterator silently makes the next HTTP send transmit an - empty payload. Also clears vcrpy's sticky ``_was_iter`` / ``_was_file`` - flags, which otherwise make the ``body`` getter re-wrap the stored - bytes in ``iter()`` on every access (so a freshly-materialized body - would look like ``bytes_iterator`` to the next reader). - """ body = getattr(request, "body", None) if body is None or isinstance(body, (bytes, bytearray, str)): return - if not hasattr(body, "__iter__"): + if not hasattr(body, "__next__"): return try: chunks = list(body) except TypeError: return - out = b"" - if chunks: - first = chunks[0] - if isinstance(first, int): - try: - out = bytes(chunks) - except (TypeError, ValueError): - out = b"" - elif isinstance(first, (bytes, bytearray)): - try: - out = b"".join(c if isinstance(c, bytes) else bytes(c) for c in chunks) - except (TypeError, ValueError): - out = b"" - elif isinstance(first, str): - try: - out = "".join(chunks).encode("utf-8") - except (TypeError, ValueError): - out = b"" + out = _coalesce_chunks_to_bytes(chunks) + if out is None: + method = getattr(request, "method", "?") + uri = getattr(request, "uri", getattr(request, "url", "?")) + first_type = type(chunks[0]).__name__ if chunks else "empty" + vcr_diag_write_line( + f"[vcr-materialize] FALLBACK: {method} {uri} chunk type " + f"{first_type!r} not coerced to bytes; storing b''" + ) + out = b"" try: request.body = out @@ -633,6 +593,22 @@ def _materialize_iterable_body(request) -> None: pass +def _coalesce_chunks_to_bytes(chunks): + if not chunks: + return b"" + first = chunks[0] + try: + if isinstance(first, int): + return bytes(chunks) + if isinstance(first, (bytes, bytearray)): + return b"".join(c if isinstance(c, bytes) else bytes(c) for c in chunks) + if isinstance(first, str): + return "".join(chunks).encode("utf-8") + except (TypeError, ValueError): + return None + return None + + def _key_fingerprint_matcher(r1, r2) -> None: def _fp(req): for value in _iter_header_values( diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index 3e543a1a6e6e..050f9879fbd7 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -212,14 +212,6 @@ def save_cassette(cassette_path, cassette_dict, serializer): def _log_episode_body_hashes(key: str, cassette_dict) -> None: - """Record a per-episode body SHA-256 for every cassette save. - - Lets two consecutive CI runs be diffed: if the same test records a - different hash run-to-run, the live request body varies; if both - runs record the same hash but the matcher still misses, the bug is - in the matcher itself. Negligible cost (~1ms hashing + ~200B file - write per saved episode; nothing at all when cassettes replay). - """ import hashlib from tests._vcr_conftest_common import vcr_diag_write_line diff --git a/tests/image_gen_tests/test_image_edits.py b/tests/image_gen_tests/test_image_edits.py index d6600ecc94f4..ca8ec3bbe32e 100644 --- a/tests/image_gen_tests/test_image_edits.py +++ b/tests/image_gen_tests/test_image_edits.py @@ -103,9 +103,6 @@ async def test_openai_image_edit_litellm_sdk(self, sync_mode): pwd = os.path.dirname(os.path.realpath(__file__)) -# Fixtures must be raw ``bytes``, not ``BytesIO``: an SDK retry that -# reads a BytesIO twice gets an empty second body, which records as a -# divergent VCR episode and eventually trips MAX_EPISODES_PER_CASSETTE. def _read_image_bytes(filename: str) -> bytes: with open(os.path.join(pwd, filename), "rb") as f: return f.read() From bfa1cb8af151e762d5265e7bed3fc4e226aad30a Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 17 May 2026 16:23:35 +0000 Subject: [PATCH 14/17] fix(tests): JSON-encode dict bodies in canonical_body for stable matching aiohttp stubs store the json kwarg as a dict; the fallback that compared all dicts as b"" caused concurrent presidio analyze calls to be served the wrong cassette episode. JSON-encode with sort_keys for stable bytes. --- tests/_vcr_conftest_common.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index ccaa76e70983..7f62cfd97d9a 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -301,6 +301,14 @@ def _canonical_body(request) -> tuple[bytes, str]: return bytes(body), pre_type if isinstance(body, str): return body.encode("utf-8"), pre_type + if isinstance(body, (dict, list)): + try: + return ( + json.dumps(body, sort_keys=True, separators=(",", ":")).encode("utf-8"), + pre_type, + ) + except (TypeError, ValueError): + pass method = getattr(request, "method", "?") uri = getattr(request, "uri", getattr(request, "url", "?")) vcr_diag_write_line( From f41a2b1ae3d8f84760cb91ed6e75dfb187eb8c6d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 17 May 2026 16:52:01 +0000 Subject: [PATCH 15/17] fix(tests): guard emit_vcr_diagnostic_log against multi-conftest re-emission Co-authored-by: Yassin Kortam --- tests/_vcr_conftest_common.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index 7f62cfd97d9a..ad2d6bbc0e2c 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -73,7 +73,13 @@ def reset_vcr_diag_dir() -> None: pass +_vcr_diagnostic_log_emitted = False + + def emit_vcr_diagnostic_log(terminalreporter) -> None: + global _vcr_diagnostic_log_emitted + if _vcr_diagnostic_log_emitted: + return directory = _vcr_diag_dir() if not os.path.isdir(directory): return @@ -83,6 +89,7 @@ def emit_vcr_diagnostic_log(terminalreporter) -> None: return if not files: return + _vcr_diagnostic_log_emitted = True terminalreporter.write_sep("=", "VCR DIAGNOSTIC LOG", bold=True) terminalreporter.write_line( f" source dir: {directory} (also archived as a CI artifact)" From dc59efb2fcedde7935433e4bdcfe108d9bf1e9bd Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 17 May 2026 20:20:21 +0000 Subject: [PATCH 16/17] fix(tests): globalize multipart-boundary pin + stabilize whisper fixtures Diagnostic shows audio_testing was silently re-recording 50+ live Whisper episodes per CI run (over MAX_EPISODES_PER_CASSETTE, so the persister refused to save). Two changes: * Move the session-autouse _pin_multipart_boundary fixture into the shared _vcr_conftest_common module so every VCR-using suite picks it up via a single import. image_gen had it inline; the other 12 suites silently lacked it. * Replace the module-level open("rb") audio file handles in test_whisper with cached bytes + a per-call (filename, bytes, mimetype) tuple, mirroring the image_edits raw-bytes pattern. Stops the file-pointer- at-EOF bug where the second test got an empty multipart body. --- tests/_vcr_conftest_common.py | 8 +++++ tests/audio_tests/conftest.py | 3 +- tests/audio_tests/test_whisper.py | 39 +++++++++++++-------- tests/guardrails_tests/conftest.py | 3 +- tests/image_gen_tests/conftest.py | 12 ++----- tests/litellm_utils_tests/conftest.py | 3 +- tests/llm_responses_api_testing/conftest.py | 3 +- tests/llm_translation/conftest.py | 3 +- tests/local_testing/conftest.py | 3 +- tests/logging_callback_tests/conftest.py | 3 +- tests/ocr_tests/conftest.py | 3 +- tests/pass_through_unit_tests/conftest.py | 3 +- tests/router_unit_tests/conftest.py | 3 +- tests/search_tests/conftest.py | 3 +- tests/unified_google_tests/conftest.py | 3 +- 15 files changed, 59 insertions(+), 36 deletions(-) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index ad2d6bbc0e2c..5a9f3346b619 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -185,6 +185,14 @@ def _init_with_fixed_boundary(self, data, files, boundary=None, **kwargs): ) +@pytest.fixture(scope="session", autouse=True) +def _pin_multipart_boundary(): + monkeypatch = pytest.MonkeyPatch() + pin_httpx_multipart_boundary(monkeypatch) + yield + monkeypatch.undo() + + def _scrub_response(response): if not isinstance(response, dict): return response diff --git a/tests/audio_tests/conftest.py b/tests/audio_tests/conftest.py index 5c9977aad811..c4ff576e5bd9 100644 --- a/tests/audio_tests/conftest.py +++ b/tests/audio_tests/conftest.py @@ -5,8 +5,9 @@ sys.path.insert(0, os.path.abspath("../..")) -from tests._vcr_conftest_common import ( # noqa: E402 +from tests._vcr_conftest_common import ( # noqa: E402,F401 VerboseReporterState, + _pin_multipart_boundary, apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, diff --git a/tests/audio_tests/test_whisper.py b/tests/audio_tests/test_whisper.py index cdf079f8cb4f..243d27614b10 100644 --- a/tests/audio_tests/test_whisper.py +++ b/tests/audio_tests/test_whisper.py @@ -23,12 +23,21 @@ print(pwd) file_path = os.path.join(pwd, "gettysburg.wav") +file2_path = os.path.join(pwd, "eagle.wav") -audio_file = open(file_path, "rb") +with open(file_path, "rb") as _f: + _GETTYSBURG_BYTES = _f.read() +with open(file2_path, "rb") as _f: + _EAGLE_BYTES = _f.read() -file2_path = os.path.join(pwd, "eagle.wav") -audio_file2 = open(file2_path, "rb") +def _audio_file(): + return ("gettysburg.wav", _GETTYSBURG_BYTES, "audio/wav") + + +def _audio_file2(): + return ("eagle.wav", _EAGLE_BYTES, "audio/wav") + load_dotenv() @@ -44,7 +53,7 @@ async def _run_transcription( ): transcript = await litellm.atranscription( model=model, - file=audio_file, + file=_audio_file(), api_key=api_key, api_base=api_base, response_format=response_format, @@ -101,7 +110,7 @@ async def test_transcription_caching(): response_1 = await litellm.atranscription( model="whisper-1", - file=audio_file, + file=_audio_file(), ) await asyncio.sleep(5) @@ -110,7 +119,7 @@ async def test_transcription_caching(): response_2 = await litellm.atranscription( model="whisper-1", - file=audio_file, + file=_audio_file(), ) print("response_1", response_1) @@ -122,7 +131,7 @@ async def test_transcription_caching(): response_3 = await litellm.atranscription( model="whisper-1", - file=audio_file2, + file=_audio_file2(), ) print("response_3", response_3) print("response3 hidden params", response_3._hidden_params) @@ -146,7 +155,7 @@ async def test_whisper_log_pre_call(): with patch.object(custom_logger, "log_pre_api_call") as mock_log_pre_call: await litellm.atranscription( model="whisper-1", - file=audio_file, + file=_audio_file(), ) mock_log_pre_call.assert_called_once() @@ -165,7 +174,7 @@ async def test_whisper_log_pre_call(): with patch.object(custom_logger, "log_pre_api_call") as mock_log_pre_call: await litellm.atranscription( model="whisper-1", - file=audio_file, + file=_audio_file(), ) mock_log_pre_call.assert_called_once() @@ -177,7 +186,7 @@ async def test_gpt_4o_transcribe(): from unittest.mock import patch, MagicMock await litellm.atranscription( - model="openai/gpt-4o-transcribe", file=audio_file, response_format="json" + model="openai/gpt-4o-transcribe", file=_audio_file(), response_format="json" ) @@ -187,7 +196,9 @@ async def test_gpt_4o_transcribe_model_mapping(): # Test GPT-4o mini transcribe response = await litellm.atranscription( - model="openai/gpt-4o-mini-transcribe", file=audio_file, response_format="json" + model="openai/gpt-4o-mini-transcribe", + file=_audio_file(), + response_format="json", ) # Check that the response contains the correct model in hidden params @@ -198,7 +209,7 @@ async def test_gpt_4o_transcribe_model_mapping(): # Test GPT-4o transcribe response2 = await litellm.atranscription( - model="openai/gpt-4o-transcribe", file=audio_file, response_format="json" + model="openai/gpt-4o-transcribe", file=_audio_file(), response_format="json" ) # Check that the response contains the correct model in hidden params @@ -209,7 +220,7 @@ async def test_gpt_4o_transcribe_model_mapping(): # Test traditional whisper-1 still works response3 = await litellm.atranscription( - model="openai/whisper-1", file=audio_file, response_format="json" + model="openai/whisper-1", file=_audio_file(), response_format="json" ) # Check that the response contains the correct model in hidden params @@ -262,7 +273,7 @@ class MockTranscriptionResponse(PydanticBaseModel): # Make the transcription call response = await litellm.atranscription( model="azure/whisper-1", - file=audio_file, + file=_audio_file(), response_format="json", api_key="test-api-key", api_base="https://my-endpoint-europe-berri-992.openai.azure.com/", diff --git a/tests/guardrails_tests/conftest.py b/tests/guardrails_tests/conftest.py index c3ea3239a90f..f2f65645c3db 100644 --- a/tests/guardrails_tests/conftest.py +++ b/tests/guardrails_tests/conftest.py @@ -16,8 +16,9 @@ ) # Adds the parent directory to the system path import litellm -from tests._vcr_conftest_common import ( # noqa: E402 +from tests._vcr_conftest_common import ( # noqa: E402,F401 VerboseReporterState, + _pin_multipart_boundary, apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, diff --git a/tests/image_gen_tests/conftest.py b/tests/image_gen_tests/conftest.py index 5091230355a0..9f808c11161f 100644 --- a/tests/image_gen_tests/conftest.py +++ b/tests/image_gen_tests/conftest.py @@ -9,14 +9,14 @@ ) # Adds the parent directory to the system path import litellm # noqa: E402,F401 -from tests._vcr_conftest_common import ( # noqa: E402 +from tests._vcr_conftest_common import ( # noqa: E402,F401 VerboseReporterState, + _pin_multipart_boundary, apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, emit_vcr_diagnostic_log, install_live_call_probe, - pin_httpx_multipart_boundary, record_vcr_outcome, register_persister_if_enabled, reset_vcr_diag_dir, @@ -36,14 +36,6 @@ def event_loop(): loop.close() -@pytest.fixture(scope="session", autouse=True) -def _pin_multipart_boundary(): - monkeypatch = pytest.MonkeyPatch() - pin_httpx_multipart_boundary(monkeypatch) - yield - monkeypatch.undo() - - @pytest.fixture(scope="module") def vcr_config(): return vcr_config_dict() diff --git a/tests/litellm_utils_tests/conftest.py b/tests/litellm_utils_tests/conftest.py index 120d58ba1fb3..418ee76a399c 100644 --- a/tests/litellm_utils_tests/conftest.py +++ b/tests/litellm_utils_tests/conftest.py @@ -12,8 +12,9 @@ ) # Adds the parent directory to the system path import litellm # noqa: E402,F401 -from tests._vcr_conftest_common import ( # noqa: E402 +from tests._vcr_conftest_common import ( # noqa: E402,F401 VerboseReporterState, + _pin_multipart_boundary, apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py index 4dee8cd15a79..1928b540dad0 100644 --- a/tests/llm_responses_api_testing/conftest.py +++ b/tests/llm_responses_api_testing/conftest.py @@ -13,8 +13,9 @@ import litellm # noqa: E402 -from tests._vcr_conftest_common import ( # noqa: E402 +from tests._vcr_conftest_common import ( # noqa: E402,F401 VerboseReporterState, + _pin_multipart_boundary, apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index 5e02a48b5fcd..d346dae43084 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -18,8 +18,9 @@ import litellm # noqa: E402 -from tests._vcr_conftest_common import ( # noqa: E402 +from tests._vcr_conftest_common import ( # noqa: E402,F401 VerboseReporterState, + _pin_multipart_boundary, apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, diff --git a/tests/local_testing/conftest.py b/tests/local_testing/conftest.py index af81f55b1341..6a746041f156 100644 --- a/tests/local_testing/conftest.py +++ b/tests/local_testing/conftest.py @@ -22,8 +22,9 @@ ) # Adds the parent directory to the system path import litellm -from tests._vcr_conftest_common import ( # noqa: E402 +from tests._vcr_conftest_common import ( # noqa: E402,F401 VerboseReporterState, + _pin_multipart_boundary, apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, diff --git a/tests/logging_callback_tests/conftest.py b/tests/logging_callback_tests/conftest.py index 47c03bfd273e..6dde85f2ca72 100644 --- a/tests/logging_callback_tests/conftest.py +++ b/tests/logging_callback_tests/conftest.py @@ -19,8 +19,9 @@ ) # Adds the parent directory to the system path import litellm -from tests._vcr_conftest_common import ( # noqa: E402 +from tests._vcr_conftest_common import ( # noqa: E402,F401 VerboseReporterState, + _pin_multipart_boundary, apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, diff --git a/tests/ocr_tests/conftest.py b/tests/ocr_tests/conftest.py index 0cce6aa6bc19..94790bd7aa3e 100644 --- a/tests/ocr_tests/conftest.py +++ b/tests/ocr_tests/conftest.py @@ -12,8 +12,9 @@ sys.path.insert(0, os.path.abspath("../..")) -from tests._vcr_conftest_common import ( # noqa: E402 +from tests._vcr_conftest_common import ( # noqa: E402,F401 VerboseReporterState, + _pin_multipart_boundary, apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, diff --git a/tests/pass_through_unit_tests/conftest.py b/tests/pass_through_unit_tests/conftest.py index 70dc6224f490..390e14b7f119 100644 --- a/tests/pass_through_unit_tests/conftest.py +++ b/tests/pass_through_unit_tests/conftest.py @@ -5,8 +5,9 @@ sys.path.insert(0, os.path.abspath("../..")) -from tests._vcr_conftest_common import ( # noqa: E402 +from tests._vcr_conftest_common import ( # noqa: E402,F401 VerboseReporterState, + _pin_multipart_boundary, apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, diff --git a/tests/router_unit_tests/conftest.py b/tests/router_unit_tests/conftest.py index e28bc32387f9..6a8f3e589f48 100644 --- a/tests/router_unit_tests/conftest.py +++ b/tests/router_unit_tests/conftest.py @@ -12,8 +12,9 @@ ) # Adds the parent directory to the system path import litellm # noqa: E402,F401 -from tests._vcr_conftest_common import ( # noqa: E402 +from tests._vcr_conftest_common import ( # noqa: E402,F401 VerboseReporterState, + _pin_multipart_boundary, apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, diff --git a/tests/search_tests/conftest.py b/tests/search_tests/conftest.py index 57226abd7ccb..78ba19a77241 100644 --- a/tests/search_tests/conftest.py +++ b/tests/search_tests/conftest.py @@ -13,8 +13,9 @@ sys.path.insert(0, os.path.abspath("../..")) -from tests._vcr_conftest_common import ( # noqa: E402 +from tests._vcr_conftest_common import ( # noqa: E402,F401 VerboseReporterState, + _pin_multipart_boundary, apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, diff --git a/tests/unified_google_tests/conftest.py b/tests/unified_google_tests/conftest.py index 75e010a72063..5b4f57b8036b 100644 --- a/tests/unified_google_tests/conftest.py +++ b/tests/unified_google_tests/conftest.py @@ -12,8 +12,9 @@ ) # Adds the parent directory to the system path import litellm # noqa: E402,F401 -from tests._vcr_conftest_common import ( # noqa: E402 +from tests._vcr_conftest_common import ( # noqa: E402,F401 VerboseReporterState, + _pin_multipart_boundary, apply_vcr_auto_marker_to_items, emit_cassette_cache_session_banner, emit_vcr_classification_summary, From ea771621a841bb3775c4c4d0984bb90ee8d66080 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 17 May 2026 21:33:27 +0000 Subject: [PATCH 17/17] chore(tests): drop per-episode body-hash dump and redundant emit guard --- tests/_vcr_conftest_common.py | 7 ------- tests/_vcr_redis_persister.py | 30 ------------------------------ 2 files changed, 37 deletions(-) diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index 5a9f3346b619..cb43f1abbdd4 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -73,13 +73,7 @@ def reset_vcr_diag_dir() -> None: pass -_vcr_diagnostic_log_emitted = False - - def emit_vcr_diagnostic_log(terminalreporter) -> None: - global _vcr_diagnostic_log_emitted - if _vcr_diagnostic_log_emitted: - return directory = _vcr_diag_dir() if not os.path.isdir(directory): return @@ -89,7 +83,6 @@ def emit_vcr_diagnostic_log(terminalreporter) -> None: return if not files: return - _vcr_diagnostic_log_emitted = True terminalreporter.write_sep("=", "VCR DIAGNOSTIC LOG", bold=True) terminalreporter.write_line( f" source dir: {directory} (also archived as a CI artifact)" diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index 050f9879fbd7..7fdb7267a382 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -168,7 +168,6 @@ def save_cassette(cassette_path, cassette_dict, serializer): key = redis_key_for(cassette_path) passed = _passed_by_cassette_key.pop(key, True) episode_count = len(cassette_dict.get("requests", []) or []) - _log_episode_body_hashes(key, cassette_dict) if episode_count > MAX_EPISODES_PER_CASSETTE: _log.warning( "VCR redis save refused for %s; cassette has %d episodes " @@ -211,35 +210,6 @@ def save_cassette(cassette_path, cassette_dict, serializer): return _RedisPersister -def _log_episode_body_hashes(key: str, cassette_dict) -> None: - import hashlib - - from tests._vcr_conftest_common import vcr_diag_write_line - - requests = cassette_dict.get("requests", []) or [] - for i, req in enumerate(requests): - body = getattr(req, "body", None) - if body is None: - body_bytes = b"" - elif isinstance(body, (bytes, bytearray)): - body_bytes = bytes(body) - elif isinstance(body, str): - body_bytes = body.encode("utf-8") - else: - vcr_diag_write_line( - f"[vcr-episode-body-hash] {key} episode[{i}]: body type=" - f"{type(body).__name__!r} not bytes/bytearray/str -- cannot hash" - ) - continue - method = getattr(req, "method", "?") - uri = getattr(req, "uri", getattr(req, "url", "?")) - vcr_diag_write_line( - f"[vcr-episode-body-hash] {key} episode[{i}] {method} {uri} " - f"body sha256={hashlib.sha256(body_bytes).hexdigest()} " - f"len={len(body_bytes)} preview={body_bytes[:120]!r}" - ) - - def filter_non_2xx_response(response): if not isinstance(response, dict): return response