From 8da870f464cc574b27238c80318a559c432d8c7d Mon Sep 17 00:00:00 2001
From: Yassin Kortam <yassin@berri.ai>
Date: Wed, 8 Jul 2026 09:46:19 +0300
Subject: [PATCH] test(benchmarks): run shared logging executor inline to make
 CodSpeed measurements deterministic

---
 tests/benchmarks/conftest.py        | 36 +++++++++++++++++++++++++++++
 tests/benchmarks/test_benchmarks.py | 21 +++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100644 tests/benchmarks/conftest.py

diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py
new file mode 100644
index 000000000000..c9b31cfb7d77
--- /dev/null
+++ b/tests/benchmarks/conftest.py
@@ -0,0 +1,36 @@
+"""Shared setup keeping CodSpeed measurements hermetic.
+
+CodSpeed's callgrind instrumentation counts instructions from every thread while
+a measurement window is open, and valgrind serializes all threads onto one
+virtual CPU. Work deferred to litellm's shared logging executor would therefore
+be attributed to whichever benchmark the valgrind scheduler resumes it under,
+flipping results between runs. Running the executor inline keeps each
+benchmark's cost self-contained and deterministic.
+"""
+
+from collections.abc import Callable, Iterator
+from concurrent.futures import Future
+from typing import ParamSpec, TypeVar
+
+import pytest
+
+from litellm.litellm_core_utils.thread_pool_executor import executor
+
+P = ParamSpec("P")
+R = TypeVar("R")
+
+
+def _submit_inline(fn: Callable[P, R], /, *args: P.args, **kwargs: P.kwargs) -> Future[R]:
+    future: Future[R] = Future()
+    try:
+        future.set_result(fn(*args, **kwargs))
+    except BaseException as exc:
+        future.set_exception(exc)
+    return future
+
+
+@pytest.fixture(autouse=True, scope="session")
+def inline_logging_executor() -> Iterator[None]:
+    executor.submit = _submit_inline
+    yield
+    del executor.submit
diff --git a/tests/benchmarks/test_benchmarks.py b/tests/benchmarks/test_benchmarks.py
index 123dad93e11a..59b3e0b6d5cb 100644
--- a/tests/benchmarks/test_benchmarks.py
+++ b/tests/benchmarks/test_benchmarks.py
@@ -6,10 +6,13 @@
 resolution, and cost calculation.
 """
 
+import threading
+
 import pytest
 
 import litellm
 from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
+from litellm.litellm_core_utils.thread_pool_executor import executor
 from litellm.litellm_core_utils.token_counter import token_counter
 
 
@@ -205,3 +208,21 @@ def test_get_model_cost_key_exact_match():
 def test_get_model_cost_key_case_insensitive():
     """Benchmark model cost key lookup with case-insensitive fallback."""
     litellm.utils._get_model_cost_key("GPT-4o")
+
+
+# ---------------------------------------------------------------------------
+# Measurement hermeticity guard
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.benchmark
+def test_logging_executor_runs_inline():
+    """Guard that the shared logging executor runs submissions inline.
+
+    Deferred submissions execute on worker threads, and callgrind attributes
+    their instructions to whichever benchmark's measurement window is open when
+    the valgrind scheduler resumes them, making results nondeterministic.
+    """
+    future = executor.submit(threading.get_ident)
+    assert future.done()
+    assert future.result() == threading.get_ident()