From 7736d14d22bdb9a2c53f74275e2399ff437e2175 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Tue, 30 Jun 2026 12:12:27 -0700 Subject: [PATCH 1/2] ci(codspeed): pin benchmark runner to ubuntu-24.04 ubuntu-latest resolves to different runner images between the BASE (main/staging) and HEAD (PR) runs, so CodSpeed reports 'Different runtime environments detected' and emits false-positive regressions (e.g. a -25.2% swing on test_completion_multi_turn in #31684, an MCP auth fix with no LLM code changes). Pinning the runner to a fixed image keeps BASE and HEAD on the same hardware so 1 ms swings on a ~3 ms benchmark stop blocking unrelated PRs. Fixes #31738 --- .github/workflows/codspeed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 1fad82827ff4..daaa2d23a910 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -23,7 +23,7 @@ concurrency: jobs: benchmarks: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 15 steps: From 00bf227ae32c9676858de112623cbbca00b5962b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 30 Jun 2026 13:19:23 -0700 Subject: [PATCH 2/2] ci(codspeed): stop running benchmarks on litellm_internal_staging The CodSpeed check flip-flops on internal staging and on PRs targeting it (e.g. "+11.75% improvement" on one run, "-25.36% regression" on the next) because the comparison flags "different runtime environments" and the benchmarks are only 3-4 ms, so sub-millisecond runner noise swings the result by 25-30%. Pinning the runner to ubuntu-24.04 in this PR helps the head side, but the internal_staging base is still recorded on the old unpinned runner, so comparisons keep flapping until the pin merges and the base is re-baselined. Until that settles, the red X's on internal staging make the OSS project look unhealthy and confuse contributors, so drop the litellm_internal_staging push and pull_request triggers and keep CodSpeed running on main only. --- .github/workflows/codspeed.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index daaa2d23a910..49f1d906069a 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -4,11 +4,9 @@ on: push: branches: - main - - litellm_internal_staging pull_request: branches: - main - - litellm_internal_staging # Allow CodSpeed to trigger backtest performance analysis # in order to generate initial data workflow_dispatch: