From f491dd5fb8460a4f5f5bbb1d200a39dcf844b04f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 20 Jun 2026 17:25:49 +0000 Subject: [PATCH 1/4] feat: add CI-parity mode and truncation-proof summary to strict ruff gate --- Makefile | 9 +- scripts/ruff_strict_gate.py | 115 +++++++++++++++----- tests/test_litellm/test_ruff_strict_gate.py | 10 ++ 3 files changed, 103 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index 6183dff15560..ff1717c1ab57 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \ info lint lint-dev format \ lint-basedpyright lint-basedpyright-budget-update \ - lint-ruff-budget lint-ruff-budget-update lint-budget-update \ + lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \ install-dev install-proxy-dev install-test-deps install-hooks \ install-helm-unittest check-circular-imports check-import-safety @@ -28,6 +28,7 @@ help: @echo " make lint-basedpyright-budget-update - Re-capture the basedpyright per-rule budget (ratchet)" @echo " make lint-black - Check Black formatting (matches CI)" @echo " make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its ceiling" + @echo " make lint-gate - Strict ruff gate in CI-parity mode (fetches staging, simulates the merge)" @echo " make lint-ruff-budget-update - Re-capture per-rule baselines in ruff-strict-budget.json (ratchet)" @echo " make lint-budget-update - Re-capture all ratchet budgets (ruff + basedpyright)" @echo " make check-circular-imports - Check for circular imports" @@ -134,6 +135,12 @@ lint-black: format-check lint-ruff-budget: install-dev $(UV_RUN) python scripts/ruff_strict_gate.py +# CI-parity strict gate: fetch the target branch fresh and count against a +# throwaway merge into HEAD, so a local pass means the CI check will pass too. +lint-gate: install-dev + git fetch origin litellm_internal_staging + $(UV_RUN) python scripts/ruff_strict_gate.py --ci-parity --base origin/litellm_internal_staging + lint-ruff-budget-update: install-dev $(UV_RUN) python scripts/ruff_strict_gate.py --update diff --git a/scripts/ruff_strict_gate.py b/scripts/ruff_strict_gate.py index 5951a1215ed3..fd9dd9a9efc6 100644 --- a/scripts/ruff_strict_gate.py +++ b/scripts/ruff_strict_gate.py @@ -5,6 +5,12 @@ gate counts each rule across the whole tree and fails when a rule is both over its ceiling and higher than the base it merges into, so a change is blamed for the violations it adds, never for drift that already exists in the base. + +By default the base is the merge-base of the current branch, which is fast but +diverges from CI: CI runs against the synthetic merge ref, so its base is the +*current* tip of the target branch plus whatever drift landed since you forked. +Pass --ci-parity to reproduce CI exactly by counting against a throwaway merge +of the base into HEAD. """ import argparse @@ -40,6 +46,12 @@ class Breach(NamedTuple): added: int +class GateInputs(NamedTuple): + head: list + base: dict + changed: dict + + def _run(cmd: list, cwd: Path = REPO_ROOT) -> str: proc = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) if proc.returncode not in (0, 1): @@ -56,14 +68,14 @@ def _ruff_json(cwd: Path, config: Path) -> list: return json.loads(raw or "[]") -def head_violations() -> list: +def collect_violations(root: Path, config: Path) -> list: out = [] - for item in _ruff_json(REPO_ROOT, STRICT_CONFIG): + for item in _ruff_json(root, config): name = Path(item["filename"]) rel = ( - (name if name.is_absolute() else REPO_ROOT / name) + (name if name.is_absolute() else root / name) .resolve() - .relative_to(REPO_ROOT) + .relative_to(root) .as_posix() ) out.append(Violation(rel, item["location"]["row"], item["code"])) @@ -80,23 +92,14 @@ def base_counts(ref: str) -> dict: try: _run(["git", "worktree", "add", "--detach", str(worktree), ref]) shutil.copy(STRICT_CONFIG, worktree / "ruff-strict.toml") - items = _ruff_json(worktree, worktree / "ruff-strict.toml") - return dict(Counter(item["code"] for item in items)) + return count_by_rule( + collect_violations(worktree, worktree / "ruff-strict.toml") + ) finally: _run(["git", "worktree", "remove", "--force", str(worktree)]) shutil.rmtree(parent, ignore_errors=True) -def evaluate(head: dict, base: dict, budget: dict) -> list: - breaches = [] - for rule, spec in budget.items(): - cap = spec["baseline"] + spec["slack"] - total = head.get(rule, 0) - if total > cap and total > base.get(rule, 0): - breaches.append(Breach(rule, total, cap, total - base.get(rule, 0))) - return sorted(breaches) - - def parse_changed_lines(diff_text: str) -> dict: changed: dict = {} path = None @@ -110,24 +113,59 @@ def parse_changed_lines(diff_text: str) -> dict: return changed +def evaluate(head: dict, base: dict, budget: dict) -> list: + breaches = [] + for rule, spec in budget.items(): + cap = spec["baseline"] + spec["slack"] + total = head.get(rule, 0) + if total > cap and total > base.get(rule, 0): + breaches.append(Breach(rule, total, cap, total - base.get(rule, 0))) + return sorted(breaches) + + def introduced(violations: list, changed: dict) -> list: return [v for v in violations if v.line in changed.get(v.file, set())] -def cmd_check(base: str) -> None: - budget = json.loads(BUDGET_PATH.read_text()) - head = head_violations() +def gather_fast(base: str) -> GateInputs: base_point = _run(["git", "merge-base", base, "HEAD"]).strip() or base - breaches = evaluate(count_by_rule(head), base_counts(base_point), budget) - if not breaches: - print(f"OK: every strict rule is within its codebase ceiling (base {base})") - return - new = introduced( - head, - parse_changed_lines( - _run(["git", "diff", base_point, "--unified=0", "--no-color", "--", TARGET]) - ), + diff = _run(["git", "diff", base_point, "--unified=0", "--no-color", "--", TARGET]) + return GateInputs( + collect_violations(REPO_ROOT, STRICT_CONFIG), + base_counts(base_point), + parse_changed_lines(diff), ) + + +def gather_ci_parity(base: str) -> GateInputs: + parent = Path(tempfile.mkdtemp(prefix="ruff_merge_")) + worktree = parent / "wt" + try: + _run(["git", "worktree", "add", "--detach", str(worktree), "HEAD"]) + merge = subprocess.run( + ["git", "merge", "--no-commit", "--no-ff", base], + cwd=worktree, + capture_output=True, + text=True, + ) + if merge.returncode != 0: + _run(["git", "merge", "--abort"], cwd=worktree) + raise SystemExit( + f"ci-parity: merging {base} into HEAD conflicts; rebase your branch first" + ) + shutil.copy(STRICT_CONFIG, worktree / "ruff-strict.toml") + head = collect_violations(worktree, worktree / "ruff-strict.toml") + diff = _run( + ["git", "diff", base, "--unified=0", "--no-color", "--", TARGET], + cwd=worktree, + ) + return GateInputs(head, base_counts(base), parse_changed_lines(diff)) + finally: + _run(["git", "worktree", "remove", "--force", str(worktree)]) + shutil.rmtree(parent, ignore_errors=True) + + +def report(breaches: list, new: list, base: str) -> None: print(f"FAIL: strict-rule totals exceed their ceiling (base {base}):") for breach in breaches: print( @@ -138,12 +176,24 @@ def cmd_check(base: str) -> None: print( "Reduce the new violations or remove an equal number elsewhere; the ceiling is baseline + slack in ruff-strict-budget.json." ) + summary = "; ".join(f"{b.rule} {b.total}/{b.cap} (+{b.added})" for b in breaches) + print(f"BREACHED RULES: {summary}") + + +def cmd_check(base: str, ci_parity: bool) -> None: + budget = json.loads(BUDGET_PATH.read_text()) + inputs = gather_ci_parity(base) if ci_parity else gather_fast(base) + breaches = evaluate(count_by_rule(inputs.head), inputs.base, budget) + if not breaches: + print(f"OK: every strict rule is within its codebase ceiling (base {base})") + return + report(breaches, introduced(inputs.head, inputs.changed), base) raise SystemExit(1) def cmd_update() -> None: budget = json.loads(BUDGET_PATH.read_text()) - head = count_by_rule(head_violations()) + head = count_by_rule(collect_violations(REPO_ROOT, STRICT_CONFIG)) for rule in budget: budget[rule]["baseline"] = head.get(rule, 0) BUDGET_PATH.write_text(json.dumps(budget, indent=2, sort_keys=True) + "\n") @@ -154,8 +204,13 @@ def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base", default=DEFAULT_BASE) parser.add_argument("--update", action="store_true") + parser.add_argument( + "--ci-parity", + action="store_true", + help="count against a throwaway merge of --base into HEAD, matching CI", + ) args = parser.parse_args() - cmd_update() if args.update else cmd_check(args.base) + cmd_update() if args.update else cmd_check(args.base, args.ci_parity) if __name__ == "__main__": diff --git a/tests/test_litellm/test_ruff_strict_gate.py b/tests/test_litellm/test_ruff_strict_gate.py index 22255f0555ec..96852e3a8a5b 100644 --- a/tests/test_litellm/test_ruff_strict_gate.py +++ b/tests/test_litellm/test_ruff_strict_gate.py @@ -82,3 +82,13 @@ def test_introduced_keeps_only_violations_on_changed_lines(): @pytest.mark.parametrize("hunk", ["@@ -1 +1 @@", "@@ -1,0 +1,2 @@"]) def test_parse_changed_lines_handles_single_and_ranged_hunks(hunk): assert gate.parse_changed_lines(f"+++ b/litellm/a.py\n{hunk}\n")["litellm/a.py"] + + +def test_report_emits_breached_rules_as_final_line(capsys): + # CI surfaces only the tail of the log, so the breached-rule summary (rule, + # total/cap, added) must be the last line or it gets truncated away. + breaches = sorted([gate.Breach("UP045", 530, 529, 1), gate.Breach("ANN401", 12, 10, 2)]) + new = [gate.Violation("litellm/types/llms/bedrock.py", 16, "UP045")] + gate.report(breaches, new, "origin/litellm_internal_staging") + last = capsys.readouterr().out.strip().splitlines()[-1] + assert last == "BREACHED RULES: ANN401 12/10 (+2); UP045 530/529 (+1)" From 61d167ecf15373aacddfda33713ff9b65e00d381 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 20 Jun 2026 17:39:20 +0000 Subject: [PATCH 2/4] refactor: tolerant worktree cleanup and concrete GateInputs types --- scripts/ruff_strict_gate.py | 46 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/scripts/ruff_strict_gate.py b/scripts/ruff_strict_gate.py index fd9dd9a9efc6..066e7b163753 100644 --- a/scripts/ruff_strict_gate.py +++ b/scripts/ruff_strict_gate.py @@ -14,6 +14,7 @@ """ import argparse +import contextlib import json import re import shutil @@ -21,6 +22,7 @@ import sys import tempfile from collections import Counter +from collections.abc import Iterator from pathlib import Path from typing import NamedTuple @@ -47,9 +49,9 @@ class Breach(NamedTuple): class GateInputs(NamedTuple): - head: list - base: dict - changed: dict + head: list[Violation] + base: dict[str, int] + changed: dict[str, set[int]] def _run(cmd: list, cwd: Path = REPO_ROOT) -> str: @@ -86,18 +88,29 @@ def count_by_rule(violations: list) -> dict: return dict(Counter(v.code for v in violations)) -def base_counts(ref: str) -> dict: - parent = Path(tempfile.mkdtemp(prefix="ruff_base_")) +@contextlib.contextmanager +def _temp_worktree(ref: str) -> Iterator[Path]: + parent = Path(tempfile.mkdtemp(prefix="ruff_wt_")) worktree = parent / "wt" + _run(["git", "worktree", "add", "--detach", str(worktree), ref]) try: - _run(["git", "worktree", "add", "--detach", str(worktree), ref]) + yield worktree + finally: + subprocess.run( + ["git", "worktree", "remove", "--force", str(worktree)], + cwd=REPO_ROOT, + capture_output=True, + text=True, + ) + shutil.rmtree(parent, ignore_errors=True) + + +def base_counts(ref: str) -> dict: + with _temp_worktree(ref) as worktree: shutil.copy(STRICT_CONFIG, worktree / "ruff-strict.toml") return count_by_rule( collect_violations(worktree, worktree / "ruff-strict.toml") ) - finally: - _run(["git", "worktree", "remove", "--force", str(worktree)]) - shutil.rmtree(parent, ignore_errors=True) def parse_changed_lines(diff_text: str) -> dict: @@ -138,10 +151,7 @@ def gather_fast(base: str) -> GateInputs: def gather_ci_parity(base: str) -> GateInputs: - parent = Path(tempfile.mkdtemp(prefix="ruff_merge_")) - worktree = parent / "wt" - try: - _run(["git", "worktree", "add", "--detach", str(worktree), "HEAD"]) + with _temp_worktree("HEAD") as worktree: merge = subprocess.run( ["git", "merge", "--no-commit", "--no-ff", base], cwd=worktree, @@ -149,7 +159,12 @@ def gather_ci_parity(base: str) -> GateInputs: text=True, ) if merge.returncode != 0: - _run(["git", "merge", "--abort"], cwd=worktree) + subprocess.run( + ["git", "merge", "--abort"], + cwd=worktree, + capture_output=True, + text=True, + ) raise SystemExit( f"ci-parity: merging {base} into HEAD conflicts; rebase your branch first" ) @@ -160,9 +175,6 @@ def gather_ci_parity(base: str) -> GateInputs: cwd=worktree, ) return GateInputs(head, base_counts(base), parse_changed_lines(diff)) - finally: - _run(["git", "worktree", "remove", "--force", str(worktree)]) - shutil.rmtree(parent, ignore_errors=True) def report(breaches: list, new: list, base: str) -> None: From bdce6c7b4e7b99728fe9a2c16fb051a9ac8cfa45 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 20 Jun 2026 17:45:25 +0000 Subject: [PATCH 3/4] fix: clean up temp dir when git worktree add fails --- scripts/ruff_strict_gate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ruff_strict_gate.py b/scripts/ruff_strict_gate.py index 066e7b163753..c5973a2da408 100644 --- a/scripts/ruff_strict_gate.py +++ b/scripts/ruff_strict_gate.py @@ -92,8 +92,8 @@ def count_by_rule(violations: list) -> dict: def _temp_worktree(ref: str) -> Iterator[Path]: parent = Path(tempfile.mkdtemp(prefix="ruff_wt_")) worktree = parent / "wt" - _run(["git", "worktree", "add", "--detach", str(worktree), ref]) try: + _run(["git", "worktree", "add", "--detach", str(worktree), ref]) yield worktree finally: subprocess.run( From 497447aa8e15e6a0724f2b48a109e036f022ec2b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 20 Jun 2026 17:52:28 +0000 Subject: [PATCH 4/4] fix: align lint-gate with CI by dropping unused --ci-parity path The lint-gate Makefile target invoked ruff_strict_gate.py with --ci-parity, which counted violations on a throwaway merge of base into HEAD against base counts at the base tip. CI in test-linting.yml runs the same script without --ci-parity on a PR-head checkout, taking the gather_fast path that counts on the live tree against base counts at the merge-base. A local pass could therefore disagree with CI. Drop --ci-parity from the Makefile and remove the now-unused gather_ci_parity branch and flag so there is one code path that both local and CI exercise. The docstring claim that CI runs against the synthetic merge ref was also wrong; the workflow checks out github.event.pull_request.head.sha. --- Makefile | 6 ++--- scripts/ruff_strict_gate.py | 47 +++++-------------------------------- 2 files changed, 9 insertions(+), 44 deletions(-) diff --git a/Makefile b/Makefile index ff1717c1ab57..27150aec9389 100644 --- a/Makefile +++ b/Makefile @@ -135,11 +135,11 @@ lint-black: format-check lint-ruff-budget: install-dev $(UV_RUN) python scripts/ruff_strict_gate.py -# CI-parity strict gate: fetch the target branch fresh and count against a -# throwaway merge into HEAD, so a local pass means the CI check will pass too. +# Strict gate, invoked the same way CI does in test-linting.yml so a local pass +# means the CI check will pass too. lint-gate: install-dev git fetch origin litellm_internal_staging - $(UV_RUN) python scripts/ruff_strict_gate.py --ci-parity --base origin/litellm_internal_staging + $(UV_RUN) python scripts/ruff_strict_gate.py --base origin/litellm_internal_staging lint-ruff-budget-update: install-dev $(UV_RUN) python scripts/ruff_strict_gate.py --update diff --git a/scripts/ruff_strict_gate.py b/scripts/ruff_strict_gate.py index c5973a2da408..9c406b8482bb 100644 --- a/scripts/ruff_strict_gate.py +++ b/scripts/ruff_strict_gate.py @@ -6,11 +6,8 @@ its ceiling and higher than the base it merges into, so a change is blamed for the violations it adds, never for drift that already exists in the base. -By default the base is the merge-base of the current branch, which is fast but -diverges from CI: CI runs against the synthetic merge ref, so its base is the -*current* tip of the target branch plus whatever drift landed since you forked. -Pass --ci-parity to reproduce CI exactly by counting against a throwaway merge -of the base into HEAD. +The base is the merge-base of the current branch with --base; this matches CI, +which checks out the PR head sha and runs the gate against the PR's base sha. """ import argparse @@ -140,7 +137,7 @@ def introduced(violations: list, changed: dict) -> list: return [v for v in violations if v.line in changed.get(v.file, set())] -def gather_fast(base: str) -> GateInputs: +def gather(base: str) -> GateInputs: base_point = _run(["git", "merge-base", base, "HEAD"]).strip() or base diff = _run(["git", "diff", base_point, "--unified=0", "--no-color", "--", TARGET]) return GateInputs( @@ -150,33 +147,6 @@ def gather_fast(base: str) -> GateInputs: ) -def gather_ci_parity(base: str) -> GateInputs: - with _temp_worktree("HEAD") as worktree: - merge = subprocess.run( - ["git", "merge", "--no-commit", "--no-ff", base], - cwd=worktree, - capture_output=True, - text=True, - ) - if merge.returncode != 0: - subprocess.run( - ["git", "merge", "--abort"], - cwd=worktree, - capture_output=True, - text=True, - ) - raise SystemExit( - f"ci-parity: merging {base} into HEAD conflicts; rebase your branch first" - ) - shutil.copy(STRICT_CONFIG, worktree / "ruff-strict.toml") - head = collect_violations(worktree, worktree / "ruff-strict.toml") - diff = _run( - ["git", "diff", base, "--unified=0", "--no-color", "--", TARGET], - cwd=worktree, - ) - return GateInputs(head, base_counts(base), parse_changed_lines(diff)) - - def report(breaches: list, new: list, base: str) -> None: print(f"FAIL: strict-rule totals exceed their ceiling (base {base}):") for breach in breaches: @@ -192,9 +162,9 @@ def report(breaches: list, new: list, base: str) -> None: print(f"BREACHED RULES: {summary}") -def cmd_check(base: str, ci_parity: bool) -> None: +def cmd_check(base: str) -> None: budget = json.loads(BUDGET_PATH.read_text()) - inputs = gather_ci_parity(base) if ci_parity else gather_fast(base) + inputs = gather(base) breaches = evaluate(count_by_rule(inputs.head), inputs.base, budget) if not breaches: print(f"OK: every strict rule is within its codebase ceiling (base {base})") @@ -216,13 +186,8 @@ def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base", default=DEFAULT_BASE) parser.add_argument("--update", action="store_true") - parser.add_argument( - "--ci-parity", - action="store_true", - help="count against a throwaway merge of --base into HEAD, matching CI", - ) args = parser.parse_args() - cmd_update() if args.update else cmd_check(args.base, args.ci_parity) + cmd_update() if args.update else cmd_check(args.base) if __name__ == "__main__":