From cbb2113703e03d8ea2a698d328c17adcb85c0365 Mon Sep 17 00:00:00 2001 From: Tim Stranske Date: Sat, 25 Jul 2026 17:42:09 -0500 Subject: [PATCH] maint-78: derive pilot candidates from the registry + auto-trigger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move 1 of the self-feeding verifier-model promotion system (#2819). The pilot candidate list was hand-maintained and had already drifted: it listed models that were no longer current while omitting now-current ones (e.g. claude-opus-4-8). A stale candidate list means a superseded model stays the incumbent forever because nothing ever pilots its replacement. - tools/refresh_model_eval_candidates.py: derive_candidates() builds the candidate set straight from config/model_registry.json — incumbent = the profile's reviewed selection, candidates = every other current, non-blocked, same-provider model whose positioning is a plausible verifier tier (excludes efficient / coding-worker-profile). --write regenerates the config; --check is a drift gate. - tests: derivation logic + a gate test asserting the committed config/model_eval_candidates.json equals the derivation, so a catalog change that forgets to refresh candidates reddens CI. - config/model_eval_candidates.json: regenerated from the registry (adds claude-fable-5, claude-opus-4-8, and a github-models candidate). - maint-78: auto-run on catalog change (push to main touching the registry) + weekly cron, and refresh candidates from the registry before each run so the pilot always tests every now-current model. Promotion is unchanged — the pilot only narrows candidates; approval still requires the 75-case corpus + policy gate. Co-Authored-By: Claude Opus 4.8 --- .../maint-78-model-evaluation-pilot.yml | 14 ++ config/model_eval_candidates.json | 52 ++++++- .../test_refresh_model_eval_candidates.py | 135 ++++++++++++++++++ tools/refresh_model_eval_candidates.py | 122 ++++++++++++++++ 4 files changed, 316 insertions(+), 7 deletions(-) create mode 100644 tests/tools/test_refresh_model_eval_candidates.py create mode 100644 tools/refresh_model_eval_candidates.py diff --git a/.github/workflows/maint-78-model-evaluation-pilot.yml b/.github/workflows/maint-78-model-evaluation-pilot.yml index 2cce4d7a3..9d31a92b2 100644 --- a/.github/workflows/maint-78-model-evaluation-pilot.yml +++ b/.github/workflows/maint-78-model-evaluation-pilot.yml @@ -2,6 +2,16 @@ name: Maint 78 Model Evaluation Pilot on: workflow_dispatch: {} + # Auto-run when the catalog changes so a newly-current model is piloted without + # anyone editing the candidate list (stranske/Workflows#2819, move 1). + push: + branches: [main] + paths: + - config/model_registry.json + - config/model_eval_candidates.json + # Weekly safety net in case a catalog change landed via a path the push filter missed. + schedule: + - cron: "17 6 * * 1" permissions: contents: read @@ -17,6 +27,10 @@ jobs: with: persist-credentials: false - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + - name: Refresh candidates from the registry + # Derive the candidate set from the current catalog so the pilot always + # tests every now-current model, even if the committed file lagged. + run: python -m tools.refresh_model_eval_candidates --write - name: Run paired 30-case pilot env: # Cross-repo read token; github.token is scoped to Workflows only. diff --git a/config/model_eval_candidates.json b/config/model_eval_candidates.json index 583db08d5..da058d269 100644 --- a/config/model_eval_candidates.json +++ b/config/model_eval_candidates.json @@ -1,11 +1,49 @@ { "candidates": [ - {"provider":"openai","model_id":"gpt-5.4","role":"incumbent"}, - {"provider":"openai","model_id":"gpt-5.6-terra","role":"candidate"}, - {"provider":"openai","model_id":"gpt-5.6-sol","role":"candidate"}, - {"provider":"anthropic","model_id":"claude-opus-4-6","role":"incumbent"}, - {"provider":"anthropic","model_id":"claude-opus-4-8","role":"candidate"}, - {"provider":"anthropic","model_id":"claude-sonnet-5","role":"candidate"}, - {"provider":"github-models","model_id":"codex-mini-latest","role":"incumbent"} + { + "provider": "anthropic", + "model_id": "claude-opus-4-6", + "role": "incumbent" + }, + { + "provider": "anthropic", + "model_id": "claude-fable-5", + "role": "candidate" + }, + { + "provider": "anthropic", + "model_id": "claude-opus-4-8", + "role": "candidate" + }, + { + "provider": "anthropic", + "model_id": "claude-sonnet-5", + "role": "candidate" + }, + { + "provider": "github-models", + "model_id": "codex-mini-latest", + "role": "incumbent" + }, + { + "provider": "github-models", + "model_id": "openai/gpt-5", + "role": "candidate" + }, + { + "provider": "openai", + "model_id": "gpt-5.4", + "role": "incumbent" + }, + { + "provider": "openai", + "model_id": "gpt-5.6-sol", + "role": "candidate" + }, + { + "provider": "openai", + "model_id": "gpt-5.6-terra", + "role": "candidate" + } ] } diff --git a/tests/tools/test_refresh_model_eval_candidates.py b/tests/tools/test_refresh_model_eval_candidates.py new file mode 100644 index 000000000..7e4374703 --- /dev/null +++ b/tests/tools/test_refresh_model_eval_candidates.py @@ -0,0 +1,135 @@ +"""Tests for tools/refresh_model_eval_candidates.py (registry-derived pilot candidates).""" + +from __future__ import annotations + +import json + +from tools import refresh_model_eval_candidates as rc + + +def _registry(): + return { + "selections": [ + {"provider": "openai", "profile": "verifier-balanced", "model_id": "gpt-5.4"}, + { + "provider": "anthropic", + "profile": "verifier-balanced", + "model_id": "claude-opus-4-6", + }, + {"provider": "openai", "profile": "some-other-profile", "model_id": "gpt-x"}, + ], + "models": [ + { + "provider": "openai", + "model_id": "gpt-5.4", + "lifecycle": "current", + "positioning": "incumbent-verifier", + }, + { + "provider": "openai", + "model_id": "gpt-5.6-terra", + "lifecycle": "current", + "positioning": "balanced", + }, + { + "provider": "openai", + "model_id": "gpt-5.6-luna", + "lifecycle": "current", + "positioning": "efficient", + }, # excluded + { + "provider": "openai", + "model_id": "gpt-5.5", + "lifecycle": "compatibility", + "positioning": "frontier", + }, # excluded (not current) + { + "provider": "openai", + "model_id": "gpt-blocked", + "lifecycle": "current", + "positioning": "frontier", + "blocked": True, + }, # excluded (blocked) + { + "provider": "anthropic", + "model_id": "claude-opus-4-6", + "lifecycle": "current", + "positioning": "incumbent-verifier", + }, + { + "provider": "anthropic", + "model_id": "claude-opus-4-8", + "lifecycle": "current", + "positioning": "high-capability", + }, + ], + } + + +def test_derive_picks_incumbent_and_verifier_candidates(): + out = rc.derive_candidates(_registry())["candidates"] + keys = [(c["provider"], c["model_id"], c["role"]) for c in out] + assert ("openai", "gpt-5.4", "incumbent") in keys + assert ("openai", "gpt-5.6-terra", "candidate") in keys + assert ("anthropic", "claude-opus-4-6", "incumbent") in keys + assert ("anthropic", "claude-opus-4-8", "candidate") in keys + + +def test_derive_excludes_efficient_noncurrent_and_blocked(): + models = {c["model_id"] for c in rc.derive_candidates(_registry())["candidates"]} + assert "gpt-5.6-luna" not in models # efficient + assert "gpt-5.5" not in models # not current + assert "gpt-blocked" not in models # blocked + + +def test_derive_only_uses_the_target_profile(): + # the some-other-profile openai selection must not become an incumbent + incumbents = { + c["model_id"] + for c in rc.derive_candidates(_registry())["candidates"] + if c["role"] == "incumbent" + } + assert "gpt-x" not in incumbents + assert incumbents == {"gpt-5.4", "claude-opus-4-6"} + + +def test_derive_is_deterministic_and_sorted(): + a = rc.derive_candidates(_registry()) + b = rc.derive_candidates(_registry()) + assert a == b + # candidates within a provider are sorted by model_id + anth = [c["model_id"] for c in a["candidates"] if c["provider"] == "anthropic"] + assert anth == sorted( + anth, key=lambda m: (m != "claude-opus-4-6", m) + ) # incumbent first, then sorted + + +def test_check_detects_drift(tmp_path, capsys): + reg = tmp_path / "reg.json" + cand = tmp_path / "cand.json" + reg.write_text(json.dumps(_registry())) + cand.write_text( + json.dumps( + {"candidates": [{"provider": "openai", "model_id": "stale", "role": "incumbent"}]} + ) + ) + rc_code = rc.main(["--registry", str(reg), "--candidates", str(cand), "--check"]) + assert rc_code == 1 # drifted + + +def test_write_then_check_roundtrips(tmp_path): + reg = tmp_path / "reg.json" + cand = tmp_path / "cand.json" + reg.write_text(json.dumps(_registry())) + assert rc.main(["--registry", str(reg), "--candidates", str(cand), "--write"]) == 0 + assert rc.main(["--registry", str(reg), "--candidates", str(cand), "--check"]) == 0 + + +def test_committed_candidates_match_registry_derivation(): + """Drift gate: the shipped config/model_eval_candidates.json must equal the derivation.""" + registry = json.loads(rc.DEFAULT_REGISTRY_PATH.read_text(encoding="utf-8")) + committed = json.loads(rc.DEFAULT_CANDIDATES_PATH.read_text(encoding="utf-8")) + assert committed == rc.derive_candidates(registry), ( + "config/model_eval_candidates.json is out of sync with config/model_registry.json; " + "run `python -m tools.refresh_model_eval_candidates --write`" + ) diff --git a/tools/refresh_model_eval_candidates.py b/tools/refresh_model_eval_candidates.py new file mode 100644 index 000000000..3fc2fa62d --- /dev/null +++ b/tools/refresh_model_eval_candidates.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Derive the verifier-pilot candidate set from the model registry. + +Part of the self-feeding verifier-model promotion system (stranske/Workflows#2819), +move 1: candidates should never be hand-maintained (they drifted — a defunct +``claude-sonnet-4-6`` was listed while the current ``claude-opus-4-8`` was omitted). +Instead derive them from ``config/model_registry.json`` so a catalog change +automatically produces the right pilot candidates. + +For each provider selected for the target profile: + - incumbent = that profile's reviewed selection for the provider + - candidates = every OTHER current, non-blocked, same-provider catalogued model + whose positioning is not clearly non-verifier (``efficient``, + ``coding-worker-profile``). + +``--write`` regenerates ``config/model_eval_candidates.json``; ``--check`` exits 1 +if the committed file differs from the derived set (a drift gate). +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +_REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_REGISTRY_PATH = _REPO_ROOT / "config" / "model_registry.json" +DEFAULT_CANDIDATES_PATH = _REPO_ROOT / "config" / "model_eval_candidates.json" +DEFAULT_PROFILE = "verifier-balanced" + +# Positionings that are not verifier candidates (cost/speed tiers, worker profiles). +EXCLUDED_POSITIONINGS = frozenset({"efficient", "coding-worker-profile"}) + + +def derive_candidates( + registry: dict[str, Any], *, profile: str = DEFAULT_PROFILE +) -> dict[str, Any]: + """Return the candidate set derived from the registry (pure function).""" + incumbents: dict[str, str] = { + str(sel.get("provider", "")): str(sel.get("model_id", "")) + for sel in registry.get("selections", []) + if sel.get("profile") == profile and sel.get("provider") and sel.get("model_id") + } + models = registry.get("models", []) + + candidates: list[dict[str, str]] = [] + for provider in sorted(incumbents): + incumbent = incumbents[provider] + candidates.append({"provider": provider, "model_id": incumbent, "role": "incumbent"}) + alternatives = sorted( + str(m.get("model_id", "")) + for m in models + if str(m.get("provider", "")) == provider + and str(m.get("model_id", "")) != incumbent + and m.get("lifecycle") == "current" + and not m.get("blocked", False) + and str(m.get("positioning", "")) not in EXCLUDED_POSITIONINGS + ) + for model_id in alternatives: + candidates.append({"provider": provider, "model_id": model_id, "role": "candidate"}) + return {"candidates": candidates} + + +def _load(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def _serialize(candidates: dict[str, Any]) -> str: + return json.dumps(candidates, indent=2) + "\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Derive verifier-pilot candidates from the registry." + ) + parser.add_argument("--registry", type=Path, default=DEFAULT_REGISTRY_PATH) + parser.add_argument("--candidates", type=Path, default=DEFAULT_CANDIDATES_PATH) + parser.add_argument("--profile", default=DEFAULT_PROFILE) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--write", action="store_true", help="Regenerate the candidates file.") + group.add_argument( + "--check", action="store_true", help="Exit 1 if the committed file has drifted." + ) + args = parser.parse_args(argv) + + try: + registry = _load(args.registry) + except (OSError, json.JSONDecodeError) as exc: + print(f"cannot read registry: {exc}", file=sys.stderr) + return 2 + + derived = derive_candidates(registry, profile=args.profile) + if not derived["candidates"]: + print(f"no selections for profile {args.profile!r}; nothing to derive", file=sys.stderr) + return 2 + + if args.write: + args.candidates.write_text(_serialize(derived), encoding="utf-8") + print(f"wrote {len(derived['candidates'])} candidate rows to {args.candidates}") + return 0 + + # --check + try: + committed = _load(args.candidates) + except (OSError, json.JSONDecodeError) as exc: + print(f"cannot read candidates file: {exc}", file=sys.stderr) + return 1 + if committed == derived: + print("candidates are in sync with the registry.") + return 0 + print( + "candidates have DRIFTED from the registry. Run " + "`python -m tools.refresh_model_eval_candidates --write` and commit.", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main())