maint-78: derive pilot candidates from the registry + auto-trigger (#2819 move 1)

.github/workflows/maint-78-model-evaluation-pilot.yml

@@ -2,6 +2,16 @@ name: Maint 78 Model Evaluation Pilot
 
 on:
   workflow_dispatch: {}
+  # Auto-run when the catalog changes so a newly-current model is piloted without
+  # anyone editing the candidate list (stranske/Workflows#2819, move 1).
+  push:
+    branches: [main]
+    paths:
+      - config/model_registry.json
+      - config/model_eval_candidates.json
+  # Weekly safety net in case a catalog change landed via a path the push filter missed.
+  schedule:
+    - cron: "17 6 * * 1"
 
 permissions:
   contents: read
@@ -17,6 +27,10 @@ jobs:
         with:
           persist-credentials: false
       - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+      - name: Refresh candidates from the registry
+        # Derive the candidate set from the current catalog so the pilot always
+        # tests every now-current model, even if the committed file lagged.
+        run: python -m tools.refresh_model_eval_candidates --write
       - name: Run paired 30-case pilot
         env:
           # Cross-repo read token; github.token is scoped to Workflows only.

config/model_eval_candidates.json

@@ -1,11 +1,49 @@
 {
   "candidates": [
-    {"provider":"openai","model_id":"gpt-5.4","role":"incumbent"},
-    {"provider":"openai","model_id":"gpt-5.6-terra","role":"candidate"},
-    {"provider":"openai","model_id":"gpt-5.6-sol","role":"candidate"},
-    {"provider":"anthropic","model_id":"claude-opus-4-6","role":"incumbent"},
-    {"provider":"anthropic","model_id":"claude-opus-4-8","role":"candidate"},
-    {"provider":"anthropic","model_id":"claude-sonnet-5","role":"candidate"},
-    {"provider":"github-models","model_id":"codex-mini-latest","role":"incumbent"}
+    {
+      "provider": "anthropic",
+      "model_id": "claude-opus-4-6",
+      "role": "incumbent"
+    },
+    {
+      "provider": "anthropic",
+      "model_id": "claude-fable-5",
+      "role": "candidate"
+    },
+    {
+      "provider": "anthropic",
+      "model_id": "claude-opus-4-8",
+      "role": "candidate"
+    },
+    {
+      "provider": "anthropic",
+      "model_id": "claude-sonnet-5",
+      "role": "candidate"
+    },
+    {
+      "provider": "github-models",
+      "model_id": "codex-mini-latest",
+      "role": "incumbent"
+    },
+    {
+      "provider": "github-models",
+      "model_id": "openai/gpt-5",
+      "role": "candidate"
+    },
+    {
+      "provider": "openai",
+      "model_id": "gpt-5.4",
+      "role": "incumbent"
+    },
+    {
+      "provider": "openai",
+      "model_id": "gpt-5.6-sol",
+      "role": "candidate"
+    },
+    {
+      "provider": "openai",
+      "model_id": "gpt-5.6-terra",
+      "role": "candidate"
+    }
   ]
 }

tests/tools/test_refresh_model_eval_candidates.py

@@ -0,0 +1,135 @@
+"""Tests for tools/refresh_model_eval_candidates.py (registry-derived pilot candidates)."""
+
+from __future__ import annotations
+
+import json
+
+from tools import refresh_model_eval_candidates as rc
+
+
+def _registry():
+    return {
+        "selections": [
+            {"provider": "openai", "profile": "verifier-balanced", "model_id": "gpt-5.4"},
+            {
+                "provider": "anthropic",
+                "profile": "verifier-balanced",
+                "model_id": "claude-opus-4-6",
+            },
+            {"provider": "openai", "profile": "some-other-profile", "model_id": "gpt-x"},
+        ],
+        "models": [
+            {
+                "provider": "openai",
+                "model_id": "gpt-5.4",
+                "lifecycle": "current",
+                "positioning": "incumbent-verifier",
+            },
+            {
+                "provider": "openai",
+                "model_id": "gpt-5.6-terra",
+                "lifecycle": "current",
+                "positioning": "balanced",
+            },
+            {
+                "provider": "openai",
+                "model_id": "gpt-5.6-luna",
+                "lifecycle": "current",
+                "positioning": "efficient",
+            },  # excluded
+            {
+                "provider": "openai",
+                "model_id": "gpt-5.5",
+                "lifecycle": "compatibility",
+                "positioning": "frontier",
+            },  # excluded (not current)
+            {
+                "provider": "openai",
+                "model_id": "gpt-blocked",
+                "lifecycle": "current",
+                "positioning": "frontier",
+                "blocked": True,
+            },  # excluded (blocked)
+            {
+                "provider": "anthropic",
+                "model_id": "claude-opus-4-6",
+                "lifecycle": "current",
+                "positioning": "incumbent-verifier",
+            },
+            {
+                "provider": "anthropic",
+                "model_id": "claude-opus-4-8",
+                "lifecycle": "current",
+                "positioning": "high-capability",
+            },
+        ],
+    }
+
+
+def test_derive_picks_incumbent_and_verifier_candidates():
+    out = rc.derive_candidates(_registry())["candidates"]
+    keys = [(c["provider"], c["model_id"], c["role"]) for c in out]
+    assert ("openai", "gpt-5.4", "incumbent") in keys
+    assert ("openai", "gpt-5.6-terra", "candidate") in keys
+    assert ("anthropic", "claude-opus-4-6", "incumbent") in keys
+    assert ("anthropic", "claude-opus-4-8", "candidate") in keys
+
+
+def test_derive_excludes_efficient_noncurrent_and_blocked():
+    models = {c["model_id"] for c in rc.derive_candidates(_registry())["candidates"]}
+    assert "gpt-5.6-luna" not in models  # efficient
+    assert "gpt-5.5" not in models  # not current
+    assert "gpt-blocked" not in models  # blocked
+
+
+def test_derive_only_uses_the_target_profile():
+    # the some-other-profile openai selection must not become an incumbent
+    incumbents = {
+        c["model_id"]
+        for c in rc.derive_candidates(_registry())["candidates"]
+        if c["role"] == "incumbent"
+    }
+    assert "gpt-x" not in incumbents
+    assert incumbents == {"gpt-5.4", "claude-opus-4-6"}
+
+
+def test_derive_is_deterministic_and_sorted():
+    a = rc.derive_candidates(_registry())
+    b = rc.derive_candidates(_registry())
+    assert a == b
+    # candidates within a provider are sorted by model_id
+    anth = [c["model_id"] for c in a["candidates"] if c["provider"] == "anthropic"]
+    assert anth == sorted(
+        anth, key=lambda m: (m != "claude-opus-4-6", m)
+    )  # incumbent first, then sorted
+
+
+def test_check_detects_drift(tmp_path, capsys):
+    reg = tmp_path / "reg.json"
+    cand = tmp_path / "cand.json"
+    reg.write_text(json.dumps(_registry()))
+    cand.write_text(
+        json.dumps(
+            {"candidates": [{"provider": "openai", "model_id": "stale", "role": "incumbent"}]}
+        )
+    )
+    rc_code = rc.main(["--registry", str(reg), "--candidates", str(cand), "--check"])
+    assert rc_code == 1  # drifted
+
+
+def test_write_then_check_roundtrips(tmp_path):
+    reg = tmp_path / "reg.json"
+    cand = tmp_path / "cand.json"
+    reg.write_text(json.dumps(_registry()))
+    assert rc.main(["--registry", str(reg), "--candidates", str(cand), "--write"]) == 0
+    assert rc.main(["--registry", str(reg), "--candidates", str(cand), "--check"]) == 0
+
+
+def test_committed_candidates_match_registry_derivation():
+    """Drift gate: the shipped config/model_eval_candidates.json must equal the derivation."""
+    registry = json.loads(rc.DEFAULT_REGISTRY_PATH.read_text(encoding="utf-8"))
+    committed = json.loads(rc.DEFAULT_CANDIDATES_PATH.read_text(encoding="utf-8"))
+    assert committed == rc.derive_candidates(registry), (
+        "config/model_eval_candidates.json is out of sync with config/model_registry.json; "
+        "run `python -m tools.refresh_model_eval_candidates --write`"
+    )

tools/refresh_model_eval_candidates.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""Derive the verifier-pilot candidate set from the model registry.
+
+Part of the self-feeding verifier-model promotion system (stranske/Workflows#2819),
+move 1: candidates should never be hand-maintained (they drifted — a defunct
+``claude-sonnet-4-6`` was listed while the current ``claude-opus-4-8`` was omitted).
+Instead derive them from ``config/model_registry.json`` so a catalog change
+automatically produces the right pilot candidates.
+
+For each provider selected for the target profile:
+  - incumbent  = that profile's reviewed selection for the provider
+  - candidates = every OTHER current, non-blocked, same-provider catalogued model
+                 whose positioning is not clearly non-verifier (``efficient``,
+                 ``coding-worker-profile``).
+
+``--write`` regenerates ``config/model_eval_candidates.json``; ``--check`` exits 1
+if the committed file differs from the derived set (a drift gate).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+_REPO_ROOT = Path(__file__).resolve().parent.parent
+DEFAULT_REGISTRY_PATH = _REPO_ROOT / "config" / "model_registry.json"
+DEFAULT_CANDIDATES_PATH = _REPO_ROOT / "config" / "model_eval_candidates.json"
+DEFAULT_PROFILE = "verifier-balanced"
+
+# Positionings that are not verifier candidates (cost/speed tiers, worker profiles).
+EXCLUDED_POSITIONINGS = frozenset({"efficient", "coding-worker-profile"})
+
+
+def derive_candidates(
+    registry: dict[str, Any], *, profile: str = DEFAULT_PROFILE
+) -> dict[str, Any]:
+    """Return the candidate set derived from the registry (pure function)."""
+    incumbents: dict[str, str] = {
+        str(sel.get("provider", "")): str(sel.get("model_id", ""))
+        for sel in registry.get("selections", [])
+        if sel.get("profile") == profile and sel.get("provider") and sel.get("model_id")
+    }
+    models = registry.get("models", [])
+
+    candidates: list[dict[str, str]] = []
+    for provider in sorted(incumbents):
+        incumbent = incumbents[provider]
+        candidates.append({"provider": provider, "model_id": incumbent, "role": "incumbent"})
+        alternatives = sorted(
+            str(m.get("model_id", ""))
+            for m in models
+            if str(m.get("provider", "")) == provider
+            and str(m.get("model_id", "")) != incumbent
+            and m.get("lifecycle") == "current"
+            and not m.get("blocked", False)
+            and str(m.get("positioning", "")) not in EXCLUDED_POSITIONINGS
+        )
+        for model_id in alternatives:
+            candidates.append({"provider": provider, "model_id": model_id, "role": "candidate"})
+    return {"candidates": candidates}
+
+
+def _load(path: Path) -> dict[str, Any]:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def _serialize(candidates: dict[str, Any]) -> str:
+    return json.dumps(candidates, indent=2) + "\n"
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        description="Derive verifier-pilot candidates from the registry."
+    )
+    parser.add_argument("--registry", type=Path, default=DEFAULT_REGISTRY_PATH)
+    parser.add_argument("--candidates", type=Path, default=DEFAULT_CANDIDATES_PATH)
+    parser.add_argument("--profile", default=DEFAULT_PROFILE)
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument("--write", action="store_true", help="Regenerate the candidates file.")
+    group.add_argument(
+        "--check", action="store_true", help="Exit 1 if the committed file has drifted."
+    )
+    args = parser.parse_args(argv)
+
+    try:
+        registry = _load(args.registry)
+    except (OSError, json.JSONDecodeError) as exc:
+        print(f"cannot read registry: {exc}", file=sys.stderr)
+        return 2
+
+    derived = derive_candidates(registry, profile=args.profile)
+    if not derived["candidates"]:
+        print(f"no selections for profile {args.profile!r}; nothing to derive", file=sys.stderr)
+        return 2
+
+    if args.write:
+        args.candidates.write_text(_serialize(derived), encoding="utf-8")
+        print(f"wrote {len(derived['candidates'])} candidate rows to {args.candidates}")
+        return 0
+
+    # --check
+    try:
+        committed = _load(args.candidates)
+    except (OSError, json.JSONDecodeError) as exc:
+        print(f"cannot read candidates file: {exc}", file=sys.stderr)
+        return 1
+    if committed == derived:
+        print("candidates are in sync with the registry.")
+        return 0
+    print(
+        "candidates have DRIFTED from the registry. Run "
+        "`python -m tools.refresh_model_eval_candidates --write` and commit.",
+        file=sys.stderr,
+    )
+    return 1
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())