feat(model-eval): per-provider preflight tolerance (one bad candidate no longer aborts the pilot) — part of #2819

tests/tools/test_run_model_eval_pilot.py

@@ -315,3 +315,73 @@ def test_main_reports_malformed_corpus_without_traceback(monkeypatch, tmp_path,
     assert (
         "pilot preflight error: pilot corpus requires at least one case" in capsys.readouterr().err
     )
+
+
+# --- per-provider preflight tolerance (partition_usable_candidates) --------------
+
+
+def _cands(*triples):
+    return {"candidates": [{"provider": p, "model_id": m, "role": r} for p, m, r in triples]}
+
+
+def test_partition_drops_provider_whose_incumbent_is_unusable():
+    # github incumbent 404s -> drop github entirely, keep openai/anthropic.
+    cands = _cands(
+        ("openai", "gpt-5.4", "incumbent"),
+        ("openai", "gpt-5.6-terra", "candidate"),
+        ("anthropic", "claude-opus-4-6", "incumbent"),
+        ("anthropic", "claude-opus-4-8", "candidate"),
+        ("github-models", "codex-mini-latest", "incumbent"),
+    )
+    remaining, dropped, fatal = pilot.partition_usable_candidates(
+        cands, ["github-models/codex-mini-latest"]
+    )
+    assert fatal is None
+    keys = {pilot._cand_key(e) for e in remaining["candidates"]}
+    assert keys == {
+        "openai/gpt-5.4",
+        "openai/gpt-5.6-terra",
+        "anthropic/claude-opus-4-6",
+        "anthropic/claude-opus-4-8",
+    }
+    assert any("github-models" in d for d in dropped)
+
+
+def test_partition_drops_provider_with_no_usable_candidate():
+    # anthropic incumbent ok but its only candidate is unusable -> drop anthropic; openai remains.
+    cands = _cands(
+        ("openai", "gpt-5.4", "incumbent"),
+        ("openai", "gpt-5.6-terra", "candidate"),
+        ("anthropic", "claude-opus-4-6", "incumbent"),
+        ("anthropic", "claude-opus-4-8", "candidate"),
+    )
+    remaining, dropped, fatal = pilot.partition_usable_candidates(
+        cands, ["anthropic/claude-opus-4-8"]
+    )
+    assert fatal is None
+    keys = {pilot._cand_key(e) for e in remaining["candidates"]}
+    assert keys == {"openai/gpt-5.4", "openai/gpt-5.6-terra"}
+    assert any("anthropic: no usable candidate" in d for d in dropped)
+
+
+def test_partition_fatal_when_no_provider_pair_survives():
+    cands = _cands(
+        ("openai", "gpt-5.4", "incumbent"),
+        ("openai", "gpt-5.6-terra", "candidate"),
+    )
+    remaining, dropped, fatal = pilot.partition_usable_candidates(
+        cands, ["openai/gpt-5.6-terra"]  # incumbent ok, only candidate dead -> nothing to compare
+    )
+    assert fatal is not None
+    assert remaining["candidates"] == []
+
+
+def test_partition_noop_when_all_usable():
+    cands = _cands(
+        ("openai", "gpt-5.4", "incumbent"),
+        ("openai", "gpt-5.6-terra", "candidate"),
+    )
+    remaining, dropped, fatal = pilot.partition_usable_candidates(cands, [])
+    assert fatal is None
+    assert dropped == []
+    assert len(remaining["candidates"]) == 2

tools/run_model_eval_pilot.py

@@ -254,6 +254,50 @@ def unusable_candidates(report: dict[str, Any]) -> list[str]:
     ]
 
 
+def _cand_key(entry: dict[str, Any]) -> str:
+    return f"{entry.get('provider', '')}/{entry.get('model_id', '')}"
+
+
+def partition_usable_candidates(
+    candidates: dict[str, Any], unusable: list[str]
+) -> tuple[dict[str, Any], list[str], str | None]:
+    """Drop unusable candidates per provider so one bad model/provider can't abort the pilot.
+
+    A provider is retained only if its incumbent AND at least one candidate are usable
+    (you cannot benchmark a provider without a working baseline or without something to
+    compare). Returns ``(remaining_candidates, dropped_notes, fatal_reason_or_None)``;
+    ``fatal_reason`` is set only when no provider has a usable incumbent+candidate pair.
+    """
+    unusable_set = set(unusable)
+    by_provider: dict[str, list[dict[str, Any]]] = {}
+    for entry in candidates.get("candidates", []):
+        by_provider.setdefault(str(entry.get("provider", "")), []).append(entry)
+
+    kept: list[dict[str, Any]] = []
+    dropped: list[str] = []
+    for provider, group in by_provider.items():
+        incumbents = [e for e in group if e.get("role") == "incumbent"]
+        cands = [e for e in group if e.get("role") != "incumbent"]
+        usable_incumbents = [e for e in incumbents if _cand_key(e) not in unusable_set]
+        usable_cands = [e for e in cands if _cand_key(e) not in unusable_set]
+        if not usable_incumbents:
+            dropped.append(
+                f"{provider}: incumbent unusable ({', '.join(_cand_key(e) for e in incumbents) or 'none'})"
+            )
+            continue
+        if not usable_cands:
+            dropped.append(f"{provider}: no usable candidate")
+            continue
+        newly = [_cand_key(e) for e in group if _cand_key(e) in unusable_set]
+        if newly:
+            dropped.append(f"{provider}: dropped {', '.join(newly)}")
+        kept.extend(usable_incumbents + usable_cands)
+
+    remaining = {**candidates, "candidates": kept}
+    fatal = None if kept else "no provider has a usable incumbent + candidate pair"
+    return remaining, dropped, fatal
+
+
 def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("--corpus", type=Path, default=ROOT / "config/model_eval_pilot.json")
@@ -281,21 +325,24 @@ def main() -> int:
     preflight_output.write_text(json.dumps(preflight, indent=2) + "\n")
     unusable = unusable_candidates(preflight)
     if unusable:
-        print(
-            "pilot preflight error: candidates produced no schema-valid rows: "
-            + ", ".join(unusable),
-            file=sys.stderr,
-        )
-        return 1
+        candidates, dropped, fatal = partition_usable_candidates(candidates, unusable)
+        for note in dropped:
+            print(f"pilot preflight: skipping {note}", file=sys.stderr)
+        if fatal:
+            print(
+                f"pilot preflight error: {fatal} (unusable: {', '.join(unusable)})", file=sys.stderr
+            )
+            return 1
     payload = run_pilot(corpus, candidates, token=token)
     args.output.write_text(json.dumps(payload, indent=2) + "\n")
     unusable = unusable_candidates(payload)
     if unusable:
-        print(
-            "pilot error: candidates produced no schema-valid rows: " + ", ".join(unusable),
-            file=sys.stderr,
-        )
-        return 1
+        _, dropped, fatal = partition_usable_candidates(candidates, unusable)
+        for note in dropped:
+            print(f"pilot: {note} (post-run)", file=sys.stderr)
+        if fatal:
+            print(f"pilot error: {fatal} (unusable: {', '.join(unusable)})", file=sys.stderr)
+            return 1
     return 0