diff --git a/tests/tools/test_run_model_eval_pilot.py b/tests/tools/test_run_model_eval_pilot.py index e685dd25a..1e7d6b552 100644 --- a/tests/tools/test_run_model_eval_pilot.py +++ b/tests/tools/test_run_model_eval_pilot.py @@ -315,3 +315,73 @@ def test_main_reports_malformed_corpus_without_traceback(monkeypatch, tmp_path, assert ( "pilot preflight error: pilot corpus requires at least one case" in capsys.readouterr().err ) + + +# --- per-provider preflight tolerance (partition_usable_candidates) -------------- + + +def _cands(*triples): + return {"candidates": [{"provider": p, "model_id": m, "role": r} for p, m, r in triples]} + + +def test_partition_drops_provider_whose_incumbent_is_unusable(): + # github incumbent 404s -> drop github entirely, keep openai/anthropic. + cands = _cands( + ("openai", "gpt-5.4", "incumbent"), + ("openai", "gpt-5.6-terra", "candidate"), + ("anthropic", "claude-opus-4-6", "incumbent"), + ("anthropic", "claude-opus-4-8", "candidate"), + ("github-models", "codex-mini-latest", "incumbent"), + ) + remaining, dropped, fatal = pilot.partition_usable_candidates( + cands, ["github-models/codex-mini-latest"] + ) + assert fatal is None + keys = {pilot._cand_key(e) for e in remaining["candidates"]} + assert keys == { + "openai/gpt-5.4", + "openai/gpt-5.6-terra", + "anthropic/claude-opus-4-6", + "anthropic/claude-opus-4-8", + } + assert any("github-models" in d for d in dropped) + + +def test_partition_drops_provider_with_no_usable_candidate(): + # anthropic incumbent ok but its only candidate is unusable -> drop anthropic; openai remains. + cands = _cands( + ("openai", "gpt-5.4", "incumbent"), + ("openai", "gpt-5.6-terra", "candidate"), + ("anthropic", "claude-opus-4-6", "incumbent"), + ("anthropic", "claude-opus-4-8", "candidate"), + ) + remaining, dropped, fatal = pilot.partition_usable_candidates( + cands, ["anthropic/claude-opus-4-8"] + ) + assert fatal is None + keys = {pilot._cand_key(e) for e in remaining["candidates"]} + assert keys == {"openai/gpt-5.4", "openai/gpt-5.6-terra"} + assert any("anthropic: no usable candidate" in d for d in dropped) + + +def test_partition_fatal_when_no_provider_pair_survives(): + cands = _cands( + ("openai", "gpt-5.4", "incumbent"), + ("openai", "gpt-5.6-terra", "candidate"), + ) + remaining, dropped, fatal = pilot.partition_usable_candidates( + cands, ["openai/gpt-5.6-terra"] # incumbent ok, only candidate dead -> nothing to compare + ) + assert fatal is not None + assert remaining["candidates"] == [] + + +def test_partition_noop_when_all_usable(): + cands = _cands( + ("openai", "gpt-5.4", "incumbent"), + ("openai", "gpt-5.6-terra", "candidate"), + ) + remaining, dropped, fatal = pilot.partition_usable_candidates(cands, []) + assert fatal is None + assert dropped == [] + assert len(remaining["candidates"]) == 2 diff --git a/tools/run_model_eval_pilot.py b/tools/run_model_eval_pilot.py index d68eb2a69..6875625ba 100644 --- a/tools/run_model_eval_pilot.py +++ b/tools/run_model_eval_pilot.py @@ -254,6 +254,50 @@ def unusable_candidates(report: dict[str, Any]) -> list[str]: ] +def _cand_key(entry: dict[str, Any]) -> str: + return f"{entry.get('provider', '')}/{entry.get('model_id', '')}" + + +def partition_usable_candidates( + candidates: dict[str, Any], unusable: list[str] +) -> tuple[dict[str, Any], list[str], str | None]: + """Drop unusable candidates per provider so one bad model/provider can't abort the pilot. + + A provider is retained only if its incumbent AND at least one candidate are usable + (you cannot benchmark a provider without a working baseline or without something to + compare). Returns ``(remaining_candidates, dropped_notes, fatal_reason_or_None)``; + ``fatal_reason`` is set only when no provider has a usable incumbent+candidate pair. + """ + unusable_set = set(unusable) + by_provider: dict[str, list[dict[str, Any]]] = {} + for entry in candidates.get("candidates", []): + by_provider.setdefault(str(entry.get("provider", "")), []).append(entry) + + kept: list[dict[str, Any]] = [] + dropped: list[str] = [] + for provider, group in by_provider.items(): + incumbents = [e for e in group if e.get("role") == "incumbent"] + cands = [e for e in group if e.get("role") != "incumbent"] + usable_incumbents = [e for e in incumbents if _cand_key(e) not in unusable_set] + usable_cands = [e for e in cands if _cand_key(e) not in unusable_set] + if not usable_incumbents: + dropped.append( + f"{provider}: incumbent unusable ({', '.join(_cand_key(e) for e in incumbents) or 'none'})" + ) + continue + if not usable_cands: + dropped.append(f"{provider}: no usable candidate") + continue + newly = [_cand_key(e) for e in group if _cand_key(e) in unusable_set] + if newly: + dropped.append(f"{provider}: dropped {', '.join(newly)}") + kept.extend(usable_incumbents + usable_cands) + + remaining = {**candidates, "candidates": kept} + fatal = None if kept else "no provider has a usable incumbent + candidate pair" + return remaining, dropped, fatal + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--corpus", type=Path, default=ROOT / "config/model_eval_pilot.json") @@ -281,21 +325,24 @@ def main() -> int: preflight_output.write_text(json.dumps(preflight, indent=2) + "\n") unusable = unusable_candidates(preflight) if unusable: - print( - "pilot preflight error: candidates produced no schema-valid rows: " - + ", ".join(unusable), - file=sys.stderr, - ) - return 1 + candidates, dropped, fatal = partition_usable_candidates(candidates, unusable) + for note in dropped: + print(f"pilot preflight: skipping {note}", file=sys.stderr) + if fatal: + print( + f"pilot preflight error: {fatal} (unusable: {', '.join(unusable)})", file=sys.stderr + ) + return 1 payload = run_pilot(corpus, candidates, token=token) args.output.write_text(json.dumps(payload, indent=2) + "\n") unusable = unusable_candidates(payload) if unusable: - print( - "pilot error: candidates produced no schema-valid rows: " + ", ".join(unusable), - file=sys.stderr, - ) - return 1 + _, dropped, fatal = partition_usable_candidates(candidates, unusable) + for note in dropped: + print(f"pilot: {note} (post-run)", file=sys.stderr) + if fatal: + print(f"pilot error: {fatal} (unusable: {', '.join(unusable)})", file=sys.stderr) + return 1 return 0