fix(llm): install provider clients for model pilot

.github/workflows/maint-78-model-evaluation-pilot.yml

@@ -21,10 +21,12 @@ jobs:
         env:
           # Cross-repo read token; github.token is scoped to Workflows only.
           GH_TOKEN: ${{ secrets.OWNER_PR_PAT }}
+          GITHUB_TOKEN: ${{ github.token }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           CLAUDE_API_STRANSKE: ${{ secrets.CLAUDE_API_STRANSKE }}
         run: |
-          uv run --extra dev python -m tools.run_model_eval_pilot --output pilot-results.json
+          uv run --extra dev --extra langchain \
+            python -m tools.run_model_eval_pilot --output pilot-results.json
       - name: Summarize pilot
         if: always()
         run: |

tests/tools/test_run_model_eval_pilot.py

@@ -1,6 +1,6 @@
 from types import SimpleNamespace
 
-from tools.run_model_eval_pilot import run_pilot
+from tools.run_model_eval_pilot import run_pilot, unusable_candidates
 
 
 def test_run_pilot_records_paired_verdicts() -> None:
@@ -77,3 +77,16 @@ def test_run_pilot_captures_failure_with_case_and_candidate_metadata() -> None:
             "error": "fetch failed",
         }
     ]
+
+
+def test_unusable_candidates_requires_valid_evidence_per_candidate() -> None:
+    report = {
+        "results": [
+            {"provider": "openai", "model_id": "working", "schema_valid": True},
+            {"provider": "openai", "model_id": "working", "schema_valid": False},
+            {"provider": "anthropic", "model_id": "broken", "schema_valid": False},
+        ]
+    }
+
+    assert unusable_candidates(report) == ["anthropic/broken"]
+    assert unusable_candidates({"results": []}) == ["<no-results>"]

tests/workflows/test_model_eval_pilot_workflow.py

@@ -14,6 +14,9 @@ def test_model_eval_pilot_runs_as_importable_module() -> None:
     upload = next(step for step in steps if "actions/upload-artifact@" in step.get("uses", ""))
 
     assert pilot["run"].count("python -m tools.run_model_eval_pilot") == 1
+    assert pilot["run"].count("--extra langchain") == 1
+    assert pilot["env"]["GH_TOKEN"] == "${{ secrets.OWNER_PR_PAT }}"
+    assert pilot["env"]["GITHUB_TOKEN"] == "${{ github.token }}"
     assert "python tools/run_model_eval_pilot.py" not in pilot["run"]
     assert summary["if"] == "always()"
     assert "if [ ! -f pilot-results.json ]" in summary["run"]

tools/run_model_eval_pilot.py

@@ -6,6 +6,7 @@
 import argparse
 import json
 import os
+import sys
 import time
 from collections.abc import Callable
 from pathlib import Path
@@ -75,6 +76,19 @@ def run_pilot(
     }
 
 
+def unusable_candidates(report: dict[str, Any]) -> list[str]:
+    """Return candidates that produced no schema-valid evaluation row."""
+    if not report.get("results"):
+        return ["<no-results>"]
+    health: dict[tuple[str, str], bool] = {}
+    for row in report.get("results", []):
+        if not isinstance(row, dict):
+            continue
+        key = (str(row.get("provider", "")), str(row.get("model_id", "")))
+        health[key] = health.get(key, False) or row.get("schema_valid") is True
+    return [f"{provider}/{model}" for (provider, model), usable in health.items() if not usable]
+
+
 def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("--corpus", type=Path, default=ROOT / "config/model_eval_pilot.json")
@@ -92,6 +106,13 @@ def main() -> int:
         token=token,
     )
     args.output.write_text(json.dumps(payload, indent=2) + "\n")
+    unusable = unusable_candidates(payload)
+    if unusable:
+        print(
+            "pilot error: candidates produced no schema-valid rows: " + ", ".join(unusable),
+            file=sys.stderr,
+        )
+        return 1
     return 0