diff --git a/config/model_eval_corpus_staging.json b/config/model_eval_corpus_staging.json index 9dc33fedb..19cc2a21c 100644 --- a/config/model_eval_corpus_staging.json +++ b/config/model_eval_corpus_staging.json @@ -1,5 +1,42 @@ { "schema": "verifier-corpus-staging/v1", "note": "FYI-only. Auto-expiring candidate cases pending stability or adjudication; nothing here gates anything.", - "cases": [] + "cases": [ + { + "case_id": "workflows-2541", + "repo": "stranske/Workflows", + "pr": 2541, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "trend_model_project-5667", + "repo": "stranske/Trend_Model_Project", + "pr": 5667, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "portable-alpha-extension-model-2046", + "repo": "stranske/Portable-Alpha-Extension-Model", + "pr": 2046, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "fine-art-archive-169", + "repo": "stranske/Fine-Art-Archive", + "pr": 169, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + } + ] } diff --git a/config/model_eval_pilot.json b/config/model_eval_pilot.json index 827d76940..491918bc7 100644 --- a/config/model_eval_pilot.json +++ b/config/model_eval_pilot.json @@ -1,38 +1,407 @@ { "schema": "workflows-verifier-pilot-corpus/v1", - "corpus_version": "verifier-balanced-pilot-2026-07-12", + "corpus_version": "verifier-balanced-pilot-2026-07-12+harvest21", "purpose": "Paired 30-case pilot used to narrow model candidates before the 75-case approval benchmark.", "adjudication_rule": "PASS requires durable completion disposition; NON_PASS requires a verifier-driven follow-up or unresolved completion gap.", "cases": [ - {"case_id":"workflows-2744","repo":"stranske/Workflows","pr":2744,"expected_verdict":"PASS","category":"clean-pass"}, - {"case_id":"workflows-2746","repo":"stranske/Workflows","pr":2746,"expected_verdict":"PASS","category":"clean-pass"}, - {"case_id":"workflows-2747","repo":"stranske/Workflows","pr":2747,"expected_verdict":"PASS","category":"clean-pass"}, - {"case_id":"pension-704","repo":"stranske/Pension-Data","pr":704,"expected_verdict":"PASS","category":"clean-pass"}, - {"case_id":"pension-702","repo":"stranske/Pension-Data","pr":702,"expected_verdict":"PASS","category":"stale-verifier-claim"}, - {"case_id":"workflows-2755","repo":"stranske/Workflows","pr":2755,"expected_verdict":"NON_PASS","category":"follow-up-required"}, - {"case_id":"workflows-2756","repo":"stranske/Workflows","pr":2756,"expected_verdict":"PASS","category":"clean-pass"}, - {"case_id":"workflows-2757","repo":"stranske/Workflows","pr":2757,"expected_verdict":"PASS","category":"review-thread-debt"}, - {"case_id":"pension-708","repo":"stranske/Pension-Data","pr":708,"expected_verdict":"PASS","category":"stale-verifier-claim"}, - {"case_id":"pension-711","repo":"stranske/Pension-Data","pr":711,"expected_verdict":"PASS","category":"clean-pass"}, - {"case_id":"pension-712","repo":"stranske/Pension-Data","pr":712,"expected_verdict":"PASS","category":"clean-pass"}, - {"case_id":"pension-713","repo":"stranske/Pension-Data","pr":713,"expected_verdict":"PASS","category":"stale-verifier-claim"}, - {"case_id":"pension-714","repo":"stranske/Pension-Data","pr":714,"expected_verdict":"PASS","category":"clean-pass"}, - {"case_id":"manager-1408","repo":"stranske/Manager-Database","pr":1408,"expected_verdict":"PASS","category":"clean-pass"}, - {"case_id":"manager-1411","repo":"stranske/Manager-Database","pr":1411,"expected_verdict":"PASS","category":"clean-pass"}, - {"case_id":"manager-1407","repo":"stranske/Manager-Database","pr":1407,"expected_verdict":"PASS","category":"clean-pass"}, - {"case_id":"manager-1412","repo":"stranske/Manager-Database","pr":1412,"expected_verdict":"PASS","category":"review-thread-debt"}, - {"case_id":"fine-art-237","repo":"stranske/Fine-Art-Archive","pr":237,"expected_verdict":"NON_PASS","category":"follow-up-required"}, - {"case_id":"fine-art-244","repo":"stranske/Fine-Art-Archive","pr":244,"expected_verdict":"PASS","category":"clean-pass"}, - {"case_id":"fine-art-255","repo":"stranske/Fine-Art-Archive","pr":255,"expected_verdict":"PASS","category":"clean-pass"}, - {"case_id":"fine-art-256","repo":"stranske/Fine-Art-Archive","pr":256,"expected_verdict":"PASS","category":"clean-pass"}, - {"case_id":"fine-art-257","repo":"stranske/Fine-Art-Archive","pr":257,"expected_verdict":"NON_PASS","category":"follow-up-required"}, - {"case_id":"inv-man-768","repo":"stranske/Inv-Man-Intake","pr":768,"expected_verdict":"PASS","category":"stale-verifier-claim"}, - {"case_id":"inv-man-781","repo":"stranske/Inv-Man-Intake","pr":781,"expected_verdict":"PASS","category":"clean-pass"}, - {"case_id":"inv-man-778","repo":"stranske/Inv-Man-Intake","pr":778,"expected_verdict":"PASS","category":"clean-pass"}, - {"case_id":"inv-man-776","repo":"stranske/Inv-Man-Intake","pr":776,"expected_verdict":"PASS","category":"clean-pass"}, - {"case_id":"inv-man-774","repo":"stranske/Inv-Man-Intake","pr":774,"expected_verdict":"PASS","category":"clean-pass"}, - {"case_id":"trip-1494","repo":"stranske/trip-planner","pr":1494,"expected_verdict":"PASS","category":"stale-verifier-claim"}, - {"case_id":"lms-393","repo":"stranske/learning-management-system","pr":393,"expected_verdict":"PASS","category":"clean-pass"}, - {"case_id":"workflows-2403","repo":"stranske/Workflows","pr":2403,"expected_verdict":"NON_PASS","category":"missing-acceptance-criterion"} + { + "case_id": "workflows-2744", + "repo": "stranske/Workflows", + "pr": 2744, + "expected_verdict": "PASS", + "category": "clean-pass" + }, + { + "case_id": "workflows-2746", + "repo": "stranske/Workflows", + "pr": 2746, + "expected_verdict": "PASS", + "category": "clean-pass" + }, + { + "case_id": "workflows-2747", + "repo": "stranske/Workflows", + "pr": 2747, + "expected_verdict": "PASS", + "category": "clean-pass" + }, + { + "case_id": "pension-704", + "repo": "stranske/Pension-Data", + "pr": 704, + "expected_verdict": "PASS", + "category": "clean-pass" + }, + { + "case_id": "pension-702", + "repo": "stranske/Pension-Data", + "pr": 702, + "expected_verdict": "PASS", + "category": "stale-verifier-claim" + }, + { + "case_id": "workflows-2755", + "repo": "stranske/Workflows", + "pr": 2755, + "expected_verdict": "NON_PASS", + "category": "follow-up-required" + }, + { + "case_id": "workflows-2756", + "repo": "stranske/Workflows", + "pr": 2756, + "expected_verdict": "PASS", + "category": "clean-pass" + }, + { + "case_id": "workflows-2757", + "repo": "stranske/Workflows", + "pr": 2757, + "expected_verdict": "PASS", + "category": "review-thread-debt" + }, + { + "case_id": "pension-708", + "repo": "stranske/Pension-Data", + "pr": 708, + "expected_verdict": "PASS", + "category": "stale-verifier-claim" + }, + { + "case_id": "pension-711", + "repo": "stranske/Pension-Data", + "pr": 711, + "expected_verdict": "PASS", + "category": "clean-pass" + }, + { + "case_id": "pension-712", + "repo": "stranske/Pension-Data", + "pr": 712, + "expected_verdict": "PASS", + "category": "clean-pass" + }, + { + "case_id": "pension-713", + "repo": "stranske/Pension-Data", + "pr": 713, + "expected_verdict": "PASS", + "category": "stale-verifier-claim" + }, + { + "case_id": "pension-714", + "repo": "stranske/Pension-Data", + "pr": 714, + "expected_verdict": "PASS", + "category": "clean-pass" + }, + { + "case_id": "manager-1408", + "repo": "stranske/Manager-Database", + "pr": 1408, + "expected_verdict": "PASS", + "category": "clean-pass" + }, + { + "case_id": "manager-1411", + "repo": "stranske/Manager-Database", + "pr": 1411, + "expected_verdict": "PASS", + "category": "clean-pass" + }, + { + "case_id": "manager-1407", + "repo": "stranske/Manager-Database", + "pr": 1407, + "expected_verdict": "PASS", + "category": "clean-pass" + }, + { + "case_id": "manager-1412", + "repo": "stranske/Manager-Database", + "pr": 1412, + "expected_verdict": "PASS", + "category": "review-thread-debt" + }, + { + "case_id": "fine-art-237", + "repo": "stranske/Fine-Art-Archive", + "pr": 237, + "expected_verdict": "NON_PASS", + "category": "follow-up-required" + }, + { + "case_id": "fine-art-244", + "repo": "stranske/Fine-Art-Archive", + "pr": 244, + "expected_verdict": "PASS", + "category": "clean-pass" + }, + { + "case_id": "fine-art-255", + "repo": "stranske/Fine-Art-Archive", + "pr": 255, + "expected_verdict": "PASS", + "category": "clean-pass" + }, + { + "case_id": "fine-art-256", + "repo": "stranske/Fine-Art-Archive", + "pr": 256, + "expected_verdict": "PASS", + "category": "clean-pass" + }, + { + "case_id": "fine-art-257", + "repo": "stranske/Fine-Art-Archive", + "pr": 257, + "expected_verdict": "NON_PASS", + "category": "follow-up-required" + }, + { + "case_id": "inv-man-768", + "repo": "stranske/Inv-Man-Intake", + "pr": 768, + "expected_verdict": "PASS", + "category": "stale-verifier-claim" + }, + { + "case_id": "inv-man-781", + "repo": "stranske/Inv-Man-Intake", + "pr": 781, + "expected_verdict": "PASS", + "category": "clean-pass" + }, + { + "case_id": "inv-man-778", + "repo": "stranske/Inv-Man-Intake", + "pr": 778, + "expected_verdict": "PASS", + "category": "clean-pass" + }, + { + "case_id": "inv-man-776", + "repo": "stranske/Inv-Man-Intake", + "pr": 776, + "expected_verdict": "PASS", + "category": "clean-pass" + }, + { + "case_id": "inv-man-774", + "repo": "stranske/Inv-Man-Intake", + "pr": 774, + "expected_verdict": "PASS", + "category": "clean-pass" + }, + { + "case_id": "trip-1494", + "repo": "stranske/trip-planner", + "pr": 1494, + "expected_verdict": "PASS", + "category": "stale-verifier-claim" + }, + { + "case_id": "lms-393", + "repo": "stranske/learning-management-system", + "pr": 393, + "expected_verdict": "PASS", + "category": "clean-pass" + }, + { + "case_id": "workflows-2403", + "repo": "stranske/Workflows", + "pr": 2403, + "expected_verdict": "NON_PASS", + "category": "missing-acceptance-criterion" + }, + { + "case_id": "workflows-2545", + "repo": "stranske/Workflows", + "pr": 2545, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "workflows-2544", + "repo": "stranske/Workflows", + "pr": 2544, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "workflows-2543", + "repo": "stranske/Workflows", + "pr": 2543, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "workflows-2540", + "repo": "stranske/Workflows", + "pr": 2540, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "workflows-2539", + "repo": "stranske/Workflows", + "pr": 2539, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "workflows-2538", + "repo": "stranske/Workflows", + "pr": 2538, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "workflows-2537", + "repo": "stranske/Workflows", + "pr": 2537, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "workflows-2536", + "repo": "stranske/Workflows", + "pr": 2536, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "workflows-2535", + "repo": "stranske/Workflows", + "pr": 2535, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "workflows-2534", + "repo": "stranske/Workflows", + "pr": 2534, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "workflows-2533", + "repo": "stranske/Workflows", + "pr": 2533, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "workflows-2532", + "repo": "stranske/Workflows", + "pr": 2532, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "workflows-2531", + "repo": "stranske/Workflows", + "pr": 2531, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "workflows-2530", + "repo": "stranske/Workflows", + "pr": 2530, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "workflows-2528", + "repo": "stranske/Workflows", + "pr": 2528, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "workflows-2527", + "repo": "stranske/Workflows", + "pr": 2527, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "workflows-2524", + "repo": "stranske/Workflows", + "pr": 2524, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "workflows-2523", + "repo": "stranske/Workflows", + "pr": 2523, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "workflows-2522", + "repo": "stranske/Workflows", + "pr": 2522, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "workflows-2520", + "repo": "stranske/Workflows", + "pr": 2520, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + }, + { + "case_id": "workflows-2519", + "repo": "stranske/Workflows", + "pr": 2519, + "expected_verdict": "PASS", + "category": "clean-pass", + "provenance": "harvested", + "harvested_at": "2026-07-27" + } ] }