feat(llm): add evidence-gated model evaluation pipeline (#2710)

.github/scripts/gate_summary.py

@@ -361,9 +361,9 @@ def summarize(context: SummaryContext) -> SummaryResult:
     # files changed.
     elif not context.python_required and python_result == "skipped":
         lines.append("- Python CI skipped: no Python-code changes detected.")
-    elif python_result == "cancelled":
+    elif python_result in {"cancelled", "abandoned"}:
         state = "pending"
-        description = "Python CI cancelled; waiting for rerun."
+        description = f"Python CI {python_result}; waiting for rerun."
     elif python_result not in ("success", "skipped") or (
         python_result == "skipped" and context.run_core
     ):

.github/sync-manifest.yml

@@ -682,7 +682,13 @@ scripts:
     description: "Embedding provider registry used by synced semantic matching helpers"
 
   - source: tools/check_model_registry_freshness.py
-    description: "Model-registry freshness gate - offline check that flags stale/blocked/dominated model pins so old models do not get stuck as primary"
+    description: "Model-registry freshness gate - validates dated decisions, evidence, lifecycle, and profile-based slots"
+
+  - source: tools/discover_model_catalog.py
+    description: "Advisory provider-catalog discovery - proposes new model candidates without changing reviewed selections"
+
+  - source: tools/evaluate_model_benchmark.py
+    description: "Deterministic paired model benchmark evaluator - computes confidence-bound quality gates and cost/latency ranking"
 
   # Research-backplane run-contract/v1 validator (run locally + in the conformance gate)
   - source: scripts/validate_run_contract.py
@@ -747,8 +753,15 @@ llm_config:
   - source: config/model_registry.json
     description: "Model registry - available LLM models and their capabilities"
 
+  - source: config/model_selection_policy.json
+    description: "Model selection policy - benchmark gates, measurements, optimization order, and review triggers"
+
 # Documentation synced to consumer repos
 docs:
+  - source: docs/MODEL_SELECTION_POLICY.md
+    target: docs/MODEL_SELECTION_POLICY.md
+    description: "Auditable auxiliary-model evaluation, selection, and refresh policy"
+
   - source: docs/AGENT_ISSUE_FORMAT.md
     target: docs/AGENT_ISSUE_FORMAT.md
     description: "Format specification for agent-compatible issues"

.github/workflows/maint-77-model-registry-freshness.yml

@@ -1,11 +1,8 @@
 name: Maint 77 Model Registry Freshness
 
-# Detects when the canonical LLM model configuration has gone stale so old models
-# do not get stuck as the primary ones indefinitely. OFFLINE + deterministic:
-# it only reads config/model_registry.json + config/llm_slots.json (no provider
-# API calls, no secrets). On staleness it opens/refreshes a single tracking issue;
-# a human reviews + refreshes the registry, which then propagates to consumers via
-# the existing maint-68 sync (model_registry.json is copy-synced).
+# The PR gate validates registry decisions, evidence, and slots offline. Scheduled
+# and manual runs additionally query provider catalogs when credentials exist.
+# Catalog changes are advisory candidates; they never change a selection.
 #
 # This does NOT change model selection and does NOT touch the sync workflow.
 
@@ -17,8 +14,11 @@ on:
   pull_request:
     paths:
       - config/model_registry.json
+      - config/model_selection_policy.json
       - config/llm_slots.json
       - tools/check_model_registry_freshness.py
+      - tools/discover_model_catalog.py
+      - tools/evaluate_model_benchmark.py
 
 permissions:
   contents: read
@@ -32,6 +32,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       rc: ${{ steps.gate.outputs.rc }}
+      discovery_drift: ${{ steps.discovery.outputs.drift || 'false' }}
     permissions:
       contents: read
     steps:
@@ -72,8 +73,45 @@ jobs:
           echo "::error::Model registry/slots are stale — see job summary."
           exit 1
 
+      - name: Discover provider catalog drift
+        if: github.event_name != 'pull_request'
+        id: discovery
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          CLAUDE_API_STRANSKE: ${{ secrets.CLAUDE_API_STRANSKE }}
+        run: |
+          set +e
+          python3 tools/discover_model_catalog.py --output catalog-discovery.json
+          rc=$?
+          if [ "$rc" = "1" ]; then
+            echo "drift=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "drift=false" >> "$GITHUB_OUTPUT"
+          fi
+          if [ "$rc" = "2" ]; then
+            echo "::error::Catalog discovery configuration error."
+            exit 2
+          fi
+          {
+            echo '### Provider catalog discovery'
+            echo '```json'
+            cat catalog-discovery.json
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload catalog discovery
+        if: github.event_name != 'pull_request' && always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: model-catalog-discovery-${{ github.run_id }}
+          path: catalog-discovery.json
+          if-no-files-found: ignore
+
   tracking-issue:
-    if: github.event_name != 'pull_request' && needs.freshness.outputs.rc == '1'
+    if: >-
+      github.event_name != 'pull_request' &&
+      (needs.freshness.outputs.rc == '1' || needs.freshness.outputs.discovery_drift == 'true')
     needs: freshness
     runs-on: ubuntu-latest
     permissions:
@@ -91,39 +129,57 @@ jobs:
           python-version: '3.14'
 
       - name: Recreate freshness report
+        id: reports
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          CLAUDE_API_STRANSKE: ${{ secrets.CLAUDE_API_STRANSKE }}
         run: |
           set +e
           python3 tools/check_model_registry_freshness.py --json > freshness.json
-          rc=$?
+          freshness_rc=$?
+          python3 tools/discover_model_catalog.py --output catalog-discovery.json
+          discovery_rc=$?
           cat freshness.json
-          if [ "$rc" = "2" ]; then
+          cat catalog-discovery.json
+          if [ "$freshness_rc" = "2" ] || [ "$discovery_rc" = "2" ]; then
             echo "::error::Freshness gate configuration error."
             exit 2
           fi
-          if [ "$rc" != "1" ]; then
-            echo "::notice::Freshness finding cleared before issue update."
+          if [ "$freshness_rc" != "1" ] && [ "$discovery_rc" != "1" ]; then
+            echo "::notice::Freshness and catalog findings cleared before issue update."
             exit 0
           fi
 
       - name: Open or refresh tracking issue (scheduled/dispatch only)
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          TITLE="🧭 LLM model registry needs review (freshness gate)"
+          TITLE="LLM model registry needs evidence review"
           BODY_FILE=issue_body.md
           {
-            echo "The model-registry freshness gate (\`tools/check_model_registry_freshness.py\`, maint-77) flagged stale or dominated model configuration."
+            echo "Maint-77 found an overdue/unproved selection or provider catalog drift."
+            echo "Catalog additions are candidates only and must not be auto-selected."
             echo ""
-            echo "Refresh \`config/model_registry.json\` (and \`config/llm_slots.json\` if pins are dominated/blocked), bump \`review_by\`, and merge — the update propagates to consumers via maint-68 (model_registry.json is copy-synced)."
+            echo "Follow \`docs/MODEL_SELECTION_POLICY.md\`: refresh facts, run the paired"
+            echo "adjudicated benchmark, attach evidence, and update the explicit selection"
+            echo "and review date. Maint-68 then propagates the registry."
             echo ""
             echo "**Findings:**"
             echo '```json'
             cat freshness.json
             echo '```'
             echo ""
-            echo "_Run:_ ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "**Catalog discovery:**"
+            echo '```json'
+            cat catalog-discovery.json
+            echo '```'
+            echo ""
+            echo "_Run:_ ${{ github.server_url }}/${{ github.repository }}/actions/runs/"
+            echo "${{ github.run_id }}"
           } > "$BODY_FILE"
-          EXISTING=$(gh issue list --search "$TITLE in:title" --state open --json number --jq '.[0].number' 2>/dev/null)
+          EXISTING=$(gh issue list --search "$TITLE in:title" --state open \
+            --json number --jq '.[0].number' 2>/dev/null)
           if [ -n "$EXISTING" ]; then
             gh issue comment "$EXISTING" --body-file "$BODY_FILE"
             echo "::warning::Refreshed existing freshness issue #$EXISTING"

.github/workflows/maint-78-model-evaluation-pilot.yml

@@ -0,0 +1,46 @@
+name: Maint 78 Model Evaluation Pilot
+
+on:
+  workflow_dispatch: {}
+
+permissions:
+  contents: read
+  pull-requests: read
+  models: read
+
+jobs:
+  pilot:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+      - name: Run paired 30-case pilot
+        env:
+          # Cross-repo read token; github.token is scoped to Workflows only.
+          GH_TOKEN: ${{ secrets.OWNER_PR_PAT }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          CLAUDE_API_STRANSKE: ${{ secrets.CLAUDE_API_STRANSKE }}
+        run: |
+          uv run --extra dev python tools/run_model_eval_pilot.py --output pilot-results.json
+      - name: Summarize pilot
+        if: always()
+        run: |
+          {
+            echo '## Verifier model pilot'
+            echo 'The pilot narrows candidates only; approval still requires the 75-case corpus.'
+            echo '```json'
+            jq '{schema, corpus_version, rows:(.results|length),
+              errors:([.results[]|select(.schema_valid == false)]|length)}' \
+              pilot-results.json
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        if: always()
+        with:
+          name: verifier-model-pilot-${{ github.run_id }}
+          path: |
+            pilot-results.json
+          if-no-files-found: error

README.md

@@ -76,7 +76,7 @@ Each stage completes, then dispatches auto-pilot again with the next step name.
 
 ### Verification Pipeline
 
-After PR merge, applying a `verify:*` label (typically `verify:evaluate` via auto-pilot, or `verify:compare` for dual-model mode) triggers the verifier. In `compare` mode, two LLM providers (gpt-5.4 + claude-sonnet-4-6) independently evaluate the diff against acceptance criteria with unanimous PASS required. On CONCERNS or FAIL, maintainers or automation can apply the `verify:create-new-pr` label to trigger a 4-round LLM pipeline that generates a follow-up issue (analyze -> tasks -> acceptance criteria -> format).
+After PR merge, applying a `verify:*` label (typically `verify:evaluate` via auto-pilot, or `verify:compare` for dual-model mode) triggers the verifier. In `compare` mode, two LLM providers (gpt-5.4 + claude-opus-4-6) independently evaluate the diff against acceptance criteria with unanimous PASS required. On CONCERNS or FAIL, maintainers or automation can apply the `verify:create-new-pr` label to trigger a 4-round LLM pipeline that generates a follow-up issue (analyze -> tasks -> acceptance criteria -> format).
 
 **Live verifier and pipeline metrics** are surfaced through the weekly summary tracker — see [issue #2211](https://github.com/stranske/Workflows/issues/2211) (durable auto-bot tracker, posted Mondays at 06:00 UTC) and the LangSmith dashboard wired by [`maint-80-langsmith-metrics-dashboard.yml`](.github/workflows/maint-80-langsmith-metrics-dashboard.yml). The original Feb 2026 baseline (40-PR sample, first-fix 35%, avg chain depth 2.7) is preserved at [`docs/analysis/verify-compare-40pr-evaluation-feb-2026.md`](docs/analysis/verify-compare-40pr-evaluation-feb-2026.md) for historical comparison.
 

config/llm_slots.json

@@ -1,20 +1,20 @@
 {
-  "purpose": "Auxiliary judge/evaluator slots only; do not treat these as coding-worker execution profiles.",
+  "purpose": "Auxiliary judge/evaluator provider preferences. Model versions resolve from explicit, reviewed registry decisions.",
   "slots": [
     {
       "name": "slot1",
       "provider": "openai",
-      "model": "gpt-5.4"
+      "profile": "verifier-balanced"
     },
     {
       "name": "slot2",
       "provider": "anthropic",
-      "quality_tier": "T5"
+      "profile": "verifier-balanced"
     },
     {
       "name": "slot3",
       "provider": "github-models",
-      "model": "codex-mini-latest"
+      "profile": "verifier-balanced"
     }
   ]
 }

config/model_eval_candidates.json

@@ -0,0 +1,10 @@
+{
+  "candidates": [
+    {"provider":"openai","model_id":"gpt-5.4","role":"incumbent"},
+    {"provider":"openai","model_id":"gpt-5.6-terra","role":"candidate"},
+    {"provider":"openai","model_id":"gpt-5.6-sol","role":"candidate"},
+    {"provider":"anthropic","model_id":"claude-opus-4-6","role":"incumbent"},
+    {"provider":"anthropic","model_id":"claude-sonnet-4-6","role":"candidate"},
+    {"provider":"github-models","model_id":"codex-mini-latest","role":"incumbent"}
+  ]
+}

config/model_eval_pilot.json

@@ -0,0 +1,38 @@
+{
+  "schema": "workflows-verifier-pilot-corpus/v1",
+  "corpus_version": "verifier-balanced-pilot-2026-07-12",
+  "purpose": "Paired 30-case pilot used to narrow model candidates before the 75-case approval benchmark.",
+  "adjudication_rule": "PASS requires durable completion disposition; NON_PASS requires a verifier-driven follow-up or unresolved completion gap.",
+  "cases": [
+    {"case_id":"workflows-2744","repo":"stranske/Workflows","pr":2744,"expected_verdict":"PASS","category":"clean-pass"},
+    {"case_id":"workflows-2746","repo":"stranske/Workflows","pr":2746,"expected_verdict":"PASS","category":"clean-pass"},
+    {"case_id":"workflows-2747","repo":"stranske/Workflows","pr":2747,"expected_verdict":"PASS","category":"clean-pass"},
+    {"case_id":"pension-704","repo":"stranske/Pension-Data","pr":704,"expected_verdict":"PASS","category":"clean-pass"},
+    {"case_id":"pension-702","repo":"stranske/Pension-Data","pr":702,"expected_verdict":"PASS","category":"stale-verifier-claim"},
+    {"case_id":"workflows-2755","repo":"stranske/Workflows","pr":2755,"expected_verdict":"NON_PASS","category":"follow-up-required"},
+    {"case_id":"workflows-2756","repo":"stranske/Workflows","pr":2756,"expected_verdict":"PASS","category":"clean-pass"},
+    {"case_id":"workflows-2757","repo":"stranske/Workflows","pr":2757,"expected_verdict":"PASS","category":"review-thread-debt"},
+    {"case_id":"pension-708","repo":"stranske/Pension-Data","pr":708,"expected_verdict":"PASS","category":"stale-verifier-claim"},
+    {"case_id":"pension-711","repo":"stranske/Pension-Data","pr":711,"expected_verdict":"PASS","category":"clean-pass"},
+    {"case_id":"pension-712","repo":"stranske/Pension-Data","pr":712,"expected_verdict":"PASS","category":"clean-pass"},
+    {"case_id":"pension-713","repo":"stranske/Pension-Data","pr":713,"expected_verdict":"PASS","category":"stale-verifier-claim"},
+    {"case_id":"pension-714","repo":"stranske/Pension-Data","pr":714,"expected_verdict":"PASS","category":"clean-pass"},
+    {"case_id":"manager-1408","repo":"stranske/Manager-Database","pr":1408,"expected_verdict":"PASS","category":"clean-pass"},
+    {"case_id":"manager-1411","repo":"stranske/Manager-Database","pr":1411,"expected_verdict":"PASS","category":"clean-pass"},
+    {"case_id":"manager-1407","repo":"stranske/Manager-Database","pr":1407,"expected_verdict":"PASS","category":"clean-pass"},
+    {"case_id":"manager-1412","repo":"stranske/Manager-Database","pr":1412,"expected_verdict":"PASS","category":"review-thread-debt"},
+    {"case_id":"fine-art-237","repo":"stranske/Fine-Art-Archive","pr":237,"expected_verdict":"NON_PASS","category":"follow-up-required"},
+    {"case_id":"fine-art-244","repo":"stranske/Fine-Art-Archive","pr":244,"expected_verdict":"PASS","category":"clean-pass"},
+    {"case_id":"fine-art-255","repo":"stranske/Fine-Art-Archive","pr":255,"expected_verdict":"PASS","category":"clean-pass"},
+    {"case_id":"fine-art-256","repo":"stranske/Fine-Art-Archive","pr":256,"expected_verdict":"PASS","category":"clean-pass"},
+    {"case_id":"fine-art-257","repo":"stranske/Fine-Art-Archive","pr":257,"expected_verdict":"NON_PASS","category":"follow-up-required"},
+    {"case_id":"inv-man-768","repo":"stranske/Inv-Man-Intake","pr":768,"expected_verdict":"PASS","category":"stale-verifier-claim"},
+    {"case_id":"inv-man-781","repo":"stranske/Inv-Man-Intake","pr":781,"expected_verdict":"PASS","category":"clean-pass"},
+    {"case_id":"inv-man-778","repo":"stranske/Inv-Man-Intake","pr":778,"expected_verdict":"PASS","category":"clean-pass"},
+    {"case_id":"inv-man-776","repo":"stranske/Inv-Man-Intake","pr":776,"expected_verdict":"PASS","category":"clean-pass"},
+    {"case_id":"inv-man-774","repo":"stranske/Inv-Man-Intake","pr":774,"expected_verdict":"PASS","category":"clean-pass"},
+    {"case_id":"trip-1494","repo":"stranske/trip-planner","pr":1494,"expected_verdict":"PASS","category":"stale-verifier-claim"},
+    {"case_id":"lms-393","repo":"stranske/learning-management-system","pr":393,"expected_verdict":"PASS","category":"clean-pass"},
+    {"case_id":"workflows-2403","repo":"stranske/Workflows","pr":2403,"expected_verdict":"NON_PASS","category":"missing-acceptance-criterion"}
+  ]
+}