maint-79: grow verifier corpus from realized PR outcomes (#2819 move 2)

.github/workflows/maint-79-verifier-corpus-harvest.yml

@@ -0,0 +1,106 @@
+name: Maint 79 Verifier Corpus Harvest
+
+# Grows the frozen verifier evaluation corpus from realized PR outcomes
+# (stranske/Workflows#2819, move 2). High-confidence cases (stable merges,
+# reverts, resolved verifier follow-ups) auto-promote via an auto-merging PR;
+# ambiguous cases land in an FYI-only, auto-expiring staging file. No human
+# action is required — the PR is the audit trail, not a gate.
+#
+# On pull_request touching the harvester/config it only runs the unit tests
+# (the corpus is grown from live data, so there is no deterministic drift gate).
+
+on:
+  schedule:
+    # Weekly, Monday 05:40 UTC (after maint-68 sync + maint-77 freshness).
+    - cron: "40 5 * * 1"
+  workflow_dispatch:
+    inputs:
+      write:
+        description: "Persist and open a PR (false = dry-run summary only)."
+        type: boolean
+        default: true
+  pull_request:
+    paths:
+      - tools/harvest_verifier_corpus.py
+      - tests/tools/test_harvest_verifier_corpus.py
+      - config/model_selection_policy.json
+      - config/model_eval_corpus_staging.json
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.14"
+      - name: Unit tests
+        run: python -m pytest tests/tools/test_harvest_verifier_corpus.py -q
+
+  harvest:
+    if: github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.14"
+      - name: Harvest realized-outcome cases
+        env:
+          # Cross-repo read; github.token is scoped to Workflows only.
+          GH_TOKEN: ${{ secrets.OWNER_PR_PAT }}
+        run: |
+          mode=""
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.write }}" != "true" ]; then
+            mode="dry-run"
+            python -m tools.harvest_verifier_corpus | tee harvest.log
+          else
+            python -m tools.harvest_verifier_corpus --write | tee harvest.log
+          fi
+          {
+            echo '## Verifier corpus harvest'
+            echo '```'
+            cat harvest.log
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+      - name: Open auto-merging corpus-growth PR
+        if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.write)
+        id: cpr
+        uses: peter-evans/create-pull-request@v8
+        with:
+          token: ${{ secrets.OWNER_PR_PAT }}
+          branch: verifier-corpus-harvest/auto
+          base: main
+          title: "corpus: harvest realized-outcome verifier cases"
+          commit-message: "corpus: grow verifier eval corpus from realized PR outcomes"
+          labels: "automation,model-selection"
+          delete-branch: true
+          body: |
+            Automated corpus growth (stranske/Workflows#2819 move 2).
+
+            High-confidence cases derived from realized PR outcomes — see the run
+            summary for the promoted case list. Ambiguous cases were routed to the
+            auto-expiring staging file, not here.
+
+            Expected verdicts here come from what the world already adjudicated by
+            merging or reverting each PR. The semantic NON_PASS categories
+            (stale-verifier-claim, review-thread-debt, missing-acceptance-criterion)
+            remain owner-sourced and are never machine-added.
+      - name: Enable auto-merge
+        if: steps.cpr.outputs.pull-request-number
+        env:
+          GH_TOKEN: ${{ secrets.OWNER_PR_PAT }}
+        run: gh pr merge "${{ steps.cpr.outputs.pull-request-number }}" --squash --auto

config/model_eval_corpus_staging.json

@@ -0,0 +1,5 @@
+{
+  "schema": "verifier-corpus-staging/v1",
+  "note": "FYI-only. Auto-expiring candidate cases pending stability or adjudication; nothing here gates anything.",
+  "cases": []
+}

config/model_selection_policy.json

@@ -54,6 +54,34 @@
         "pricing_change": true,
         "material_prompt_or_workload_change": true,
         "quality_gate_breach": true
+      },
+      "corpus_growth": {
+        "enabled": true,
+        "stability_days": 30,
+        "staging_expiry_days": 60,
+        "harvest_window_days": 60,
+        "max_corpus_size": 150,
+        "auto_promote_confidence": "high",
+        "category_caps": {
+          "clean-pass": 40,
+          "regression-after-merge": 20,
+          "follow-up-required": 20
+        },
+        "note": "Move 2 of #2819. High-confidence realized-outcome cases (stable merges, reverts, resolved verifier follow-ups) auto-grow the frozen corpus; ambiguous cases stage FYI-only and auto-expire. The stale-verifier-claim / review-thread-debt / missing-acceptance-criterion categories are NOT machine-labelable and remain owner-sourced.",
+        "source_repos": [
+          "stranske/Workflows",
+          "stranske/Travel-Plan-Permission",
+          "stranske/Trend_Model_Project",
+          "stranske/Portable-Alpha-Extension-Model",
+          "stranske/Counter_Risk",
+          "stranske/Manager-Database",
+          "stranske/Inv-Man-Intake",
+          "stranske/Pension-Data",
+          "stranske/Ready",
+          "stranske/trip-planner",
+          "stranske/learning-management-system",
+          "stranske/Fine-Art-Archive"
+        ]
       }
     }
   }

tests/tools/test_harvest_verifier_corpus.py

@@ -0,0 +1,164 @@
+"""Tests for tools/harvest_verifier_corpus.py (realized-outcome corpus growth)."""
+
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+
+from tools import harvest_verifier_corpus as hv
+
+NOW = datetime(2026, 7, 25, tzinfo=UTC)
+
+
+def _rec(pr, *, merged=True, days_ago=40, reverted=False, followup=False, resolved=False):
+    merged_at = None
+    if merged:
+        merged_at = datetime(2026, 7, 25, tzinfo=UTC).fromordinal(NOW.toordinal() - days_ago)
+        merged_at = merged_at.replace(tzinfo=UTC).isoformat()
+    return {
+        "repo": "stranske/Demo",
+        "pr": pr,
+        "merged": merged,
+        "merged_at": merged_at,
+        "reverted": reverted,
+        "verifier_followup": followup,
+        "followup_resolved": resolved,
+    }
+
+
+def test_stable_merge_is_high_confidence_pass():
+    label = hv.classify(_rec(1, days_ago=40), now=NOW, stability_days=30)
+    assert label == {"expected_verdict": "PASS", "category": "clean-pass", "confidence": "high"}
+
+
+def test_recent_merge_is_low_confidence_pass():
+    label = hv.classify(_rec(2, days_ago=5), now=NOW, stability_days=30)
+    assert label["expected_verdict"] == "PASS"
+    assert label["confidence"] == "low"  # not yet stable -> staging
+
+
+def test_reverted_is_high_confidence_non_pass():
+    label = hv.classify(_rec(3, days_ago=10, reverted=True), now=NOW, stability_days=30)
+    assert label == {
+        "expected_verdict": "NON_PASS",
+        "category": "regression-after-merge",
+        "confidence": "high",
+    }
+
+
+def test_resolved_followup_is_high_confidence_non_pass():
+    label = hv.classify(_rec(4, followup=True, resolved=True), now=NOW, stability_days=30)
+    assert label["category"] == "follow-up-required"
+    assert label["confidence"] == "high"
+
+
+def test_unresolved_followup_stages():
+    label = hv.classify(_rec(5, followup=True, resolved=False), now=NOW, stability_days=30)
+    assert label["expected_verdict"] == "NON_PASS"
+    assert label["confidence"] == "low"
+
+
+def test_unmerged_pr_yields_no_signal():
+    assert hv.classify(_rec(6, merged=False), now=NOW, stability_days=30) is None
+
+
+def test_partition_splits_promote_and_stage():
+    recs = [_rec(1, days_ago=40), _rec(2, days_ago=3), _rec(3, reverted=True)]
+    promote, stage = hv.partition(recs, now=NOW, stability_days=30)
+    assert {c["pr"] for c in promote} == {1, 3}
+    assert {c["pr"] for c in stage} == {2}
+
+
+def test_grow_corpus_dedups_and_caps_and_bumps_version():
+    corpus = {"corpus_version": "v2026-07-12", "cases": [{"repo": "stranske/Demo", "pr": 1}]}
+    promote = [
+        hv.to_case(
+            _rec(1, days_ago=40), {"expected_verdict": "PASS", "category": "clean-pass"}, now=NOW
+        ),
+        hv.to_case(
+            _rec(9, days_ago=40), {"expected_verdict": "PASS", "category": "clean-pass"}, now=NOW
+        ),
+    ]
+    grown, added = hv.grow_corpus(corpus, promote, max_size=150)
+    assert [c["pr"] for c in added] == [9]  # pr 1 already present -> deduped
+    assert grown["corpus_version"] == "v2026-07-12+harvest1"
+
+    capped, added2 = hv.grow_corpus(corpus, promote, max_size=1)  # already 1 case -> cap hit
+    assert added2 == [] and capped is corpus
+
+
+def test_grow_corpus_respects_per_category_caps():
+    corpus = {"cases": []}
+    promote = [
+        hv.to_case(
+            _rec(n, days_ago=40), {"expected_verdict": "PASS", "category": "clean-pass"}, now=NOW
+        )
+        for n in range(10)
+    ]
+    grown, added = hv.grow_corpus(corpus, promote, max_size=150, category_caps={"clean-pass": 3})
+    assert len(added) == 3  # capped at 3 clean-pass despite 10 offered
+    assert len(grown["cases"]) == 3
+
+
+def test_grow_corpus_noop_returns_original():
+    corpus = {"cases": [{"repo": "stranske/Demo", "pr": 1}]}
+    grown, added = hv.grow_corpus(corpus, [], max_size=150)
+    assert added == [] and grown is corpus
+
+
+def test_staging_auto_expires_old_cases():
+    old = {"repo": "stranske/Demo", "pr": 100, "harvested_at": "2026-01-01"}  # >60d ago
+    fresh = {"repo": "stranske/Demo", "pr": 101, "harvested_at": NOW.date().isoformat()}
+    out = hv.prune_staging({"cases": [old]}, [fresh], now=NOW, expiry_days=60)
+    prs = {c["pr"] for c in out["cases"]}
+    assert prs == {101}  # old one expired out, fresh retained
+
+
+def test_main_dry_run_and_write(tmp_path, capsys):
+    policy = {
+        "profiles": {
+            "verifier-balanced": {
+                "corpus_growth": {
+                    "enabled": True,
+                    "stability_days": 30,
+                    "staging_expiry_days": 60,
+                    "max_corpus_size": 150,
+                    "source_repos": [],
+                }
+            }
+        }
+    }
+    corpus = {"corpus_version": "v1", "cases": []}
+    records = [_rec(1, days_ago=40), _rec(2, days_ago=2)]
+    pol_p = tmp_path / "policy.json"
+    cor_p = tmp_path / "corpus.json"
+    stg_p = tmp_path / "staging.json"
+    rec_p = tmp_path / "records.json"
+    pol_p.write_text(json.dumps(policy))
+    cor_p.write_text(json.dumps(corpus))
+    rec_p.write_text(json.dumps(records))
+
+    argv = [
+        "--policy",
+        str(pol_p),
+        "--corpus",
+        str(cor_p),
+        "--staging",
+        str(stg_p),
+        "--from-json",
+        str(rec_p),
+        "--write",
+    ]
+    assert hv.main(argv) == 0
+    grown = json.loads(cor_p.read_text())
+    assert [c["pr"] for c in grown["cases"]] == [1]  # only the stable merge promoted
+    staged = json.loads(stg_p.read_text())
+    assert [c["pr"] for c in staged["cases"]] == [2]  # recent merge staged
+
+
+def test_main_respects_disabled_flag(tmp_path):
+    pol_p = tmp_path / "policy.json"
+    pol_p.write_text(
+        json.dumps({"profiles": {"verifier-balanced": {"corpus_growth": {"enabled": False}}}})
+    )
+    assert hv.main(["--policy", str(pol_p), "--from-json", str(tmp_path / "none.json")]) == 0

tests/tools/test_model_eval_pilot.py

@@ -2,21 +2,43 @@
 import json
 from pathlib import Path
 
+# The owner-adjudicated seed is digest-frozen: it may never be silently edited.
+# Machine-harvested cases (provenance == "harvested") are allowed to grow the
+# corpus over time (stranske/Workflows#2819 move 2) and are validated
+# structurally rather than by digest.
+FROZEN_SEED_DIGEST = "a823e6fca7b3e16d5c6a8cff685ea9288cacedacc1ce7d9ca4c7af887227f287"
+MACHINE_LABELABLE_CATEGORIES = {
+    "clean-pass",
+    "regression-after-merge",
+    "follow-up-required",
+}
 
-def test_pilot_corpus_is_frozen_and_well_formed() -> None:
+
+def _load_cases():
     root = Path(__file__).resolve().parents[2]
     payload = json.loads((root / "config" / "model_eval_pilot.json").read_text())
-    cases = payload["cases"]
+    return payload, payload["cases"]
+
+
+def test_pilot_seed_is_frozen() -> None:
+    payload, cases = _load_cases()
+    seed = [case for case in cases if case.get("provenance") != "harvested"]
 
     assert payload["schema"] == "workflows-verifier-pilot-corpus/v1"
-    assert payload["corpus_version"] == "verifier-balanced-pilot-2026-07-12"
-    assert len(cases) == 30
-    assert len({case["case_id"] for case in cases}) == 30
-    assert all(case["repo"].startswith("stranske/") for case in cases)
-    assert all(isinstance(case["pr"], int) and case["pr"] > 0 for case in cases)
-    assert {case["expected_verdict"] for case in cases} == {"PASS", "NON_PASS"}
+    # Version keeps its adjudicated base; harvest runs append a "+harvestN" suffix.
+    assert payload["corpus_version"].startswith("verifier-balanced-pilot-2026-07-12")
+    assert len(seed) == 30
+
+    identity = [
+        {key: case[key] for key in ("case_id", "repo", "pr", "expected_verdict", "category")}
+        for case in seed
+    ]
+    digest = hashlib.sha256(
+        json.dumps(identity, sort_keys=True, separators=(",", ":")).encode()
+    ).hexdigest()
+    assert digest == FROZEN_SEED_DIGEST, "the owner-adjudicated seed cases must not change"
 
-    categories = {case["category"] for case in cases}
+    categories = {case["category"] for case in seed}
     assert {
         "clean-pass",
         "missing-acceptance-criterion",
@@ -25,11 +47,19 @@ def test_pilot_corpus_is_frozen_and_well_formed() -> None:
         "follow-up-required",
     } <= categories
 
-    identity = [
-        {key: case[key] for key in ("case_id", "repo", "pr", "expected_verdict", "category")}
-        for case in cases
-    ]
-    digest = hashlib.sha256(
-        json.dumps(identity, sort_keys=True, separators=(",", ":")).encode()
-    ).hexdigest()
-    assert digest == "a823e6fca7b3e16d5c6a8cff685ea9288cacedacc1ce7d9ca4c7af887227f287"
+
+def test_corpus_is_well_formed_and_ids_unique() -> None:
+    _payload, cases = _load_cases()
+    assert len({case["case_id"] for case in cases}) == len(cases)
+    assert all(case["repo"].startswith("stranske/") for case in cases)
+    assert all(isinstance(case["pr"], int) and case["pr"] > 0 for case in cases)
+    assert {case["expected_verdict"] for case in cases} <= {"PASS", "NON_PASS"}
+
+
+def test_harvested_cases_are_constrained() -> None:
+    _payload, cases = _load_cases()
+    harvested = [case for case in cases if case.get("provenance") == "harvested"]
+    for case in harvested:
+        # Harvesting may only produce categories it can label from realized outcomes.
+        assert case["category"] in MACHINE_LABELABLE_CATEGORIES
+        assert "harvested_at" in case