diff --git a/.github/workflows/maint-77-model-registry-freshness.yml b/.github/workflows/maint-77-model-registry-freshness.yml index 0fb639b79..b35027072 100644 --- a/.github/workflows/maint-77-model-registry-freshness.yml +++ b/.github/workflows/maint-77-model-registry-freshness.yml @@ -32,6 +32,7 @@ jobs: runs-on: ubuntu-latest outputs: rc: ${{ steps.gate.outputs.rc }} + has_advisory: ${{ steps.gate.outputs.has_advisory }} discovery_drift: ${{ steps.discovery.outputs.drift || 'false' }} permissions: contents: read @@ -47,6 +48,8 @@ jobs: python-version: '3.14' # On PRs, fail the job on staleness so registry/slot changes are gated. + # The default gate fails only on structural findings (danger); a merely + # overdue review is advisory and never fails, so it cannot block work. # On schedule/dispatch, never fail the run — open a tracking issue instead. - name: Run freshness gate id: gate @@ -55,6 +58,8 @@ jobs: python3 tools/check_model_registry_freshness.py --json > freshness.json rc=$? echo "rc=$rc" >> "$GITHUB_OUTPUT" + has_advisory=$(python3 -c "import json;print('true' if json.load(open('freshness.json')).get('advisory') else 'false')") + echo "has_advisory=$has_advisory" >> "$GITHUB_OUTPUT" cat freshness.json { echo '### Model registry freshness' @@ -67,10 +72,12 @@ jobs: exit 2 fi - - name: Fail PRs on staleness + # Only structural/dangerous findings fail a model-config PR. An overdue + # review is advisory and is surfaced by the scheduled tracking issue. + - name: Fail PRs on structural findings if: github.event_name == 'pull_request' && steps.gate.outputs.rc == '1' run: | - echo "::error::Model registry/slots are stale — see job summary." + echo "::error::Model registry/slots have a structural problem — see job summary." exit 1 - name: Discover provider catalog drift @@ -111,7 +118,9 @@ jobs: tracking-issue: if: >- github.event_name != 'pull_request' && - (needs.freshness.outputs.rc == '1' || needs.freshness.outputs.discovery_drift == 'true') + (needs.freshness.outputs.rc == '1' || + needs.freshness.outputs.has_advisory == 'true' || + needs.freshness.outputs.discovery_drift == 'true') needs: freshness runs-on: ubuntu-latest permissions: diff --git a/docs/MODEL_SELECTION_POLICY.md b/docs/MODEL_SELECTION_POLICY.md index c002bbd94..39fd73dbe 100644 --- a/docs/MODEL_SELECTION_POLICY.md +++ b/docs/MODEL_SELECTION_POLICY.md @@ -64,6 +64,25 @@ records. Do not hand-enter aggregate rates or recommendation rankings. An approved evidence record uses `kind: workload-benchmark` and `status: passed`. The freshness gate rejects an approved decision without it. +### Advisory vs. blocking findings + +`tools/check_model_registry_freshness.py` separates its findings into two +classes, and by default only one of them fails the gate: + +- **Blocking (structural):** a malformed registry, a selection or slot that + points at an absent or blocked model, or an *approved* selection with no + passing workload-benchmark evidence. These mean work could be wrong, so they + fail the gate (exit 1). +- **Advisory (cadence):** a review whose `review_by` date has simply passed + (`review_overdue`, `provisional_overdue`, `selection_review_overdue`). A due + review is not a danger signal — the provisional incumbents remain a valid + runtime baseline — so it never fails the default gate and never blocks + unrelated work. It is surfaced instead by the `maint-77` scheduled run, which + opens a non-blocking tracking issue. + +Pass `--strict` to fail on *any* finding (used where a PR itself edits model +configuration and should be proven fresh before merging). + ## Incumbents and Candidates The existing OpenAI, Anthropic, and GitHub Models verifier choices are recorded diff --git a/tests/test_check_model_registry_freshness.py b/tests/test_check_model_registry_freshness.py index a51c08afe..a43522eaa 100644 --- a/tests/test_check_model_registry_freshness.py +++ b/tests/test_check_model_registry_freshness.py @@ -319,7 +319,16 @@ def test_main_exit_codes(tmp_path: Path): "2026-07-10", ] assert gate.main(common) == 0 + # An overdue review is advisory: it must NOT fail the default gate (so it + # cannot block unrelated fleet-wide work)... registry_path.write_text(json.dumps(_registry(review_by="2026-07-01")), encoding="utf-8") + assert gate.main(common) == 0 + # ...but --strict still fails on it, for callers gating a model-config change. + assert gate.main([*common, "--strict"]) == 1 + # A structural finding (selection references an absent model) always blocks. + structural = _registry() + structural["selections"][0]["model_id"] = "missing" + registry_path.write_text(json.dumps(structural), encoding="utf-8") assert gate.main(common) == 1 assert gate.main([*common[:-1], "not-a-date"]) == 2 diff --git a/tools/check_model_registry_freshness.py b/tools/check_model_registry_freshness.py index f95e1c0ed..95ea69251 100644 --- a/tools/check_model_registry_freshness.py +++ b/tools/check_model_registry_freshness.py @@ -30,6 +30,30 @@ DEFAULT_MAX_AGE_DAYS = 30 VALID_SELECTION_STATUSES = {"provisional", "approved"} +# Time-cadence findings mean "a review is due", NOT "this work is dangerous". +# They are advisory: reported and surfaced as a tracking issue, but they never +# fail the gate (and so never block unrelated fleet-wide work). Only structural +# findings — a malformed registry, an absent/blocked model, an APPROVED +# selection with no passing evidence — indicate work could be wrong, and those +# still block. Use --strict to fail on any finding (e.g. gating a PR that itself +# edits model config). +ADVISORY_FINDING_KINDS = frozenset( + { + "review_overdue", + "provisional_overdue", + "selection_review_overdue", + } +) + + +def partition_findings( + findings: list[dict[str, str]], +) -> tuple[list[dict[str, str]], list[dict[str, str]]]: + """Split findings into (blocking, advisory).""" + advisory = [f for f in findings if f.get("kind") in ADVISORY_FINDING_KINDS] + blocking = [f for f in findings if f.get("kind") not in ADVISORY_FINDING_KINDS] + return blocking, advisory + def _normalize_provider(provider: str) -> str: normalized = (provider or "").strip().lower() @@ -403,6 +427,11 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--max-age-days", type=int, default=DEFAULT_MAX_AGE_DAYS) parser.add_argument("--today", type=str, default=None) parser.add_argument("--json", action="store_true") + parser.add_argument( + "--strict", + action="store_true", + help="Fail (exit 1) on ANY finding, including advisory cadence findings.", + ) args = parser.parse_args(argv) try: @@ -421,15 +450,34 @@ def main(argv: list[str] | None = None) -> int: max_age_days=args.max_age_days, policy=policy, ) + blocking, advisory = partition_findings(findings) if args.json: - print(json.dumps({"fresh": not findings, "findings": findings}, indent=2)) + print( + json.dumps( + { + "fresh": not findings, + "ok": not blocking, + "blocking": blocking, + "advisory": advisory, + "findings": findings, + }, + indent=2, + ) + ) elif findings: - print(f"Model registry freshness: {len(findings)} finding(s):") - for finding in findings: - print(f" [{finding['kind']}] {finding['detail']}") + print(f"Model registry freshness: {len(blocking)} blocking, {len(advisory)} advisory:") + for finding in blocking: + print(f" [BLOCK] [{finding['kind']}] {finding['detail']}") + for finding in advisory: + print(f" [advisory] [{finding['kind']}] {finding['detail']}") else: print("Model registry is fresh: decisions, evidence, and slots are consistent.") - return 1 if findings else 0 + + if args.strict: + return 1 if findings else 0 + # Default: only structural/dangerous findings fail the gate. A merely-overdue + # review is advisory and must never block unrelated work. + return 1 if blocking else 0 if __name__ == "__main__": # pragma: no cover