diff --git a/.github/workflows/health-78-backplane-contract.yml b/.github/workflows/health-78-backplane-contract.yml index 8f101771d..2d51b3fcf 100644 --- a/.github/workflows/health-78-backplane-contract.yml +++ b/.github/workflows/health-78-backplane-contract.yml @@ -35,6 +35,18 @@ on: - 'scripts/validate_run_contract.py' - 'scripts/validate_backplane_registry.py' - 'tests/fixtures/backplane/**' + # Weekly warning-only freshness surfacing (mirrors the langsmith contract's + # scheduled path). Reference-run staleness is a ::warning:: here, not a failure, + # so an aged reference baseline is visible on cadence without wedging CI. Use the + # manual dispatch (enforce_freshness=true) for a deliberate freshness gate. + schedule: + - cron: '43 6 * * 1' + workflow_dispatch: + inputs: + enforce_freshness: + description: 'Fail the run on reference-run staleness (default: warning-only)' + type: boolean + default: false permissions: contents: read @@ -78,9 +90,18 @@ jobs: | tee artifacts/backplane-contract/self-smoke.txt - name: Registry lifecycle and evidence validation + # Structural defects always fail. Reference-run staleness is warning-only + # (surfaced as a ::warning:: annotation) unless a manual dispatch opts into + # enforcement, so an expired freshness window never wedges the fleet. + env: + ENFORCE_FRESHNESS: ${{ github.event.inputs.enforce_freshness == 'true' }} run: | set -o pipefail - python scripts/validate_backplane_registry.py --json \ + enforce_flag="" + if [ "${ENFORCE_FRESHNESS}" = "true" ]; then + enforce_flag="--enforce-freshness" + fi + python scripts/validate_backplane_registry.py --json ${enforce_flag} \ | tee artifacts/backplane-contract/registry-status.json - name: Documented invalid-fixture smoke (must exit 1) diff --git a/scripts/validate_backplane_registry.py b/scripts/validate_backplane_registry.py index cd6f0932c..7720e17fe 100644 --- a/scripts/validate_backplane_registry.py +++ b/scripts/validate_backplane_registry.py @@ -362,30 +362,58 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--strict", action="store_true", - help="Compatibility flag; validation is always strict.", + help="Compatibility flag; structural validation is always strict.", + ) + parser.add_argument( + "--enforce-freshness", + action="store_true", + help=( + "Also fail on reference-run staleness. Default is warning-only: staleness " + "is an operational freshness signal (a human-reviewed reference baseline " + "aging past the window), surfaced but non-blocking, so an expired window " + "does not wedge unrelated CI. Set this for a deliberate freshness gate " + "(mirrors the langsmith contract's opt-in enforce_block)." + ), ) args = parser.parse_args(argv) registry = _load_json(args.registry) findings = validate_registry(registry) + blocking = blocking_findings(findings) + stale = [f for f in findings if f.severity == STALE_SEVERITY] + fail = bool(blocking) or (args.enforce_freshness and bool(stale)) report = { "generated_at": datetime.now(UTC).isoformat(), "ok": not findings, + "blocking_ok": not blocking, "finding_count": len(findings), - "findings": [{"path": f.path, "message": f.message} for f in findings], + "blocking_count": len(blocking), + "stale_count": len(stale), + "findings": [ + {"path": f.path, "message": f.message, "severity": f.severity} for f in findings + ], "participants": _reference_rows(registry), } + # Emit staleness as a GitHub Actions warning annotation on stderr regardless of + # output mode, so the freshness signal surfaces in the Actions UI even when the + # JSON report is piped to a file. Non-blocking unless --enforce-freshness. + for finding in stale: + print(f"::warning::{finding.path}: {finding.message}", file=sys.stderr) + if args.json: print(json.dumps(report, indent=2, sort_keys=True)) - elif findings: - print("Backplane registry validation failed:", file=sys.stderr) - for finding in findings: - print(f"- {finding.path}: {finding.message}", file=sys.stderr) else: - print("Backplane registry validation passed.") - - return 0 if not findings else 1 + if blocking: + print("Backplane registry validation failed:", file=sys.stderr) + for finding in blocking: + print(f"- {finding.path}: {finding.message}", file=sys.stderr) + if not findings: + print("Backplane registry validation passed.") + elif not blocking and not args.enforce_freshness: + print("Backplane registry structurally valid (reference-run freshness warnings only).") + + return 1 if fail else 0 if __name__ == "__main__": diff --git a/tests/test_backplane_registry.py b/tests/test_backplane_registry.py index c2ae3de76..accad445a 100644 --- a/tests/test_backplane_registry.py +++ b/tests/test_backplane_registry.py @@ -60,6 +60,50 @@ def test_structural_defects_remain_blocking_despite_staleness_carveout() -> None assert vbr.blocking_findings(findings) +def _write_stale_registry(tmp_path: Path) -> Path: + registry = copy.deepcopy(_registry()) + entry = _pension_conformant_entry(registry) + entry["reference_run_evidence"]["generated_at"] = "2026-01-01T00:00:00Z" + registry_path = tmp_path / "registry.json" + registry_path.write_text(json.dumps(registry), encoding="utf-8") + return registry_path + + +def test_cli_staleness_is_warning_only_by_default( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + registry_path = _write_stale_registry(tmp_path) + + # Warning-only: a stale-but-structurally-valid registry must not fail the job, + # so an expired freshness window can't wedge unrelated CI. + assert vbr.main(["--json", str(registry_path)]) == 0 + + report = json.loads(capsys.readouterr().out) + assert report["stale_count"] >= 1 + assert report["blocking_count"] == 0 + assert report["blocking_ok"] is True + assert report["ok"] is False # not fully clean; freshness is surfaced + + +def test_cli_enforce_freshness_makes_staleness_blocking(tmp_path: Path) -> None: + registry_path = _write_stale_registry(tmp_path) + + assert vbr.main(["--enforce-freshness", str(registry_path)]) == 1 + + +def test_cli_emits_github_warning_annotation_for_staleness( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + registry_path = _write_stale_registry(tmp_path) + + vbr.main([str(registry_path)]) + + # Warning annotation is emitted on stderr so it surfaces even in --json mode. + err = capsys.readouterr().err + assert "::warning::" in err + assert "reference run is stale" in err + + def test_parent_issue_and_inactive_participants_are_explicit() -> None: registry = _registry()