From dc86ed204e6d1c3289a77b4f9a47b0c3d6c736c2 Mon Sep 17 00:00:00 2001 From: Tim Stranske Date: Fri, 24 Jul 2026 17:49:46 -0500 Subject: [PATCH] Make backplane reference-run freshness warning-only + surface on a weekly cadence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #2821. That PR stopped reference-run staleness from failing the REQUIRED suite; this addresses the underlying blocker: staleness still hard-failed the dedicated health-78 lane (its CLI exited 1), so the only way to clear the red was a manual reference-run refresh — which nothing schedules, so it recurs weekly. The contract docs establish the intended model: sibling freshness contracts (langsmith-observability-contract.md) make the scheduled path WARNING-ONLY with opt-in enforcement (enforce_block). The reference_run_evidence is a human-reviewed baseline (carries verifier issue/PR + disposition comment), so a 7-day wall-clock hard-fail on it is mis-calibrated. Align with the documented pattern: - validate_backplane_registry.py `main()`: exit non-zero only on BLOCKING (structural) findings. Reference-run staleness is emitted as a `::warning::` annotation (on stderr, so it surfaces even in --json mode) and is non-blocking by default. New `--enforce-freshness` opts into failing on staleness (mirrors langsmith's enforce_block). Report gains blocking_ok/blocking_count/stale_count and per-finding severity. Structural defects still fail (test-gated). - health-78: add a weekly `schedule:` (warning-only cadence surfacing) and a `workflow_dispatch` `enforce_freshness` input for a deliberate gate. With main() now warning-only, the lane goes green-with-warning on an aged baseline instead of stuck-red — no manual intervention needed to keep CI green. Tests: staleness warning-only (exit 0) with stale_count surfaced; --enforce-freshness fails; ::warning:: annotation emitted; structural defects still block (deliberate-break). 27 pass; black/ruff/actionlint clean. NOT in scope (needs a real cross-repo run, won't fabricate provenance): auto- REGENERATING the evidence by re-emitting the Pension-Data reference conformance run and writing back real provenance. Speccing that separately. Co-Authored-By: Claude Opus 4.8 --- .../health-78-backplane-contract.yml | 23 +++++++++- scripts/validate_backplane_registry.py | 46 +++++++++++++++---- tests/test_backplane_registry.py | 44 ++++++++++++++++++ 3 files changed, 103 insertions(+), 10 deletions(-) diff --git a/.github/workflows/health-78-backplane-contract.yml b/.github/workflows/health-78-backplane-contract.yml index 8f101771d..2d51b3fcf 100644 --- a/.github/workflows/health-78-backplane-contract.yml +++ b/.github/workflows/health-78-backplane-contract.yml @@ -35,6 +35,18 @@ on: - 'scripts/validate_run_contract.py' - 'scripts/validate_backplane_registry.py' - 'tests/fixtures/backplane/**' + # Weekly warning-only freshness surfacing (mirrors the langsmith contract's + # scheduled path). Reference-run staleness is a ::warning:: here, not a failure, + # so an aged reference baseline is visible on cadence without wedging CI. Use the + # manual dispatch (enforce_freshness=true) for a deliberate freshness gate. + schedule: + - cron: '43 6 * * 1' + workflow_dispatch: + inputs: + enforce_freshness: + description: 'Fail the run on reference-run staleness (default: warning-only)' + type: boolean + default: false permissions: contents: read @@ -78,9 +90,18 @@ jobs: | tee artifacts/backplane-contract/self-smoke.txt - name: Registry lifecycle and evidence validation + # Structural defects always fail. Reference-run staleness is warning-only + # (surfaced as a ::warning:: annotation) unless a manual dispatch opts into + # enforcement, so an expired freshness window never wedges the fleet. + env: + ENFORCE_FRESHNESS: ${{ github.event.inputs.enforce_freshness == 'true' }} run: | set -o pipefail - python scripts/validate_backplane_registry.py --json \ + enforce_flag="" + if [ "${ENFORCE_FRESHNESS}" = "true" ]; then + enforce_flag="--enforce-freshness" + fi + python scripts/validate_backplane_registry.py --json ${enforce_flag} \ | tee artifacts/backplane-contract/registry-status.json - name: Documented invalid-fixture smoke (must exit 1) diff --git a/scripts/validate_backplane_registry.py b/scripts/validate_backplane_registry.py index cd6f0932c..7720e17fe 100644 --- a/scripts/validate_backplane_registry.py +++ b/scripts/validate_backplane_registry.py @@ -362,30 +362,58 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--strict", action="store_true", - help="Compatibility flag; validation is always strict.", + help="Compatibility flag; structural validation is always strict.", + ) + parser.add_argument( + "--enforce-freshness", + action="store_true", + help=( + "Also fail on reference-run staleness. Default is warning-only: staleness " + "is an operational freshness signal (a human-reviewed reference baseline " + "aging past the window), surfaced but non-blocking, so an expired window " + "does not wedge unrelated CI. Set this for a deliberate freshness gate " + "(mirrors the langsmith contract's opt-in enforce_block)." + ), ) args = parser.parse_args(argv) registry = _load_json(args.registry) findings = validate_registry(registry) + blocking = blocking_findings(findings) + stale = [f for f in findings if f.severity == STALE_SEVERITY] + fail = bool(blocking) or (args.enforce_freshness and bool(stale)) report = { "generated_at": datetime.now(UTC).isoformat(), "ok": not findings, + "blocking_ok": not blocking, "finding_count": len(findings), - "findings": [{"path": f.path, "message": f.message} for f in findings], + "blocking_count": len(blocking), + "stale_count": len(stale), + "findings": [ + {"path": f.path, "message": f.message, "severity": f.severity} for f in findings + ], "participants": _reference_rows(registry), } + # Emit staleness as a GitHub Actions warning annotation on stderr regardless of + # output mode, so the freshness signal surfaces in the Actions UI even when the + # JSON report is piped to a file. Non-blocking unless --enforce-freshness. + for finding in stale: + print(f"::warning::{finding.path}: {finding.message}", file=sys.stderr) + if args.json: print(json.dumps(report, indent=2, sort_keys=True)) - elif findings: - print("Backplane registry validation failed:", file=sys.stderr) - for finding in findings: - print(f"- {finding.path}: {finding.message}", file=sys.stderr) else: - print("Backplane registry validation passed.") - - return 0 if not findings else 1 + if blocking: + print("Backplane registry validation failed:", file=sys.stderr) + for finding in blocking: + print(f"- {finding.path}: {finding.message}", file=sys.stderr) + if not findings: + print("Backplane registry validation passed.") + elif not blocking and not args.enforce_freshness: + print("Backplane registry structurally valid (reference-run freshness warnings only).") + + return 1 if fail else 0 if __name__ == "__main__": diff --git a/tests/test_backplane_registry.py b/tests/test_backplane_registry.py index c2ae3de76..accad445a 100644 --- a/tests/test_backplane_registry.py +++ b/tests/test_backplane_registry.py @@ -60,6 +60,50 @@ def test_structural_defects_remain_blocking_despite_staleness_carveout() -> None assert vbr.blocking_findings(findings) +def _write_stale_registry(tmp_path: Path) -> Path: + registry = copy.deepcopy(_registry()) + entry = _pension_conformant_entry(registry) + entry["reference_run_evidence"]["generated_at"] = "2026-01-01T00:00:00Z" + registry_path = tmp_path / "registry.json" + registry_path.write_text(json.dumps(registry), encoding="utf-8") + return registry_path + + +def test_cli_staleness_is_warning_only_by_default( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + registry_path = _write_stale_registry(tmp_path) + + # Warning-only: a stale-but-structurally-valid registry must not fail the job, + # so an expired freshness window can't wedge unrelated CI. + assert vbr.main(["--json", str(registry_path)]) == 0 + + report = json.loads(capsys.readouterr().out) + assert report["stale_count"] >= 1 + assert report["blocking_count"] == 0 + assert report["blocking_ok"] is True + assert report["ok"] is False # not fully clean; freshness is surfaced + + +def test_cli_enforce_freshness_makes_staleness_blocking(tmp_path: Path) -> None: + registry_path = _write_stale_registry(tmp_path) + + assert vbr.main(["--enforce-freshness", str(registry_path)]) == 1 + + +def test_cli_emits_github_warning_annotation_for_staleness( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + registry_path = _write_stale_registry(tmp_path) + + vbr.main([str(registry_path)]) + + # Warning annotation is emitted on stderr so it surfaces even in --json mode. + err = capsys.readouterr().err + assert "::warning::" in err + assert "reference run is stale" in err + + def test_parent_issue_and_inactive_participants_are_explicit() -> None: registry = _registry()