Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 22 additions & 1 deletion .github/workflows/health-78-backplane-contract.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,18 @@ on:
- 'scripts/validate_run_contract.py'
- 'scripts/validate_backplane_registry.py'
- 'tests/fixtures/backplane/**'
# Weekly warning-only freshness surfacing (mirrors the langsmith contract's
# scheduled path). Reference-run staleness is a ::warning:: here, not a failure,
# so an aged reference baseline is visible on cadence without wedging CI. Use the
# manual dispatch (enforce_freshness=true) for a deliberate freshness gate.
schedule:

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Document the new scheduled and manual workflow contract

The workflow now supports a weekly schedule and manual freshness enforcement, but the inspected inventories still describe health-78-backplane-contract.yml as PR/push-only (docs/ci/WORKFLOWS.md:199 and docs/ci/WORKFLOW_SYSTEM.md:737). Operators relying on those canonical docs will not discover the cadence or enforce_freshness gate, so update both inventories and the backplane contract documentation alongside this trigger change.

AGENTS.md reference: AGENTS.md:L60-L65

Useful? React with 👍 / 👎.

- cron: '43 6 * * 1'
workflow_dispatch:
inputs:
enforce_freshness:
description: 'Fail the run on reference-run staleness (default: warning-only)'
type: boolean
default: false

permissions:
contents: read
Expand Down Expand Up @@ -78,9 +90,18 @@ jobs:
| tee artifacts/backplane-contract/self-smoke.txt

- name: Registry lifecycle and evidence validation
# Structural defects always fail. Reference-run staleness is warning-only
# (surfaced as a ::warning:: annotation) unless a manual dispatch opts into
# enforcement, so an expired freshness window never wedges the fleet.
env:
ENFORCE_FRESHNESS: ${{ github.event.inputs.enforce_freshness == 'true' }}
run: |
set -o pipefail
python scripts/validate_backplane_registry.py --json \
enforce_flag=""
if [ "${ENFORCE_FRESHNESS}" = "true" ]; then
enforce_flag="--enforce-freshness"
fi
python scripts/validate_backplane_registry.py --json ${enforce_flag} \
| tee artifacts/backplane-contract/registry-status.json

- name: Documented invalid-fixture smoke (must exit 1)
Expand Down
46 changes: 37 additions & 9 deletions scripts/validate_backplane_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -362,30 +362,58 @@ def main(argv: list[str] | None = None) -> int:
parser.add_argument(
"--strict",
action="store_true",
help="Compatibility flag; validation is always strict.",
help="Compatibility flag; structural validation is always strict.",
)
parser.add_argument(
"--enforce-freshness",
action="store_true",
help=(
"Also fail on reference-run staleness. Default is warning-only: staleness "
"is an operational freshness signal (a human-reviewed reference baseline "
"aging past the window), surfaced but non-blocking, so an expired window "
"does not wedge unrelated CI. Set this for a deliberate freshness gate "
"(mirrors the langsmith contract's opt-in enforce_block)."
),
)
args = parser.parse_args(argv)

registry = _load_json(args.registry)
findings = validate_registry(registry)
blocking = blocking_findings(findings)
stale = [f for f in findings if f.severity == STALE_SEVERITY]
fail = bool(blocking) or (args.enforce_freshness and bool(stale))
report = {
"generated_at": datetime.now(UTC).isoformat(),
"ok": not findings,
"blocking_ok": not blocking,
"finding_count": len(findings),
"findings": [{"path": f.path, "message": f.message} for f in findings],
"blocking_count": len(blocking),
"stale_count": len(stale),
"findings": [
{"path": f.path, "message": f.message, "severity": f.severity} for f in findings
],
"participants": _reference_rows(registry),
}

# Emit staleness as a GitHub Actions warning annotation on stderr regardless of
# output mode, so the freshness signal surfaces in the Actions UI even when the
# JSON report is piped to a file. Non-blocking unless --enforce-freshness.
for finding in stale:
print(f"::warning::{finding.path}: {finding.message}", file=sys.stderr)

if args.json:
print(json.dumps(report, indent=2, sort_keys=True))
elif findings:
print("Backplane registry validation failed:", file=sys.stderr)
for finding in findings:
print(f"- {finding.path}: {finding.message}", file=sys.stderr)
else:
print("Backplane registry validation passed.")

return 0 if not findings else 1
if blocking:
print("Backplane registry validation failed:", file=sys.stderr)
for finding in blocking:
print(f"- {finding.path}: {finding.message}", file=sys.stderr)
if not findings:
print("Backplane registry validation passed.")
elif not blocking and not args.enforce_freshness:
print("Backplane registry structurally valid (reference-run freshness warnings only).")

return 1 if fail else 0


if __name__ == "__main__":
Expand Down
44 changes: 44 additions & 0 deletions tests/test_backplane_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,50 @@ def test_structural_defects_remain_blocking_despite_staleness_carveout() -> None
assert vbr.blocking_findings(findings)


def _write_stale_registry(tmp_path: Path) -> Path:
registry = copy.deepcopy(_registry())
entry = _pension_conformant_entry(registry)
entry["reference_run_evidence"]["generated_at"] = "2026-01-01T00:00:00Z"
registry_path = tmp_path / "registry.json"
registry_path.write_text(json.dumps(registry), encoding="utf-8")
return registry_path


def test_cli_staleness_is_warning_only_by_default(
tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
registry_path = _write_stale_registry(tmp_path)

# Warning-only: a stale-but-structurally-valid registry must not fail the job,
# so an expired freshness window can't wedge unrelated CI.
assert vbr.main(["--json", str(registry_path)]) == 0

report = json.loads(capsys.readouterr().out)
assert report["stale_count"] >= 1
assert report["blocking_count"] == 0
assert report["blocking_ok"] is True
assert report["ok"] is False # not fully clean; freshness is surfaced


def test_cli_enforce_freshness_makes_staleness_blocking(tmp_path: Path) -> None:
registry_path = _write_stale_registry(tmp_path)

assert vbr.main(["--enforce-freshness", str(registry_path)]) == 1


def test_cli_emits_github_warning_annotation_for_staleness(
tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
registry_path = _write_stale_registry(tmp_path)

vbr.main([str(registry_path)])

# Warning annotation is emitted on stderr so it surfaces even in --json mode.
err = capsys.readouterr().err
assert "::warning::" in err
assert "reference run is stale" in err


def test_parent_issue_and_inactive_participants_are_explicit() -> None:
registry = _registry()

Expand Down
Loading