diff --git a/.github/workflows/health-68-consumer-sync-drift.yml b/.github/workflows/health-68-consumer-sync-drift.yml index 642ed81a90..49de8b9152 100644 --- a/.github/workflows/health-68-consumer-sync-drift.yml +++ b/.github/workflows/health-68-consumer-sync-drift.yml @@ -15,6 +15,9 @@ on: workflow_run: workflows: [Merge Sync PRs] types: [completed] + schedule: + # Daily self-healing cadence; workflow_run remains debounced below. + - cron: '15 6 * * *' workflow_dispatch: inputs: repos: @@ -24,10 +27,10 @@ on: permissions: contents: read - issues: write + actions: read concurrency: - group: consumer-sync-drift-${{ github.repository }}-${{ github.ref }} + group: health-68-consumer-sync-drift cancel-in-progress: true jobs: @@ -37,28 +40,87 @@ jobs: github.event_name != 'workflow_run' || (github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_branch == 'main') runs-on: ubuntu-latest + outputs: + drift_failed: ${{ steps.compare.outcome == 'failure' }} steps: - name: Checkout Workflows repo - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Setup API client uses: ./.github/actions/setup-api-client with: - secrets: ${{ toJSON(secrets) }} github_token: ${{ github.token }} + - name: Debounce workflow_run fan-out + id: debounce + if: github.event_name == 'workflow_run' + env: + GH_TOKEN: ${{ github.token }} + DEBOUNCE_MINUTES: '30' + run: | + set -euo pipefail + latest_created="$( + node -e ' + const { Octokit } = require("@octokit/rest"); + const { createTokenAwareRetry } = require("./.github/scripts/github-api-with-retry.js"); + const core = { info: () => {}, warning: console.warn, debug: () => {} }; + const [owner, repo] = process.env.GITHUB_REPOSITORY.split("/"); + (async () => { + const github = new Octokit({ auth: process.env.GH_TOKEN || process.env.GITHUB_TOKEN }); + const { withRetry } = await createTokenAwareRetry({ + github, + core, + env: process.env, + task: "health-68-consumer-sync-drift-debounce", + capabilities: ["actions:read"], + }); + const response = await withRetry(() => github.rest.actions.listWorkflowRuns({ + owner, + repo, + workflow_id: "health-68-consumer-sync-drift.yml", + branch: "main", + per_page: 20, + })); + process.stdout.write(JSON.stringify(response.data)); + })().catch((error) => { + console.error(error); + process.exit(1); + }); + ' | jq -r --argjson current "$GITHUB_RUN_ID" \ + '[.workflow_runs[] | select(.id != $current) | select(.conclusion == "success" or .conclusion == "failure" or .conclusion == "timed_out")][0].created_at // empty' + )" + if [ -z "$latest_created" ]; then + echo "No prior executable Health 68 run; continuing." + echo "skip=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + latest_epoch="$(date -d "$latest_created" +%s)" + now_epoch="$(date -u +%s)" + age_minutes=$(( (now_epoch - latest_epoch) / 60 )) + echo "Last executable Health 68 run was ${age_minutes} minutes ago at ${latest_created}." + if [ "$age_minutes" -lt "$DEBOUNCE_MINUTES" ]; then + echo "Skipping debounced workflow_run fan-out (< ${DEBOUNCE_MINUTES} minutes)." + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "skip=false" >> "$GITHUB_OUTPUT" - name: Setup Python + if: steps.debounce.outputs.skip != 'true' uses: actions/setup-python@v7 with: python-version: '3.14' - name: Install dependencies + if: steps.debounce.outputs.skip != 'true' run: pip install pyyaml requests - name: Compare consumer repos to templates id: compare + if: steps.debounce.outputs.skip != 'true' env: CONSUMER_SYNC_DRIFT_REPORT_JSON: artifacts/consumer-sync-drift-report.json INPUT_REPOS: ${{ inputs.repos }} @@ -74,7 +136,7 @@ jobs: --report-json "$CONSUMER_SYNC_DRIFT_REPORT_JSON" - name: Upload drift report - if: ${{ always() }} + if: ${{ always() && steps.debounce.outputs.skip != 'true' }} uses: actions/upload-artifact@v7 with: name: consumer-sync-drift-report @@ -82,8 +144,32 @@ jobs: if-no-files-found: warn retention-days: 14 + publish-drift: + name: Publish actionable drift tracker update + needs: check-drift + if: ${{ always() && needs.check-drift.outputs.drift_failed == 'true' }} + runs-on: ubuntu-latest + permissions: + contents: read + issues: write + steps: + - name: Checkout Workflows repo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup API client + uses: ./.github/actions/setup-api-client + with: + github_token: ${{ github.token }} + + - name: Download drift report + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: consumer-sync-drift-report + path: artifacts + - name: Create drift issue - if: ${{ failure() }} uses: actions/github-script@v9 with: script: | diff --git a/.github/workflows/health-71-sync-health-check.yml b/.github/workflows/health-71-sync-health-check.yml index f0f1cc34c4..873ba18d9d 100644 --- a/.github/workflows/health-71-sync-health-check.yml +++ b/.github/workflows/health-71-sync-health-check.yml @@ -39,7 +39,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Export load-balancer tokens uses: ./.github/actions/setup-api-client - name: Install GitHub API dependencies @@ -281,3 +283,45 @@ jobs: echo "**Hours since last success:** ${{ steps.check.outputs.hours_since_success }}" fi } >> "$GITHUB_STEP_SUMMARY" + + - name: Assert durable tracker workflow liveness + id: tracker_liveness + continue-on-error: true + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + pip install pyyaml + set +e + python3 scripts/check_durable_tracker_liveness.py --comment-on-failure + checker_status=$? + set -e + case "$checker_status" in + 0) + echo "status=healthy" >> "$GITHUB_OUTPUT" + ;; + 1) + echo "status=unhealthy" >> "$GITHUB_OUTPUT" + exit 1 + ;; + *) + echo "status=error" >> "$GITHUB_OUTPUT" + exit "$checker_status" + ;; + esac + + - name: Tracker liveness summary + if: always() && steps.tracker_liveness.outputs.status != 'healthy' + env: + TRACKER_LIVENESS_STATUS: ${{ steps.tracker_liveness.outputs.status }} + run: | + { + echo "## Durable tracker liveness" + echo "" + if [ "$TRACKER_LIVENESS_STATUS" = "unhealthy" ]; then + echo "At least one durable tracker source workflow has no executable run inside its configured cadence." + echo "FYI comments were posted on the affected tracker issue(s); no gate was opened or closed." + else + echo "The tracker-liveness checker encountered a coverage, setup, or runtime error; no liveness disposition was inferred." + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/config/durable_tracker_liveness.yml b/config/durable_tracker_liveness.yml new file mode 100644 index 0000000000..d22d341469 --- /dev/null +++ b/config/durable_tracker_liveness.yml @@ -0,0 +1,21 @@ +# Workflows monitored by Health 71 durable-tracker liveness assertions. +# Each entry must match a row in docs/ops/DURABLE_TRACKING_ISSUES.md. +trackers: + - workflow: agents-weekly-metrics.yml + issue: 2211 + max_age_hours: 192 + - workflow: maint-82-sync-dependency-campaign.yml + issue: 1836 + max_age_hours: 1 + - workflow: health-83-dependency-sync-efficiency.yml + issue: 2897 + max_age_hours: 192 + - workflow: health-68-consumer-sync-drift.yml + issue: 2210 + max_age_hours: 48 + - workflow: maint-69-sync-integration-repo.yml + issue: 2470 + event_driven: true + - workflow: maint-80-langsmith-metrics-dashboard.yml + issue: 2415 + max_age_hours: 192 diff --git a/docs/WORKFLOW_GUIDE.md b/docs/WORKFLOW_GUIDE.md index 47e544c77b..5f51ba37fd 100644 --- a/docs/WORKFLOW_GUIDE.md +++ b/docs/WORKFLOW_GUIDE.md @@ -78,7 +78,7 @@ _Inline Gate helper_ - **`health-44-gate-branch-protection.yml`** — Enforces branch-protection policy via `tools/enforce_gate_branch_protection.py` when the PAT is configured (now triggered on PRs or by the consolidated sweep) and skips minting extra GitHub App tokens because enforcement already runs with the configured `BRANCH_PROTECTION_TOKEN`. - **`health-50-security-scan.yml`** — Security scanning workflow triggered on push, PR, and schedule. Runs CodeQL vulnerability scans using the configured PAT priority list without minting additional GitHub App tokens. - **`health-67-integration-sync-check.yml`** — Daily + event-driven comparison between `templates/integration-repo/` and the Workflows-Integration-Tests repo. Manual dispatchers can now toggle the CI/versions/input checks individually to avoid cloning/running sections they don't need while still filing drift issues when enabled checks detect problems. -- **`health-68-consumer-sync-drift.yml`** — Daily + change-triggered drift detector for consumer repos. Uses the shared `scripts/list_registered_consumer_repos.py` helper to build the repo list (also accepts a manual override) before running `scripts/check_consumer_sync_drift.py`, and files/updates the `consumer-sync` issue when inconsistencies surface. +- **`health-68-consumer-sync-drift.yml`** — Daily self-healing drift detector for consumer repos, also triggered by qualifying template/manifest/script pushes, manual dispatch, and debounced successful main-branch `Merge Sync PRs` completions. Uses the shared `scripts/list_registered_consumer_repos.py` helper to build the repo list (also accepts a manual override) before running `scripts/check_consumer_sync_drift.py`, and files/updates the `consumer-sync` issue when inconsistencies surface. - **`health-69-consumer-sync-shadow-evidence.yml`** — Weekly, change-triggered, and manual read-only producer for the typed consumer-sync plan plus a non-authorizing Orchestrator handoff artifact. It has `contents: read` only and cannot mutate consumers or promote the shadow capability. - **`health-71-sync-health-check.yml`** — Daily monitor that inspects the recent `maint-68-sync-consumer-repos` runs. Manual dispatches can tweak `lookback_runs`/`max_age_hours`, and the workflow now reuses `scripts/list_registered_consumer_repos.py` (instead of ad-hoc parsing) without minting an extra GitHub App token. - **`health-72-template-sync.yml`** — Keeps manifest-declared exact template-sync files in sync between Workflows and the consumer template. For PRs from this repo it auto-runs `scripts/sync_templates.sh` + pushes deltas, then validates via `scripts/validate_template_sync.py`, relying only on the installation token. diff --git a/docs/ops/DURABLE_TRACKING_ISSUES.md b/docs/ops/DURABLE_TRACKING_ISSUES.md index 13948520ac..467d418a07 100644 --- a/docs/ops/DURABLE_TRACKING_ISSUES.md +++ b/docs/ops/DURABLE_TRACKING_ISSUES.md @@ -38,7 +38,7 @@ failure-notification path. | [#2211](https://github.com/stranske/Workflows/issues/2211) | Agent metrics weekly summary | [`agents-weekly-metrics.yml`](../../.github/workflows/agents-weekly-metrics.yml) | Mondays 06:00 UTC | New comment per run | | [#1836](https://github.com/stranske/Workflows/issues/1836) | Sync/Dependabot campaign queue | [`maint-82-sync-dependency-campaign.yml`](../../.github/workflows/maint-82-sync-dependency-campaign.yml) + [`.github/scripts/sync_dependency_campaign.js`](../../.github/scripts/sync_dependency_campaign.js) | Every 10 minutes + Mondays 10:30 UTC | Body rewritten in place; due transient delivery lanes are serialized, while actionable delivery blockers do not suppress corrective candidate work | | [#2897](https://github.com/stranske/Workflows/issues/2897) | Dependency/sync maintenance-efficiency advisory report | [`health-83-dependency-sync-efficiency.yml`](../../.github/workflows/health-83-dependency-sync-efficiency.yml) | Weekly Mondays 11:15 UTC + `workflow_dispatch` | Comment appended only on material-evidence fingerprint change | -| [#2210](https://github.com/stranske/Workflows/issues/2210) | 🔄 Consumer repo drift detected | [`health-68-consumer-sync-drift.yml`](../../.github/workflows/health-68-consumer-sync-drift.yml) | After each successful main-branch `Merge Sync PRs` run, qualifying manifest/template/script/tool pushes, or manual dispatch | Body rewritten in place, but only for actionable states | +| [#2210](https://github.com/stranske/Workflows/issues/2210) | 🔄 Consumer repo drift detected | [`health-68-consumer-sync-drift.yml`](../../.github/workflows/health-68-consumer-sync-drift.yml) | Daily 06:15 UTC schedule, debounced successful main-branch `Merge Sync PRs` runs, qualifying manifest/template/script/tool pushes, or manual dispatch | Body rewritten in place, but only for actionable states | | [#2470](https://github.com/stranske/Workflows/issues/2470) | 🚨 Integration-Tests Sync Failed - Action Required | [`maint-69-sync-integration-repo.yml`](../../.github/workflows/maint-69-sync-integration-repo.yml) | On qualifying template/config pushes or manual dispatch | Stuck-window marker in body + recovery comment | | [#2415](https://github.com/stranske/Workflows/issues/2415) | 📊 LangSmith Trace Coverage Dashboard | [`maint-80-langsmith-metrics-dashboard.yml`](../../.github/workflows/maint-80-langsmith-metrics-dashboard.yml) | Mondays 09:00 UTC + manual dispatch | Body rewritten in place | diff --git a/scripts/check_durable_tracker_liveness.py b/scripts/check_durable_tracker_liveness.py new file mode 100644 index 0000000000..5603dcfc89 --- /dev/null +++ b/scripts/check_durable_tracker_liveness.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""Assert durable tracker source workflows executed inside their cadence.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[1] +CONFIG_PATH = REPO_ROOT / "config" / "durable_tracker_liveness.yml" +TRACKER_DOC = REPO_ROOT / "docs" / "ops" / "DURABLE_TRACKING_ISSUES.md" +EXECUTABLE_CONCLUSIONS = frozenset({"success", "failure", "cancelled", "timed_out"}) + + +def _github_token() -> str: + token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") + if token: + return token + return subprocess.check_output(["gh", "auth", "token"], text=True).strip() + + +def _load_config() -> list[dict[str, Any]]: + data = yaml.safe_load(CONFIG_PATH.read_text(encoding="utf-8")) + trackers = data.get("trackers") + if not isinstance(trackers, list) or not trackers: + raise ValueError(f"{CONFIG_PATH} must define a non-empty trackers list") + return trackers + + +def tracker_doc_workflows() -> set[str]: + import re + + text = TRACKER_DOC.read_text(encoding="utf-8") + section_match = re.search( + r"## Current durable trackers\n\n\| Issue.*?\n\|[-| ]+\n(.*?)(?:\n\n|\n### )", + text, + re.DOTALL, + ) + assert section_match, "DURABLE_TRACKING_ISSUES.md is missing the tracker table" + table_body = section_match.group(1) + workflows = re.findall(r"\[`[^`]+`\]\([^)]*/([^/)]+)\)", table_body) + return {name for name in workflows if name.endswith(".yml")} + + +def _latest_executable_run(repo: str, workflow_file: str, token: str) -> dict[str, Any] | None: + raw = subprocess.check_output( + [ + "gh", + "api", + f"repos/{repo}/actions/workflows/{workflow_file}/runs", + "--method", + "GET", + "-f", + "per_page=100", + "--jq", + ".workflow_runs", + ], + text=True, + env={**os.environ, "GH_TOKEN": token}, + ) + runs = json.loads(raw) + if not isinstance(runs, list): + return None + for run in runs: + if not isinstance(run, dict): + continue + conclusion = str(run.get("conclusion") or "") + if conclusion in EXECUTABLE_CONCLUSIONS: + return run + return None + + +def _hours_since(iso_timestamp: str) -> float: + created = datetime.fromisoformat(iso_timestamp.replace("Z", "+00:00")) + return (datetime.now(UTC) - created).total_seconds() / 3600.0 + + +def evaluate_trackers(repo: str, token: str | None = None) -> list[dict[str, Any]]: + auth = token or _github_token() + results: list[dict[str, Any]] = [] + for entry in _load_config(): + workflow = str(entry["workflow"]) + issue = int(entry["issue"]) + if entry.get("event_driven") is True: + results.append( + { + "workflow": workflow, + "issue": issue, + "healthy": True, + "reason": "event-driven workflow excluded from age-based liveness", + } + ) + continue + max_age_hours = float(entry["max_age_hours"]) + latest = _latest_executable_run(repo, workflow, auth) + if latest is None: + results.append( + { + "workflow": workflow, + "issue": issue, + "healthy": False, + "reason": "no executable run found (only action_required/skipped)", + } + ) + continue + hours = _hours_since(str(latest["created_at"])) + healthy = hours <= max_age_hours + results.append( + { + "workflow": workflow, + "issue": issue, + "healthy": healthy, + "latest_conclusion": latest.get("conclusion"), + "latest_created_at": latest.get("created_at"), + "hours_since": round(hours, 2), + "max_age_hours": max_age_hours, + "run_url": latest.get("html_url"), + } + ) + return results + + +def _comment_on_tracker(repo: str, issue: int, body: str, token: str) -> None: + subprocess.run( + ["gh", "issue", "comment", str(issue), "--repo", repo, "--body", body], + check=True, + env={**os.environ, "GH_TOKEN": token}, + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY", "stranske/Workflows")) + parser.add_argument("--comment-on-failure", action="store_true") + parser.add_argument("--json", action="store_true") + args = parser.parse_args(argv) + + configured = {str(entry["workflow"]) for entry in _load_config()} + documented = tracker_doc_workflows() + missing_from_config = sorted(documented - configured) + extra_in_config = sorted(configured - documented) + if missing_from_config or extra_in_config: + print( + "Tracker workflow coverage mismatch: " + f"missing_from_config={missing_from_config} extra_in_config={extra_in_config}", + file=sys.stderr, + ) + return 2 + + token = _github_token() + results = evaluate_trackers(args.repo, token) + unhealthy = [item for item in results if not item.get("healthy")] + + if args.json: + print(json.dumps({"results": results, "unhealthy_count": len(unhealthy)}, indent=2)) + else: + for item in results: + status = "healthy" if item.get("healthy") else "UNHEALTHY" + print(f"{item['workflow']}\t{status}\t{item}") + + if args.comment_on_failure and unhealthy: + for item in unhealthy: + body = ( + "## Durable tracker liveness alert\n\n" + f"Source workflow `{item['workflow']}` has no executable run inside its " + f"{item.get('max_age_hours', '?')}h cadence.\n\n" + f"- Latest executable run: {item.get('latest_created_at', 'none')}\n" + f"- Conclusion: {item.get('latest_conclusion', 'n/a')}\n" + f"- Hours since: {item.get('hours_since', 'n/a')}\n" + f"- Run URL: {item.get('run_url', 'n/a')}\n\n" + "Confirm liveness from workflow run history, not tracker comment activity." + ) + _comment_on_tracker(args.repo, int(item["issue"]), body, token) + + return 1 if unhealthy else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/docs/test_durable_tracker_drift_states.py b/tests/docs/test_durable_tracker_drift_states.py index b99f43957f..409c4c97ac 100644 --- a/tests/docs/test_durable_tracker_drift_states.py +++ b/tests/docs/test_durable_tracker_drift_states.py @@ -56,17 +56,17 @@ def test_tracker_doc_documents_every_state_the_checker_emits() -> None: ) -def test_tracker_doc_does_not_promise_a_cron_health_68_no_longer_has() -> None: +def test_tracker_doc_documents_daily_schedule_for_health_68() -> None: text = TRACKER_DOC.read_text(encoding="utf-8") workflow = HEALTH_68.read_text(encoding="utf-8") row = _ROW_2210.search(text) assert row, "missing #2210 cadence row in the durable tracker table" cadence = row.group(1) - assert "schedule:" not in workflow, "Health 68 regained a cron; update the tracker doc row" - assert "05:10" not in cadence and "cron" not in cadence.lower() + assert "schedule:" in workflow, "Health 68 must declare a daily schedule trigger" + assert "daily" in cadence.lower() or "06:15" in cadence assert "Merge Sync PRs" in cadence - assert "successful main-branch" in cadence + assert "successful main-branch" in cadence or "debounced" in cadence assert "qualifying" in cadence and "push" in cadence assert "manual dispatch" in cadence diff --git a/tests/workflows/test_durable_tracker_liveness.py b/tests/workflows/test_durable_tracker_liveness.py new file mode 100644 index 0000000000..780244dad8 --- /dev/null +++ b/tests/workflows/test_durable_tracker_liveness.py @@ -0,0 +1,179 @@ +"""Offline gate: every durable tracker workflow has a Health 71 liveness assertion.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import yaml +from scripts import check_durable_tracker_liveness +from scripts.check_durable_tracker_liveness import tracker_doc_workflows + +TRACKER_DOC = Path("docs/ops/DURABLE_TRACKING_ISSUES.md") +LIVENESS_CONFIG = Path("config/durable_tracker_liveness.yml") +HEALTH_71 = Path(".github/workflows/health-71-sync-health-check.yml") + +EXPECTED_TRACKERS = { + "agents-weekly-metrics.yml": (2211, 192), + "maint-82-sync-dependency-campaign.yml": (1836, 1), + "health-83-dependency-sync-efficiency.yml": (2897, 192), + "health-68-consumer-sync-drift.yml": (2210, 48), + "maint-69-sync-integration-repo.yml": (2470, None), + "maint-80-langsmith-metrics-dashboard.yml": (2415, 192), +} + + +def _configured_workflows() -> set[str]: + config = yaml.safe_load(LIVENESS_CONFIG.read_text(encoding="utf-8")) + return {str(entry["workflow"]) for entry in config["trackers"]} + + +def test_every_tracked_workflow_has_a_liveness_assertion() -> None: + documented = tracker_doc_workflows() + configured = _configured_workflows() + missing = sorted(documented - configured) + extra = sorted(configured - documented) + assert not missing, f"tracker table workflows missing liveness config: {missing}" + assert not extra, f"liveness config has undocumented workflows: {extra}" + + +def test_liveness_config_has_literal_expected_tracker_contract() -> None: + config = yaml.safe_load(LIVENESS_CONFIG.read_text(encoding="utf-8")) + observed = { + entry["workflow"]: (int(entry["issue"]), entry.get("max_age_hours")) + for entry in config["trackers"] + } + assert observed == EXPECTED_TRACKERS + + +def test_evaluate_trackers_classifies_event_driven_recent_stale_and_absent_runs( + monkeypatch, +) -> None: + trackers = [ + {"workflow": "event.yml", "issue": 1, "event_driven": True}, + {"workflow": "recent.yml", "issue": 2, "max_age_hours": 24}, + {"workflow": "stale.yml", "issue": 3, "max_age_hours": 24}, + {"workflow": "absent.yml", "issue": 4, "max_age_hours": 24}, + ] + runs = { + "recent.yml": { + "conclusion": "success", + "created_at": "recent", + "html_url": "https://run/recent", + }, + "stale.yml": { + "conclusion": "failure", + "created_at": "stale", + "html_url": "https://run/stale", + }, + "absent.yml": None, + } + monkeypatch.setattr(check_durable_tracker_liveness, "_load_config", lambda: trackers) + monkeypatch.setattr( + check_durable_tracker_liveness, + "_latest_executable_run", + lambda _repo, workflow, _token: runs[workflow], + ) + monkeypatch.setattr( + check_durable_tracker_liveness, + "_hours_since", + lambda created: 1.0 if created == "recent" else 25.0, + ) + + assert check_durable_tracker_liveness.evaluate_trackers("stranske/Workflows", "token") == [ + { + "workflow": "event.yml", + "issue": 1, + "healthy": True, + "reason": "event-driven workflow excluded from age-based liveness", + }, + { + "workflow": "recent.yml", + "issue": 2, + "healthy": True, + "latest_conclusion": "success", + "latest_created_at": "recent", + "hours_since": 1.0, + "max_age_hours": 24.0, + "run_url": "https://run/recent", + }, + { + "workflow": "stale.yml", + "issue": 3, + "healthy": False, + "latest_conclusion": "failure", + "latest_created_at": "stale", + "hours_since": 25.0, + "max_age_hours": 24.0, + "run_url": "https://run/stale", + }, + { + "workflow": "absent.yml", + "issue": 4, + "healthy": False, + "reason": "no executable run found (only action_required/skipped)", + }, + ] + + +def test_health_71_invokes_durable_tracker_liveness_check() -> None: + text = HEALTH_71.read_text(encoding="utf-8") + assert "check_durable_tracker_liveness.py" in text + assert "--comment-on-failure" in text + + +def test_event_driven_tracker_is_not_subject_to_age_liveness() -> None: + config = yaml.safe_load(LIVENESS_CONFIG.read_text(encoding="utf-8")) + tracker = next( + entry + for entry in config["trackers"] + if entry["workflow"] == "maint-69-sync-integration-repo.yml" + ) + + assert tracker["event_driven"] is True + assert "max_age_hours" not in tracker + + +def test_tracker_run_lookup_uses_get_for_query_parameters(monkeypatch) -> None: + captured: list[str] = [] + + def fake_check_output(command, **_kwargs): + captured.extend(command) + return "[]" + + monkeypatch.setattr( + check_durable_tracker_liveness.subprocess, "check_output", fake_check_output + ) + + assert ( + check_durable_tracker_liveness._latest_executable_run( + "stranske/Workflows", "health-68-consumer-sync-drift.yml", "token" + ) + is None + ) + method_index = captured.index("--method") + assert captured[method_index + 1] == "GET" + + +def test_tracker_doc_table_links_match_config() -> None: + text = TRACKER_DOC.read_text(encoding="utf-8") + section_match = re.search( + r"## Current durable trackers\n\n\| Issue.*?\n\|[-| ]+\n(.*?)(?:\n\n|\n### )", + text, + re.DOTALL, + ) + assert section_match + issue_to_workflow: dict[int, str] = {} + for row in section_match.group(1).splitlines(): + issue_match = re.search(r"\[#(\d+)\]", row) + workflow_match = re.search(r"/([^/)]+?\.yml)\)", row) + if issue_match and workflow_match: + issue_to_workflow[int(issue_match.group(1))] = workflow_match.group(1) + config = yaml.safe_load(LIVENESS_CONFIG.read_text(encoding="utf-8")) + for entry in config["trackers"]: + issue = int(entry["issue"]) + workflow = str(entry["workflow"]) + assert issue_to_workflow.get(issue) == workflow, ( + f"config issue #{issue} maps to {workflow}, " + f"tracker doc maps to {issue_to_workflow.get(issue)}" + ) diff --git a/tests/workflows/test_health_68_liveness.py b/tests/workflows/test_health_68_liveness.py new file mode 100644 index 0000000000..dfd79bd74f --- /dev/null +++ b/tests/workflows/test_health_68_liveness.py @@ -0,0 +1,131 @@ +"""Offline gates for Health 68 trigger and execution liveness.""" + +from __future__ import annotations + +import os +import re +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +import pytest +import yaml + +WORKFLOW = Path(".github/workflows/health-68-consumer-sync-drift.yml") +TRACKER_DOC = Path("docs/ops/DURABLE_TRACKING_ISSUES.md") +LIVENESS_CONFIG = Path("config/durable_tracker_liveness.yml") + + +def _workflow_triggers() -> dict: + data = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + return data.get(True) or data.get("on") or {} + + +def test_consumer_drift_detector_has_a_schedule() -> None: + triggers = _workflow_triggers() + assert "schedule" in triggers, "Health 68 must declare a schedule trigger for self-healing" + + +def test_consumer_drift_detector_debounces_workflow_run() -> None: + text = WORKFLOW.read_text(encoding="utf-8") + assert "Debounce workflow_run fan-out" in text + assert "github.event_name == 'workflow_run'" in text + assert "| jq -r --argjson current" in text + assert '.conclusion == "cancelled"' not in text + assert 'branch: "main"' in text or "branch: 'main'" in text + + +def test_consumer_drift_debounce_filters_main_before_ordering() -> None: + """Non-main runs must not suppress workflow_run fan-out for main.""" + runs = [ + { + "id": 2, + "head_branch": "feature/test", + "conclusion": "success", + "created_at": "2026-08-23T07:20:00Z", + }, + { + "id": 1, + "head_branch": "main", + "conclusion": "success", + "created_at": "2026-08-23T07:00:00Z", + }, + ] + + def selected(values: list[dict]) -> int | None: + eligible = [ + run for run in values if run["conclusion"] in {"success", "failure", "timed_out"} + ] + return eligible[0]["id"] if eligible else None + + assert selected(runs) == 2 + main_only = [run for run in runs if run["head_branch"] == "main"] + assert selected(main_only) == 1 + + +@pytest.mark.skipif( + os.environ.get("RUN_LIVE_HEALTH_68_PROBE") != "1" + or not (os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN")), + reason="RUN_LIVE_HEALTH_68_PROBE=1 and GH_TOKEN or GITHUB_TOKEN required for live Health 68 execution probe", +) +def test_consumer_drift_detector_executed_recently() -> None: + repo = os.environ.get("GITHUB_REPOSITORY", "stranske/Workflows") + token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") + payload = subprocess.check_output( + [ + "gh", + "api", + f"repos/{repo}/actions/workflows/health-68-consumer-sync-drift.yml/runs", + "--paginate", + "-q", + ".workflow_runs[]", + ], + text=True, + env={**os.environ, "GH_TOKEN": token}, + ) + latest_executable: str | None = None + for line in payload.splitlines(): + line = line.strip() + if not line: + continue + import json + + run = json.loads(line) + conclusion = str(run.get("conclusion") or "") + if conclusion not in {"success", "failure", "cancelled", "timed_out"}: + continue + latest_executable = str(run.get("created_at")) + break + + assert latest_executable, "no executable Health 68 run found" + created = datetime.fromisoformat(latest_executable.replace("Z", "+00:00")) + hours = (datetime.now(UTC) - created).total_seconds() / 3600.0 + assert hours <= 48, f"newest executable Health 68 run is {latest_executable} ({hours:.1f}h old)" + + +def test_health_68_issue_publish_job_is_split() -> None: + text = WORKFLOW.read_text(encoding="utf-8") + assert "publish-drift:" in text + assert "check-drift:" in text + assert text.index("publish-drift:") > text.index("check-drift:") + publish_section = text.split("publish-drift:", 1)[1] + assert "issues: write" in publish_section + assert "Create drift issue" in publish_section + + +def test_tracker_doc_lists_daily_schedule_for_health_68() -> None: + text = TRACKER_DOC.read_text(encoding="utf-8") + row = re.search( + r"\| \[#2210\].*?\|.*?\|.*?\| (.*?) \|", + text, + re.DOTALL, + ) + assert row, "missing #2210 cadence row" + cadence = row.group(1) + assert "schedule" in cadence.lower() or "daily" in cadence.lower() + + +def test_liveness_config_includes_health_68() -> None: + config = yaml.safe_load(LIVENESS_CONFIG.read_text(encoding="utf-8")) + workflows = {entry["workflow"] for entry in config["trackers"]} + assert "health-68-consumer-sync-drift.yml" in workflows