From 788aaf5080aa8ba1c31a1bef9742161a92de5585 Mon Sep 17 00:00:00 2001 From: Codex Automation Date: Sun, 23 Aug 2026 01:50:16 -0500 Subject: [PATCH 01/10] fix(health): restore Health 68 execution and durable-tracker liveness Add a daily schedule, debounced workflow_run fan-out, and a split publish job so the drift check can execute without workflow-level issues:write holds. Extend Health 71 with FYI durable-tracker liveness assertions backed by offline gates. Co-authored-by: Cursor --- .../health-68-consumer-sync-drift.yml | 67 ++++++- .../workflows/health-71-sync-health-check.yml | 25 +++ config/durable_tracker_liveness.yml | 21 +++ docs/ops/DURABLE_TRACKING_ISSUES.md | 2 +- scripts/check_durable_tracker_liveness.py | 175 ++++++++++++++++++ .../docs/test_durable_tracker_drift_states.py | 8 +- .../test_durable_tracker_liveness.py | 58 ++++++ tests/workflows/test_health_68_liveness.py | 101 ++++++++++ 8 files changed, 447 insertions(+), 10 deletions(-) create mode 100644 config/durable_tracker_liveness.yml create mode 100644 scripts/check_durable_tracker_liveness.py create mode 100644 tests/workflows/test_durable_tracker_liveness.py create mode 100644 tests/workflows/test_health_68_liveness.py diff --git a/.github/workflows/health-68-consumer-sync-drift.yml b/.github/workflows/health-68-consumer-sync-drift.yml index 642ed81a90..8345fc072e 100644 --- a/.github/workflows/health-68-consumer-sync-drift.yml +++ b/.github/workflows/health-68-consumer-sync-drift.yml @@ -15,6 +15,9 @@ on: workflow_run: workflows: [Merge Sync PRs] types: [completed] + schedule: + # Daily self-healing cadence; workflow_run remains debounced below. + - cron: '15 6 * * *' workflow_dispatch: inputs: repos: @@ -24,10 +27,10 @@ on: permissions: contents: read - issues: write + actions: read concurrency: - group: consumer-sync-drift-${{ github.repository }}-${{ github.ref }} + group: health-68-consumer-sync-drift cancel-in-progress: true jobs: @@ -37,6 +40,8 @@ jobs: github.event_name != 'workflow_run' || (github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_branch == 'main') runs-on: ubuntu-latest + outputs: + drift_failed: ${{ steps.compare.outcome == 'failure' }} steps: - name: Checkout Workflows repo uses: actions/checkout@v7 @@ -47,18 +52,47 @@ jobs: secrets: ${{ toJSON(secrets) }} github_token: ${{ github.token }} - + - name: Debounce workflow_run fan-out + id: debounce + if: github.event_name == 'workflow_run' + env: + GH_TOKEN: ${{ github.token }} + DEBOUNCE_MINUTES: '30' + run: | + set -euo pipefail + latest_created="$( + gh api "repos/${GITHUB_REPOSITORY}/actions/workflows/health-68-consumer-sync-drift.yml/runs?per_page=20" \ + --jq '[.workflow_runs[] | select(.id != '"${GITHUB_RUN_ID}"') | select(.conclusion == "success" or .conclusion == "failure" or .conclusion == "cancelled" or .conclusion == "timed_out")][0].created_at // empty']' + ) + if [ -z "$latest_created" ]; then + echo "No prior executable Health 68 run; continuing." + echo "skip=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + latest_epoch="$(date -d "$latest_created" +%s)" + now_epoch="$(date -u +%s)" + age_minutes=$(( (now_epoch - latest_epoch) / 60 )) + echo "Last executable Health 68 run was ${age_minutes} minutes ago at ${latest_created}." + if [ "$age_minutes" -lt "$DEBOUNCE_MINUTES" ]; then + echo "Skipping debounced workflow_run fan-out (< ${DEBOUNCE_MINUTES} minutes)." + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "skip=false" >> "$GITHUB_OUTPUT" - name: Setup Python + if: steps.debounce.outputs.skip != 'true' uses: actions/setup-python@v7 with: python-version: '3.14' - name: Install dependencies + if: steps.debounce.outputs.skip != 'true' run: pip install pyyaml requests - name: Compare consumer repos to templates id: compare + if: steps.debounce.outputs.skip != 'true' env: CONSUMER_SYNC_DRIFT_REPORT_JSON: artifacts/consumer-sync-drift-report.json INPUT_REPOS: ${{ inputs.repos }} @@ -74,7 +108,7 @@ jobs: --report-json "$CONSUMER_SYNC_DRIFT_REPORT_JSON" - name: Upload drift report - if: ${{ always() }} + if: ${{ always() && steps.debounce.outputs.skip != 'true' }} uses: actions/upload-artifact@v7 with: name: consumer-sync-drift-report @@ -82,8 +116,31 @@ jobs: if-no-files-found: warn retention-days: 14 + publish-drift: + name: Publish actionable drift tracker update + needs: check-drift + if: ${{ needs.check-drift.result == 'failure' }} + runs-on: ubuntu-latest + permissions: + contents: read + issues: write + steps: + - name: Checkout Workflows repo + uses: actions/checkout@v7 + + - name: Setup API client + uses: ./.github/actions/setup-api-client + with: + secrets: ${{ toJSON(secrets) }} + github_token: ${{ github.token }} + + - name: Download drift report + uses: actions/download-artifact@v6 + with: + name: consumer-sync-drift-report + path: artifacts + - name: Create drift issue - if: ${{ failure() }} uses: actions/github-script@v9 with: script: | diff --git a/.github/workflows/health-71-sync-health-check.yml b/.github/workflows/health-71-sync-health-check.yml index f0f1cc34c4..f2ea6707fd 100644 --- a/.github/workflows/health-71-sync-health-check.yml +++ b/.github/workflows/health-71-sync-health-check.yml @@ -281,3 +281,28 @@ jobs: echo "**Hours since last success:** ${{ steps.check.outputs.hours_since_success }}" fi } >> "$GITHUB_STEP_SUMMARY" + + - name: Assert durable tracker workflow liveness + id: tracker_liveness + continue-on-error: true + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + pip install pyyaml + if python3 scripts/check_durable_tracker_liveness.py --comment-on-failure; then + echo "status=healthy" >> "$GITHUB_OUTPUT" + else + echo "status=unhealthy" >> "$GITHUB_OUTPUT" + exit 1 + fi + + - name: Tracker liveness summary + if: always() && steps.tracker_liveness.outcome == 'failure' + run: | + { + echo "## Durable tracker liveness" + echo "" + echo "At least one durable tracker source workflow has no executable run inside its configured cadence." + echo "FYI comments were posted on the affected tracker issue(s); no gate was opened or closed." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/config/durable_tracker_liveness.yml b/config/durable_tracker_liveness.yml new file mode 100644 index 0000000000..34d7463c0a --- /dev/null +++ b/config/durable_tracker_liveness.yml @@ -0,0 +1,21 @@ +# Workflows monitored by Health 71 durable-tracker liveness assertions. +# Each entry must match a row in docs/ops/DURABLE_TRACKING_ISSUES.md. +trackers: + - workflow: agents-weekly-metrics.yml + issue: 2211 + max_age_hours: 192 + - workflow: maint-82-sync-dependency-campaign.yml + issue: 1836 + max_age_hours: 1 + - workflow: health-83-dependency-sync-efficiency.yml + issue: 2897 + max_age_hours: 192 + - workflow: health-68-consumer-sync-drift.yml + issue: 2210 + max_age_hours: 48 + - workflow: maint-69-sync-integration-repo.yml + issue: 2470 + max_age_hours: 168 + - workflow: maint-80-langsmith-metrics-dashboard.yml + issue: 2415 + max_age_hours: 192 diff --git a/docs/ops/DURABLE_TRACKING_ISSUES.md b/docs/ops/DURABLE_TRACKING_ISSUES.md index 13948520ac..467d418a07 100644 --- a/docs/ops/DURABLE_TRACKING_ISSUES.md +++ b/docs/ops/DURABLE_TRACKING_ISSUES.md @@ -38,7 +38,7 @@ failure-notification path. | [#2211](https://github.com/stranske/Workflows/issues/2211) | Agent metrics weekly summary | [`agents-weekly-metrics.yml`](../../.github/workflows/agents-weekly-metrics.yml) | Mondays 06:00 UTC | New comment per run | | [#1836](https://github.com/stranske/Workflows/issues/1836) | Sync/Dependabot campaign queue | [`maint-82-sync-dependency-campaign.yml`](../../.github/workflows/maint-82-sync-dependency-campaign.yml) + [`.github/scripts/sync_dependency_campaign.js`](../../.github/scripts/sync_dependency_campaign.js) | Every 10 minutes + Mondays 10:30 UTC | Body rewritten in place; due transient delivery lanes are serialized, while actionable delivery blockers do not suppress corrective candidate work | | [#2897](https://github.com/stranske/Workflows/issues/2897) | Dependency/sync maintenance-efficiency advisory report | [`health-83-dependency-sync-efficiency.yml`](../../.github/workflows/health-83-dependency-sync-efficiency.yml) | Weekly Mondays 11:15 UTC + `workflow_dispatch` | Comment appended only on material-evidence fingerprint change | -| [#2210](https://github.com/stranske/Workflows/issues/2210) | 🔄 Consumer repo drift detected | [`health-68-consumer-sync-drift.yml`](../../.github/workflows/health-68-consumer-sync-drift.yml) | After each successful main-branch `Merge Sync PRs` run, qualifying manifest/template/script/tool pushes, or manual dispatch | Body rewritten in place, but only for actionable states | +| [#2210](https://github.com/stranske/Workflows/issues/2210) | 🔄 Consumer repo drift detected | [`health-68-consumer-sync-drift.yml`](../../.github/workflows/health-68-consumer-sync-drift.yml) | Daily 06:15 UTC schedule, debounced successful main-branch `Merge Sync PRs` runs, qualifying manifest/template/script/tool pushes, or manual dispatch | Body rewritten in place, but only for actionable states | | [#2470](https://github.com/stranske/Workflows/issues/2470) | 🚨 Integration-Tests Sync Failed - Action Required | [`maint-69-sync-integration-repo.yml`](../../.github/workflows/maint-69-sync-integration-repo.yml) | On qualifying template/config pushes or manual dispatch | Stuck-window marker in body + recovery comment | | [#2415](https://github.com/stranske/Workflows/issues/2415) | 📊 LangSmith Trace Coverage Dashboard | [`maint-80-langsmith-metrics-dashboard.yml`](../../.github/workflows/maint-80-langsmith-metrics-dashboard.yml) | Mondays 09:00 UTC + manual dispatch | Body rewritten in place | diff --git a/scripts/check_durable_tracker_liveness.py b/scripts/check_durable_tracker_liveness.py new file mode 100644 index 0000000000..0b5fb3e696 --- /dev/null +++ b/scripts/check_durable_tracker_liveness.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""Assert durable tracker source workflows executed inside their cadence.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[1] +CONFIG_PATH = REPO_ROOT / "config" / "durable_tracker_liveness.yml" +TRACKER_DOC = REPO_ROOT / "docs" / "ops" / "DURABLE_TRACKING_ISSUES.md" +EXECUTABLE_CONCLUSIONS = frozenset({"success", "failure", "cancelled", "timed_out"}) + + +def _github_token() -> str: + token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") + if token: + return token + return subprocess.check_output(["gh", "auth", "token"], text=True).strip() + + +def _load_config() -> list[dict[str, Any]]: + data = yaml.safe_load(CONFIG_PATH.read_text(encoding="utf-8")) + trackers = data.get("trackers") + if not isinstance(trackers, list) or not trackers: + raise ValueError(f"{CONFIG_PATH} must define a non-empty trackers list") + return trackers + + +def tracker_doc_workflows() -> set[str]: + import re + + text = TRACKER_DOC.read_text(encoding="utf-8") + section_match = re.search( + r"## Current durable trackers\n\n\| Issue.*?\n\|[-| ]+\n(.*?)(?:\n\n|\n### )", + text, + re.DOTALL, + ) + assert section_match, "DURABLE_TRACKING_ISSUES.md is missing the tracker table" + table_body = section_match.group(1) + workflows = re.findall(r"\[`[^`]+`\]\([^)]*/([^/)]+)\)", table_body) + return {name for name in workflows if name.endswith(".yml")} + + +def _latest_executable_run(repo: str, workflow_file: str, token: str) -> dict[str, Any] | None: + raw = subprocess.check_output( + [ + "gh", + "api", + f"repos/{repo}/actions/workflows/{workflow_file}/runs", + "-f", + "per_page=100", + "--jq", + ".workflow_runs", + ], + text=True, + env={**os.environ, "GH_TOKEN": token}, + ) + runs = json.loads(raw) + if not isinstance(runs, list): + return None + for run in runs: + if not isinstance(run, dict): + continue + conclusion = str(run.get("conclusion") or "") + if conclusion in EXECUTABLE_CONCLUSIONS: + return run + return None + + +def _hours_since(iso_timestamp: str) -> float: + created = datetime.fromisoformat(iso_timestamp.replace("Z", "+00:00")) + return (datetime.now(UTC) - created).total_seconds() / 3600.0 + + +def evaluate_trackers(repo: str, token: str | None = None) -> list[dict[str, Any]]: + auth = token or _github_token() + results: list[dict[str, Any]] = [] + for entry in _load_config(): + workflow = str(entry["workflow"]) + issue = int(entry["issue"]) + max_age_hours = float(entry["max_age_hours"]) + latest = _latest_executable_run(repo, workflow, auth) + if latest is None: + results.append( + { + "workflow": workflow, + "issue": issue, + "healthy": False, + "reason": "no executable run found (only action_required/skipped)", + } + ) + continue + hours = _hours_since(str(latest["created_at"])) + healthy = hours <= max_age_hours + results.append( + { + "workflow": workflow, + "issue": issue, + "healthy": healthy, + "latest_conclusion": latest.get("conclusion"), + "latest_created_at": latest.get("created_at"), + "hours_since": round(hours, 2), + "max_age_hours": max_age_hours, + "run_url": latest.get("html_url"), + } + ) + return results + + +def _comment_on_tracker(repo: str, issue: int, body: str, token: str) -> None: + subprocess.run( + ["gh", "issue", "comment", str(issue), "--repo", repo, "--body", body], + check=True, + env={**os.environ, "GH_TOKEN": token}, + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY", "stranske/Workflows")) + parser.add_argument("--comment-on-failure", action="store_true") + parser.add_argument("--json", action="store_true") + args = parser.parse_args(argv) + + configured = {str(entry["workflow"]) for entry in _load_config()} + documented = tracker_doc_workflows() + missing_from_config = sorted(documented - configured) + extra_in_config = sorted(configured - documented) + if missing_from_config or extra_in_config: + print( + "Tracker workflow coverage mismatch: " + f"missing_from_config={missing_from_config} extra_in_config={extra_in_config}", + file=sys.stderr, + ) + return 2 + + token = _github_token() + results = evaluate_trackers(args.repo, token) + unhealthy = [item for item in results if not item.get("healthy")] + + if args.json: + print(json.dumps({"results": results, "unhealthy_count": len(unhealthy)}, indent=2)) + else: + for item in results: + status = "healthy" if item.get("healthy") else "UNHEALTHY" + print(f"{item['workflow']}\t{status}\t{item}") + + if args.comment_on_failure and unhealthy: + for item in unhealthy: + body = ( + "## Durable tracker liveness alert\n\n" + f"Source workflow `{item['workflow']}` has no executable run inside its " + f"{item.get('max_age_hours', '?')}h cadence.\n\n" + f"- Latest executable run: {item.get('latest_created_at', 'none')}\n" + f"- Conclusion: {item.get('latest_conclusion', 'n/a')}\n" + f"- Hours since: {item.get('hours_since', 'n/a')}\n" + f"- Run URL: {item.get('run_url', 'n/a')}\n\n" + "Confirm liveness from workflow run history, not tracker comment activity." + ) + _comment_on_tracker(args.repo, int(item["issue"]), body, token) + + return 1 if unhealthy else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/docs/test_durable_tracker_drift_states.py b/tests/docs/test_durable_tracker_drift_states.py index b99f43957f..409c4c97ac 100644 --- a/tests/docs/test_durable_tracker_drift_states.py +++ b/tests/docs/test_durable_tracker_drift_states.py @@ -56,17 +56,17 @@ def test_tracker_doc_documents_every_state_the_checker_emits() -> None: ) -def test_tracker_doc_does_not_promise_a_cron_health_68_no_longer_has() -> None: +def test_tracker_doc_documents_daily_schedule_for_health_68() -> None: text = TRACKER_DOC.read_text(encoding="utf-8") workflow = HEALTH_68.read_text(encoding="utf-8") row = _ROW_2210.search(text) assert row, "missing #2210 cadence row in the durable tracker table" cadence = row.group(1) - assert "schedule:" not in workflow, "Health 68 regained a cron; update the tracker doc row" - assert "05:10" not in cadence and "cron" not in cadence.lower() + assert "schedule:" in workflow, "Health 68 must declare a daily schedule trigger" + assert "daily" in cadence.lower() or "06:15" in cadence assert "Merge Sync PRs" in cadence - assert "successful main-branch" in cadence + assert "successful main-branch" in cadence or "debounced" in cadence assert "qualifying" in cadence and "push" in cadence assert "manual dispatch" in cadence diff --git a/tests/workflows/test_durable_tracker_liveness.py b/tests/workflows/test_durable_tracker_liveness.py new file mode 100644 index 0000000000..bd2087ebe0 --- /dev/null +++ b/tests/workflows/test_durable_tracker_liveness.py @@ -0,0 +1,58 @@ +"""Offline gate: every durable tracker workflow has a Health 71 liveness assertion.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import yaml + +from scripts.check_durable_tracker_liveness import tracker_doc_workflows + +TRACKER_DOC = Path("docs/ops/DURABLE_TRACKING_ISSUES.md") +LIVENESS_CONFIG = Path("config/durable_tracker_liveness.yml") +HEALTH_71 = Path(".github/workflows/health-71-sync-health-check.yml") + + +def _configured_workflows() -> set[str]: + config = yaml.safe_load(LIVENESS_CONFIG.read_text(encoding="utf-8")) + return {str(entry["workflow"]) for entry in config["trackers"]} + + +def test_every_tracked_workflow_has_a_liveness_assertion() -> None: + documented = tracker_doc_workflows() + configured = _configured_workflows() + missing = sorted(documented - configured) + extra = sorted(configured - documented) + assert not missing, f"tracker table workflows missing liveness config: {missing}" + assert not extra, f"liveness config has undocumented workflows: {extra}" + + +def test_health_71_invokes_durable_tracker_liveness_check() -> None: + text = HEALTH_71.read_text(encoding="utf-8") + assert "check_durable_tracker_liveness.py" in text + assert "--comment-on-failure" in text + + +def test_tracker_doc_table_links_match_config() -> None: + text = TRACKER_DOC.read_text(encoding="utf-8") + section_match = re.search( + r"## Current durable trackers\n\n\| Issue.*?\n\|[-| ]+\n(.*?)(?:\n\n|\n### )", + text, + re.DOTALL, + ) + assert section_match + issue_to_workflow: dict[int, str] = {} + for row in section_match.group(1).splitlines(): + issue_match = re.search(r"\[#(\d+)\]", row) + workflow_match = re.search(r"/([^/)]+?\.yml)\)", row) + if issue_match and workflow_match: + issue_to_workflow[int(issue_match.group(1))] = workflow_match.group(1) + config = yaml.safe_load(LIVENESS_CONFIG.read_text(encoding="utf-8")) + for entry in config["trackers"]: + issue = int(entry["issue"]) + workflow = str(entry["workflow"]) + assert issue_to_workflow.get(issue) == workflow, ( + f"config issue #{issue} maps to {workflow}, " + f"tracker doc maps to {issue_to_workflow.get(issue)}" + ) diff --git a/tests/workflows/test_health_68_liveness.py b/tests/workflows/test_health_68_liveness.py new file mode 100644 index 0000000000..616bf64869 --- /dev/null +++ b/tests/workflows/test_health_68_liveness.py @@ -0,0 +1,101 @@ +"""Offline gates for Health 68 trigger and execution liveness.""" + +from __future__ import annotations + +import os +import re +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +import pytest +import yaml + +WORKFLOW = Path(".github/workflows/health-68-consumer-sync-drift.yml") +TRACKER_DOC = Path("docs/ops/DURABLE_TRACKING_ISSUES.md") +LIVENESS_CONFIG = Path("config/durable_tracker_liveness.yml") + + +def _workflow_triggers() -> dict: + data = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + return data.get(True) or data.get("on") or {} + + +def test_consumer_drift_detector_has_a_schedule() -> None: + triggers = _workflow_triggers() + assert "schedule" in triggers, "Health 68 must declare a schedule trigger for self-healing" + + +def test_consumer_drift_detector_debounces_workflow_run() -> None: + text = WORKFLOW.read_text(encoding="utf-8") + assert "Debounce workflow_run fan-out" in text + assert "github.event_name == 'workflow_run'" in text + + +@pytest.mark.skipif( + not (os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN")), + reason="GH_TOKEN or GITHUB_TOKEN required for live Health 68 execution probe", +) +def test_consumer_drift_detector_executed_recently() -> None: + repo = os.environ.get("GITHUB_REPOSITORY", "stranske/Workflows") + token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") + payload = subprocess.check_output( + [ + "gh", + "api", + f"repos/{repo}/actions/workflows/health-68-consumer-sync-drift.yml/runs", + "--paginate", + "-q", + ".workflow_runs[]", + ], + text=True, + env={**os.environ, "GH_TOKEN": token}, + ) + latest_executable: str | None = None + for line in payload.splitlines(): + line = line.strip() + if not line: + continue + import json + + run = json.loads(line) + conclusion = str(run.get("conclusion") or "") + if conclusion not in {"success", "failure", "cancelled", "timed_out"}: + continue + latest_executable = str(run.get("created_at")) + break + + assert latest_executable, "no executable Health 68 run found" + created = datetime.fromisoformat(latest_executable.replace("Z", "+00:00")) + hours = (datetime.now(UTC) - created).total_seconds() / 3600.0 + assert hours <= 48, ( + f"newest executable Health 68 run is {latest_executable} ({hours:.1f}h old)" + ) + + +def test_health_68_issue_publish_job_is_split() -> None: + text = WORKFLOW.read_text(encoding="utf-8") + assert "publish-drift:" in text + assert "check-drift:" in text + assert text.index("publish-drift:") > text.index("check-drift:") + publish_section = text.split("publish-drift:", 1)[1] + assert "issues: write" in publish_section + assert "Create drift issue" in publish_section + + +def test_tracker_doc_lists_daily_schedule_for_health_68() -> None: + text = TRACKER_DOC.read_text(encoding="utf-8") + row = re.search( + r"\| \[#2210\].*?\|.*?\|.*?\| (.*?) \|", + text, + re.DOTALL, + ) + assert row, "missing #2210 cadence row" + cadence = row.group(1) + assert "schedule" in cadence.lower() or "daily" in cadence.lower() + + +def test_liveness_config_includes_health_68() -> None: + config = yaml.safe_load(LIVENESS_CONFIG.read_text(encoding="utf-8")) + workflows = {entry["workflow"] for entry in config["trackers"]} + assert "health-68-consumer-sync-drift.yml" in workflows From 804fe1aac3a4eb1033e651332aaf9ee33357e69c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 23 Aug 2026 06:59:43 +0000 Subject: [PATCH 02/10] chore(autofix): formatting/lint --- tests/workflows/test_durable_tracker_liveness.py | 1 - tests/workflows/test_health_68_liveness.py | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/workflows/test_durable_tracker_liveness.py b/tests/workflows/test_durable_tracker_liveness.py index bd2087ebe0..82ca067cc6 100644 --- a/tests/workflows/test_durable_tracker_liveness.py +++ b/tests/workflows/test_durable_tracker_liveness.py @@ -6,7 +6,6 @@ from pathlib import Path import yaml - from scripts.check_durable_tracker_liveness import tracker_doc_workflows TRACKER_DOC = Path("docs/ops/DURABLE_TRACKING_ISSUES.md") diff --git a/tests/workflows/test_health_68_liveness.py b/tests/workflows/test_health_68_liveness.py index 616bf64869..0d6f520884 100644 --- a/tests/workflows/test_health_68_liveness.py +++ b/tests/workflows/test_health_68_liveness.py @@ -68,9 +68,7 @@ def test_consumer_drift_detector_executed_recently() -> None: assert latest_executable, "no executable Health 68 run found" created = datetime.fromisoformat(latest_executable.replace("Z", "+00:00")) hours = (datetime.now(UTC) - created).total_seconds() / 3600.0 - assert hours <= 48, ( - f"newest executable Health 68 run is {latest_executable} ({hours:.1f}h old)" - ) + assert hours <= 48, f"newest executable Health 68 run is {latest_executable} ({hours:.1f}h old)" def test_health_68_issue_publish_job_is_split() -> None: From 3394369e02e04e4c6a5775cfc7ffa594a25653cb Mon Sep 17 00:00:00 2001 From: Codex Automation Date: Sun, 23 Aug 2026 02:11:36 -0500 Subject: [PATCH 03/10] fix(health): harden drift liveness checks --- .../health-68-consumer-sync-drift.yml | 2 +- config/durable_tracker_liveness.yml | 2 +- scripts/check_durable_tracker_liveness.py | 12 +++++++ .../test_durable_tracker_liveness.py | 32 +++++++++++++++++++ tests/workflows/test_health_68_liveness.py | 2 ++ 5 files changed, 48 insertions(+), 2 deletions(-) diff --git a/.github/workflows/health-68-consumer-sync-drift.yml b/.github/workflows/health-68-consumer-sync-drift.yml index 8345fc072e..ef4dd2a371 100644 --- a/.github/workflows/health-68-consumer-sync-drift.yml +++ b/.github/workflows/health-68-consumer-sync-drift.yml @@ -62,7 +62,7 @@ jobs: set -euo pipefail latest_created="$( gh api "repos/${GITHUB_REPOSITORY}/actions/workflows/health-68-consumer-sync-drift.yml/runs?per_page=20" \ - --jq '[.workflow_runs[] | select(.id != '"${GITHUB_RUN_ID}"') | select(.conclusion == "success" or .conclusion == "failure" or .conclusion == "cancelled" or .conclusion == "timed_out")][0].created_at // empty']' + --jq '[.workflow_runs[] | select(.id != '"${GITHUB_RUN_ID}"') | select(.conclusion == "success" or .conclusion == "failure" or .conclusion == "timed_out")][0].created_at // empty' ) if [ -z "$latest_created" ]; then echo "No prior executable Health 68 run; continuing." diff --git a/config/durable_tracker_liveness.yml b/config/durable_tracker_liveness.yml index 34d7463c0a..d22d341469 100644 --- a/config/durable_tracker_liveness.yml +++ b/config/durable_tracker_liveness.yml @@ -15,7 +15,7 @@ trackers: max_age_hours: 48 - workflow: maint-69-sync-integration-repo.yml issue: 2470 - max_age_hours: 168 + event_driven: true - workflow: maint-80-langsmith-metrics-dashboard.yml issue: 2415 max_age_hours: 192 diff --git a/scripts/check_durable_tracker_liveness.py b/scripts/check_durable_tracker_liveness.py index 0b5fb3e696..5603dcfc89 100644 --- a/scripts/check_durable_tracker_liveness.py +++ b/scripts/check_durable_tracker_liveness.py @@ -56,6 +56,8 @@ def _latest_executable_run(repo: str, workflow_file: str, token: str) -> dict[st "gh", "api", f"repos/{repo}/actions/workflows/{workflow_file}/runs", + "--method", + "GET", "-f", "per_page=100", "--jq", @@ -87,6 +89,16 @@ def evaluate_trackers(repo: str, token: str | None = None) -> list[dict[str, Any for entry in _load_config(): workflow = str(entry["workflow"]) issue = int(entry["issue"]) + if entry.get("event_driven") is True: + results.append( + { + "workflow": workflow, + "issue": issue, + "healthy": True, + "reason": "event-driven workflow excluded from age-based liveness", + } + ) + continue max_age_hours = float(entry["max_age_hours"]) latest = _latest_executable_run(repo, workflow, auth) if latest is None: diff --git a/tests/workflows/test_durable_tracker_liveness.py b/tests/workflows/test_durable_tracker_liveness.py index 82ca067cc6..a6c2f8a309 100644 --- a/tests/workflows/test_durable_tracker_liveness.py +++ b/tests/workflows/test_durable_tracker_liveness.py @@ -6,6 +6,7 @@ from pathlib import Path import yaml +from scripts import check_durable_tracker_liveness from scripts.check_durable_tracker_liveness import tracker_doc_workflows TRACKER_DOC = Path("docs/ops/DURABLE_TRACKING_ISSUES.md") @@ -33,6 +34,37 @@ def test_health_71_invokes_durable_tracker_liveness_check() -> None: assert "--comment-on-failure" in text +def test_event_driven_tracker_is_not_subject_to_age_liveness() -> None: + config = yaml.safe_load(LIVENESS_CONFIG.read_text(encoding="utf-8")) + tracker = next( + entry + for entry in config["trackers"] + if entry["workflow"] == "maint-69-sync-integration-repo.yml" + ) + + assert tracker["event_driven"] is True + assert "max_age_hours" not in tracker + + +def test_tracker_run_lookup_uses_get_for_query_parameters(monkeypatch) -> None: + captured: list[str] = [] + + def fake_check_output(command, **_kwargs): + captured.extend(command) + return "[]" + + monkeypatch.setattr(check_durable_tracker_liveness.subprocess, "check_output", fake_check_output) + + assert ( + check_durable_tracker_liveness._latest_executable_run( + "stranske/Workflows", "health-68-consumer-sync-drift.yml", "token" + ) + is None + ) + method_index = captured.index("--method") + assert captured[method_index + 1] == "GET" + + def test_tracker_doc_table_links_match_config() -> None: text = TRACKER_DOC.read_text(encoding="utf-8") section_match = re.search( diff --git a/tests/workflows/test_health_68_liveness.py b/tests/workflows/test_health_68_liveness.py index 0d6f520884..51d9030ae8 100644 --- a/tests/workflows/test_health_68_liveness.py +++ b/tests/workflows/test_health_68_liveness.py @@ -30,6 +30,8 @@ def test_consumer_drift_detector_debounces_workflow_run() -> None: text = WORKFLOW.read_text(encoding="utf-8") assert "Debounce workflow_run fan-out" in text assert "github.event_name == 'workflow_run'" in text + assert "--jq '[.workflow_runs[]" in text + assert ".conclusion == \"cancelled\"" not in text @pytest.mark.skipif( From a670af52a949a0365f506aa406c196891efc6fbc Mon Sep 17 00:00:00 2001 From: Codex Automation Date: Sun, 23 Aug 2026 02:16:09 -0500 Subject: [PATCH 04/10] fix(health): lint debounce workflow --- .github/workflows/health-68-consumer-sync-drift.yml | 5 +++-- docs/WORKFLOW_GUIDE.md | 2 +- tests/workflows/test_health_68_liveness.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/health-68-consumer-sync-drift.yml b/.github/workflows/health-68-consumer-sync-drift.yml index ef4dd2a371..aa90b4080b 100644 --- a/.github/workflows/health-68-consumer-sync-drift.yml +++ b/.github/workflows/health-68-consumer-sync-drift.yml @@ -62,8 +62,9 @@ jobs: set -euo pipefail latest_created="$( gh api "repos/${GITHUB_REPOSITORY}/actions/workflows/health-68-consumer-sync-drift.yml/runs?per_page=20" \ - --jq '[.workflow_runs[] | select(.id != '"${GITHUB_RUN_ID}"') | select(.conclusion == "success" or .conclusion == "failure" or .conclusion == "timed_out")][0].created_at // empty' - ) + | jq -r --argjson current "$GITHUB_RUN_ID" \ + '[.workflow_runs[] | select(.id != $current) | select(.conclusion == "success" or .conclusion == "failure" or .conclusion == "timed_out")][0].created_at // empty' + )" if [ -z "$latest_created" ]; then echo "No prior executable Health 68 run; continuing." echo "skip=false" >> "$GITHUB_OUTPUT" diff --git a/docs/WORKFLOW_GUIDE.md b/docs/WORKFLOW_GUIDE.md index 47e544c77b..5f51ba37fd 100644 --- a/docs/WORKFLOW_GUIDE.md +++ b/docs/WORKFLOW_GUIDE.md @@ -78,7 +78,7 @@ _Inline Gate helper_ - **`health-44-gate-branch-protection.yml`** — Enforces branch-protection policy via `tools/enforce_gate_branch_protection.py` when the PAT is configured (now triggered on PRs or by the consolidated sweep) and skips minting extra GitHub App tokens because enforcement already runs with the configured `BRANCH_PROTECTION_TOKEN`. - **`health-50-security-scan.yml`** — Security scanning workflow triggered on push, PR, and schedule. Runs CodeQL vulnerability scans using the configured PAT priority list without minting additional GitHub App tokens. - **`health-67-integration-sync-check.yml`** — Daily + event-driven comparison between `templates/integration-repo/` and the Workflows-Integration-Tests repo. Manual dispatchers can now toggle the CI/versions/input checks individually to avoid cloning/running sections they don't need while still filing drift issues when enabled checks detect problems. -- **`health-68-consumer-sync-drift.yml`** — Daily + change-triggered drift detector for consumer repos. Uses the shared `scripts/list_registered_consumer_repos.py` helper to build the repo list (also accepts a manual override) before running `scripts/check_consumer_sync_drift.py`, and files/updates the `consumer-sync` issue when inconsistencies surface. +- **`health-68-consumer-sync-drift.yml`** — Daily self-healing drift detector for consumer repos, also triggered by qualifying template/manifest/script pushes, manual dispatch, and debounced successful main-branch `Merge Sync PRs` completions. Uses the shared `scripts/list_registered_consumer_repos.py` helper to build the repo list (also accepts a manual override) before running `scripts/check_consumer_sync_drift.py`, and files/updates the `consumer-sync` issue when inconsistencies surface. - **`health-69-consumer-sync-shadow-evidence.yml`** — Weekly, change-triggered, and manual read-only producer for the typed consumer-sync plan plus a non-authorizing Orchestrator handoff artifact. It has `contents: read` only and cannot mutate consumers or promote the shadow capability. - **`health-71-sync-health-check.yml`** — Daily monitor that inspects the recent `maint-68-sync-consumer-repos` runs. Manual dispatches can tweak `lookback_runs`/`max_age_hours`, and the workflow now reuses `scripts/list_registered_consumer_repos.py` (instead of ad-hoc parsing) without minting an extra GitHub App token. - **`health-72-template-sync.yml`** — Keeps manifest-declared exact template-sync files in sync between Workflows and the consumer template. For PRs from this repo it auto-runs `scripts/sync_templates.sh` + pushes deltas, then validates via `scripts/validate_template_sync.py`, relying only on the installation token. diff --git a/tests/workflows/test_health_68_liveness.py b/tests/workflows/test_health_68_liveness.py index 51d9030ae8..b03f1efcf9 100644 --- a/tests/workflows/test_health_68_liveness.py +++ b/tests/workflows/test_health_68_liveness.py @@ -30,7 +30,7 @@ def test_consumer_drift_detector_debounces_workflow_run() -> None: text = WORKFLOW.read_text(encoding="utf-8") assert "Debounce workflow_run fan-out" in text assert "github.event_name == 'workflow_run'" in text - assert "--jq '[.workflow_runs[]" in text + assert "| jq -r --argjson current" in text assert ".conclusion == \"cancelled\"" not in text From 38f4d655ca3afbef8685ebbaff3da0d38f2e94e2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 23 Aug 2026 07:17:39 +0000 Subject: [PATCH 05/10] chore(autofix): formatting/lint --- tests/workflows/test_durable_tracker_liveness.py | 4 +++- tests/workflows/test_health_68_liveness.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/workflows/test_durable_tracker_liveness.py b/tests/workflows/test_durable_tracker_liveness.py index a6c2f8a309..36a83a9e38 100644 --- a/tests/workflows/test_durable_tracker_liveness.py +++ b/tests/workflows/test_durable_tracker_liveness.py @@ -53,7 +53,9 @@ def fake_check_output(command, **_kwargs): captured.extend(command) return "[]" - monkeypatch.setattr(check_durable_tracker_liveness.subprocess, "check_output", fake_check_output) + monkeypatch.setattr( + check_durable_tracker_liveness.subprocess, "check_output", fake_check_output + ) assert ( check_durable_tracker_liveness._latest_executable_run( diff --git a/tests/workflows/test_health_68_liveness.py b/tests/workflows/test_health_68_liveness.py index b03f1efcf9..ede2fbb9b7 100644 --- a/tests/workflows/test_health_68_liveness.py +++ b/tests/workflows/test_health_68_liveness.py @@ -31,7 +31,7 @@ def test_consumer_drift_detector_debounces_workflow_run() -> None: assert "Debounce workflow_run fan-out" in text assert "github.event_name == 'workflow_run'" in text assert "| jq -r --argjson current" in text - assert ".conclusion == \"cancelled\"" not in text + assert '.conclusion == "cancelled"' not in text @pytest.mark.skipif( From e77b8fcc1710291d024889664ccd096ba3531652 Mon Sep 17 00:00:00 2001 From: Codex Automation Date: Sun, 23 Aug 2026 02:26:27 -0500 Subject: [PATCH 06/10] fix(health): route debounce API through retry client --- .../health-68-consumer-sync-drift.yml | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/.github/workflows/health-68-consumer-sync-drift.yml b/.github/workflows/health-68-consumer-sync-drift.yml index aa90b4080b..f0e154965d 100644 --- a/.github/workflows/health-68-consumer-sync-drift.yml +++ b/.github/workflows/health-68-consumer-sync-drift.yml @@ -61,8 +61,33 @@ jobs: run: | set -euo pipefail latest_created="$( - gh api "repos/${GITHUB_REPOSITORY}/actions/workflows/health-68-consumer-sync-drift.yml/runs?per_page=20" \ - | jq -r --argjson current "$GITHUB_RUN_ID" \ + node -e ' + const { Octokit } = require("@octokit/rest"); + const { createTokenAwareRetry } = require("./.github/scripts/github-api-with-retry.js"); + const core = { info: () => {}, warning: console.warn, debug: () => {} }; + const [owner, repo] = process.env.GITHUB_REPOSITORY.split("/"); + + (async () => { + const github = new Octokit({ auth: process.env.GH_TOKEN || process.env.GITHUB_TOKEN }); + const { withRetry } = await createTokenAwareRetry({ + github, + core, + env: process.env, + task: "health-68-consumer-sync-drift-debounce", + capabilities: ["actions:read"], + }); + const response = await withRetry(() => github.rest.actions.listWorkflowRuns({ + owner, + repo, + workflow_id: "health-68-consumer-sync-drift.yml", + per_page: 20, + })); + process.stdout.write(JSON.stringify(response.data)); + })().catch((error) => { + console.error(error); + process.exit(1); + }); + ' | jq -r --argjson current "$GITHUB_RUN_ID" \ '[.workflow_runs[] | select(.id != $current) | select(.conclusion == "success" or .conclusion == "failure" or .conclusion == "timed_out")][0].created_at // empty' )" if [ -z "$latest_created" ]; then From 37adb2baf11a1b68b3eb8b9d214213656e9ea1c1 Mon Sep 17 00:00:00 2001 From: Codex Automation Date: Sun, 23 Aug 2026 02:29:16 -0500 Subject: [PATCH 07/10] fix(health): preserve drift and liveness outcomes --- .../health-68-consumer-sync-drift.yml | 2 +- .../workflows/health-71-sync-health-check.yml | 33 ++++++++---- .../test_durable_tracker_liveness.py | 50 +++++++++++++++++++ tests/workflows/test_health_68_liveness.py | 5 +- 4 files changed, 78 insertions(+), 12 deletions(-) diff --git a/.github/workflows/health-68-consumer-sync-drift.yml b/.github/workflows/health-68-consumer-sync-drift.yml index f0e154965d..5c60600851 100644 --- a/.github/workflows/health-68-consumer-sync-drift.yml +++ b/.github/workflows/health-68-consumer-sync-drift.yml @@ -145,7 +145,7 @@ jobs: publish-drift: name: Publish actionable drift tracker update needs: check-drift - if: ${{ needs.check-drift.result == 'failure' }} + if: ${{ always() && needs.check-drift.outputs.drift_failed == 'true' }} runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/health-71-sync-health-check.yml b/.github/workflows/health-71-sync-health-check.yml index f2ea6707fd..3d5d4af01f 100644 --- a/.github/workflows/health-71-sync-health-check.yml +++ b/.github/workflows/health-71-sync-health-check.yml @@ -290,19 +290,34 @@ jobs: run: | set -euo pipefail pip install pyyaml - if python3 scripts/check_durable_tracker_liveness.py --comment-on-failure; then - echo "status=healthy" >> "$GITHUB_OUTPUT" - else - echo "status=unhealthy" >> "$GITHUB_OUTPUT" - exit 1 - fi + set +e + python3 scripts/check_durable_tracker_liveness.py --comment-on-failure + checker_status=$? + set -e + case "$checker_status" in + 0) + echo "status=healthy" >> "$GITHUB_OUTPUT" + ;; + 1) + echo "status=unhealthy" >> "$GITHUB_OUTPUT" + exit 1 + ;; + *) + echo "status=error" >> "$GITHUB_OUTPUT" + exit "$checker_status" + ;; + esac - name: Tracker liveness summary - if: always() && steps.tracker_liveness.outcome == 'failure' + if: always() && steps.tracker_liveness.outputs.status != 'healthy' run: | { echo "## Durable tracker liveness" echo "" - echo "At least one durable tracker source workflow has no executable run inside its configured cadence." - echo "FYI comments were posted on the affected tracker issue(s); no gate was opened or closed." + if [ "${{ steps.tracker_liveness.outputs.status }}" = "unhealthy" ]; then + echo "At least one durable tracker source workflow has no executable run inside its configured cadence." + echo "FYI comments were posted on the affected tracker issue(s); no gate was opened or closed." + else + echo "The tracker-liveness checker encountered a coverage, setup, or runtime error; no liveness disposition was inferred." + fi } >> "$GITHUB_STEP_SUMMARY" diff --git a/tests/workflows/test_durable_tracker_liveness.py b/tests/workflows/test_durable_tracker_liveness.py index 36a83a9e38..6c8a7c44ae 100644 --- a/tests/workflows/test_durable_tracker_liveness.py +++ b/tests/workflows/test_durable_tracker_liveness.py @@ -13,6 +13,15 @@ LIVENESS_CONFIG = Path("config/durable_tracker_liveness.yml") HEALTH_71 = Path(".github/workflows/health-71-sync-health-check.yml") +EXPECTED_TRACKERS = { + "agents-weekly-metrics.yml": (2211, 192), + "maint-82-sync-dependency-campaign.yml": (1836, 1), + "health-83-dependency-sync-efficiency.yml": (2897, 192), + "health-68-consumer-sync-drift.yml": (2210, 48), + "maint-69-sync-integration-repo.yml": (2470, None), + "maint-80-langsmith-metrics-dashboard.yml": (2415, 192), +} + def _configured_workflows() -> set[str]: config = yaml.safe_load(LIVENESS_CONFIG.read_text(encoding="utf-8")) @@ -28,6 +37,47 @@ def test_every_tracked_workflow_has_a_liveness_assertion() -> None: assert not extra, f"liveness config has undocumented workflows: {extra}" +def test_liveness_config_has_literal_expected_tracker_contract() -> None: + config = yaml.safe_load(LIVENESS_CONFIG.read_text(encoding="utf-8")) + observed = { + entry["workflow"]: (int(entry["issue"]), entry.get("max_age_hours")) + for entry in config["trackers"] + } + assert observed == EXPECTED_TRACKERS + + +def test_evaluate_trackers_classifies_event_driven_recent_stale_and_absent_runs(monkeypatch) -> None: + trackers = [ + {"workflow": "event.yml", "issue": 1, "event_driven": True}, + {"workflow": "recent.yml", "issue": 2, "max_age_hours": 24}, + {"workflow": "stale.yml", "issue": 3, "max_age_hours": 24}, + {"workflow": "absent.yml", "issue": 4, "max_age_hours": 24}, + ] + runs = { + "recent.yml": {"conclusion": "success", "created_at": "recent", "html_url": "https://run/recent"}, + "stale.yml": {"conclusion": "failure", "created_at": "stale", "html_url": "https://run/stale"}, + "absent.yml": None, + } + monkeypatch.setattr(check_durable_tracker_liveness, "_load_config", lambda: trackers) + monkeypatch.setattr( + check_durable_tracker_liveness, + "_latest_executable_run", + lambda _repo, workflow, _token: runs[workflow], + ) + monkeypatch.setattr( + check_durable_tracker_liveness, + "_hours_since", + lambda created: 1.0 if created == "recent" else 25.0, + ) + + assert check_durable_tracker_liveness.evaluate_trackers("stranske/Workflows", "token") == [ + {"workflow": "event.yml", "issue": 1, "healthy": True, "reason": "event-driven workflow excluded from age-based liveness"}, + {"workflow": "recent.yml", "issue": 2, "healthy": True, "latest_conclusion": "success", "latest_created_at": "recent", "hours_since": 1.0, "max_age_hours": 24.0, "run_url": "https://run/recent"}, + {"workflow": "stale.yml", "issue": 3, "healthy": False, "latest_conclusion": "failure", "latest_created_at": "stale", "hours_since": 25.0, "max_age_hours": 24.0, "run_url": "https://run/stale"}, + {"workflow": "absent.yml", "issue": 4, "healthy": False, "reason": "no executable run found (only action_required/skipped)"}, + ] + + def test_health_71_invokes_durable_tracker_liveness_check() -> None: text = HEALTH_71.read_text(encoding="utf-8") assert "check_durable_tracker_liveness.py" in text diff --git a/tests/workflows/test_health_68_liveness.py b/tests/workflows/test_health_68_liveness.py index ede2fbb9b7..13d5c2252f 100644 --- a/tests/workflows/test_health_68_liveness.py +++ b/tests/workflows/test_health_68_liveness.py @@ -35,8 +35,9 @@ def test_consumer_drift_detector_debounces_workflow_run() -> None: @pytest.mark.skipif( - not (os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN")), - reason="GH_TOKEN or GITHUB_TOKEN required for live Health 68 execution probe", + os.environ.get("RUN_LIVE_HEALTH_68_PROBE") != "1" + or not (os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN")), + reason="RUN_LIVE_HEALTH_68_PROBE=1 and GH_TOKEN or GITHUB_TOKEN required for live Health 68 execution probe", ) def test_consumer_drift_detector_executed_recently() -> None: repo = os.environ.get("GITHUB_REPOSITORY", "stranske/Workflows") From 0f9effa32bedf9be8aeb7cbc48a710d3e807fdcf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 23 Aug 2026 07:35:59 +0000 Subject: [PATCH 08/10] chore(autofix): formatting/lint --- .../test_durable_tracker_liveness.py | 52 ++++++++++++++++--- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/tests/workflows/test_durable_tracker_liveness.py b/tests/workflows/test_durable_tracker_liveness.py index 6c8a7c44ae..780244dad8 100644 --- a/tests/workflows/test_durable_tracker_liveness.py +++ b/tests/workflows/test_durable_tracker_liveness.py @@ -46,7 +46,9 @@ def test_liveness_config_has_literal_expected_tracker_contract() -> None: assert observed == EXPECTED_TRACKERS -def test_evaluate_trackers_classifies_event_driven_recent_stale_and_absent_runs(monkeypatch) -> None: +def test_evaluate_trackers_classifies_event_driven_recent_stale_and_absent_runs( + monkeypatch, +) -> None: trackers = [ {"workflow": "event.yml", "issue": 1, "event_driven": True}, {"workflow": "recent.yml", "issue": 2, "max_age_hours": 24}, @@ -54,8 +56,16 @@ def test_evaluate_trackers_classifies_event_driven_recent_stale_and_absent_runs( {"workflow": "absent.yml", "issue": 4, "max_age_hours": 24}, ] runs = { - "recent.yml": {"conclusion": "success", "created_at": "recent", "html_url": "https://run/recent"}, - "stale.yml": {"conclusion": "failure", "created_at": "stale", "html_url": "https://run/stale"}, + "recent.yml": { + "conclusion": "success", + "created_at": "recent", + "html_url": "https://run/recent", + }, + "stale.yml": { + "conclusion": "failure", + "created_at": "stale", + "html_url": "https://run/stale", + }, "absent.yml": None, } monkeypatch.setattr(check_durable_tracker_liveness, "_load_config", lambda: trackers) @@ -71,10 +81,38 @@ def test_evaluate_trackers_classifies_event_driven_recent_stale_and_absent_runs( ) assert check_durable_tracker_liveness.evaluate_trackers("stranske/Workflows", "token") == [ - {"workflow": "event.yml", "issue": 1, "healthy": True, "reason": "event-driven workflow excluded from age-based liveness"}, - {"workflow": "recent.yml", "issue": 2, "healthy": True, "latest_conclusion": "success", "latest_created_at": "recent", "hours_since": 1.0, "max_age_hours": 24.0, "run_url": "https://run/recent"}, - {"workflow": "stale.yml", "issue": 3, "healthy": False, "latest_conclusion": "failure", "latest_created_at": "stale", "hours_since": 25.0, "max_age_hours": 24.0, "run_url": "https://run/stale"}, - {"workflow": "absent.yml", "issue": 4, "healthy": False, "reason": "no executable run found (only action_required/skipped)"}, + { + "workflow": "event.yml", + "issue": 1, + "healthy": True, + "reason": "event-driven workflow excluded from age-based liveness", + }, + { + "workflow": "recent.yml", + "issue": 2, + "healthy": True, + "latest_conclusion": "success", + "latest_created_at": "recent", + "hours_since": 1.0, + "max_age_hours": 24.0, + "run_url": "https://run/recent", + }, + { + "workflow": "stale.yml", + "issue": 3, + "healthy": False, + "latest_conclusion": "failure", + "latest_created_at": "stale", + "hours_since": 25.0, + "max_age_hours": 24.0, + "run_url": "https://run/stale", + }, + { + "workflow": "absent.yml", + "issue": 4, + "healthy": False, + "reason": "no executable run found (only action_required/skipped)", + }, ] From 14fc8c1d87b85f50ed66380979eac5a64a9ad244 Mon Sep 17 00:00:00 2001 From: Codex Automation Date: Sun, 23 Aug 2026 02:42:31 -0500 Subject: [PATCH 09/10] fix(health): harden debounce branch filter and workflow security Filter Health 68 debounce evidence to main-branch runs, pin checkout and artifact actions, stop over-provisioning secrets in drift publish, and route tracker summary through env to avoid template injection. Co-authored-by: Cursor --- .../health-68-consumer-sync-drift.yml | 13 +++++--- .../workflows/health-71-sync-health-check.yml | 8 +++-- tests/workflows/test_health_68_liveness.py | 31 +++++++++++++++++++ 3 files changed, 45 insertions(+), 7 deletions(-) diff --git a/.github/workflows/health-68-consumer-sync-drift.yml b/.github/workflows/health-68-consumer-sync-drift.yml index 5c60600851..49de8b9152 100644 --- a/.github/workflows/health-68-consumer-sync-drift.yml +++ b/.github/workflows/health-68-consumer-sync-drift.yml @@ -44,12 +44,13 @@ jobs: drift_failed: ${{ steps.compare.outcome == 'failure' }} steps: - name: Checkout Workflows repo - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Setup API client uses: ./.github/actions/setup-api-client with: - secrets: ${{ toJSON(secrets) }} github_token: ${{ github.token }} - name: Debounce workflow_run fan-out @@ -80,6 +81,7 @@ jobs: owner, repo, workflow_id: "health-68-consumer-sync-drift.yml", + branch: "main", per_page: 20, })); process.stdout.write(JSON.stringify(response.data)); @@ -152,16 +154,17 @@ jobs: issues: write steps: - name: Checkout Workflows repo - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Setup API client uses: ./.github/actions/setup-api-client with: - secrets: ${{ toJSON(secrets) }} github_token: ${{ github.token }} - name: Download drift report - uses: actions/download-artifact@v6 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: consumer-sync-drift-report path: artifacts diff --git a/.github/workflows/health-71-sync-health-check.yml b/.github/workflows/health-71-sync-health-check.yml index 3d5d4af01f..873ba18d9d 100644 --- a/.github/workflows/health-71-sync-health-check.yml +++ b/.github/workflows/health-71-sync-health-check.yml @@ -39,7 +39,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Export load-balancer tokens uses: ./.github/actions/setup-api-client - name: Install GitHub API dependencies @@ -310,11 +312,13 @@ jobs: - name: Tracker liveness summary if: always() && steps.tracker_liveness.outputs.status != 'healthy' + env: + TRACKER_LIVENESS_STATUS: ${{ steps.tracker_liveness.outputs.status }} run: | { echo "## Durable tracker liveness" echo "" - if [ "${{ steps.tracker_liveness.outputs.status }}" = "unhealthy" ]; then + if [ "$TRACKER_LIVENESS_STATUS" = "unhealthy" ]; then echo "At least one durable tracker source workflow has no executable run inside its configured cadence." echo "FYI comments were posted on the affected tracker issue(s); no gate was opened or closed." else diff --git a/tests/workflows/test_health_68_liveness.py b/tests/workflows/test_health_68_liveness.py index 13d5c2252f..27712a6f4a 100644 --- a/tests/workflows/test_health_68_liveness.py +++ b/tests/workflows/test_health_68_liveness.py @@ -32,6 +32,37 @@ def test_consumer_drift_detector_debounces_workflow_run() -> None: assert "github.event_name == 'workflow_run'" in text assert "| jq -r --argjson current" in text assert '.conclusion == "cancelled"' not in text + assert "branch: \"main\"" in text or "branch: 'main'" in text + + +def test_consumer_drift_debounce_filters_main_before_ordering() -> None: + """Non-main runs must not suppress workflow_run fan-out for main.""" + runs = [ + { + "id": 2, + "head_branch": "feature/test", + "conclusion": "success", + "created_at": "2026-08-23T07:20:00Z", + }, + { + "id": 1, + "head_branch": "main", + "conclusion": "success", + "created_at": "2026-08-23T07:00:00Z", + }, + ] + + def selected(values: list[dict]) -> int | None: + eligible = [ + run + for run in values + if run["conclusion"] in {"success", "failure", "timed_out"} + ] + return eligible[0]["id"] if eligible else None + + assert selected(runs) == 2 + main_only = [run for run in runs if run["head_branch"] == "main"] + assert selected(main_only) == 1 @pytest.mark.skipif( From 251c6bb0f35b6a79f838ad0e4743030fd95cfe30 Mon Sep 17 00:00:00 2001 From: Codex Automation Date: Sun, 23 Aug 2026 03:09:01 -0500 Subject: [PATCH 10/10] style(health): format liveness regression test --- tests/workflows/test_health_68_liveness.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/workflows/test_health_68_liveness.py b/tests/workflows/test_health_68_liveness.py index 27712a6f4a..dfd79bd74f 100644 --- a/tests/workflows/test_health_68_liveness.py +++ b/tests/workflows/test_health_68_liveness.py @@ -32,7 +32,7 @@ def test_consumer_drift_detector_debounces_workflow_run() -> None: assert "github.event_name == 'workflow_run'" in text assert "| jq -r --argjson current" in text assert '.conclusion == "cancelled"' not in text - assert "branch: \"main\"" in text or "branch: 'main'" in text + assert 'branch: "main"' in text or "branch: 'main'" in text def test_consumer_drift_debounce_filters_main_before_ordering() -> None: @@ -54,9 +54,7 @@ def test_consumer_drift_debounce_filters_main_before_ordering() -> None: def selected(values: list[dict]) -> int | None: eligible = [ - run - for run in values - if run["conclusion"] in {"success", "failure", "timed_out"} + run for run in values if run["conclusion"] in {"success", "failure", "timed_out"} ] return eligible[0]["id"] if eligible else None