diff --git a/.github/workflows/health-68-consumer-sync-drift.yml b/.github/workflows/health-68-consumer-sync-drift.yml index 23f577787..a9f3635a9 100644 --- a/.github/workflows/health-68-consumer-sync-drift.yml +++ b/.github/workflows/health-68-consumer-sync-drift.yml @@ -43,6 +43,12 @@ jobs: outputs: drift_failed: ${{ steps.compare.outcome == 'failure' }} drift_clean: ${{ steps.compare.outcome == 'success' }} + # Did this run COMPARE, as distinct from merely concluding? A debounced run + # concludes `success` with every comparison step `skipped`, so the job + # conclusion cannot answer that and any oracle reading it counts a no-op as + # evidence of life. Kept beside the two drift outputs so the three are read + # together. + compared: ${{ steps.compare.outcome != 'skipped' }} steps: - name: Checkout Workflows repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -60,14 +66,28 @@ jobs: env: GH_TOKEN: ${{ github.token }} DEBOUNCE_MINUTES: '30' + # The step whose conclusion decides whether a run COMPARED. Must match the + # `name:` of the compare step below, and the `require_step` value in + # config/durable_tracker_liveness.yml. + COMPARE_STEP_NAME: 'Compare consumer repos to templates' + # How many prior runs to probe for that step before giving up and running. + COMPARE_PROBE_LIMIT: '15' run: | set -euo pipefail + # Age is measured from the last run that actually COMPARED, never from the + # last run that merely concluded. The old selector accepted any + # success/failure/timed_out run, which INCLUDED this step's own debounced + # no-ops, so the clock reset on nothing happening and the effective interval + # drifted with trigger volume instead of tracking real comparisons. latest_created="$( node -e ' const { Octokit } = require("@octokit/rest"); const { createTokenAwareRetry } = require("./.github/scripts/github-api-with-retry.js"); const core = { info: () => {}, warning: console.warn, debug: () => {} }; const [owner, repo] = process.env.GITHUB_REPOSITORY.split("/"); + const stepName = process.env.COMPARE_STEP_NAME; + const probeLimit = Number(process.env.COMPARE_PROBE_LIMIT || "15"); + const currentId = String(process.env.GITHUB_RUN_ID || ""); (async () => { const github = new Octokit({ auth: process.env.GH_TOKEN || process.env.GITHUB_TOKEN }); @@ -85,25 +105,46 @@ jobs: branch: "main", per_page: 20, })); - process.stdout.write(JSON.stringify(response.data)); + const executable = new Set(["success", "failure", "timed_out"]); + const candidates = (response.data.workflow_runs || []) + .filter((run) => String(run.id) !== currentId) + .filter((run) => executable.has(String(run.conclusion || ""))) + .slice(0, probeLimit); + for (const run of candidates) { + const jobs = await withRetry(() => github.rest.actions.listJobsForWorkflowRun({ + owner, + repo, + run_id: run.id, + per_page: 100, + })); + const ran = (jobs.data.jobs || []).some((job) => + (job.steps || []).some((step) => + step.name === stepName && step.conclusion !== "skipped" + ) + ); + if (ran) { + process.stdout.write(String(run.created_at || "")); + return; + } + } + process.stdout.write(""); })().catch((error) => { console.error(error); process.exit(1); }); - ' | jq -r --argjson current "$GITHUB_RUN_ID" \ - '[.workflow_runs[] | select(.id != $current) | select(.conclusion == "success" or .conclusion == "failure" or .conclusion == "timed_out")][0].created_at // empty' + ' )" if [ -z "$latest_created" ]; then - echo "No prior executable Health 68 run; continuing." + echo "No prior Health 68 run COMPARED within the probed window; continuing." echo "skip=false" >> "$GITHUB_OUTPUT" exit 0 fi latest_epoch="$(date -d "$latest_created" +%s)" now_epoch="$(date -u +%s)" age_minutes=$(( (now_epoch - latest_epoch) / 60 )) - echo "Last executable Health 68 run was ${age_minutes} minutes ago at ${latest_created}." + echo "Last Health 68 run that COMPARED was ${age_minutes} minutes ago at ${latest_created}." if [ "$age_minutes" -lt "$DEBOUNCE_MINUTES" ]; then - echo "Skipping debounced workflow_run fan-out (< ${DEBOUNCE_MINUTES} minutes)." + echo "Skipping debounced workflow_run fan-out (< ${DEBOUNCE_MINUTES} minutes since the last comparison)." echo "skip=true" >> "$GITHUB_OUTPUT" exit 0 fi diff --git a/config/durable_tracker_liveness.yml b/config/durable_tracker_liveness.yml index dcc5b24f9..45ba9d95b 100644 --- a/config/durable_tracker_liveness.yml +++ b/config/durable_tracker_liveness.yml @@ -23,3 +23,25 @@ trackers: - workflow: health-40-repo-selfcheck.yml issue: 3218 max_age_hours: 192 + +# Workflows whose EXECUTION is monitored but which own no durable tracker issue, so +# they have no row in docs/ops/DURABLE_TRACKING_ISSUES.md's tracker table and are +# excluded from the config-vs-doc coverage equality. They are reported by +# `check_durable_tracker_liveness.py` and count toward its exit code; they are never +# commented on, because there is no durable issue to comment on. +execution_liveness: + # Health 68 concludes `success` while its comparison steps are SKIPPED by the + # workflow_run debounce, so the job conclusion says only that the workflow was + # triggered. `require_step` moves the liveness question from "did a run happen" + # to "did a run compare". Measured 2026-08-24: the seven most recent `success` + # runs all had this step `skipped`; the newest run that actually compared was + # 32774227027, which FAILED. + # + # #2210 is deliberately absent: it is a TRANSIENT alert this workflow opens and + # closes (docs/ops/DURABLE_TRACKING_ISSUES.md, "Distinguishing trackers from + # transient alerts") and it is currently CLOSED. Listing it as a durable tracker + # would be a false machine-readable claim. + - workflow: health-68-consumer-sync-drift.yml + issue: null + max_age_hours: 48 + require_step: Compare consumer repos to templates diff --git a/scripts/check_durable_tracker_liveness.py b/scripts/check_durable_tracker_liveness.py index ecdf457e9..d36da79ff 100644 --- a/scripts/check_durable_tracker_liveness.py +++ b/scripts/check_durable_tracker_liveness.py @@ -33,6 +33,11 @@ TRACKER_DOC = REPO_ROOT / "docs" / "ops" / "DURABLE_TRACKING_ISSUES.md" EXECUTABLE_CONCLUSIONS = frozenset({"success", "failure", "cancelled", "timed_out"}) +# How many job-level-executable runs to probe per page when a tracker configures +# `require_step`. Bounded so a workflow that has been debouncing for weeks costs a +# fixed number of jobs calls rather than one per run in its history. +STEP_PROBE_LIMIT = 20 + def _github_token() -> str: token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") @@ -42,11 +47,30 @@ def _github_token() -> str: def _load_config() -> list[dict[str, Any]]: + """Every monitored workflow: durable trackers first, then execution-only entries. + + `execution_liveness` entries are monitored the same way but own NO durable + tracker issue, so they are excluded from the config-vs-doc coverage equality in + `main()` and are never commented on. Health 68 is the motivating case: its #2210 + is a TRANSIENT alert that the workflow itself opens and closes (see + docs/ops/DURABLE_TRACKING_ISSUES.md, "Distinguishing trackers from transient + alerts"), so listing it under `trackers:` would assert a durable relationship + that does not exist — the same false machine-readable claim #3244 is about. + """ data = yaml.safe_load(CONFIG_PATH.read_text(encoding="utf-8")) trackers = data.get("trackers") if not isinstance(trackers, list) or not trackers: raise ValueError(f"{CONFIG_PATH} must define a non-empty trackers list") - return trackers + entries = [{**entry, "durable": True} for entry in trackers] + execution_only = data.get("execution_liveness") or [] + if not isinstance(execution_only, list): + raise ValueError(f"{CONFIG_PATH} execution_liveness must be a list when present") + entries.extend({**entry, "durable": False} for entry in execution_only) + return entries + + +def _durable_tracker_workflows() -> set[str]: + return {str(entry["workflow"]) for entry in _load_config() if entry.get("durable")} def tracker_doc_workflows() -> set[str]: @@ -64,11 +88,37 @@ def tracker_doc_workflows() -> set[str]: return {name for name in workflows if name.endswith(".yml")} +def run_step_conclusion( + repo: str, + run_id: Any, + step_name: str, + token: str, +) -> str | None: + """Conclusion of ``step_name`` in ``run_id``, or None when the step is absent. + + None means "this run has no such step" — a different fact from "the step ran + and was skipped", which returns ``"skipped"``. Collapsing the two is the + defect this module is being fixed for, one level up. + """ + payload = _gh_api(f"repos/{repo}/actions/runs/{run_id}/jobs?per_page=100", token) + jobs = payload.get("jobs") + if not isinstance(jobs, list): + return None + for job in jobs: + if not isinstance(job, dict): + continue + for step in job.get("steps") or []: + if isinstance(step, dict) and str(step.get("name") or "") == step_name: + return str(step.get("conclusion") or "") + return None + + def _latest_executable_run( repo: str, workflow_file: str, token: str, allowed_events: frozenset[str] | None = None, + require_step: str | None = None, ) -> dict[str, Any] | None: """Newest run of ``workflow_file`` that actually executed, or None. @@ -76,6 +126,13 @@ def _latest_executable_run( failed lookup RAISES (via the wrapper) rather than returning None, because None reads as "no executable run" and would blame the workflow for the checker's own inability to look. + + ``require_step`` narrows "executed" from the JOB conclusion to a named STEP. + A job whose work step was skipped still concludes ``success`` — Health 68's + debounce does exactly that — so without this the newest run is evidence that + the workflow was TRIGGERED, never that it did anything. With it, a run counts + only when the named step reached a conclusion other than ``skipped``. + When ``require_step`` is None the job conclusion is used, unchanged. """ base_path = f"repos/{repo}/actions/workflows/{workflow_file}/runs?per_page=100" events: tuple[str | None, ...] = tuple(sorted(allowed_events)) if allowed_events else (None,) @@ -92,15 +149,25 @@ def _latest_executable_run( runs = payload.get("workflow_runs") if not isinstance(runs, list): break - candidate = next( - ( - run - for run in runs - if isinstance(run, dict) - and str(run.get("conclusion") or "") in EXECUTABLE_CONCLUSIONS - ), - None, - ) + executable = [ + run + for run in runs + if isinstance(run, dict) + and str(run.get("conclusion") or "") in EXECUTABLE_CONCLUSIONS + ] + candidate = None + if require_step is None: + candidate = executable[0] if executable else None + else: + # Newest-first, and only over runs that already cleared the job-level + # filter, so the extra jobs call is paid once per plausible run rather + # than once per run in history. + for run in executable[:STEP_PROBE_LIMIT]: + conclusion = run_step_conclusion(repo, run.get("id"), require_step, token) + if conclusion is not None and conclusion != "skipped": + candidate = dict(run) + candidate["required_step_conclusion"] = conclusion + break if candidate is not None: candidates.append(candidate) break @@ -141,7 +208,8 @@ def evaluate_trackers(repo: str, token: str | None = None) -> list[dict[str, Any results: list[dict[str, Any]] = [] for entry in _load_config(): workflow = str(entry["workflow"]) - issue = int(entry["issue"]) + raw_issue = entry.get("issue") + issue = int(raw_issue) if raw_issue is not None else None if entry.get("event_driven") is True: results.append( { @@ -159,6 +227,8 @@ def evaluate_trackers(repo: str, token: str | None = None) -> list[dict[str, Any if isinstance(configured_events, list) and configured_events else None ) + require_step = entry.get("require_step") + require_step = str(require_step) if require_step else None latest = _latest_executable_run(repo, workflow, auth, allowed_events) if latest is None: results.append( @@ -174,20 +244,52 @@ def evaluate_trackers(repo: str, token: str | None = None) -> list[dict[str, Any ) continue hours = _hours_since(str(latest["created_at"])) - healthy = hours <= max_age_hours - results.append( - { - "workflow": workflow, - "issue": issue, - "healthy": healthy, - "latest_conclusion": latest.get("conclusion"), - "latest_created_at": latest.get("created_at"), - "latest_event": latest.get("event"), - "hours_since": round(hours, 2), - "max_age_hours": max_age_hours, - "run_url": latest.get("html_url"), - } - ) + result: dict[str, Any] = { + "workflow": workflow, + "issue": issue, + "healthy": hours <= max_age_hours, + "latest_conclusion": latest.get("conclusion"), + "latest_created_at": latest.get("created_at"), + "latest_event": latest.get("event"), + "hours_since": round(hours, 2), + "max_age_hours": max_age_hours, + "run_url": latest.get("html_url"), + } + + # THE BLOCKING QUANTITY AND THE DRAINABLE QUANTITY, SIDE BY SIDE. + # `hours_since` alone answers "was this workflow triggered recently", which a + # debounced no-op satisfies forever. `hours_since_executing_run` answers "did + # it DO anything recently", which is the number the tracker actually depends + # on. Reporting only the first is what let seven consecutive comparison-free + # `success` runs read as health. + if require_step is not None: + result["require_step"] = require_step + executed = _latest_executable_run(repo, workflow, auth, allowed_events, require_step) + if executed is None: + # Distinguishable from "no runs at all" above: runs exist, they + # concluded, and not one of them ran the step. Never silently reuse + # `hours_since` here — that would rebuild the very defect this branch + # exists to detect, one level up. + result["latest_executing_created_at"] = None + result["hours_since_executing_run"] = None + result["healthy"] = False + result["reason"] = ( + f"no run in the {STEP_PROBE_LIMIT} newest executable runs ran step " + f"{require_step!r}; every one of them concluded without doing the work, " + f"so the newest run at {latest.get('created_at')} is evidence the workflow " + f"was triggered, not that it executed." + ) + else: + executing_hours = _hours_since(str(executed["created_at"])) + result["latest_executing_created_at"] = executed.get("created_at") + result["latest_executing_conclusion"] = executed.get("conclusion") + result["required_step_conclusion"] = executed.get("required_step_conclusion") + result["hours_since_executing_run"] = round(executing_hours, 2) + result["executing_run_url"] = executed.get("html_url") + # Health is decided by the EXECUTING run. The bare run age stays in the + # payload so a reader can see the gap between the two. + result["healthy"] = executing_hours <= max_age_hours + results.append(result) return results @@ -206,7 +308,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--json", action="store_true") args = parser.parse_args(argv) - configured = {str(entry["workflow"]) for entry in _load_config()} + configured = _durable_tracker_workflows() documented = tracker_doc_workflows() missing_from_config = sorted(documented - configured) extra_in_config = sorted(configured - documented) @@ -231,6 +333,21 @@ def main(argv: list[str] | None = None) -> int: if args.comment_on_failure and unhealthy: for item in unhealthy: + # An execution_liveness entry has no durable tracker to comment on. It is + # still reported and still fails the exit code; it just has no issue. + if item.get("issue") is None: + continue + require_step = item.get("require_step") + executed_lines = "" + if require_step: + executed_lines = ( + f"- Required step: `{require_step}`\n" + f"- Latest run that RAN that step: " + f"{item.get('latest_executing_created_at') or 'none in recent history'}\n" + f"- Hours since that run: {item.get('hours_since_executing_run', 'n/a')}\n" + f"- That run's step conclusion: " + f"{item.get('required_step_conclusion', 'n/a')}\n" + ) body = ( "## Durable tracker liveness alert\n\n" f"Source workflow `{item['workflow']}` has no executable run inside its " @@ -238,8 +355,10 @@ def main(argv: list[str] | None = None) -> int: f"- Latest executable run: {item.get('latest_created_at', 'none')}\n" f"- Conclusion: {item.get('latest_conclusion', 'n/a')}\n" f"- Hours since: {item.get('hours_since', 'n/a')}\n" - f"- Run URL: {item.get('run_url', 'n/a')}\n\n" - "Confirm liveness from workflow run history, not tracker comment activity." + f"- Run URL: {item.get('run_url', 'n/a')}\n" + + executed_lines + + (f"\n{item['reason']}\n" if item.get("reason") else "") + + "\nConfirm liveness from workflow run history, not tracker comment activity." ) _comment_on_tracker(args.repo, int(item["issue"]), body, token) diff --git a/tests/workflows/test_durable_tracker_liveness.py b/tests/workflows/test_durable_tracker_liveness.py index 4d27c3d13..ce6d3505a 100644 --- a/tests/workflows/test_durable_tracker_liveness.py +++ b/tests/workflows/test_durable_tracker_liveness.py @@ -119,6 +119,175 @@ def test_evaluate_trackers_classifies_event_driven_recent_stale_and_absent_runs( ] +def test_liveness_ignores_runs_whose_comparison_step_was_skipped(monkeypatch) -> None: + """A run that concluded without running the work step is not evidence of life. + + Health 68's debounce skips the comparison steps while the JOB still concludes + `success`, so the bare conclusion says only that the workflow was triggered. + Measured live 2026-08-24: the seven newest `success` runs all had this step + `skipped`, and the newest run that actually compared had FAILED four hours + earlier — the oracle called that healthy. + + Two runs go in: a newer `success` whose step was skipped, and an older + `success` whose step ran. The reported age must be measured from the OLDER one. + """ + # THE WIRING, ASSERTED AGAINST THE SHIPPED CONFIG, NOT THE MONKEYPATCHED ONE. + # The behaviour below is exercised through a stub config, so on its own it would + # keep passing after `require_step` was deleted from the real file -- a gate that + # cannot notice its own disconnection. Both halves have to be here. + shipped = yaml.safe_load(LIVENESS_CONFIG.read_text(encoding="utf-8")) + health_68 = next( + entry + for entry in shipped.get("execution_liveness") or [] + if entry["workflow"] == "health-68-consumer-sync-drift.yml" + ) + assert health_68.get("require_step") == "Compare consumer repos to templates", ( + "health-68 has no require_step in the shipped config, so in production the " + "checker is back to reading the bare job conclusion" + ) + + newer = { + "id": 2, + "conclusion": "success", + "created_at": "2026-08-24T22:18:09Z", + "html_url": "https://run/newer", + } + older = { + "id": 1, + "conclusion": "success", + "created_at": "2026-08-24T20:29:59Z", + "html_url": "https://run/older", + } + step_conclusions = {2: "skipped", 1: "success"} + + monkeypatch.setattr( + check_durable_tracker_liveness, + "_load_config", + lambda: [ + { + "workflow": "health-68-consumer-sync-drift.yml", + "issue": None, + "max_age_hours": 48, + "require_step": "Compare consumer repos to templates", + "durable": False, + } + ], + ) + monkeypatch.setattr( + check_durable_tracker_liveness, + "_gh_api", + lambda path, _token: {"workflow_runs": [newer, older]}, + ) + monkeypatch.setattr( + check_durable_tracker_liveness, + "run_step_conclusion", + lambda _repo, run_id, _step, _token: step_conclusions[run_id], + ) + ages = {"2026-08-24T22:18:09Z": 1.0, "2026-08-24T20:29:59Z": 3.0} + monkeypatch.setattr(check_durable_tracker_liveness, "_hours_since", lambda ts: ages[ts]) + + (result,) = check_durable_tracker_liveness.evaluate_trackers("stranske/Workflows", "token") + + # The bare run age still reports the NEWER run -- both numbers are published. + assert result["latest_created_at"] == newer["created_at"] + assert result["hours_since"] == 1.0 + # Liveness is measured from the OLDER run, the one that actually compared. + assert result["latest_executing_created_at"] == older["created_at"] + assert result["hours_since_executing_run"] == 3.0 + assert result["required_step_conclusion"] == "success" + + +def test_liveness_says_so_when_nothing_in_history_ran_the_step(monkeypatch) -> None: + """ "Ran and compared nothing" must not be silently reported as an age. + + The fix for #3243 must not rebuild #3243 one level up: when no run in the probed + window executed the required step, the checker has to SAY that, not fall back to + the bare run age and look healthy. + """ + runs = [ + {"id": n, "conclusion": "success", "created_at": f"2026-08-24T2{n}:00:00Z"} + for n in range(3) + ] + monkeypatch.setattr( + check_durable_tracker_liveness, + "_load_config", + lambda: [ + { + "workflow": "health-68-consumer-sync-drift.yml", + "issue": None, + "max_age_hours": 48, + "require_step": "Compare consumer repos to templates", + "durable": False, + } + ], + ) + monkeypatch.setattr( + check_durable_tracker_liveness, "_gh_api", lambda path, _token: {"workflow_runs": runs} + ) + monkeypatch.setattr( + check_durable_tracker_liveness, + "run_step_conclusion", + lambda _repo, _run_id, _step, _token: "skipped", + ) + monkeypatch.setattr(check_durable_tracker_liveness, "_hours_since", lambda _ts: 0.1) + + (result,) = check_durable_tracker_liveness.evaluate_trackers("stranske/Workflows", "token") + + assert result["healthy"] is False, "0.1h since a no-op run must not read as healthy" + assert result["hours_since_executing_run"] is None + assert result["latest_executing_created_at"] is None + assert "Compare consumer repos to templates" in result["reason"] + assert "triggered, not that it executed" in result["reason"] + + +def test_run_step_conclusion_distinguishes_absent_from_skipped(monkeypatch) -> None: + """`None` (no such step) and `"skipped"` (step ran, was skipped) are different facts.""" + payload = { + "jobs": [ + { + "name": "Validate consumer repo drift", + "steps": [ + {"name": "Debounce workflow_run fan-out", "conclusion": "success"}, + {"name": "Compare consumer repos to templates", "conclusion": "skipped"}, + ], + } + ] + } + monkeypatch.setattr(check_durable_tracker_liveness, "_gh_api", lambda _p, _t: payload) + + assert ( + check_durable_tracker_liveness.run_step_conclusion( + "o/r", 1, "Compare consumer repos to templates", "t" + ) + == "skipped" + ) + assert check_durable_tracker_liveness.run_step_conclusion("o/r", 1, "No Such Step", "t") is None + + +def test_execution_liveness_entries_are_excluded_from_tracker_doc_coverage() -> None: + """An execution-only entry must not be required to have a durable-tracker row. + + #2210 is a TRANSIENT alert Health 68 opens and closes, and it is currently + closed. Listing health-68 under `trackers:` would assert a durable relationship + that does not exist. + """ + config = yaml.safe_load(LIVENESS_CONFIG.read_text(encoding="utf-8")) + execution_only = {str(entry["workflow"]) for entry in config.get("execution_liveness") or []} + assert "health-68-consumer-sync-drift.yml" in execution_only + + durable = check_durable_tracker_liveness._durable_tracker_workflows() + assert "health-68-consumer-sync-drift.yml" not in durable + assert durable == tracker_doc_workflows() + + entry = next( + item + for item in config["execution_liveness"] + if item["workflow"] == "health-68-consumer-sync-drift.yml" + ) + assert entry["require_step"] == "Compare consumer repos to templates" + assert entry["issue"] is None, "a transient alert is not a durable tracker to comment on" + + def test_health_71_invokes_durable_tracker_liveness_check() -> None: text = HEALTH_71.read_text(encoding="utf-8") assert "check_durable_tracker_liveness.py" in text diff --git a/tests/workflows/test_health_68_liveness.py b/tests/workflows/test_health_68_liveness.py index f274400f5..aa2ce0eae 100644 --- a/tests/workflows/test_health_68_liveness.py +++ b/tests/workflows/test_health_68_liveness.py @@ -23,14 +23,71 @@ def test_consumer_drift_detector_has_a_schedule() -> None: assert "schedule" in triggers, "Health 68 must declare a schedule trigger for self-healing" +COMPARE_STEP = "Compare consumer repos to templates" + + def test_consumer_drift_detector_debounces_workflow_run() -> None: + """The debounce must clock the last run that COMPARED, not the last that concluded. + + The old selector accepted any success/failure/timed_out run, which included this + step's own debounced no-ops, so the 30-minute clock reset on nothing happening. + """ text = WORKFLOW.read_text(encoding="utf-8") assert "Debounce workflow_run fan-out" in text assert "github.event_name == 'workflow_run'" in text - assert "| jq -r --argjson current" in text assert '.conclusion == "cancelled"' not in text assert 'branch: "main"' in text or "branch: 'main'" in text + # The selector reads a per-step comparison marker... + assert "COMPARE_STEP_NAME" in text + assert f"COMPARE_STEP_NAME: '{COMPARE_STEP}'" in text + assert "listJobsForWorkflowRun" in text + assert 'step.conclusion !== "skipped"' in text + + # ...and no longer selects purely on the run conclusion. The old jq selector is + # pinned as ABSENT so restoring it is a test failure, not a silent regression. + assert ( + "--argjson current" not in text + ), "the bare-conclusion jq selector is back; it counts debounced no-ops as runs" + + +def test_debounce_step_name_matches_the_step_it_measures() -> None: + """One name, defined once, consumed by the workflow AND the liveness config. + + A matching pair of literals drifts; renaming the compare step without renaming + the marker would leave the debounce measuring a step that no longer exists and + silently falling back to "nothing compared, run anyway". + """ + import yaml as _yaml + + text = WORKFLOW.read_text(encoding="utf-8") + data = _yaml.safe_load(text) + steps = data["jobs"]["check-drift"]["steps"] + step_names = [str(step.get("name") or "") for step in steps] + assert COMPARE_STEP in step_names, "the step the debounce measures must exist" + + debounce = next(step for step in steps if step.get("name") == "Debounce workflow_run fan-out") + assert debounce["env"]["COMPARE_STEP_NAME"] == COMPARE_STEP + + config = _yaml.safe_load( + Path("config/durable_tracker_liveness.yml").read_text(encoding="utf-8") + ) + entry = next( + item + for item in config["execution_liveness"] + if item["workflow"] == "health-68-consumer-sync-drift.yml" + ) + assert entry["require_step"] == COMPARE_STEP + + +def test_check_drift_publishes_whether_it_compared() -> None: + """The job must say whether it did the work, not only that it concluded.""" + import yaml as _yaml + + data = _yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + outputs = data["jobs"]["check-drift"]["outputs"] + assert outputs["compared"] == "${{ steps.compare.outcome != 'skipped' }}" + def test_consumer_drift_debounce_filters_main_before_ordering() -> None: """Non-main runs must not suppress workflow_run fan-out for main.""" @@ -60,44 +117,107 @@ def selected(values: list[dict]) -> int | None: assert selected(main_only) == 1 +LIVE_PROBE_SKIP_REASON = "GH_TOKEN or GITHUB_TOKEN required for the live Health 68 execution probe" + + @pytest.mark.skipif( - os.environ.get("RUN_LIVE_HEALTH_68_PROBE") != "1" - or not (os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN")), - reason="RUN_LIVE_HEALTH_68_PROBE=1 and GH_TOKEN or GITHUB_TOKEN required for live Health 68 execution probe", + not (os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN")), + reason=LIVE_PROBE_SKIP_REASON, ) def test_consumer_drift_detector_executed_recently() -> None: + """The newest run that COMPARED must be recent — not the newest run. + + #3179's Implementation Notes specified a probe that skips only when no token is + present; the shipped version also required RUN_LIVE_HEALTH_68_PROBE=1, so it never + ran in CI even where a token existed. That extra term is gone. + + The probe also no longer accepts a bare run conclusion. Measured live 2026-08-24, + the seven newest `success` runs had this step `skipped` and the newest run that + actually compared had FAILED — a bare-conclusion probe called that healthy. + """ + import json + repo = os.environ.get("GITHUB_REPOSITORY", "stranske/Workflows") token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") payload = subprocess.check_output( [ "gh", "api", - f"repos/{repo}/actions/workflows/health-68-consumer-sync-drift.yml/runs", - "--paginate", + f"repos/{repo}/actions/workflows/health-68-consumer-sync-drift.yml/runs" + "?per_page=100&branch=main", "-q", ".workflow_runs[]", ], text=True, env={**os.environ, "GH_TOKEN": token}, ) - latest_executable: str | None = None + latest_comparing: str | None = None + probed = 0 for line in payload.splitlines(): line = line.strip() if not line: continue - import json - run = json.loads(line) conclusion = str(run.get("conclusion") or "") if conclusion not in {"success", "failure", "cancelled", "timed_out"}: continue - latest_executable = str(run.get("created_at")) - break - - assert latest_executable, "no executable Health 68 run found" - created = datetime.fromisoformat(latest_executable.replace("Z", "+00:00")) + if probed >= 20: + break + probed += 1 + steps = subprocess.check_output( + [ + "gh", + "api", + f"repos/{repo}/actions/runs/{run['id']}/jobs?per_page=100", + "-q", + f'.jobs[].steps[] | select(.name=="{COMPARE_STEP}") | .conclusion', + ], + text=True, + env={**os.environ, "GH_TOKEN": token}, + ).split() + if any(step != "skipped" for step in steps): + latest_comparing = str(run.get("created_at")) + break + + assert latest_comparing, ( + f"no Health 68 run in the {probed} newest executable runs ran {COMPARE_STEP!r}; " + "every one concluded without comparing anything" + ) + created = datetime.fromisoformat(latest_comparing.replace("Z", "+00:00")) hours = (datetime.now(UTC) - created).total_seconds() / 3600.0 - assert hours <= 48, f"newest executable Health 68 run is {latest_executable} ({hours:.1f}h old)" + assert ( + hours <= 48 + ), f"newest Health 68 run that COMPARED is {latest_comparing} ({hours:.1f}h old)" + + +def test_live_probe_skips_only_on_a_missing_token() -> None: + """The skip condition must name the missing variable and nothing else. + + RUN_LIVE_HEALTH_68_PROBE made the probe unrunnable in CI even with a token, which + is a gate whose drain is switched off by default. + """ + # Pin the READ, not the name: this file is allowed to explain the flag in prose, + # and it does. The literal is split so the needle cannot match its own line. + source = Path(__file__).read_text(encoding="utf-8") + needle = 'environ.get("RUN_LIVE' + '_HEALTH_68_PROBE")' + assert ( + needle not in source + ), "the opt-in flag is being read again; the live probe will never run in CI" + + assert "GH_TOKEN" in LIVE_PROBE_SKIP_REASON + assert "GITHUB_TOKEN" in LIVE_PROBE_SKIP_REASON + + marker = next( + mark + for mark in test_consumer_drift_detector_executed_recently.pytestmark + if mark.name == "skipif" + ) + assert marker.kwargs["reason"] == LIVE_PROBE_SKIP_REASON + + # Behavioural, and stronger than the source pin: the condition must be exactly + # "no token". Any extra opt-in term makes this differ whenever a token is present. + expected_skip = not (os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN")) + assert bool(marker.args[0]) is bool(expected_skip) def test_health_68_issue_publish_job_is_split() -> None: