From cfe726f88e91c350164e71a3af13823187f38ea9 Mon Sep 17 00:00:00 2001 From: Test Runner Date: Sun, 12 Jul 2026 10:59:33 -0700 Subject: [PATCH 1/2] feat(OMN-14466): RT-7 fail-loud release/deploy drift monitor Standalone scheduled monitor that surfaces every silent release/deploy divergence same-day instead of after six weeks of green-and-unread silence. Per repo it compares PyPI-published version vs latest v* tag vs main/dev pyproject version, flags [tool.uv.sources] git-override drift (main carrying an override dev already dropped), and checks the last release.yml / runtime-rebuild-trigger.yml run. On divergence it writes a friction record into the existing friction registry (.onex_state/friction/, consumed by node_friction_triage_orchestrator -> Linear) and exits non-zero so the scheduled workflow goes RED. No new store; reuses the env_sync_alert friction precedent. - scripts/release_drift_monitor_lib.py: pure evaluator (no I/O), unit-tested against EXISTS-but-WRONG (fires) and aligned (green) facts. - scripts/release_drift_monitor.py: I/O shell (gh api + PyPI JSON) + friction emit. - .github/workflows/release-drift-monitor.yml: scheduled (every 6h) + dispatch. - tests/scripts/test_release_drift_monitor.py: 12 tests, incl. live-drift RED. dod_evidence: live run against GitHub/PyPI on 2026-07-12 exits 1 with 13 findings incl. omnibase_infra PyPI 0.36.1 behind tag v0.38.3 (P1), omnibase_infra release.yml last run = failure (P1), onex_change_control main stale omnibase-core [tool.uv.sources] override (P1). blind=false, 0 probe errors. 12/12 unit tests pass; ruff + mypy clean; pre-commit green. Epic OMN-13674. Design: docs/plans/2026-07-12-mechanical-release-trains.md section 4 (RT-7). --- .github/workflows/release-drift-monitor.yml | 102 +++++ scripts/release_drift_monitor.py | 445 ++++++++++++++++++++ scripts/release_drift_monitor_lib.py | 384 +++++++++++++++++ tests/scripts/test_release_drift_monitor.py | 283 +++++++++++++ 4 files changed, 1214 insertions(+) create mode 100644 .github/workflows/release-drift-monitor.yml create mode 100644 scripts/release_drift_monitor.py create mode 100644 scripts/release_drift_monitor_lib.py create mode 100644 tests/scripts/test_release_drift_monitor.py diff --git a/.github/workflows/release-drift-monitor.yml b/.github/workflows/release-drift-monitor.yml new file mode 100644 index 0000000000..ffa209ab06 --- /dev/null +++ b/.github/workflows/release-drift-monitor.yml @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +# +# RT-7 fail-loud release/deploy drift monitor (OMN-14466). +# +# Scheduled, same-day surfacing of every silent release/deploy failure across +# the OmniNode repos: PyPI-published version vs latest v* tag vs main/dev +# pyproject version, [tool.uv.sources] git-override drift (main vs dev), and the +# health of the last release.yml / runtime-rebuild-trigger.yml run. +# +# On divergence the monitor exits non-zero -> this workflow goes RED (the +# durable, observable, same-day signal) AND writes friction records into the +# existing friction registry (.onex_state/friction/, consumed by +# node_friction_triage_orchestrator -> Linear). It builds no new store. +# +# This is what would have caught "PyPI omnibase-infra stuck at 0.36.1 for six +# weeks" (main is at 0.38.4) on day one instead of after six weeks of silence. +# +# Token: uses CROSS_REPO_PAT when configured (needed to read the sibling repos +# via the GitHub API). With only GITHUB_TOKEN, sibling probes fail and the +# monitor exits 2 (BLIND) rather than a false green -- a visibility gap is never +# reported as clean. Design: docs/plans/2026-07-12-mechanical-release-trains.md §4. + +name: Release Drift Monitor + +on: + schedule: + # Every 6 hours at minute 17 (offset to avoid thundering herd). + - cron: "17 */6 * * *" + workflow_dispatch: + +# Never overlap runs; the newest schedule/dispatch wins. +concurrency: + group: release-drift-monitor + cancel-in-progress: true + +jobs: + drift-monitor: + name: Release / Deploy Drift Monitor + runs-on: >- + ${{ + fromJSON(vars.OMNI_TRUSTED_CI_RUNS_ON_JSON || '["self-hosted","omnibase-ci"]') + }} + timeout-minutes: 15 + + permissions: + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + + - name: Check CROSS_REPO_PAT is configured + env: + CROSS_REPO_PAT: ${{ secrets.CROSS_REPO_PAT }} + run: | + if [ -z "$CROSS_REPO_PAT" ]; then + echo "::warning::CROSS_REPO_PAT is not set — sibling-repo probes will" + echo "::warning::fail and the monitor will exit 2 (BLIND), not a false green." + fi + + - name: Run release/deploy drift monitor (fail-loud) + id: monitor + env: + # CROSS_REPO_PAT has org read scope; GITHUB_TOKEN is this-repo-only and + # will make sibling probes fail (surfaced as BLIND, never silent-green). + GH_TOKEN: ${{ secrets.CROSS_REPO_PAT || secrets.GITHUB_TOKEN }} + run: | + set +e + uv run --no-project --with packaging --with pyyaml \ + python scripts/release_drift_monitor.py --json | tee drift-report.json + code=${PIPESTATUS[0]} + { + echo "## RT-7 release/deploy drift monitor" + echo "" + if [ "$code" = "0" ]; then + echo "**Result:** clean — PyPI/tag/main/dev aligned, workflows healthy." + elif [ "$code" = "1" ]; then + echo "**Result:** :red_circle: DRIFT DETECTED (exit 1)." + else + echo "**Result:** :warning: BLIND — could not gather some facts (exit $code)." + fi + echo "" + echo '```json' + cat drift-report.json + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + exit "$code" + + - name: Upload drift report + friction records + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: release-drift-report + path: | + drift-report.json + .onex_state/friction/release-drift-*.yaml + if-no-files-found: ignore + retention-days: 30 diff --git a/scripts/release_drift_monitor.py b/scripts/release_drift_monitor.py new file mode 100644 index 0000000000..9af56e1aab --- /dev/null +++ b/scripts/release_drift_monitor.py @@ -0,0 +1,445 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""RT-7 fail-loud release/deploy drift monitor -- I/O shell (OMN-14466). + +Gathers, for every monitored OmniNode repo, the facts the pure evaluator +(``release_drift_monitor_lib.py``) needs -- PyPI-published version, latest ``v*`` +tag, ``main``/``dev`` pyproject versions + ``[tool.uv.sources]`` git overrides, +dev-ahead-of-main commit count, and the last ``release.yml`` / +``runtime-rebuild-trigger.yml`` run -- via the GitHub REST API (``gh api``) and +the PyPI JSON API. It then evaluates drift and, on divergence, writes a friction +record into the existing friction registry (``.onex_state/friction/``, consumed +by ``node_friction_triage_orchestrator`` -> Linear) and exits non-zero so a +scheduled run goes RED. + +This is a **standalone monitor** (RT-7 seam): it does not modify +``stage_workspace.sh``, ``deploy-runtime.sh``, ``release.yml``, or the trigger +scripts. It reuses the existing friction substrate -- it does NOT build a new +store (design §4 RT-7). The friction-emit shape mirrors +``node_env_sync_alert_effect``, the platform's existing scheduled drift -> +friction -> Linear precedent. + +SCOPE NOTE (per-lane running-signature): the design also calls for a per-lane +"running-signature vs intended-ref" check. That requires reading a live ``.201`` +container signature (SSH / introspection endpoint) and is delivered by the RT-6 +deploy-readback lane; it is out of band for a CI-schedulable monitor. RT-7 here +covers the repo/PyPI/tag/workflow axes that ARE observable from CI, which is +what catches the six-week PyPI-stuck class. No fake lane check is emitted. + +Usage:: + + uv run python scripts/release_drift_monitor.py # human report, exit 1 on drift + uv run python scripts/release_drift_monitor.py --json # machine-readable report + uv run python scripts/release_drift_monitor.py --no-emit # do not write friction files + uv run python scripts/release_drift_monitor.py --warn-only # always exit 0 (report only) + +Exit codes: + 0 -- clean: every fact gathered, nothing diverged + 1 -- divergence detected (RED, the fail-loud signal) + 2 -- the monitor could not gather some facts and found no divergence + (a blind run, surfaced explicitly -- never silently green) +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import subprocess +import sys +import tomllib +import urllib.error +import urllib.request +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import yaml +from release_drift_monitor_lib import ( + DriftReport, + DriftThresholds, + RepoFacts, + WorkflowRun, + evaluate_all, + exit_code_for, +) + +_ORG = "OmniNode-ai" +_PYPI_TIMEOUT = 15 +_RELEASE_WORKFLOW = "release.yml" +_TRIGGER_WORKFLOW = "runtime-rebuild-trigger.yml" + + +@dataclass(frozen=True) +class RepoConfig: + """A repo to monitor and the PyPI package it publishes (None if not published).""" + + repo: str + pypi_package: str | None + + +# Default monitor set. onex_change_control publishes nothing to PyPI, so its +# PyPI-based checks are skipped (pypi_package=None) -- only override-drift and +# workflow health apply there. +DEFAULT_REPOS: tuple[RepoConfig, ...] = ( + RepoConfig("omnibase_core", "omnibase-core"), + RepoConfig("omnibase_infra", "omnibase-infra"), + RepoConfig("omnibase_spi", "omnibase-spi"), + RepoConfig("omnibase_compat", "omnibase-compat"), + RepoConfig("omnimarket", "omnimarket"), + RepoConfig("onex_change_control", None), +) + + +# --------------------------------------------------------------------------- # +# GitHub / PyPI probes. Every probe returns (value, error_message | None) so a # +# gather failure is carried as a probe_error rather than silently dropped. # +# --------------------------------------------------------------------------- # +def _gh_api(path: str, jq: str | None = None) -> tuple[Any, str | None]: + """Call ``gh api `` (optionally with a --jq filter). Returns (data, error).""" + args = ["gh", "api", path] + if jq is not None: + args += ["--jq", jq] + try: + proc = subprocess.run(args, capture_output=True, text=True, check=False) + except FileNotFoundError: + return None, "gh CLI not found" + if proc.returncode != 0: + stderr = proc.stderr.strip().splitlines() + detail = stderr[-1] if stderr else f"exit {proc.returncode}" + return None, detail + out = proc.stdout.strip() + if not out: + return None, None + if jq is not None: + # --jq output is raw text/lines, not necessarily JSON. + return out, None + try: + return json.loads(out), None + except json.JSONDecodeError as exc: + return None, f"invalid JSON from gh api {path}: {exc}" + + +def _pypi_latest(package: str) -> tuple[str | None, str | None]: + """Latest published version on PyPI. (None, None) if the package is absent.""" + url = f"https://pypi.org/pypi/{package}/json" # url-authority-ok: PyPI public JSON API, fixed canonical third-party endpoint (no ONEX routing authority) + try: + with urllib.request.urlopen(url, timeout=_PYPI_TIMEOUT) as resp: # noqa: S310 (https literal) + data = json.loads(resp.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + if exc.code == 404: + return None, f"PyPI package {package} not found (404)" + return None, f"PyPI HTTP {exc.code} for {package}" + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc: + return None, f"PyPI probe failed for {package}: {exc}" + return data.get("info", {}).get("version"), None + + +def _latest_tag(repo: str) -> tuple[str | None, str | None]: + """Highest semver ``v*`` tag for a repo (semver-sorted in Python).""" + from release_drift_monitor_lib import parse_version + + data, err = _gh_api(f"repos/{_ORG}/{repo}/tags?per_page=100", jq=".[].name") + if err is not None: + return None, f"tags: {err}" + if not data: + return None, None + best_raw: str | None = None + best_ver = None + for name in str(data).splitlines(): + name = name.strip() + ver = parse_version(name) + if ver is None: + continue + if best_ver is None or ver > best_ver: + best_ver = ver + best_raw = name + return best_raw, None + + +def _pyproject_at( + repo: str, ref: str +) -> tuple[str | None, tuple[str, ...], str | None]: + """Return (project.version, git-override package names, error) for pyproject at ref.""" + data, err = _gh_api( + f"repos/{_ORG}/{repo}/contents/pyproject.toml?ref={ref}", jq=".content" + ) + if err is not None: + return None, (), f"pyproject@{ref}: {err}" + if not data: + return None, (), f"pyproject@{ref}: empty response" + try: + raw = base64.b64decode(str(data)).decode("utf-8") + parsed = tomllib.loads(raw) + except (ValueError, tomllib.TOMLDecodeError) as exc: + return None, (), f"pyproject@{ref}: decode failed: {exc}" + version = parsed.get("project", {}).get("version") + overrides = _git_overrides(parsed) + return (str(version) if version is not None else None), overrides, None + + +def _git_overrides(pyproject: dict[str, Any]) -> tuple[str, ...]: + """Package names in [tool.uv.sources] that pin a git source (git= or rev=).""" + sources = pyproject.get("tool", {}).get("uv", {}).get("sources", {}) + names: list[str] = [] + for pkg, spec in sources.items(): + if isinstance(spec, dict) and ("git" in spec or "rev" in spec): + names.append(pkg) + return tuple(sorted(names)) + + +def _dev_ahead(repo: str) -> tuple[int | None, str | None]: + """Commits ``dev`` is ahead of ``main`` (GitHub compare ahead_by).""" + data, err = _gh_api(f"repos/{_ORG}/{repo}/compare/main...dev", jq=".ahead_by") + if err is not None: + return None, f"compare main...dev: {err}" + if data is None or str(data).strip() == "": + return None, None + try: + return int(str(data).strip()), None + except ValueError: + return None, f"compare main...dev: non-int ahead_by {data!r}" + + +def _workflow_last_run( + repo: str, workflow: str +) -> tuple[WorkflowRun | None, str | None]: + """Last run of a workflow (any branch). exists=False when the workflow is absent.""" + data, err = _gh_api( + f"repos/{_ORG}/{repo}/actions/workflows/{workflow}/runs?per_page=1", + jq=".workflow_runs[0] | {conclusion, created_at, status}", + ) + if err is not None: + if "Not Found" in err or "404" in err: + return WorkflowRun(name=workflow, exists=False), None + return None, f"{workflow} runs: {err}" + if not data: + # Workflow exists but has no runs. + return WorkflowRun(name=workflow, exists=True, conclusion=None), None + try: + parsed = json.loads(str(data)) + except json.JSONDecodeError as exc: + return None, f"{workflow} runs: invalid JSON: {exc}" + return ( + WorkflowRun( + name=workflow, + exists=True, + conclusion=parsed.get("conclusion"), + created_at=parsed.get("created_at"), + ), + None, + ) + + +def gather_repo_facts(cfg: RepoConfig) -> RepoFacts: + """Gather all monitored facts for one repo, accumulating any probe errors.""" + errors: list[str] = [] + + pypi_version: str | None = None + if cfg.pypi_package is not None: + pypi_version, err = _pypi_latest(cfg.pypi_package) + if err is not None: + errors.append(err) + + latest_tag, err = _latest_tag(cfg.repo) + if err is not None: + errors.append(err) + + main_version, main_overrides, err = _pyproject_at(cfg.repo, "main") + if err is not None: + errors.append(err) + dev_version, dev_overrides, err = _pyproject_at(cfg.repo, "dev") + if err is not None: + errors.append(err) + + dev_ahead, err = _dev_ahead(cfg.repo) + if err is not None: + errors.append(err) + + release_run, err = _workflow_last_run(cfg.repo, _RELEASE_WORKFLOW) + if err is not None: + errors.append(err) + trigger_run, err = _workflow_last_run(cfg.repo, _TRIGGER_WORKFLOW) + if err is not None: + errors.append(err) + + return RepoFacts( + repo=cfg.repo, + pypi_package=cfg.pypi_package, + pypi_version=pypi_version, + latest_tag=latest_tag, + main_version=main_version, + dev_version=dev_version, + dev_ahead_commits=dev_ahead, + main_git_overrides=main_overrides, + dev_git_overrides=dev_overrides, + release_run=release_run, + trigger_run=trigger_run, + probe_errors=tuple(errors), + ) + + +# --------------------------------------------------------------------------- # +# Friction emission -- reuse the existing registry (no new store). # +# --------------------------------------------------------------------------- # +def _resolve_friction_dir(explicit: str | None) -> Path: + """Resolve the friction registry directory (mirrors node_env_sync_alert_effect).""" + if explicit: + return Path(explicit) + state_dir = os.environ.get("ONEX_STATE_DIR") + if state_dir: + return Path(state_dir) / "friction" + omni_home = os.environ.get("OMNI_HOME") + if omni_home: + return Path(omni_home) / ".onex_state" / "friction" + return Path(".onex_state") / "friction" + + +def emit_friction(report: DriftReport, friction_dir: Path) -> list[str]: + """Write one friction YAML per finding signature; return the paths written.""" + if not report.findings: + return [] + friction_dir.mkdir(parents=True, exist_ok=True) + occurred_at = datetime.now(tz=UTC).isoformat() + written: list[str] = [] + for finding in report.findings: + event = { + "event_type": "release_deploy_drift", + "occurred_at": occurred_at, + "severity": finding.severity, + "drift_signature": finding.signature, + "code": finding.code, + "repo": finding.repo, + "title": f"Release/deploy drift ({finding.code}): {finding.summary}", + "summary": finding.summary, + "detail": finding.detail, + "source": "release_drift_monitor", + } + safe = finding.signature.replace("/", "-").replace(":", "-") + path = friction_dir / f"release-drift-{safe}.yaml" + path.write_text(yaml.safe_dump(event, sort_keys=True), encoding="utf-8") + written.append(str(path)) + return written + + +# --------------------------------------------------------------------------- # +# Rendering # +# --------------------------------------------------------------------------- # +def _render_human(report: DriftReport, friction_paths: list[str]) -> str: + lines: list[str] = [] + lines.append("=" * 74) + lines.append("RT-7 release/deploy drift monitor") + lines.append(f"generated_at: {report.generated_at}") + lines.append(f"repos_checked: {report.repos_checked}") + lines.append("=" * 74) + if report.findings: + lines.append(f"DRIFT DETECTED -- {len(report.findings)} finding(s):") + for finding in report.findings: + lines.append("") + lines.append(f" [{finding.severity}] {finding.code} -- {finding.repo}") + lines.append(f" {finding.summary}") + lines.append(f" {finding.detail}") + else: + lines.append( + "No drift detected -- PyPI/tag/main/dev aligned, workflows healthy." + ) + if report.probe_errors: + lines.append("") + lines.append( + f"PROBE ERRORS ({len(report.probe_errors)}) -- monitor visibility gaps:" + ) + for err in report.probe_errors: + lines.append(f" ! {err}") + if friction_paths: + lines.append("") + lines.append("friction records written:") + for path in friction_paths: + lines.append(f" - {path}") + lines.append("=" * 74) + return "\n".join(lines) + + +def _report_to_dict(report: DriftReport, friction_paths: list[str]) -> dict[str, Any]: + return { + "generated_at": report.generated_at, + "repos_checked": report.repos_checked, + "diverged": report.diverged, + "blind": report.blind, + "findings": [ + { + "code": f.code, + "severity": f.severity, + "repo": f.repo, + "signature": f.signature, + "summary": f.summary, + "detail": f.detail, + } + for f in report.findings + ], + "probe_errors": list(report.probe_errors), + "friction_paths": friction_paths, + } + + +# --------------------------------------------------------------------------- # +# CLI # +# --------------------------------------------------------------------------- # +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="RT-7 release/deploy drift monitor") + parser.add_argument( + "--json", action="store_true", help="machine-readable JSON output" + ) + parser.add_argument( + "--no-emit", action="store_true", help="do not write friction records" + ) + parser.add_argument( + "--warn-only", + action="store_true", + help="always exit 0 (report drift without failing the run)", + ) + parser.add_argument( + "--friction-dir", default=None, help="override the friction registry directory" + ) + parser.add_argument( + "--dev-ahead-warn", + type=int, + default=DriftThresholds.dev_ahead_warn, + help="dev-ahead-of-main commit count that fires DEV_AHEAD_OF_MAIN", + ) + parser.add_argument( + "--release-stale-days", + type=float, + default=DriftThresholds.release_stale_days, + help="release-run age (main ahead of tag) that fires RELEASE_TRAIN_STALLED", + ) + args = parser.parse_args(argv) + + thresholds = DriftThresholds( + dev_ahead_warn=args.dev_ahead_warn, + release_stale_days=args.release_stale_days, + ) + + facts = [gather_repo_facts(cfg) for cfg in DEFAULT_REPOS] + report = evaluate_all(facts, thresholds) + + friction_paths: list[str] = [] + if not args.no_emit: + friction_paths = emit_friction(report, _resolve_friction_dir(args.friction_dir)) + + if args.json: + print(json.dumps(_report_to_dict(report, friction_paths), indent=2)) + else: + print(_render_human(report, friction_paths)) + + if args.warn_only: + return 0 + exit_code: int = exit_code_for(report) + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/release_drift_monitor_lib.py b/scripts/release_drift_monitor_lib.py new file mode 100644 index 0000000000..e545017fdc --- /dev/null +++ b/scripts/release_drift_monitor_lib.py @@ -0,0 +1,384 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Pure drift-evaluation logic for the RT-7 release/deploy drift monitor (OMN-14466). + +Given already-gathered facts about each monitored repo -- its PyPI-published +version, latest ``v*`` tag, ``main``/``dev`` pyproject versions, ``[tool.uv.sources]`` +git overrides on each branch, dev-ahead-of-main commit count, and the last +``release.yml`` / ``runtime-rebuild-trigger.yml`` workflow run -- this module +decides which drift findings fire. + +WHY A SEPARATE PURE MODULE +-------------------------- +The fire/no-fire decision has **no I/O**: every input is a plain frozen +dataclass the CLI shell (``release_drift_monitor.py``) populates from ``gh api`` +and the PyPI JSON API. That separation makes the decision unit-testable against +both **EXISTS-but-WRONG** facts (must fire) and **aligned** facts (must stay +green) -- the negative control an enforcement check needs so a green result +proves *"checked and matched"*, not *"couldn't see anything"*. + +DESIGN (taxonomy Class-3 / Class-6, docs/plans/2026-07-12-mechanical-release-trains.md +§4 RT-7): every stage between "merged" and "released/running" is a silent +producer -- green and unread. This monitor reads the observable end-state +(PyPI vs tag vs main vs dev, and the workflow that should have moved them) and +fires the moment they diverge, so "PyPI stuck at 0.36.1 for six weeks" is a +same-day RED, not a six-week silence. + +FRESHNESS / FAIL-LOUD: a fact the shell could not gather is carried as a +``probe_error`` on the RepoFacts, never dropped. ``exit_code_for`` returns 2 +when the monitor could not see (probe errors, no findings) so a blind run is +never mistaken for a clean run -- "stale -> explicit, never silently served as +current". +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import UTC, datetime + +from packaging.version import InvalidVersion, Version + +# --------------------------------------------------------------------------- # +# Severity ordering (used for sorting / summary only; not a gate threshold). # +# --------------------------------------------------------------------------- # +_SEVERITY_ORDER = {"P1": 0, "P2": 1, "P3": 2} + + +def parse_version(raw: str | None) -> Version | None: + """Parse a version string (with or without a leading ``v``); None if unparseable.""" + if not raw: + return None + candidate = raw[1:] if raw.startswith("v") else raw + try: + return Version(candidate) + except InvalidVersion: + return None + + +def _age_days(iso_ts: str | None, now: datetime) -> float | None: + """Age in days of an ISO-8601 timestamp relative to ``now``; None if unparseable.""" + if not iso_ts: + return None + text = iso_ts.replace("Z", "+00:00") + try: + when = datetime.fromisoformat(text) + except ValueError: + return None + if when.tzinfo is None: + when = when.replace(tzinfo=UTC) + return (now - when).total_seconds() / 86400.0 + + +# --------------------------------------------------------------------------- # +# Fact model -- populated by the I/O shell, consumed here with zero I/O. # +# --------------------------------------------------------------------------- # +@dataclass(frozen=True) +class WorkflowRun: + """The last run of a release/deploy workflow for one repo.""" + + name: str + exists: bool = True + conclusion: str | None = None # "success" | "failure" | "cancelled" | None + created_at: str | None = None # ISO-8601 + + +@dataclass(frozen=True) +class RepoFacts: + """Everything the monitor knows about one repo this cycle. + + ``pypi_package`` is None for repos that are intentionally not published to + PyPI (e.g. onex_change_control) -- PyPI-based checks are then skipped, not + treated as drift. + """ + + repo: str + pypi_package: str | None = None + pypi_version: str | None = None + latest_tag: str | None = None + main_version: str | None = None + dev_version: str | None = None + dev_ahead_commits: int | None = None + main_git_overrides: tuple[str, ...] = () + dev_git_overrides: tuple[str, ...] = () + release_run: WorkflowRun | None = None + trigger_run: WorkflowRun | None = None + probe_errors: tuple[str, ...] = () + + +@dataclass(frozen=True) +class DriftFinding: + """One divergence the monitor is firing on.""" + + code: str + severity: str # "P1" | "P2" | "P3" + repo: str + signature: str # stable dedup key -> friction filename; repeated runs overwrite + summary: str + detail: str + + +@dataclass(frozen=True) +class DriftThresholds: + """Tunable thresholds for the count/age-based findings.""" + + dev_ahead_warn: int = 25 # dev commits ahead of main before DEV_AHEAD_OF_MAIN fires + release_stale_days: float = ( + 7.0 # release-run age (main-ahead-of-tag) before STALLED fires + ) + + +@dataclass(frozen=True) +class DriftReport: + """Aggregate result for one monitor cycle.""" + + findings: tuple[DriftFinding, ...] + repos_checked: int + probe_errors: tuple[str, ...] + generated_at: str + thresholds: DriftThresholds = field(default_factory=DriftThresholds) + + @property + def diverged(self) -> bool: + return len(self.findings) > 0 + + @property + def blind(self) -> bool: + """True when the monitor could not gather some facts (partial visibility).""" + return len(self.probe_errors) > 0 + + +# --------------------------------------------------------------------------- # +# The checks. Each is a small pure predicate over RepoFacts. # +# --------------------------------------------------------------------------- # +def evaluate_repo( + facts: RepoFacts, + thresholds: DriftThresholds, + now: datetime | None = None, +) -> list[DriftFinding]: + """Return every drift finding that fires for one repo.""" + now = now or datetime.now(tz=UTC) + findings: list[DriftFinding] = [] + + pypi = parse_version(facts.pypi_version) + tag = parse_version(facts.latest_tag) + main = parse_version(facts.main_version) + + # 1) PyPI behind the latest tag -> a tag was cut but the publish never + # landed (stuck/failed publish). This is the "PyPI stuck at 0.36.1 while + # v0.38.3 is tagged" case -- the six-week silence. + if pypi is not None and tag is not None and pypi < tag: + findings.append( + DriftFinding( + code="PYPI_BEHIND_TAG", + severity="P1", + repo=facts.repo, + signature=f"{facts.repo}:pypi-behind-tag", + summary=( + f"{facts.pypi_package} PyPI {pypi} is behind latest tag v{tag}" + ), + detail=( + f"A v{tag} tag exists but PyPI still serves {pypi}. The " + "publish step never landed for at least one release -- " + "downstream pins resolving from PyPI run stale code." + ), + ) + ) + + # 2) main bumped past the latest tag -> a version was set on main that was + # never tagged/released (the auto-tag chain stopped firing). + if main is not None and tag is not None and main > tag: + findings.append( + DriftFinding( + code="MAIN_AHEAD_OF_TAG", + severity="P2", + repo=facts.repo, + signature=f"{facts.repo}:main-ahead-of-tag", + summary=f"main pyproject {main} was never tagged (latest tag v{tag})", + detail=( + f"main declares {main} but the highest release tag is v{tag}. " + "A release was merged as a version bump but no tag/publish " + "followed -- the release train did not advance." + ), + ) + ) + + # 3) main behind the highest released artifact (PyPI or tag) -> stale main + # lineage running behind what has already shipped. + released_candidates = [v for v in (pypi, tag) if v is not None] + released = max(released_candidates) if released_candidates else None + if main is not None and released is not None and main < released: + findings.append( + DriftFinding( + code="MAIN_BEHIND_RELEASED", + severity="P2", + repo=facts.repo, + signature=f"{facts.repo}:main-behind-released", + summary=f"main pyproject {main} is behind released {released}", + detail=( + f"main is at {main} but the highest released version " + f"(max of PyPI {pypi} / tag {tag}) is {released}. main " + "lineage has not caught up to what shipped." + ), + ) + ) + + # 4) large unreleased dev->main backlog (the "six weeks of merged, never + # released" signal at commit granularity). + if ( + facts.dev_ahead_commits is not None + and facts.dev_ahead_commits >= thresholds.dev_ahead_warn + ): + findings.append( + DriftFinding( + code="DEV_AHEAD_OF_MAIN", + severity="P3", + repo=facts.repo, + signature=f"{facts.repo}:dev-ahead-of-main", + summary=( + f"dev is {facts.dev_ahead_commits} commits ahead of main " + f"(>= {thresholds.dev_ahead_warn})" + ), + detail=( + f"{facts.dev_ahead_commits} commits merged to dev have not " + "promoted to main -- a growing unreleased backlog." + ), + ) + ) + + # 5) a [tool.uv.sources] git override still present on main that dev already + # dropped -> main resolves a pinned git SHA while dev runs the released + # dep (the OCC omnibase-core rev=f325e759 case). + stale_overrides = tuple( + pkg for pkg in facts.main_git_overrides if pkg not in facts.dev_git_overrides + ) + for pkg in stale_overrides: + findings.append( + DriftFinding( + code="MAIN_STALE_UV_OVERRIDE", + severity="P1", + repo=facts.repo, + signature=f"{facts.repo}:stale-uv-override:{pkg}", + summary=( + f"main carries a stale [tool.uv.sources] git override for " + f"{pkg} that dev has removed" + ), + detail=( + f"main's pyproject pins {pkg} to a git rev via " + "[tool.uv.sources] while dev already dropped it. main builds " + "resolve the pinned SHA, not the released dep -- the " + '"fix merged but not live" residual.' + ), + ) + ) + + # 6) the release workflow's last run failed outright. + if ( + facts.release_run is not None + and facts.release_run.exists + and facts.release_run.conclusion == "failure" + ): + findings.append( + DriftFinding( + code="RELEASE_WORKFLOW_FAILED", + severity="P1", + repo=facts.repo, + signature=f"{facts.repo}:release-workflow-failed", + summary=f"{facts.release_run.name} last run concluded 'failure'", + detail=( + f"The most recent {facts.release_run.name} run " + f"({facts.release_run.created_at}) failed. The release chain " + "is broken; no publish has succeeded since." + ), + ) + ) + + # 7) release workflow stale while main is ahead of the tag -> the train has + # stalled (a bump landed, but no release run has moved it in N days). + if ( + main is not None + and tag is not None + and main > tag + and facts.release_run is not None + ): + age = _age_days(facts.release_run.created_at, now) + if ( + facts.release_run.exists + and age is not None + and age >= thresholds.release_stale_days + ): + findings.append( + DriftFinding( + code="RELEASE_TRAIN_STALLED", + severity="P2", + repo=facts.repo, + signature=f"{facts.repo}:release-train-stalled", + summary=( + f"main {main} > tag v{tag} but {facts.release_run.name} " + f"last ran {age:.0f}d ago (>= {thresholds.release_stale_days:.0f}d)" + ), + detail=( + f"main is ahead of the latest tag yet the release " + f"workflow has not run in {age:.0f} days -- the release " + "train has stalled with unreleased bumps pending." + ), + ) + ) + + return findings + + +def evaluate_all( + facts: list[RepoFacts], + thresholds: DriftThresholds | None = None, + now: datetime | None = None, +) -> DriftReport: + """Evaluate every repo and aggregate into a single report.""" + thresholds = thresholds or DriftThresholds() + now = now or datetime.now(tz=UTC) + all_findings: list[DriftFinding] = [] + all_probe_errors: list[str] = [] + for repo_facts in facts: + all_findings.extend(evaluate_repo(repo_facts, thresholds, now)) + all_probe_errors.extend( + f"{repo_facts.repo}: {err}" for err in repo_facts.probe_errors + ) + all_findings.sort( + key=lambda f: (_SEVERITY_ORDER.get(f.severity, 9), f.repo, f.code) + ) + return DriftReport( + findings=tuple(all_findings), + repos_checked=len(facts), + probe_errors=tuple(all_probe_errors), + generated_at=now.isoformat(), + thresholds=thresholds, + ) + + +def exit_code_for(report: DriftReport) -> int: + """Map a report to a process exit code (fail-loud). + + * 1 -- divergence found (RED, the fail-loud drift signal) + * 2 -- the monitor could not see (probe errors) and found no divergence; + a blind run must never be mistaken for a clean run + * 0 -- clean: every fact gathered, nothing diverged + """ + if report.diverged: + return 1 + if report.blind: + return 2 + return 0 + + +__all__ = [ + "DriftFinding", + "DriftReport", + "DriftThresholds", + "RepoFacts", + "WorkflowRun", + "evaluate_all", + "evaluate_repo", + "exit_code_for", + "parse_version", +] diff --git a/tests/scripts/test_release_drift_monitor.py b/tests/scripts/test_release_drift_monitor.py new file mode 100644 index 0000000000..09810d7df0 --- /dev/null +++ b/tests/scripts/test_release_drift_monitor.py @@ -0,0 +1,283 @@ +# SPDX-FileCopyrightText: 2025 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT + +"""Tests for the RT-7 release/deploy drift monitor pure evaluator (OMN-14466). + +Two obligations, per feedback_prove_red_against_exists_but_wrong: + +1. RED reproduces on the CURRENT live drift. Each ``test_live_*`` feeds the + evaluator the EXACT facts observed on 2026-07-12 (infra PyPI 0.36.1 vs tag + v0.38.3 vs main 0.38.4; core main 0.46.6 behind PyPI/tag 0.46.7; OCC main + still pinning the omnibase-core git override while dev is clean; infra + release.yml last run = failure) and asserts the monitor FIRES. +2. GREEN is discriminating, not vacuous. ``test_aligned_repo_is_green`` proves + an aligned repo produces zero findings, and its sibling flips one field to + confirm the same check goes RED -- a green here means "checked and matched", + not "couldn't see anything". +""" + +from __future__ import annotations + +import importlib.util +import sys +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_LIB = _REPO_ROOT / "scripts" / "release_drift_monitor_lib.py" + +# A fixed "now" so the age-based findings are deterministic. Matches the +# 2026-07-12 observation date used for the live-drift fixtures below. +_NOW = datetime(2026, 7, 12, 17, 0, tzinfo=UTC) + + +def _load_lib() -> Any: + mod_name = "release_drift_monitor_lib_under_test" + spec = importlib.util.spec_from_file_location(mod_name, _LIB) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + sys.modules[mod_name] = mod + spec.loader.exec_module(mod) + return mod + + +_lib = _load_lib() + +RepoFacts = _lib.RepoFacts +WorkflowRun = _lib.WorkflowRun +DriftThresholds = _lib.DriftThresholds +evaluate_repo = _lib.evaluate_repo +evaluate_all = _lib.evaluate_all +exit_code_for = _lib.exit_code_for + + +def _codes(facts: Any, thresholds: DriftThresholds | None = None) -> set[str]: + findings = evaluate_repo(facts, thresholds or DriftThresholds(), now=_NOW) + return {f.code for f in findings} + + +# --------------------------------------------------------------------------- # +# 1. RED reproduces on the CURRENT live drift (2026-07-12 observations). # +# --------------------------------------------------------------------------- # +@pytest.mark.unit +def test_live_infra_drift_fires() -> None: + """infra: PyPI 0.36.1 behind tag v0.38.3; main 0.38.4 never tagged; release failed.""" + facts = RepoFacts( + repo="omnibase_infra", + pypi_package="omnibase-infra", + pypi_version="0.36.1", + latest_tag="v0.38.3", + main_version="0.38.4", + dev_version="0.38.4", + dev_ahead_commits=294, + release_run=WorkflowRun( + name="release.yml", + exists=True, + conclusion="failure", + created_at="2026-05-27T21:12:23Z", + ), + ) + codes = _codes(facts) + # The six-week-stuck-PyPI signal and its siblings must all fire. + assert "PYPI_BEHIND_TAG" in codes # 0.36.1 < v0.38.3 + assert "MAIN_AHEAD_OF_TAG" in codes # 0.38.4 never tagged + assert "RELEASE_WORKFLOW_FAILED" in codes # last release.yml run failed + assert "RELEASE_TRAIN_STALLED" in codes # main>tag and release run 46d stale + assert "DEV_AHEAD_OF_MAIN" in codes # 294 >= 25 + # main is AHEAD of the released set here, so this must NOT fire. + assert "MAIN_BEHIND_RELEASED" not in codes + + +@pytest.mark.unit +def test_live_occ_stale_override_fires() -> None: + """OCC: main still pins the omnibase-core git override that dev removed.""" + facts = RepoFacts( + repo="onex_change_control", + pypi_package=None, # not published to PyPI -> no PyPI checks + pypi_version=None, + latest_tag="v0.5.1", + main_version="0.5.1", + dev_version="0.5.1", + main_git_overrides=("omnibase-core",), + dev_git_overrides=(), + release_run=WorkflowRun(name="release.yml", exists=False), + ) + findings = evaluate_repo(facts, DriftThresholds(), now=_NOW) + codes = {f.code for f in findings} + assert "MAIN_STALE_UV_OVERRIDE" in codes + stale = next(f for f in findings if f.code == "MAIN_STALE_UV_OVERRIDE") + assert stale.signature == "onex_change_control:stale-uv-override:omnibase-core" + # No PyPI package -> no PyPI-based finding, no crash. + assert "PYPI_BEHIND_TAG" not in codes + + +@pytest.mark.unit +def test_live_core_main_behind_released_fires() -> None: + """core: main 0.46.6 is behind PyPI 0.46.7 / tag v0.46.7 (stale main lineage).""" + facts = RepoFacts( + repo="omnibase_core", + pypi_package="omnibase-core", + pypi_version="0.46.7", + latest_tag="v0.46.7", + main_version="0.46.6", + dev_version="0.46.8", + dev_ahead_commits=170, + release_run=WorkflowRun( + name="release.yml", + exists=True, + conclusion="success", + created_at="2026-07-10T00:00:00Z", + ), + ) + codes = _codes(facts) + assert "MAIN_BEHIND_RELEASED" in codes # 0.46.6 < 0.46.7 + # PyPI equals the tag here, so the publish is NOT behind the tag. + assert "PYPI_BEHIND_TAG" not in codes + # main is not ahead of the tag, so no missing-tag / stalled finding. + assert "MAIN_AHEAD_OF_TAG" not in codes + assert "RELEASE_TRAIN_STALLED" not in codes + + +# --------------------------------------------------------------------------- # +# 2. GREEN is discriminating (aligned passes; a one-field break fires). # +# --------------------------------------------------------------------------- # +def _aligned_facts(**overrides: Any) -> Any: + base: dict[str, Any] = { + "repo": "omnibase_example", + "pypi_package": "omnibase-example", + "pypi_version": "1.4.0", + "latest_tag": "v1.4.0", + "main_version": "1.4.0", + "dev_version": "1.4.0", + "dev_ahead_commits": 0, + "main_git_overrides": (), + "dev_git_overrides": (), + "release_run": WorkflowRun( + name="release.yml", + exists=True, + conclusion="success", + created_at="2026-07-11T00:00:00Z", + ), + } + base.update(overrides) + return RepoFacts(**base) + + +@pytest.mark.unit +def test_aligned_repo_is_green() -> None: + """Negative control: fully aligned repo produces zero findings.""" + assert _codes(_aligned_facts()) == set() + + +@pytest.mark.unit +def test_aligned_repo_goes_red_when_pypi_slips() -> None: + """Same aligned fixture, PyPI dropped one patch behind the tag -> fires. + + Proves the green above is 'checked and matched', not vacuous absence. + """ + assert "PYPI_BEHIND_TAG" in _codes(_aligned_facts(pypi_version="1.3.9")) + + +@pytest.mark.unit +def test_aligned_repo_goes_red_when_override_added() -> None: + assert "MAIN_STALE_UV_OVERRIDE" in _codes( + _aligned_facts(main_git_overrides=("omnibase-core",), dev_git_overrides=()) + ) + + +# --------------------------------------------------------------------------- # +# Threshold / edge behaviour # +# --------------------------------------------------------------------------- # +@pytest.mark.unit +def test_dev_ahead_backlog_threshold() -> None: + below = _aligned_facts(dev_ahead_commits=24) + at = _aligned_facts(dev_ahead_commits=25) + assert "DEV_AHEAD_OF_MAIN" not in _codes(below) + assert "DEV_AHEAD_OF_MAIN" in _codes(at) + + +@pytest.mark.unit +def test_release_stalled_needs_main_ahead_and_stale_run() -> None: + # main ahead of tag but the release run is RECENT -> not stalled. + recent = _aligned_facts( + main_version="1.5.0", + latest_tag="v1.4.0", + release_run=WorkflowRun( + name="release.yml", + exists=True, + conclusion="success", + created_at="2026-07-11T00:00:00Z", # 1d before _NOW + ), + ) + assert "RELEASE_TRAIN_STALLED" not in _codes(recent) + # main ahead of tag AND the run is stale -> stalled. + stale = _aligned_facts( + main_version="1.5.0", + latest_tag="v1.4.0", + release_run=WorkflowRun( + name="release.yml", + exists=True, + conclusion="success", + created_at="2026-06-01T00:00:00Z", # ~41d before _NOW + ), + ) + assert "RELEASE_TRAIN_STALLED" in _codes(stale) + + +@pytest.mark.unit +def test_shared_override_on_both_branches_is_not_drift() -> None: + """An override present on BOTH main and dev is intentional, not stale.""" + facts = _aligned_facts( + main_git_overrides=("omnibase-core",), + dev_git_overrides=("omnibase-core",), + ) + assert "MAIN_STALE_UV_OVERRIDE" not in _codes(facts) + + +# --------------------------------------------------------------------------- # +# Aggregation + fail-loud exit codes # +# --------------------------------------------------------------------------- # +@pytest.mark.unit +def test_evaluate_all_aggregates_and_exit_code_is_red() -> None: + infra = RepoFacts( + repo="omnibase_infra", + pypi_package="omnibase-infra", + pypi_version="0.36.1", + latest_tag="v0.38.3", + main_version="0.38.4", + dev_version="0.38.4", + release_run=WorkflowRun( + name="release.yml", + exists=True, + conclusion="failure", + created_at="2026-05-27T21:12:23Z", + ), + ) + report = evaluate_all([infra, _aligned_facts()], DriftThresholds(), now=_NOW) + assert report.diverged is True + assert report.repos_checked == 2 + assert exit_code_for(report) == 1 + # Findings are sorted P1 before P2. + severities = [f.severity for f in report.findings] + assert severities == sorted(severities, key={"P1": 0, "P2": 1, "P3": 2}.get) + + +@pytest.mark.unit +def test_aligned_only_exit_code_is_zero() -> None: + report = evaluate_all([_aligned_facts()], DriftThresholds(), now=_NOW) + assert report.diverged is False + assert report.blind is False + assert exit_code_for(report) == 0 + + +@pytest.mark.unit +def test_probe_error_is_not_silently_green() -> None: + """A blind run (probe errors, no findings) must exit 2, never 0.""" + blind = _aligned_facts(probe_errors=("tags: gh api failed",)) + report = evaluate_all([blind], DriftThresholds(), now=_NOW) + assert report.diverged is False + assert report.blind is True + assert exit_code_for(report) == 2 # blind, not clean From 7e81e2db42912768aa2d7978a0c1d18c91982454 Mon Sep 17 00:00:00 2001 From: Test Runner Date: Sun, 12 Jul 2026 12:11:00 -0700 Subject: [PATCH 2/2] chore(OMN-14466): refresh CI after OCC evidence