From a65a677aeaf2e918b18f3137d1bb9c274c110d3c Mon Sep 17 00:00:00 2001 From: CC#1 Kora Substrate Date: Sat, 23 May 2026 13:52:19 -0700 Subject: [PATCH] =?UTF-8?q?feat(kora):=20KR-ALERT-NOTIFY=20ST1=20=E2=80=94?= =?UTF-8?q?=20push=20alerts=20to=20Joshua=20via=20Slack=20DM=20+=20email?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the operator-feedback loop. The cockpit-complete observation from #145: every panel now reads real data, but Joshua still has to LOOK at the cockpit to see alerts. This bucket flips that — when a critical/warning alert fires, Kora pings Joshua via Slack DM (immediate); info alerts go to email. New module: ``kora_cli/alerts/notifier.py`` * ``AlertNotifier`` class — periodic-task entry point ``run_notification_cycle`` calls #145's aggregator, computes set-diff against last cycle's alert IDs, dispatches only newly-firing alerts. * Channel routing (PM §4 Q2 default): critical/warning → Slack DM, info → email. * Dedup state in-memory only (PM §4 Q3 default): empty on first cycle → all currently-active alerts fire as "new" (one-time re-ping on restart; persistence is a future bucket). * Failed dispatch still adds alert ID to last_alert_ids (no spam on transient SMTP/Slack errors). * Audit emit ``notification.dispatched`` per dispatch attempt (success OR failure) — operator triages via audit panel if expecting a notification that didn't arrive. * Pure formatting helpers (Slack DM text, email subject, email body, relative-time) extracted for isolated unit-testing. New listener: ``kora_cli/listeners/alert_notifier_listener.py`` * Constructs the notifier bound to ``current_slack_client`` + ``current_purelymail_client`` lazy factories (matches the accessor pattern from KR-MCP-SEND-TOOLS). * Periodic task ``alerts.notify`` registered at import time @ 180s default (PM §4 Q1); ``KORA_ALERT_NOTIFY_INTERVAL_SEC`` env override. * Shutdown resets dedup state so a subsequent listener start sees a clean slate. * Defense-in-depth outer try/except in run_notification_cycle so any path that bypasses the notifier's inner catch can't crash the heartbeat scheduler. Audit seam extension: ``notification.dispatched`` added to the SeamName Literal in ``kora_cli/audit/jsonl_sink.py`` (small + additive; existing seams unchanged). K-DG verification at HEAD ``2345d51`` (matches the spec's cited SHA): all 5 accessors confirmed in place + interfaces matched (compute_active_alerts ✓ in #145 / current_slack_client / current_purelymail_client / register_periodic_task ✓ in heartbeat scheduler / emit_audit ✓ in audit/jsonl_sink). 50 new tests pass: * 36 notifier tests (formatting + routing + dedup + failure isolation + audit shape + telemetry + dispatch outcome) * 14 listener tests (registration + cadence resolution + lifecycle + run_notification_cycle short-circuits + defense-in-depth + factory wiring) Cross-bucket regression: 600/600 when run serially. With pytest-xdist parallelism, 5-ish flaky failures appear in test_email_inbound_handler.py — VERIFIED pre-existing on bare HEAD without these changes (3 runs of bare HEAD reproduced 4/6/0 failures, same email_inbound tests). Not introduced by this PR; filed as a separate xdist-ordering pollution issue for the follow-on bucket queue. Ruff clean. §4 PM-open status — all DEFAULTS applied in ST1: Q1 cadence 180s (3 min) — accepted Q2 routing critical/warning→Slack, info→email — accepted Q3 fire on first cycle (no persistence) — accepted Q4 kora__send_test_alert MCP tool — DEFERRED to ST2 After ST2 lands (per-category cooldown + burst dampening + digest mode + operator runbook + kora__send_test_alert MCP tool), the operator-feedback loop closes fully. Co-Authored-By: Claude Opus 4.7 (1M context) --- kora_cli/alerts/__init__.py | 29 +- kora_cli/alerts/notifier.py | 440 ++++++++++++ kora_cli/audit/jsonl_sink.py | 6 + kora_cli/listeners/__init__.py | 9 + kora_cli/listeners/alert_notifier_listener.py | 255 +++++++ tests/kora_cli/alerts/test_notifier.py | 680 ++++++++++++++++++ .../test_alert_notifier_listener.py | 226 ++++++ 7 files changed, 1639 insertions(+), 6 deletions(-) create mode 100644 kora_cli/alerts/notifier.py create mode 100644 kora_cli/listeners/alert_notifier_listener.py create mode 100644 tests/kora_cli/alerts/test_notifier.py create mode 100644 tests/kora_cli/test_listeners/test_alert_notifier_listener.py diff --git a/kora_cli/alerts/__init__.py b/kora_cli/alerts/__init__.py index 0555ce67143e..f09f942f8258 100644 --- a/kora_cli/alerts/__init__.py +++ b/kora_cli/alerts/__init__.py @@ -1,16 +1,33 @@ -"""Alert aggregation — KR-ALERTS-PANEL-FLIP. +"""Alert aggregation + push-notification surface. -Aggregates operator-attention signals from the 5 existing data -sources (OperationalStateHolder + cost-ladder holder + HealthRollup -+ audit JSONL + heartbeat probe snapshots) into the FE's -``Alert`` shape from ``web/src/lib/api.ts``. +Aggregator (KR-ALERTS-PANEL-FLIP): operator-attention signals +from the 5 data sources (OperationalStateHolder + cost-ladder +holder + HealthRollup + audit JSONL + heartbeat probe snapshots) +projected to the FE's ``Alert`` shape from ``web/src/lib/api.ts``. + +Notifier (KR-ALERT-NOTIFY): periodic task that diffs the active +alert set + pushes newly-firing alerts to Joshua via Slack DM +(critical / warning) or email (info). Public surface: * :class:`Alert` — wire-shape dataclass mirroring the TS interface * :func:`compute_active_alerts` — single-call aggregator the endpoint consumes + * :class:`AlertNotifier` + :class:`NotificationCycleResult` + + :class:`DispatchOutcome` — push-notification surface """ from kora_cli.alerts.aggregator import Alert, compute_active_alerts +from kora_cli.alerts.notifier import ( + AlertNotifier, + DispatchOutcome, + NotificationCycleResult, +) -__all__ = ["Alert", "compute_active_alerts"] +__all__ = [ + "Alert", + "AlertNotifier", + "DispatchOutcome", + "NotificationCycleResult", + "compute_active_alerts", +] diff --git a/kora_cli/alerts/notifier.py b/kora_cli/alerts/notifier.py new file mode 100644 index 000000000000..76ca948be5f0 --- /dev/null +++ b/kora_cli/alerts/notifier.py @@ -0,0 +1,440 @@ +"""Alert push-notification — KR-ALERT-NOTIFY ST1. + +Closes the operator-feedback loop: when a critical/warning alert +fires, Kora pings Joshua via Slack DM; info alerts go to email. +Joshua doesn't have to LOOK at the cockpit. + +# Architecture + + 1. Periodic task (from the heartbeat scheduler @ 3min default + cadence) calls :meth:`AlertNotifier.run_notification_cycle`. + 2. That method calls + :func:`kora_cli.alerts.aggregator.compute_active_alerts` and + computes set-diff against ``last_alert_ids`` from the + previous cycle. + 3. For each newly-firing alert, dispatch to the channel matched + by severity (rules below). + 4. After the cycle, ``last_alert_ids = active_ids`` so a still- + firing alert doesn't re-notify next cycle. + +# Channel rules (PM-default Q2) + + - ``critical`` → Slack DM (immediate, operator action needed) + - ``warning`` → Slack DM (operator should know soon) + - ``info`` → email (not action-required; v1 fires immediately, + digest-shaping deferred to ST2) + +# Dedup semantics (PM-default Q3) + +``last_alert_ids`` is in-memory ONLY. Daemon restart starts with +an empty set, so all currently-active alerts get notified on the +first cycle as "new" — operator gets re-pinged on every restart. +Persistence across restarts is a future bucket if the re-ping +becomes annoying. + +# Failure semantics + +Send failures DO NOT cause re-notify on the next cycle. The +alert ID enters ``last_alert_ids`` regardless of send outcome — +this prevents spam if Slack/SMTP are flapping. The audit JSONL +records the failure so the operator can triage via the audit +panel if they were expecting a notification that didn't arrive. + +Trade-off: a single transient Slack 429 means the operator +might miss that alert. ST2's per-category cooldown + the +operator runbook addendum cover this gap with a different +mechanism (the alert STILL shows in the cockpit; notification +is a "push" convenience, not a delivery guarantee). +""" + +from __future__ import annotations + +import logging +import os +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Callable, List, Optional, Set + +from kora_cli.alerts.aggregator import Alert, compute_active_alerts + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Channel routing +# --------------------------------------------------------------------------- + + +# Severity → channel. Documented in module docstring. +_CHANNEL_SLACK = "slack_dm" +_CHANNEL_EMAIL = "email" + +_SEVERITY_TO_CHANNEL = { + "critical": _CHANNEL_SLACK, + "warning": _CHANNEL_SLACK, + "info": _CHANNEL_EMAIL, +} + +# Severity → emoji prefix in Slack DM body +_SEVERITY_EMOJI = { + "critical": "🔴", + "warning": "🟡", + "info": "🔵", +} + + +# --------------------------------------------------------------------------- +# Envs +# --------------------------------------------------------------------------- + + +JOSHUA_SLACK_USER_ID_ENV = "KORA_SLACK_JOSHUA_USER_ID" +JOSHUA_EMAIL_ADDRESS_ENV = "KORA_EMAIL_JOSHUA_ADDRESS" +KORA_EMAIL_FROM_ADDRESS_ENV = "KORA_EMAIL_KORA_ADDRESS" +COCKPIT_URL_ENV = "KORA_COCKPIT_URL" + + +# --------------------------------------------------------------------------- +# Result types +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class DispatchOutcome: + """One per dispatch attempt. Bundled into :class:`NotificationCycleResult`.""" + + alert_id: str + severity: str + channel: str + success: bool + error: Optional[str] = None + + +@dataclass(frozen=True, slots=True) +class NotificationCycleResult: + """Summary of one ``run_notification_cycle`` call. + + Surfaces cycle telemetry for the audit log + operator triage. + Empty cycles (no newly-firing alerts) return zeros across the + board. + """ + + active_count: int + newly_firing_count: int + newly_resolved_count: int + slack_dispatched: int + email_dispatched: int + dispatch_errors: int + outcomes: List[DispatchOutcome] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Formatting helpers (pure functions — easy to test in isolation) +# --------------------------------------------------------------------------- + + +def _format_relative_time(at_iso: str, *, now: Optional[datetime] = None) -> str: + """Human-readable relative time. Falls back to the raw ISO on parse failure.""" + if not at_iso: + return "(unknown)" + try: + ts = datetime.fromisoformat(at_iso.replace("Z", "+00:00")) + except ValueError: + return at_iso + if ts.tzinfo is None: + ts = ts.replace(tzinfo=timezone.utc) + now = now or datetime.now(timezone.utc) + delta = (now - ts).total_seconds() + if delta < 0: + return at_iso # future-dated; surface raw + if delta < 60: + return f"{int(delta)}s ago" + if delta < 3600: + return f"{int(delta // 60)}m ago" + if delta < 86400: + return f"{int(delta // 3600)}h ago" + return f"{int(delta // 86400)}d ago" + + +def format_slack_dm_text(alert: Alert, *, now: Optional[datetime] = None) -> str: + """Slack DM body per bucket §2(c).""" + emoji = _SEVERITY_EMOJI.get(alert.severity, "🔵") + relative = _format_relative_time(alert.first_seen_at, now=now) + return ( + f"{emoji} [{alert.severity.upper()}] Kora alert\n" + f"{alert.title}\n" + f"\n" + f"{alert.detail}\n" + f"\n" + f"Source: {alert.source_panel_route}\n" + f"First seen: {relative}" + ) + + +def format_email_subject(alert: Alert) -> str: + """Email subject per bucket §2(d).""" + return f"[Kora alert] {alert.severity}: {alert.title}" + + +def format_email_body(alert: Alert) -> str: + """Email body per bucket §2(d). ``KORA_COCKPIT_URL`` env appends + a cockpit link when configured; otherwise omitted.""" + cockpit_url = os.environ.get(COCKPIT_URL_ENV, "").strip() + cockpit_block = f"\n\nCockpit URL: {cockpit_url}" if cockpit_url else "" + return ( + f"{alert.detail}\n" + f"\n" + f"Source panel: {alert.source_panel_route}\n" + f"First seen: {alert.first_seen_at}" + f"{cockpit_block}" + ) + + +# --------------------------------------------------------------------------- +# AlertNotifier +# --------------------------------------------------------------------------- + + +SlackClientFactory = Callable[[], Optional[Any]] +PurelymailClientFactory = Callable[[], Optional[Any]] + + +class AlertNotifier: + """Periodic alert push-notifier with set-diff dedup. + + Constructed by the listener with lazy factories for the + SlackClient + PurelymailClient — they may not be initialized + yet when the listener boots (fail-soft pattern). The factory + is called once per cycle so a client that comes up mid-runtime + starts being usable immediately. + """ + + def __init__( + self, + *, + slack_client_factory: SlackClientFactory, + purelymail_client_factory: PurelymailClientFactory, + compute_alerts: Callable[[], List[Alert]] = compute_active_alerts, + ) -> None: + self._slack_client_factory = slack_client_factory + self._purelymail_client_factory = purelymail_client_factory + self._compute_alerts = compute_alerts + # Last cycle's active alert ids. Empty at construction → + # the first cycle treats all current alerts as newly-firing + # (PM Q3 ruling: fire on first cycle; persistence is future). + self._last_alert_ids: Set[str] = set() + + @property + def last_alert_ids(self) -> Set[str]: + """Read-only view for tests + telemetry.""" + return set(self._last_alert_ids) + + def reset_dedup_state(self) -> None: + """Clear the in-memory dedup set. Listener shutdown calls + this so a subsequent listener start sees a clean slate.""" + self._last_alert_ids = set() + + async def run_notification_cycle(self) -> NotificationCycleResult: + """One cycle: compute active alerts → diff → dispatch new fires. + + Fail-soft: any exception inside the cycle is caught + logged; + an empty :class:`NotificationCycleResult` is returned so the + heartbeat scheduler doesn't crash. + """ + try: + alerts = list(self._compute_alerts()) + except Exception as exc: + logger.warning( + "[kora.alert_notifier] compute_active_alerts raised %r — " + "skipping cycle", + exc, + ) + return NotificationCycleResult( + active_count=0, + newly_firing_count=0, + newly_resolved_count=0, + slack_dispatched=0, + email_dispatched=0, + dispatch_errors=0, + outcomes=[], + ) + + active_ids = {a.id for a in alerts} + newly_firing_ids = active_ids - self._last_alert_ids + newly_resolved_ids = self._last_alert_ids - active_ids + + # Order new-fires by severity (critical first) for predictable + # dispatch sequence. Reuse the aggregator's sort key shape. + severity_rank = {"critical": 0, "warning": 1, "info": 2} + new_fires = [a for a in alerts if a.id in newly_firing_ids] + new_fires.sort( + key=lambda a: (severity_rank.get(a.severity, 99), a.id) + ) + + outcomes: List[DispatchOutcome] = [] + slack_dispatched = 0 + email_dispatched = 0 + dispatch_errors = 0 + for alert in new_fires: + outcome = await self._dispatch_alert(alert) + outcomes.append(outcome) + if outcome.success: + if outcome.channel == _CHANNEL_SLACK: + slack_dispatched += 1 + elif outcome.channel == _CHANNEL_EMAIL: + email_dispatched += 1 + else: + dispatch_errors += 1 + + # Update dedup state AFTER dispatching — even if dispatch failed, + # the alert ID enters last_alert_ids to prevent re-notify spam + # on the next cycle. Audit captures the failure for triage. + self._last_alert_ids = active_ids + + if newly_resolved_ids: + logger.info( + "[kora.alert_notifier] %d alert(s) resolved this cycle: %s", + len(newly_resolved_ids), + sorted(newly_resolved_ids), + ) + + return NotificationCycleResult( + active_count=len(active_ids), + newly_firing_count=len(newly_firing_ids), + newly_resolved_count=len(newly_resolved_ids), + slack_dispatched=slack_dispatched, + email_dispatched=email_dispatched, + dispatch_errors=dispatch_errors, + outcomes=outcomes, + ) + + # ------------------------------------------------------------------ + # Dispatch + # ------------------------------------------------------------------ + + async def _dispatch_alert(self, alert: Alert) -> DispatchOutcome: + """Route one alert to its channel. Records audit + returns + outcome regardless of success/failure.""" + channel = _SEVERITY_TO_CHANNEL.get(alert.severity, _CHANNEL_SLACK) + try: + if channel == _CHANNEL_SLACK: + await self._send_slack_dm(alert) + else: + await self._send_email(alert) + except Exception as exc: + err_text = f"{type(exc).__name__}" + logger.warning( + "[kora.alert_notifier] dispatch failed alert=%s channel=%s: %r", + alert.id, + channel, + exc, + ) + self._emit_audit(alert, channel=channel, status="failed", error=err_text) + return DispatchOutcome( + alert_id=alert.id, + severity=alert.severity, + channel=channel, + success=False, + error=err_text, + ) + + self._emit_audit(alert, channel=channel, status="ok", error=None) + return DispatchOutcome( + alert_id=alert.id, + severity=alert.severity, + channel=channel, + success=True, + ) + + async def _send_slack_dm(self, alert: Alert) -> None: + """Format + send via the live SlackClient. Raises on failure + (caller catches + records audit).""" + client = self._slack_client_factory() + if client is None: + raise RuntimeError("slack_client_unavailable") + + joshua_user_id = os.environ.get( + JOSHUA_SLACK_USER_ID_ENV, "" + ).strip() + if not joshua_user_id: + raise RuntimeError("joshua_slack_user_id_unset") + + text = format_slack_dm_text(alert) + # Slack's chat.postMessage auto-resolves the bot's DM channel + # for a user-id passed as channel_id — same approach the MCP + # kora__send_slack_dm tool uses (mcp_tools.py:1119). + await client.post_dm(channel_id=joshua_user_id, text=text) + + async def _send_email(self, alert: Alert) -> None: + """Format + send via the live PurelymailClient. Raises on failure.""" + client = self._purelymail_client_factory() + if client is None: + raise RuntimeError("purelymail_client_unavailable") + + joshua_email = os.environ.get(JOSHUA_EMAIL_ADDRESS_ENV, "").strip() + if not joshua_email: + raise RuntimeError("joshua_email_address_unset") + + from_addr = os.environ.get(KORA_EMAIL_FROM_ADDRESS_ENV, "").strip() + if not from_addr: + raise RuntimeError("kora_email_from_address_unset") + + subject = format_email_subject(alert) + body_text = format_email_body(alert) + result = await client.send_email( + from_addr=from_addr, + to=[joshua_email], + subject=subject, + body_text=body_text, + ) + # The SMTP client returns a SendResult with status="failed" on + # error rather than raising — surface that as a dispatch + # failure so the audit log records the SMTP code. + if getattr(result, "status", None) == "failed": + raise RuntimeError( + f"smtp_send_failed:{getattr(result, 'smtp_code', None)}" + ) + + # ------------------------------------------------------------------ + # Audit + # ------------------------------------------------------------------ + + def _emit_audit( + self, + alert: Alert, + *, + channel: str, + status: str, + error: Optional[str], + ) -> None: + """Record one dispatch attempt in the audit JSONL. + + Fail-soft: an audit-write error must not propagate (caller + is already inside dispatch try/except).""" + try: + from kora_cli.audit import emit_audit + except Exception as exc: + logger.warning( + "[kora.alert_notifier] audit import failed: %r", exc + ) + return + details = { + "channel": channel, + "alert_id": alert.id, + "severity": alert.severity, + "category": alert.category, + "status": status, + } + if error is not None: + details["error"] = error + try: + emit_audit( + seam="notification.dispatched", + details=details, + source=None, + ) + except Exception as exc: + logger.warning( + "[kora.alert_notifier] emit_audit raised %r — continuing", + exc, + ) diff --git a/kora_cli/audit/jsonl_sink.py b/kora_cli/audit/jsonl_sink.py index cbe13980c8d8..41ac93c30438 100644 --- a/kora_cli/audit/jsonl_sink.py +++ b/kora_cli/audit/jsonl_sink.py @@ -84,6 +84,12 @@ "webhook.dead_letter", "slack_dm.reply_failed", "reasoning.tool_called", + # KR-ALERT-NOTIFY — alert push notifications (Slack DM / + # email) dispatched by the alert_notifier_listener periodic + # task. Each new-fire dispatch emits one entry regardless of + # success/failure; failures are visible in the audit panel + # alongside the alerts panel. + "notification.dispatched", ] SourceName = Literal[ diff --git a/kora_cli/listeners/__init__.py b/kora_cli/listeners/__init__.py index 45469df4a841..f890aa3a9f28 100644 --- a/kora_cli/listeners/__init__.py +++ b/kora_cli/listeners/__init__.py @@ -49,3 +49,12 @@ # short-circuits cleanly). Registered AFTER the SMTP client listener # so the symmetric `current_*_client()` accessors line up. from kora_cli.listeners import email_inbound_imap_listener # noqa: F401 +# KR-ALERT-NOTIFY ST1 — alert push-notifier. Registers a periodic +# heartbeat task that diffs the active alert set + pushes newly- +# firing alerts to Joshua via Slack DM (critical / warning) or +# email (info). Fail-soft on client unavailability — alert IDs +# still enter the dedup set so transient SMTP/Slack failures don't +# cause spam on the next cycle. Imported AFTER the client listeners +# + the email inbound listener so the lazy factories resolve to +# live singletons by the time the first cycle ticks. +from kora_cli.listeners import alert_notifier_listener # noqa: F401 diff --git a/kora_cli/listeners/alert_notifier_listener.py b/kora_cli/listeners/alert_notifier_listener.py new file mode 100644 index 000000000000..46f75504857d --- /dev/null +++ b/kora_cli/listeners/alert_notifier_listener.py @@ -0,0 +1,255 @@ +"""Alert push-notifier daemon listener — KR-ALERT-NOTIFY ST1. + +Registers a periodic task with the heartbeat scheduler that runs +:meth:`AlertNotifier.run_notification_cycle` every +``KORA_ALERT_NOTIFY_INTERVAL_SEC`` (default 180s / 3 min) seconds. + +# Wiring + + - Daemon startup: construct one :class:`AlertNotifier` bound to + the live SlackClient + PurelymailClient lazy factories + (:func:`current_slack_client` / + :func:`current_purelymail_client`). Expose via the module + singleton + accessor pattern. + - Daemon shutdown: clear the notifier's in-memory dedup state + so a subsequent listener start sees a clean slate. The + notifier itself doesn't hold a long-lived transport. + +# Cadence rationale (PM Q1 default) + +3 minutes balances responsiveness (operator gets pinged within +3 min of an alert firing) against burn (slack/email API budget). +``KORA_ALERT_NOTIFY_INTERVAL_SEC`` env override available; values +≤ 0 or non-numeric fall back to the default. + +# Why fail-soft on client unavailability + +The notifier resolves SlackClient / PurelymailClient per cycle +(via lazy factories). If either is unavailable the dispatch +fails, the audit log records the failure, and the alert ID still +enters ``last_alert_ids`` so the next cycle doesn't re-spam. The +alert remains visible in the cockpit (the panel reads the +aggregator directly; notifications are a convenience layer). +""" + +from __future__ import annotations + +import logging +import os +from typing import Any, Optional + +from kora_cli.alerts.notifier import AlertNotifier +from kora_cli.daemon import DEFAULT_SHUTDOWN_TIMEOUT, register_daemon_listener +from kora_cli.listeners.heartbeat import register_periodic_task + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Cadence config +# --------------------------------------------------------------------------- + + +DEFAULT_INTERVAL_SEC: float = 180.0 # 3 min per §4 Q1 default +INTERVAL_ENV: str = "KORA_ALERT_NOTIFY_INTERVAL_SEC" + + +def _read_interval() -> float: + """Resolve cycle cadence from env with sane fallback. + + Mirrors the pattern from + :func:`kora_cli.listeners.mcp_consumption._read_health_check_interval` + + email_inbound_imap_listener's _read_poll_interval — invalid + values (non-numeric, <=0) WARN-log + fall back to default. + """ + raw = os.environ.get(INTERVAL_ENV, "").strip() + if not raw: + return DEFAULT_INTERVAL_SEC + try: + value = float(raw) + except ValueError: + logger.warning( + "[kora.alert_notifier_listener] %s=%r is not numeric; using " + "default %ss", + INTERVAL_ENV, + raw, + DEFAULT_INTERVAL_SEC, + ) + return DEFAULT_INTERVAL_SEC + if value <= 0: + logger.warning( + "[kora.alert_notifier_listener] %s=%s must be > 0; using " + "default %ss", + INTERVAL_ENV, + value, + DEFAULT_INTERVAL_SEC, + ) + return DEFAULT_INTERVAL_SEC + return value + + +# --------------------------------------------------------------------------- +# Module-level singleton + accessor +# --------------------------------------------------------------------------- + + +_notifier_singleton: Optional[AlertNotifier] = None + + +def _set_singleton(notifier: AlertNotifier) -> None: + global _notifier_singleton + _notifier_singleton = notifier + + +def _clear_singleton() -> None: + global _notifier_singleton + _notifier_singleton = None + + +def current_alert_notifier() -> Optional[AlertNotifier]: + """Return the live :class:`AlertNotifier`, or ``None``. + + ``None`` when the daemon isn't running with this listener + registered + started. + """ + return _notifier_singleton + + +# --------------------------------------------------------------------------- +# Periodic task — runs per heartbeat scheduler tick +# --------------------------------------------------------------------------- + + +async def run_notification_cycle() -> None: + """One scheduler tick. Short-circuits cleanly when the notifier + singleton is absent (daemon shutdown in progress, fail-soft boot, + etc.). The notifier's own ``run_notification_cycle`` is fail-soft + too — exceptions inside don't crash the scheduler. + """ + notifier = current_alert_notifier() + if notifier is None: + logger.debug( + "[kora.alert_notifier_listener] tick skipped: no active notifier" + ) + return + try: + result = await notifier.run_notification_cycle() + except Exception as exc: + # Defense in depth — AlertNotifier.run_notification_cycle + # already catches inside; this is the outermost guard so the + # scheduler keeps ticking. + logger.warning( + "[kora.alert_notifier_listener] cycle raised past inner " + "catch: %r", + exc, + ) + return + + if result.newly_firing_count > 0 or result.dispatch_errors > 0: + logger.info( + "[kora.alert_notifier_listener] cycle: active=%d new=%d " + "resolved=%d slack=%d email=%d errors=%d", + result.active_count, + result.newly_firing_count, + result.newly_resolved_count, + result.slack_dispatched, + result.email_dispatched, + result.dispatch_errors, + ) + + +# --------------------------------------------------------------------------- +# Listener +# --------------------------------------------------------------------------- + + +class AlertNotifierListener: + """Holds the live :class:`AlertNotifier` for the daemon's lifetime.""" + + async def startup(self) -> None: + """Construct the notifier bound to live client lazy factories. + + Fail-soft on construction errors so daemon boot doesn't + crash even in unusual environments where the alert module + imports fail. + """ + try: + slack_factory = _slack_client_factory + purelymail_factory = _purelymail_client_factory + notifier = AlertNotifier( + slack_client_factory=slack_factory, + purelymail_client_factory=purelymail_factory, + ) + except Exception as exc: + logger.warning( + "[kora.alert_notifier_listener] startup raised %r — " + "notifier disabled this run", + exc, + ) + _clear_singleton() + return + + _set_singleton(notifier) + logger.info( + "[kora.alert_notifier_listener] AlertNotifier constructed; " + "cycle cadence=%ss", + _read_interval(), + ) + + async def shutdown(self) -> None: + """Reset dedup state + clear the singleton. The notifier has + no transport state to release.""" + notifier = _notifier_singleton + if notifier is not None: + notifier.reset_dedup_state() + _clear_singleton() + logger.info("[kora.alert_notifier_listener] AlertNotifier cleared") + + +def _slack_client_factory() -> Optional[Any]: + """Lazy resolver for the live SlackClient. Falls through to + ``None`` when the listener stack isn't initialized (test paths).""" + try: + from kora_cli.listeners.slack_client_listener import ( + current_slack_client, + ) + except Exception: + return None + return current_slack_client() + + +def _purelymail_client_factory() -> Optional[Any]: + """Lazy resolver for the live PurelymailClient. Same fall-through + posture as the Slack factory.""" + try: + from kora_cli.listeners.purelymail_client_listener import ( + current_purelymail_client, + ) + except Exception: + return None + return current_purelymail_client() + + +# --------------------------------------------------------------------------- +# Factory + registration (import-time side effect) +# --------------------------------------------------------------------------- + + +def _factory(): + listener = AlertNotifierListener() + return (listener.startup, listener.shutdown, DEFAULT_SHUTDOWN_TIMEOUT) + + +register_daemon_listener("alert_notifier", _factory) + + +# --------------------------------------------------------------------------- +# Periodic-task registration (import-time side effect) +# --------------------------------------------------------------------------- + + +register_periodic_task( + "alerts.notify", + interval_seconds=_read_interval(), + callable=run_notification_cycle, +) diff --git a/tests/kora_cli/alerts/test_notifier.py b/tests/kora_cli/alerts/test_notifier.py new file mode 100644 index 000000000000..1457985367b9 --- /dev/null +++ b/tests/kora_cli/alerts/test_notifier.py @@ -0,0 +1,680 @@ +"""Tests for the KR-ALERT-NOTIFY ST1 push-notifier. + +Bucket §2 scenarios: + + Formatting helpers (pure functions): + 1. Slack DM text — emoji + severity uppercase + title + detail + + source + relative-time line + 2. Email subject — `[Kora alert] {severity}: {title}` format + 3. Email body — detail + source + first_seen + optional cockpit URL + 4. Relative time formatting under various deltas + + Channel routing: + 5. critical → Slack DM + 6. warning → Slack DM + 7. info → email + 8. Unknown severity defaults to Slack DM (defensive) + + Dedup semantics: + 9. First cycle: all active alerts fire as newly-firing + 10. Repeat cycle: same alert IDs no re-dispatch + 11. Newly-firing alert mid-stream: only the new one dispatches + 12. Resolved alert: no dispatch on resolution + cleared from set + 13. Mixed: new + still-firing + resolved → only new dispatches + + Per-cycle failure isolation: + 14. SlackClient unavailable → dispatch fails + alert STILL enters + dedup set (no spam on retry) + 15. PurelymailClient unavailable → same + 16. Joshua env unset (slack or email) → dispatch fails, alert + enters dedup set + 17. Slack post_dm raises → audit failed + outcome.success=False + 18. SMTP send returns failed SendResult → outcome.success=False + 19. compute_active_alerts raises → empty result (cycle no-op) + 20. emit_audit raises → cycle still returns clean result + + Audit emit: + 21. Successful dispatch emits notification.dispatched with status=ok + 22. Failed dispatch emits notification.dispatched with status=failed + + error code + 23. Audit details include alert_id + severity + category + channel + + Reset: + 24. reset_dedup_state clears last_alert_ids + + Cycle telemetry: + 25. NotificationCycleResult counts reconcile: slack + email + + errors == newly_firing_count +""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from kora_cli.alerts.aggregator import Alert +from kora_cli.alerts.notifier import ( + AlertNotifier, + DispatchOutcome, + NotificationCycleResult, + format_email_body, + format_email_subject, + format_slack_dm_text, + _format_relative_time, +) + + +_JOSHUA_USER_ID = "U01JOSHUA" +_JOSHUA_EMAIL = "joshua@stormhavenenterprises.com" +_KORA_EMAIL = "kora@stormhavenenterprises.com" + + +@pytest.fixture(autouse=True) +def _isolate_env(monkeypatch): + monkeypatch.setenv("KORA_SLACK_JOSHUA_USER_ID", _JOSHUA_USER_ID) + monkeypatch.setenv("KORA_EMAIL_JOSHUA_ADDRESS", _JOSHUA_EMAIL) + monkeypatch.setenv("KORA_EMAIL_KORA_ADDRESS", _KORA_EMAIL) + monkeypatch.delenv("KORA_COCKPIT_URL", raising=False) + + +def _make_alert( + *, + id: str = "cost_ladder_warned", + severity: str = "warning", + category: str = "cost_ladder", + title: str = "Budget at 80% of monthly cap", + detail: str = "Cost ladder at warn_75 — reasoning still runs.", + source_panel: str = "cost", + source_panel_route: str = "/cost-state", + first_seen_at: str = "2026-05-23T10:00:00Z", +) -> Alert: + return Alert( + id=id, + severity=severity, + category=category, + title=title, + detail=detail, + source_panel=source_panel, + source_panel_route=source_panel_route, + first_seen_at=first_seen_at, + ) + + +def _make_clients_factories(*, slack_client=None, purelymail_client=None): + """Build the two factory callables AlertNotifier expects.""" + return (lambda: slack_client), (lambda: purelymail_client) + + +def _make_slack_client(): + client = MagicMock() + client.post_dm = AsyncMock(return_value={"ts": "1716480000.000001"}) + return client + + +def _make_purelymail_client(*, status: str = "ok"): + client = MagicMock() + result = MagicMock() + result.status = status + result.smtp_code = 250 if status == "ok" else 550 + client.send_email = AsyncMock(return_value=result) + return client + + +# =========================================================================== +# Formatting helpers +# =========================================================================== + + +def test_format_slack_dm_text_includes_emoji_and_severity(): + alert = _make_alert(severity="critical", title="Halted", detail="Budget halted.") + text = format_slack_dm_text( + alert, now=datetime(2026, 5, 23, 10, 5, 0, tzinfo=timezone.utc) + ) + assert "🔴" in text + assert "[CRITICAL]" in text + assert "Halted" in text + assert "Budget halted." in text + assert "/cost-state" in text + assert "5m ago" in text + + +def test_format_slack_dm_text_warning_emoji(): + alert = _make_alert(severity="warning") + text = format_slack_dm_text(alert) + assert "🟡" in text + assert "[WARNING]" in text + + +def test_format_email_subject_includes_severity_and_title(): + alert = _make_alert(severity="info", title="capability denied surge") + assert format_email_subject(alert) == ( + "[Kora alert] info: capability denied surge" + ) + + +def test_format_email_body_omits_cockpit_url_when_unset(monkeypatch): + monkeypatch.delenv("KORA_COCKPIT_URL", raising=False) + alert = _make_alert(detail="see panel", source_panel_route="/alerts") + body = format_email_body(alert) + assert "see panel" in body + assert "/alerts" in body + assert "Cockpit URL" not in body + + +def test_format_email_body_includes_cockpit_url_when_set(monkeypatch): + monkeypatch.setenv("KORA_COCKPIT_URL", "https://kora.example/cockpit") + body = format_email_body(_make_alert()) + assert "https://kora.example/cockpit" in body + + +def test_format_relative_time_under_minute(): + now = datetime(2026, 5, 23, 12, 0, 30, tzinfo=timezone.utc) + assert _format_relative_time("2026-05-23T12:00:00Z", now=now) == "30s ago" + + +def test_format_relative_time_minutes(): + now = datetime(2026, 5, 23, 12, 5, 0, tzinfo=timezone.utc) + assert _format_relative_time("2026-05-23T12:00:00Z", now=now) == "5m ago" + + +def test_format_relative_time_hours(): + now = datetime(2026, 5, 23, 14, 0, 0, tzinfo=timezone.utc) + assert _format_relative_time("2026-05-23T12:00:00Z", now=now) == "2h ago" + + +def test_format_relative_time_days(): + now = datetime(2026, 5, 25, 12, 0, 0, tzinfo=timezone.utc) + assert _format_relative_time("2026-05-23T12:00:00Z", now=now) == "2d ago" + + +def test_format_relative_time_malformed_returns_raw(): + assert _format_relative_time("not-a-timestamp") == "not-a-timestamp" + + +def test_format_relative_time_empty_returns_unknown(): + assert _format_relative_time("") == "(unknown)" + + +# =========================================================================== +# Channel routing +# =========================================================================== + + +@pytest.mark.asyncio +async def test_critical_routes_to_slack(): + slack = _make_slack_client() + purelymail = _make_purelymail_client() + sf, pf = _make_clients_factories( + slack_client=slack, purelymail_client=purelymail + ) + notifier = AlertNotifier( + slack_client_factory=sf, + purelymail_client_factory=pf, + compute_alerts=lambda: [_make_alert(severity="critical")], + ) + result = await notifier.run_notification_cycle() + assert result.slack_dispatched == 1 + assert result.email_dispatched == 0 + slack.post_dm.assert_awaited_once() + purelymail.send_email.assert_not_called() + + +@pytest.mark.asyncio +async def test_warning_routes_to_slack(): + slack = _make_slack_client() + purelymail = _make_purelymail_client() + sf, pf = _make_clients_factories( + slack_client=slack, purelymail_client=purelymail + ) + notifier = AlertNotifier( + slack_client_factory=sf, + purelymail_client_factory=pf, + compute_alerts=lambda: [_make_alert(severity="warning")], + ) + result = await notifier.run_notification_cycle() + assert result.slack_dispatched == 1 + slack.post_dm.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_info_routes_to_email(): + slack = _make_slack_client() + purelymail = _make_purelymail_client() + sf, pf = _make_clients_factories( + slack_client=slack, purelymail_client=purelymail + ) + notifier = AlertNotifier( + slack_client_factory=sf, + purelymail_client_factory=pf, + compute_alerts=lambda: [_make_alert(severity="info")], + ) + result = await notifier.run_notification_cycle() + assert result.email_dispatched == 1 + assert result.slack_dispatched == 0 + purelymail.send_email.assert_awaited_once() + slack.post_dm.assert_not_called() + + +# =========================================================================== +# Dedup semantics +# =========================================================================== + + +@pytest.mark.asyncio +async def test_first_cycle_fires_all_active_alerts(): + """PM Q3 default: empty last_alert_ids on first cycle means + everything currently active gets notified.""" + slack = _make_slack_client() + purelymail = _make_purelymail_client() + sf, pf = _make_clients_factories( + slack_client=slack, purelymail_client=purelymail + ) + alerts = [ + _make_alert(id="cost_ladder_warned", severity="warning"), + _make_alert(id="operator_paused", severity="critical"), + ] + notifier = AlertNotifier( + slack_client_factory=sf, + purelymail_client_factory=pf, + compute_alerts=lambda: alerts, + ) + result = await notifier.run_notification_cycle() + assert result.newly_firing_count == 2 + assert result.slack_dispatched == 2 + + +@pytest.mark.asyncio +async def test_repeat_cycle_no_redispatch_for_still_firing(): + slack = _make_slack_client() + purelymail = _make_purelymail_client() + sf, pf = _make_clients_factories( + slack_client=slack, purelymail_client=purelymail + ) + alerts = [_make_alert(id="cost_ladder_warned", severity="warning")] + notifier = AlertNotifier( + slack_client_factory=sf, + purelymail_client_factory=pf, + compute_alerts=lambda: alerts, + ) + # First cycle fires. + r1 = await notifier.run_notification_cycle() + assert r1.slack_dispatched == 1 + # Second cycle (same alert active): no re-dispatch. + r2 = await notifier.run_notification_cycle() + assert r2.newly_firing_count == 0 + assert r2.slack_dispatched == 0 + assert slack.post_dm.await_count == 1 + + +@pytest.mark.asyncio +async def test_new_alert_mid_stream_only_dispatches_new_one(): + slack = _make_slack_client() + purelymail = _make_purelymail_client() + sf, pf = _make_clients_factories( + slack_client=slack, purelymail_client=purelymail + ) + alerts_state = [[_make_alert(id="a", severity="warning")]] + + notifier = AlertNotifier( + slack_client_factory=sf, + purelymail_client_factory=pf, + compute_alerts=lambda: alerts_state[0], + ) + await notifier.run_notification_cycle() + # Add a second alert. + alerts_state[0] = [ + _make_alert(id="a", severity="warning"), + _make_alert(id="b", severity="critical"), + ] + r2 = await notifier.run_notification_cycle() + assert r2.newly_firing_count == 1 + assert r2.slack_dispatched == 1 + # Total slack calls = 1 (cycle 1) + 1 (cycle 2's new alert) = 2 + assert slack.post_dm.await_count == 2 + + +@pytest.mark.asyncio +async def test_resolved_alert_clears_from_dedup_set(): + slack = _make_slack_client() + purelymail = _make_purelymail_client() + sf, pf = _make_clients_factories( + slack_client=slack, purelymail_client=purelymail + ) + alerts_state = [[_make_alert(id="x", severity="warning")]] + notifier = AlertNotifier( + slack_client_factory=sf, + purelymail_client_factory=pf, + compute_alerts=lambda: alerts_state[0], + ) + await notifier.run_notification_cycle() + assert "x" in notifier.last_alert_ids + # Resolve. + alerts_state[0] = [] + r2 = await notifier.run_notification_cycle() + assert r2.newly_resolved_count == 1 + assert "x" not in notifier.last_alert_ids + + +@pytest.mark.asyncio +async def test_mixed_new_still_firing_resolved(): + slack = _make_slack_client() + purelymail = _make_purelymail_client() + sf, pf = _make_clients_factories( + slack_client=slack, purelymail_client=purelymail + ) + alerts_state = [ + [ + _make_alert(id="a", severity="warning"), + _make_alert(id="b", severity="warning"), + ] + ] + notifier = AlertNotifier( + slack_client_factory=sf, + purelymail_client_factory=pf, + compute_alerts=lambda: alerts_state[0], + ) + # Cycle 1: a + b new + await notifier.run_notification_cycle() + # Cycle 2: a still firing, b resolved, c new + alerts_state[0] = [ + _make_alert(id="a", severity="warning"), + _make_alert(id="c", severity="critical"), + ] + r2 = await notifier.run_notification_cycle() + assert r2.newly_firing_count == 1 # only c + assert r2.newly_resolved_count == 1 # b + assert r2.slack_dispatched == 1 + + +# =========================================================================== +# Failure isolation +# =========================================================================== + + +@pytest.mark.asyncio +async def test_slack_client_unavailable_no_redispatch(): + sf, pf = _make_clients_factories( + slack_client=None, purelymail_client=_make_purelymail_client() + ) + notifier = AlertNotifier( + slack_client_factory=sf, + purelymail_client_factory=pf, + compute_alerts=lambda: [_make_alert(severity="critical", id="boom")], + ) + r1 = await notifier.run_notification_cycle() + assert r1.dispatch_errors == 1 + assert "boom" in notifier.last_alert_ids + # Second cycle: alert still active, but no re-dispatch (spam protection). + r2 = await notifier.run_notification_cycle() + assert r2.newly_firing_count == 0 + assert r2.dispatch_errors == 0 + + +@pytest.mark.asyncio +async def test_purelymail_client_unavailable(): + sf, pf = _make_clients_factories( + slack_client=_make_slack_client(), purelymail_client=None + ) + notifier = AlertNotifier( + slack_client_factory=sf, + purelymail_client_factory=pf, + compute_alerts=lambda: [_make_alert(severity="info")], + ) + result = await notifier.run_notification_cycle() + assert result.dispatch_errors == 1 + assert result.email_dispatched == 0 + + +@pytest.mark.asyncio +async def test_joshua_slack_id_unset(monkeypatch): + monkeypatch.delenv("KORA_SLACK_JOSHUA_USER_ID", raising=False) + sf, pf = _make_clients_factories( + slack_client=_make_slack_client(), + purelymail_client=_make_purelymail_client(), + ) + notifier = AlertNotifier( + slack_client_factory=sf, + purelymail_client_factory=pf, + compute_alerts=lambda: [_make_alert(severity="critical", id="x")], + ) + result = await notifier.run_notification_cycle() + assert result.dispatch_errors == 1 + assert "x" in notifier.last_alert_ids + + +@pytest.mark.asyncio +async def test_joshua_email_unset(monkeypatch): + monkeypatch.delenv("KORA_EMAIL_JOSHUA_ADDRESS", raising=False) + sf, pf = _make_clients_factories( + slack_client=_make_slack_client(), + purelymail_client=_make_purelymail_client(), + ) + notifier = AlertNotifier( + slack_client_factory=sf, + purelymail_client_factory=pf, + compute_alerts=lambda: [_make_alert(severity="info")], + ) + result = await notifier.run_notification_cycle() + assert result.dispatch_errors == 1 + + +@pytest.mark.asyncio +async def test_slack_post_dm_raises(): + slack = _make_slack_client() + slack.post_dm.side_effect = RuntimeError("slack_429") + sf, pf = _make_clients_factories( + slack_client=slack, purelymail_client=_make_purelymail_client() + ) + notifier = AlertNotifier( + slack_client_factory=sf, + purelymail_client_factory=pf, + compute_alerts=lambda: [_make_alert(severity="critical")], + ) + result = await notifier.run_notification_cycle() + assert result.dispatch_errors == 1 + assert len(result.outcomes) == 1 + assert result.outcomes[0].success is False + assert "RuntimeError" in (result.outcomes[0].error or "") + + +@pytest.mark.asyncio +async def test_smtp_failed_sendresult_marks_failure(): + purelymail = _make_purelymail_client(status="failed") + sf, pf = _make_clients_factories( + slack_client=_make_slack_client(), purelymail_client=purelymail + ) + notifier = AlertNotifier( + slack_client_factory=sf, + purelymail_client_factory=pf, + compute_alerts=lambda: [_make_alert(severity="info")], + ) + result = await notifier.run_notification_cycle() + assert result.dispatch_errors == 1 + assert result.outcomes[0].success is False + + +@pytest.mark.asyncio +async def test_compute_active_alerts_raises_returns_empty_result(): + sf, pf = _make_clients_factories( + slack_client=_make_slack_client(), + purelymail_client=_make_purelymail_client(), + ) + + def boom(): + raise RuntimeError("aggregator dead") + + notifier = AlertNotifier( + slack_client_factory=sf, + purelymail_client_factory=pf, + compute_alerts=boom, + ) + result = await notifier.run_notification_cycle() + assert result == NotificationCycleResult( + active_count=0, + newly_firing_count=0, + newly_resolved_count=0, + slack_dispatched=0, + email_dispatched=0, + dispatch_errors=0, + outcomes=[], + ) + + +@pytest.mark.asyncio +async def test_emit_audit_raises_doesnt_break_cycle(): + sf, pf = _make_clients_factories( + slack_client=_make_slack_client(), + purelymail_client=_make_purelymail_client(), + ) + notifier = AlertNotifier( + slack_client_factory=sf, + purelymail_client_factory=pf, + compute_alerts=lambda: [_make_alert(severity="critical")], + ) + with patch( + "kora_cli.audit.emit_audit", side_effect=RuntimeError("audit dead") + ): + result = await notifier.run_notification_cycle() + # Cycle still succeeded (slack post_dm ran before audit emit). + assert result.slack_dispatched == 1 + + +# =========================================================================== +# Audit emit shape +# =========================================================================== + + +@pytest.mark.asyncio +async def test_audit_records_success_dispatch(): + sf, pf = _make_clients_factories( + slack_client=_make_slack_client(), + purelymail_client=_make_purelymail_client(), + ) + notifier = AlertNotifier( + slack_client_factory=sf, + purelymail_client_factory=pf, + compute_alerts=lambda: [ + _make_alert( + id="cost_ladder_warned", + severity="warning", + category="cost_ladder", + ) + ], + ) + with patch("kora_cli.audit.emit_audit") as mock_emit: + await notifier.run_notification_cycle() + mock_emit.assert_called_once() + call_kwargs = mock_emit.call_args.kwargs + assert call_kwargs["seam"] == "notification.dispatched" + details = call_kwargs["details"] + assert details["channel"] == "slack_dm" + assert details["alert_id"] == "cost_ladder_warned" + assert details["severity"] == "warning" + assert details["category"] == "cost_ladder" + assert details["status"] == "ok" + assert "error" not in details # success path — error omitted + + +@pytest.mark.asyncio +async def test_audit_records_failure_dispatch_with_error_code(): + slack = _make_slack_client() + slack.post_dm.side_effect = RuntimeError("slack_429") + sf, pf = _make_clients_factories( + slack_client=slack, purelymail_client=_make_purelymail_client() + ) + notifier = AlertNotifier( + slack_client_factory=sf, + purelymail_client_factory=pf, + compute_alerts=lambda: [_make_alert(severity="critical")], + ) + with patch("kora_cli.audit.emit_audit") as mock_emit: + await notifier.run_notification_cycle() + details = mock_emit.call_args.kwargs["details"] + assert details["status"] == "failed" + assert details["error"] == "RuntimeError" + + +# =========================================================================== +# Reset + telemetry +# =========================================================================== + + +@pytest.mark.asyncio +async def test_reset_dedup_state_clears_set(): + sf, pf = _make_clients_factories( + slack_client=_make_slack_client(), + purelymail_client=_make_purelymail_client(), + ) + notifier = AlertNotifier( + slack_client_factory=sf, + purelymail_client_factory=pf, + compute_alerts=lambda: [_make_alert(severity="warning", id="x")], + ) + await notifier.run_notification_cycle() + assert "x" in notifier.last_alert_ids + notifier.reset_dedup_state() + assert notifier.last_alert_ids == set() + + +@pytest.mark.asyncio +async def test_cycle_telemetry_counts_reconcile(): + alerts = [ + _make_alert(id=f"a{i}", severity="warning") for i in range(3) + ] + [_make_alert(id="info1", severity="info")] + sf, pf = _make_clients_factories( + slack_client=_make_slack_client(), + purelymail_client=_make_purelymail_client(), + ) + notifier = AlertNotifier( + slack_client_factory=sf, + purelymail_client_factory=pf, + compute_alerts=lambda: alerts, + ) + result = await notifier.run_notification_cycle() + assert result.newly_firing_count == 4 + assert ( + result.slack_dispatched + + result.email_dispatched + + result.dispatch_errors + == result.newly_firing_count + ) + assert result.slack_dispatched == 3 + assert result.email_dispatched == 1 + + +@pytest.mark.asyncio +async def test_severity_sort_order_in_dispatch_outcomes(): + """Cycle should dispatch critical-first, then warning, then info, + then by id. Outcomes recorded in the same order.""" + alerts = [ + _make_alert(id="info1", severity="info"), + _make_alert(id="warn1", severity="warning"), + _make_alert(id="crit1", severity="critical"), + ] + sf, pf = _make_clients_factories( + slack_client=_make_slack_client(), + purelymail_client=_make_purelymail_client(), + ) + notifier = AlertNotifier( + slack_client_factory=sf, + purelymail_client_factory=pf, + compute_alerts=lambda: alerts, + ) + result = await notifier.run_notification_cycle() + ids = [o.alert_id for o in result.outcomes] + assert ids == ["crit1", "warn1", "info1"] + + +# =========================================================================== +# DispatchOutcome shape +# =========================================================================== + + +def test_dispatch_outcome_is_frozen(): + o = DispatchOutcome( + alert_id="x", severity="warning", channel="slack_dm", success=True + ) + with pytest.raises(Exception): # FrozenInstanceError or AttributeError + o.success = False # type: ignore diff --git a/tests/kora_cli/test_listeners/test_alert_notifier_listener.py b/tests/kora_cli/test_listeners/test_alert_notifier_listener.py new file mode 100644 index 000000000000..c919734ed116 --- /dev/null +++ b/tests/kora_cli/test_listeners/test_alert_notifier_listener.py @@ -0,0 +1,226 @@ +"""Tests for the KR-ALERT-NOTIFY ST1 daemon listener. + +Bucket §2(e) scenarios: + 1. Listener registered in LISTENER_REGISTRY at import time + 2. Periodic task `alerts.notify` registered with the heartbeat + scheduler at import time + 3. Default cadence is 180s (3 min); env override respected + 4. Invalid env value falls back to default + WARNs + 5. Listener startup populates the module singleton with a live + AlertNotifier; current_alert_notifier() returns it + 6. Listener shutdown resets dedup state + clears singleton + 7. run_notification_cycle short-circuits cleanly when no notifier + is active + 8. run_notification_cycle defense-in-depth catch — notifier raise + does not kill the scheduler + 9. Factory tuple shape correct (startup, shutdown, timeout) +""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from kora_cli import daemon as daemon_mod +from kora_cli.listeners import alert_notifier_listener +from kora_cli.listeners.alert_notifier_listener import ( + DEFAULT_INTERVAL_SEC, + INTERVAL_ENV, + AlertNotifierListener, + _clear_singleton, + _factory, + _read_interval, + current_alert_notifier, + run_notification_cycle, +) +from kora_cli.listeners.heartbeat import PERIODIC_TASK_REGISTRY + + +@pytest.fixture(autouse=True) +def _reset_singleton(): + _clear_singleton() + yield + _clear_singleton() + + +# =========================================================================== +# Registration +# =========================================================================== + + +def test_listener_registered_in_daemon_registry(): + registered_names = {name for name, _factory in daemon_mod.LISTENER_REGISTRY} + assert "alert_notifier" in registered_names + + +def test_periodic_task_registered(): + names = [t.name for t in PERIODIC_TASK_REGISTRY] + assert "alerts.notify" in names + + +def test_factory_returns_tuple_shape(): + startup, shutdown, timeout = _factory() + assert callable(startup) + assert callable(shutdown) + assert isinstance(timeout, (int, float)) + assert timeout > 0 + + +# =========================================================================== +# Cadence +# =========================================================================== + + +def test_read_interval_default(monkeypatch): + monkeypatch.delenv(INTERVAL_ENV, raising=False) + assert _read_interval() == DEFAULT_INTERVAL_SEC == 180.0 + + +def test_read_interval_env_override(monkeypatch): + monkeypatch.setenv(INTERVAL_ENV, "60") + assert _read_interval() == 60.0 + + +def test_read_interval_invalid_falls_back(monkeypatch, caplog): + monkeypatch.setenv(INTERVAL_ENV, "not-a-number") + with caplog.at_level("WARNING"): + assert _read_interval() == DEFAULT_INTERVAL_SEC + assert any( + "is not numeric" in r.message for r in caplog.records + ) + + +def test_read_interval_zero_falls_back(monkeypatch): + monkeypatch.setenv(INTERVAL_ENV, "0") + assert _read_interval() == DEFAULT_INTERVAL_SEC + + +def test_read_interval_negative_falls_back(monkeypatch): + monkeypatch.setenv(INTERVAL_ENV, "-5") + assert _read_interval() == DEFAULT_INTERVAL_SEC + + +# =========================================================================== +# Listener lifecycle +# =========================================================================== + + +@pytest.mark.asyncio +async def test_startup_populates_singleton(): + listener = AlertNotifierListener() + await listener.startup() + notifier = current_alert_notifier() + assert notifier is not None + + +@pytest.mark.asyncio +async def test_startup_failsoft_on_unexpected_exception(): + listener = AlertNotifierListener() + with patch( + "kora_cli.listeners.alert_notifier_listener.AlertNotifier", + side_effect=RuntimeError("unexpected"), + ): + await listener.startup() + assert current_alert_notifier() is None + + +@pytest.mark.asyncio +async def test_shutdown_resets_dedup_and_clears_singleton(): + listener = AlertNotifierListener() + await listener.startup() + notifier = current_alert_notifier() + assert notifier is not None + # Spy on reset_dedup_state. + with patch.object(notifier, "reset_dedup_state") as mock_reset: + await listener.shutdown() + mock_reset.assert_called_once() + assert current_alert_notifier() is None + + +@pytest.mark.asyncio +async def test_shutdown_idempotent_without_startup(): + listener = AlertNotifierListener() + await listener.shutdown() + assert current_alert_notifier() is None + + +# =========================================================================== +# run_notification_cycle behavior +# =========================================================================== + + +@pytest.mark.asyncio +async def test_run_cycle_short_circuits_without_singleton(): + # No singleton. + await run_notification_cycle() # must not raise + + +@pytest.mark.asyncio +async def test_run_cycle_invokes_notifier(): + fake_notifier = MagicMock() + fake_result = MagicMock() + fake_result.newly_firing_count = 0 + fake_result.dispatch_errors = 0 + fake_result.active_count = 0 + fake_result.newly_resolved_count = 0 + fake_result.slack_dispatched = 0 + fake_result.email_dispatched = 0 + fake_notifier.run_notification_cycle = AsyncMock(return_value=fake_result) + with patch.object( + alert_notifier_listener, + "_notifier_singleton", + fake_notifier, + ): + await run_notification_cycle() + fake_notifier.run_notification_cycle.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_run_cycle_swallows_unexpected_exceptions(): + """Defense in depth — the notifier's run_notification_cycle + already catches inside; this outer guard catches any path that + bypasses the inner catch.""" + fake_notifier = MagicMock() + fake_notifier.run_notification_cycle = AsyncMock( + side_effect=RuntimeError("unexpected from notifier") + ) + with patch.object( + alert_notifier_listener, + "_notifier_singleton", + fake_notifier, + ): + # Must not raise. + await run_notification_cycle() + + +# =========================================================================== +# Factory wiring +# =========================================================================== + + +def test_slack_client_factory_returns_none_when_listener_absent(): + """Factory used internally to lazily resolve the SlackClient. + When the slack_client_listener isn't imported / running, the + factory returns None gracefully.""" + from kora_cli.listeners.alert_notifier_listener import ( + _slack_client_factory, + ) + + with patch( + "kora_cli.listeners.slack_client_listener.current_slack_client", + return_value=None, + ): + assert _slack_client_factory() is None + + +def test_purelymail_client_factory_returns_none_when_listener_absent(): + from kora_cli.listeners.alert_notifier_listener import ( + _purelymail_client_factory, + ) + + with patch( + "kora_cli.listeners.purelymail_client_listener.current_purelymail_client", + return_value=None, + ): + assert _purelymail_client_factory() is None