diff --git a/kora_cli/audit/jsonl_sink.py b/kora_cli/audit/jsonl_sink.py index 41ac93c30438..6cbf1a19690d 100644 --- a/kora_cli/audit/jsonl_sink.py +++ b/kora_cli/audit/jsonl_sink.py @@ -90,6 +90,14 @@ # success/failure; failures are visible in the audit panel # alongside the alerts panel. "notification.dispatched", + # KR-PROBE-AUDIT-AND-CONVERT — per-probe issue-detection wake + # signal. Probe cron post-hook detects an issue criterion + # crossing, writes one of these to flag that Kora's reasoning + # SHOULD investigate. The consumer side (reasoning-engine + # wake) is a follow-on bucket (KR-PROBE-WAKE-CONSUMER); v1 + # ships the emission for operator visibility via the audit + # panel + alerts panel without invoking LLM. + "probe.wake_requested", ] SourceName = Literal[ diff --git a/kora_cli/heartbeat_probes/runner.py b/kora_cli/heartbeat_probes/runner.py index a1bc7803d5d8..f62287cc6432 100644 --- a/kora_cli/heartbeat_probes/runner.py +++ b/kora_cli/heartbeat_probes/runner.py @@ -119,6 +119,25 @@ async def run_all_probes( ) results[probe.name] = snapshot _snapshot_cache[probe.name] = snapshot + + # KR-PROBE-AUDIT-AND-CONVERT — post-cycle issue-detection hook. + # Classifies each snapshot into Issue objects + emits one + # ``probe.wake_requested`` audit row per Issue. Routine probing + # remains $0 LLM cost; the audit emission is in-memory + JSONL + # append. Fail-soft per the wake emitter's own contract — any + # exception here logs + continues. + try: + from kora_cli.probes import detect_issues, emit_wake_event + + for issue in detect_issues(results.values()): + emit_wake_event(issue) + except Exception as exc: + logger.warning( + "[kora.heartbeat_probes] post-cycle issue detection raised " + "%r — wake events not emitted this cycle", + exc, + ) + return results diff --git a/kora_cli/probes/__init__.py b/kora_cli/probes/__init__.py new file mode 100644 index 000000000000..dfff6fa2be1f --- /dev/null +++ b/kora_cli/probes/__init__.py @@ -0,0 +1,49 @@ +"""Probe issue-detection + fix-attempt envelope declarations + wake-event +emission — KR-PROBE-AUDIT-AND-CONVERT (Lock R3-8 (b)). + +Lives ALONGSIDE the existing ``kora_cli/heartbeat_probes/`` package +(the cheap cron observers). This package is the issue-detection + +wake layer: pure functions that classify ServiceHealthSnapshot +observations into operator-attention Issue objects, declarative +fix-attempt envelopes (all default OFF per fail-CLOSED discipline), +and the wake-event emitter that writes ``probe.wake_requested`` +audit rows. + +Per spec §2 Phase 1 audit: all 5 probes +(supabase / fly / vercel / sentry / doppler) are already cheap- +cron-only ($0 LLM cost). No conversion was needed in Phase 2 — +the package ships the issue-detection + envelope + wake layer +on top of the existing cheap probes. + +Public surface: + * :class:`Issue` — typed probe issue (per-probe criteria firing) + * :func:`detect_issues` — pure function: snapshots → issues + * :class:`FixEnvelope` — declarative fix-attempt envelope + * ``ENVELOPES`` — per-probe envelope table (env-gated; default OFF) + * :func:`is_envelope_enabled` — env-gated enable check + * :func:`emit_wake_event` — audit-row emitter +""" + +from kora_cli.probes.fix_envelopes import ( + ENVELOPES, + FixEnvelope, + is_envelope_enabled, +) +from kora_cli.probes.issue_detector import ( + Issue, + IssueSeverity, + detect_issue_for_snapshot, + detect_issues, +) +from kora_cli.probes.wake_emitter import emit_wake_event + +__all__ = [ + "ENVELOPES", + "FixEnvelope", + "Issue", + "IssueSeverity", + "detect_issue_for_snapshot", + "detect_issues", + "emit_wake_event", + "is_envelope_enabled", +] diff --git a/kora_cli/probes/fix_envelopes.py b/kora_cli/probes/fix_envelopes.py new file mode 100644 index 000000000000..5b0b246722b9 --- /dev/null +++ b/kora_cli/probes/fix_envelopes.py @@ -0,0 +1,199 @@ +"""Per-probe fix-attempt envelope declarations — KR-PROBE-AUDIT-AND-CONVERT. + +Declarative envelopes only. Each envelope: + + 1. Names a single narrow auto-fix action (e.g., "restart 1 fly + machine"). + 2. Documents what's IN the envelope (eligible failures) and + what's OUT (operator-required). + 3. Is gated on a per-probe enable env (default OFF per + ``feedback-fail-closed-by-default-for-security-infra``). + +v1 SCOPE — declarations + envelope-enabled gate ONLY. Actual +fix-attempt execution (calling the Fly API to restart a machine, +etc.) is deferred to a follow-on bucket +(KR-PROBE-AUTOFIX-EXECUTION) so the capability-matrix / +SECDEF / audit story can be reviewed before Kora's daemon starts +mutating cloud infrastructure. + +This module ships the envelope vocabulary + the env-flag plumbing; +the wake-event consumer (also follow-on) will read these envelopes +to decide what Kora's reasoning loop is permitted to attempt. + +# Envelope severity by probe + +| Probe | v1 envelope | Why operator-required (rest) | +|---|---|---| +| supabase | (none) | Substrate is THE critical surface. Any automated retry/recovery touches Joshua's whole data layer. Operator decides. | +| fly | restart 1 unhealthy machine in 1 app | Deploy rollbacks + scale changes + multi-machine actions risk cascading outage. | +| vercel | (none) | Failed deploys may indicate code issues; rolling back blindly can revert intended changes. | +| sentry | (none) | Investigation-only — Sentry issues themselves are bugs in code Kora can't fix at runtime. | +| doppler | (none) | Credential surface. Auto-rotation could lock the runtime out of itself. Operator decides. | + +Only ``fly`` has a v1 envelope; everything else is observation-only +operator-attention. This matches the spec's Phase 4 explicit +guidance. + +# Operator opt-in pattern + +``` +# Default — all envelopes disabled +$ unset KORA_PROBE_AUTOFIX_FLY_ENABLED + +# Per-probe opt-in (operator reviews the envelope first) +$ export KORA_PROBE_AUTOFIX_FLY_ENABLED=true +``` + +Per the fail-CLOSED memory, ``true`` / ``1`` / ``yes`` / ``on`` +enable; everything else (including ``false`` / unset) keeps the +envelope OFF. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from typing import Dict, FrozenSet + + +@dataclass(frozen=True, slots=True) +class FixEnvelope: + """One probe's fix-attempt envelope declaration. + + Wire-stable shape — declarative; the executor (follow-on + bucket) reads these to know what's allowed. + """ + + probe: str + fix_name: str # short stable id (e.g., "restart_unhealthy_machine") + enable_env: str # env var name; truthy value enables + description: str # operator-readable summary + in_envelope: FrozenSet[str] = field(default_factory=frozenset) + out_of_envelope: FrozenSet[str] = field(default_factory=frozenset) + requires_capability: str = "" # capability-matrix gate (when executor lands) + + +# --------------------------------------------------------------------------- +# Per-probe envelope table +# --------------------------------------------------------------------------- + + +ENABLE_ENV_SUPABASE = "KORA_PROBE_AUTOFIX_SUPABASE_ENABLED" +ENABLE_ENV_FLY = "KORA_PROBE_AUTOFIX_FLY_ENABLED" +ENABLE_ENV_VERCEL = "KORA_PROBE_AUTOFIX_VERCEL_ENABLED" +ENABLE_ENV_SENTRY = "KORA_PROBE_AUTOFIX_SENTRY_ENABLED" +ENABLE_ENV_DOPPLER = "KORA_PROBE_AUTOFIX_DOPPLER_ENABLED" + + +_FLY_ENVELOPE = FixEnvelope( + probe="fly", + fix_name="restart_unhealthy_machine", + enable_env=ENABLE_ENV_FLY, + description=( + "Restart exactly ONE Fly machine whose state != 'started' " + "via the Machines API. No multi-machine action, no deploy " + "rollback, no scale change." + ), + in_envelope=frozenset( + { + "single_machine_not_started", + # Health-endpoint-fails-on-one-machine (per spec §2 Phase 4) + # subsumed here — the "not_started" + "stopped" flag captures + # the operator-recoverable cases without requiring per-app + # health-endpoint introspection. + } + ), + out_of_envelope=frozenset( + { + "deploy_rollback", + "scale_up", + "scale_down", + "multi_machine_restart", + "machine_destroy", + "app_create", + "config_change", + } + ), + # Capability literal reserved; the executor follow-on bucket + # will add the cap-matrix gate. v1 declares the requirement so + # the matrix audit can include this expected surface. + requires_capability="probe_autofix_fly_restart", +) + + +# v1 — only the fly envelope is non-trivial; the others ship as +# explicit "none" declarations so a future bucket can populate them +# without touching the envelope-resolution code path. +_SUPABASE_ENVELOPE = FixEnvelope( + probe="supabase", + fix_name="(none)", + enable_env=ENABLE_ENV_SUPABASE, + description=( + "No v1 auto-fix envelope. Substrate is critical — operator " + "decides on any recovery action." + ), +) + +_VERCEL_ENVELOPE = FixEnvelope( + probe="vercel", + fix_name="(none)", + enable_env=ENABLE_ENV_VERCEL, + description=( + "No v1 auto-fix envelope. Failed deploys may indicate code " + "issues; rolling back blindly can revert intended changes. " + "Operator decides." + ), +) + +_SENTRY_ENVELOPE = FixEnvelope( + probe="sentry", + fix_name="(none)", + enable_env=ENABLE_ENV_SENTRY, + description=( + "No v1 auto-fix envelope. Sentry issues themselves are bugs " + "in code Kora can't fix at runtime. Investigation-only." + ), +) + +_DOPPLER_ENVELOPE = FixEnvelope( + probe="doppler", + fix_name="(none)", + enable_env=ENABLE_ENV_DOPPLER, + description=( + "No v1 auto-fix envelope. Credential surface — auto-rotation " + "could lock the runtime out of itself. Operator decides." + ), +) + + +ENVELOPES: Dict[str, FixEnvelope] = { + "supabase": _SUPABASE_ENVELOPE, + "fly": _FLY_ENVELOPE, + "vercel": _VERCEL_ENVELOPE, + "sentry": _SENTRY_ENVELOPE, + "doppler": _DOPPLER_ENVELOPE, +} + + +# Truthy env values per the fail-CLOSED memory + the AUTO_REPLY env +# pattern in email_inbound_handler. Anything else (unset, "false", +# garbage) keeps the envelope OFF. +_TRUTHY_VALUES = frozenset({"true", "1", "yes", "on"}) + + +def is_envelope_enabled(probe: str) -> bool: + """Return ``True`` iff the probe's auto-fix envelope env is + explicitly truthy AND the envelope is non-empty (fix_name != + "(none)"). + + Fail-CLOSED default per ``feedback-fail-closed-by-default-for- + security-infra``: operator must explicitly opt in, AND a v1 + envelope must actually exist for the probe. + """ + envelope = ENVELOPES.get(probe) + if envelope is None: + return False + if envelope.fix_name == "(none)": + return False + raw = os.environ.get(envelope.enable_env, "").strip().lower() + return raw in _TRUTHY_VALUES diff --git a/kora_cli/probes/issue_detector.py b/kora_cli/probes/issue_detector.py new file mode 100644 index 000000000000..504db251c259 --- /dev/null +++ b/kora_cli/probes/issue_detector.py @@ -0,0 +1,232 @@ +"""Per-probe issue-criteria detection — KR-PROBE-AUDIT-AND-CONVERT. + +Pure functions that classify a :class:`ServiceHealthSnapshot` +observation into an :class:`Issue` when the per-probe criterion +fires. Routine probing remains in +``kora_cli/heartbeat_probes/`` and stays $0 LLM cost; this module +is a READ-only classifier layered on top. + +# Per-probe criteria (spec §2 Phase 3) + +| Probe | Criterion → severity | +|---|---| +| supabase | status=unhealthy → ``critical`` (connection failure / probe error). status=degraded → ``warning`` (high connections pct, when surfaced). | +| fly | status=unhealthy → ``critical`` (no apps reachable / no machines started). status=degraded → ``warning`` (some machines unhealthy / staging app failing). | +| vercel | status=unhealthy → ``critical`` (API error). status=degraded → ``warning`` (error_rate_24h > 10%). | +| sentry | status=unhealthy → ``warning`` (API unreachable — itself a low-priority issue). status=degraded → ``warning`` (>10 unresolved issues). | +| doppler | status=unhealthy → ``critical`` (credential surface unreachable). status=degraded → ``warning`` (oldest_secret_age_days > 180). | + +``unknown`` status (cache warming / auth env unset) → no Issue. +That's an operator-config state, not a probe-detected issue. + +``healthy`` → no Issue. + +# Consecutive-failure debouncing + +Per spec example ("supabase: connection failure ≥3 consecutive +probes"), the issue-detector's caller may pass an observation +history. This module's ``detect_issues`` accepts the LATEST +observation only (single-cycle decision); the caller (the +periodic post-hook task) is responsible for maintaining a debounce +buffer. v1 ships the single-cycle classifier; debounce buffering +is a follow-on for tuning false-positive rates. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Iterable, List, Literal, Optional + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Issue value type +# --------------------------------------------------------------------------- + + +IssueSeverity = Literal["critical", "warning", "info"] + + +@dataclass(frozen=True, slots=True) +class Issue: + """One probe-derived operator-attention issue. + + Wire-shape mirror of the ``Alert`` model in + ``kora_cli/alerts/aggregator.py`` so a consumer that already + handles Alerts can render Issues with no shape adapter. + + ``probe`` names which probe surfaced it; ``category`` aligns + to the alerts vocabulary (``service_unhealthy`` from the + existing alerts taxonomy). + """ + + id: str + probe: str + severity: IssueSeverity + category: str + title: str + detail: str + details: dict # snapshot.details verbatim, for the consumer to triage + + +# --------------------------------------------------------------------------- +# Per-probe classifier table +# --------------------------------------------------------------------------- + + +def detect_issue_for_snapshot(snapshot: object) -> Optional[Issue]: + """Classify ONE :class:`ServiceHealthSnapshot` into an Issue (or + ``None`` when no criterion fires). + + Snapshot is duck-typed via ``getattr`` so callers don't have to + import the Pydantic class — this module stays decoupled from + the heartbeat_probes package's import graph. + """ + name = getattr(snapshot, "name", "") or "" + status = getattr(snapshot, "status", "") or "" + error = getattr(snapshot, "error", None) + details = getattr(snapshot, "details", {}) or {} + + if status in ("healthy", "unknown"): + return None + if status not in ("degraded", "unhealthy"): + return None + if name not in _PER_PROBE_RULES: + return None + + rule = _PER_PROBE_RULES[name] + severity = rule["unhealthy_severity" if status == "unhealthy" else "degraded_severity"] + title_fn = rule["unhealthy_title" if status == "unhealthy" else "degraded_title"] + title = title_fn(error=error, details=details) + detail = rule["detail_template"].format( + status=status, + error=error or "(no error)", + ) + + return Issue( + id=f"probe_issue:{name}:{status}", + probe=name, + severity=severity, + category="service_unhealthy", + title=title, + detail=detail, + details=dict(details), + ) + + +def detect_issues(snapshots: Iterable[object]) -> List[Issue]: + """Run the classifier across every snapshot; return non-``None`` + Issues. Order matches the input snapshots iteration order. + """ + out: List[Issue] = [] + for snap in snapshots: + try: + issue = detect_issue_for_snapshot(snap) + except Exception as exc: + logger.warning( + "[kora.probes] detect_issue_for_snapshot raised %r — " + "skipping that snapshot", + exc, + ) + continue + if issue is not None: + out.append(issue) + return out + + +# Per-probe rule literal. Each entry maps status → severity + title. +# Keeping the criteria in code (not external config) so an audit can +# diff them across versions. +_PER_PROBE_RULES = { + "supabase": { + "unhealthy_severity": "critical", + "degraded_severity": "warning", + "unhealthy_title": lambda *, error, details: ( + "Supabase unreachable" + + (f": {error}" if error else "") + ), + "degraded_title": lambda *, error, details: ( + "Supabase degraded " + f"({details.get('connections_pct', 'unknown')}% connections)" + ), + "detail_template": ( + "Supabase probe reported status={status}. error={error!r}. " + "Substrate-write impact: any write to Sea_Tickets / event_log " + "/ snapshots may fail until Supabase recovers." + ), + }, + "fly": { + "unhealthy_severity": "critical", + "degraded_severity": "warning", + "unhealthy_title": lambda *, error, details: ( + "Fly app(s) unreachable" + + (f": {error}" if error else "") + ), + "degraded_title": lambda *, error, details: ( + f"Fly degraded " + f"({details.get('apps_running', 'unknown')} app(s) running)" + ), + "detail_template": ( + "Fly probe reported status={status}. error={error!r}. " + "Deploy-control impact: app machines may be unreachable; " + "scale-down / restart actions may not complete." + ), + }, + "vercel": { + "unhealthy_severity": "critical", + "degraded_severity": "warning", + "unhealthy_title": lambda *, error, details: ( + "Vercel API unreachable" + + (f": {error}" if error else "") + ), + "degraded_title": lambda *, error, details: ( + f"Vercel error-rate elevated " + f"({float(details.get('error_rate_24h', 0)) * 100:.1f}%)" + ), + "detail_template": ( + "Vercel probe reported status={status}. error={error!r}. " + "Recent-deploy impact: if a production deploy is failing, " + "the website may be serving stale state." + ), + }, + "sentry": { + # Sentry being unreachable is operator-attention but NOT + # critical for the runtime — the Kora daemon still works, + # operator just loses error-aggregation visibility for the + # duration. Keeping unhealthy as warning (NOT critical). + "unhealthy_severity": "warning", + "degraded_severity": "warning", + "unhealthy_title": lambda *, error, details: ( + "Sentry API unreachable" + + (f": {error}" if error else "") + ), + "degraded_title": lambda *, error, details: ( + f"Sentry: {details.get('unresolved_issues', 'unknown')} " + "unresolved issue(s)" + ), + "detail_template": ( + "Sentry probe reported status={status}. error={error!r}. " + "Observability impact: error-aggregation visibility is " + "degraded; runtime itself unaffected." + ), + }, + "doppler": { + "unhealthy_severity": "critical", + "degraded_severity": "warning", + "unhealthy_title": lambda *, error, details: ( + "Doppler API unreachable" + + (f": {error}" if error else "") + ), + "degraded_title": lambda *, error, details: ( + f"Doppler secret-age elevated " + f"({details.get('oldest_secret_age_days', 'unknown')} days)" + ), + "detail_template": ( + "Doppler probe reported status={status}. error={error!r}. " + "Credential-surface impact: secret reads + rotations may " + "fail; redeploys may surface stale envs." + ), + }, +} diff --git a/kora_cli/probes/wake_emitter.py b/kora_cli/probes/wake_emitter.py new file mode 100644 index 000000000000..6ca9aa8dd681 --- /dev/null +++ b/kora_cli/probes/wake_emitter.py @@ -0,0 +1,122 @@ +"""Probe wake-event emitter — KR-PROBE-AUDIT-AND-CONVERT. + +Writes one ``probe.wake_requested`` audit row when a probe's +issue criterion fires (per :mod:`kora_cli.probes.issue_detector`). + +# Architecture decision (proposed via STOP-ASK) + +Spec §4 STOP-ASK #1 noted no existing wake-Kora-on-event mechanism +exists for non-alert events. The PR body's audit table proposes +this seam: an audit JSONL row is the conduit. A follow-on bucket +(KR-PROBE-WAKE-CONSUMER) will register a watcher on the audit +log that, when fresh ``probe.wake_requested`` rows appear AND the +relevant envelope is enabled, invokes the reasoning engine with +``route="probe_investigation"`` (the telemetry literal already +accepted by PR #161). + +v1 ships JUST the emission. Benefits: + + * Operator-visible: the audit panel already renders this seam + (rendering is shape-driven; ``probe.wake_requested`` rows + appear alongside the other 5 seams). + * Alerts integration: the snapshot's ``alerts.by_category`` + bucket will reflect probe-derived issues via the existing + ``service_unhealthy`` aggregator rule (also unchanged). + * No risk of cost surprises: zero LLM cost on the emission + path; the consumer side is gated separately. + +# Wake-event payload + +Audit row ``details`` shape (consumed by the future wake-listener ++ surfaced in the audit panel): + + - ``probe``: which probe surfaced the issue + - ``severity``: critical | warning | info + - ``category``: matches alerts vocabulary (``service_unhealthy``) + - ``title``: short title + - ``detail``: longer detail (operator-readable) + - ``snapshot_details``: the probe's own ``details`` dict + - ``envelope_enabled``: bool — is the per-probe auto-fix + envelope opted in? Lets the consumer branch on "investigate + only" vs "investigate + attempt fix" + - ``envelope_fix_name``: short id of the envelope (or "(none)") + +# Telemetry route literal + +When the future consumer side invokes reasoning in response to a +wake event, it bills via ``record_inference(route="probe_investigation")`` +(already accepted by PR #161's taxonomy). This module does NOT +itself invoke reasoning — emission is $0 LLM cost. +""" + +from __future__ import annotations + +import logging +from typing import Optional + +from kora_cli.probes.fix_envelopes import ENVELOPES, is_envelope_enabled +from kora_cli.probes.issue_detector import Issue + +logger = logging.getLogger(__name__) + + +def emit_wake_event(issue: Issue) -> None: + """Write one ``probe.wake_requested`` audit row for ``issue``. + + Fail-soft: an audit-write error logs + returns; never raises + (the probe runner's post-hook can't crash the heartbeat + scheduler). + + The audit row is the wake-event conduit. The consumer side + (reasoning-engine wake listener) is a follow-on bucket; v1 + surfaces the event to the audit panel so operator visibility + is the immediate observable. + """ + try: + from kora_cli.audit import emit_audit + except Exception as exc: + logger.warning( + "[kora.probes.wake] audit import failed: %r — wake event " + "not recorded for issue=%s", + exc, + issue.id, + ) + return + + envelope = ENVELOPES.get(issue.probe) + envelope_fix_name = envelope.fix_name if envelope is not None else "(none)" + envelope_enabled = is_envelope_enabled(issue.probe) + + details = { + "probe": issue.probe, + "severity": issue.severity, + "category": issue.category, + "title": issue.title, + "detail": issue.detail, + "snapshot_details": dict(issue.details), + "envelope_enabled": envelope_enabled, + "envelope_fix_name": envelope_fix_name, + } + try: + emit_audit( + seam="probe.wake_requested", + details=details, + source=None, + ) + except Exception as exc: + logger.warning( + "[kora.probes.wake] emit_audit raised %r — wake event " + "not recorded for issue=%s", + exc, + issue.id, + ) + return + + logger.info( + "[kora.probes.wake] probe=%s severity=%s envelope_enabled=%s " + "envelope=%s", + issue.probe, + issue.severity, + envelope_enabled, + envelope_fix_name, + ) diff --git a/tests/kora_cli/probes/__init__.py b/tests/kora_cli/probes/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/kora_cli/probes/test_fix_envelopes.py b/tests/kora_cli/probes/test_fix_envelopes.py new file mode 100644 index 000000000000..72790dcff7ed --- /dev/null +++ b/tests/kora_cli/probes/test_fix_envelopes.py @@ -0,0 +1,129 @@ +"""Tests for KR-PROBE-AUDIT-AND-CONVERT — fix-envelope declarations. + +Bucket §2 Phase 4 scenarios: + 1. Each of the 5 probes has an envelope entry + 2. ``fly`` envelope has a concrete fix_name (the only non-none v1) + 3. supabase / vercel / sentry / doppler envelopes are explicit "(none)" + 4. is_envelope_enabled default False (env unset) + 5. is_envelope_enabled True only with truthy env AND non-none envelope + 6. Truthy values: true / 1 / yes / on (case-insensitive) + 7. is_envelope_enabled is False for "(none)" envelopes even if env truthy + 8. Unknown probe → is_envelope_enabled False + 9. requires_capability field on fly envelope reserves the cap literal +""" + +from __future__ import annotations + +import pytest + +from kora_cli.probes.fix_envelopes import ( + ENABLE_ENV_DOPPLER, + ENABLE_ENV_FLY, + ENABLE_ENV_SENTRY, + ENABLE_ENV_SUPABASE, + ENABLE_ENV_VERCEL, + ENVELOPES, + is_envelope_enabled, +) + + +@pytest.fixture(autouse=True) +def _isolate_env(monkeypatch): + """All envelope envs unset by default — fail-CLOSED.""" + for env in ( + ENABLE_ENV_SUPABASE, + ENABLE_ENV_FLY, + ENABLE_ENV_VERCEL, + ENABLE_ENV_SENTRY, + ENABLE_ENV_DOPPLER, + ): + monkeypatch.delenv(env, raising=False) + + +# =========================================================================== +# Declaration shape +# =========================================================================== + + +def test_every_probe_has_envelope_entry(): + assert set(ENVELOPES.keys()) == { + "supabase", + "fly", + "vercel", + "sentry", + "doppler", + } + + +def test_fly_envelope_is_only_non_none_v1(): + """v1 ships only the fly envelope. Spec §2 Phase 4 makes this + explicit; the others are documented as operator-required.""" + fly = ENVELOPES["fly"] + assert fly.fix_name == "restart_unhealthy_machine" + assert "single_machine_not_started" in fly.in_envelope + # OUT-of-envelope reflects the spec's "anything multi-machine" + # exclusion — operator decides. + assert "deploy_rollback" in fly.out_of_envelope + assert "scale_up" in fly.out_of_envelope + assert "multi_machine_restart" in fly.out_of_envelope + + +def test_other_probes_envelope_explicit_none(): + for probe in ("supabase", "vercel", "sentry", "doppler"): + assert ENVELOPES[probe].fix_name == "(none)" + # in_envelope set is empty for "(none)" envelopes. + assert ENVELOPES[probe].in_envelope == frozenset() + + +def test_fly_envelope_reserves_capability_literal(): + """Capability-matrix gate literal is RESERVED in the envelope + declaration so when the executor follow-on bucket lands, the + cap-matrix audit can include this expected surface.""" + assert ENVELOPES["fly"].requires_capability == "probe_autofix_fly_restart" + + +# =========================================================================== +# is_envelope_enabled — fail-CLOSED defaults +# =========================================================================== + + +def test_default_all_disabled(): + """No envs set → every probe's envelope is disabled.""" + for probe in ENVELOPES: + assert is_envelope_enabled(probe) is False + + +def test_fly_envelope_enables_with_true(monkeypatch): + monkeypatch.setenv(ENABLE_ENV_FLY, "true") + assert is_envelope_enabled("fly") is True + + +def test_fly_envelope_enables_with_alternate_truthy(monkeypatch): + for val in ("1", "yes", "on", "TRUE", "Yes", "ON"): + monkeypatch.setenv(ENABLE_ENV_FLY, val) + assert is_envelope_enabled("fly") is True, ( + f"value {val!r} should enable but didn't" + ) + + +def test_fly_envelope_stays_off_for_falsy(monkeypatch): + for val in ("false", "0", "no", "off", "", " ", "False", "garbage"): + monkeypatch.setenv(ENABLE_ENV_FLY, val) + assert is_envelope_enabled("fly") is False, ( + f"value {val!r} should NOT enable but did" + ) + + +def test_none_envelopes_stay_off_even_with_truthy_env(monkeypatch): + """Setting KORA_PROBE_AUTOFIX_SUPABASE_ENABLED=true MUST NOT + enable a non-existent envelope — fail-CLOSED.""" + monkeypatch.setenv(ENABLE_ENV_SUPABASE, "true") + monkeypatch.setenv(ENABLE_ENV_VERCEL, "1") + monkeypatch.setenv(ENABLE_ENV_SENTRY, "yes") + monkeypatch.setenv(ENABLE_ENV_DOPPLER, "on") + for probe in ("supabase", "vercel", "sentry", "doppler"): + assert is_envelope_enabled(probe) is False + + +def test_unknown_probe_disabled(): + assert is_envelope_enabled("not_a_real_probe") is False diff --git a/tests/kora_cli/probes/test_issue_detector.py b/tests/kora_cli/probes/test_issue_detector.py new file mode 100644 index 000000000000..e4db0f8c7053 --- /dev/null +++ b/tests/kora_cli/probes/test_issue_detector.py @@ -0,0 +1,235 @@ +"""Tests for KR-PROBE-AUDIT-AND-CONVERT — issue detector. + +Bucket §2 Phase 3 scenarios: + 1. healthy snapshot → no Issue + 2. unknown snapshot → no Issue (cache-warming / auth-unset is + operator-config state, not probe-detected issue) + 3. Each probe × {unhealthy, degraded} → Issue with right severity + 4. supabase unhealthy → critical + 5. fly unhealthy → critical; degraded → warning + 6. vercel unhealthy → critical; degraded → warning + 7. sentry — both unhealthy + degraded → warning + (Sentry-unreachable isn't critical for runtime; observability-only) + 8. doppler unhealthy → critical; degraded → warning + 9. Unknown probe name → no Issue (defensive) + 10. detect_issues across multiple snapshots returns list in order + 11. detect_issue handles bad snapshot (missing attrs) gracefully +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import pytest + +from kora_cli.probes.issue_detector import ( + Issue, + detect_issue_for_snapshot, + detect_issues, +) + + +@dataclass(frozen=True) +class _FakeSnap: + name: str + status: str + error: str | None = None + details: dict | None = None + + +def _snap(**kw) -> _FakeSnap: + return _FakeSnap(details=kw.pop("details", {}) or {}, **kw) + + +# =========================================================================== +# healthy / unknown — no Issue +# =========================================================================== + + +def test_healthy_supabase_no_issue(): + assert detect_issue_for_snapshot(_snap(name="supabase", status="healthy")) is None + + +def test_healthy_fly_no_issue(): + assert detect_issue_for_snapshot(_snap(name="fly", status="healthy")) is None + + +def test_unknown_status_no_issue(): + """Auth env unset / cache warming surfaces as 'unknown' status — + that's operator-config state, not a probe-detected issue.""" + assert detect_issue_for_snapshot(_snap(name="supabase", status="unknown")) is None + assert detect_issue_for_snapshot(_snap(name="fly", status="unknown")) is None + + +def test_garbage_status_no_issue(): + assert detect_issue_for_snapshot(_snap(name="supabase", status="weird")) is None + + +# =========================================================================== +# supabase +# =========================================================================== + + +def test_supabase_unhealthy_critical(): + issue = detect_issue_for_snapshot( + _snap(name="supabase", status="unhealthy", error="HTTP 502") + ) + assert issue is not None + assert issue.severity == "critical" + assert issue.probe == "supabase" + assert issue.category == "service_unhealthy" + assert "Supabase" in issue.title + assert "Substrate" in issue.detail + + +def test_supabase_degraded_warning(): + issue = detect_issue_for_snapshot( + _snap( + name="supabase", + status="degraded", + details={"connections_pct": 85}, + ) + ) + assert issue is not None + assert issue.severity == "warning" + + +# =========================================================================== +# fly +# =========================================================================== + + +def test_fly_unhealthy_critical(): + issue = detect_issue_for_snapshot( + _snap(name="fly", status="unhealthy", error="HTTP 401") + ) + assert issue is not None + assert issue.severity == "critical" + assert "Fly" in issue.title + + +def test_fly_degraded_warning(): + issue = detect_issue_for_snapshot( + _snap( + name="fly", + status="degraded", + details={"apps_running": 1, "deploys_last_24h": "unknown"}, + ) + ) + assert issue is not None + assert issue.severity == "warning" + + +# =========================================================================== +# vercel +# =========================================================================== + + +def test_vercel_unhealthy_critical(): + issue = detect_issue_for_snapshot( + _snap(name="vercel", status="unhealthy", error="HTTP 500") + ) + assert issue is not None + assert issue.severity == "critical" + assert "Vercel" in issue.title + + +def test_vercel_degraded_warning_with_error_rate(): + issue = detect_issue_for_snapshot( + _snap( + name="vercel", + status="degraded", + details={"deployments_last_24h": 20, "error_rate_24h": 0.15}, + ) + ) + assert issue is not None + assert issue.severity == "warning" + assert "15.0%" in issue.title + + +# =========================================================================== +# sentry (warning, not critical, for unreachable — observability-only) +# =========================================================================== + + +def test_sentry_unhealthy_is_warning_not_critical(): + """Sentry-unreachable shouldn't wake the operator at critical + severity — the runtime works fine without Sentry. Spec §2 Phase 3 + + module docstring documents this exception.""" + issue = detect_issue_for_snapshot( + _snap(name="sentry", status="unhealthy", error="HTTP 503") + ) + assert issue is not None + assert issue.severity == "warning" + + +def test_sentry_degraded_warning_with_unresolved_count(): + issue = detect_issue_for_snapshot( + _snap( + name="sentry", + status="degraded", + details={"unresolved_issues": 42}, + ) + ) + assert issue is not None + assert issue.severity == "warning" + assert "42" in issue.title + + +# =========================================================================== +# doppler +# =========================================================================== + + +def test_doppler_unhealthy_critical(): + issue = detect_issue_for_snapshot( + _snap(name="doppler", status="unhealthy", error="HTTP 401") + ) + assert issue is not None + assert issue.severity == "critical" + + +def test_doppler_degraded_warning_with_secret_age(): + issue = detect_issue_for_snapshot( + _snap( + name="doppler", + status="degraded", + details={"oldest_secret_age_days": 200, "projects_total": 5}, + ) + ) + assert issue is not None + assert issue.severity == "warning" + assert "200" in issue.title + + +# =========================================================================== +# Unknown probe + defensive +# =========================================================================== + + +def test_unknown_probe_name_no_issue(): + assert ( + detect_issue_for_snapshot(_snap(name="not_a_real_probe", status="unhealthy")) + is None + ) + + +def test_detect_issues_multi(): + snaps = [ + _snap(name="supabase", status="healthy"), + _snap(name="fly", status="unhealthy", error="HTTP 500"), + _snap(name="vercel", status="degraded", details={"error_rate_24h": 0.12}), + ] + issues = detect_issues(snaps) + assert len(issues) == 2 + assert {i.probe for i in issues} == {"fly", "vercel"} + + +def test_detect_issue_missing_attrs_no_raise(): + """A duck-typed object missing some expected attrs returns None + safely (no AttributeError).""" + + class Empty: + pass + + assert detect_issue_for_snapshot(Empty()) is None diff --git a/tests/kora_cli/probes/test_runner_post_hook.py b/tests/kora_cli/probes/test_runner_post_hook.py new file mode 100644 index 000000000000..4a9b36af1b73 --- /dev/null +++ b/tests/kora_cli/probes/test_runner_post_hook.py @@ -0,0 +1,148 @@ +"""Integration tests for the post-cycle hook wired into +``kora_cli/heartbeat_probes/runner.py`` — KR-PROBE-AUDIT-AND-CONVERT. + +Scenarios: + 1. Routine cycle with all-healthy snapshots → NO wake events emitted + 2. Cycle with one unhealthy snapshot → ONE wake event for that probe + 3. Cycle with multiple issues → one wake event per affected probe + 4. Hook failure path: detect_issues raises → runner still completes + cycle + populates cache; cycle does NOT crash + 5. Routine cycle does NOT invoke LLM (no reasoning import triggered) +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from unittest.mock import AsyncMock, patch + +import pytest + +from kora_cli.heartbeat_probes.runner import run_all_probes +from kora_cli.heartbeat_probes.types import ServiceHealthSnapshot + + +def _mock_probe(name: str, status: str = "healthy", error=None, details=None): + """Build a fake probe object whose ``check()`` returns a + pre-baked ServiceHealthSnapshot.""" + + class MockProbe: + pass + + probe = MockProbe() + probe.name = name + snap = ServiceHealthSnapshot( + name=name, + status=status, + latency_ms=10, + last_check_at=datetime.now(timezone.utc), + details=details or {}, + error=error, + ) + probe.check = AsyncMock(return_value=snap) + return probe + + +@pytest.mark.asyncio +async def test_all_healthy_no_wake_events(): + probes = [ + _mock_probe("supabase", "healthy"), + _mock_probe("fly", "healthy"), + _mock_probe("vercel", "healthy"), + _mock_probe("sentry", "healthy"), + _mock_probe("doppler", "healthy"), + ] + with patch("kora_cli.audit.emit_audit") as mock_emit: + await run_all_probes(probes) + mock_emit.assert_not_called() + + +@pytest.mark.asyncio +async def test_one_unhealthy_one_wake_event(): + probes = [ + _mock_probe("supabase", "healthy"), + _mock_probe("fly", "unhealthy", error="HTTP 500"), + _mock_probe("vercel", "healthy"), + ] + with patch("kora_cli.audit.emit_audit") as mock_emit: + await run_all_probes(probes) + # Only the fly issue should have emitted. + assert mock_emit.call_count == 1 + call = mock_emit.call_args + assert call.kwargs["seam"] == "probe.wake_requested" + assert call.kwargs["details"]["probe"] == "fly" + + +@pytest.mark.asyncio +async def test_multiple_issues_emit_per_probe(): + probes = [ + _mock_probe("supabase", "unhealthy", error="HTTP 502"), + _mock_probe("fly", "degraded", details={"apps_running": 1}), + _mock_probe("vercel", "healthy"), + _mock_probe("sentry", "degraded", details={"unresolved_issues": 42}), + _mock_probe("doppler", "healthy"), + ] + with patch("kora_cli.audit.emit_audit") as mock_emit: + await run_all_probes(probes) + # 3 issues fired: supabase critical + fly warning + sentry warning. + assert mock_emit.call_count == 3 + emitted_probes = { + c.kwargs["details"]["probe"] for c in mock_emit.call_args_list + } + assert emitted_probes == {"supabase", "fly", "sentry"} + + +@pytest.mark.asyncio +async def test_hook_failure_does_not_crash_cycle(caplog): + """If detect_issues raises, the runner's hook logs + continues — + the cache is still populated, the cycle completes successfully.""" + probes = [ + _mock_probe("supabase", "healthy"), + _mock_probe("fly", "healthy"), + ] + with patch( + "kora_cli.probes.detect_issues", + side_effect=RuntimeError("detector exploded"), + ): + with caplog.at_level("WARNING"): + results = await run_all_probes(probes) + # Cycle still completed; cache was populated. + assert "supabase" in results + assert "fly" in results + # Warning logged. + assert any( + "post-cycle issue detection raised" in r.message + for r in caplog.records + ) + + +@pytest.mark.asyncio +async def test_routine_cycle_does_not_invoke_llm(): + """Spec §2 Phase 2 invariant — the routine probing path is $0 + LLM cost. We assert no reasoning-engine respond() call happens + during run_all_probes even with all 5 probes failing. + + The runner hook ONLY emits audit events; it does NOT invoke + reasoning. The consumer side (reasoning wake) is a follow-on. + """ + probes = [ + _mock_probe("supabase", "unhealthy", error="HTTP 500"), + _mock_probe("fly", "unhealthy", error="HTTP 500"), + _mock_probe("vercel", "unhealthy", error="HTTP 500"), + _mock_probe("sentry", "unhealthy", error="HTTP 500"), + _mock_probe("doppler", "unhealthy", error="HTTP 500"), + ] + # Patch potential reasoning entry points so any accidental call + # would surface as a test failure. + with patch("kora_cli.audit.emit_audit"): + try: + from kora_cli.reasoning.anthropic_engine import ( + AnthropicReasoningEngine, + ) + with patch.object( + AnthropicReasoningEngine, "respond", new=AsyncMock(side_effect=AssertionError("reasoning invoked from routine probe path")) + ): + await run_all_probes(probes) + except ImportError: + # Reasoning module not importable in test env — that's + # itself proof the routine probe path doesn't import it. + await run_all_probes(probes) diff --git a/tests/kora_cli/probes/test_wake_emitter.py b/tests/kora_cli/probes/test_wake_emitter.py new file mode 100644 index 000000000000..c945b0bdecb8 --- /dev/null +++ b/tests/kora_cli/probes/test_wake_emitter.py @@ -0,0 +1,178 @@ +"""Tests for KR-PROBE-AUDIT-AND-CONVERT — wake-event emitter. + +Scenarios: + 1. emit_wake_event writes a probe.wake_requested audit row + 2. Audit row details include probe + severity + category + title + 3. envelope_enabled reflects the env state (default False; True + when fly env truthy) + 4. envelope_fix_name reflects the per-probe envelope (none for + non-fly probes; "restart_unhealthy_machine" for fly) + 5. Audit-write failure logs + doesn't raise + 6. emit_audit raises → wake_emitter logs + continues (fail-soft) + 7. cheap-cron contract: routine probing path doesn't invoke LLM +""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from kora_cli.probes import ( + Issue, + emit_wake_event, +) +from kora_cli.probes.fix_envelopes import ENABLE_ENV_FLY + + +@pytest.fixture(autouse=True) +def _isolate_env(monkeypatch): + monkeypatch.delenv(ENABLE_ENV_FLY, raising=False) + + +def _make_fly_issue(severity="critical") -> Issue: + return Issue( + id="probe_issue:fly:unhealthy", + probe="fly", + severity=severity, + category="service_unhealthy", + title="Fly app(s) unreachable: HTTP 401", + detail="Fly probe reported status=unhealthy.", + details={"apps_running": 0, "deploys_last_24h": "unknown"}, + ) + + +def _make_supabase_issue(severity="critical") -> Issue: + return Issue( + id="probe_issue:supabase:unhealthy", + probe="supabase", + severity=severity, + category="service_unhealthy", + title="Supabase unreachable", + detail="Supabase probe reported status=unhealthy.", + details={"connections_pct": "unknown"}, + ) + + +# =========================================================================== +# Audit emission +# =========================================================================== + + +def test_emit_wake_writes_audit_row(): + issue = _make_fly_issue() + with patch("kora_cli.audit.emit_audit") as mock_emit: + emit_wake_event(issue) + mock_emit.assert_called_once() + call_kwargs = mock_emit.call_args.kwargs + assert call_kwargs["seam"] == "probe.wake_requested" + details = call_kwargs["details"] + assert details["probe"] == "fly" + assert details["severity"] == "critical" + assert details["category"] == "service_unhealthy" + assert "Fly" in details["title"] + assert details["snapshot_details"] == { + "apps_running": 0, + "deploys_last_24h": "unknown", + } + + +def test_envelope_enabled_default_false_in_audit(): + """When env unset → envelope_enabled False even for fly (the only + probe with a non-none envelope).""" + issue = _make_fly_issue() + with patch("kora_cli.audit.emit_audit") as mock_emit: + emit_wake_event(issue) + details = mock_emit.call_args.kwargs["details"] + assert details["envelope_enabled"] is False + assert details["envelope_fix_name"] == "restart_unhealthy_machine" + + +def test_envelope_enabled_true_when_env_set(monkeypatch): + monkeypatch.setenv(ENABLE_ENV_FLY, "true") + issue = _make_fly_issue() + with patch("kora_cli.audit.emit_audit") as mock_emit: + emit_wake_event(issue) + details = mock_emit.call_args.kwargs["details"] + assert details["envelope_enabled"] is True + assert details["envelope_fix_name"] == "restart_unhealthy_machine" + + +def test_supabase_envelope_stays_disabled_even_with_env_set(monkeypatch): + """Setting an env for a probe with a "(none)" envelope must not + enable a non-existent fix — fail-CLOSED.""" + monkeypatch.setenv("KORA_PROBE_AUTOFIX_SUPABASE_ENABLED", "true") + issue = _make_supabase_issue() + with patch("kora_cli.audit.emit_audit") as mock_emit: + emit_wake_event(issue) + details = mock_emit.call_args.kwargs["details"] + assert details["envelope_enabled"] is False + assert details["envelope_fix_name"] == "(none)" + + +# =========================================================================== +# Fail-soft +# =========================================================================== + + +def test_emit_audit_raises_logs_no_raise(caplog): + def boom(**kw): + raise RuntimeError("audit write failed") + + issue = _make_fly_issue() + with patch("kora_cli.audit.emit_audit", side_effect=boom): + with caplog.at_level("WARNING"): + emit_wake_event(issue) # MUST not raise + assert any("emit_audit raised" in r.message for r in caplog.records) + + +def test_audit_import_failure_no_raise(monkeypatch, caplog): + """Defense in depth — if the audit module itself can't be + imported (unusual but possible in partial-install paths), the + emitter logs + returns.""" + import builtins + + original_import = builtins.__import__ + + def importer(name, *args, **kwargs): + if name == "kora_cli.audit": + raise ImportError("simulated audit import fail") + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", importer) + issue = _make_fly_issue() + with caplog.at_level("WARNING"): + emit_wake_event(issue) + assert any("audit import failed" in r.message for r in caplog.records) + + +# =========================================================================== +# Cheap-cron contract — no LLM invoked +# =========================================================================== + + +def test_emit_wake_does_not_import_reasoning(): + """The emitter must NEVER invoke (or even import) the reasoning + engine. Spec §2 + the cheap-cron architecture pins this — actual + reasoning invocation belongs in the wake-listener follow-on.""" + import sys + + # Snapshot module set before; emit; verify no reasoning modules + # got newly imported. + before = { + m for m in sys.modules if "reasoning" in m or "anthropic" in m + } + issue = _make_fly_issue() + with patch("kora_cli.audit.emit_audit"): + emit_wake_event(issue) + after = { + m for m in sys.modules if "reasoning" in m or "anthropic" in m + } + # Module set didn't grow during emit. (If the test process imported + # reasoning earlier — e.g., via a fixture loading the engine — the + # set may have entries; we only check delta.) + new_imports = after - before + assert new_imports == set(), ( + f"emit_wake_event triggered unexpected reasoning imports: " + f"{new_imports}" + )