feat(alerts): complete alerting system overhaul — retry limits, HMAC signing, severity, resolution notifications

src/alerts/dispatcher.ts

@@ -1,5 +1,5 @@
 import type Database from "better-sqlite3";
-import { getUndeliveredAlerts, markAlertDelivered } from "../db/repositories.js";
+import { getUndeliveredAlerts, markAlertDelivered, incrementRetryCount, MAX_RETRY_COUNT } from "../db/repositories.js";
 import { buildAlertEvent, type AlertEvent } from "./types.js";
 import { sendWebhookAlert } from "./webhook.js";
 import { sendSlackAlert } from "./slack.js";
@@ -10,12 +10,14 @@ const logger = getLogger().child({ component: "AlertDispatcher" });
 // ─── Public contract ─────────────────────────────────────────────────────────
 
 export interface DeliveryResult {
-    /** Total alerts processed (includes email-skipped and failed). */
+    /** Total alerts processed (includes failed). */
     attempted: number;
     /** Alerts successfully sent and marked delivered = 1. */
     delivered: number;
-    /** Alerts that threw during delivery — left as delivered = 0 for retry. */
+    /** Alerts that threw during delivery — retry count incremented. */
     failed: number;
+    /** Alerts that exceeded max retries and were abandoned. */
+    abandoned: number;
     /** Error messages for each failed delivery. */
     errors: string[];
 }
@@ -27,11 +29,8 @@ export interface DeliveryResult {
  * dispatch them to the appropriate channel handler.
  *
  * Per-alert errors are caught and collected — this function never throws.
- * Failed deliveries are left with `delivered = 0` so the next daemon cycle
- * retries them automatically.
- *
- * Email channel type is not yet implemented and will be counted as attempted
- * but not delivered or failed.
+ * Failed deliveries have their retry_count incremented. Alerts exceeding
+ * MAX_RETRY_COUNT are excluded from future queries automatically.
  */
 export async function deliverPendingAlerts(
     db: Database.Database,
@@ -41,6 +40,7 @@ export async function deliverPendingAlerts(
         attempted: 0,
         delivered: 0,
         failed: 0,
+        abandoned: 0,
         errors: [],
     };
 
@@ -68,7 +68,7 @@ export async function deliverPendingAlerts(
         });
 
         try {
-            await route(alert.channelType, alert.channelTarget, event);
+            await route(alert.channelType, alert.channelTarget, event, alert.webhookSecret);
             markAlertDelivered(db, alert.alertFiredId);
             result.delivered++;
 
@@ -81,40 +81,67 @@ export async function deliverPendingAlerts(
             result.failed++;
             result.errors.push(message);
 
-            logger.warn(
-                `Alert delivery failed — id: ${alert.alertFiredId}, ` +
-                `channel: ${alert.channelType}, error: ${message}. Will retry next cycle.`,
-            );
+            incrementRetryCount(db, alert.alertFiredId);
+            const nextRetry = alert.retryCount + 1;
+
+            if (nextRetry >= MAX_RETRY_COUNT) {
+                result.abandoned++;
+                logger.error(
+                    `Alert abandoned after ${MAX_RETRY_COUNT} retries — id: ${alert.alertFiredId}, ` +
+                    `channel: ${alert.channelType}, error: ${message}`,
+                );
+            } else {
+                logger.warn(
+                    `Alert delivery failed (attempt ${nextRetry}/${MAX_RETRY_COUNT}) — ` +
+                    `id: ${alert.alertFiredId}, channel: ${alert.channelType}, error: ${message}`,
+                );
+            }
         }
     }
 
     logger.debug(
         `Dispatcher finished — attempted: ${result.attempted}, ` +
-        `delivered: ${result.delivered}, failed: ${result.failed}`,
+        `delivered: ${result.delivered}, failed: ${result.failed}, abandoned: ${result.abandoned}`,
     );
 
     return result;
 }
 
+/**
+ * Deliver a single AlertEvent directly (used for resolution notifications).
+ * Returns true on success, false on failure.
+ */
+export async function deliverSingleAlert(
+    channelType: string,
+    channelTarget: string,
+    event: AlertEvent,
+    webhookSecret?: string | null,
+): Promise<boolean> {
+    try {
+        await route(channelType, channelTarget, event, webhookSecret ?? null);
+        return true;
+    } catch (err: unknown) {
+        const message = err instanceof Error ? err.message : String(err);
+        logger.warn(`Resolution alert delivery failed — channel: ${channelType}, error: ${message}`);
+        return false;
+    }
+}
+
 // ─── Private ─────────────────────────────────────────────────────────────────
 
 async function route(
     channelType: string,
     channelTarget: string,
     event: AlertEvent,
+    webhookSecret: string | null,
 ): Promise<void> {
     switch (channelType) {
         case "webhook":
-            await sendWebhookAlert(channelTarget, event);
+            await sendWebhookAlert(channelTarget, event, webhookSecret);
             break;
         case "slack":
             await sendSlackAlert(channelTarget, event);
             break;
-        case "email":
-            // Email delivery is not yet implemented.
-            // Log and return without marking as delivered or failed.
-            logger.debug(`Email delivery not yet implemented — skipping alert to ${channelTarget}`);
-            break;
         default:
             throw new Error(`Unknown channel type: ${channelType}`);
     }

src/alerts/slack.ts

@@ -1,21 +1,48 @@
 import type { AlertEvent } from "./types.js";
+import { loadConfig } from "../utils/config.js";
 import { getLogger } from "../logging/index.js";
 
 const logger = getLogger().child({ component: "SlackHandler" });
 const SLACK_API_URL = "https://slack.com/api/chat.postMessage";
 const TIMEOUT_MS = 10_000;
 
+// ─── Token resolution ─────────────────────────────────────────────────────────
+
+/**
+ * Resolve the Slack Bot Token from config or environment.
+ * Priority: SENTINEL_SLACK_TOKEN env var > config.slackToken
+ */
+function resolveSlackToken(): string {
+    const envToken = process.env["SENTINEL_SLACK_TOKEN"];
+    if (envToken) return envToken;
+
+    const config = loadConfig();
+    if (config.slackToken) return config.slackToken;
+
+    throw new Error(
+        "Slack token not configured. Set SENTINEL_SLACK_TOKEN environment variable " +
+        "or add slackToken to ~/.soroban-sentinel/config.yaml.",
+    );
+}
+
 // ─── Block Kit builder ────────────────────────────────────────────────────────
 
 interface SlackBlock {
     type: string;
     [key: string]: unknown;
 }
 
+function severityEmoji(event: AlertEvent): string {
+    if (event.type === "alert_resolved") return "✅";
+    if (event.severity === "critical") return "🔴";
+    return "⚠️";
+}
+
 function buildBlocks(event: AlertEvent): SlackBlock[] {
-    const isAlert = event.type === "threshold_crossed";
-    const icon = isAlert ? "⚠️" : "✅";
-    const status = isAlert ? "TTL Warning" : "Alert Resolved";
+    const icon = severityEmoji(event);
+    const status = event.type === "threshold_crossed"
+        ? `TTL ${event.severity === "critical" ? "CRITICAL" : "Warning"}`
+        : "Alert Resolved";
     const contractDisplay = event.contractName ?? event.contractId;
 
     const header: SlackBlock = {
@@ -54,7 +81,7 @@ function buildBlocks(event: AlertEvent): SlackBlock[] {
         elements: [
             {
                 type: "mrkdwn",
-                text: `Run \`sentinel status ${event.contractId}\` for details.`,
+                text: `Severity: *${event.severity}* | Run \`sentinel status ${event.contractId}\` for details.`,
             },
         ],
     };
@@ -63,9 +90,10 @@ function buildBlocks(event: AlertEvent): SlackBlock[] {
 }
 
 function buildFallbackText(event: AlertEvent): string {
-    const isAlert = event.type === "threshold_crossed";
-    const icon = isAlert ? "⚠️" : "✅";
-    const status = isAlert ? "TTL Warning" : "Alert Resolved";
+    const icon = severityEmoji(event);
+    const status = event.type === "threshold_crossed"
+        ? `TTL ${event.severity === "critical" ? "CRITICAL" : "Warning"}`
+        : "Alert Resolved";
     const contractDisplay = event.contractName ?? event.contractId;
 
     return (
@@ -81,18 +109,12 @@ function buildFallbackText(event: AlertEvent): string {
 /**
  * Send an AlertEvent to a Slack channel via the Slack Web API.
  *
- * Reads the Bot Token from `process.env.SENTINEL_SLACK_TOKEN`.
+ * Resolves the Bot Token from env (SENTINEL_SLACK_TOKEN) or config (slackToken).
  * Throws when the token is absent, the network fails, or Slack returns ok: false.
  * The caller (dispatcher) handles retry via the `delivered` flag.
  */
 export async function sendSlackAlert(channel: string, event: AlertEvent): Promise<void> {
-    const token = process.env["SENTINEL_SLACK_TOKEN"];
-    if (!token) {
-        throw new Error(
-            "SENTINEL_SLACK_TOKEN environment variable is not set. " +
-            "Export your Slack Bot Token before starting the daemon.",
-        );
-    }
+    const token = resolveSlackToken();
 
     logger.debug(`Sending Slack alert to ${channel}`, { type: event.type, contractId: event.contractId });
 

src/alerts/types.ts

@@ -2,9 +2,13 @@ import { formatTimeToCloseLedger } from "../utils/formatting.js";
 
 // ─── Core event type ─────────────────────────────────────────────────────────
 
+export type AlertSeverity = "critical" | "warning" | "info";
+
 export interface AlertEvent {
     /** Whether this is a new threshold crossing or a resolved alert. */
     type: "threshold_crossed" | "alert_resolved";
+    /** Severity based on how close to expiry the entry is. */
+    severity: AlertSeverity;
     contractId: string;
     contractName: string | null;
     network: string;
@@ -29,6 +33,19 @@ export interface AlertEvent {
 
 // ─── Helpers ─────────────────────────────────────────────────────────────────
 
+/**
+ * Compute alert severity from remaining TTL.
+ * - critical: less than 25% of threshold remaining
+ * - warning:  less than threshold (but above 25%)
+ * - info:     used for resolution events
+ */
+export function computeSeverity(remainingTTL: number, thresholdLedgers: number, isResolution: boolean): AlertSeverity {
+    if (isResolution) return "info";
+    if (remainingTTL <= 0) return "critical";
+    if (remainingTTL < thresholdLedgers * 0.25) return "critical";
+    return "warning";
+}
+
 /**
  * Build an AlertEvent from raw data.  Keeps the assembly logic in one place
  * so both the dispatcher and any future test fixtures share it.
@@ -47,6 +64,7 @@ export function buildAlertEvent(opts: {
 }): AlertEvent {
     return {
         type: opts.type,
+        severity: computeSeverity(opts.remainingTTL, opts.configuredLedgers, opts.type === "alert_resolved"),
         contractId: opts.contractId,
         contractName: opts.contractName,
         network: opts.network,

src/alerts/webhook.ts

@@ -1,3 +1,4 @@
+import { createHmac } from "node:crypto";
 import type { AlertEvent } from "./types.js";
 import { getLogger } from "../logging/index.js";
 
@@ -8,21 +9,33 @@ const TIMEOUT_MS = 10_000;
 /**
  * Send an AlertEvent to a webhook URL via HTTP POST.
  *
+ * If a `secret` is provided, the request includes an `X-Sentinel-Signature`
+ * header with an HMAC-SHA256 hex digest of the body, allowing receivers to
+ * verify authenticity.
+ *
  * Throws on any non-2xx response or network error.
  * The caller (dispatcher) is responsible for retry logic via the `delivered` flag.
  */
-export async function sendWebhookAlert(url: string, event: AlertEvent): Promise<void> {
+export async function sendWebhookAlert(url: string, event: AlertEvent, secret?: string | null): Promise<void> {
     logger.debug(`Sending webhook alert to ${url}`, { type: event.type, contractId: event.contractId });
 
+    const body = JSON.stringify(event);
+    const headers: Record<string, string> = { "Content-Type": "application/json" };
+
+    if (secret) {
+        const signature = createHmac("sha256", secret).update(body).digest("hex");
+        headers["X-Sentinel-Signature"] = `sha256=${signature}`;
+    }
+
     const controller = new AbortController();
     const timeout = setTimeout(() => controller.abort(), TIMEOUT_MS);
 
     let response: Response;
     try {
         response = await fetch(url, {
             method: "POST",
-            headers: { "Content-Type": "application/json" },
-            body: JSON.stringify(event),
+            headers,
+            body,
             signal: controller.signal,
         });
     } finally {