diff --git a/src/alerts/dispatcher.ts b/src/alerts/dispatcher.ts index 8606b210..ebbd71e7 100644 --- a/src/alerts/dispatcher.ts +++ b/src/alerts/dispatcher.ts @@ -1,5 +1,5 @@ import type Database from "better-sqlite3"; -import { getUndeliveredAlerts, markAlertDelivered } from "../db/repositories.js"; +import { getUndeliveredAlerts, markAlertDelivered, incrementRetryCount, MAX_RETRY_COUNT } from "../db/repositories.js"; import { buildAlertEvent, type AlertEvent } from "./types.js"; import { sendWebhookAlert } from "./webhook.js"; import { sendSlackAlert } from "./slack.js"; @@ -10,12 +10,14 @@ const logger = getLogger().child({ component: "AlertDispatcher" }); // ─── Public contract ───────────────────────────────────────────────────────── export interface DeliveryResult { - /** Total alerts processed (includes email-skipped and failed). */ + /** Total alerts processed (includes failed). */ attempted: number; /** Alerts successfully sent and marked delivered = 1. */ delivered: number; - /** Alerts that threw during delivery — left as delivered = 0 for retry. */ + /** Alerts that threw during delivery — retry count incremented. */ failed: number; + /** Alerts that exceeded max retries and were abandoned. */ + abandoned: number; /** Error messages for each failed delivery. */ errors: string[]; } @@ -27,11 +29,8 @@ export interface DeliveryResult { * dispatch them to the appropriate channel handler. * * Per-alert errors are caught and collected — this function never throws. - * Failed deliveries are left with `delivered = 0` so the next daemon cycle - * retries them automatically. - * - * Email channel type is not yet implemented and will be counted as attempted - * but not delivered or failed. + * Failed deliveries have their retry_count incremented. Alerts exceeding + * MAX_RETRY_COUNT are excluded from future queries automatically. */ export async function deliverPendingAlerts( db: Database.Database, @@ -41,6 +40,7 @@ export async function deliverPendingAlerts( attempted: 0, delivered: 0, failed: 0, + abandoned: 0, errors: [], }; @@ -68,7 +68,7 @@ export async function deliverPendingAlerts( }); try { - await route(alert.channelType, alert.channelTarget, event); + await route(alert.channelType, alert.channelTarget, event, alert.webhookSecret); markAlertDelivered(db, alert.alertFiredId); result.delivered++; @@ -81,40 +81,67 @@ export async function deliverPendingAlerts( result.failed++; result.errors.push(message); - logger.warn( - `Alert delivery failed — id: ${alert.alertFiredId}, ` + - `channel: ${alert.channelType}, error: ${message}. Will retry next cycle.`, - ); + incrementRetryCount(db, alert.alertFiredId); + const nextRetry = alert.retryCount + 1; + + if (nextRetry >= MAX_RETRY_COUNT) { + result.abandoned++; + logger.error( + `Alert abandoned after ${MAX_RETRY_COUNT} retries — id: ${alert.alertFiredId}, ` + + `channel: ${alert.channelType}, error: ${message}`, + ); + } else { + logger.warn( + `Alert delivery failed (attempt ${nextRetry}/${MAX_RETRY_COUNT}) — ` + + `id: ${alert.alertFiredId}, channel: ${alert.channelType}, error: ${message}`, + ); + } } } logger.debug( `Dispatcher finished — attempted: ${result.attempted}, ` + - `delivered: ${result.delivered}, failed: ${result.failed}`, + `delivered: ${result.delivered}, failed: ${result.failed}, abandoned: ${result.abandoned}`, ); return result; } +/** + * Deliver a single AlertEvent directly (used for resolution notifications). + * Returns true on success, false on failure. + */ +export async function deliverSingleAlert( + channelType: string, + channelTarget: string, + event: AlertEvent, + webhookSecret?: string | null, +): Promise { + try { + await route(channelType, channelTarget, event, webhookSecret ?? null); + return true; + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + logger.warn(`Resolution alert delivery failed — channel: ${channelType}, error: ${message}`); + return false; + } +} + // ─── Private ───────────────────────────────────────────────────────────────── async function route( channelType: string, channelTarget: string, event: AlertEvent, + webhookSecret: string | null, ): Promise { switch (channelType) { case "webhook": - await sendWebhookAlert(channelTarget, event); + await sendWebhookAlert(channelTarget, event, webhookSecret); break; case "slack": await sendSlackAlert(channelTarget, event); break; - case "email": - // Email delivery is not yet implemented. - // Log and return without marking as delivered or failed. - logger.debug(`Email delivery not yet implemented — skipping alert to ${channelTarget}`); - break; default: throw new Error(`Unknown channel type: ${channelType}`); } diff --git a/src/alerts/slack.ts b/src/alerts/slack.ts index 2fe0ec2f..6786c82a 100644 --- a/src/alerts/slack.ts +++ b/src/alerts/slack.ts @@ -1,10 +1,30 @@ import type { AlertEvent } from "./types.js"; +import { loadConfig } from "../utils/config.js"; import { getLogger } from "../logging/index.js"; const logger = getLogger().child({ component: "SlackHandler" }); const SLACK_API_URL = "https://slack.com/api/chat.postMessage"; const TIMEOUT_MS = 10_000; +// ─── Token resolution ───────────────────────────────────────────────────────── + +/** + * Resolve the Slack Bot Token from config or environment. + * Priority: SENTINEL_SLACK_TOKEN env var > config.slackToken + */ +function resolveSlackToken(): string { + const envToken = process.env["SENTINEL_SLACK_TOKEN"]; + if (envToken) return envToken; + + const config = loadConfig(); + if (config.slackToken) return config.slackToken; + + throw new Error( + "Slack token not configured. Set SENTINEL_SLACK_TOKEN environment variable " + + "or add slackToken to ~/.soroban-sentinel/config.yaml.", + ); +} + // ─── Block Kit builder ──────────────────────────────────────────────────────── interface SlackBlock { @@ -12,10 +32,17 @@ interface SlackBlock { [key: string]: unknown; } +function severityEmoji(event: AlertEvent): string { + if (event.type === "alert_resolved") return "✅"; + if (event.severity === "critical") return "🔴"; + return "⚠️"; +} + function buildBlocks(event: AlertEvent): SlackBlock[] { - const isAlert = event.type === "threshold_crossed"; - const icon = isAlert ? "⚠️" : "✅"; - const status = isAlert ? "TTL Warning" : "Alert Resolved"; + const icon = severityEmoji(event); + const status = event.type === "threshold_crossed" + ? `TTL ${event.severity === "critical" ? "CRITICAL" : "Warning"}` + : "Alert Resolved"; const contractDisplay = event.contractName ?? event.contractId; const header: SlackBlock = { @@ -54,7 +81,7 @@ function buildBlocks(event: AlertEvent): SlackBlock[] { elements: [ { type: "mrkdwn", - text: `Run \`sentinel status ${event.contractId}\` for details.`, + text: `Severity: *${event.severity}* | Run \`sentinel status ${event.contractId}\` for details.`, }, ], }; @@ -63,9 +90,10 @@ function buildBlocks(event: AlertEvent): SlackBlock[] { } function buildFallbackText(event: AlertEvent): string { - const isAlert = event.type === "threshold_crossed"; - const icon = isAlert ? "⚠️" : "✅"; - const status = isAlert ? "TTL Warning" : "Alert Resolved"; + const icon = severityEmoji(event); + const status = event.type === "threshold_crossed" + ? `TTL ${event.severity === "critical" ? "CRITICAL" : "Warning"}` + : "Alert Resolved"; const contractDisplay = event.contractName ?? event.contractId; return ( @@ -81,18 +109,12 @@ function buildFallbackText(event: AlertEvent): string { /** * Send an AlertEvent to a Slack channel via the Slack Web API. * - * Reads the Bot Token from `process.env.SENTINEL_SLACK_TOKEN`. + * Resolves the Bot Token from env (SENTINEL_SLACK_TOKEN) or config (slackToken). * Throws when the token is absent, the network fails, or Slack returns ok: false. * The caller (dispatcher) handles retry via the `delivered` flag. */ export async function sendSlackAlert(channel: string, event: AlertEvent): Promise { - const token = process.env["SENTINEL_SLACK_TOKEN"]; - if (!token) { - throw new Error( - "SENTINEL_SLACK_TOKEN environment variable is not set. " + - "Export your Slack Bot Token before starting the daemon.", - ); - } + const token = resolveSlackToken(); logger.debug(`Sending Slack alert to ${channel}`, { type: event.type, contractId: event.contractId }); diff --git a/src/alerts/types.ts b/src/alerts/types.ts index 618bf6dd..8fd0fd02 100644 --- a/src/alerts/types.ts +++ b/src/alerts/types.ts @@ -2,9 +2,13 @@ import { formatTimeToCloseLedger } from "../utils/formatting.js"; // ─── Core event type ───────────────────────────────────────────────────────── +export type AlertSeverity = "critical" | "warning" | "info"; + export interface AlertEvent { /** Whether this is a new threshold crossing or a resolved alert. */ type: "threshold_crossed" | "alert_resolved"; + /** Severity based on how close to expiry the entry is. */ + severity: AlertSeverity; contractId: string; contractName: string | null; network: string; @@ -29,6 +33,19 @@ export interface AlertEvent { // ─── Helpers ───────────────────────────────────────────────────────────────── +/** + * Compute alert severity from remaining TTL. + * - critical: less than 25% of threshold remaining + * - warning: less than threshold (but above 25%) + * - info: used for resolution events + */ +export function computeSeverity(remainingTTL: number, thresholdLedgers: number, isResolution: boolean): AlertSeverity { + if (isResolution) return "info"; + if (remainingTTL <= 0) return "critical"; + if (remainingTTL < thresholdLedgers * 0.25) return "critical"; + return "warning"; +} + /** * Build an AlertEvent from raw data. Keeps the assembly logic in one place * so both the dispatcher and any future test fixtures share it. @@ -47,6 +64,7 @@ export function buildAlertEvent(opts: { }): AlertEvent { return { type: opts.type, + severity: computeSeverity(opts.remainingTTL, opts.configuredLedgers, opts.type === "alert_resolved"), contractId: opts.contractId, contractName: opts.contractName, network: opts.network, diff --git a/src/alerts/webhook.ts b/src/alerts/webhook.ts index 163f1116..d216a909 100644 --- a/src/alerts/webhook.ts +++ b/src/alerts/webhook.ts @@ -1,3 +1,4 @@ +import { createHmac } from "node:crypto"; import type { AlertEvent } from "./types.js"; import { getLogger } from "../logging/index.js"; @@ -8,12 +9,24 @@ const TIMEOUT_MS = 10_000; /** * Send an AlertEvent to a webhook URL via HTTP POST. * + * If a `secret` is provided, the request includes an `X-Sentinel-Signature` + * header with an HMAC-SHA256 hex digest of the body, allowing receivers to + * verify authenticity. + * * Throws on any non-2xx response or network error. * The caller (dispatcher) is responsible for retry logic via the `delivered` flag. */ -export async function sendWebhookAlert(url: string, event: AlertEvent): Promise { +export async function sendWebhookAlert(url: string, event: AlertEvent, secret?: string | null): Promise { logger.debug(`Sending webhook alert to ${url}`, { type: event.type, contractId: event.contractId }); + const body = JSON.stringify(event); + const headers: Record = { "Content-Type": "application/json" }; + + if (secret) { + const signature = createHmac("sha256", secret).update(body).digest("hex"); + headers["X-Sentinel-Signature"] = `sha256=${signature}`; + } + const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), TIMEOUT_MS); @@ -21,8 +34,8 @@ export async function sendWebhookAlert(url: string, event: AlertEvent): Promise< try { response = await fetch(url, { method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(event), + headers, + body, signal: controller.signal, }); } finally { diff --git a/src/commands/alerts.ts b/src/commands/alerts.ts index 1d9d1d0e..b426b197 100644 --- a/src/commands/alerts.ts +++ b/src/commands/alerts.ts @@ -1,27 +1,33 @@ import { Command } from "commander"; import chalk from "chalk"; +import { randomBytes } from "node:crypto"; import { getDatabase } from "../db/database.js"; import { insertAlertConfig, getAlertConfigsForContract, + getAlertConfigById, deleteAlertConfig, getContract, + getAlertHistory, } from "../db/repositories.js"; -import { formatContractID } from "../utils/formatting.js"; +import { formatContractID, formatTimeToCloseLedger } from "../utils/formatting.js"; +import { deliverSingleAlert } from "../alerts/dispatcher.js"; +import { buildAlertEvent } from "../alerts/types.js"; export function registerAlertsCommand(program: Command): void { const alerts = program .command("alerts") .description("Manage alert configurations"); + // ── alerts add ───────────────────────────────────────────────────── alerts .command("add") .description("Add a new alert configuration") .requiredOption("--contract ", "The contract ID to alert on") - .requiredOption("--type ", "The notification channel type ('webhook', 'slack', or 'email')") + .requiredOption("--type ", "The notification channel type ('webhook' or 'slack')") .option("--url ", "Webhook URL (required if --type is webhook)") .option("--channel ", "Slack channel (required if --type is slack)") - .option("--email ", "Email address (required if --type is email)") + .option("--secret ", "HMAC secret for webhook signing (auto-generated if omitted for webhooks)") .requiredOption("--threshold ", "Threshold in number of ledgers", (val) => parseInt(val, 10)) .action((options) => { const contractId = options.contract; @@ -41,12 +47,15 @@ export function registerAlertsCommand(program: Command): void { } let target = ""; + let webhookSecret: string | undefined; + if (options.type === "webhook") { if (!options.url) { console.error(chalk.red("Error: --url is required when --type is webhook.")); process.exit(1); } target = options.url; + webhookSecret = options.secret ?? randomBytes(32).toString("hex"); } else if (options.type === "slack") { if (!options.channel) { console.error(chalk.red("Error: --channel is required when --type is slack.")); @@ -54,13 +63,10 @@ export function registerAlertsCommand(program: Command): void { } target = options.channel; } else if (options.type === "email") { - if (!options.email) { - console.error(chalk.red("Error: --email is required when --type is email.")); - process.exit(1); - } - target = options.email; + console.error(chalk.red("Error: Email alerting is not yet implemented. Use 'webhook' or 'slack'.")); + process.exit(1); } else { - console.error(chalk.red("Error: --type must be 'webhook', 'slack', or 'email'.")); + console.error(chalk.red("Error: --type must be 'webhook' or 'slack'.")); process.exit(1); } @@ -69,6 +75,7 @@ export function registerAlertsCommand(program: Command): void { channel_type: options.type, channel_target: target, threshold_ledgers: threshold, + webhook_secret: webhookSecret, }); console.log( @@ -76,8 +83,14 @@ export function registerAlertsCommand(program: Command): void { `Successfully added alert config: type=${options.type}, target=${target}, threshold=${threshold} ledgers` ) ); + + if (webhookSecret) { + console.log(` ${chalk.bold("Webhook secret:")} ${webhookSecret}`); + console.log(chalk.dim(" Save this secret — it signs payloads via X-Sentinel-Signature header.")); + } }); + // ── alerts list ──────────────────────────────────────────────────── alerts .command("list") .description("List alert configurations for a contract") @@ -102,16 +115,19 @@ export function registerAlertsCommand(program: Command): void { console.log(chalk.bold(` Alert Configurations for ${contract.name ?? formatContractID(contractId)}`)); console.log(); for (const config of configs) { + const signed = config.webhook_secret ? chalk.green(" [signed]") : ""; console.log( ` ID: ${chalk.cyan(config.id.toString().padEnd(4))} | ` + `Type: ${chalk.yellow(config.channel_type.padEnd(8))} | ` + `Target: ${chalk.green(config.channel_target.padEnd(30))} | ` + - `Threshold: ${chalk.magenta(config.threshold_ledgers.toLocaleString())} ledgers` + `Threshold: ${chalk.magenta(config.threshold_ledgers.toLocaleString())} ledgers` + + signed ); } console.log(); }); + // ── alerts remove ────────────────────────────────────────────────── alerts .command("remove") .description("Remove an alert configuration") @@ -127,4 +143,98 @@ export function registerAlertsCommand(program: Command): void { deleteAlertConfig(db, id); console.log(chalk.green(`Successfully removed alert config ID ${id}.`)); }); + + // ── alerts test ──────────────────────────────────────────────────── + alerts + .command("test") + .description("Send a test alert to verify channel connectivity") + .requiredOption("--id ", "The alert configuration ID to test") + .action(async (options) => { + const id = parseInt(options.id, 10); + if (isNaN(id)) { + console.error(chalk.red("Error: --id must be a number.")); + process.exit(1); + } + + const db = getDatabase(); + const config = getAlertConfigById(db, id); + if (!config) { + console.error(chalk.red(`Error: Alert config ID ${id} not found.`)); + process.exit(1); + } + + const testEvent = buildAlertEvent({ + type: "threshold_crossed", + contractId: config.contract_id, + contractName: null, + network: "testnet", + entryKeyXdr: "TEST_ENTRY_KEY", + entryType: "instance", + entryLabel: "test-entry", + configuredLedgers: config.threshold_ledgers, + remainingTTL: Math.floor(config.threshold_ledgers * 0.5), + firedAtLedger: 0, + }); + + console.log(`Sending test alert to ${config.channel_type}:${config.channel_target}...`); + + const success = await deliverSingleAlert( + config.channel_type, + config.channel_target, + testEvent, + config.webhook_secret, + ); + + if (success) { + console.log(chalk.green("Test alert delivered successfully.")); + } else { + console.error(chalk.red("Test alert delivery failed. Check logs for details.")); + process.exit(1); + } + }); + + // ── alerts history ───────────────────────────────────────────────── + alerts + .command("history") + .description("Show alert history for a contract") + .requiredOption("--contract ", "The contract ID to show history for") + .option("--limit ", "Max number of records to show", "20") + .action((options) => { + const contractId = options.contract; + const limit = parseInt(options.limit, 10); + const db = getDatabase(); + + const contract = getContract(db, contractId); + if (!contract) { + console.error(chalk.red(`Error: Contract ${formatContractID(contractId)} is not registered.`)); + process.exit(1); + } + + const history = getAlertHistory(db, contractId, limit > 0 ? limit : undefined); + if (history.length === 0) { + console.log(chalk.yellow("No alert history found.")); + return; + } + + const displayName = contract.name ?? formatContractID(contractId); + console.log(`\n${chalk.bold("Alert History")} — ${chalk.cyan(displayName)}\n`); + + for (const record of history) { + const statusIcon = record.resolved ? chalk.green("✓") : chalk.yellow("●"); + const deliveryIcon = record.delivered ? chalk.green("✓") : chalk.red("✗"); + const label = record.entryLabel ?? record.entryType; + const ttlDisplay = formatTimeToCloseLedger(record.ttlAtFire); + + console.log( + ` ${statusIcon} ${chalk.dim(record.firedAt)} | ` + + `${label} | TTL: ${record.ttlAtFire.toLocaleString()} (${ttlDisplay}) | ` + + `${record.channelType}→${deliveryIcon} | ` + + `retries: ${record.retryCount}` + ); + if (record.resolvedAt) { + console.log(chalk.dim(` Resolved: ${record.resolvedAt}`)); + } + } + console.log(); + }); } diff --git a/src/core/monitor.ts b/src/core/monitor.ts index 55dd6377..648802ff 100644 --- a/src/core/monitor.ts +++ b/src/core/monitor.ts @@ -5,11 +5,14 @@ import { upsertEntry, updateLastCheckedLedger, getAlertConfigsForContract, + getAlertConfigById, hasUnresolvedAlert, recordAlertFired, resolveAlerts, } from "../db/repositories.js"; import { StellarRpcClient } from "../rpc/client.js"; +import { deliverSingleAlert } from "../alerts/dispatcher.js"; +import { buildAlertEvent } from "../alerts/types.js"; import { getLogger } from "../logging/index.js"; const logger = getLogger().child({ component: "MonitorCycle" }); @@ -83,7 +86,7 @@ export async function runMonitorCycle( result.contractsChecked++; try { - await processContract(db, client, contract.id, result); + await processContract(db, client, contract.id, network, result); } catch (error: unknown) { // Fault isolation: record the failure, move to next contract. const message = error instanceof Error ? error.message : String(error); @@ -116,6 +119,7 @@ async function processContract( db: Database.Database, client: StellarRpcClient, contractId: string, + network: string, result: MonitorCycleResult, ): Promise { const entries = getEntriesForContract(db, contractId); @@ -196,7 +200,7 @@ async function processContract( } else { // 5. TTL is at or above threshold — resolve any open alert. if (hasUnresolvedAlert(db, alertConfig.id, entry.id)) { - resolveAlerts(db, entry.id); + const resolvedConfigIds = resolveAlerts(db, entry.id); result.alertsResolved++; logger.info( @@ -205,6 +209,33 @@ async function processContract( `remainingTTL: ${remainingTTL}, ` + `threshold: ${alertConfig.threshold_ledgers}`, ); + + // Send resolution notifications (best-effort, errors don't block). + for (const configId of resolvedConfigIds) { + const config = getAlertConfigById(db, configId); + if (!config) continue; + + const event = buildAlertEvent({ + type: "alert_resolved", + contractId, + contractName: null, + network, + entryKeyXdr: entry.entry_key_xdr, + entryType: entry.entry_type, + entryLabel: entry.label, + configuredLedgers: config.threshold_ledgers, + remainingTTL, + firedAtLedger: rpcResult.latestLedger, + }); + + // Fire and forget — resolution is best-effort + void deliverSingleAlert( + config.channel_type, + config.channel_target, + event, + config.webhook_secret, + ); + } } } } diff --git a/src/db/database.ts b/src/db/database.ts index 91ee9e9f..62578b5a 100644 --- a/src/db/database.ts +++ b/src/db/database.ts @@ -42,6 +42,8 @@ export function getDatabase(customPath?: string): Database.Database { const migrations = [ `ALTER TABLE alerts_fired ADD COLUMN delivered INTEGER NOT NULL DEFAULT 0`, `ALTER TABLE alerts_fired ADD COLUMN delivered_at TEXT`, + `ALTER TABLE alerts_fired ADD COLUMN retry_count INTEGER NOT NULL DEFAULT 0`, + `ALTER TABLE alert_configs ADD COLUMN webhook_secret TEXT`, ]; for (const sql of migrations) { try { db.exec(sql); } catch { /* column already exists — no-op */ } diff --git a/src/db/repositories.ts b/src/db/repositories.ts index b973d517..18bfa19c 100644 --- a/src/db/repositories.ts +++ b/src/db/repositories.ts @@ -37,9 +37,10 @@ export interface ExtensionPolicy { export interface AlertConfig { id: number; contract_id: string; - channel_type: "email" | "slack" | "webhook"; + channel_type: "slack" | "webhook"; channel_target: string; threshold_ledgers: number; + webhook_secret: string | null; created_at: Date; } @@ -171,11 +172,19 @@ export function insertAlertConfig(db: Database.Database, config: { channel_type: string; channel_target: string; threshold_ledgers: number; + webhook_secret?: string; }): void { db.prepare(` - INSERT INTO alert_configs (contract_id, channel_type, channel_target, threshold_ledgers) - VALUES (@contract_id, @channel_type, @channel_target, @threshold_ledgers) - `).run(config); + INSERT INTO alert_configs (contract_id, channel_type, channel_target, threshold_ledgers, webhook_secret) + VALUES (@contract_id, @channel_type, @channel_target, @threshold_ledgers, @webhook_secret) + `).run({ + ...config, + webhook_secret: config.webhook_secret ?? null, + }); +} + +export function getAlertConfigById(db: Database.Database, id: number): AlertConfig | undefined { + return db.prepare("SELECT * FROM alert_configs WHERE id = ?").get(id) as AlertConfig | undefined; } export function getAlertConfigsForContract(db: Database.Database, contractId: string): AlertConfig[] { @@ -208,11 +217,20 @@ export function hasUnresolvedAlert(db: Database.Database, alertConfigId: number, return row !== undefined; } -export function resolveAlerts(db: Database.Database, entryId: number): void { - db.prepare(` - UPDATE alerts_fired SET resolved = 1, resolved_at = datetime('now') +export function resolveAlerts(db: Database.Database, entryId: number): number[] { + const rows = db.prepare(` + SELECT alert_config_id FROM alerts_fired WHERE contract_entry_id = ? AND resolved = 0 - `).run(entryId); + `).all(entryId) as { alert_config_id: number }[]; + + if (rows.length > 0) { + db.prepare(` + UPDATE alerts_fired SET resolved = 1, resolved_at = datetime('now') + WHERE contract_entry_id = ? AND resolved = 0 + `).run(entryId); + } + + return rows.map(r => r.alert_config_id); } // ---------------------------- Database Access Functions For Other Schema: ExtensionRecord---------------------------- @@ -264,18 +282,24 @@ export interface UndeliveredAlert { entryKeyXdr: string; entryType: string; entryLabel: string | null; - channelType: "webhook" | "slack" | "email"; + channelType: "webhook" | "slack"; channelTarget: string; thresholdLedgers: number; + webhookSecret: string | null; /** TTL remaining at the moment the alert fired (ttl_at_fire). */ remainingTTL: number; firedAtLedger: number; firedAt: string; + retryCount: number; } +/** Maximum number of delivery attempts before giving up on an alert. */ +export const MAX_RETRY_COUNT = 5; + /** * Return all undelivered (delivered = 0) alerts for the given network, * joining alerts_fired → alert_configs → contract_entries → contracts. + * Alerts that have exceeded MAX_RETRY_COUNT are excluded. */ export function getUndeliveredAlerts( db: Database.Database, @@ -295,17 +319,20 @@ export function getUndeliveredAlerts( ac.channel_type AS channelType, ac.channel_target AS channelTarget, ac.threshold_ledgers AS thresholdLedgers, + ac.webhook_secret AS webhookSecret, af.ttl_at_fire AS remainingTTL, af.fired_at_ledger AS firedAtLedger, - af.fired_at AS firedAt + af.fired_at AS firedAt, + af.retry_count AS retryCount FROM alerts_fired af JOIN alert_configs ac ON ac.id = af.alert_config_id JOIN contract_entries ce ON ce.id = af.contract_entry_id JOIN contracts c ON c.id = ce.contract_id WHERE af.delivered = 0 + AND af.retry_count < ? AND c.network = ? ORDER BY af.fired_at ASC - `).all(network) as UndeliveredAlert[]; + `).all(MAX_RETRY_COUNT, network) as UndeliveredAlert[]; return rows; } @@ -321,3 +348,66 @@ export function markAlertDelivered(db: Database.Database, alertFiredId: number): WHERE id = ? `).run(alertFiredId); } + +/** + * Increment the retry count for a failed alert delivery. + */ +export function incrementRetryCount(db: Database.Database, alertFiredId: number): void { + db.prepare(` + UPDATE alerts_fired + SET retry_count = retry_count + 1 + WHERE id = ? + `).run(alertFiredId); +} + +/** + * Get alert history for a contract. Returns fired alerts with config and entry info. + */ +export interface AlertHistoryRecord { + alertFiredId: number; + channelType: string; + channelTarget: string; + entryKeyXdr: string; + entryType: string; + entryLabel: string | null; + thresholdLedgers: number; + ttlAtFire: number; + firedAtLedger: number; + firedAt: string; + resolved: number; + resolvedAt: string | null; + delivered: number; + deliveredAt: string | null; + retryCount: number; +} + +export function getAlertHistory(db: Database.Database, contractId: string, limit?: number): AlertHistoryRecord[] { + const sql = ` + SELECT + af.id AS alertFiredId, + ac.channel_type AS channelType, + ac.channel_target AS channelTarget, + ce.entry_key_xdr AS entryKeyXdr, + ce.entry_type AS entryType, + ce.label AS entryLabel, + ac.threshold_ledgers AS thresholdLedgers, + af.ttl_at_fire AS ttlAtFire, + af.fired_at_ledger AS firedAtLedger, + af.fired_at AS firedAt, + af.resolved AS resolved, + af.resolved_at AS resolvedAt, + af.delivered AS delivered, + af.delivered_at AS deliveredAt, + af.retry_count AS retryCount + FROM alerts_fired af + JOIN alert_configs ac ON ac.id = af.alert_config_id + JOIN contract_entries ce ON ce.id = af.contract_entry_id + WHERE ac.contract_id = ? + ORDER BY af.fired_at DESC + ${limit ? "LIMIT ?" : ""} + `; + return (limit + ? db.prepare(sql).all(contractId, limit) + : db.prepare(sql).all(contractId) + ) as AlertHistoryRecord[]; +} diff --git a/src/db/schema.sql b/src/db/schema.sql index 62b752d5..61c4e450 100644 --- a/src/db/schema.sql +++ b/src/db/schema.sql @@ -37,9 +37,10 @@ CREATE TABLE IF NOT EXISTS extension_policies ( CREATE TABLE IF NOT EXISTS alert_configs ( id INTEGER PRIMARY KEY AUTOINCREMENT, contract_id TEXT NOT NULL REFERENCES contracts(id) ON DELETE CASCADE, - channel_type TEXT NOT NULL CHECK(channel_type IN ('email', 'slack', 'webhook')), + channel_type TEXT NOT NULL CHECK(channel_type IN ('slack', 'webhook')), channel_target TEXT NOT NULL, threshold_ledgers INTEGER NOT NULL, + webhook_secret TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP ); @@ -53,7 +54,8 @@ CREATE TABLE IF NOT EXISTS alerts_fired ( resolved BOOLEAN NOT NULL DEFAULT 0, resolved_at TEXT, delivered INTEGER NOT NULL DEFAULT 0, - delivered_at TEXT + delivered_at TEXT, + retry_count INTEGER NOT NULL DEFAULT 0 ); CREATE TABLE IF NOT EXISTS extension_history ( diff --git a/tests/alerts/dispatcher.test.ts b/tests/alerts/dispatcher.test.ts index eb0f85b2..a74f4425 100644 --- a/tests/alerts/dispatcher.test.ts +++ b/tests/alerts/dispatcher.test.ts @@ -6,6 +6,7 @@ import { upsertEntry, insertAlertConfig, recordAlertFired, + MAX_RETRY_COUNT, } from "../../src/db/repositories"; // ─── Mocks ─────────────────────────────────────────────────────────────────── @@ -33,10 +34,11 @@ function seedContractWithAlert( network?: string; entryKeyXdr?: string; entryType?: string; - channelType?: "webhook" | "slack" | "email"; + channelType?: "webhook" | "slack"; channelTarget?: string; thresholdLedgers?: number; ttlAtFire?: number; + webhookSecret?: string; } ): { entryId: number; alertConfigId: number; alertFiredId: number } { const network = opts.network ?? "testnet"; @@ -64,6 +66,7 @@ function seedContractWithAlert( channel_type: opts.channelType ?? "webhook", channel_target: opts.channelTarget ?? "https://example.com/hook", threshold_ledgers: opts.thresholdLedgers ?? 20_000, + webhook_secret: opts.webhookSecret, }); const config = db @@ -104,6 +107,7 @@ describe("deliverPendingAlerts", () => { expect(result).toHaveProperty("attempted"); expect(result).toHaveProperty("delivered"); expect(result).toHaveProperty("failed"); + expect(result).toHaveProperty("abandoned"); expect(result).toHaveProperty("errors"); expect(Array.isArray(result.errors)).toBe(true); }); @@ -114,6 +118,7 @@ describe("deliverPendingAlerts", () => { expect(result.attempted).toBe(0); expect(result.delivered).toBe(0); expect(result.failed).toBe(0); + expect(result.abandoned).toBe(0); expect(result.errors).toHaveLength(0); }); }); @@ -150,7 +155,7 @@ describe("deliverPendingAlerts", () => { expect(mockSendWebhookAlert).not.toHaveBeenCalled(); }); - it("calls sendWebhookAlert with the correct URL and event payload", async () => { + it("calls sendWebhookAlert with the correct URL, event payload, and secret", async () => { mockSendWebhookAlert.mockResolvedValue(undefined); seedContractWithAlert(db, { contractId: "CTEST1234", @@ -159,16 +164,19 @@ describe("deliverPendingAlerts", () => { channelTarget: "https://ops.example.com/hook", thresholdLedgers: 15_000, ttlAtFire: 7_000, + webhookSecret: "test-secret-123", }); await deliverPendingAlerts(db, "testnet"); - const [url, event] = mockSendWebhookAlert.mock.calls[0]!; + const [url, event, secret] = mockSendWebhookAlert.mock.calls[0]!; expect(url).toBe("https://ops.example.com/hook"); + expect(secret).toBe("test-secret-123"); expect(event.type).toBe("threshold_crossed"); expect(event.contractId).toBe("CTEST1234"); expect(event.contractName).toBe("test-contract"); expect(event.network).toBe("testnet"); + expect(event.severity).toMatch(/^(warning|critical)$/); expect(event.threshold.configuredLedgers).toBe(15_000); expect(event.threshold.currentRemainingLedgers).toBe(7_000); expect(typeof event.threshold.approximateTimeRemaining).toBe("string"); @@ -189,21 +197,6 @@ describe("deliverPendingAlerts", () => { expect(channel).toBe("#my-alerts"); expect(event.type).toBe("threshold_crossed"); }); - - it("does not call any handler for email channel type (not yet implemented)", async () => { - seedContractWithAlert(db, { - contractId: "CA", - channelType: "email", - channelTarget: "ops@example.com", - }); - - const result = await deliverPendingAlerts(db, "testnet"); - - expect(mockSendWebhookAlert).not.toHaveBeenCalled(); - expect(mockSendSlackAlert).not.toHaveBeenCalled(); - // email should be counted as skipped — not failed, not delivered - expect(result.attempted).toBe(1); - }); }); // ========================================================================= @@ -259,23 +252,63 @@ describe("deliverPendingAlerts", () => { expect(mockSendWebhookAlert).toHaveBeenCalledTimes(1); let row = db - .prepare("SELECT delivered FROM alerts_fired WHERE id = ?") - .get(alertFiredId) as { delivered: number }; + .prepare("SELECT delivered, retry_count FROM alerts_fired WHERE id = ?") + .get(alertFiredId) as { delivered: number; retry_count: number }; expect(row.delivered).toBe(0); + expect(row.retry_count).toBe(1); // Second cycle — succeeds await deliverPendingAlerts(db, "testnet"); expect(mockSendWebhookAlert).toHaveBeenCalledTimes(2); row = db - .prepare("SELECT delivered FROM alerts_fired WHERE id = ?") - .get(alertFiredId) as { delivered: number }; + .prepare("SELECT delivered, retry_count FROM alerts_fired WHERE id = ?") + .get(alertFiredId) as { delivered: number; retry_count: number }; expect(row.delivered).toBe(1); }); }); // ========================================================================= - // 4. ERROR RESILIENCE + // 4. RETRY LIMITS + // ========================================================================= + describe("Retry limits", () => { + it("stops retrying after MAX_RETRY_COUNT failures", async () => { + mockSendWebhookAlert.mockRejectedValue(new Error("permanent failure")); + const { alertFiredId } = seedContractWithAlert(db, { contractId: "CA" }); + + // Run MAX_RETRY_COUNT cycles — each should attempt delivery + for (let i = 0; i < MAX_RETRY_COUNT; i++) { + await deliverPendingAlerts(db, "testnet"); + } + + expect(mockSendWebhookAlert).toHaveBeenCalledTimes(MAX_RETRY_COUNT); + + // Next cycle should NOT attempt delivery — alert excluded by retry cap + await deliverPendingAlerts(db, "testnet"); + expect(mockSendWebhookAlert).toHaveBeenCalledTimes(MAX_RETRY_COUNT); + + const row = db + .prepare("SELECT retry_count, delivered FROM alerts_fired WHERE id = ?") + .get(alertFiredId) as { retry_count: number; delivered: number }; + expect(row.retry_count).toBe(MAX_RETRY_COUNT); + expect(row.delivered).toBe(0); + }); + + it("reports abandoned count in result", async () => { + mockSendWebhookAlert.mockRejectedValue(new Error("fail")); + const { alertFiredId } = seedContractWithAlert(db, { contractId: "CA" }); + + // Set retry_count to MAX_RETRY_COUNT - 1 so next failure abandons it + db.prepare("UPDATE alerts_fired SET retry_count = ? WHERE id = ?") + .run(MAX_RETRY_COUNT - 1, alertFiredId); + + const result = await deliverPendingAlerts(db, "testnet"); + expect(result.abandoned).toBe(1); + }); + }); + + // ========================================================================= + // 5. ERROR RESILIENCE // ========================================================================= describe("Error resilience", () => { it("never throws even if all deliveries fail", async () => { @@ -328,7 +361,7 @@ describe("deliverPendingAlerts", () => { }); // ========================================================================= - // 5. COUNTING + // 6. COUNTING // ========================================================================= describe("Result counting", () => { it("counts attempted as total alerts processed regardless of outcome", async () => { @@ -371,7 +404,7 @@ describe("deliverPendingAlerts", () => { }); // ========================================================================= - // 6. NETWORK ISOLATION + // 7. NETWORK ISOLATION // ========================================================================= describe("Network isolation", () => { it("only delivers alerts for the specified network", async () => { @@ -396,7 +429,7 @@ describe("deliverPendingAlerts", () => { }); // ========================================================================= - // 7. PAYLOAD CORRECTNESS + // 8. PAYLOAD CORRECTNESS // ========================================================================= describe("Payload correctness", () => { it("event timestamp is a valid ISO 8601 string", async () => { @@ -445,5 +478,15 @@ describe("deliverPendingAlerts", () => { expect(typeof event.threshold.approximateTimeRemaining).toBe("string"); expect(event.threshold.approximateTimeRemaining.length).toBeGreaterThan(0); }); + + it("event includes severity field", async () => { + mockSendWebhookAlert.mockResolvedValue(undefined); + seedContractWithAlert(db, { contractId: "CA", ttlAtFire: 1_000 }); + + await deliverPendingAlerts(db, "testnet"); + + const [, event] = mockSendWebhookAlert.mock.calls[0]!; + expect(event.severity).toBe("critical"); + }); }); }); diff --git a/tests/alerts/slack.test.ts b/tests/alerts/slack.test.ts index 053e5228..5bb81433 100644 --- a/tests/alerts/slack.test.ts +++ b/tests/alerts/slack.test.ts @@ -15,6 +15,7 @@ const VALID_TOKEN = "xoxb-test-slack-bot-token"; function makeAlertEvent(overrides: Partial = {}): AlertEvent { return { type: "threshold_crossed", + severity: "warning", contractId: "CDEF1234ABCD5678", contractName: "my-defi-pool", network: "mainnet", @@ -66,20 +67,20 @@ describe("sendSlackAlert", () => { // 1. TOKEN VALIDATION // ========================================================================= describe("Token validation", () => { - it("throws a clear error when SENTINEL_SLACK_TOKEN is not set", async () => { + it("throws a clear error when no Slack token is configured", async () => { delete process.env["SENTINEL_SLACK_TOKEN"]; await expect( sendSlackAlert("#oncall", makeAlertEvent()), - ).rejects.toThrow("SENTINEL_SLACK_TOKEN"); + ).rejects.toThrow(/[Ss]lack token/); }); - it("throws when SENTINEL_SLACK_TOKEN is an empty string", async () => { + it("throws when SENTINEL_SLACK_TOKEN is an empty string and no config token", async () => { process.env["SENTINEL_SLACK_TOKEN"] = ""; await expect( sendSlackAlert("#oncall", makeAlertEvent()), - ).rejects.toThrow("SENTINEL_SLACK_TOKEN"); + ).rejects.toThrow(/[Ss]lack token/); }); it("uses the token in the Authorization header", async () => { diff --git a/tests/alerts/webhook.test.ts b/tests/alerts/webhook.test.ts index c94a06e4..9ac50892 100644 --- a/tests/alerts/webhook.test.ts +++ b/tests/alerts/webhook.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { createHmac } from "node:crypto"; // ─── Mock fetch before importing the module under test ──────────────────────── @@ -13,6 +14,7 @@ import type { AlertEvent } from "../../src/alerts/types"; function makeAlertEvent(overrides: Partial = {}): AlertEvent { return { type: "threshold_crossed", + severity: "warning", contractId: "CDEF1234ABCD5678", contractName: "my-defi-pool", network: "testnet", @@ -33,7 +35,6 @@ function makeAlertEvent(overrides: Partial = {}): AlertEvent { } function makeOkResponse(status = 200): Response { - // 204 No Content must not have a body (Response constructor enforces this) if (status === 204) { return new Response(null, { status }); } @@ -122,7 +123,54 @@ describe("sendWebhookAlert", () => { }); // ========================================================================= - // 2. SUCCESS HANDLING + // 2. HMAC SIGNING + // ========================================================================= + describe("HMAC signing", () => { + it("does not include X-Sentinel-Signature header when no secret provided", async () => { + mockFetch.mockResolvedValue(makeOkResponse()); + + await sendWebhookAlert("https://example.com/hook", makeAlertEvent()); + + const [, options] = mockFetch.mock.calls[0]!; + expect(options.headers["X-Sentinel-Signature"]).toBeUndefined(); + }); + + it("includes X-Sentinel-Signature header when secret is provided", async () => { + mockFetch.mockResolvedValue(makeOkResponse()); + const secret = "my-webhook-secret"; + + await sendWebhookAlert("https://example.com/hook", makeAlertEvent(), secret); + + const [, options] = mockFetch.mock.calls[0]!; + expect(options.headers["X-Sentinel-Signature"]).toBeDefined(); + expect(options.headers["X-Sentinel-Signature"]).toMatch(/^sha256=[a-f0-9]{64}$/); + }); + + it("signature is a valid HMAC-SHA256 of the body", async () => { + mockFetch.mockResolvedValue(makeOkResponse()); + const secret = "test-secret-key"; + const event = makeAlertEvent(); + + await sendWebhookAlert("https://example.com/hook", event, secret); + + const [, options] = mockFetch.mock.calls[0]!; + const body = options.body as string; + const expectedSig = createHmac("sha256", secret).update(body).digest("hex"); + expect(options.headers["X-Sentinel-Signature"]).toBe(`sha256=${expectedSig}`); + }); + + it("does not include signature when secret is null", async () => { + mockFetch.mockResolvedValue(makeOkResponse()); + + await sendWebhookAlert("https://example.com/hook", makeAlertEvent(), null); + + const [, options] = mockFetch.mock.calls[0]!; + expect(options.headers["X-Sentinel-Signature"]).toBeUndefined(); + }); + }); + + // ========================================================================= + // 3. SUCCESS HANDLING // ========================================================================= describe("Success handling", () => { it("resolves without throwing on 200", async () => { @@ -148,7 +196,7 @@ describe("sendWebhookAlert", () => { }); // ========================================================================= - // 3. ERROR HANDLING + // 4. ERROR HANDLING // ========================================================================= describe("Error handling", () => { it("throws on 400 Bad Request", async () => { @@ -159,22 +207,6 @@ describe("sendWebhookAlert", () => { ).rejects.toThrow("400"); }); - it("throws on 401 Unauthorized", async () => { - mockFetch.mockResolvedValue(makeErrorResponse(401)); - - await expect( - sendWebhookAlert("https://example.com/hook", makeAlertEvent()), - ).rejects.toThrow("401"); - }); - - it("throws on 404 Not Found", async () => { - mockFetch.mockResolvedValue(makeErrorResponse(404)); - - await expect( - sendWebhookAlert("https://example.com/hook", makeAlertEvent()), - ).rejects.toThrow("404"); - }); - it("throws on 500 Internal Server Error", async () => { mockFetch.mockResolvedValue(makeErrorResponse(500)); @@ -183,14 +215,6 @@ describe("sendWebhookAlert", () => { ).rejects.toThrow("500"); }); - it("throws on 503 Service Unavailable", async () => { - mockFetch.mockResolvedValue(makeErrorResponse(503)); - - await expect( - sendWebhookAlert("https://example.com/hook", makeAlertEvent()), - ).rejects.toThrow("503"); - }); - it("throws when fetch itself rejects (network unreachable)", async () => { mockFetch.mockRejectedValue(new Error("ECONNREFUSED")); @@ -198,18 +222,10 @@ describe("sendWebhookAlert", () => { sendWebhookAlert("https://example.com/hook", makeAlertEvent()), ).rejects.toThrow("ECONNREFUSED"); }); - - it("throws when fetch rejects with a non-Error value", async () => { - mockFetch.mockRejectedValue("network gone"); - - await expect( - sendWebhookAlert("https://example.com/hook", makeAlertEvent()), - ).rejects.toBeDefined(); - }); }); // ========================================================================= - // 4. REQUEST CONFIGURATION + // 5. REQUEST CONFIGURATION // ========================================================================= describe("Request configuration", () => { it("sets a signal for abort / timeout control", async () => { @@ -218,11 +234,10 @@ describe("sendWebhookAlert", () => { await sendWebhookAlert("https://example.com/hook", makeAlertEvent()); const [, options] = mockFetch.mock.calls[0]!; - // signal must be an AbortSignal instance expect(options.signal).toBeDefined(); }); - it("does not send extra unexpected top-level keys in the body", async () => { + it("body includes severity field", async () => { mockFetch.mockResolvedValue(makeOkResponse()); const event = makeAlertEvent(); @@ -237,6 +252,7 @@ describe("sendWebhookAlert", () => { "entry", "firedAtLedger", "network", + "severity", "threshold", "timestamp", "type", diff --git a/tests/commands/alerts.test.ts b/tests/commands/alerts.test.ts index 68e6290d..2da5b493 100644 --- a/tests/commands/alerts.test.ts +++ b/tests/commands/alerts.test.ts @@ -101,33 +101,30 @@ describe("alerts command", () => { }); }); - it("adds an email alert configuration", () => { + it("rejects email alert type as not yet implemented", () => { const program = new Command(); registerAlertsCommand(program); - program.parse([ - "node", - "sentinel", - "alerts", - "add", - "--contract", - contractID, - "--type", - "email", - "--email", - "test@example.com", - "--threshold", - "3000", - ]); + expect(() => { + program.parse([ + "node", + "sentinel", + "alerts", + "add", + "--contract", + contractID, + "--type", + "email", + "--url", + "https://example.com", + "--threshold", + "3000", + ]); + }).toThrow("process.exit called"); - const configs = getAlertConfigsForContract(mockDb, contractID); - expect(configs).toHaveLength(1); - expect(configs[0]).toMatchObject({ - contract_id: contractID, - channel_type: "email", - channel_target: "test@example.com", - threshold_ledgers: 3000, - }); + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining("not yet implemented") + ); }); it("fails if contract is not registered", () => { diff --git a/tests/db/alert_delivery.test.ts b/tests/db/alert_delivery.test.ts index 6d797895..54801f9c 100644 --- a/tests/db/alert_delivery.test.ts +++ b/tests/db/alert_delivery.test.ts @@ -22,7 +22,7 @@ function seedFull( entryKeyXdr?: string; entryType?: string; liveUntil?: number; - channelType?: "webhook" | "slack" | "email"; + channelType?: "webhook" | "slack"; channelTarget?: string; thresholdLedgers?: number; ttlAtFire?: number; @@ -128,6 +128,8 @@ describe("getUndeliveredAlerts", () => { expect(typeof alert.alertFiredId).toBe("number"); expect(typeof alert.entryId).toBe("number"); expect(typeof alert.alertConfigId).toBe("number"); + expect(alert.retryCount).toBe(0); + expect(alert.webhookSecret).toBeNull(); }); it("returns multiple undelivered alerts", () => {