feat(datadog): loud MacroDB prod CPU and wait alerts

.github/workflows/sync-macrodb-datadog-alerts.yml

@@ -0,0 +1,35 @@
+name: Sync MacroDB Datadog alerts
+
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - 'infra/stacks/datadog-macrodb-alerts/monitors.ts'
+      - 'infra/stacks/datadog-macrodb-alerts/sync.ts'
+      - '.github/workflows/sync-macrodb-datadog-alerts.yml'
+
+concurrency:
+  group: sync-macrodb-datadog-alerts
+  cancel-in-progress: false
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+        with:
+          persist-credentials: false
+
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: 1.2.0
+
+      - name: Upsert MacroDB Datadog monitors
+        working-directory: infra/stacks/datadog-macrodb-alerts
+        env:
+          DD_API_KEY: ${{ secrets.DD_API_KEY }}
+          DD_APP_KEY: ${{ secrets.DD_APP_KEY }}
+          DD_SITE: us5.datadoghq.com
+        run: bun run sync.ts

infra/stacks/datadog-macrodb-alerts/Pulumi.dev.yaml

@@ -0,0 +1,3 @@
+config:
+  aws:region: us-east-1
+  datadog:validate: "false"

infra/stacks/datadog-macrodb-alerts/Pulumi.prod.yaml

@@ -0,0 +1,2 @@
+config:
+  aws:region: us-east-1

infra/stacks/datadog-macrodb-alerts/Pulumi.yaml

@@ -0,0 +1,3 @@
+name: datadog-macrodb-alerts
+runtime: nodejs
+description: Datadog monitors for MacroDB primary and read replica

infra/stacks/datadog-macrodb-alerts/index.ts

@@ -0,0 +1,33 @@
+import * as datadog from '@pulumi/datadog';
+import * as pulumi from '@pulumi/pulumi';
+import { MACRODB_PROD_MONITORS } from './monitors';
+
+const stack = pulumi.getStack();
+
+if (stack === 'prod') {
+  for (const spec of MACRODB_PROD_MONITORS) {
+    new datadog.Monitor(spec.id, {
+      name: spec.name,
+      type: spec.type,
+      query: spec.query,
+      message: spec.message,
+      tags: spec.tags,
+      priority: String(spec.priority),
+      monitorThresholds: {
+        critical: spec.thresholds.critical,
+        warning: spec.thresholds.warning,
+        criticalRecovery: spec.thresholds.criticalRecovery,
+        warningRecovery: spec.thresholds.warningRecovery,
+      },
+      requireFullWindow: spec.requireFullWindow,
+      evaluationDelay: spec.evaluationDelay,
+      notifyNoData: spec.notifyNoData,
+      noDataTimeframe: spec.noDataTimeframe,
+      includeTags: true,
+      newGroupDelay: 60,
+      timeoutH: 0,
+    });
+  }
+}
+
+export const monitorCount = stack === 'prod' ? MACRODB_PROD_MONITORS.length : 0;

infra/stacks/datadog-macrodb-alerts/monitors.ts

@@ -0,0 +1,170 @@
+/**
+ * MacroDB prod alert registry.
+ *
+ * Each entry becomes one Datadog metric monitor. Deploy only on the prod stack.
+ * Notifications target the Macro Alerts channel via @webhook-macro-alert-hook.
+ */
+
+export type MonitorSpec = {
+  /** Pulumi resource name. Stable across deploys. */
+  id: string;
+  name: string;
+  type: 'query alert';
+  query: string;
+  message: string;
+  tags: string[];
+  thresholds: {
+    critical: string;
+    warning?: string;
+    criticalRecovery?: string;
+    warningRecovery?: string;
+  };
+  priority: number;
+  requireFullWindow?: boolean;
+  evaluationDelay?: number;
+  notifyNoData?: boolean;
+  noDataTimeframe?: number;
+};
+
+/** RDS instance identifiers for MacroDB prod primary and read replica. */
+export const MACRODB_PROD_DB_IDS =
+  'dbinstanceidentifier:macro-db-prod OR dbinstanceidentifier:macro-db-prod-read-replica';
+
+/**
+ * DBM `database_instance` tags (FQDNs) for MacroDB prod primary and read replica.
+ */
+export const MACRODB_PROD_DATABASE_INSTANCES =
+  'database_instance:macro-db-prod.ctkwyjgndnfr.us-east-1.rds.amazonaws.com OR database_instance:macro-db-prod-read-replica.ctkwyjgndnfr.us-east-1.rds.amazonaws.com';
+
+export const MACRODB_PROD_READ_REPLICA_DATABASE_INSTANCE =
+  'database_instance:macro-db-prod-read-replica.ctkwyjgndnfr.us-east-1.rds.amazonaws.com';
+
+const ALERTS_CHANNEL = '@webhook-macro-alert-hook';
+
+const COMMON_TAGS = [
+  'env:prod',
+  'service:macrodb',
+  'team:infra',
+  'managed:datadog-macrodb-alerts',
+];
+
+function alertMessage(body: string): string {
+  return `{{#is_alert}}
+🚨 ${body}
+
+${ALERTS_CHANNEL}
+{{/is_alert}}
+{{#is_warning}}
+⚠️ ${body}
+
+${ALERTS_CHANNEL}
+{{/is_warning}}
+{{#is_recovery}}
+✅ Recovered: ${body}
+
+${ALERTS_CHANNEL}
+{{/is_recovery}}`;
+}
+
+/**
+ * Prod MacroDB monitors.
+ *
+ * Thresholds come from 7d baselines (primary CPU avg ~4% / max ~14%, replica
+ * max ~36%, lock-wait spikes, blocked_connections mostly 0 with rare spikes,
+ * replica lag usually <1s with rare multi-second bumps). Existing UI monitor
+ * "[PROD] RDS High CPU Utilization" stays at 95%/1h and is too quiet; these
+ * fire earlier into Alerts.
+ */
+export const MACRODB_PROD_MONITORS: MonitorSpec[] = [
+  {
+    id: 'macrodb-cpu-high',
+    name: '[PROD] MacroDB CPU high',
+    type: 'query alert',
+    query: `avg(last_10m):avg:aws.rds.cpuutilization{${MACRODB_PROD_DB_IDS}} by {dbinstanceidentifier}.weighted() > 70`,
+    message: alertMessage(
+      'MacroDB {{dbinstanceidentifier.name}} CPU is {{value}}% (threshold {{threshold}}%). Check DBM activity and top queries.'
+    ),
+    tags: [...COMMON_TAGS, 'signal:cpu'],
+    thresholds: {
+      critical: '70',
+      warning: '50',
+    },
+    priority: 2,
+    requireFullWindow: true,
+    evaluationDelay: 60,
+    notifyNoData: false,
+  },
+  {
+    id: 'macrodb-lock-waits-high',
+    name: '[PROD] MacroDB lock waits high',
+    type: 'query alert',
+    query: `avg(last_10m):sum:postgresql.activity.waits{(${MACRODB_PROD_DATABASE_INSTANCES}) AND wait_event_type:Lock} by {database_instance} > 15`,
+    message: alertMessage(
+      'MacroDB {{database_instance.name}} has {{value}} sessions in Lock waits (threshold {{threshold}}). Check blockers and long transactions in DBM.'
+    ),
+    tags: [...COMMON_TAGS, 'signal:lock-waits'],
+    thresholds: {
+      critical: '15',
+      warning: '8',
+    },
+    priority: 1,
+    requireFullWindow: false,
+    evaluationDelay: 60,
+    notifyNoData: false,
+  },
+  {
+    id: 'macrodb-blocked-connections',
+    name: '[PROD] MacroDB blocked connections',
+    type: 'query alert',
+    query: `avg(last_10m):avg:postgresql.activity.blocked_connections{${MACRODB_PROD_DATABASE_INSTANCES}} by {database_instance} > 5`,
+    message: alertMessage(
+      'MacroDB {{database_instance.name}} has {{value}} blocked connections (threshold {{threshold}}). Lock contention is active.'
+    ),
+    tags: [...COMMON_TAGS, 'signal:blocked-connections'],
+    thresholds: {
+      critical: '5',
+      warning: '2',
+    },
+    priority: 1,
+    requireFullWindow: false,
+    evaluationDelay: 60,
+    notifyNoData: false,
+  },
+  {
+    id: 'macrodb-io-waits-high',
+    name: '[PROD] MacroDB IO waits high',
+    type: 'query alert',
+    query: `avg(last_15m):sum:postgresql.activity.waits{(${MACRODB_PROD_DATABASE_INSTANCES}) AND wait_event_type:IO} by {database_instance} > 25`,
+    message: alertMessage(
+      'MacroDB {{database_instance.name}} has {{value}} sessions in IO waits (threshold {{threshold}}). Check storage IOPS, WAL, and heavy queries.'
+    ),
+    tags: [...COMMON_TAGS, 'signal:io-waits'],
+    thresholds: {
+      critical: '25',
+      warning: '15',
+    },
+    priority: 2,
+    requireFullWindow: true,
+    evaluationDelay: 60,
+    notifyNoData: false,
+  },
+  {
+    id: 'macrodb-replication-lag',
+    name: '[PROD] MacroDB read replica lag',
+    type: 'query alert',
+    query: `avg(last_5m):avg:postgresql.replication_delay{${MACRODB_PROD_READ_REPLICA_DATABASE_INSTANCE}} > 60`,
+    message: alertMessage(
+      'MacroDB read replica lag is {{value}}s (threshold {{threshold}}s). Reads may be stale; check replica CPU/IO and primary WAL pressure.'
+    ),
+    tags: [...COMMON_TAGS, 'signal:replication-lag'],
+    thresholds: {
+      critical: '60',
+      warning: '15',
+    },
+    priority: 2,
+    requireFullWindow: true,
+    evaluationDelay: 60,
+    notifyNoData: true,
+    noDataTimeframe: 20,
+  },
+];

infra/stacks/datadog-macrodb-alerts/package.json

@@ -0,0 +1,7 @@
+{
+	"name": "datadog-macrodb-alerts-stack",
+	"version": "0.0.0",
+	"private": true,
+	"license": "MIT",
+	"main": "index.ts"
+}