perf(events): has_embedding column + maintenance triggers

db/migrations/20260517000000_events_has_embedding.sql

@@ -0,0 +1,56 @@
+-- migrate:up
+
+-- has_embedding mirrors "does event_embeddings have a row for this event_id".
+-- The embed-backfill scheduler today does a 1.27s seq scan + hash anti-join
+-- on the 1.15M-row events table every 5 min (pg_stat_statements rank #1 by
+-- total_exec_time, see lobu#767 postmortem). With this column + a partial
+-- index (added in a follow-up migration after backfill), the scheduler
+-- becomes a tiny index lookup over the actually-missing-embedding rows.
+--
+-- This migration only adds the column and the maintenance triggers. The
+-- column is intentionally:
+--   * nullable: ADD COLUMN <bool> NULL is O(1) metadata-only in PG 11+; a
+--     DEFAULT would rewrite all 1.15M rows under ACCESS EXCLUSIVE (the same
+--     trap that timed out 20260516200000_events_search_tsv).
+--   * not backfilled here: existing rows are populated by a batched script
+--     (scripts/backfill-events-has-embedding.sql) that runs outside the
+--     Helm hook so it can pace itself with statement_timeout headroom.
+--
+-- Until backfill finishes, has_embedding IS NULL means "unknown" for old
+-- rows; new rows get has_embedding flipped by the triggers below. The
+-- partial index (next migration) treats NULL the same as FALSE so the
+-- scheduler keeps working.
+
+ALTER TABLE public.events ADD COLUMN has_embedding boolean;
+
+CREATE OR REPLACE FUNCTION public.event_embeddings_after_insert() RETURNS trigger AS $$
+BEGIN
+  UPDATE public.events SET has_embedding = true WHERE id = NEW.event_id;
+  RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION public.event_embeddings_after_delete() RETURNS trigger AS $$
+BEGIN
+  UPDATE public.events SET has_embedding = false WHERE id = OLD.event_id;
+  RETURN OLD;
+END;
+$$ LANGUAGE plpgsql;
+
+DROP TRIGGER IF EXISTS trg_event_embeddings_after_insert ON public.event_embeddings;
+CREATE TRIGGER trg_event_embeddings_after_insert
+  AFTER INSERT ON public.event_embeddings
+  FOR EACH ROW EXECUTE FUNCTION public.event_embeddings_after_insert();
+
+DROP TRIGGER IF EXISTS trg_event_embeddings_after_delete ON public.event_embeddings;
+CREATE TRIGGER trg_event_embeddings_after_delete
+  AFTER DELETE ON public.event_embeddings
+  FOR EACH ROW EXECUTE FUNCTION public.event_embeddings_after_delete();
+
+-- migrate:down
+
+DROP TRIGGER IF EXISTS trg_event_embeddings_after_insert ON public.event_embeddings;
+DROP TRIGGER IF EXISTS trg_event_embeddings_after_delete ON public.event_embeddings;
+DROP FUNCTION IF EXISTS public.event_embeddings_after_insert();
+DROP FUNCTION IF EXISTS public.event_embeddings_after_delete();
+ALTER TABLE public.events DROP COLUMN IF EXISTS has_embedding;

scripts/backfill-events-has-embedding.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+# Batched backfill for events.has_embedding (added in migration
+# 20260517000000_events_has_embedding.sql).
+#
+# Why batched and not in the migration: a single UPDATE over 1.15M rows
+# would hold an exclusive row-level lock for ~minute on a hot table while
+# WAL-amplifying every row. Doing it in batches of 10k means each transaction
+# is bounded (~1s) so VACUUM can keep up, autovacuum doesn't fall behind, and
+# the API stays responsive.
+#
+# Idempotent: only touches rows where has_embedding IS NULL, so re-running
+# after a partial run picks up where it left off.
+#
+# Run from the repo root:
+#   DATABASE_URL="postgres://..." ./scripts/backfill-events-has-embedding.sh
+#
+# Operational note: takes ~5-10 min for 1.15M rows on a small Postgres. Safe
+# to ctrl-C and resume; safe to run alongside live traffic.
+
+set -euo pipefail
+
+if [ -z "${DATABASE_URL:-}" ]; then
+  echo "DATABASE_URL not set" >&2
+  exit 1
+fi
+
+BATCH_SIZE="${BATCH_SIZE:-10000}"
+SLEEP_BETWEEN_BATCHES="${SLEEP_BETWEEN_BATCHES:-0.1}"
+
+echo "[backfill] BATCH_SIZE=$BATCH_SIZE SLEEP_BETWEEN_BATCHES=${SLEEP_BETWEEN_BATCHES}s"
+
+remaining_count() {
+  psql "$DATABASE_URL" -tAc "SELECT count(*) FROM public.events WHERE has_embedding IS NULL"
+}
+
+total_remaining=$(remaining_count)
+echo "[backfill] $total_remaining rows have has_embedding IS NULL"
+
+if [ "$total_remaining" = "0" ]; then
+  echo "[backfill] nothing to do"
+  exit 0
+fi
+
+batches=0
+while true; do
+  # Each batch is one transaction. Update up to $BATCH_SIZE rows where the
+  # column is still unknown. UPDATE FROM joins to event_embeddings to set
+  # the right value; rows with no matching embedding row get FALSE.
+  rows_updated=$(psql "$DATABASE_URL" -tAc "
+    WITH batch AS (
+      SELECT id FROM public.events
+       WHERE has_embedding IS NULL
+       LIMIT $BATCH_SIZE
+       FOR UPDATE SKIP LOCKED
+    )
+    UPDATE public.events e
+       SET has_embedding = (emb.event_id IS NOT NULL)
+      FROM batch b
+      LEFT JOIN public.event_embeddings emb ON emb.event_id = b.id
+     WHERE e.id = b.id
+    RETURNING 1
+  " | wc -l | tr -d ' ')
+
+  batches=$((batches + 1))
+
+  if [ "$rows_updated" = "0" ]; then
+    echo "[backfill] no rows updated in batch $batches — done"
+    break
+  fi
+
+  if [ $((batches % 10)) -eq 0 ]; then
+    current_remaining=$(remaining_count)
+    echo "[backfill] batch $batches: ~${rows_updated} rows updated this batch, ${current_remaining} remaining"
+  fi
+
+  sleep "$SLEEP_BETWEEN_BATCHES"
+done
+
+final_remaining=$(remaining_count)
+echo "[backfill] complete: $final_remaining rows still have has_embedding IS NULL (expect 0)"