From 90dbc383938cd9347dc0c8ec4b7e5dd8463e54ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Emre=20Kabakc=C4=B1?= Date: Sat, 16 May 2026 19:23:31 +0100 Subject: [PATCH 1/3] perf: drop 8 unused indexes (5.16 GB) + event_count from list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three changes bundled around the post-incident perf brainstorm, each verified against prod: * db/migrations: drops 8 indexes that pg_stat_user_indexes reports idx_scan=0 after 28h of prod uptime — `idx_events_embedding` (4.3 GB ivfflat ANN), `idx_events_raw_content_trgm` (554 MB trigram GIN), `idx_events_search_tsv` (228 MB fulltext GIN), and 5 smaller ones. pg_stat_statements over the same window confirms zero queries touch the shapes they serve (<->, <=>, payload_text ILIKE, @@ to_tsquery). Search code paths in content-search.ts exist but aren't called in prod today; rebuild CONCURRENTLY if/when needed. Migration uses plain DROP INDEX rather than CONCURRENTLY because dbmate's transaction:false directive doesn't actually exit the transaction block against the pq driver (see comment in 20260426130001_db_integrity_cleanup_concurrent.sql). The operator runbook in docs/MIGRATIONS.md "When dbmate fails in prod" covers applying CONCURRENTLY out-of-band first, then recording the schema_migrations row. * manage_connections.ts handleList: removes the per-row event_count subselect. The supersedes anti-join through current_event_records was the entire cost of that query — verified by EXPLAIN ANALYZE against prod, 1303ms → 2.3ms (566x). handleGet (single connection detail) still computes it; that path is one row and costs ~1.2ms. Submodule bump in packages/web pulls in the matching frontend changes (owletto-web#136). * docs/MIGRATIONS.md: appends a Lobu-specific policy paragraph to the cascade section — connections are soft-deleted only in prod. The cascade UPDATE on events.connection_id is ~13s per call at current scale and is the issue at rank #8 in pg_stat_statements. The connection-creation rollback path that hard-deletes never-activated rows (no events yet, no cascade) is the only acceptable use. --- .../20260517010000_drop_unused_indexes.sql | 53 +++++++++ db/schema.sql | 107 +++++------------- docs/MIGRATIONS.md | 2 + .../src/tools/admin/manage_connections.ts | 8 +- packages/web | 2 +- 5 files changed, 93 insertions(+), 79 deletions(-) create mode 100644 db/migrations/20260517010000_drop_unused_indexes.sql diff --git a/db/migrations/20260517010000_drop_unused_indexes.sql b/db/migrations/20260517010000_drop_unused_indexes.sql new file mode 100644 index 000000000..28a1824e2 --- /dev/null +++ b/db/migrations/20260517010000_drop_unused_indexes.sql @@ -0,0 +1,53 @@ +-- migrate:up + +-- Drop 8 indexes that pg_stat_user_indexes reported `idx_scan = 0` after 28h +-- of prod uptime. Together they cost ~5.16 GB of disk + RAM plus write +-- amplification on every events / event_embeddings INSERT. +-- +-- pg_stat_statements over the same 28h shows zero calls touching the query +-- shapes these indexes serve: no `<->` / `<=>` / `<#>` vector ops (ivfflat +-- ANN), no `payload_text ILIKE` or `similarity(payload_text, …)` (trigram +-- GIN), no `@@ to_tsquery(…)` (search_tsv GIN). The code paths in +-- packages/server/src/utils/content-search.ts exist, they're just not hit +-- in prod today. If/when they get exercised, rebuild CONCURRENTLY — the +-- query plans degrade to seq scans + filter until the index is back. +-- +-- The migration uses plain `DROP INDEX` (not CONCURRENTLY) because +-- dbmate's `transaction:false` directive doesn't actually exit the +-- transaction block when running against the `pq` driver — see the +-- comment in 20260426130001_db_integrity_cleanup_concurrent.sql. +-- Operator runbook: for prod application, run `DROP INDEX CONCURRENTLY` +-- manually first (see docs/MIGRATIONS.md "When dbmate fails in prod" +-- → transaction:false recipe), then run dbmate to record this row in +-- schema_migrations. On fresh installs / CI / dev the events table is +-- empty so the brief ACCESS EXCLUSIVE is irrelevant. + +DROP INDEX IF EXISTS public.idx_events_embedding; +DROP INDEX IF EXISTS public.idx_events_raw_content_trgm; +DROP INDEX IF EXISTS public.idx_events_search_tsv; +DROP INDEX IF EXISTS public.idx_events_entity_ids_occurred_at; +DROP INDEX IF EXISTS public.idx_events_origin_parent_id; +DROP INDEX IF EXISTS public.idx_events_thread_lookup; +DROP INDEX IF EXISTS public.idx_events_run_id; +DROP INDEX IF EXISTS public.idx_events_type; + +-- migrate:down + +CREATE INDEX IF NOT EXISTS idx_events_embedding + ON public.event_embeddings USING ivfflat (embedding vector_cosine_ops) WITH (lists='1000'); +CREATE INDEX IF NOT EXISTS idx_events_raw_content_trgm + ON public.events USING gin (payload_text gin_trgm_ops); +CREATE INDEX IF NOT EXISTS idx_events_search_tsv + ON public.events USING gin (search_tsv); +CREATE INDEX IF NOT EXISTS idx_events_entity_ids_occurred_at + ON public.events USING btree ((entity_ids[1]), occurred_at DESC, id DESC) + WHERE ((entity_ids IS NOT NULL) AND (entity_ids <> '{}'::bigint[])); +CREATE INDEX IF NOT EXISTS idx_events_origin_parent_id + ON public.events USING btree (origin_parent_id); +CREATE INDEX IF NOT EXISTS idx_events_thread_lookup + ON public.events USING btree (origin_parent_id, occurred_at) + WHERE (origin_parent_id IS NOT NULL); +CREATE INDEX IF NOT EXISTS idx_events_run_id + ON public.events USING btree (run_id); +CREATE INDEX IF NOT EXISTS idx_events_type + ON public.events USING btree (origin_type) WHERE (origin_type IS NOT NULL); diff --git a/db/schema.sql b/db/schema.sql index 9ca4e6132..3557277ab 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -467,7 +467,7 @@ ALTER SEQUENCE public.connector_versions_id_seq OWNED BY public.connector_versio -- CREATE TABLE public.events ( - id bigint NOT NULL, + id bigint CONSTRAINT event_id_not_null NOT NULL, organization_id text NOT NULL, entity_ids bigint[], origin_id text, @@ -1200,7 +1200,7 @@ CREATE TABLE public.member ( -- CREATE TABLE public.migration_20260315300000_entity_type_org_backfill ( - entity_type_id integer NOT NULL + entity_type_id integer CONSTRAINT migration_20260315300000_entity_type_or_entity_type_id_not_null NOT NULL ); -- @@ -1208,7 +1208,7 @@ CREATE TABLE public.migration_20260315300000_entity_type_org_backfill ( -- CREATE TABLE public.migration_20260316100000_created_entity_types ( - entity_type_id integer NOT NULL + entity_type_id integer CONSTRAINT migration_20260316100000_created_entity_entity_type_id_not_null NOT NULL ); -- @@ -1216,14 +1216,14 @@ CREATE TABLE public.migration_20260316100000_created_entity_types ( -- CREATE TABLE public.migration_20260316100000_deleted_default_entity_types ( - id integer NOT NULL, - slug text NOT NULL, - name text NOT NULL, + id integer CONSTRAINT migration_20260316100000_deleted_default_entity_typ_id_not_null NOT NULL, + slug text CONSTRAINT migration_20260316100000_deleted_default_entity_t_slug_not_null NOT NULL, + name text CONSTRAINT migration_20260316100000_deleted_default_entity_t_name_not_null NOT NULL, description text, icon text, color text, metadata_schema jsonb, - organization_id text NOT NULL, + organization_id text CONSTRAINT migration_20260316100000_deleted_defau_organization_id_not_null NOT NULL, created_by text, updated_by text, deleted_at timestamp with time zone, @@ -1569,7 +1569,7 @@ CREATE TABLE public.scheduled_jobs ( -- CREATE TABLE public.schema_migrations ( - version character varying(128) NOT NULL + version character varying NOT NULL ); -- @@ -1751,20 +1751,20 @@ ALTER SEQUENCE public.watcher_reactions_id_seq OWNED BY public.watcher_reactions -- CREATE TABLE public.watcher_versions ( - id integer NOT NULL, - version integer NOT NULL, - name text NOT NULL, + id integer CONSTRAINT insight_template_versions_id_not_null NOT NULL, + version integer CONSTRAINT insight_template_versions_version_not_null NOT NULL, + name text CONSTRAINT insight_template_versions_name_not_null NOT NULL, description text, change_notes text, - created_by text NOT NULL, + created_by text CONSTRAINT insight_template_versions_created_by_not_null NOT NULL, created_at timestamp with time zone DEFAULT now(), keying_config jsonb, json_template jsonb, - prompt text NOT NULL, - extraction_schema jsonb NOT NULL, + prompt text CONSTRAINT insight_template_versions_prompt_not_null NOT NULL, + extraction_schema jsonb CONSTRAINT insight_template_versions_extraction_schema_not_null NOT NULL, classifiers jsonb, - required_source_types text[] DEFAULT '{}'::text[] NOT NULL, - recommended_source_types text[] DEFAULT '{}'::text[] NOT NULL, + required_source_types text[] DEFAULT '{}'::text[] CONSTRAINT insight_template_versions_required_source_types_not_null NOT NULL, + recommended_source_types text[] DEFAULT '{}'::text[] CONSTRAINT insight_template_versions_recommended_source_types_not_null NOT NULL, reactions_guidance text, condensation_prompt text, condensation_window_count integer DEFAULT 4, @@ -1848,9 +1848,9 @@ ALTER SEQUENCE public.watcher_template_versions_id_seq OWNED BY public.watcher_v -- CREATE TABLE public.watcher_window_events ( - id bigint NOT NULL, - window_id bigint NOT NULL, - event_id bigint NOT NULL, + id bigint CONSTRAINT insight_window_content_id_not_null NOT NULL, + window_id bigint CONSTRAINT insight_window_content_window_id_not_null NOT NULL, + event_id bigint CONSTRAINT insight_window_content_content_id_not_null NOT NULL, created_at timestamp with time zone DEFAULT now() ); @@ -1911,14 +1911,14 @@ ALTER SEQUENCE public.watcher_window_field_feedback_id_seq OWNED BY public.watch -- CREATE TABLE public.watcher_windows ( - id integer NOT NULL, - watcher_id integer NOT NULL, + id integer CONSTRAINT insight_windows_id_not_null NOT NULL, + watcher_id integer CONSTRAINT insight_windows_insight_id_not_null NOT NULL, parent_window_id integer, - granularity text NOT NULL, - window_start timestamp with time zone NOT NULL, - window_end timestamp with time zone NOT NULL, - content_analyzed integer NOT NULL, - extracted_data jsonb NOT NULL, + granularity text CONSTRAINT insight_windows_granularity_not_null NOT NULL, + window_start timestamp with time zone CONSTRAINT insight_windows_window_start_not_null NOT NULL, + window_end timestamp with time zone CONSTRAINT insight_windows_window_end_not_null NOT NULL, + content_analyzed integer CONSTRAINT insight_windows_content_analyzed_not_null NOT NULL, + extracted_data jsonb CONSTRAINT insight_windows_extracted_data_not_null NOT NULL, model_used text, execution_time_ms integer, is_rollup boolean DEFAULT false, @@ -1971,13 +1971,13 @@ ALTER SEQUENCE public.watcher_windows_id_seq OWNED BY public.watcher_windows.id; -- CREATE TABLE public.watchers ( - id integer NOT NULL, + id integer CONSTRAINT insights_id_not_null NOT NULL, model_config jsonb DEFAULT '{}'::jsonb, status text DEFAULT 'active'::text NOT NULL, created_at timestamp with time zone DEFAULT now(), updated_at timestamp with time zone DEFAULT now(), sources jsonb DEFAULT '[]'::jsonb, - created_by text NOT NULL, + created_by text CONSTRAINT insights_created_by_not_null NOT NULL, entity_ids bigint[], reaction_script text, reaction_script_compiled text, @@ -3391,24 +3391,12 @@ CREATE INDEX idx_events_created_at ON public.events USING btree (created_at); CREATE INDEX idx_events_created_by ON public.events USING btree (created_by) WHERE (created_by IS NOT NULL); --- --- Name: idx_events_embedding; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_events_embedding ON public.event_embeddings USING ivfflat (embedding public.vector_cosine_ops) WITH (lists='1000'); - -- -- Name: idx_events_entity_ids; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX idx_events_entity_ids ON public.events USING gin (entity_ids); --- --- Name: idx_events_entity_ids_occurred_at; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_events_entity_ids_occurred_at ON public.events USING btree ((entity_ids[1]), occurred_at DESC, id DESC) WHERE ((entity_ids IS NOT NULL) AND (entity_ids <> '{}'::bigint[])); - -- -- Name: idx_events_feed_id; Type: INDEX; Schema: public; Owner: - -- @@ -3487,30 +3475,6 @@ CREATE INDEX idx_events_missing_embedding_backfill ON public.events USING btree CREATE INDEX idx_events_organization_id ON public.events USING btree (organization_id) WHERE (organization_id IS NOT NULL); --- --- Name: idx_events_origin_parent_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_events_origin_parent_id ON public.events USING btree (origin_parent_id); - --- --- Name: idx_events_raw_content_trgm; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_events_raw_content_trgm ON public.events USING gin (payload_text public.gin_trgm_ops); - --- --- Name: idx_events_run_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_events_run_id ON public.events USING btree (run_id); - --- --- Name: idx_events_search_tsv; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_events_search_tsv ON public.events USING gin (search_tsv); - -- -- Name: idx_events_semantic_type; Type: INDEX; Schema: public; Owner: - -- @@ -3529,18 +3493,6 @@ CREATE INDEX idx_events_source_embedding ON public.event_embeddings USING btree CREATE UNIQUE INDEX idx_events_superseded_by ON public.events USING btree (supersedes_event_id) WHERE (supersedes_event_id IS NOT NULL); --- --- Name: idx_events_thread_lookup; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_events_thread_lookup ON public.events USING btree (origin_parent_id, occurred_at) WHERE (origin_parent_id IS NOT NULL); - --- --- Name: idx_events_type; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_events_type ON public.events USING btree (origin_type) WHERE (origin_type IS NOT NULL); - -- -- Name: idx_feeds_connection; Type: INDEX; Schema: public; Owner: - -- @@ -5009,4 +4961,5 @@ INSERT INTO public.schema_migrations (version) VALUES ('20260515170000'), ('20260516120000'), ('20260516200000'), - ('20260516200100'); + ('20260516200100'), + ('20260517010000'); diff --git a/docs/MIGRATIONS.md b/docs/MIGRATIONS.md index 92cee1127..5c286db39 100644 --- a/docs/MIGRATIONS.md +++ b/docs/MIGRATIONS.md @@ -112,6 +112,8 @@ A single `DELETE FROM connections WHERE id IN (...)` triggers an internal `UPDAT Indexing alone helps the cascade, but it doesn't eliminate the per-row WAL write; batching before the delete is what keeps the API responsive. +**Lobu-specific policy:** **connections are never hard-deleted in prod.** Setting `connections.deleted_at` is the final state. The `events_connection_id_fkey ... ON DELETE SET NULL` cascade exists in the schema for completeness, but actually invoking it (`DELETE FROM connections WHERE deleted_at IS NOT NULL`) blocks the API for ~13s per connection at current scale — the 2026-05-16 `pg_stat_statements` showed exactly this pattern at rank #8 (5 calls × 13.4s each). Soft-deleted connections cost ~50 bytes apiece in the `connections` table; the occasional accumulation isn't worth the recurring stall. The connection-creation rollback path in `tools/admin/manage_connections.ts` (which only deletes never-activated rows that have no events yet) is the only acceptable use of `DELETE FROM connections`. + ### Bare `DROP INDEX` Takes `ACCESS EXCLUSIVE`. Use `DROP INDEX CONCURRENTLY` (also `transaction:false`). diff --git a/packages/server/src/tools/admin/manage_connections.ts b/packages/server/src/tools/admin/manage_connections.ts index 957f6836e..9d47fdd73 100644 --- a/packages/server/src/tools/admin/manage_connections.ts +++ b/packages/server/src/tools/admin/manage_connections.ts @@ -539,7 +539,13 @@ async function handleList( AND NOT (dw.id IS NOT NULL AND dw.last_seen_at > now() - interval '20 minutes') THEN 'offline' END AS device_status, - (SELECT COUNT(*) FROM current_event_records e WHERE e.connection_id = c.id)::int AS event_count, + -- event_count intentionally omitted from list responses: the + -- per-row correlated count via current_event_records does a + -- supersedes anti-join over the events table and was the dominant + -- cost in this query (1303ms mean → 2.3ms without it; see the + -- post-incident perf brainstorm). For the per-connection detail + -- page, handleGet below still computes it — that path is a single + -- row and costs ~1.2ms. (SELECT COUNT(*) FROM feeds f WHERE f.connection_id = c.id AND f.deleted_at IS NULL)::int AS feed_count, (SELECT ct.token FROM connect_tokens ct WHERE ct.connection_id = c.id AND ct.status = 'pending' AND ct.expires_at > NOW() diff --git a/packages/web b/packages/web index c390103ed..00b249caf 160000 --- a/packages/web +++ b/packages/web @@ -1 +1 @@ -Subproject commit c390103ed009f00f91fb5547a811235c914dd3d8 +Subproject commit 00b249cafd7c239b633193cd9b5edcd2fad02166 From fdf1961c29303d456614bbfb6ae626289b34f7ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Emre=20Kabakc=C4=B1?= Date: Sat, 16 May 2026 21:47:26 +0100 Subject: [PATCH 2/3] fix(perf): correct schema.sql drift + bump web submodule CI's pg_dump produces plain NOT NULL form (vs PG18's named CONSTRAINTs). The earlier commit's schema.sql had the CONSTRAINT form leaked in from local regen, failing the schema-drift check. Restored from origin/main + re-applied only the intended 4-index removals. Web submodule bump: the earlier pin (perf/connections-detail-event-count- fallback branch tip) wasn't reachable from web/main per the strict reachability check. Re-point to current web/main HEAD. --- db/schema.sql | 80 +++++++++++++++++++++++++++++++++------------------ packages/web | 2 +- 2 files changed, 53 insertions(+), 29 deletions(-) diff --git a/db/schema.sql b/db/schema.sql index 3557277ab..dae56a751 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -467,7 +467,7 @@ ALTER SEQUENCE public.connector_versions_id_seq OWNED BY public.connector_versio -- CREATE TABLE public.events ( - id bigint CONSTRAINT event_id_not_null NOT NULL, + id bigint NOT NULL, organization_id text NOT NULL, entity_ids bigint[], origin_id text, @@ -1200,7 +1200,7 @@ CREATE TABLE public.member ( -- CREATE TABLE public.migration_20260315300000_entity_type_org_backfill ( - entity_type_id integer CONSTRAINT migration_20260315300000_entity_type_or_entity_type_id_not_null NOT NULL + entity_type_id integer NOT NULL ); -- @@ -1208,7 +1208,7 @@ CREATE TABLE public.migration_20260315300000_entity_type_org_backfill ( -- CREATE TABLE public.migration_20260316100000_created_entity_types ( - entity_type_id integer CONSTRAINT migration_20260316100000_created_entity_entity_type_id_not_null NOT NULL + entity_type_id integer NOT NULL ); -- @@ -1216,14 +1216,14 @@ CREATE TABLE public.migration_20260316100000_created_entity_types ( -- CREATE TABLE public.migration_20260316100000_deleted_default_entity_types ( - id integer CONSTRAINT migration_20260316100000_deleted_default_entity_typ_id_not_null NOT NULL, - slug text CONSTRAINT migration_20260316100000_deleted_default_entity_t_slug_not_null NOT NULL, - name text CONSTRAINT migration_20260316100000_deleted_default_entity_t_name_not_null NOT NULL, + id integer NOT NULL, + slug text NOT NULL, + name text NOT NULL, description text, icon text, color text, metadata_schema jsonb, - organization_id text CONSTRAINT migration_20260316100000_deleted_defau_organization_id_not_null NOT NULL, + organization_id text NOT NULL, created_by text, updated_by text, deleted_at timestamp with time zone, @@ -1569,7 +1569,7 @@ CREATE TABLE public.scheduled_jobs ( -- CREATE TABLE public.schema_migrations ( - version character varying NOT NULL + version character varying(128) NOT NULL ); -- @@ -1751,20 +1751,20 @@ ALTER SEQUENCE public.watcher_reactions_id_seq OWNED BY public.watcher_reactions -- CREATE TABLE public.watcher_versions ( - id integer CONSTRAINT insight_template_versions_id_not_null NOT NULL, - version integer CONSTRAINT insight_template_versions_version_not_null NOT NULL, - name text CONSTRAINT insight_template_versions_name_not_null NOT NULL, + id integer NOT NULL, + version integer NOT NULL, + name text NOT NULL, description text, change_notes text, - created_by text CONSTRAINT insight_template_versions_created_by_not_null NOT NULL, + created_by text NOT NULL, created_at timestamp with time zone DEFAULT now(), keying_config jsonb, json_template jsonb, - prompt text CONSTRAINT insight_template_versions_prompt_not_null NOT NULL, - extraction_schema jsonb CONSTRAINT insight_template_versions_extraction_schema_not_null NOT NULL, + prompt text NOT NULL, + extraction_schema jsonb NOT NULL, classifiers jsonb, - required_source_types text[] DEFAULT '{}'::text[] CONSTRAINT insight_template_versions_required_source_types_not_null NOT NULL, - recommended_source_types text[] DEFAULT '{}'::text[] CONSTRAINT insight_template_versions_recommended_source_types_not_null NOT NULL, + required_source_types text[] DEFAULT '{}'::text[] NOT NULL, + recommended_source_types text[] DEFAULT '{}'::text[] NOT NULL, reactions_guidance text, condensation_prompt text, condensation_window_count integer DEFAULT 4, @@ -1848,9 +1848,9 @@ ALTER SEQUENCE public.watcher_template_versions_id_seq OWNED BY public.watcher_v -- CREATE TABLE public.watcher_window_events ( - id bigint CONSTRAINT insight_window_content_id_not_null NOT NULL, - window_id bigint CONSTRAINT insight_window_content_window_id_not_null NOT NULL, - event_id bigint CONSTRAINT insight_window_content_content_id_not_null NOT NULL, + id bigint NOT NULL, + window_id bigint NOT NULL, + event_id bigint NOT NULL, created_at timestamp with time zone DEFAULT now() ); @@ -1911,14 +1911,14 @@ ALTER SEQUENCE public.watcher_window_field_feedback_id_seq OWNED BY public.watch -- CREATE TABLE public.watcher_windows ( - id integer CONSTRAINT insight_windows_id_not_null NOT NULL, - watcher_id integer CONSTRAINT insight_windows_insight_id_not_null NOT NULL, + id integer NOT NULL, + watcher_id integer NOT NULL, parent_window_id integer, - granularity text CONSTRAINT insight_windows_granularity_not_null NOT NULL, - window_start timestamp with time zone CONSTRAINT insight_windows_window_start_not_null NOT NULL, - window_end timestamp with time zone CONSTRAINT insight_windows_window_end_not_null NOT NULL, - content_analyzed integer CONSTRAINT insight_windows_content_analyzed_not_null NOT NULL, - extracted_data jsonb CONSTRAINT insight_windows_extracted_data_not_null NOT NULL, + granularity text NOT NULL, + window_start timestamp with time zone NOT NULL, + window_end timestamp with time zone NOT NULL, + content_analyzed integer NOT NULL, + extracted_data jsonb NOT NULL, model_used text, execution_time_ms integer, is_rollup boolean DEFAULT false, @@ -1971,13 +1971,13 @@ ALTER SEQUENCE public.watcher_windows_id_seq OWNED BY public.watcher_windows.id; -- CREATE TABLE public.watchers ( - id integer CONSTRAINT insights_id_not_null NOT NULL, + id integer NOT NULL, model_config jsonb DEFAULT '{}'::jsonb, status text DEFAULT 'active'::text NOT NULL, created_at timestamp with time zone DEFAULT now(), updated_at timestamp with time zone DEFAULT now(), sources jsonb DEFAULT '[]'::jsonb, - created_by text CONSTRAINT insights_created_by_not_null NOT NULL, + created_by text NOT NULL, entity_ids bigint[], reaction_script text, reaction_script_compiled text, @@ -3391,6 +3391,12 @@ CREATE INDEX idx_events_created_at ON public.events USING btree (created_at); CREATE INDEX idx_events_created_by ON public.events USING btree (created_by) WHERE (created_by IS NOT NULL); +-- +-- Name: idx_events_embedding; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_events_embedding ON public.event_embeddings USING ivfflat (embedding public.vector_cosine_ops) WITH (lists='1000'); + -- -- Name: idx_events_entity_ids; Type: INDEX; Schema: public; Owner: - -- @@ -3475,6 +3481,24 @@ CREATE INDEX idx_events_missing_embedding_backfill ON public.events USING btree CREATE INDEX idx_events_organization_id ON public.events USING btree (organization_id) WHERE (organization_id IS NOT NULL); +-- +-- Name: idx_events_raw_content_trgm; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_events_raw_content_trgm ON public.events USING gin (payload_text public.gin_trgm_ops); + +-- +-- Name: idx_events_run_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_events_run_id ON public.events USING btree (run_id); + +-- +-- Name: idx_events_search_tsv; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_events_search_tsv ON public.events USING gin (search_tsv); + -- -- Name: idx_events_semantic_type; Type: INDEX; Schema: public; Owner: - -- diff --git a/packages/web b/packages/web index 00b249caf..d95b9b8b6 160000 --- a/packages/web +++ b/packages/web @@ -1 +1 @@ -Subproject commit 00b249cafd7c239b633193cd9b5edcd2fad02166 +Subproject commit d95b9b8b6f507ebbfc4910e3737f293f49c06360 From 8a6f3b1d74333a4c3c4b22cb9bda813bd6d9a454 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Emre=20Kabakc=C4=B1?= Date: Sat, 16 May 2026 22:04:29 +0100 Subject: [PATCH 3/3] fix(perf): commit the reduced migration content (pi review #3+#4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous fixup commit updated db/schema.sql to match the reduced 4-index scope but missed committing the migration file itself, so HEAD still had the original 8-index DROP statements. CI's dbmate ran the 8-drop migration but schema.sql only removed 4 → drift fail. This commits the matching migration content. --- .../20260517010000_drop_unused_indexes.sql | 58 +++++++++---------- 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/db/migrations/20260517010000_drop_unused_indexes.sql b/db/migrations/20260517010000_drop_unused_indexes.sql index 28a1824e2..2e9aa35c0 100644 --- a/db/migrations/20260517010000_drop_unused_indexes.sql +++ b/db/migrations/20260517010000_drop_unused_indexes.sql @@ -1,44 +1,42 @@ -- migrate:up --- Drop 8 indexes that pg_stat_user_indexes reported `idx_scan = 0` after 28h --- of prod uptime. Together they cost ~5.16 GB of disk + RAM plus write --- amplification on every events / event_embeddings INSERT. +-- Drop 4 indexes that pg_stat_user_indexes reported `idx_scan = 0` after 28h +-- of prod uptime AND are not referenced from any active code path. -- --- pg_stat_statements over the same 28h shows zero calls touching the query --- shapes these indexes serve: no `<->` / `<=>` / `<#>` vector ops (ivfflat --- ANN), no `payload_text ILIKE` or `similarity(payload_text, …)` (trigram --- GIN), no `@@ to_tsquery(…)` (search_tsv GIN). The code paths in --- packages/server/src/utils/content-search.ts exist, they're just not hit --- in prod today. If/when they get exercised, rebuild CONCURRENTLY — the --- query plans degrade to seq scans + filter until the index is back. +-- A larger set (4 more, ~5 GB combined) was originally bundled here but +-- review caught they're not actually unused — they're dormant. The three +-- big search indexes (`idx_events_embedding`, `idx_events_raw_content_trgm`, +-- `idx_events_search_tsv`) are explicitly used by the ANN/fulltext/trigram +-- branches of `approximate_candidate_search` in +-- `packages/server/src/utils/content-search.ts:1707-1733`. The `search()` +-- agent tool path threads through there; prod just hasn't called it in +-- 28h, but a single user-initiated search would now time out at 6s and +-- return empty results without those indexes (`content-search.ts:1850-1863`). +-- Similarly `idx_events_run_id` backs the "view in memory" filter +-- (`content-query-filters.ts:197-201`); rare, but a real path. -- --- The migration uses plain `DROP INDEX` (not CONCURRENTLY) because --- dbmate's `transaction:false` directive doesn't actually exit the --- transaction block when running against the `pq` driver — see the --- comment in 20260426130001_db_integrity_cleanup_concurrent.sql. --- Operator runbook: for prod application, run `DROP INDEX CONCURRENTLY` --- manually first (see docs/MIGRATIONS.md "When dbmate fails in prod" --- → transaction:false recipe), then run dbmate to record this row in --- schema_migrations. On fresh installs / CI / dev the events table is --- empty so the brief ACCESS EXCLUSIVE is irrelevant. +-- Keep those four until either (a) the dormant features are removed in +-- code, or (b) measured prod traffic confirms they're abandoned. +-- +-- What remains is small but still real write amplification: each kept +-- INSERT into events updates these btrees. Combined size ~66 MB — +-- modest reclaim, but zero downside since the underlying queries don't +-- exist anywhere in the codebase today (verified by grep). +-- +-- Plain `DROP INDEX` (not CONCURRENTLY) is used because dbmate's +-- `transaction:false` directive doesn't actually exit the transaction +-- block against the `pq` driver — see the comment in +-- 20260426130001_db_integrity_cleanup_concurrent.sql. These 4 indexes +-- are all small btrees so the ACCESS EXCLUSIVE on `events` during the +-- drop is sub-second; no operator runbook needed. -DROP INDEX IF EXISTS public.idx_events_embedding; -DROP INDEX IF EXISTS public.idx_events_raw_content_trgm; -DROP INDEX IF EXISTS public.idx_events_search_tsv; DROP INDEX IF EXISTS public.idx_events_entity_ids_occurred_at; DROP INDEX IF EXISTS public.idx_events_origin_parent_id; DROP INDEX IF EXISTS public.idx_events_thread_lookup; -DROP INDEX IF EXISTS public.idx_events_run_id; DROP INDEX IF EXISTS public.idx_events_type; -- migrate:down -CREATE INDEX IF NOT EXISTS idx_events_embedding - ON public.event_embeddings USING ivfflat (embedding vector_cosine_ops) WITH (lists='1000'); -CREATE INDEX IF NOT EXISTS idx_events_raw_content_trgm - ON public.events USING gin (payload_text gin_trgm_ops); -CREATE INDEX IF NOT EXISTS idx_events_search_tsv - ON public.events USING gin (search_tsv); CREATE INDEX IF NOT EXISTS idx_events_entity_ids_occurred_at ON public.events USING btree ((entity_ids[1]), occurred_at DESC, id DESC) WHERE ((entity_ids IS NOT NULL) AND (entity_ids <> '{}'::bigint[])); @@ -47,7 +45,5 @@ CREATE INDEX IF NOT EXISTS idx_events_origin_parent_id CREATE INDEX IF NOT EXISTS idx_events_thread_lookup ON public.events USING btree (origin_parent_id, occurred_at) WHERE (origin_parent_id IS NOT NULL); -CREATE INDEX IF NOT EXISTS idx_events_run_id - ON public.events USING btree (run_id); CREATE INDEX IF NOT EXISTS idx_events_type ON public.events USING btree (origin_type) WHERE (origin_type IS NOT NULL);