From 2321a85232e238fbac96404c18af21cb6090ee2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Tue, 26 May 2026 15:42:16 +0200 Subject: [PATCH] chore(db): drop indexes that are unused or redundant with composite indexes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Code audit identified 9 indexes on memory_links, entities, documents, and unit_entities that are either dead (no code path exercises them) or fully covered by composite indexes the planner already prefers. See the migration docstring for the per-index rationale. Also fixes two stale comments that referenced indexes which no longer match the code paths: - link_expansion_retrieval.py claimed entity expansion uses idx_memory_links_entity_covering, but the CTE traverses unit_entities, not memory_links — that's why the covering index has no code path exercising it. - memory_engine.py referenced idx_memory_links_bank_link_type, which was never created on PostgreSQL (only the bank_id column exists). The skills/hindsight-docs/ regen is a drive-by from the pre-commit hook catching up with embeddings-provider docs that landed on main earlier. --- .../e1b2c3d4f5a6_drop_unused_indexes.py | 132 ++++++++++++++++++ .../hindsight_api/engine/memory_engine.py | 6 +- .../engine/search/link_expansion_retrieval.py | 12 +- .../references/developer/configuration.md | 13 +- 4 files changed, 150 insertions(+), 13 deletions(-) create mode 100644 hindsight-api-slim/hindsight_api/alembic/versions/e1b2c3d4f5a6_drop_unused_indexes.py diff --git a/hindsight-api-slim/hindsight_api/alembic/versions/e1b2c3d4f5a6_drop_unused_indexes.py b/hindsight-api-slim/hindsight_api/alembic/versions/e1b2c3d4f5a6_drop_unused_indexes.py new file mode 100644 index 0000000000..8c598bd98b --- /dev/null +++ b/hindsight-api-slim/hindsight_api/alembic/versions/e1b2c3d4f5a6_drop_unused_indexes.py @@ -0,0 +1,132 @@ +"""Drop indexes that are unused or redundant with composite indexes. + +Code audit identified the following indexes as either dead (no code path +exercises them) or fully covered by composite indexes the planner already +prefers: + +memory_links: +1. idx_memory_links_entity_covering — entity co-occurrence expansion was + rewritten to traverse unit_entities instead of memory_links, so no code + path filters memory_links on (link_type = 'entity'). +2. idx_memory_links_from_unit — redundant. idx_memory_links_from_type_weight + (from_unit_id, link_type, weight DESC) leads with the same column and + answers every from_unit_id = X query. +3. idx_memory_links_to_unit — redundant. idx_memory_links_to_type_weight + (to_unit_id, link_type, weight DESC) leads with the same column. +4. idx_memory_links_link_type — no application query filters on link_type + alone; the composite indexes above serve every (from/to + link_type) + predicate. + +entities: +5. idx_entities_canonical_name — superseded by + entities_canonical_name_lower_trgm_idx (case-insensitive lookups). +6. entities_canonical_name_trgm_idx — superseded by the lowercase variant + in migration 2eee35aa3cfc, but the original was never dropped on schemas + that ran the prior migration. + +documents: +7. idx_documents_retain_params — GIN index on retain_params JSONB; no query + uses jsonb containment on this column. +8. idx_documents_content_hash — content-hash lookups happen on the chunks + table (chunks.content_hash, indexed separately). + +unit_entities: +9. idx_unit_entities_entity — defensive drop. Migration h3i4j5k6l7m8 already + issues DROP INDEX IF EXISTS for this; this re-runs the drop idempotently + to cover any schema that missed the previous migration. + +All drops use CONCURRENTLY + IF EXISTS so they neither block writers nor +fail on schemas where the index is already gone. + +Revision ID: e1b2c3d4f5a6 +Revises: p4q5r6s7t8u9 +Create Date: 2026-05-26 +""" + +from collections.abc import Sequence + +from alembic import context, op + +from hindsight_api.alembic._dialect import run_for_dialect + +revision: str = "e1b2c3d4f5a6" +down_revision: str | Sequence[str] | None = "p4q5r6s7t8u9" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +_PG_INDEXES_TO_DROP: tuple[str, ...] = ( + "idx_memory_links_entity_covering", + "idx_memory_links_from_unit", + "idx_memory_links_to_unit", + "idx_memory_links_link_type", + "idx_entities_canonical_name", + "entities_canonical_name_trgm_idx", + "idx_documents_retain_params", + "idx_documents_content_hash", + "idx_unit_entities_entity", +) + + +def _schema_prefix() -> str: + schema = context.config.get_main_option("target_schema") + return f'"{schema}".' if schema else "" + + +def _pg_upgrade() -> None: + schema = _schema_prefix() + # DROP INDEX CONCURRENTLY cannot run inside a transaction block; commit + # the Alembic transaction and issue each statement in its own implicit + # autocommit transaction. IF EXISTS makes each statement idempotent + # across schemas that already dropped (or never had) the index. + for index_name in _PG_INDEXES_TO_DROP: + op.execute("COMMIT") + op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}{index_name}") + + +def _pg_downgrade() -> None: + schema = _schema_prefix() + + # Recreate the dropped indexes in the same shape the prior migrations used, + # so a downgrade leaves the schema in the state the previous head expected. + op.execute("COMMIT") + op.execute( + f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_entity_covering " + f"ON {schema}memory_links(from_unit_id) " + f"INCLUDE (to_unit_id, entity_id) " + f"WHERE link_type = 'entity'" + ) + op.execute("COMMIT") + op.execute( + f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_from_unit ON {schema}memory_links(from_unit_id)" + ) + op.execute("COMMIT") + op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_to_unit ON {schema}memory_links(to_unit_id)") + op.execute("COMMIT") + op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_link_type ON {schema}memory_links(link_type)") + op.execute("COMMIT") + op.execute( + f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_entities_canonical_name ON {schema}entities(canonical_name)" + ) + op.execute("COMMIT") + op.execute( + f"CREATE INDEX CONCURRENTLY IF NOT EXISTS entities_canonical_name_trgm_idx " + f"ON {schema}entities USING GIN (canonical_name gin_trgm_ops)" + ) + op.execute("COMMIT") + op.execute( + f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_documents_retain_params " + f"ON {schema}documents USING GIN (retain_params)" + ) + op.execute("COMMIT") + op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_documents_content_hash ON {schema}documents(content_hash)") + op.execute("COMMIT") + op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_unit_entities_entity ON {schema}unit_entities(entity_id)") + + +def upgrade() -> None: + run_for_dialect(pg=_pg_upgrade) + + +def downgrade() -> None: + run_for_dialect(pg=_pg_downgrade) diff --git a/hindsight-api-slim/hindsight_api/engine/memory_engine.py b/hindsight-api-slim/hindsight_api/engine/memory_engine.py index 88796f3c19..e215dc9c31 100644 --- a/hindsight-api-slim/hindsight_api/engine/memory_engine.py +++ b/hindsight-api-slim/hindsight_api/engine/memory_engine.py @@ -6996,10 +6996,8 @@ async def get_bank_stats( bank_id, ) - # Link stats — filter on ml.bank_id (indexed) instead of joining through mu.bank_id. - # With the idx_memory_links_bank_link_type index this turns a full-table hash join - # into an indexed scan + PK lookups. link_counts and link_counts_by_fact_type are - # derived in Python from the breakdown. + # Link stats — filter on ml.bank_id directly instead of joining through mu.bank_id. + # link_counts and link_counts_by_fact_type are derived in Python from the breakdown. link_breakdown_stats = await conn.fetch( f""" SELECT mu.fact_type, ml.link_type, COUNT(*) as count diff --git a/hindsight-api-slim/hindsight_api/engine/search/link_expansion_retrieval.py b/hindsight-api-slim/hindsight_api/engine/search/link_expansion_retrieval.py index 8fc4a24739..6e903d95e1 100644 --- a/hindsight-api-slim/hindsight_api/engine/search/link_expansion_retrieval.py +++ b/hindsight-api-slim/hindsight_api/engine/search/link_expansion_retrieval.py @@ -283,12 +283,12 @@ async def _expand_combined( score transformations. The three CTEs share one connection slot — important for asyncpg which does not allow concurrent queries on the same connection. - Index coverage (requires migration d2e3f4a5b6c7): - entity: idx_memory_links_entity_covering (from_unit_id) INCLUDE (to_unit_id, entity_id) - WHERE link_type = 'entity' → index-only scan, no heap reads - semantic incoming: - idx_memory_links_to_type_weight (to_unit_id, link_type, weight DESC) - → replaces costly BitmapAnd of two separate scans + Index coverage: + entity: idx_unit_entities_entity_unit (entity_id, unit_id) — entity + expansion traverses unit_entities, not memory_links. + semantic: idx_memory_links_from_type_weight / _to_type_weight + (from_unit_id|to_unit_id, link_type, weight DESC) serve both + outgoing and incoming sides as single composite index scans. """ config = get_config() ml = fq_table("memory_links") diff --git a/skills/hindsight-docs/references/developer/configuration.md b/skills/hindsight-docs/references/developer/configuration.md index f6cef88a82..adf62fe75f 100644 --- a/skills/hindsight-docs/references/developer/configuration.md +++ b/skills/hindsight-docs/references/developer/configuration.md @@ -453,7 +453,7 @@ two slots that retain/consolidation cannot consume. | Variable | Description | Default | |----------|-------------|---------| -| `HINDSIGHT_API_EMBEDDINGS_PROVIDER` | Provider: `local`, `tei`, `openai`, `openrouter`, `cohere`, `google`, `litellm`, or `litellm-sdk` | `local` | +| `HINDSIGHT_API_EMBEDDINGS_PROVIDER` | Provider: `local`, `tei`, `openai`, `openai-codex`, `openrouter`, `cohere`, `google`, `litellm`, or `litellm-sdk` | `local` | | `HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL` | Model for local provider | `BAAI/bge-small-en-v1.5` | | `HINDSIGHT_API_EMBEDDINGS_LOCAL_TRUST_REMOTE_CODE` | Allow loading models with custom code (security risk, disabled by default) | `false` | | `HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU` | Force CPU mode for local embeddings (avoids MPS/XPC issues on macOS) | `false` | @@ -462,6 +462,7 @@ two slots that retain/consolidation cannot consume. | `HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL` | OpenAI embedding model | `text-embedding-3-small` | | `HINDSIGHT_API_EMBEDDINGS_OPENAI_BASE_URL` | Custom base URL for OpenAI-compatible API (e.g., Azure OpenAI) | - | | `HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE` | Max inputs per `embeddings.create` call for `openai`/`openrouter` providers — lower this when the upstream endpoint enforces stricter limits (e.g. DashScope caps at 10) | `100` | +| `HINDSIGHT_API_EMBEDDINGS_OPENAI_DIMENSIONS` | Optional requested output dimensions for OpenAI `text-embedding-3` models (e.g., `384` to match an existing pgvector schema) | - | | `HINDSIGHT_API_EMBEDDINGS_OPENROUTER_API_KEY` | OpenRouter API key for embeddings (falls back to `HINDSIGHT_API_OPENROUTER_API_KEY`, then `HINDSIGHT_API_LLM_API_KEY`) | - | | `HINDSIGHT_API_EMBEDDINGS_OPENROUTER_MODEL` | OpenRouter embedding model | `perplexity/pplx-embed-v1-0.6b` | | `HINDSIGHT_API_EMBEDDINGS_COHERE_API_KEY` | Cohere API key for embeddings | - | @@ -520,8 +521,14 @@ export HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL=BAAI/bge-small-en-v1.5 # OpenAI - cloud-based embeddings export HINDSIGHT_API_EMBEDDINGS_PROVIDER=openai -export HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY=sk-xxxxxxxxxxxx # or reuses HINDSIGHT_API_LLM_API_KEY -export HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL=text-embedding-3-small # 1536 dimensions +export HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY=*** # or reuses HINDSIGHT_API_LLM_API_KEY +export HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL=text-embedding-3-small # 1536 dimensions by default +# export HINDSIGHT_API_EMBEDDINGS_OPENAI_DIMENSIONS=384 # optional reduced output size + +# OpenAI Codex OAuth - uses existing ChatGPT/Codex login, no API key needed +export HINDSIGHT_API_EMBEDDINGS_PROVIDER=openai-codex +export HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL=text-embedding-3-small # 1536 dimensions by default +# export HINDSIGHT_API_EMBEDDINGS_OPENAI_DIMENSIONS=384 # optional reduced output size # Azure OpenAI - embeddings via Azure endpoint export HINDSIGHT_API_EMBEDDINGS_PROVIDER=openai