chore(db): drop 9 unused/redundant indexes

hindsight-api-slim/hindsight_api/alembic/versions/e1b2c3d4f5a6_drop_unused_indexes.py

@@ -0,0 +1,132 @@
+"""Drop indexes that are unused or redundant with composite indexes.
+
+Code audit identified the following indexes as either dead (no code path
+exercises them) or fully covered by composite indexes the planner already
+prefers:
+
+memory_links:
+1. idx_memory_links_entity_covering — entity co-occurrence expansion was
+   rewritten to traverse unit_entities instead of memory_links, so no code
+   path filters memory_links on (link_type = 'entity').
+2. idx_memory_links_from_unit — redundant.  idx_memory_links_from_type_weight
+   (from_unit_id, link_type, weight DESC) leads with the same column and
+   answers every from_unit_id = X query.
+3. idx_memory_links_to_unit — redundant.  idx_memory_links_to_type_weight
+   (to_unit_id, link_type, weight DESC) leads with the same column.
+4. idx_memory_links_link_type — no application query filters on link_type
+   alone; the composite indexes above serve every (from/to + link_type)
+   predicate.
+
+entities:
+5. idx_entities_canonical_name — superseded by
+   entities_canonical_name_lower_trgm_idx (case-insensitive lookups).
+6. entities_canonical_name_trgm_idx — superseded by the lowercase variant
+   in migration 2eee35aa3cfc, but the original was never dropped on schemas
+   that ran the prior migration.
+
+documents:
+7. idx_documents_retain_params — GIN index on retain_params JSONB; no query
+   uses jsonb containment on this column.
+8. idx_documents_content_hash — content-hash lookups happen on the chunks
+   table (chunks.content_hash, indexed separately).
+
+unit_entities:
+9. idx_unit_entities_entity — defensive drop.  Migration h3i4j5k6l7m8 already
+   issues DROP INDEX IF EXISTS for this; this re-runs the drop idempotently
+   to cover any schema that missed the previous migration.
+
+All drops use CONCURRENTLY + IF EXISTS so they neither block writers nor
+fail on schemas where the index is already gone.
+
+Revision ID: e1b2c3d4f5a6
+Revises: p4q5r6s7t8u9
+Create Date: 2026-05-26
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+from hindsight_api.alembic._dialect import run_for_dialect
+
+revision: str = "e1b2c3d4f5a6"
+down_revision: str | Sequence[str] | None = "p4q5r6s7t8u9"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+_PG_INDEXES_TO_DROP: tuple[str, ...] = (
+    "idx_memory_links_entity_covering",
+    "idx_memory_links_from_unit",
+    "idx_memory_links_to_unit",
+    "idx_memory_links_link_type",
+    "idx_entities_canonical_name",
+    "entities_canonical_name_trgm_idx",
+    "idx_documents_retain_params",
+    "idx_documents_content_hash",
+    "idx_unit_entities_entity",
+)
+
+
+def _schema_prefix() -> str:
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def _pg_upgrade() -> None:
+    schema = _schema_prefix()
+    # DROP INDEX CONCURRENTLY cannot run inside a transaction block; commit
+    # the Alembic transaction and issue each statement in its own implicit
+    # autocommit transaction.  IF EXISTS makes each statement idempotent
+    # across schemas that already dropped (or never had) the index.
+    for index_name in _PG_INDEXES_TO_DROP:
+        op.execute("COMMIT")
+        op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}{index_name}")
+
+
+def _pg_downgrade() -> None:
+    schema = _schema_prefix()
+
+    # Recreate the dropped indexes in the same shape the prior migrations used,
+    # so a downgrade leaves the schema in the state the previous head expected.
+    op.execute("COMMIT")
+    op.execute(
+        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_entity_covering "
+        f"ON {schema}memory_links(from_unit_id) "
+        f"INCLUDE (to_unit_id, entity_id) "
+        f"WHERE link_type = 'entity'"
+    )
+    op.execute("COMMIT")
+    op.execute(
+        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_from_unit ON {schema}memory_links(from_unit_id)"
+    )
+    op.execute("COMMIT")
+    op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_to_unit ON {schema}memory_links(to_unit_id)")
+    op.execute("COMMIT")
+    op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_link_type ON {schema}memory_links(link_type)")
+    op.execute("COMMIT")
+    op.execute(
+        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_entities_canonical_name ON {schema}entities(canonical_name)"
+    )
+    op.execute("COMMIT")
+    op.execute(
+        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS entities_canonical_name_trgm_idx "
+        f"ON {schema}entities USING GIN (canonical_name gin_trgm_ops)"
+    )
+    op.execute("COMMIT")
+    op.execute(
+        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_documents_retain_params "
+        f"ON {schema}documents USING GIN (retain_params)"
+    )
+    op.execute("COMMIT")
+    op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_documents_content_hash ON {schema}documents(content_hash)")
+    op.execute("COMMIT")
+    op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_unit_entities_entity ON {schema}unit_entities(entity_id)")
+
+
+def upgrade() -> None:
+    run_for_dialect(pg=_pg_upgrade)
+
+
+def downgrade() -> None:
+    run_for_dialect(pg=_pg_downgrade)

hindsight-api-slim/hindsight_api/engine/memory_engine.py

@@ -6996,10 +6996,8 @@ async def get_bank_stats(
                 bank_id,
             )
 
-            # Link stats — filter on ml.bank_id (indexed) instead of joining through mu.bank_id.
-            # With the idx_memory_links_bank_link_type index this turns a full-table hash join
-            # into an indexed scan + PK lookups.  link_counts and link_counts_by_fact_type are
-            # derived in Python from the breakdown.
+            # Link stats — filter on ml.bank_id directly instead of joining through mu.bank_id.
+            # link_counts and link_counts_by_fact_type are derived in Python from the breakdown.
             link_breakdown_stats = await conn.fetch(
                 f"""
                 SELECT mu.fact_type, ml.link_type, COUNT(*) as count

hindsight-api-slim/hindsight_api/engine/search/link_expansion_retrieval.py

@@ -283,12 +283,12 @@ async def _expand_combined(
         score transformations.  The three CTEs share one connection slot — important
         for asyncpg which does not allow concurrent queries on the same connection.
 
-        Index coverage (requires migration d2e3f4a5b6c7):
-          entity:   idx_memory_links_entity_covering (from_unit_id) INCLUDE (to_unit_id, entity_id)
-                    WHERE link_type = 'entity'  → index-only scan, no heap reads
-          semantic incoming:
-                    idx_memory_links_to_type_weight (to_unit_id, link_type, weight DESC)
-                    → replaces costly BitmapAnd of two separate scans
+        Index coverage:
+          entity:   idx_unit_entities_entity_unit (entity_id, unit_id) — entity
+                    expansion traverses unit_entities, not memory_links.
+          semantic: idx_memory_links_from_type_weight / _to_type_weight
+                    (from_unit_id|to_unit_id, link_type, weight DESC) serve both
+                    outgoing and incoming sides as single composite index scans.
         """
         config = get_config()
         ml = fq_table("memory_links")

skills/hindsight-docs/references/developer/configuration.md

@@ -453,7 +453,7 @@ two slots that retain/consolidation cannot consume.
 
 | Variable | Description | Default |
 |----------|-------------|---------|
-| `HINDSIGHT_API_EMBEDDINGS_PROVIDER` | Provider: `local`, `tei`, `openai`, `openrouter`, `cohere`, `google`, `litellm`, or `litellm-sdk` | `local` |
+| `HINDSIGHT_API_EMBEDDINGS_PROVIDER` | Provider: `local`, `tei`, `openai`, `openai-codex`, `openrouter`, `cohere`, `google`, `litellm`, or `litellm-sdk` | `local` |
 | `HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL` | Model for local provider | `BAAI/bge-small-en-v1.5` |
 | `HINDSIGHT_API_EMBEDDINGS_LOCAL_TRUST_REMOTE_CODE` | Allow loading models with custom code (security risk, disabled by default) | `false` |
 | `HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU` | Force CPU mode for local embeddings (avoids MPS/XPC issues on macOS) | `false` |
@@ -462,6 +462,7 @@ two slots that retain/consolidation cannot consume.
 | `HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL` | OpenAI embedding model | `text-embedding-3-small` |
 | `HINDSIGHT_API_EMBEDDINGS_OPENAI_BASE_URL` | Custom base URL for OpenAI-compatible API (e.g., Azure OpenAI) | - |
 | `HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE` | Max inputs per `embeddings.create` call for `openai`/`openrouter` providers — lower this when the upstream endpoint enforces stricter limits (e.g. DashScope caps at 10) | `100` |
+| `HINDSIGHT_API_EMBEDDINGS_OPENAI_DIMENSIONS` | Optional requested output dimensions for OpenAI `text-embedding-3` models (e.g., `384` to match an existing pgvector schema) | - |
 | `HINDSIGHT_API_EMBEDDINGS_OPENROUTER_API_KEY` | OpenRouter API key for embeddings (falls back to `HINDSIGHT_API_OPENROUTER_API_KEY`, then `HINDSIGHT_API_LLM_API_KEY`) | - |
 | `HINDSIGHT_API_EMBEDDINGS_OPENROUTER_MODEL` | OpenRouter embedding model | `perplexity/pplx-embed-v1-0.6b` |
 | `HINDSIGHT_API_EMBEDDINGS_COHERE_API_KEY` | Cohere API key for embeddings | - |
@@ -520,8 +521,14 @@ export HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL=BAAI/bge-small-en-v1.5
 
 # OpenAI - cloud-based embeddings
 export HINDSIGHT_API_EMBEDDINGS_PROVIDER=openai
-export HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY=sk-xxxxxxxxxxxx  # or reuses HINDSIGHT_API_LLM_API_KEY
-export HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL=text-embedding-3-small  # 1536 dimensions
+export HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY=***  # or reuses HINDSIGHT_API_LLM_API_KEY
+export HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL=text-embedding-3-small  # 1536 dimensions by default
+# export HINDSIGHT_API_EMBEDDINGS_OPENAI_DIMENSIONS=384  # optional reduced output size
+
+# OpenAI Codex OAuth - uses existing ChatGPT/Codex login, no API key needed
+export HINDSIGHT_API_EMBEDDINGS_PROVIDER=openai-codex
+export HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL=text-embedding-3-small  # 1536 dimensions by default
+# export HINDSIGHT_API_EMBEDDINGS_OPENAI_DIMENSIONS=384  # optional reduced output size
 
 # Azure OpenAI - embeddings via Azure endpoint
 export HINDSIGHT_API_EMBEDDINGS_PROVIDER=openai