feat(api): add ParadeDB pg_search as Citus-compatible BM25 backend

docker/docker-compose/pg_search/Dockerfile

@@ -0,0 +1,7 @@
+# PostgreSQL with pgvector and ParadeDB pg_search extensions.
+#
+# The official ParadeDB image ships PostgreSQL with pg_search and pgvector
+# already installed, so no build steps are required. We pin to the PG17
+# variant for parity with the other Hindsight docker-compose examples
+# (vchord, pg_textsearch).
+FROM paradedb/paradedb:latest-pg17

docker/docker-compose/pg_search/docker-compose.yaml

@@ -0,0 +1,93 @@
+name: hindsight
+# Docker Compose file for Hindsight with PostgreSQL and ParadeDB pg_search.
+#
+# pg_search is the only BM25 backend supported by Hindsight that works with
+# Citus, so this is the recommended setup for horizontally scaled deployments.
+#
+# Usage:
+#   docker compose -f docker/docker-compose/pg_search/docker-compose.yaml up -d
+#
+# Required environment variables:
+# - HINDSIGHT_DB_PASSWORD: Password for the PostgreSQL user
+# - Configure LLM provider variables as needed (see the hindsight service)
+#
+# Optional environment variables with defaults:
+# - HINDSIGHT_VERSION: Hindsight application version (default: latest)
+# - HINDSIGHT_DB_USER: PostgreSQL user (default: hindsight_user)
+# - HINDSIGHT_DB_NAME: PostgreSQL database name (default: hindsight_db)
+
+services:
+  db:
+    # Use ParadeDB image which bundles pgvector + pg_search
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: hindsight-db
+    restart: always
+    ports:
+      - "5437:5432"
+    environment:
+      POSTGRES_USER: ${HINDSIGHT_DB_USER:-hindsight_user}
+      POSTGRES_PASSWORD: ${HINDSIGHT_DB_PASSWORD:-hindsight_password}
+      POSTGRES_DB: ${HINDSIGHT_DB_NAME:-hindsight_db}
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    networks:
+      - hindsight-net
+
+  pg-search-init:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    depends_on:
+      - db
+    environment:
+      - PGPASSWORD=${HINDSIGHT_DB_PASSWORD:-hindsight_password}
+    command: >
+      bash -c "
+        echo 'Waiting for PostgreSQL to be ready...';
+        until pg_isready -h hindsight-db -p 5432 -U hindsight_user; do
+          echo 'PostgreSQL is unavailable - sleeping';
+          sleep 2;
+        done;
+        echo 'PostgreSQL is ready - creating hindsight_db database';
+        psql -h hindsight-db -p 5432 -U hindsight_user -c 'CREATE DATABASE hindsight_db;' 2>/dev/null || echo 'Database already exists';
+        echo 'Creating extensions in hindsight_db database';
+        psql -h hindsight-db -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS vector CASCADE;';
+        psql -h hindsight-db -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS pg_search CASCADE;';
+        echo 'Database and extensions created successfully';
+      "
+    restart: "no"
+    networks:
+      - hindsight-net
+
+  hindsight:
+    image: ghcr.io/vectorize-io/hindsight:${HINDSIGHT_VERSION:-latest}
+    container_name: hindsight-app
+    ports:
+      - "8888:8888"
+      - "9999:9999"
+    environment:
+      # LLM Configuration
+      HINDSIGHT_API_LLM_PROVIDER: ${HINDSIGHT_API_LLM_PROVIDER:-openai}
+      HINDSIGHT_API_LLM_API_KEY: ${OPENAI_API_KEY:-your-api-key}
+
+      # Database Configuration
+      HINDSIGHT_API_DATABASE_URL: postgresql://${HINDSIGHT_DB_USER:-hindsight_user}:${HINDSIGHT_DB_PASSWORD:-hindsight_password}@db:5432/${HINDSIGHT_DB_NAME:-hindsight_db}
+
+      # Vector and Text Search Extensions
+      HINDSIGHT_API_VECTOR_EXTENSION: pgvector
+      HINDSIGHT_API_TEXT_SEARCH_EXTENSION: pg_search
+
+    depends_on:
+      - db
+    networks:
+      - hindsight-net
+
+
+networks:
+  hindsight-net:
+    driver: bridge
+
+volumes:
+  pg_data:

hindsight-api-slim/hindsight_api/alembic/versions/5a366d414dce_initial_schema.py

@@ -92,8 +92,8 @@ def _vector_index_using_clause(ext: str) -> str:
 def _detect_text_search_extension() -> str:
     """
     Detect or validate text search extension: 'native', 'vchord', 'pg_textsearch',
-    or 'pgroonga'. Respects HINDSIGHT_API_TEXT_SEARCH_EXTENSION env var. Creates
-    the extension if needed.
+    'pgroonga', or 'pg_search'. Respects HINDSIGHT_API_TEXT_SEARCH_EXTENSION env var.
+    Creates the extension if needed.
 
     pgroonga is treated as native here so the initial schema still creates valid
     tsvector columns. ensure_text_search_extension() at startup converts the
@@ -126,6 +126,18 @@ def _detect_text_search_extension() -> str:
                 # Extension truly doesn't exist - re-raise the error
                 raise
         return "pg_textsearch"
+    elif text_search_extension == "pg_search":
+        # ParadeDB pg_search — true BM25 over base columns, Citus-compatible.
+        try:
+            op.execute("CREATE EXTENSION IF NOT EXISTS pg_search CASCADE")
+        except Exception:
+            # Extension might already exist or user lacks permissions - verify it exists
+            conn = op.get_bind()
+            result = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'pg_search'")).fetchone()
+            if not result:
+                # Extension truly doesn't exist - re-raise the error
+                raise
+        return "pg_search"
     elif text_search_extension == "native":
         return "native"
     elif text_search_extension == "pgroonga":
@@ -135,7 +147,7 @@ def _detect_text_search_extension() -> str:
     else:
         raise ValueError(
             f"Invalid HINDSIGHT_API_TEXT_SEARCH_EXTENSION: {text_search_extension}. "
-            "Must be 'native', 'vchord', 'pg_textsearch', or 'pgroonga'"
+            "Must be 'native', 'vchord', 'pg_textsearch', 'pgroonga', or 'pg_search'"
         )
 
 
@@ -294,8 +306,9 @@ def _pg_upgrade() -> None:
             ALTER TABLE memory_units
             ADD COLUMN search_vector bm25_catalog.bm25vector
         """)
-    elif text_search_ext == "pg_textsearch":
-        # Timescale pg_textsearch: dummy TEXT column for consistency (indexes operate on base columns directly)
+    elif text_search_ext in ("pg_textsearch", "pg_search"):
+        # Timescale pg_textsearch / ParadeDB pg_search: dummy TEXT column for
+        # consistency (indexes operate on base columns directly).
         op.execute("""
             ALTER TABLE memory_units
             ADD COLUMN search_vector TEXT
@@ -360,6 +373,14 @@ def _pg_upgrade() -> None:
             USING bm25(text)
             WITH (text_config='english')
         """)
+    elif text_search_ext == "pg_search":
+        # ParadeDB pg_search BM25 index on (id, text, context). The key_field
+        # reloption is required and must match the table's primary key column.
+        op.execute("""
+            CREATE INDEX idx_memory_units_text_search ON memory_units
+            USING bm25 (id, text, context)
+            WITH (key_field='id')
+        """)
     else:  # native
         # Native PostgreSQL GIN index
         op.execute("""

hindsight-api-slim/hindsight_api/alembic/versions/a2b3c4d5e6f7_add_text_signals_column.py

@@ -7,6 +7,7 @@
 - vchord: text_signals included in tokenize() at insert time
 - native: search_vector GENERATED column regenerated to include text_signals
 - pg_textsearch: no change (index only supports a single base column)
+- pg_search: BM25 index dropped and recreated to include text_signals
 
 Revision ID: a2b3c4d5e6f7
 Revises: z1u2v3w4x5y6
@@ -62,6 +63,15 @@ def _pg_upgrade() -> None:
             CREATE INDEX IF NOT EXISTS idx_memory_units_text_search
             ON {table} USING gin(search_vector)
         """)
+    elif text_search_ext == "pg_search":
+        # ParadeDB pg_search: drop the existing BM25 index and recreate it
+        # to include text_signals alongside text and context.
+        op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_text_search")
+        op.execute(f"""
+            CREATE INDEX idx_memory_units_text_search ON {table}
+            USING bm25 (id, text, context, text_signals)
+            WITH (key_field='id')
+        """)
 
     # vchord: tokenize() call in fact_storage.py is updated to include text_signals at insert time
     # pg_textsearch: no change — index operates on the base `text` column only
@@ -86,6 +96,14 @@ def _pg_downgrade() -> None:
             CREATE INDEX idx_memory_units_text_search
             ON {table} USING gin(search_vector)
         """)
+    elif text_search_ext == "pg_search":
+        # Restore the original (id, text, context) BM25 index without text_signals.
+        op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_text_search")
+        op.execute(f"""
+            CREATE INDEX idx_memory_units_text_search ON {table}
+            USING bm25 (id, text, context)
+            WITH (key_field='id')
+        """)
 
     op.execute(f"ALTER TABLE {table} DROP COLUMN IF EXISTS text_signals")
 

hindsight-api-slim/hindsight_api/alembic/versions/n9i0j1k2l3m4_learnings_and_pinned_reflections.py

@@ -95,9 +95,15 @@ def _vector_index_using_clause(ext: str) -> str:
 
 def _detect_text_search_extension() -> str:
     """
-    Detect or validate text search extension: 'native', 'vchord', or 'pg_textsearch'.
-    Respects HINDSIGHT_API_TEXT_SEARCH_EXTENSION env var.
+    Detect or validate text search extension: 'native', 'vchord', 'pg_textsearch',
+    'pgroonga', or 'pg_search'. Respects HINDSIGHT_API_TEXT_SEARCH_EXTENSION env var.
     Creates the extension if needed.
+
+    pgroonga is treated as native here so this migration still creates valid
+    tsvector columns; ensure_text_search_extension() at startup converts the
+    reflections table (renamed from pinned_reflections in p1k2l3m4n5o6) to
+    pgroonga structures. The learnings table is dropped in p1k2l3m4n5o6 so its
+    transient native-style column never reaches steady state.
     """
     text_search_extension = os.getenv("HINDSIGHT_API_TEXT_SEARCH_EXTENSION", "native").lower()
 
@@ -125,11 +131,26 @@ def _detect_text_search_extension() -> str:
                 # Extension truly doesn't exist - re-raise the error
                 raise
         return "pg_textsearch"
+    elif text_search_extension == "pg_search":
+        # ParadeDB pg_search — true BM25 over base columns, Citus-compatible.
+        try:
+            op.execute("CREATE EXTENSION IF NOT EXISTS pg_search CASCADE")
+        except Exception:
+            conn = op.get_bind()
+            result = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'pg_search'")).fetchone()
+            if not result:
+                raise
+        return "pg_search"
     elif text_search_extension == "native":
         return "native"
+    elif text_search_extension == "pgroonga":
+        # Treat as native here; ensure_text_search_extension() converts the
+        # reflections table to pgroonga structures at runtime.
+        return "native"
     else:
         raise ValueError(
-            f"Invalid HINDSIGHT_API_TEXT_SEARCH_EXTENSION: {text_search_extension}. Must be 'native', 'vchord', or 'pg_textsearch'"
+            f"Invalid HINDSIGHT_API_TEXT_SEARCH_EXTENSION: {text_search_extension}. "
+            "Must be 'native', 'vchord', 'pg_textsearch', 'pgroonga', or 'pg_search'"
         )
 
 
@@ -200,6 +221,17 @@ def _pg_upgrade() -> None:
             CREATE INDEX idx_learnings_text_search ON {schema}learnings
             USING bm25(text) WITH (text_config='english')
         """)
+    elif text_search_ext == "pg_search":
+        # ParadeDB pg_search: dummy TEXT column; BM25 index is built directly over (id, text)
+        # with key_field='id' (matches the table's primary key).
+        op.execute(f"""
+            ALTER TABLE {schema}learnings ADD COLUMN search_vector TEXT
+        """)
+        op.execute(f"""
+            CREATE INDEX idx_learnings_text_search ON {schema}learnings
+            USING bm25 (id, text)
+            WITH (key_field='id')
+        """)
     else:  # native
         # Native PostgreSQL: tsvector with automatic generation
         op.execute(f"""
@@ -264,6 +296,17 @@ def _pg_upgrade() -> None:
             USING bm25(content)
             WITH (text_config='english')
         """)
+    elif text_search_ext == "pg_search":
+        # ParadeDB pg_search: dummy TEXT column; BM25 index over (id, name, content)
+        # with key_field='id'.
+        op.execute(f"""
+            ALTER TABLE {schema}pinned_reflections ADD COLUMN search_vector TEXT
+        """)
+        op.execute(f"""
+            CREATE INDEX idx_pinned_reflections_text_search ON {schema}pinned_reflections
+            USING bm25 (id, name, content)
+            WITH (key_field='id')
+        """)
     else:  # native
         # Native PostgreSQL: tsvector with automatic generation
         op.execute(f"""

hindsight-api-slim/hindsight_api/config.py

@@ -566,13 +566,14 @@ def normalize_config_dict(config: dict[str, Any]) -> dict[str, Any]:
 # Vector extension (pgvector, vchord, pgvectorscale, or AlloyDB ScaNN)
 DEFAULT_VECTOR_EXTENSION = "pgvector"  # Options: "pgvector", "vchord", "pgvectorscale", "scann"
 
-# Text search extension (native PostgreSQL, vchord BM25, Timescale pg_textsearch, or pgroonga)
-DEFAULT_TEXT_SEARCH_EXTENSION = "native"  # Options: "native", "vchord", "pg_textsearch", "pgroonga"
+# Text search extension (native PostgreSQL, vchord BM25, Timescale pg_textsearch,
+# pgroonga, or ParadeDB pg_search)
+DEFAULT_TEXT_SEARCH_EXTENSION = "native"  # Options: "native", "vchord", "pg_textsearch", "pgroonga", "pg_search"
 
 # PostgreSQL text search dictionary used by the native tsvector backend. Only
 # affects text_search_extension == "native"; other backends use their own
 # tokenizers (vchord: llmlingua2, pg_textsearch: hardcoded english,
-# pgroonga: TokenBigram polyglot).
+# pgroonga: TokenBigram polyglot, pg_search: per-field Tantivy tokenizer).
 DEFAULT_TEXT_SEARCH_EXTENSION_NATIVE_LANGUAGE = "english"
 
 # LiteLLM defaults
@@ -907,10 +908,11 @@ class HindsightConfig:
     migration_database_url: str | None
     database_schema: str
     vector_extension: str  # "pgvector", "vchord", "pgvectorscale", or "scann"
-    text_search_extension: str  # "native", "vchord", "pg_textsearch", or "pgroonga"
+    text_search_extension: str  # "native", "vchord", "pg_textsearch", "pgroonga", or "pg_search"
     # PostgreSQL text search dictionary for the "native" backend (ignored by
     # other backends). Only the "native" backend reads this field; pgroonga
-    # uses TokenBigram, vchord uses llmlingua2, pg_textsearch hardcodes english.
+    # uses TokenBigram, vchord uses llmlingua2, pg_textsearch hardcodes english,
+    # pg_search uses Tantivy per-field tokenizers.
     text_search_extension_native_language: str
     # When set, every LLM-generated artifact (retain facts, consolidation
     # observations, reflect responses) is forced into this language regardless
@@ -1381,7 +1383,7 @@ def validate(self) -> None:
         validate_extension(self.vector_extension)
 
         # Validate text_search_extension
-        valid_text_search = ("native", "vchord", "pg_textsearch", "pgroonga")
+        valid_text_search = ("native", "vchord", "pg_textsearch", "pgroonga", "pg_search")
         if self.text_search_extension not in valid_text_search:
             raise ValueError(
                 f"Invalid text_search_extension: {self.text_search_extension}. Must be one of: {', '.join(valid_text_search)}"

hindsight-api-slim/hindsight_api/engine/consolidation/consolidator.py

@@ -1415,9 +1415,16 @@ async def _create_observation_directly(
                     tokenize($3, 'llmlingua2')::bm25_catalog.bm25vector)
             RETURNING id
         """
-    else:  # native or pg_textsearch
-        # Native PostgreSQL: search_vector is GENERATED ALWAYS, don't include it
-        # pg_textsearch: indexes operate on base columns directly, don't populate search_vector
+    else:  # native, pg_textsearch, pgroonga, or pg_search
+        # pg_textsearch / pgroonga / pg_search: indexes operate on base text
+        # columns directly, so the dummy search_vector column is left NULL.
+        # Native: the migration p4q5r6s7t8u9 dropped the GENERATED expression on
+        # search_vector to allow per-deployment language configuration; the
+        # batch insert path in ops_postgresql.insert_facts_batch now populates
+        # it via to_tsvector($lang, ...). This single-observation INSERT does
+        # not, so observations under the native backend currently land with
+        # NULL search_vector and are not BM25-searchable until reflected/
+        # re-ingested. Tracking a separate fix for that gap.
         query = f"""
             INSERT INTO {fq_table("memory_units")} (
                 id, bank_id, text, fact_type, embedding, proof_count, source_memory_ids, history,

hindsight-api-slim/hindsight_api/engine/db/ops_postgresql.py

@@ -141,9 +141,9 @@ async def insert_facts_batch(
                 RETURNING id
             """
         else:
-            # pg_textsearch and pgroonga: search_vector is a dummy TEXT column;
-            # the actual full-text index operates on the base text columns
-            # directly, so we don't populate search_vector at insert time.
+            # pg_textsearch, pgroonga, and pg_search: search_vector is a dummy
+            # TEXT column; the actual full-text index operates on the base text
+            # columns directly, so we don't populate search_vector at insert time.
             query = f"""
                 WITH input_data AS (
                     SELECT * FROM unnest(

hindsight-api-slim/hindsight_api/engine/sql/postgresql.py

@@ -203,6 +203,20 @@ def build_bm25_arm(
                 f"AND (COALESCE(text, '') || ' ' || COALESCE(context, '') || ' ' || COALESCE(text_signals, '')) "
                 f"&@~ {text_param}"
             )
+        elif text_search_extension == "pg_search":
+            # ParadeDB pg_search: BM25 index over (id, text, context, text_signals)
+            # with key_field='id'. The @@@ operator on the key_field requires a
+            # field-qualified query (`text:foo`); to keep the bind-parameter form,
+            # we fan the query out across all indexed text fields with paradedb.boolean.
+            bm25_score_expr = "paradedb.score(id)"
+            bm25_order_by = "paradedb.score(id) DESC"
+            bm25_where_filter = (
+                f"AND id @@@ paradedb.boolean(should => ARRAY["
+                f"paradedb.match('text', {text_param}), "
+                f"paradedb.match('context', {text_param}), "
+                f"paradedb.match('text_signals', {text_param})"
+                f"])"
+            )
         else:  # native tsvector
             # bm25_language is validated as a PG identifier in HindsightConfig.validate(),
             # so embedding it as a SQL literal here is safe.
@@ -233,7 +247,7 @@ def prepare_bm25_text(
         *,
         text_search_extension: str = "native",
     ) -> str:
-        if text_search_extension in ("vchord", "pg_textsearch", "pgroonga"):
+        if text_search_extension in ("vchord", "pg_textsearch", "pgroonga", "pg_search"):
             return query_text
         # native tsvector: join tokens with OR operator
         return " | ".join(tokens)