diff --git a/docker/docker-compose/pg_search/Dockerfile b/docker/docker-compose/pg_search/Dockerfile new file mode 100644 index 0000000000..5d4b6a9979 --- /dev/null +++ b/docker/docker-compose/pg_search/Dockerfile @@ -0,0 +1,7 @@ +# PostgreSQL with pgvector and ParadeDB pg_search extensions. +# +# The official ParadeDB image ships PostgreSQL with pg_search and pgvector +# already installed, so no build steps are required. We pin to the PG17 +# variant for parity with the other Hindsight docker-compose examples +# (vchord, pg_textsearch). +FROM paradedb/paradedb:latest-pg17 diff --git a/docker/docker-compose/pg_search/docker-compose.yaml b/docker/docker-compose/pg_search/docker-compose.yaml new file mode 100644 index 0000000000..764c38fbb7 --- /dev/null +++ b/docker/docker-compose/pg_search/docker-compose.yaml @@ -0,0 +1,93 @@ +name: hindsight +# Docker Compose file for Hindsight with PostgreSQL and ParadeDB pg_search. +# +# pg_search is the only BM25 backend supported by Hindsight that works with +# Citus, so this is the recommended setup for horizontally scaled deployments. +# +# Usage: +# docker compose -f docker/docker-compose/pg_search/docker-compose.yaml up -d +# +# Required environment variables: +# - HINDSIGHT_DB_PASSWORD: Password for the PostgreSQL user +# - Configure LLM provider variables as needed (see the hindsight service) +# +# Optional environment variables with defaults: +# - HINDSIGHT_VERSION: Hindsight application version (default: latest) +# - HINDSIGHT_DB_USER: PostgreSQL user (default: hindsight_user) +# - HINDSIGHT_DB_NAME: PostgreSQL database name (default: hindsight_db) + +services: + db: + # Use ParadeDB image which bundles pgvector + pg_search + build: + context: . + dockerfile: Dockerfile + container_name: hindsight-db + restart: always + ports: + - "5437:5432" + environment: + POSTGRES_USER: ${HINDSIGHT_DB_USER:-hindsight_user} + POSTGRES_PASSWORD: ${HINDSIGHT_DB_PASSWORD:-hindsight_password} + POSTGRES_DB: ${HINDSIGHT_DB_NAME:-hindsight_db} + volumes: + - pg_data:/var/lib/postgresql/data + networks: + - hindsight-net + + pg-search-init: + build: + context: . + dockerfile: Dockerfile + depends_on: + - db + environment: + - PGPASSWORD=${HINDSIGHT_DB_PASSWORD:-hindsight_password} + command: > + bash -c " + echo 'Waiting for PostgreSQL to be ready...'; + until pg_isready -h hindsight-db -p 5432 -U hindsight_user; do + echo 'PostgreSQL is unavailable - sleeping'; + sleep 2; + done; + echo 'PostgreSQL is ready - creating hindsight_db database'; + psql -h hindsight-db -p 5432 -U hindsight_user -c 'CREATE DATABASE hindsight_db;' 2>/dev/null || echo 'Database already exists'; + echo 'Creating extensions in hindsight_db database'; + psql -h hindsight-db -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS vector CASCADE;'; + psql -h hindsight-db -p 5432 -U hindsight_user -d hindsight_db -c 'CREATE EXTENSION IF NOT EXISTS pg_search CASCADE;'; + echo 'Database and extensions created successfully'; + " + restart: "no" + networks: + - hindsight-net + + hindsight: + image: ghcr.io/vectorize-io/hindsight:${HINDSIGHT_VERSION:-latest} + container_name: hindsight-app + ports: + - "8888:8888" + - "9999:9999" + environment: + # LLM Configuration + HINDSIGHT_API_LLM_PROVIDER: ${HINDSIGHT_API_LLM_PROVIDER:-openai} + HINDSIGHT_API_LLM_API_KEY: ${OPENAI_API_KEY:-your-api-key} + + # Database Configuration + HINDSIGHT_API_DATABASE_URL: postgresql://${HINDSIGHT_DB_USER:-hindsight_user}:${HINDSIGHT_DB_PASSWORD:-hindsight_password}@db:5432/${HINDSIGHT_DB_NAME:-hindsight_db} + + # Vector and Text Search Extensions + HINDSIGHT_API_VECTOR_EXTENSION: pgvector + HINDSIGHT_API_TEXT_SEARCH_EXTENSION: pg_search + + depends_on: + - db + networks: + - hindsight-net + + +networks: + hindsight-net: + driver: bridge + +volumes: + pg_data: diff --git a/hindsight-api-slim/hindsight_api/alembic/versions/5a366d414dce_initial_schema.py b/hindsight-api-slim/hindsight_api/alembic/versions/5a366d414dce_initial_schema.py index 55d05d7287..643ab41de5 100644 --- a/hindsight-api-slim/hindsight_api/alembic/versions/5a366d414dce_initial_schema.py +++ b/hindsight-api-slim/hindsight_api/alembic/versions/5a366d414dce_initial_schema.py @@ -92,8 +92,8 @@ def _vector_index_using_clause(ext: str) -> str: def _detect_text_search_extension() -> str: """ Detect or validate text search extension: 'native', 'vchord', 'pg_textsearch', - or 'pgroonga'. Respects HINDSIGHT_API_TEXT_SEARCH_EXTENSION env var. Creates - the extension if needed. + 'pgroonga', or 'pg_search'. Respects HINDSIGHT_API_TEXT_SEARCH_EXTENSION env var. + Creates the extension if needed. pgroonga is treated as native here so the initial schema still creates valid tsvector columns. ensure_text_search_extension() at startup converts the @@ -126,6 +126,18 @@ def _detect_text_search_extension() -> str: # Extension truly doesn't exist - re-raise the error raise return "pg_textsearch" + elif text_search_extension == "pg_search": + # ParadeDB pg_search — true BM25 over base columns, Citus-compatible. + try: + op.execute("CREATE EXTENSION IF NOT EXISTS pg_search CASCADE") + except Exception: + # Extension might already exist or user lacks permissions - verify it exists + conn = op.get_bind() + result = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'pg_search'")).fetchone() + if not result: + # Extension truly doesn't exist - re-raise the error + raise + return "pg_search" elif text_search_extension == "native": return "native" elif text_search_extension == "pgroonga": @@ -135,7 +147,7 @@ def _detect_text_search_extension() -> str: else: raise ValueError( f"Invalid HINDSIGHT_API_TEXT_SEARCH_EXTENSION: {text_search_extension}. " - "Must be 'native', 'vchord', 'pg_textsearch', or 'pgroonga'" + "Must be 'native', 'vchord', 'pg_textsearch', 'pgroonga', or 'pg_search'" ) @@ -294,8 +306,9 @@ def _pg_upgrade() -> None: ALTER TABLE memory_units ADD COLUMN search_vector bm25_catalog.bm25vector """) - elif text_search_ext == "pg_textsearch": - # Timescale pg_textsearch: dummy TEXT column for consistency (indexes operate on base columns directly) + elif text_search_ext in ("pg_textsearch", "pg_search"): + # Timescale pg_textsearch / ParadeDB pg_search: dummy TEXT column for + # consistency (indexes operate on base columns directly). op.execute(""" ALTER TABLE memory_units ADD COLUMN search_vector TEXT @@ -360,6 +373,14 @@ def _pg_upgrade() -> None: USING bm25(text) WITH (text_config='english') """) + elif text_search_ext == "pg_search": + # ParadeDB pg_search BM25 index on (id, text, context). The key_field + # reloption is required and must match the table's primary key column. + op.execute(""" + CREATE INDEX idx_memory_units_text_search ON memory_units + USING bm25 (id, text, context) + WITH (key_field='id') + """) else: # native # Native PostgreSQL GIN index op.execute(""" diff --git a/hindsight-api-slim/hindsight_api/alembic/versions/a2b3c4d5e6f7_add_text_signals_column.py b/hindsight-api-slim/hindsight_api/alembic/versions/a2b3c4d5e6f7_add_text_signals_column.py index 5f21ac42b8..bca2b24837 100644 --- a/hindsight-api-slim/hindsight_api/alembic/versions/a2b3c4d5e6f7_add_text_signals_column.py +++ b/hindsight-api-slim/hindsight_api/alembic/versions/a2b3c4d5e6f7_add_text_signals_column.py @@ -7,6 +7,7 @@ - vchord: text_signals included in tokenize() at insert time - native: search_vector GENERATED column regenerated to include text_signals - pg_textsearch: no change (index only supports a single base column) +- pg_search: BM25 index dropped and recreated to include text_signals Revision ID: a2b3c4d5e6f7 Revises: z1u2v3w4x5y6 @@ -62,6 +63,15 @@ def _pg_upgrade() -> None: CREATE INDEX IF NOT EXISTS idx_memory_units_text_search ON {table} USING gin(search_vector) """) + elif text_search_ext == "pg_search": + # ParadeDB pg_search: drop the existing BM25 index and recreate it + # to include text_signals alongside text and context. + op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_text_search") + op.execute(f""" + CREATE INDEX idx_memory_units_text_search ON {table} + USING bm25 (id, text, context, text_signals) + WITH (key_field='id') + """) # vchord: tokenize() call in fact_storage.py is updated to include text_signals at insert time # pg_textsearch: no change — index operates on the base `text` column only @@ -86,6 +96,14 @@ def _pg_downgrade() -> None: CREATE INDEX idx_memory_units_text_search ON {table} USING gin(search_vector) """) + elif text_search_ext == "pg_search": + # Restore the original (id, text, context) BM25 index without text_signals. + op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_text_search") + op.execute(f""" + CREATE INDEX idx_memory_units_text_search ON {table} + USING bm25 (id, text, context) + WITH (key_field='id') + """) op.execute(f"ALTER TABLE {table} DROP COLUMN IF EXISTS text_signals") diff --git a/hindsight-api-slim/hindsight_api/alembic/versions/n9i0j1k2l3m4_learnings_and_pinned_reflections.py b/hindsight-api-slim/hindsight_api/alembic/versions/n9i0j1k2l3m4_learnings_and_pinned_reflections.py index 52a750eb92..f07a75658f 100644 --- a/hindsight-api-slim/hindsight_api/alembic/versions/n9i0j1k2l3m4_learnings_and_pinned_reflections.py +++ b/hindsight-api-slim/hindsight_api/alembic/versions/n9i0j1k2l3m4_learnings_and_pinned_reflections.py @@ -95,9 +95,15 @@ def _vector_index_using_clause(ext: str) -> str: def _detect_text_search_extension() -> str: """ - Detect or validate text search extension: 'native', 'vchord', or 'pg_textsearch'. - Respects HINDSIGHT_API_TEXT_SEARCH_EXTENSION env var. + Detect or validate text search extension: 'native', 'vchord', 'pg_textsearch', + 'pgroonga', or 'pg_search'. Respects HINDSIGHT_API_TEXT_SEARCH_EXTENSION env var. Creates the extension if needed. + + pgroonga is treated as native here so this migration still creates valid + tsvector columns; ensure_text_search_extension() at startup converts the + reflections table (renamed from pinned_reflections in p1k2l3m4n5o6) to + pgroonga structures. The learnings table is dropped in p1k2l3m4n5o6 so its + transient native-style column never reaches steady state. """ text_search_extension = os.getenv("HINDSIGHT_API_TEXT_SEARCH_EXTENSION", "native").lower() @@ -125,11 +131,26 @@ def _detect_text_search_extension() -> str: # Extension truly doesn't exist - re-raise the error raise return "pg_textsearch" + elif text_search_extension == "pg_search": + # ParadeDB pg_search — true BM25 over base columns, Citus-compatible. + try: + op.execute("CREATE EXTENSION IF NOT EXISTS pg_search CASCADE") + except Exception: + conn = op.get_bind() + result = conn.execute(text("SELECT 1 FROM pg_extension WHERE extname = 'pg_search'")).fetchone() + if not result: + raise + return "pg_search" elif text_search_extension == "native": return "native" + elif text_search_extension == "pgroonga": + # Treat as native here; ensure_text_search_extension() converts the + # reflections table to pgroonga structures at runtime. + return "native" else: raise ValueError( - f"Invalid HINDSIGHT_API_TEXT_SEARCH_EXTENSION: {text_search_extension}. Must be 'native', 'vchord', or 'pg_textsearch'" + f"Invalid HINDSIGHT_API_TEXT_SEARCH_EXTENSION: {text_search_extension}. " + "Must be 'native', 'vchord', 'pg_textsearch', 'pgroonga', or 'pg_search'" ) @@ -200,6 +221,17 @@ def _pg_upgrade() -> None: CREATE INDEX idx_learnings_text_search ON {schema}learnings USING bm25(text) WITH (text_config='english') """) + elif text_search_ext == "pg_search": + # ParadeDB pg_search: dummy TEXT column; BM25 index is built directly over (id, text) + # with key_field='id' (matches the table's primary key). + op.execute(f""" + ALTER TABLE {schema}learnings ADD COLUMN search_vector TEXT + """) + op.execute(f""" + CREATE INDEX idx_learnings_text_search ON {schema}learnings + USING bm25 (id, text) + WITH (key_field='id') + """) else: # native # Native PostgreSQL: tsvector with automatic generation op.execute(f""" @@ -264,6 +296,17 @@ def _pg_upgrade() -> None: USING bm25(content) WITH (text_config='english') """) + elif text_search_ext == "pg_search": + # ParadeDB pg_search: dummy TEXT column; BM25 index over (id, name, content) + # with key_field='id'. + op.execute(f""" + ALTER TABLE {schema}pinned_reflections ADD COLUMN search_vector TEXT + """) + op.execute(f""" + CREATE INDEX idx_pinned_reflections_text_search ON {schema}pinned_reflections + USING bm25 (id, name, content) + WITH (key_field='id') + """) else: # native # Native PostgreSQL: tsvector with automatic generation op.execute(f""" diff --git a/hindsight-api-slim/hindsight_api/config.py b/hindsight-api-slim/hindsight_api/config.py index ffd9eec9e0..73aecceaeb 100644 --- a/hindsight-api-slim/hindsight_api/config.py +++ b/hindsight-api-slim/hindsight_api/config.py @@ -566,13 +566,14 @@ def normalize_config_dict(config: dict[str, Any]) -> dict[str, Any]: # Vector extension (pgvector, vchord, pgvectorscale, or AlloyDB ScaNN) DEFAULT_VECTOR_EXTENSION = "pgvector" # Options: "pgvector", "vchord", "pgvectorscale", "scann" -# Text search extension (native PostgreSQL, vchord BM25, Timescale pg_textsearch, or pgroonga) -DEFAULT_TEXT_SEARCH_EXTENSION = "native" # Options: "native", "vchord", "pg_textsearch", "pgroonga" +# Text search extension (native PostgreSQL, vchord BM25, Timescale pg_textsearch, +# pgroonga, or ParadeDB pg_search) +DEFAULT_TEXT_SEARCH_EXTENSION = "native" # Options: "native", "vchord", "pg_textsearch", "pgroonga", "pg_search" # PostgreSQL text search dictionary used by the native tsvector backend. Only # affects text_search_extension == "native"; other backends use their own # tokenizers (vchord: llmlingua2, pg_textsearch: hardcoded english, -# pgroonga: TokenBigram polyglot). +# pgroonga: TokenBigram polyglot, pg_search: per-field Tantivy tokenizer). DEFAULT_TEXT_SEARCH_EXTENSION_NATIVE_LANGUAGE = "english" # LiteLLM defaults @@ -907,10 +908,11 @@ class HindsightConfig: migration_database_url: str | None database_schema: str vector_extension: str # "pgvector", "vchord", "pgvectorscale", or "scann" - text_search_extension: str # "native", "vchord", "pg_textsearch", or "pgroonga" + text_search_extension: str # "native", "vchord", "pg_textsearch", "pgroonga", or "pg_search" # PostgreSQL text search dictionary for the "native" backend (ignored by # other backends). Only the "native" backend reads this field; pgroonga - # uses TokenBigram, vchord uses llmlingua2, pg_textsearch hardcodes english. + # uses TokenBigram, vchord uses llmlingua2, pg_textsearch hardcodes english, + # pg_search uses Tantivy per-field tokenizers. text_search_extension_native_language: str # When set, every LLM-generated artifact (retain facts, consolidation # observations, reflect responses) is forced into this language regardless @@ -1381,7 +1383,7 @@ def validate(self) -> None: validate_extension(self.vector_extension) # Validate text_search_extension - valid_text_search = ("native", "vchord", "pg_textsearch", "pgroonga") + valid_text_search = ("native", "vchord", "pg_textsearch", "pgroonga", "pg_search") if self.text_search_extension not in valid_text_search: raise ValueError( f"Invalid text_search_extension: {self.text_search_extension}. Must be one of: {', '.join(valid_text_search)}" diff --git a/hindsight-api-slim/hindsight_api/engine/consolidation/consolidator.py b/hindsight-api-slim/hindsight_api/engine/consolidation/consolidator.py index d267631a9d..58db6d4d3f 100644 --- a/hindsight-api-slim/hindsight_api/engine/consolidation/consolidator.py +++ b/hindsight-api-slim/hindsight_api/engine/consolidation/consolidator.py @@ -1415,9 +1415,16 @@ async def _create_observation_directly( tokenize($3, 'llmlingua2')::bm25_catalog.bm25vector) RETURNING id """ - else: # native or pg_textsearch - # Native PostgreSQL: search_vector is GENERATED ALWAYS, don't include it - # pg_textsearch: indexes operate on base columns directly, don't populate search_vector + else: # native, pg_textsearch, pgroonga, or pg_search + # pg_textsearch / pgroonga / pg_search: indexes operate on base text + # columns directly, so the dummy search_vector column is left NULL. + # Native: the migration p4q5r6s7t8u9 dropped the GENERATED expression on + # search_vector to allow per-deployment language configuration; the + # batch insert path in ops_postgresql.insert_facts_batch now populates + # it via to_tsvector($lang, ...). This single-observation INSERT does + # not, so observations under the native backend currently land with + # NULL search_vector and are not BM25-searchable until reflected/ + # re-ingested. Tracking a separate fix for that gap. query = f""" INSERT INTO {fq_table("memory_units")} ( id, bank_id, text, fact_type, embedding, proof_count, source_memory_ids, history, diff --git a/hindsight-api-slim/hindsight_api/engine/db/ops_postgresql.py b/hindsight-api-slim/hindsight_api/engine/db/ops_postgresql.py index 259dad641f..cd5d9b178e 100644 --- a/hindsight-api-slim/hindsight_api/engine/db/ops_postgresql.py +++ b/hindsight-api-slim/hindsight_api/engine/db/ops_postgresql.py @@ -141,9 +141,9 @@ async def insert_facts_batch( RETURNING id """ else: - # pg_textsearch and pgroonga: search_vector is a dummy TEXT column; - # the actual full-text index operates on the base text columns - # directly, so we don't populate search_vector at insert time. + # pg_textsearch, pgroonga, and pg_search: search_vector is a dummy + # TEXT column; the actual full-text index operates on the base text + # columns directly, so we don't populate search_vector at insert time. query = f""" WITH input_data AS ( SELECT * FROM unnest( diff --git a/hindsight-api-slim/hindsight_api/engine/sql/postgresql.py b/hindsight-api-slim/hindsight_api/engine/sql/postgresql.py index 76df38e249..b9acadf5d6 100644 --- a/hindsight-api-slim/hindsight_api/engine/sql/postgresql.py +++ b/hindsight-api-slim/hindsight_api/engine/sql/postgresql.py @@ -203,6 +203,20 @@ def build_bm25_arm( f"AND (COALESCE(text, '') || ' ' || COALESCE(context, '') || ' ' || COALESCE(text_signals, '')) " f"&@~ {text_param}" ) + elif text_search_extension == "pg_search": + # ParadeDB pg_search: BM25 index over (id, text, context, text_signals) + # with key_field='id'. The @@@ operator on the key_field requires a + # field-qualified query (`text:foo`); to keep the bind-parameter form, + # we fan the query out across all indexed text fields with paradedb.boolean. + bm25_score_expr = "paradedb.score(id)" + bm25_order_by = "paradedb.score(id) DESC" + bm25_where_filter = ( + f"AND id @@@ paradedb.boolean(should => ARRAY[" + f"paradedb.match('text', {text_param}), " + f"paradedb.match('context', {text_param}), " + f"paradedb.match('text_signals', {text_param})" + f"])" + ) else: # native tsvector # bm25_language is validated as a PG identifier in HindsightConfig.validate(), # so embedding it as a SQL literal here is safe. @@ -233,7 +247,7 @@ def prepare_bm25_text( *, text_search_extension: str = "native", ) -> str: - if text_search_extension in ("vchord", "pg_textsearch", "pgroonga"): + if text_search_extension in ("vchord", "pg_textsearch", "pgroonga", "pg_search"): return query_text # native tsvector: join tokens with OR operator return " | ".join(tokens) diff --git a/hindsight-api-slim/hindsight_api/migrations.py b/hindsight-api-slim/hindsight_api/migrations.py index 769327d45c..11525c4964 100644 --- a/hindsight-api-slim/hindsight_api/migrations.py +++ b/hindsight-api-slim/hindsight_api/migrations.py @@ -845,6 +845,11 @@ def ensure_text_search_extension( # and so the column-type mismatch detection above keeps working. target_column_type = "text" target_index_type = "pgroonga" + elif text_search_extension == "pg_search": + # ParadeDB: same column type / access method as pg_textsearch. + # Disambiguated below by inspecting the index reloptions (key_field). + target_column_type = "text" + target_index_type = "bm25" else: # native target_column_type = "tsvector" target_index_type = "gin" @@ -882,16 +887,18 @@ def ensure_text_search_extension( if not current_column_info: logger.warning(f"No search_vector column found for {table_name}, will create it") - mismatched_tables.append((table_name, None, None)) + mismatched_tables.append((table_name, None, None, False)) continue # Check column type (udt_name contains the actual type: tsvector, bm25vector, etc.) current_column_type = current_column_info[1] # udt_name - # Get current index type + # Get current index type and definition. The definition lets us + # disambiguate pg_textsearch vs pg_search (both register a `bm25` + # access method but only pg_search uses the `key_field` reloption). current_index_info = conn.execute( text(""" - SELECT am.amname + SELECT am.amname, pi.indexdef FROM pg_indexes pi JOIN pg_class c ON c.relname = pi.indexname JOIN pg_am am ON am.oid = c.relam @@ -903,10 +910,21 @@ def ensure_text_search_extension( ).fetchone() current_index_type = current_index_info[0] if current_index_info else None + current_index_def = current_index_info[1] if current_index_info else None + + # Detect pg_search specifically (vs pg_textsearch) via the key_field reloption + current_is_pg_search = bool(current_index_def and "key_field" in current_index_def) + want_pg_search = text_search_extension == "pg_search" # Check if column and index types match target column_matches = current_column_type == target_column_type index_matches = current_index_type == target_index_type if current_index_type else False + # When both target and current sit at column=text/index=bm25, the + # access-method check alone can't tell pg_textsearch from pg_search — + # require the key_field reloption to agree with the configured backend. + if column_matches and index_matches and target_index_type == "bm25" and target_column_type == "text": + if current_is_pg_search != want_pg_search: + index_matches = False if not (column_matches and index_matches): logger.info( @@ -914,7 +932,7 @@ def ensure_text_search_extension( f"column={current_column_type} (want {target_column_type}), " f"index={current_index_type} (want {target_index_type})" ) - mismatched_tables.append((table_name, current_column_type, current_index_type)) + mismatched_tables.append((table_name, current_column_type, current_index_type, current_is_pg_search)) # Check if table has data row_count = conn.execute(text(f"SELECT COUNT(*) FROM {schema_name}.{table_name}")).scalar() @@ -932,11 +950,12 @@ def ensure_text_search_extension( # If there's data in any mismatched table, raise error if tables_with_data: table_list = ", ".join([f"{table}({count} rows)" for table, count in tables_with_data]) - # Detect current extension from column type + index type. tsvector is - # unambiguous; text could be either pg_textsearch or pgroonga, so we - # disambiguate via the index type. + # Detect current extension from column type, index type, and (for the + # text/bm25 ambiguity) the key_field reloption. tsvector is + # unambiguous; text could be pg_textsearch, pgroonga, or pg_search. current_col_type = mismatched_tables[0][1] current_idx_type = mismatched_tables[0][2] + first_is_pg_search = mismatched_tables[0][3] if current_col_type == "tsvector": current_ext = "native" elif current_col_type == "bm25vector": @@ -944,7 +963,7 @@ def ensure_text_search_extension( elif current_col_type == "text" and current_idx_type == "pgroonga": current_ext = "pgroonga" elif current_col_type == "text": - current_ext = "pg_textsearch" + current_ext = "pg_search" if first_is_pg_search else "pg_textsearch" else: current_ext = "unknown" raise RuntimeError( @@ -959,7 +978,7 @@ def ensure_text_search_extension( # Tables are empty, safe to recreate columns/indexes logger.info(f"Recreating text search columns/indexes for {text_search_extension}") - for table_name, current_col_type, current_idx_type in mismatched_tables: + for table_name, current_col_type, current_idx_type, _was_pg_search in mismatched_tables: # Drop existing index if it exists if current_idx_type: logger.info(f"Dropping {current_idx_type} index on {table_name}") @@ -1049,6 +1068,27 @@ def ensure_text_search_extension( WITH (tokenizer='TokenBigram', normalizer='NormalizerNFKC150') """) ) + elif text_search_extension == "pg_search": + logger.info(f"Creating TEXT column on {table_name}") + # Dummy TEXT column for schema symmetry; pg_search indexes operate on base columns. + conn.execute(text(f"ALTER TABLE {schema_name}.{table_name} ADD COLUMN search_vector TEXT")) + + # ParadeDB BM25 index over the table's primary key and text columns. + # Column list mirrors what the initial / text_signals migrations create. + if table_name == "memory_units": + bm25_cols = "id, text, context, text_signals" + else: # reflections + bm25_cols = "id, name, content" + + logger.info(f"Creating ParadeDB BM25 index on {table_name}") + conn.execute( + text(f""" + CREATE INDEX idx_{table_name.replace(".", "_")}_text_search + ON {schema_name}.{table_name} + USING bm25 ({bm25_cols}) + WITH (key_field='id') + """) + ) else: # native logger.info(f"Creating tsvector column on {table_name}") # Plain tsvector column. The application populates search_vector diff --git a/hindsight-api-slim/tests/test_db_abstraction.py b/hindsight-api-slim/tests/test_db_abstraction.py index 1a0fb763ac..0f02473a6b 100644 --- a/hindsight-api-slim/tests/test_db_abstraction.py +++ b/hindsight-api-slim/tests/test_db_abstraction.py @@ -255,6 +255,23 @@ def test_build_bm25_arm_pgroonga_ignores_bm25_language(self, d): ) assert "french" not in arm + def test_build_bm25_arm_pg_search(self, d): + arm = d.build_bm25_arm( + table="schema.memory_units", cols="id, text", fact_type="world", + bank_id_param="$2", limit_param="$3", text_param="$4", + text_search_extension="pg_search", + ) + assert "paradedb.score(id)" in arm + # @@@ on the key_field requires a field-qualified query, so we + # fan the bind param out across all indexed text fields. + assert "id @@@ paradedb.boolean(should =>" in arm + assert "paradedb.match('text', $4)" in arm + assert "paradedb.match('context', $4)" in arm + assert "paradedb.match('text_signals', $4)" in arm + assert "paradedb.score(id) DESC" in arm + assert "'bm25' AS source" in arm + assert "LIMIT $3" in arm + def test_prepare_bm25_text_native(self, d): result = d.prepare_bm25_text(["hello", "world"], "hello world") assert result == "hello | world" @@ -269,6 +286,10 @@ def test_prepare_bm25_text_pgroonga(self, d): result = d.prepare_bm25_text(["hello", "world"], "hello world", text_search_extension="pgroonga") assert result == "hello world" + def test_prepare_bm25_text_pg_search(self, d): + result = d.prepare_bm25_text(["hello", "world"], "hello world", text_search_extension="pg_search") + assert result == "hello world" + # --------------------------------------------------------------------------- # OracleDialect tests (no oracledb dependency needed) diff --git a/hindsight-docs/docs/developer/configuration.md b/hindsight-docs/docs/developer/configuration.md index 2fef4ab804..79016c369f 100644 --- a/hindsight-docs/docs/developer/configuration.md +++ b/hindsight-docs/docs/developer/configuration.md @@ -140,15 +140,16 @@ If you need to switch from one extension to another: | Variable | Description | Default | |----------|-------------|---------| -| `HINDSIGHT_API_TEXT_SEARCH_EXTENSION` | Text search backend: `native`, `vchord`, `pg_textsearch`, or `pgroonga` | `native` | +| `HINDSIGHT_API_TEXT_SEARCH_EXTENSION` | Text search backend: `native`, `vchord`, `pg_textsearch`, `pgroonga`, or `pg_search` | `native` | | `HINDSIGHT_API_TEXT_SEARCH_EXTENSION_NATIVE_LANGUAGE` | PostgreSQL text search dictionary used by the `native` backend (e.g. `english`, `french`, `simple`, `zhparser`) | `english` | | `HINDSIGHT_API_LLM_OUTPUT_LANGUAGE` | When set, forces every LLM-generated artifact (retain facts, consolidation observations, reflect responses) into this language. Free-form (e.g. `Spanish`, `Japanese`). | unset | -Hindsight supports four backends for BM25 keyword retrieval: +Hindsight supports five backends for BM25 keyword retrieval: - **native** — PostgreSQL's built-in full-text search (`tsvector` + GIN). Language configurable. - **vchord** — VectorChord BM25 (uses the `llmlingua2` multilingual tokenizer). - **pg_textsearch** — Timescale's pg_textsearch extension. English-only. - **pgroonga** — pgroonga full-text search. Multilingual / CJK out of the box. +- **pg_search** — ParadeDB pg_search. True BM25; the only backend that is Citus-compatible. To switch backends: set `HINDSIGHT_API_TEXT_SEARCH_EXTENSION`. With existing data, you'll get an error and migration instructions; with an empty database the columns/indexes are recreated automatically on startup. diff --git a/hindsight-docs/docs/developer/retrieval.md b/hindsight-docs/docs/developer/retrieval.md index 9338c30b6f..be4e7cb091 100644 --- a/hindsight-docs/docs/developer/retrieval.md +++ b/hindsight-docs/docs/developer/retrieval.md @@ -63,6 +63,19 @@ No single search method handles all these well. Hindsight solves this with **TEM **Why it matters:** Ensures you never miss results that mention specific names or terms, even if they're semantically distant from your query. +**Backends:** Hindsight ships four pluggable BM25 backends, selected via +`HINDSIGHT_API_TEXT_SEARCH_EXTENSION`: + +| Backend | What it uses | Citus-compatible? | +|---|---|---| +| `native` | PostgreSQL `tsvector` + `ts_rank_cd` (TF-IDF, not true BM25) | Yes | +| `vchord` | `vchord_bm25` extension | No | +| `pg_textsearch` | Timescale `pg_textsearch` extension | No | +| `pg_search` | ParadeDB `pg_search` extension | Yes | + +If you need true BM25 ranking on a horizontally scaled Postgres (Citus) cluster, +`pg_search` is the only option. See the [`pg_search` docker-compose example](https://github.com/vectorize-io/hindsight/tree/main/docker/docker-compose/pg_search). + --- ### Graph Traversal diff --git a/skills/hindsight-docs/references/developer/configuration.md b/skills/hindsight-docs/references/developer/configuration.md index f6cef88a82..f3b45c7124 100644 --- a/skills/hindsight-docs/references/developer/configuration.md +++ b/skills/hindsight-docs/references/developer/configuration.md @@ -140,15 +140,16 @@ If you need to switch from one extension to another: | Variable | Description | Default | |----------|-------------|---------| -| `HINDSIGHT_API_TEXT_SEARCH_EXTENSION` | Text search backend: `native`, `vchord`, `pg_textsearch`, or `pgroonga` | `native` | +| `HINDSIGHT_API_TEXT_SEARCH_EXTENSION` | Text search backend: `native`, `vchord`, `pg_textsearch`, `pgroonga`, or `pg_search` | `native` | | `HINDSIGHT_API_TEXT_SEARCH_EXTENSION_NATIVE_LANGUAGE` | PostgreSQL text search dictionary used by the `native` backend (e.g. `english`, `french`, `simple`, `zhparser`) | `english` | | `HINDSIGHT_API_LLM_OUTPUT_LANGUAGE` | When set, forces every LLM-generated artifact (retain facts, consolidation observations, reflect responses) into this language. Free-form (e.g. `Spanish`, `Japanese`). | unset | -Hindsight supports four backends for BM25 keyword retrieval: +Hindsight supports five backends for BM25 keyword retrieval: - **native** — PostgreSQL's built-in full-text search (`tsvector` + GIN). Language configurable. - **vchord** — VectorChord BM25 (uses the `llmlingua2` multilingual tokenizer). - **pg_textsearch** — Timescale's pg_textsearch extension. English-only. - **pgroonga** — pgroonga full-text search. Multilingual / CJK out of the box. +- **pg_search** — ParadeDB pg_search. True BM25; the only backend that is Citus-compatible. To switch backends: set `HINDSIGHT_API_TEXT_SEARCH_EXTENSION`. With existing data, you'll get an error and migration instructions; with an empty database the columns/indexes are recreated automatically on startup. @@ -453,7 +454,7 @@ two slots that retain/consolidation cannot consume. | Variable | Description | Default | |----------|-------------|---------| -| `HINDSIGHT_API_EMBEDDINGS_PROVIDER` | Provider: `local`, `tei`, `openai`, `openrouter`, `cohere`, `google`, `litellm`, or `litellm-sdk` | `local` | +| `HINDSIGHT_API_EMBEDDINGS_PROVIDER` | Provider: `local`, `tei`, `openai`, `openai-codex`, `openrouter`, `cohere`, `google`, `litellm`, or `litellm-sdk` | `local` | | `HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL` | Model for local provider | `BAAI/bge-small-en-v1.5` | | `HINDSIGHT_API_EMBEDDINGS_LOCAL_TRUST_REMOTE_CODE` | Allow loading models with custom code (security risk, disabled by default) | `false` | | `HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU` | Force CPU mode for local embeddings (avoids MPS/XPC issues on macOS) | `false` | @@ -462,6 +463,7 @@ two slots that retain/consolidation cannot consume. | `HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL` | OpenAI embedding model | `text-embedding-3-small` | | `HINDSIGHT_API_EMBEDDINGS_OPENAI_BASE_URL` | Custom base URL for OpenAI-compatible API (e.g., Azure OpenAI) | - | | `HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE` | Max inputs per `embeddings.create` call for `openai`/`openrouter` providers — lower this when the upstream endpoint enforces stricter limits (e.g. DashScope caps at 10) | `100` | +| `HINDSIGHT_API_EMBEDDINGS_OPENAI_DIMENSIONS` | Optional requested output dimensions for OpenAI `text-embedding-3` models (e.g., `384` to match an existing pgvector schema) | - | | `HINDSIGHT_API_EMBEDDINGS_OPENROUTER_API_KEY` | OpenRouter API key for embeddings (falls back to `HINDSIGHT_API_OPENROUTER_API_KEY`, then `HINDSIGHT_API_LLM_API_KEY`) | - | | `HINDSIGHT_API_EMBEDDINGS_OPENROUTER_MODEL` | OpenRouter embedding model | `perplexity/pplx-embed-v1-0.6b` | | `HINDSIGHT_API_EMBEDDINGS_COHERE_API_KEY` | Cohere API key for embeddings | - | @@ -520,8 +522,14 @@ export HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL=BAAI/bge-small-en-v1.5 # OpenAI - cloud-based embeddings export HINDSIGHT_API_EMBEDDINGS_PROVIDER=openai -export HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY=sk-xxxxxxxxxxxx # or reuses HINDSIGHT_API_LLM_API_KEY -export HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL=text-embedding-3-small # 1536 dimensions +export HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY=*** # or reuses HINDSIGHT_API_LLM_API_KEY +export HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL=text-embedding-3-small # 1536 dimensions by default +# export HINDSIGHT_API_EMBEDDINGS_OPENAI_DIMENSIONS=384 # optional reduced output size + +# OpenAI Codex OAuth - uses existing ChatGPT/Codex login, no API key needed +export HINDSIGHT_API_EMBEDDINGS_PROVIDER=openai-codex +export HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL=text-embedding-3-small # 1536 dimensions by default +# export HINDSIGHT_API_EMBEDDINGS_OPENAI_DIMENSIONS=384 # optional reduced output size # Azure OpenAI - embeddings via Azure endpoint export HINDSIGHT_API_EMBEDDINGS_PROVIDER=openai diff --git a/skills/hindsight-docs/references/developer/retrieval.md b/skills/hindsight-docs/references/developer/retrieval.md index 9338c30b6f..be4e7cb091 100644 --- a/skills/hindsight-docs/references/developer/retrieval.md +++ b/skills/hindsight-docs/references/developer/retrieval.md @@ -63,6 +63,19 @@ No single search method handles all these well. Hindsight solves this with **TEM **Why it matters:** Ensures you never miss results that mention specific names or terms, even if they're semantically distant from your query. +**Backends:** Hindsight ships four pluggable BM25 backends, selected via +`HINDSIGHT_API_TEXT_SEARCH_EXTENSION`: + +| Backend | What it uses | Citus-compatible? | +|---|---|---| +| `native` | PostgreSQL `tsvector` + `ts_rank_cd` (TF-IDF, not true BM25) | Yes | +| `vchord` | `vchord_bm25` extension | No | +| `pg_textsearch` | Timescale `pg_textsearch` extension | No | +| `pg_search` | ParadeDB `pg_search` extension | Yes | + +If you need true BM25 ranking on a horizontally scaled Postgres (Citus) cluster, +`pg_search` is the only option. See the [`pg_search` docker-compose example](https://github.com/vectorize-io/hindsight/tree/main/docker/docker-compose/pg_search). + --- ### Graph Traversal