fix(db): unblock PostgreSQL upgrade to v0.7.x (sqlalchemy<2.1 + autocommit_block migrations)

hindsight-api-slim/hindsight_api/alembic/versions/a2b3c4d5e6f8_add_gin_index_source_memory_ids.py

@@ -40,20 +40,20 @@ def _get_schema_prefix() -> str:
 def _pg_upgrade() -> None:
     schema = _get_schema_prefix()
 
-    # CREATE INDEX CONCURRENTLY cannot run inside a transaction block.
-    # Commit the current Alembic transaction first.
-    op.execute("COMMIT")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_source_memory_ids "
-        f"ON {schema}memory_units USING GIN (source_memory_ids) "
-        f"WHERE source_memory_ids IS NOT NULL"
-    )
+    # CREATE INDEX CONCURRENTLY cannot run inside a transaction block; an
+    # autocommit_block runs it outside Alembic's migration transaction.
+    with op.get_context().autocommit_block():
+        op.execute(
+            f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_source_memory_ids "
+            f"ON {schema}memory_units USING GIN (source_memory_ids) "
+            f"WHERE source_memory_ids IS NOT NULL"
+        )
 
 
 def _pg_downgrade() -> None:
     schema = _get_schema_prefix()
-    op.execute("COMMIT")
-    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_source_memory_ids")
+    with op.get_context().autocommit_block():
+        op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_source_memory_ids")
 
 
 def upgrade() -> None:

hindsight-api-slim/hindsight_api/alembic/versions/b3c4d5e6f7g8_add_temporal_date_indexes.py

@@ -37,37 +37,35 @@ def _get_schema_prefix() -> str:
 
 def _pg_upgrade() -> None:
     schema = _get_schema_prefix()
-    # Partial index on occurred_start (covers "occurred_start BETWEEN $4 AND $5")
-    op.execute("COMMIT")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_bank_occurred_start "
-        f"ON {schema}memory_units(bank_id, fact_type, occurred_start) "
-        f"WHERE occurred_start IS NOT NULL"
-    )
-    # Partial index on occurred_end (covers "occurred_end BETWEEN $4 AND $5")
-    op.execute("COMMIT")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_bank_occurred_end "
-        f"ON {schema}memory_units(bank_id, fact_type, occurred_end) "
-        f"WHERE occurred_end IS NOT NULL"
-    )
-    # Partial index on mentioned_at (covers "mentioned_at BETWEEN $4 AND $5")
-    op.execute("COMMIT")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_bank_mentioned_at "
-        f"ON {schema}memory_units(bank_id, fact_type, mentioned_at) "
-        f"WHERE mentioned_at IS NOT NULL"
-    )
+    # CREATE INDEX CONCURRENTLY cannot run inside a transaction block; an
+    # autocommit_block runs each statement outside Alembic's migration transaction.
+    with op.get_context().autocommit_block():
+        # Partial index on occurred_start (covers "occurred_start BETWEEN $4 AND $5")
+        op.execute(
+            f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_bank_occurred_start "
+            f"ON {schema}memory_units(bank_id, fact_type, occurred_start) "
+            f"WHERE occurred_start IS NOT NULL"
+        )
+        # Partial index on occurred_end (covers "occurred_end BETWEEN $4 AND $5")
+        op.execute(
+            f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_bank_occurred_end "
+            f"ON {schema}memory_units(bank_id, fact_type, occurred_end) "
+            f"WHERE occurred_end IS NOT NULL"
+        )
+        # Partial index on mentioned_at (covers "mentioned_at BETWEEN $4 AND $5")
+        op.execute(
+            f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_bank_mentioned_at "
+            f"ON {schema}memory_units(bank_id, fact_type, mentioned_at) "
+            f"WHERE mentioned_at IS NOT NULL"
+        )
 
 
 def _pg_downgrade() -> None:
     schema = _get_schema_prefix()
-    op.execute("COMMIT")
-    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_bank_mentioned_at")
-    op.execute("COMMIT")
-    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_bank_occurred_end")
-    op.execute("COMMIT")
-    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_bank_occurred_start")
+    with op.get_context().autocommit_block():
+        op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_bank_mentioned_at")
+        op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_bank_occurred_end")
+        op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_bank_occurred_start")
 
 
 def upgrade() -> None:

hindsight-api-slim/hindsight_api/alembic/versions/c1a2b3d4e5f6_enable_pg_trgm_and_entities_trgm_index.py

@@ -47,17 +47,18 @@ def _pg_upgrade() -> None:
     schema = _get_schema_prefix()
     # GIN index on canonical_name enables sub-millisecond trigram similarity queries
     # (% operator, similarity()) instead of full-table scans across all bank entities.
-    op.execute("COMMIT")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS entities_canonical_name_trgm_idx "
-        f"ON {schema}entities USING GIN (canonical_name gin_trgm_ops)"
-    )
+    # CREATE INDEX CONCURRENTLY cannot run inside a transaction block.
+    with op.get_context().autocommit_block():
+        op.execute(
+            f"CREATE INDEX CONCURRENTLY IF NOT EXISTS entities_canonical_name_trgm_idx "
+            f"ON {schema}entities USING GIN (canonical_name gin_trgm_ops)"
+        )
 
 
 def _pg_downgrade() -> None:
     schema = _get_schema_prefix()
-    op.execute("COMMIT")
-    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}entities_canonical_name_trgm_idx")
+    with op.get_context().autocommit_block():
+        op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}entities_canonical_name_trgm_idx")
     # Note: not dropping pg_trgm extension as other indexes may depend on it
 
 

hindsight-api-slim/hindsight_api/alembic/versions/d2e3f4a5b6c7_add_memory_links_expansion_indexes.py

@@ -50,39 +50,35 @@ def _get_schema_prefix() -> str:
 def _pg_upgrade() -> None:
     schema = _get_schema_prefix()
 
-    # CREATE INDEX CONCURRENTLY cannot run inside a transaction block.
-    # Commit the current Alembic transaction, then issue each CONCURRENTLY
-    # statement in its own implicit autocommit transaction.
-    # IF NOT EXISTS makes each statement idempotent if the migration is retried.
-
-    # Index for the semantic *incoming* direction in link_expansion_retrieval.py.
-    # Replaces the BitmapAnd of idx_memory_links_to_unit ∩ idx_memory_links_link_type
-    # with a single composite index scan.
-    op.execute("COMMIT")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_to_type_weight "
-        f"ON {schema}memory_links(to_unit_id, link_type, weight DESC)"
-    )
-
-    # Covering index for entity co-occurrence expansion.
-    # Enables an index-only scan: entity_id and to_unit_id are read from the
-    # index leaf pages instead of the heap, eliminating ~2 500 random heap-page
-    # reads per expansion query.
-    op.execute("COMMIT")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_entity_covering "
-        f"ON {schema}memory_links(from_unit_id) "
-        f"INCLUDE (to_unit_id, entity_id) "
-        f"WHERE link_type = 'entity'"
-    )
+    # CREATE INDEX CONCURRENTLY cannot run inside a transaction block; an
+    # autocommit_block runs each statement outside Alembic's migration
+    # transaction.  IF NOT EXISTS makes each statement idempotent on retry.
+    with op.get_context().autocommit_block():
+        # Index for the semantic *incoming* direction in link_expansion_retrieval.py.
+        # Replaces the BitmapAnd of idx_memory_links_to_unit ∩ idx_memory_links_link_type
+        # with a single composite index scan.
+        op.execute(
+            f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_to_type_weight "
+            f"ON {schema}memory_links(to_unit_id, link_type, weight DESC)"
+        )
+
+        # Covering index for entity co-occurrence expansion.
+        # Enables an index-only scan: entity_id and to_unit_id are read from the
+        # index leaf pages instead of the heap, eliminating ~2 500 random heap-page
+        # reads per expansion query.
+        op.execute(
+            f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_entity_covering "
+            f"ON {schema}memory_links(from_unit_id) "
+            f"INCLUDE (to_unit_id, entity_id) "
+            f"WHERE link_type = 'entity'"
+        )
 
 
 def _pg_downgrade() -> None:
     schema = _get_schema_prefix()
-    op.execute("COMMIT")
-    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_links_entity_covering")
-    op.execute("COMMIT")
-    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_links_to_type_weight")
+    with op.get_context().autocommit_block():
+        op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_links_entity_covering")
+        op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_links_to_type_weight")
 
 
 def upgrade() -> None:

hindsight-api-slim/hindsight_api/alembic/versions/d4e5f6g7h8i9_gin_source_memory_ids_fastupdate_off.py

@@ -33,26 +33,27 @@ def _get_schema_prefix() -> str:
 
 def _pg_upgrade() -> None:
     schema = _get_schema_prefix()
-    # DROP + CREATE CONCURRENTLY must run outside a transaction block.
-    op.execute("COMMIT")
-    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_source_memory_ids")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_source_memory_ids "
-        f"ON {schema}memory_units USING GIN (source_memory_ids) "
-        f"WITH (fastupdate=off) "
-        f"WHERE source_memory_ids IS NOT NULL"
-    )
+    # DROP + CREATE CONCURRENTLY must run outside a transaction block; an
+    # autocommit_block runs them outside Alembic's migration transaction.
+    with op.get_context().autocommit_block():
+        op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_source_memory_ids")
+        op.execute(
+            f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_source_memory_ids "
+            f"ON {schema}memory_units USING GIN (source_memory_ids) "
+            f"WITH (fastupdate=off) "
+            f"WHERE source_memory_ids IS NOT NULL"
+        )
 
 
 def _pg_downgrade() -> None:
     schema = _get_schema_prefix()
-    op.execute("COMMIT")
-    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_source_memory_ids")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_source_memory_ids "
-        f"ON {schema}memory_units USING GIN (source_memory_ids) "
-        f"WHERE source_memory_ids IS NOT NULL"
-    )
+    with op.get_context().autocommit_block():
+        op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_units_source_memory_ids")
+        op.execute(
+            f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_units_source_memory_ids "
+            f"ON {schema}memory_units USING GIN (source_memory_ids) "
+            f"WHERE source_memory_ids IS NOT NULL"
+        )
 
 
 def upgrade() -> None:

hindsight-api-slim/hindsight_api/alembic/versions/e1b2c3d4f5a6_drop_unused_indexes.py

@@ -75,53 +75,54 @@ def _schema_prefix() -> str:
 
 def _pg_upgrade() -> None:
     schema = _schema_prefix()
-    # DROP INDEX CONCURRENTLY cannot run inside a transaction block; commit
-    # the Alembic transaction and issue each statement in its own implicit
-    # autocommit transaction.  IF EXISTS makes each statement idempotent
-    # across schemas that already dropped (or never had) the index.
-    for index_name in _PG_INDEXES_TO_DROP:
-        op.execute("COMMIT")
-        op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}{index_name}")
+    # DROP INDEX CONCURRENTLY cannot run inside a transaction block; an
+    # autocommit_block drops out of Alembic's migration transaction so each
+    # statement runs in its own autocommit.  IF EXISTS makes each statement
+    # idempotent across schemas that already dropped (or never had) the index.
+    with op.get_context().autocommit_block():
+        for index_name in _PG_INDEXES_TO_DROP:
+            op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}{index_name}")
 
 
 def _pg_downgrade() -> None:
     schema = _schema_prefix()
 
     # Recreate the dropped indexes in the same shape the prior migrations used,
     # so a downgrade leaves the schema in the state the previous head expected.
-    op.execute("COMMIT")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_entity_covering "
-        f"ON {schema}memory_links(from_unit_id) "
-        f"INCLUDE (to_unit_id, entity_id) "
-        f"WHERE link_type = 'entity'"
-    )
-    op.execute("COMMIT")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_from_unit ON {schema}memory_links(from_unit_id)"
-    )
-    op.execute("COMMIT")
-    op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_to_unit ON {schema}memory_links(to_unit_id)")
-    op.execute("COMMIT")
-    op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_link_type ON {schema}memory_links(link_type)")
-    op.execute("COMMIT")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_entities_canonical_name ON {schema}entities(canonical_name)"
-    )
-    op.execute("COMMIT")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS entities_canonical_name_trgm_idx "
-        f"ON {schema}entities USING GIN (canonical_name gin_trgm_ops)"
-    )
-    op.execute("COMMIT")
-    op.execute(
-        f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_documents_retain_params "
-        f"ON {schema}documents USING GIN (retain_params)"
-    )
-    op.execute("COMMIT")
-    op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_documents_content_hash ON {schema}documents(content_hash)")
-    op.execute("COMMIT")
-    op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_unit_entities_entity ON {schema}unit_entities(entity_id)")
+    # CREATE INDEX CONCURRENTLY cannot run inside a transaction block.
+    with op.get_context().autocommit_block():
+        op.execute(
+            f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_entity_covering "
+            f"ON {schema}memory_links(from_unit_id) "
+            f"INCLUDE (to_unit_id, entity_id) "
+            f"WHERE link_type = 'entity'"
+        )
+        op.execute(
+            f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_from_unit ON {schema}memory_links(from_unit_id)"
+        )
+        op.execute(
+            f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_to_unit ON {schema}memory_links(to_unit_id)"
+        )
+        op.execute(
+            f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_links_link_type ON {schema}memory_links(link_type)"
+        )
+        op.execute(
+            f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_entities_canonical_name ON {schema}entities(canonical_name)"
+        )
+        op.execute(
+            f"CREATE INDEX CONCURRENTLY IF NOT EXISTS entities_canonical_name_trgm_idx "
+            f"ON {schema}entities USING GIN (canonical_name gin_trgm_ops)"
+        )
+        op.execute(
+            f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_documents_retain_params "
+            f"ON {schema}documents USING GIN (retain_params)"
+        )
+        op.execute(
+            f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_documents_content_hash ON {schema}documents(content_hash)"
+        )
+        op.execute(
+            f"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_unit_entities_entity ON {schema}unit_entities(entity_id)"
+        )
 
 
 def upgrade() -> None:

hindsight-api-slim/hindsight_api/alembic/versions/e9b2c7d1f3a4_drop_entity_memory_links.py

@@ -37,33 +37,35 @@ def _pg_upgrade() -> None:
     schema = _pg_schema_prefix()
 
     # Drop the partial covering index first so the bulk DELETE doesn't churn it.
-    # CREATE/DROP INDEX CONCURRENTLY must run outside a transaction block.
-    op.execute("COMMIT")
-    op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_links_entity_covering")
-
-    # Delete entity rows. Chunked to keep individual transactions small on
-    # large banks (the perf-medium bench had ~345k entity rows; production
-    # banks can be much larger).
-    op.execute(
-        f"""
-        DO $$
-        DECLARE
-            deleted INTEGER;
-        BEGIN
-            LOOP
-                DELETE FROM {schema}memory_links
-                WHERE ctid IN (
-                    SELECT ctid FROM {schema}memory_links
-                    WHERE link_type = 'entity'
-                    LIMIT 50000
-                );
-                GET DIAGNOSTICS deleted = ROW_COUNT;
-                EXIT WHEN deleted = 0;
-                COMMIT;
-            END LOOP;
-        END$$;
-        """
-    )
+    # DROP INDEX CONCURRENTLY, and the DO block's per-batch COMMIT, both require
+    # running outside Alembic's migration transaction — an autocommit_block
+    # commits it and switches the connection to autocommit for the duration.
+    with op.get_context().autocommit_block():
+        op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {schema}idx_memory_links_entity_covering")
+
+        # Delete entity rows. Chunked to keep individual transactions small on
+        # large banks (the perf-medium bench had ~345k entity rows; production
+        # banks can be much larger).
+        op.execute(
+            f"""
+            DO $$
+            DECLARE
+                deleted INTEGER;
+            BEGIN
+                LOOP
+                    DELETE FROM {schema}memory_links
+                    WHERE ctid IN (
+                        SELECT ctid FROM {schema}memory_links
+                        WHERE link_type = 'entity'
+                        LIMIT 50000
+                    );
+                    GET DIAGNOSTICS deleted = ROW_COUNT;
+                    EXIT WHEN deleted = 0;
+                    COMMIT;
+                END LOOP;
+            END$$;
+            """
+        )
 
 
 def _pg_downgrade() -> None:

hindsight-api-slim/pyproject.toml

@@ -18,7 +18,11 @@ dependencies = [
     "fastapi[standard]>=0.120.3",
     "uvicorn>=0.38.0",
     "wsproto>=1.0.0",
-    "sqlalchemy>=2.0.44",
+    # Cap below 2.1: SQLAlchemy 2.1 switches the default `postgresql://` DBAPI
+    # from psycopg2 to psycopg (v3), which we don't ship — a bare install would
+    # fail migrations with "No module named 'psycopg'". Pin to the tested 2.0
+    # line (which keeps psycopg2 the default driver) until psycopg3 is adopted.
+    "sqlalchemy>=2.0.44,<2.1",
     "alembic>=1.17.1",
     "pgvector>=0.4.1",
     "greenlet>=3.2.4,<3.4.0", # 3.4.0 lacks arm64 wheels for manylinux_2_41