fix(state): add sessions.trigram_fts flag to drop the trigram FTS index

hermes_cli/config.py

@@ -3068,6 +3068,18 @@ def _ensure_hermes_home_managed(home: Path):
         # GBs of disk on heavy users.  Opt in only if you have an external
         # tool that consumes the JSON files directly.
         "write_json_snapshots": False,
+        # Secondary trigram FTS5 index over message content, used for
+        # substring and CJK (Chinese/Japanese/Korean) search.  It stores
+        # its own full copy of every message plus a 3-character-gram index,
+        # so it is by far the largest object in state.db (3-5x the raw
+        # message text) and fires on every message INSERT.  On heavy users
+        # it dominates DB size and slows writes enough to blow the
+        # write-lock retry budget ("database is locked").  When false, the
+        # index is dropped at startup and never recreated; substring/CJK
+        # search transparently falls back to a LIKE scan (the base
+        # ``messages_fts`` word index — used by recall and session search —
+        # is unaffected).  Default true preserves current behavior.
+        "trigram_fts": True,
     },
 
     # Contextual first-touch onboarding hints (see agent/onboarding.py).

hermes_state.py

@@ -175,6 +175,25 @@ def _delete_delegate_children(conn, parent_ids: List[str]) -> List[str]:
     "messages_fts_trigram_update",
 )
 
+# Subset of _FTS_TRIGGERS that feed the trigram index specifically. Dropped
+# (leaving the base messages_fts triggers intact) when sessions.trigram_fts
+# is disabled — see SessionDB._drop_trigram_fts.
+_TRIGRAM_FTS_TRIGGERS = (
+    "messages_fts_trigram_insert",
+    "messages_fts_trigram_delete",
+    "messages_fts_trigram_update",
+)
+
+# The base messages_fts triggers, always expected regardless of the trigram
+# flag. Used to detect base-trigger degradation independently of trigram
+# presence (comparing against the 6-tuple total would let a healthy
+# trigram-disabled DB mask 3 missing base triggers).
+_BASE_FTS_TRIGGERS = (
+    "messages_fts_insert",
+    "messages_fts_delete",
+    "messages_fts_update",
+)
+
 
 def _set_last_init_error(msg: Optional[str]) -> None:
     """Record (or clear) the most recent state.db init failure.
@@ -994,6 +1013,24 @@ def _connect_and_init():
 
     # ── Core write helper ──
 
+    @staticmethod
+    def _trigram_fts_enabled() -> bool:
+        """Read ``sessions.trigram_fts`` from config (default True).
+
+        Lazy import + best-effort: any config-load failure defaults to the
+        historical behavior (trigram enabled).  Cheap enough to call once per
+        schema init; not on the message write hot path."""
+        try:
+            from hermes_cli.config import load_config
+
+            cfg = load_config()
+            sessions = cfg.get("sessions") if isinstance(cfg, dict) else None
+            if isinstance(sessions, dict) and "trigram_fts" in sessions:
+                return bool(sessions["trigram_fts"])
+        except Exception:
+            pass
+        return True
+
     @staticmethod
     def _is_fts5_unavailable_error(exc: sqlite3.OperationalError) -> bool:
         err = str(exc).lower()
@@ -1058,6 +1095,69 @@ def _drop_fts_triggers(cursor: sqlite3.Cursor) -> None:
             except sqlite3.OperationalError:
                 pass
 
+    @staticmethod
+    def _drop_trigram_fts(cursor: sqlite3.Cursor) -> bool:
+        """Drop the trigram FTS5 index and its triggers.  Returns True only if
+        the virtual table existed and was actually removed (so the caller knows
+        a VACUUM would reclaim space).  Used when ``sessions.trigram_fts`` is
+        disabled: the trigram index is the single largest object in state.db and
+        fires on every message INSERT, so a heavy user opts out to shrink the DB
+        and speed up writes.  Substring/CJK search falls back to a LIKE scan; the
+        base ``messages_fts`` word index is untouched.
+
+        The trigger + table drops must fail closed. A concurrent writer can hold
+        the DB lock during startup (gateway + cron), and swallowing that lock on
+        a trigger or table DROP can leave a stale trigram trigger pointing at a
+        missing table (or log that the table was dropped when it was not). Acquire
+        a write transaction up front so the sequence commits as a unit; suppress
+        only known FTS-unavailable errors and propagate lock/unrelated failures.
+        """
+        started_transaction = False
+        try:
+            cursor.execute("BEGIN IMMEDIATE")
+            started_transaction = True
+            for trigger in _TRIGRAM_FTS_TRIGGERS:
+                cursor.execute(f"DROP TRIGGER IF EXISTS {trigger}")
+            row = cursor.execute(
+                "SELECT 1 FROM sqlite_master "
+                "WHERE type = 'table' AND name = 'messages_fts_trigram'"
+            ).fetchone()
+            existed = row is not None
+            cursor.execute("DROP TABLE IF EXISTS messages_fts_trigram")
+            cursor.execute("COMMIT")
+            started_transaction = False
+            return existed
+        except sqlite3.OperationalError as exc:
+            if started_transaction:
+                try:
+                    cursor.execute("ROLLBACK")
+                except sqlite3.OperationalError:
+                    pass
+            if SessionDB._is_fts5_unavailable_error(exc):
+                # The DROP TABLE needs to load the vtable module/tokenizer, so a
+                # build missing the trigram tokenizer raises here and the rollback
+                # above also undid the (tokenizer-free) trigger drops. Leaving the
+                # triggers in place means every message INSERT fires
+                # messages_fts_trigram_insert against the unusable vtable and
+                # crashes. DROP TRIGGER does not need the tokenizer, so drop them
+                # in a standalone committed step and degrade to the base FTS/LIKE
+                # path. The orphaned trigram table is harmless without triggers.
+                try:
+                    cursor.execute("BEGIN IMMEDIATE")
+                    for trigger in _TRIGRAM_FTS_TRIGGERS:
+                        cursor.execute(f"DROP TRIGGER IF EXISTS {trigger}")
+                    cursor.execute("COMMIT")
+                except sqlite3.OperationalError:
+                    # Locked/unavailable while dropping the triggers alone; leave
+                    # the DB untouched (roll back the partial trigger drop) and
+                    # report no drop so a later uncontended startup retries.
+                    try:
+                        cursor.execute("ROLLBACK")
+                    except sqlite3.OperationalError:
+                        pass
+                return False
+            raise
+
     @staticmethod
     def _fts_trigger_count(cursor: sqlite3.Cursor) -> int:
         placeholders = ",".join("?" for _ in _FTS_TRIGGERS)
@@ -1068,6 +1168,22 @@ def _fts_trigger_count(cursor: sqlite3.Cursor) -> int:
         ).fetchone()
         return int(row[0] if not isinstance(row, sqlite3.Row) else row[0])
 
+    @staticmethod
+    def _base_fts_trigger_count(cursor: sqlite3.Cursor) -> int:
+        """Count only the base messages_fts triggers (excludes trigram).
+
+        Base-trigger degradation must be detected independently of whether the
+        trigram triggers are present, otherwise a trigram-disabled DB with all
+        3 base triggers missing but 3 trigram triggers still around would tie
+        the 6-tuple total and skip the base rebuild."""
+        placeholders = ",".join("?" for _ in _BASE_FTS_TRIGGERS)
+        row = cursor.execute(
+            f"SELECT COUNT(*) FROM sqlite_master "
+            f"WHERE type = 'trigger' AND name IN ({placeholders})",
+            _BASE_FTS_TRIGGERS,
+        ).fetchone()
+        return int(row[0] if not isinstance(row, sqlite3.Row) else row[0])
+
     @staticmethod
     def _rebuild_fts_indexes(
         cursor: sqlite3.Cursor,
@@ -1494,18 +1610,26 @@ def _init_schema(self):
                                 "COALESCE(tool_calls, '') "
                                 "FROM messages"
                             )
-                        trigram_ok = self._ensure_fts_schema(
-                            cursor, "messages_fts_trigram", FTS_TRIGRAM_SQL
-                        )
-                        if trigram_ok:
-                            cursor.execute(
-                                "INSERT INTO messages_fts_trigram(rowid, content) "
-                                "SELECT id, "
-                                "COALESCE(content, '') || ' ' || "
-                                "COALESCE(tool_name, '') || ' ' || "
-                                "COALESCE(tool_calls, '') "
-                                "FROM messages"
+                        trigram_ok = False
+                        if self._trigram_fts_enabled():
+                            trigram_ok = self._ensure_fts_schema(
+                                cursor, "messages_fts_trigram", FTS_TRIGRAM_SQL
                             )
+                            if trigram_ok:
+                                cursor.execute(
+                                    "INSERT INTO messages_fts_trigram(rowid, content) "
+                                    "SELECT id, "
+                                    "COALESCE(content, '') || ' ' || "
+                                    "COALESCE(tool_name, '') || ' ' || "
+                                    "COALESCE(tool_calls, '') "
+                                    "FROM messages"
+                                )
+                        else:
+                            # Trigram disabled — don't create/backfill it here
+                            # only to drop it in the runtime FTS block below.
+                            # Saves the full trigram rebuild + WAL churn on the
+                            # opt-out startup. Also drop any pre-existing copy.
+                            self._drop_trigram_fts(cursor)
                         if not base_fts_ok:
                             fts_migrations_complete = False
                         # Track trigram availability for CJK LIKE fallback.
@@ -1572,22 +1696,59 @@ def _init_schema(self):
             # FTS5 setup. Run the DDL even when the virtual table exists so
             # CREATE TRIGGER IF NOT EXISTS repairs trigger-only degradation from
             # an earlier no-FTS5 runtime.
-            triggers_need_repair = self._fts_trigger_count(cursor) < len(_FTS_TRIGGERS)
+            trigram_wanted = self._trigram_fts_enabled()
+            # Detect trigger degradation on the base and trigram sets
+            # independently. Counting the 6-tuple total would let a healthy
+            # trigram-disabled DB (3 base + 0 trigram) tie a corrupt one
+            # (0 base + 3 stale trigram) and skip the base FTS rebuild.
+            base_triggers_ok = (
+                self._base_fts_trigger_count(cursor) == len(_BASE_FTS_TRIGGERS)
+            )
+            trigram_triggers_present = (
+                self._fts_trigger_count(cursor) - self._base_fts_trigger_count(cursor)
+            )
+            if trigram_wanted:
+                triggers_need_repair = (
+                    not base_triggers_ok
+                    or trigram_triggers_present < len(_TRIGRAM_FTS_TRIGGERS)
+                )
+            else:
+                # Trigram intentionally absent — only base health matters.
+                triggers_need_repair = not base_triggers_ok
             self._fts_enabled = self._ensure_fts_schema(cursor, "messages_fts", FTS_SQL)
 
             # Trigram FTS5 for CJK/substring search. This is optional relative
             # to the main FTS table; if it cannot be created, CJK search falls
-            # back to LIKE.
+            # back to LIKE. When sessions.trigram_fts is disabled, drop it
+            # instead of creating it — it is the largest object in state.db and
+            # fires on every message INSERT. Search transparently falls back to
+            # LIKE (see search_messages / self._trigram_available).
             if self._fts_enabled:
-                trigram_enabled = self._ensure_fts_schema(
-                    cursor, "messages_fts_trigram", FTS_TRIGRAM_SQL
-                )
-                self._trigram_available = trigram_enabled
-                if triggers_need_repair:
-                    self._rebuild_fts_indexes(
-                        cursor,
-                        include_trigram=trigram_enabled,
+                if trigram_wanted:
+                    trigram_enabled = self._ensure_fts_schema(
+                        cursor, "messages_fts_trigram", FTS_TRIGRAM_SQL
                     )
+                    self._trigram_available = trigram_enabled
+                    if triggers_need_repair:
+                        self._rebuild_fts_indexes(
+                            cursor,
+                            include_trigram=trigram_enabled,
+                        )
+                else:
+                    dropped = self._drop_trigram_fts(cursor)
+                    self._trigram_available = False
+                    if dropped:
+                        # The trigram index can be multiple GB. Dropping it
+                        # frees pages but SQLite won't shrink the file on disk
+                        # automatically — point the user at VACUUM.
+                        logger.info(
+                            "Dropped the trigram FTS index (sessions.trigram_fts "
+                            "disabled). Run VACUUM on state.db to reclaim the "
+                            "freed disk space."
+                        )
+                    if triggers_need_repair:
+                        # Base FTS only — trigram is intentionally absent.
+                        self._rebuild_fts_indexes(cursor, include_trigram=False)
 
         self._conn.commit()
 
@@ -4708,22 +4869,62 @@ def search_messages(
             if not _trigram_succeeded:
                 # Short / mixed CJK query, trigram unavailable, or trigram
                 # <3 CJK chars. Fall back to LIKE substring search.
-                # For multi-token OR queries (e.g. "广西 OR 桂林 OR 漓江"),
-                # build one LIKE condition per non-operator token so each term
-                # is matched independently (#20494).
-                non_op_tokens = [
-                    t for t in raw_query.split()
-                    if t.upper() not in {"AND", "OR", "NOT"}
-                ] or [raw_query]
-                token_clauses = []
+                # Preserve the query's boolean structure (#20494): walk the
+                # tokens left-to-right, honoring AND / OR / NOT operators so
+                # `A NOT B` excludes B and `A AND B` requires both — the flat
+                # OR-everything shape silently over-matched when this fallback
+                # became the only path for disabled-trigram installs.
                 like_params: list = []
-                for tok in non_op_tokens:
+
+                def _term_clause(tok: str) -> str:
                     esc = tok.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
-                    token_clauses.append(
-                        "(m.content LIKE ? ESCAPE '\\' OR m.tool_name LIKE ? ESCAPE '\\' OR m.tool_calls LIKE ? ESCAPE '\\')"
+                    like_params.extend([f"%{esc}%", f"%{esc}%", f"%{esc}%"])
+                    # COALESCE to '' so a NULL column yields FALSE (not NULL)
+                    # under LIKE — critical for the `AND NOT (...)` branch,
+                    # where `NOT (FALSE OR NULL)` would otherwise be NULL and
+                    # silently drop every row (tool_name/tool_calls are NULL on
+                    # plain user/assistant messages).
+                    return (
+                        "(COALESCE(m.content, '') LIKE ? ESCAPE '\\' "
+                        "OR COALESCE(m.tool_name, '') LIKE ? ESCAPE '\\' "
+                        "OR COALESCE(m.tool_calls, '') LIKE ? ESCAPE '\\')"
                     )
-                    like_params += [f"%{esc}%", f"%{esc}%", f"%{esc}%"]
-                like_where = [f"({' OR '.join(token_clauses)})"]
+
+                tokens = raw_query.split()
+                non_op_tokens = [
+                    t for t in tokens if t.upper() not in {"AND", "OR", "NOT"}
+                ] or [raw_query]
+                # Build the boolean expression. Adjacent terms without an
+                # explicit operator default to AND (FTS5 semantics). NOT
+                # attaches to the following term as `AND NOT (...)`.
+                expr_parts: list = []
+                pending_op = None  # None | "AND" | "OR" | "NOT"
+                for tok in tokens:
+                    upper = tok.upper()
+                    if upper in {"AND", "OR", "NOT"}:
+                        pending_op = upper
+                        continue
+                    clause = _term_clause(tok)
+                    if not expr_parts:
+                        expr_parts.append(clause)
+                    elif pending_op == "OR":
+                        expr_parts.append(f"OR {clause}")
+                    elif pending_op == "NOT":
+                        expr_parts.append(f"AND NOT {clause}")
+                    else:  # explicit AND or implicit adjacency
+                        expr_parts.append(f"AND {clause}")
+                    pending_op = None
+                if not expr_parts:
+                    # No CJK/word tokens survived (pure operators or empty) —
+                    # match the raw query as a single substring.
+                    expr_parts.append(_term_clause(raw_query))
+                like_where = [f"({' '.join(expr_parts)})"]
+                if not include_inactive:
+                    # Mirror the FTS/trigram paths: hide rewound/undo rows
+                    # (active=0, compacted=0) while keeping compaction-archived
+                    # rows discoverable. Without this the LIKE fallback leaked
+                    # deactivated messages for disabled-trigram installs.
+                    like_where.append("(m.active = 1 OR m.compacted = 1)")
                 if source_filter is not None:
                     like_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})")
                     like_params.extend(source_filter)