diff --git a/.env.example b/.env.example index 5c08a4acd639..0570a1e6587d 100644 --- a/.env.example +++ b/.env.example @@ -268,6 +268,17 @@ BROWSER_INACTIVITY_TIMEOUT=120 # Format: logs/session_YYYYMMDD_HHMMSS_UUID.json # Contains full conversation history in trajectory format for debugging/replay +# ============================================================================= +# SESSION DATABASE (state.db) +# ============================================================================= +# Set HERMES_DISABLE_FTS_TRIGRAM=1 to skip creating the trigram FTS5 index on +# fresh databases. The trigram index is only useful for CJK substring search +# (3+ characters); on instances that never run such queries it typically +# accounts for ~50% of state.db size. With this flag set, search_messages() +# falls back to LIKE for CJK queries. Existing databases can be reclaimed +# with SessionDB.drop_fts_trigram() — see issue #22478. +# HERMES_DISABLE_FTS_TRIGRAM=0 + # ============================================================================= # VOICE TRANSCRIPTION & OPENAI TTS # ============================================================================= diff --git a/hermes_state.py b/hermes_state.py index 913563f69b81..74b0bf016aaf 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -16,6 +16,7 @@ import json import logging +import os import random import re import sqlite3 @@ -29,6 +30,27 @@ logger = logging.getLogger(__name__) + +def _env_flag(name: str) -> bool: + """Return True when env var *name* is set to a truthy value. + + Recognizes ``1``, ``true``, ``yes``, ``on`` (case-insensitive). Any + other value — including the env var being unset — returns False. + """ + val = os.environ.get(name) + if val is None: + return False + return val.strip().lower() in {"1", "true", "yes", "on"} + + +def _fts_trigram_disabled() -> bool: + """The trigram FTS5 index doubles state.db size to support CJK substring + search. Pure-English deployments don't need it. Setting + ``HERMES_DISABLE_FTS_TRIGRAM=1`` skips creating the index and falls back + to ``LIKE`` for CJK queries. See issue #22478. + """ + return _env_flag("HERMES_DISABLE_FTS_TRIGRAM") + T = TypeVar("T") DEFAULT_DB_PATH = get_hermes_home() / "state.db" @@ -332,6 +354,9 @@ def __init__(self, db_path: Path = None): self._lock = threading.Lock() self._write_count = 0 + # Cached probe for the trigram FTS5 table — populated lazily by + # ``_has_fts_trigram()`` and invalidated by ``drop_fts_trigram()``. + self._fts_trigram_available: Optional[bool] = None try: self._conn = sqlite3.connect( str(self.db_path), @@ -442,6 +467,72 @@ def _try_wal_checkpoint(self) -> None: except Exception: pass # Best effort — never fatal. + def _has_fts_trigram(self) -> bool: + """Whether the trigram FTS5 table is present in this database. + + Cached after first probe. Invalidated by ``drop_fts_trigram()``. + ``search_messages`` uses this to route CJK queries to LIKE when + the index has been opted out via ``HERMES_DISABLE_FTS_TRIGRAM=1``. + """ + if self._fts_trigram_available is not None: + return self._fts_trigram_available + try: + with self._lock: + self._conn.execute( + "SELECT 1 FROM messages_fts_trigram LIMIT 0" + ) + self._fts_trigram_available = True + except sqlite3.OperationalError: + self._fts_trigram_available = False + return self._fts_trigram_available + + def drop_fts_trigram(self) -> None: + """Drop the trigram FTS5 index and its triggers, then VACUUM. + + Reclaims the ~half-of-state.db that the trigram index occupies on + instances that never run CJK substring searches. After this call, + ``search_messages`` falls back to LIKE for CJK queries with 3+ + characters, exactly as it does when ``HERMES_DISABLE_FTS_TRIGRAM=1`` + was set before the database was first created. Safe to call on a + database that has already had the index dropped — no-op in that + case. See issue #22478. + """ + def _drop(conn: sqlite3.Connection) -> None: + for trig in ( + "messages_fts_trigram_insert", + "messages_fts_trigram_delete", + "messages_fts_trigram_update", + ): + try: + conn.execute(f"DROP TRIGGER IF EXISTS {trig}") + except sqlite3.OperationalError: + pass + try: + conn.execute("DROP TABLE IF EXISTS messages_fts_trigram") + except sqlite3.OperationalError: + pass + + self._execute_write(_drop) + self._fts_trigram_available = False + # VACUUM cannot run inside a transaction. Run on the bare + # connection without our BEGIN IMMEDIATE wrapper. + try: + with self._lock: + self._conn.execute("VACUUM") + except sqlite3.OperationalError as exc: + logger.debug("VACUUM after drop_fts_trigram failed: %s", exc) + + def vacuum(self) -> None: + """Run SQLite ``VACUUM`` to reclaim free pages and defragment the + database file. Useful after large session deletions or after + ``drop_fts_trigram()``. Cannot run inside a transaction. + """ + try: + with self._lock: + self._conn.execute("VACUUM") + except sqlite3.OperationalError as exc: + logger.debug("VACUUM failed: %s", exc) + def close(self): """Close the database connection. @@ -584,11 +675,14 @@ def _init_schema(self): # backfills, index changes tied to a specific version step) stay # in a version-gated chain. Column additions are handled by # _reconcile_columns() above and no longer need entries here. - if current_version < 10: + if current_version < 10 and not _fts_trigram_disabled(): # v10: trigram FTS5 table for CJK/substring search. The # virtual table + triggers are created unconditionally via # FTS_TRIGRAM_SQL below, but existing rows need a one-time # backfill into the FTS index. + # + # When ``HERMES_DISABLE_FTS_TRIGRAM=1`` we skip the migration + # entirely; ``search_messages`` falls back to LIKE for CJK. try: cursor.execute("SELECT * FROM messages_fts_trigram LIMIT 0") _fts_trigram_exists = True @@ -607,14 +701,16 @@ def _init_schema(self): # overwrite, so we drop them explicitly and let the post-migration # existence checks (below) recreate them from FTS_SQL / # FTS_TRIGRAM_SQL, then backfill every message row. Fixes #16751. - for _trig in ( + _trigram_off = _fts_trigram_disabled() + _triggers_to_drop = [ "messages_fts_insert", "messages_fts_delete", "messages_fts_update", "messages_fts_trigram_insert", "messages_fts_trigram_delete", "messages_fts_trigram_update", - ): + ] + for _trig in _triggers_to_drop: try: cursor.execute(f"DROP TRIGGER IF EXISTS {_trig}") except sqlite3.OperationalError: @@ -627,7 +723,6 @@ def _init_schema(self): # Recreate virtual tables + triggers with the new inline-mode # schema that indexes content || tool_name || tool_calls. cursor.executescript(FTS_SQL) - cursor.executescript(FTS_TRIGRAM_SQL) # Backfill both indexes from every existing messages row. cursor.execute( "INSERT INTO messages_fts(rowid, content) " @@ -637,14 +732,16 @@ def _init_schema(self): "COALESCE(tool_calls, '') " "FROM messages" ) - cursor.execute( - "INSERT INTO messages_fts_trigram(rowid, content) " - "SELECT id, " - "COALESCE(content, '') || ' ' || " - "COALESCE(tool_name, '') || ' ' || " - "COALESCE(tool_calls, '') " - "FROM messages" - ) + if not _trigram_off: + cursor.executescript(FTS_TRIGRAM_SQL) + cursor.execute( + "INSERT INTO messages_fts_trigram(rowid, content) " + "SELECT id, " + "COALESCE(content, '') || ' ' || " + "COALESCE(tool_name, '') || ' ' || " + "COALESCE(tool_calls, '') " + "FROM messages" + ) if current_version < SCHEMA_VERSION: cursor.execute( "UPDATE schema_version SET version = ?", @@ -666,11 +763,17 @@ def _init_schema(self): except sqlite3.OperationalError: cursor.executescript(FTS_SQL) - # Trigram FTS5 for CJK/substring search - try: - cursor.execute("SELECT * FROM messages_fts_trigram LIMIT 0") - except sqlite3.OperationalError: - cursor.executescript(FTS_TRIGRAM_SQL) + # Trigram FTS5 for CJK/substring search. + # Skipped when ``HERMES_DISABLE_FTS_TRIGRAM=1`` — the trigram index + # roughly doubles state.db size on top of the porter index and is + # only useful for CJK substring queries with ≥3 characters. See + # issue #22478. ``search_messages`` falls back to LIKE when the + # table is absent. + if not _fts_trigram_disabled(): + try: + cursor.execute("SELECT * FROM messages_fts_trigram LIMIT 0") + except sqlite3.OperationalError: + cursor.executescript(FTS_TRIGRAM_SQL) self._conn.commit() @@ -1950,9 +2053,13 @@ def search_messages( # missing exact phrase matches. # # For queries with 3+ CJK characters, we use the trigram FTS5 table - # (indexed substring matching with ranking and snippets). For shorter - # CJK queries (1-2 chars), trigram can't match (it needs ≥9 UTF-8 - # bytes = 3 CJK chars), so we fall back to LIKE. + # (indexed substring matching with ranking and snippets) when it is + # available. When it has been opted out via + # ``HERMES_DISABLE_FTS_TRIGRAM=1`` (issue #22478) or dropped via + # ``drop_fts_trigram()``, we fall back to the LIKE substring path + # used for short queries below. For shorter CJK queries (1-2 chars), + # trigram can't match (it needs ≥9 UTF-8 bytes = 3 CJK chars), so we + # always use LIKE. is_cjk = self._contains_cjk(query) if is_cjk: raw_query = query.strip('"').strip() @@ -1970,7 +2077,10 @@ def search_messages( self._count_cjk(t) < 3 for t in _tokens_for_check ) - if cjk_count >= 3 and not _any_short_cjk: + # Also route to LIKE when the trigram FTS5 table is absent — + # opted out via ``HERMES_DISABLE_FTS_TRIGRAM=1`` (issue #22478) + # or removed via ``drop_fts_trigram()``. + if cjk_count >= 3 and not _any_short_cjk and self._has_fts_trigram(): # Trigram FTS5 path — quote each non-operator token to handle # FTS5 special chars (%, *, etc.) while preserving boolean # operators (AND, OR, NOT) for multi-term queries. @@ -2021,8 +2131,11 @@ def search_messages( else: matches = [dict(row) for row in tri_cursor.fetchall()] else: - # Short / mixed CJK query: trigram cannot match tokens with - # <3 CJK chars. Fall back to LIKE substring search. + # Short / mixed CJK query, or trigram FTS5 absent: trigram + # cannot match tokens with <3 CJK chars; trigram is also + # skipped when opted out via ``HERMES_DISABLE_FTS_TRIGRAM=1`` + # / removed via ``drop_fts_trigram()`` (issue #22478). Fall + # back to LIKE substring search. # For multi-token OR queries (e.g. "广西 OR 桂林 OR 漓江"), # build one LIKE condition per non-operator token so each term # is matched independently (#20494). diff --git a/scripts/release.py b/scripts/release.py index 65513af6abfb..fc417abfae4f 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -947,6 +947,7 @@ "zhicheng.han@mathematik.uni-goettingen.de": "hanzckernel", # PR #20311 (api-server approval events) "agentsmithlaor@gmail.com": "oferlaor", # PR #22356 salvage (cron origin sender identity) "jhin.lee@unity3d.com": "leehack", # PR #22053 salvage (telegram DM topic reply fallback) + "cotrelllucia@gmail.com": "cotrelllucia", # PR #22710 (optional FTS5 trigram index) # pander: empty email, salvaged via PR #19665 from #16126 by @ms-alan "ayman.a.kamal@hotmail.com": "A-kamal", # PR #18678 (xAI image resolution fix) } diff --git a/tests/test_hermes_state.py b/tests/test_hermes_state.py index 3bae763b9412..2a6b0627079f 100644 --- a/tests/test_hermes_state.py +++ b/tests/test_hermes_state.py @@ -1,5 +1,6 @@ """Tests for hermes_state.py — SessionDB SQLite CRUD, FTS5 search, export.""" +import os import time import pytest from pathlib import Path @@ -2943,3 +2944,138 @@ def test_v10_to_v11_upgrade_backfills_tool_fields(self, tmp_path): finally: session_db.close() + +# ========================================================================= +# Optional trigram FTS5 index — issue #22478 +# ========================================================================= + +class TestFTS5TrigramOptional: + """Regression tests for ``HERMES_DISABLE_FTS_TRIGRAM`` and + ``drop_fts_trigram()`` (issue #22478). + + The trigram FTS5 index roughly doubles state.db size and is only + useful for CJK substring search. Pure-English deployments can opt + out via the env var or by dropping the existing index. + """ + + @pytest.fixture() + def disabled_db(self, tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_DISABLE_FTS_TRIGRAM", "1") + db_path = tmp_path / "state_no_trigram.db" + session_db = SessionDB(db_path=db_path) + try: + yield session_db + finally: + session_db.close() + + @staticmethod + def _trigram_table_exists(session_db: SessionDB) -> bool: + row = session_db._conn.execute( + "SELECT name FROM sqlite_master " + "WHERE type='table' AND name='messages_fts_trigram'" + ).fetchone() + return row is not None + + def test_env_var_skips_trigram_table(self, disabled_db): + """``HERMES_DISABLE_FTS_TRIGRAM=1`` prevents the trigram virtual + table from being created on a fresh database.""" + assert not self._trigram_table_exists(disabled_db) + assert disabled_db._has_fts_trigram() is False + + def test_env_var_skips_trigram_triggers(self, disabled_db): + """No trigram triggers should exist when the index is opted out.""" + rows = disabled_db._conn.execute( + "SELECT name FROM sqlite_master " + "WHERE type='trigger' AND name LIKE 'messages_fts_trigram_%'" + ).fetchall() + assert rows == [] + + def test_porter_fts_still_works_when_trigram_disabled(self, disabled_db): + """English FTS path is unaffected by disabling the trigram index.""" + disabled_db.create_session(session_id="s1", source="cli") + disabled_db.append_message( + "s1", role="user", content="docker deployment notes" + ) + results = disabled_db.search_messages("deployment") + assert len(results) == 1 + assert results[0]["session_id"] == "s1" + + def test_cjk_search_falls_back_to_like_when_trigram_disabled( + self, disabled_db + ): + """CJK queries with ≥3 characters should fall back to LIKE rather + than crash when the trigram index is opted out.""" + disabled_db.create_session(session_id="s1", source="cli") + disabled_db.create_session(session_id="s2", source="cli") + disabled_db.append_message( + "s1", role="user", content="记忆系统已经设计完成" + ) + disabled_db.append_message( + "s2", role="user", content="今天的天气真好" + ) + results = disabled_db.search_messages("记忆系统") + assert len(results) == 1 + assert results[0]["session_id"] == "s1" + + def test_short_cjk_search_works_when_trigram_disabled(self, disabled_db): + """1-2 char CJK queries already use LIKE; that path must still work.""" + disabled_db.create_session(session_id="s1", source="cli") + disabled_db.append_message("s1", role="user", content="昨晚讨论了记忆系统") + results = disabled_db.search_messages("昨晚") + assert len(results) == 1 + + def test_inserts_dont_write_to_missing_trigram_table(self, disabled_db): + """When the trigram triggers don't exist, INSERTs must not fail.""" + disabled_db.create_session(session_id="s1", source="cli") + # Should not raise. + disabled_db.append_message("s1", role="user", content="hello world") + disabled_db.append_message( + "s1", role="assistant", content="", tool_name="web_search", + ) + results = disabled_db.search_messages("hello") + assert len(results) == 1 + + def test_drop_fts_trigram_removes_table_and_triggers(self, db): + """``drop_fts_trigram()`` removes the trigram table and its triggers.""" + # Sanity: the index exists by default. + assert self._trigram_table_exists(db) + db.drop_fts_trigram() + assert not self._trigram_table_exists(db) + rows = db._conn.execute( + "SELECT name FROM sqlite_master " + "WHERE type='trigger' AND name LIKE 'messages_fts_trigram_%'" + ).fetchall() + assert rows == [] + assert db._has_fts_trigram() is False + + def test_drop_fts_trigram_is_idempotent(self, disabled_db): + """Calling ``drop_fts_trigram()`` on a DB without the index is a no-op.""" + # Should not raise even though the table never existed. + disabled_db.drop_fts_trigram() + assert not self._trigram_table_exists(disabled_db) + + def test_search_after_drop_fts_trigram_routes_cjk_to_like(self, db): + """After dropping the trigram index, long CJK queries still return results + via the LIKE fallback.""" + db.create_session(session_id="s1", source="cli") + db.append_message("s1", role="user", content="项目管理系统的设计文档") + # Verify the trigram path works first. + results = db.search_messages("项目管理") + assert len(results) == 1 + + db.drop_fts_trigram() + + # After dropping, LIKE fallback handles the same CJK query. + results = db.search_messages("项目管理") + assert len(results) == 1 + assert results[0]["session_id"] == "s1" + + def test_vacuum_runs_without_error(self, db): + """``vacuum()`` should not raise on a small fresh database.""" + db.create_session(session_id="s1", source="cli") + db.append_message("s1", role="user", content="hello") + db.vacuum() # must not raise + # The DB should still be usable after VACUUM. + results = db.search_messages("hello") + assert len(results) == 1 + diff --git a/website/docs/developer-guide/session-storage.md b/website/docs/developer-guide/session-storage.md index 55da265595cd..4e0fa156ad2c 100644 --- a/website/docs/developer-guide/session-storage.md +++ b/website/docs/developer-guide/session-storage.md @@ -130,6 +130,27 @@ CREATE TRIGGER IF NOT EXISTS messages_fts_update AFTER UPDATE ON messages BEGIN END; ``` +### Disabling the trigram FTS5 index + +The `messages_fts_trigram` index is only used to serve CJK substring queries +with three or more characters. On instances that never run such queries it +typically accounts for **~50% of `state.db` size** because trigram tokens +expand more aggressively for CJK text than porter stemming does for English. +For deployments that don't need CJK substring search, the index can be +opted out: + +- **Fresh databases** — set `HERMES_DISABLE_FTS_TRIGRAM=1` before creating + `state.db`. The trigram virtual table and its triggers are skipped, and + `search_messages()` automatically falls back to `LIKE` for CJK queries. +- **Existing databases** — call `SessionDB.drop_fts_trigram()` to drop the + trigram table, its triggers, and run `VACUUM` to reclaim the freed pages. + After this, the database behaves as if it had been created with the env + var set. The operation is idempotent. + +Re-enabling the index later requires upgrading from a database that pre-dates +v10; the v10 backfill path will recreate it. (Manually re-creating the +trigram virtual table and re-indexing every message is also possible but +not exposed as a public API.) ## Schema Version and Migrations