diff --git a/hermes_state.py b/hermes_state.py index 80ab6dc10115b..b15f0b23026cd 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -140,7 +140,7 @@ def _delete_delegate_children(conn, parent_ids: List[str]) -> List[str]: DEFAULT_DB_PATH = get_hermes_home() / "state.db" -SCHEMA_VERSION = 20 +SCHEMA_VERSION = 21 # Cap on user-controlled FTS5 query input before regex/sanitizer processing. # Search queries do not need to be arbitrarily large, and bounding them keeps @@ -856,25 +856,31 @@ def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> Dict[str, A FTS_SQL = """ CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( - content + content, + tool_name, + tool_calls, + content='messages', + content_rowid='id' ); CREATE TRIGGER IF NOT EXISTS messages_fts_insert AFTER INSERT ON messages BEGIN - INSERT INTO messages_fts(rowid, content) VALUES ( - new.id, - COALESCE(new.content, '') || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '') + INSERT INTO messages_fts(rowid, content, tool_name, tool_calls) VALUES ( + new.id, new.content, new.tool_name, new.tool_calls ); END; CREATE TRIGGER IF NOT EXISTS messages_fts_delete AFTER DELETE ON messages BEGIN - DELETE FROM messages_fts WHERE rowid = old.id; + INSERT INTO messages_fts(messages_fts, rowid, content, tool_name, tool_calls) VALUES ( + 'delete', old.id, old.content, old.tool_name, old.tool_calls + ); END; CREATE TRIGGER IF NOT EXISTS messages_fts_update AFTER UPDATE ON messages BEGIN - DELETE FROM messages_fts WHERE rowid = old.id; - INSERT INTO messages_fts(rowid, content) VALUES ( - new.id, - COALESCE(new.content, '') || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '') + INSERT INTO messages_fts(messages_fts, rowid, content, tool_name, tool_calls) VALUES ( + 'delete', old.id, old.content, old.tool_name, old.tool_calls + ); + INSERT INTO messages_fts(rowid, content, tool_name, tool_calls) VALUES ( + new.id, new.content, new.tool_name, new.tool_calls ); END; """ @@ -886,25 +892,31 @@ def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> Dict[str, A FTS_TRIGRAM_SQL = """ CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_trigram USING fts5( content, + tool_name, + tool_calls, + content='messages', + content_rowid='id', tokenize='trigram' ); CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_insert AFTER INSERT ON messages BEGIN - INSERT INTO messages_fts_trigram(rowid, content) VALUES ( - new.id, - COALESCE(new.content, '') || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '') + INSERT INTO messages_fts_trigram(rowid, content, tool_name, tool_calls) VALUES ( + new.id, new.content, new.tool_name, new.tool_calls ); END; CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_delete AFTER DELETE ON messages BEGIN - DELETE FROM messages_fts_trigram WHERE rowid = old.id; + INSERT INTO messages_fts_trigram(messages_fts_trigram, rowid, content, tool_name, tool_calls) VALUES ( + 'delete', old.id, old.content, old.tool_name, old.tool_calls + ); END; CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_update AFTER UPDATE ON messages BEGIN - DELETE FROM messages_fts_trigram WHERE rowid = old.id; - INSERT INTO messages_fts_trigram(rowid, content) VALUES ( - new.id, - COALESCE(new.content, '') || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '') + INSERT INTO messages_fts_trigram(messages_fts_trigram, rowid, content, tool_name, tool_calls) VALUES ( + 'delete', old.id, old.content, old.tool_name, old.tool_calls + ); + INSERT INTO messages_fts_trigram(rowid, content, tool_name, tool_calls) VALUES ( + new.id, new.content, new.tool_name, new.tool_calls ); END; """ @@ -1116,25 +1128,13 @@ def _rebuild_fts_indexes( *, include_trigram: bool = True, ) -> None: - cursor.execute("DELETE FROM messages_fts") - cursor.execute( - "INSERT INTO messages_fts(rowid, content) " - "SELECT id, " - "COALESCE(content, '') || ' ' || " - "COALESCE(tool_name, '') || ' ' || " - "COALESCE(tool_calls, '') " - "FROM messages" - ) + # External-content FTS5 tables read source columns from messages. + # The special rebuild command repopulates index data from that backing table. + cursor.execute("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')") if not include_trigram: return - cursor.execute("DELETE FROM messages_fts_trigram") cursor.execute( - "INSERT INTO messages_fts_trigram(rowid, content) " - "SELECT id, " - "COALESCE(content, '') || ' ' || " - "COALESCE(tool_name, '') || ' ' || " - "COALESCE(tool_calls, '') " - "FROM messages" + "INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('rebuild')" ) def _fts_table_probe(self, cursor: sqlite3.Cursor, table_name: str) -> Optional[bool]: @@ -1497,17 +1497,22 @@ def _init_schema(self): else: fts_migrations_complete = False if current_version < 11: - # v11: re-index FTS5 tables to cover tool_name + tool_calls and - # switch from external-content to inline mode. Existing DBs have - # old-schema FTS tables and triggers that IF NOT EXISTS won't - # overwrite, so we drop them explicitly and let the post-migration - # existence checks (below) recreate them from FTS_SQL / - # FTS_TRIGRAM_SQL, then backfill every message row. Fixes #16751. + # v11: re-index FTS5 tables to cover tool_name + tool_calls. + # Existing DBs have old-schema FTS tables and triggers that IF + # NOT EXISTS won't overwrite, so drop them explicitly and let + # the post-migration existence checks recreate them from the + # current FTS_SQL / FTS_TRIGRAM_SQL definitions. Fixes #16751. if fts5_available: self._drop_fts_triggers(cursor) - for _tbl in ("messages_fts", "messages_fts_trigram"): + for _tbl, _drop_sql in ( + ("messages_fts", "DROP TABLE IF EXISTS messages_fts"), + ( + "messages_fts_trigram", + "DROP TABLE IF EXISTS messages_fts_trigram", + ), + ): try: - cursor.execute(f"DROP TABLE IF EXISTS {_tbl}") + cursor.execute(_drop_sql) except sqlite3.OperationalError as exc: if not self._is_fts5_unavailable_error(exc): raise @@ -1520,35 +1525,22 @@ def _init_schema(self): break if fts5_available: - # Recreate virtual tables + triggers with the new inline-mode - # schema that indexes content || tool_name || tool_calls. + # Recreate virtual tables + triggers with the current + # schema that indexes content, tool_name, and tool_calls. # Handle base and trigram independently — a missing # trigram tokenizer should not prevent base FTS backfill. base_fts_ok = self._ensure_fts_schema( cursor, "messages_fts", FTS_SQL ) - if base_fts_ok: - cursor.execute( - "INSERT INTO messages_fts(rowid, content) " - "SELECT id, " - "COALESCE(content, '') || ' ' || " - "COALESCE(tool_name, '') || ' ' || " - "COALESCE(tool_calls, '') " - "FROM messages" - ) trigram_ok = self._ensure_fts_schema( cursor, "messages_fts_trigram", FTS_TRIGRAM_SQL ) - if trigram_ok: - cursor.execute( - "INSERT INTO messages_fts_trigram(rowid, content) " - "SELECT id, " - "COALESCE(content, '') || ' ' || " - "COALESCE(tool_name, '') || ' ' || " - "COALESCE(tool_calls, '') " - "FROM messages" + if base_fts_ok: + self._rebuild_fts_indexes( + cursor, + include_trigram=trigram_ok, ) - if not base_fts_ok: + else: fts_migrations_complete = False # Track trigram availability for CJK LIKE fallback. self._trigram_available = trigram_ok @@ -1639,6 +1631,102 @@ def _init_schema(self): ) except sqlite3.OperationalError: pass + if current_version < 21: + # v21: switch from inline FTS5 to external-content FTS5. + # The previous inline mode duplicated indexed content in the FTS + # tables. External-content mode keeps only index data in FTS + # tables; content is read from the messages table on demand. + # + # Operational notes: + # - The messages table is the source of truth and is never + # mutated here; this migration only rebuilds derived indexes. + # - Disk space becomes reusable inside SQLite immediately, but + # the file is only returned to the filesystem after VACUUM. + # - Users who need to roll back should restore a pre-upgrade + # state.db backup; this migration intentionally updates + # derived FTS tables and schema_version. + # - Phrase queries no longer match across content/tool_name/ + # tool_calls column boundaries. That is intentional: v12's + # cross-column phrase matches were an artifact of string + # concatenation, not meaningful message text. + if fts5_available: + schema_rows = cursor.execute( + "SELECT name, sql FROM sqlite_master " + "WHERE type = 'table' AND name IN " + "('messages_fts', 'messages_fts_trigram')" + ).fetchall() + fts_schema = { + (row["name"] if isinstance(row, sqlite3.Row) else row[0]): + (row["sql"] if isinstance(row, sqlite3.Row) else row[1]) + for row in schema_rows + } + + def _uses_v21_external_schema(sql: str | None) -> bool: + normalized = "".join((sql or "").lower().split()) + return all( + marker in normalized + for marker in ( + "content='messages'", + "content_rowid='id'", + "tool_name", + "tool_calls", + ) + ) + + base_is_current = _uses_v21_external_schema( + fts_schema.get("messages_fts") + ) + trigram_sql = fts_schema.get("messages_fts_trigram") + trigram_is_current_or_absent = ( + trigram_sql is None + or _uses_v21_external_schema(trigram_sql) + ) + + if not (base_is_current and trigram_is_current_or_absent): + self._drop_fts_triggers(cursor) + for _tbl, _drop_sql in ( + ("messages_fts", "DROP TABLE IF EXISTS messages_fts"), + ( + "messages_fts_trigram", + "DROP TABLE IF EXISTS messages_fts_trigram", + ), + ): + try: + cursor.execute(_drop_sql) + except sqlite3.OperationalError as exc: + if not self._is_fts5_unavailable_error(exc): + raise + if self._is_trigram_unavailable_error(exc): + self._warn_trigram_unavailable(exc) + # An old inline trigram table remains. Keep + # the previous schema version so a runtime + # with trigram support retries the migration. + fts_migrations_complete = False + else: + self._warn_fts5_unavailable(exc) + fts5_available = False + fts_migrations_complete = False + break + + if fts5_available: + base_fts_ok = self._ensure_fts_schema( + cursor, "messages_fts", FTS_SQL + ) + trigram_ok = self._ensure_fts_schema( + cursor, "messages_fts_trigram", FTS_TRIGRAM_SQL + ) + if base_fts_ok: + self._rebuild_fts_indexes( + cursor, + include_trigram=bool(trigram_ok), + ) + else: + fts_migrations_complete = False + self._trigram_available = bool(trigram_ok) + else: + fts_migrations_complete = False + else: + fts_migrations_complete = False if current_version < SCHEMA_VERSION and fts_migrations_complete: cursor.execute( "UPDATE schema_version SET version = ?", @@ -4855,7 +4943,7 @@ def search_messages( m.id, m.session_id, m.role, - snippet(messages_fts, 0, '>>>', '<<<', '...', 40) AS snippet, + snippet(messages_fts, -1, '>>>', '<<<', '...', 40) AS snippet, m.content, m.timestamp, m.tool_name, @@ -4927,7 +5015,7 @@ def search_messages( m.id, m.session_id, m.role, - snippet(messages_fts_trigram, 0, '>>>', '<<<', '...', 40) AS snippet, + snippet(messages_fts_trigram, -1, '>>>', '<<<', '...', 40) AS snippet, m.content, m.timestamp, m.tool_name, diff --git a/tests/test_hermes_state.py b/tests/test_hermes_state.py index ada5d12c86a74..707711f8a5b58 100644 --- a/tests/test_hermes_state.py +++ b/tests/test_hermes_state.py @@ -66,6 +66,107 @@ def cursor(self, factory=None): return super().cursor(factory or _NoTrigramCursor) +class _NoTrigramExistingTableCursor(_NoTrigramCursor): + """Simulate an existing trigram table that cannot be opened or dropped.""" + + def execute(self, sql, parameters=()): + probe = sql.strip() + if probe in ( + "DROP TABLE IF EXISTS messages_fts_trigram", + "SELECT * FROM messages_fts_trigram LIMIT 0", + ): + raise sqlite3.OperationalError("no such tokenizer: trigram") + return super().execute(sql, parameters) + + +class _NoTrigramExistingTableConnection(sqlite3.Connection): + def cursor(self, factory=None): + return super().cursor(factory or _NoTrigramExistingTableCursor) + + +def _replace_fts_with_v20_inline_schema(db_path): + """Replace current FTS objects with the inline schema used by v20.""" + conn = sqlite3.connect(db_path) + try: + conn.executescript( + """ + DROP TRIGGER IF EXISTS messages_fts_insert; + DROP TRIGGER IF EXISTS messages_fts_delete; + DROP TRIGGER IF EXISTS messages_fts_update; + DROP TRIGGER IF EXISTS messages_fts_trigram_insert; + DROP TRIGGER IF EXISTS messages_fts_trigram_delete; + DROP TRIGGER IF EXISTS messages_fts_trigram_update; + DROP TABLE IF EXISTS messages_fts; + DROP TABLE IF EXISTS messages_fts_trigram; + + CREATE VIRTUAL TABLE messages_fts USING fts5(content); + CREATE VIRTUAL TABLE messages_fts_trigram USING fts5( + content, + tokenize='trigram' + ); + + CREATE TRIGGER messages_fts_insert AFTER INSERT ON messages BEGIN + INSERT INTO messages_fts(rowid, content) VALUES ( + new.id, + COALESCE(new.content, '') || ' ' || + COALESCE(new.tool_name, '') || ' ' || + COALESCE(new.tool_calls, '') + ); + END; + CREATE TRIGGER messages_fts_delete AFTER DELETE ON messages BEGIN + DELETE FROM messages_fts WHERE rowid = old.id; + END; + CREATE TRIGGER messages_fts_update AFTER UPDATE ON messages BEGIN + DELETE FROM messages_fts WHERE rowid = old.id; + INSERT INTO messages_fts(rowid, content) VALUES ( + new.id, + COALESCE(new.content, '') || ' ' || + COALESCE(new.tool_name, '') || ' ' || + COALESCE(new.tool_calls, '') + ); + END; + + CREATE TRIGGER messages_fts_trigram_insert AFTER INSERT ON messages BEGIN + INSERT INTO messages_fts_trigram(rowid, content) VALUES ( + new.id, + COALESCE(new.content, '') || ' ' || + COALESCE(new.tool_name, '') || ' ' || + COALESCE(new.tool_calls, '') + ); + END; + CREATE TRIGGER messages_fts_trigram_delete AFTER DELETE ON messages BEGIN + DELETE FROM messages_fts_trigram WHERE rowid = old.id; + END; + CREATE TRIGGER messages_fts_trigram_update AFTER UPDATE ON messages BEGIN + DELETE FROM messages_fts_trigram WHERE rowid = old.id; + INSERT INTO messages_fts_trigram(rowid, content) VALUES ( + new.id, + COALESCE(new.content, '') || ' ' || + COALESCE(new.tool_name, '') || ' ' || + COALESCE(new.tool_calls, '') + ); + END; + + INSERT INTO messages_fts(rowid, content) + SELECT id, + COALESCE(content, '') || ' ' || + COALESCE(tool_name, '') || ' ' || + COALESCE(tool_calls, '') + FROM messages; + INSERT INTO messages_fts_trigram(rowid, content) + SELECT id, + COALESCE(content, '') || ' ' || + COALESCE(tool_name, '') || ' ' || + COALESCE(tool_calls, '') + FROM messages; + UPDATE schema_version SET version = 20; + """ + ) + conn.commit() + finally: + conn.close() + + @pytest.fixture() def db(tmp_path): """Create a SessionDB with a temp database file.""" @@ -789,6 +890,67 @@ def connect_without_trigram(*args, **kwargs): finally: db.close() + def test_v21_migration_retries_inline_trigram_when_tokenizer_returns( + self, tmp_path, monkeypatch + ): + """Do not finalize v21 while an existing inline trigram table remains.""" + db_path = tmp_path / "state.db" + seeded = SessionDB(db_path=db_path) + seeded.create_session(session_id="s1", source="cli") + seeded.append_message( + "s1", + role="user", + content="legacy trigram migration 大别山项目", + tool_name="read_file", + ) + seeded.close() + _replace_fts_with_v20_inline_schema(db_path) + + real_connect = sqlite3.connect + + def connect_without_existing_trigram(*args, **kwargs): + kwargs["factory"] = _NoTrigramExistingTableConnection + return real_connect(*args, **kwargs) + + monkeypatch.setattr( + "hermes_state.sqlite3.connect", connect_without_existing_trigram + ) + without_trigram = SessionDB(db_path=db_path) + try: + version = without_trigram._conn.execute( + "SELECT version FROM schema_version" + ).fetchone()[0] + assert version == 20 + assert without_trigram._fts_enabled is True + assert without_trigram._trigram_available is False + assert len(without_trigram.search_messages("legacy")) == 1 + finally: + without_trigram.close() + + monkeypatch.setattr("hermes_state.sqlite3.connect", real_connect) + restored = SessionDB(db_path=db_path) + try: + version = restored._conn.execute( + "SELECT version FROM schema_version" + ).fetchone()[0] + assert version == 21 + trigram_sql = restored._conn.execute( + "SELECT sql FROM sqlite_master " + "WHERE name = 'messages_fts_trigram'" + ).fetchone()[0] + assert "content='messages'" in trigram_sql + + restored.append_message( + "s1", + role="assistant", + content="post migration write", + tool_name="write_file", + ) + assert len(restored.search_messages("write_file")) == 1 + assert len(restored.search_messages("大别山项目")) == 1 + finally: + restored.close() + def test_v11_migration_backfills_base_fts_when_trigram_unavailable( self, tmp_path, monkeypatch ): @@ -1512,6 +1674,54 @@ def test_search_finds_content(self, db): snippets = [r.get("snippet", "") for r in results] assert any("docker" in s.lower() or "Docker" in s for s in snippets) + def test_external_content_fts_tracks_tool_fields_and_message_lifecycle(self, db): + """FTS triggers keep external-content indexes in sync for all indexed columns.""" + db.create_session(session_id="s1", source="cli") + db.append_message( + "s1", + role="assistant", + content="initial searchable payload", + tool_name="alphatool", + tool_calls='{"name":"alphacall"}', + ) + + assert len(db.search_messages("initial")) == 1 + alpha_tool_results = db.search_messages("alphatool") + assert len(alpha_tool_results) == 1 + assert "alphatool" in alpha_tool_results[0]["snippet"] + alpha_call_results = db.search_messages("alphacall") + assert len(alpha_call_results) == 1 + assert "alphacall" in alpha_call_results[0]["snippet"] + + with db._lock: + row = db._conn.execute( + "SELECT id FROM messages WHERE session_id = ?", ("s1",) + ).fetchone() + message_id = row["id"] if hasattr(row, "keys") else row[0] + db._conn.execute( + "UPDATE messages SET content = ?, tool_name = ?, tool_calls = ? WHERE id = ?", + ( + "updated searchable payload", + "betatool", + '{"name":"betacall"}', + message_id, + ), + ) + + assert db.search_messages("initial") == [] + assert db.search_messages("alphatool") == [] + assert db.search_messages("alphacall") == [] + assert len(db.search_messages("updated")) == 1 + assert len(db.search_messages("betatool")) == 1 + assert len(db.search_messages("betacall")) == 1 + + with db._lock: + db._conn.execute("DELETE FROM messages WHERE id = ?", (message_id,)) + + assert db.search_messages("updated") == [] + assert db.search_messages("betatool") == [] + assert db.search_messages("betacall") == [] + def test_search_empty_query(self, db): assert db.search_messages("") == [] assert db.search_messages(" ") == [] @@ -3022,6 +3232,195 @@ def test_schema_version(self, db): version = cursor.fetchone()[0] assert version == SCHEMA_VERSION + def test_v12_inline_fts_migrates_to_v21_external_content(self, tmp_path): + """Upgrade inline FTS indexes to the v21 external-content schema.""" + import sqlite3 + + old_db = tmp_path / "v12.db" + conn = sqlite3.connect(old_db) + conn.executescript( + """ + CREATE TABLE schema_version (version INTEGER NOT NULL); + INSERT INTO schema_version VALUES (12); + CREATE TABLE sessions ( + id TEXT PRIMARY KEY, + source TEXT NOT NULL, + user_id TEXT, + model TEXT, + model_config TEXT, + system_prompt TEXT, + parent_session_id TEXT, + started_at REAL NOT NULL, + ended_at REAL, + end_reason TEXT, + message_count INTEGER DEFAULT 0, + tool_call_count INTEGER DEFAULT 0, + input_tokens INTEGER DEFAULT 0, + output_tokens INTEGER DEFAULT 0, + cache_read_tokens INTEGER DEFAULT 0, + cache_write_tokens INTEGER DEFAULT 0, + reasoning_tokens INTEGER DEFAULT 0, + billing_provider TEXT, + billing_base_url TEXT, + billing_mode TEXT, + estimated_cost_usd REAL, + actual_cost_usd REAL, + cost_status TEXT, + cost_source TEXT, + pricing_version TEXT, + title TEXT, + api_call_count INTEGER DEFAULT 0, + FOREIGN KEY (parent_session_id) REFERENCES sessions(id) + ); + CREATE TABLE messages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + session_id TEXT NOT NULL REFERENCES sessions(id), + role TEXT NOT NULL, + content TEXT, + tool_call_id TEXT, + tool_calls TEXT, + tool_name TEXT, + timestamp REAL NOT NULL, + token_count INTEGER, + finish_reason TEXT, + reasoning TEXT, + reasoning_content TEXT, + reasoning_details TEXT, + codex_reasoning_items TEXT, + codex_message_items TEXT + ); + CREATE VIRTUAL TABLE messages_fts USING fts5(content); + CREATE VIRTUAL TABLE messages_fts_trigram USING fts5(content, tokenize='trigram'); + INSERT INTO sessions (id, source, model, started_at, message_count) + VALUES ('s1', 'cli', 'test-model', 1700000000, 1); + INSERT INTO messages (session_id, role, content, tool_calls, tool_name, timestamp) + VALUES ('s1', 'user', 'needle 大别山项目', '[{"name":"read_file"}]', 'read_file', 1700000001); + INSERT INTO messages_fts(rowid, content) + SELECT id, COALESCE(content, '') || ' ' || COALESCE(tool_name, '') || ' ' || COALESCE(tool_calls, '') + FROM messages; + INSERT INTO messages_fts_trigram(rowid, content) + SELECT id, COALESCE(content, '') || ' ' || COALESCE(tool_name, '') || ' ' || COALESCE(tool_calls, '') + FROM messages; + """ + ) + conn.close() + + db = SessionDB(db_path=old_db) + + from hermes_state import SCHEMA_VERSION + + version = db._conn.execute("SELECT version FROM schema_version").fetchone()[0] + assert version == SCHEMA_VERSION + fts_sql = db._conn.execute( + "SELECT sql FROM sqlite_master WHERE name='messages_fts'" + ).fetchone()[0] + trigram_sql = db._conn.execute( + "SELECT sql FROM sqlite_master WHERE name='messages_fts_trigram'" + ).fetchone()[0] + assert "content='messages'" in fts_sql + assert "content_rowid='id'" in fts_sql + assert "tool_name" in fts_sql and "tool_calls" in fts_sql + assert "content='messages'" in trigram_sql + assert "tokenize='trigram'" in trigram_sql + + assert db.search_messages("needle", limit=5)[0]["session_id"] == "s1" + assert db.search_messages("read_file", limit=5)[0]["tool_name"] == "read_file" + assert db.search_messages("大别山项目", limit=5)[0]["session_id"] == "s1" + + db.append_message("s1", "assistant", "new indexed message", tool_name="write_file") + assert db.search_messages("write_file", limit=5)[0]["tool_name"] == "write_file" + db.close() + + def test_v20_inline_fts_migrates_to_v21_external_content(self, tmp_path): + """Upgrade the immediately preceding inline schema to v21.""" + db_path = tmp_path / "v20.db" + seeded = SessionDB(db_path=db_path) + seeded.create_session(session_id="s1", source="cli") + seeded.append_message( + "s1", + role="user", + content="v20 migration needle 大别山项目", + tool_name="read_file", + tool_calls='[{"name":"read_file"}]', + ) + seeded.close() + _replace_fts_with_v20_inline_schema(db_path) + + migrated = SessionDB(db_path=db_path) + try: + version = migrated._conn.execute( + "SELECT version FROM schema_version" + ).fetchone()[0] + assert version == 21 + + for table_name in ("messages_fts", "messages_fts_trigram"): + table_sql = migrated._conn.execute( + "SELECT sql FROM sqlite_master WHERE name = ?", + (table_name,), + ).fetchone()[0] + assert "content='messages'" in table_sql + assert "content_rowid='id'" in table_sql + assert "tool_name" in table_sql + assert "tool_calls" in table_sql + + shadow_tables = migrated._conn.execute( + "SELECT name FROM sqlite_master WHERE name IN " + "('messages_fts_content', 'messages_fts_trigram_content')" + ).fetchall() + assert shadow_tables == [] + + assert len(migrated.search_messages("needle")) == 1 + assert len(migrated.search_messages("read_file")) == 1 + assert len(migrated.search_messages("大别山项目")) == 1 + migrated.append_message( + "s1", + role="assistant", + content="post migration message", + tool_name="write_file", + ) + assert len(migrated.search_messages("write_file")) == 1 + finally: + migrated.close() + + def test_v21_skips_rebuild_when_fts_is_already_external( + self, tmp_path, monkeypatch + ): + """An already-current FTS layout should only need the version bump.""" + db_path = tmp_path / "already_external.db" + seeded = SessionDB(db_path=db_path) + seeded.create_session(session_id="s1", source="cli") + seeded.append_message("s1", role="user", content="already external") + seeded.close() + + conn = sqlite3.connect(db_path) + conn.execute("UPDATE schema_version SET version = 20") + conn.commit() + conn.close() + + rebuild_calls = 0 + original_rebuild = SessionDB._rebuild_fts_indexes + + def track_rebuild(cursor, *, include_trigram=True): + nonlocal rebuild_calls + rebuild_calls += 1 + return original_rebuild(cursor, include_trigram=include_trigram) + + monkeypatch.setattr( + SessionDB, + "_rebuild_fts_indexes", + staticmethod(track_rebuild), + ) + reopened = SessionDB(db_path=db_path) + try: + version = reopened._conn.execute( + "SELECT version FROM schema_version" + ).fetchone()[0] + assert version == 21 + assert rebuild_calls == 0 + assert len(reopened.search_messages("external")) == 1 + finally: + reopened.close() + def test_title_column_exists(self, db): """Verify the title column was created in the sessions table.""" cursor = db._conn.execute("PRAGMA table_info(sessions)") @@ -3312,7 +3711,7 @@ def test_migration_from_v2(self, tmp_path): conn.commit() conn.close() - # Open with SessionDB — should migrate to v9 + # Open with SessionDB — should migrate through the latest schema. migrated_db = SessionDB(db_path=db_path) # Verify migration @@ -4797,7 +5196,7 @@ def test_v10_to_v11_upgrade_backfills_tool_fields(self, tmp_path): "v11 migration must backfill tool_name into FTS" assert len(session_db.search_messages("LEGACYARG")) == 1, \ "v11 migration must backfill tool_calls JSON into FTS" - # schema_version bumped + # schema_version bumped through the latest schema. from hermes_state import SCHEMA_VERSION row = session_db._conn.execute( "SELECT version FROM schema_version LIMIT 1" diff --git a/website/docs/developer-guide/session-storage.md b/website/docs/developer-guide/session-storage.md index 55da265595cde..737ef4a8ec87a 100644 --- a/website/docs/developer-guide/session-storage.md +++ b/website/docs/developer-guide/session-storage.md @@ -105,35 +105,58 @@ Notes: ```sql CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( content, - content=messages, - content_rowid=id + tool_name, + tool_calls, + content='messages', + content_rowid='id' ); ``` -The FTS5 table is kept in sync via three triggers that fire on INSERT, UPDATE, +The base and trigram FTS5 tables use `messages` as external content, so SQLite +stores index data without keeping an additional private copy of the three +indexed columns in each FTS table. `messages` remains the source of truth. +Queries match all three columns. Phrase queries do not span column boundaries; +for example, a phrase cannot begin in `content` and continue in `tool_name`. + +The FTS5 tables are kept in sync via three triggers that fire on INSERT, UPDATE, and DELETE of the `messages` table: ```sql CREATE TRIGGER IF NOT EXISTS messages_fts_insert AFTER INSERT ON messages BEGIN - INSERT INTO messages_fts(rowid, content) VALUES (new.id, new.content); + INSERT INTO messages_fts(rowid, content, tool_name, tool_calls) VALUES ( + new.id, new.content, new.tool_name, new.tool_calls + ); END; CREATE TRIGGER IF NOT EXISTS messages_fts_delete AFTER DELETE ON messages BEGIN - INSERT INTO messages_fts(messages_fts, rowid, content) - VALUES('delete', old.id, old.content); + INSERT INTO messages_fts( + messages_fts, rowid, content, tool_name, tool_calls + ) VALUES ( + 'delete', old.id, old.content, old.tool_name, old.tool_calls + ); END; CREATE TRIGGER IF NOT EXISTS messages_fts_update AFTER UPDATE ON messages BEGIN - INSERT INTO messages_fts(messages_fts, rowid, content) - VALUES('delete', old.id, old.content); - INSERT INTO messages_fts(rowid, content) VALUES (new.id, new.content); + INSERT INTO messages_fts( + messages_fts, rowid, content, tool_name, tool_calls + ) VALUES ( + 'delete', old.id, old.content, old.tool_name, old.tool_calls + ); + INSERT INTO messages_fts(rowid, content, tool_name, tool_calls) VALUES ( + new.id, new.content, new.tool_name, new.tool_calls + ); END; ``` +The trigram table uses the same external-content columns and trigger pattern, +with `tokenize='trigram'` for CJK and substring search. Rebuilding either table +uses FTS5's special `rebuild` command so the index is repopulated from +`messages`. + ## Schema Version and Migrations -Current schema version: **11** +Current schema version: **21** The `schema_version` table stores a single integer. Simple column additions are handled declaratively by `_reconcile_columns()` (which diffs live columns against `SCHEMA_SQL` and ADDs any missing ones). The version-gated chain is reserved for data migrations and index/FTS changes that can't be expressed declaratively: @@ -150,6 +173,7 @@ The `schema_version` table stores a single integer. Simple column additions are | 9 | Add `codex_message_items` column to messages for Codex Responses message id/phase replay | | 10 | Add `messages_fts_trigram` virtual table (trigram tokenizer for CJK / substring search) and backfill existing rows | | 11 | Re-index `messages_fts` and `messages_fts_trigram` to cover `tool_name` + `tool_calls` and switch from external-content to inline mode; drop old triggers and backfill every message row | +| 21 | Switch both FTS tables to three-column external-content mode backed by `messages`; rebuild derived indexes without duplicating message text in FTS content shadow tables | Declarative column adds use `ALTER TABLE ADD COLUMN` wrapped in try/except to handle the column-already-exists case (idempotent). The version number is bumped after each successful migration block.