diff --git a/hermes_state.py b/hermes_state.py index d8e4c8961249..1cfdee785e22 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -732,6 +732,8 @@ def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> Dict[str, A ON sessions(session_key, started_at DESC); CREATE INDEX IF NOT EXISTS idx_sessions_gateway_peer ON sessions(source, user_id, chat_id, chat_type, thread_id, started_at DESC); +CREATE INDEX IF NOT EXISTS idx_sessions_handoff_state + ON sessions(handoff_state, started_at); """ FTS_SQL = """ @@ -812,6 +814,16 @@ class SessionDB: _WRITE_RETRY_MAX_S = 0.150 # 150ms # Attempt a PASSIVE WAL checkpoint every N successful writes. _CHECKPOINT_EVERY_N_WRITES = 50 + # Merge fragmented FTS5 segments every N successful writes. The message + # triggers append one segment per insert; left unmaintained these grow + # into tens of thousands of segments, so every MATCH must scan them all + # and every insert pays a growing automerge cost — which lengthens the + # write-lock hold time and starves competing writers (gateway + cron + # processes share one state.db), surfacing as "database is locked". + # 'optimize' is a no-op once the index is already merged, so an idle DB + # pays almost nothing; the cadence is deliberately coarse so the one-off + # merge cost is amortised far below the checkpoint cadence. + _OPTIMIZE_EVERY_N_WRITES = 1000 def __init__(self, db_path: Path = None, read_only: bool = False): self.db_path = db_path or DEFAULT_DB_PATH @@ -1080,10 +1092,12 @@ def _execute_write(self, fn: Callable[[sqlite3.Connection], T]) -> T: except Exception: pass raise - # Success — periodic best-effort checkpoint. + # Success — periodic best-effort checkpoint + FTS merge. self._write_count += 1 if self._write_count % self._CHECKPOINT_EVERY_N_WRITES == 0: self._try_wal_checkpoint() + if self._write_count % self._OPTIMIZE_EVERY_N_WRITES == 0: + self._try_optimize_fts() return result except sqlite3.OperationalError as exc: err_msg = str(exc).lower() @@ -1134,6 +1148,22 @@ def _try_wal_checkpoint(self) -> None: except Exception: pass # Best effort — never fatal. + def _try_optimize_fts(self) -> None: + """Best-effort FTS5 segment merge. Never raises. + + Runs on the ``_OPTIMIZE_EVERY_N_WRITES`` cadence from the write hot + path (off the lock — ``optimize_fts`` re-acquires ``self._lock`` + itself, mirroring ``_try_wal_checkpoint``). ``read_only`` connections + never reach the write path, so this is implicitly skipped for them. + Once the index is merged the 'optimize' command is close to free, so + the steady-state cost is negligible; the expensive case is only the + first merge of a long-neglected index. + """ + try: + self.optimize_fts() + except Exception: + pass # Best effort — never fatal. + def close(self): """Close the database connection. diff --git a/scripts/release.py b/scripts/release.py index 2505af4f2944..d6b41ce1782d 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -68,6 +68,7 @@ "66773372+Tranquil-Flow@users.noreply.github.com": "Tranquil-Flow", # PR #52623 salvage (auxiliary Anthropic base_url host validation; #52608) "65363919+coygeek@users.noreply.github.com": "coygeek", # PR #37735 salvage (redact provider error text at api-server HTTP boundary; #37733) "moonsong@nousresearch.local": "Tranquil-Flow", # PR #52623 salvage (auxiliary Anthropic base_url host validation; #52608) + "baris@writeme.com": "isair", # PR #50124 salvage (periodic FTS5 segment merge to curb write-lock contention; #54752) "140971685+Dr1985@users.noreply.github.com": "Dr1985", # PR #42567 salvage (launchd supervision detection + status reporting; #42524) "8180647+herbalizer404@users.noreply.github.com": "herbalizer404", # PR #49076 + #51835 salvage (auxiliary compression fallback: 403/session-usage payment errors + honor fallback chain when aux provider auth unavailable) "pyxl-dev@users.noreply.github.com": "pyxl-dev", # PR #52230 salvage (include rate-limit in auxiliary capacity-error fallback gate; #52228) diff --git a/tests/test_hermes_state.py b/tests/test_hermes_state.py index 6b623b2ba39e..4f1e54ab945a 100644 --- a/tests/test_hermes_state.py +++ b/tests/test_hermes_state.py @@ -3779,6 +3779,40 @@ def test_optimize_idempotent(self, db): # Search still works after repeated optimization. assert len(db.search_messages("repeat")) == 1 + def test_write_path_optimizes_fts_on_cadence(self, db, monkeypatch): + """Writes periodically merge FTS segments so they never accumulate + into the tens-of-thousands that lengthen the write-lock hold and + starve competing writers ("database is locked").""" + db._OPTIMIZE_EVERY_N_WRITES = 5 + calls = {"n": 0} + real_optimize = db.optimize_fts + + def _counting_optimize(): + calls["n"] += 1 + return real_optimize() + + monkeypatch.setattr(db, "optimize_fts", _counting_optimize) + # create_session is write #1; appends are #2.. -> #5 and #10 trigger. + db.create_session(session_id="s1", source="cli") + for i in range(9): + db.append_message(session_id="s1", role="user", content=f"needle {i}") + assert calls["n"] == 2 + # The auto-merge is layout-only: search is unaffected. + assert len(db.search_messages("needle")) == 9 + + def test_write_path_optimize_failure_never_breaks_write(self, db, monkeypatch): + """A failing periodic optimize must not fail the surrounding write.""" + db._OPTIMIZE_EVERY_N_WRITES = 2 + + def _boom(): + raise sqlite3.OperationalError("simulated optimize failure") + + monkeypatch.setattr(db, "optimize_fts", _boom) + db.create_session(session_id="s1", source="cli") # write #1 + # write #2 trips the cadence; the swallowed failure must not propagate. + db.append_message(session_id="s1", role="user", content="still persists") + assert len(db.get_messages("s1")) == 1 + class TestAutoMaintenance: def _make_old_ended(self, db, sid: str, days_old: int = 100):