From 69c92f427c1ecc5eaf83f6d9e87bb4a48af5295d Mon Sep 17 00:00:00 2001 From: Baris Sencan Date: Sun, 21 Jun 2026 12:35:49 +0100 Subject: [PATCH 1/3] fix(state): periodically merge FTS5 segments to curb write-lock contention The message triggers append one FTS5 segment per insert into both the porter and trigram indexes. Nothing ever called the existing optimize_fts() maintenance helper, so on a long-lived state.db these segments accumulate without bound (observed: ~34k trigram segments for ~27k messages). Every MATCH then has to scan all segments, and every insert pays a growing automerge cost that lengthens the WAL write-lock hold time. Because the gateway and cron agents are separate processes sharing one state.db, those longer holds exhaust the 1s-timeout x 15-retry budget in _execute_write and surface as repeated: Session DB creation failed (will retry next turn): database is locked Session DB append_message failed: database is locked Wire optimize_fts() into the write path on a coarse cadence (_OPTIMIZE_EVERY_N_WRITES = 1000), alongside the existing every-50-writes checkpoint. 'optimize' is effectively free once the index is already merged, so steady-state cost is negligible; only the first merge of a neglected index is expensive. The call is best-effort and never fails the surrounding write. Tests: cadence fires on the write path; a failing optimize never breaks the write. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit 583647b56e207a9b0accfd05efa2b9b251630984) --- hermes_state.py | 30 +++++++++++++++++++++++++++++- tests/test_hermes_state.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/hermes_state.py b/hermes_state.py index d8e4c8961249..667749ee238e 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -812,6 +812,16 @@ class SessionDB: _WRITE_RETRY_MAX_S = 0.150 # 150ms # Attempt a PASSIVE WAL checkpoint every N successful writes. _CHECKPOINT_EVERY_N_WRITES = 50 + # Merge fragmented FTS5 segments every N successful writes. The message + # triggers append one segment per insert; left unmaintained these grow + # into tens of thousands of segments, so every MATCH must scan them all + # and every insert pays a growing automerge cost — which lengthens the + # write-lock hold time and starves competing writers (gateway + cron + # processes share one state.db), surfacing as "database is locked". + # 'optimize' is a no-op once the index is already merged, so an idle DB + # pays almost nothing; the cadence is deliberately coarse so the one-off + # merge cost is amortised far below the checkpoint cadence. + _OPTIMIZE_EVERY_N_WRITES = 1000 def __init__(self, db_path: Path = None, read_only: bool = False): self.db_path = db_path or DEFAULT_DB_PATH @@ -1080,10 +1090,12 @@ def _execute_write(self, fn: Callable[[sqlite3.Connection], T]) -> T: except Exception: pass raise - # Success — periodic best-effort checkpoint. + # Success — periodic best-effort checkpoint + FTS merge. self._write_count += 1 if self._write_count % self._CHECKPOINT_EVERY_N_WRITES == 0: self._try_wal_checkpoint() + if self._write_count % self._OPTIMIZE_EVERY_N_WRITES == 0: + self._try_optimize_fts() return result except sqlite3.OperationalError as exc: err_msg = str(exc).lower() @@ -1134,6 +1146,22 @@ def _try_wal_checkpoint(self) -> None: except Exception: pass # Best effort — never fatal. + def _try_optimize_fts(self) -> None: + """Best-effort FTS5 segment merge. Never raises. + + Runs on the ``_OPTIMIZE_EVERY_N_WRITES`` cadence from the write hot + path (off the lock — ``optimize_fts`` re-acquires ``self._lock`` + itself, mirroring ``_try_wal_checkpoint``). ``read_only`` connections + never reach the write path, so this is implicitly skipped for them. + Once the index is merged the 'optimize' command is close to free, so + the steady-state cost is negligible; the expensive case is only the + first merge of a long-neglected index. + """ + try: + self.optimize_fts() + except Exception: + pass # Best effort — never fatal. + def close(self): """Close the database connection. diff --git a/tests/test_hermes_state.py b/tests/test_hermes_state.py index 6b623b2ba39e..4f1e54ab945a 100644 --- a/tests/test_hermes_state.py +++ b/tests/test_hermes_state.py @@ -3779,6 +3779,40 @@ def test_optimize_idempotent(self, db): # Search still works after repeated optimization. assert len(db.search_messages("repeat")) == 1 + def test_write_path_optimizes_fts_on_cadence(self, db, monkeypatch): + """Writes periodically merge FTS segments so they never accumulate + into the tens-of-thousands that lengthen the write-lock hold and + starve competing writers ("database is locked").""" + db._OPTIMIZE_EVERY_N_WRITES = 5 + calls = {"n": 0} + real_optimize = db.optimize_fts + + def _counting_optimize(): + calls["n"] += 1 + return real_optimize() + + monkeypatch.setattr(db, "optimize_fts", _counting_optimize) + # create_session is write #1; appends are #2.. -> #5 and #10 trigger. + db.create_session(session_id="s1", source="cli") + for i in range(9): + db.append_message(session_id="s1", role="user", content=f"needle {i}") + assert calls["n"] == 2 + # The auto-merge is layout-only: search is unaffected. + assert len(db.search_messages("needle")) == 9 + + def test_write_path_optimize_failure_never_breaks_write(self, db, monkeypatch): + """A failing periodic optimize must not fail the surrounding write.""" + db._OPTIMIZE_EVERY_N_WRITES = 2 + + def _boom(): + raise sqlite3.OperationalError("simulated optimize failure") + + monkeypatch.setattr(db, "optimize_fts", _boom) + db.create_session(session_id="s1", source="cli") # write #1 + # write #2 trips the cadence; the swallowed failure must not propagate. + db.append_message(session_id="s1", role="user", content="still persists") + assert len(db.get_messages("s1")) == 1 + class TestAutoMaintenance: def _make_old_ended(self, db, sid: str, days_old: int = 100): From d78a4df5613627fe59cbba6c067743b4e1e79d93 Mon Sep 17 00:00:00 2001 From: kenyonxu Date: Thu, 11 Jun 2026 13:06:19 +0800 Subject: [PATCH 2/3] fix(gateway): move handoff_state index to DEFERRED_INDEX_SQL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The index references the handoff_state column which is added by _reconcile_columns() on legacy databases. Placing it in SCHEMA_SQL causes 'no such column' errors during schema migration tests because SCHEMA_SQL runs before reconciliation. Move to DEFERRED_INDEX_SQL which runs after _reconcile_columns() — matching the existing pattern used by idx_messages_session_active. Refs: #43504, #40695 (cherry picked from commit 40ecd61d4993754e077a2bdf0c68707cd2add5f4) --- hermes_state.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hermes_state.py b/hermes_state.py index 667749ee238e..1cfdee785e22 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -732,6 +732,8 @@ def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> Dict[str, A ON sessions(session_key, started_at DESC); CREATE INDEX IF NOT EXISTS idx_sessions_gateway_peer ON sessions(source, user_id, chat_id, chat_type, thread_id, started_at DESC); +CREATE INDEX IF NOT EXISTS idx_sessions_handoff_state + ON sessions(handoff_state, started_at); """ FTS_SQL = """ From 25f51e9214fe8c68f0d02e1bc82a9fa68779c7c7 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Mon, 29 Jun 2026 14:43:39 +0530 Subject: [PATCH 3/3] chore(attribution): map baris@writeme.com -> isair for salvaged #50124 --- scripts/release.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/release.py b/scripts/release.py index 2505af4f2944..d6b41ce1782d 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -68,6 +68,7 @@ "66773372+Tranquil-Flow@users.noreply.github.com": "Tranquil-Flow", # PR #52623 salvage (auxiliary Anthropic base_url host validation; #52608) "65363919+coygeek@users.noreply.github.com": "coygeek", # PR #37735 salvage (redact provider error text at api-server HTTP boundary; #37733) "moonsong@nousresearch.local": "Tranquil-Flow", # PR #52623 salvage (auxiliary Anthropic base_url host validation; #52608) + "baris@writeme.com": "isair", # PR #50124 salvage (periodic FTS5 segment merge to curb write-lock contention; #54752) "140971685+Dr1985@users.noreply.github.com": "Dr1985", # PR #42567 salvage (launchd supervision detection + status reporting; #42524) "8180647+herbalizer404@users.noreply.github.com": "herbalizer404", # PR #49076 + #51835 salvage (auxiliary compression fallback: 403/session-usage payment errors + honor fallback chain when aux provider auth unavailable) "pyxl-dev@users.noreply.github.com": "pyxl-dev", # PR #52230 salvage (include rate-limit in auxiliary capacity-error fallback gate; #52228)