From 52ce5c25940ec1bdb914cd1ce8373867622e16bf Mon Sep 17 00:00:00 2001 From: wanliqin <101301319+wanliqin@users.noreply.github.com> Date: Sun, 16 Aug 2026 17:41:46 +0800 Subject: [PATCH 1/4] fix(state): report delete, not wal, for the indeterminate journal-mode probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When _on_disk_journal_mode probing fails (most likely a concurrent opener holding a lock), _apply_delete_for_wal_reset_bug correctly leaves the journal mode alone but returned "wal". The caller sets _wal_active = apply_wal_with_fallback(...) == "wal", so _read_ctx enabled the lock-free mode=ro read pool. If the DB is actually in DELETE rollback-journal mode, readers skip the self._lock serialized path and hit raw SQLITE_BUSY during writes — random read failures for the instance's lifetime. Return "delete" for the indeterminate case instead. Claiming "delete" costs only queuing on the write lock (slow but correct); claiming "wal" costs unserialized read errors. Closes #86515 --- hermes_state.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hermes_state.py b/hermes_state.py index 834a56b91f1b2..9a8fadccf63e0 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -1318,7 +1318,12 @@ def _apply_delete_for_wal_reset_bug( "this process does not exclusively own" ) _log_wal_reset_bug_once(db_label, kept_wal=True, indeterminate=True) - return "wal" + # Report "delete", not "wal", for the indeterminate probe: claiming + # "wal" would set _wal_active=True and enable the lock-free read pool + # on a DB that may actually be in rollback-journal mode (raw + # SQLITE_BUSY on reads); claiming "delete" only costs queuing on the + # write lock — slow but correct. + return "delete" actual = "" try: From 2b211597df007156a71dcfb9ac2e7f5fb272ee3d Mon Sep 17 00:00:00 2001 From: wanliqin <101301319+wanliqin@users.noreply.github.com> Date: Sun, 16 Aug 2026 17:43:04 +0800 Subject: [PATCH 2/4] fix(state): route four bare-SELECT read paths through _read_ctx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_compression_lock_holder, the clear_session_activity_labels fast path, get_handoff_state, and list_pending_handoffs executed bare self._conn.execute(SELECT ...) on the instance-wide shared write connection. When another thread is inside _execute_write's BEGIN IMMEDIATE transaction, these SELECTs join that uncommitted transaction and can read rows that later roll back; in non-WAL (DELETE) mode they also collide with writers and surface false "no handoff" negatives. Route all four through _read_ctx, which degrades to the self._lock serialized path automatically when WAL is inactive — matching every other read path in the module. Closes #86516 --- hermes_state.py | 69 +++++++++++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 25 deletions(-) diff --git a/hermes_state.py b/hermes_state.py index 9a8fadccf63e0..6b8b4e44745c5 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -6668,11 +6668,15 @@ def get_compression_lock_holder(self, session_id: str) -> Optional[str]: if not session_id: return None now = time.time() - row = self._conn.execute( - "SELECT holder FROM compression_locks " - "WHERE session_id = ? AND expires_at >= ?", - (session_id, now), - ).fetchone() + # Read via _read_ctx (degrades to self._lock when WAL is inactive) — + # a bare self._conn read joins any in-flight BEGIN IMMEDIATE + # transaction and can see rows that later roll back. + with self._read_ctx() as conn: + row = conn.execute( + "SELECT holder FROM compression_locks " + "WHERE session_id = ? AND expires_at >= ?", + (session_id, now), + ).fetchone() if row is None: return None return row["holder"] if isinstance(row, sqlite3.Row) else row[0] @@ -6746,11 +6750,16 @@ def clear_session_activity_labels(self, session_id: str) -> None: # No-op fast path: skip the transaction when there is nothing to # clear. Read-only, no write lock. try: - row = self._conn.execute( - "SELECT last_activity_description, last_activity_provenance " - "FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() + # Read via _read_ctx (degrades to self._lock when WAL is + # inactive) — a bare self._conn read joins any in-flight + # BEGIN IMMEDIATE transaction and can see rows that later + # roll back. + with self._read_ctx() as conn: + row = conn.execute( + "SELECT last_activity_description, last_activity_provenance " + "FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() except sqlite3.Error: row = None if row is not None: @@ -12924,12 +12933,17 @@ def get_handoff_state(self, session_id: str) -> Optional[Dict[str, Any]]: no handoff record. """ try: - cur = self._conn.execute( - "SELECT handoff_state, handoff_platform, handoff_error " - "FROM sessions WHERE id = ?", - (session_id,), - ) - row = cur.fetchone() + # Read via _read_ctx (degrades to self._lock when WAL is + # inactive) — a bare self._conn read joins any in-flight + # BEGIN IMMEDIATE transaction and can see rows that later + # roll back. + with self._read_ctx() as conn: + cur = conn.execute( + "SELECT handoff_state, handoff_platform, handoff_error " + "FROM sessions WHERE id = ?", + (session_id,), + ) + row = cur.fetchone() if not row: return None return { @@ -12946,15 +12960,20 @@ def list_pending_handoffs(self) -> List[Dict[str, Any]]: Used by the gateway's handoff watcher. """ try: - cur = self._conn.execute( - "SELECT s.*, " - "COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved " - "FROM sessions s " - "LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash " - "WHERE s.handoff_state = 'pending' " - "ORDER BY s.started_at ASC" - ) - return [self._session_row_dict(r) for r in cur.fetchall()] + # Read via _read_ctx (degrades to self._lock when WAL is + # inactive) — a bare self._conn read joins any in-flight + # BEGIN IMMEDIATE transaction and can see rows that later + # roll back. + with self._read_ctx() as conn: + cur = conn.execute( + "SELECT s.*, " + "COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved " + "FROM sessions s " + "LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash " + "WHERE s.handoff_state = 'pending' " + "ORDER BY s.started_at ASC" + ) + return [self._session_row_dict(r) for r in cur.fetchall()] except Exception: return [] From c89248de32a6d2b89c37d9c9be51d0f95ea2dcfd Mon Sep 17 00:00:00 2001 From: wanliqin <101301319+wanliqin@users.noreply.github.com> Date: Sun, 16 Aug 2026 17:44:17 +0800 Subject: [PATCH 3/4] fix(state_search): reset writable_schema in a finally so a failed demote leaves no residue _demote_legacy_fts_to_trash._stage ran PRAGMA writable_schema=ON, the sqlite_master DELETE, then =RESET as a bare sequence inside _execute_write. writable_schema is a connection-level switch that does not roll back with the transaction, so if the DELETE raised (the demote path exists precisely for pathological DBs where sqlite_master may be inconsistent) the rollback left writable_schema=ON on the long-lived self._conn until process exit: subsequent schema parsing skips integrity checks, masking corruption is_malformed_db_error would otherwise catch. Wrap the DELETE in try/finally so RESET always runs. Closes #86517 --- hermes_state_search.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/hermes_state_search.py b/hermes_state_search.py index e8d29f413ee85..64a5be05bceaa 100644 --- a/hermes_state_search.py +++ b/hermes_state_search.py @@ -699,12 +699,18 @@ def _stage(conn): ).fetchone()) if had: conn.execute("PRAGMA writable_schema=ON") - conn.execute( - "DELETE FROM sqlite_master WHERE type = 'table' " - "AND name IN ('messages_fts', 'messages_fts_trigram') " - "AND sql LIKE 'CREATE VIRTUAL TABLE%'" - ) - conn.execute("PRAGMA writable_schema=RESET") + try: + conn.execute( + "DELETE FROM sqlite_master WHERE type = 'table' " + "AND name IN ('messages_fts', 'messages_fts_trigram') " + "AND sql LIKE 'CREATE VIRTUAL TABLE%'" + ) + finally: + # writable_schema is a connection-level switch that does + # not roll back with the transaction — RESET must run even + # when the DELETE raises, or the long-lived write + # connection stays degraded until process exit. + conn.execute("PRAGMA writable_schema=RESET") shadows = [ r[0] for r in conn.execute( "SELECT name FROM sqlite_master WHERE type = 'table' " From 7cc5afe6f9a47edc7d73264548b0af90abeaa85d Mon Sep 17 00:00:00 2001 From: wanliqin <101301319+wanliqin@users.noreply.github.com> Date: Sun, 16 Aug 2026 17:46:20 +0800 Subject: [PATCH 4/4] fix(state): destroy broken read-pool connections instead of returning them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _read_ctx's finally unconditionally returned the pooled mode=ro connection to the LifoPool, whether the query succeeded or raised. After the backing file is replaced or truncated (the scenario _reconnect_after_notadb's docstring lists: forked curator inheriting and closing the write fd, external repair pass), pooled read connections fail persistently — a truncated file raises 'file is not a database' on every query, and a POSIX rename-replace keeps the connection reading the old inode, silently returning stale data. LIFO order guarantees the next checkout gets the same broken connection, and _read_open_failed_at backoff only covers open failures, not query failures. The write connection has a one-shot reconnect self-heal; the read pool had none, so the fault persisted until process restart. On sqlite3.DatabaseError, destroy the connection via _close_read_conn (which also releases its descriptor permit) instead of returning it to the pool, so the next checkout reopens — the read-side counterpart of the write-side self-heal. Closes #86518 --- hermes_state.py | 50 +++++++++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/hermes_state.py b/hermes_state.py index 6b8b4e44745c5..ae6ac42241bf7 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -3672,28 +3672,42 @@ def _read_ctx(self): """ conn = self._checkout_read_conn() if conn is not None: + broken = False try: yield conn + except sqlite3.DatabaseError: + # A DatabaseError marks the pooled read connection as broken + # or stale (backing file replaced / truncated — the same + # scenario _reconnect_after_notadb self-heals on the write + # side). Destroy it instead of returning it to the pool so + # the next checkout reopens; otherwise the LIFO order hands + # the same broken connection to every subsequent query until + # process restart. + broken = True + raise finally: - returned = False - with self._read_conns_lock: - if not self._read_conns_closed: - try: - self._read_pool.put_nowait(conn) - returned = True - except queue.Full: - pass - if not returned: - # close() has already drained the pool, so this connection - # is surplus. Close it here — dropping it on the floor is - # what leaked the fd. - # - # queue.Full is now unreachable in practice (permits and - # maxsize are both _READ_POOL_MAX, so there can never be a - # ninth connection to return), but the branch stays: it is - # load-bearing if those two ever drift apart, and a leak is - # the failure mode it prevents. + if broken: self._close_read_conn(conn) + else: + returned = False + with self._read_conns_lock: + if not self._read_conns_closed: + try: + self._read_pool.put_nowait(conn) + returned = True + except queue.Full: + pass + if not returned: + # close() has already drained the pool, so this connection + # is surplus. Close it here — dropping it on the floor is + # what leaked the fd. + # + # queue.Full is now unreachable in practice (permits and + # maxsize are both _READ_POOL_MAX, so there can never be a + # ninth connection to return), but the branch stays: it is + # load-bearing if those two ever drift apart, and a leak is + # the failure mode it prevents. + self._close_read_conn(conn) return with self._lock: yield self._conn