From fb76cd7d7bb0e035eef8ff775abb44a398fbdecb Mon Sep 17 00:00:00 2001 From: jaylfc Date: Sun, 7 Jun 2026 03:43:02 +0100 Subject: [PATCH 1/2] fix(audit): enable SQLite WAL + busy_timeout on all connections (#101) --- taosmd/_db.py | 39 +++++++++++++++++++++++++++++++++++++ taosmd/access_tracker.py | 4 +++- taosmd/archive.py | 4 +++- taosmd/browsing_history.py | 4 +++- taosmd/crystallize.py | 4 +++- taosmd/knowledge_graph.py | 4 +++- taosmd/pending_decisions.py | 4 +++- taosmd/reflect.py | 4 +++- taosmd/session_catalog.py | 4 ++-- taosmd/taosmd_backend.py | 6 +++--- taosmd/vector_memory.py | 4 +++- 11 files changed, 68 insertions(+), 13 deletions(-) create mode 100644 taosmd/_db.py diff --git a/taosmd/_db.py b/taosmd/_db.py new file mode 100644 index 00000000..f43a748d --- /dev/null +++ b/taosmd/_db.py @@ -0,0 +1,39 @@ +"""Shared SQLite connection helper. + +Every persistent store in the package opens its own SQLite database. By +default ``sqlite3.connect`` uses the rollback-journal mode, which takes an +exclusive lock for the duration of a write and lets only a single writer +touch the file at a time. When several agents (or several processes on the +same machine) share a memory store, that serialisation surfaces as +``SQLITE_BUSY`` errors. + +``connect`` centralises the fix: it enables write-ahead logging (WAL), which +lets readers proceed concurrently with a writer, and sets a busy timeout so a +contended writer waits and retries rather than failing immediately. Both are +plain PRAGMAs with no extra dependencies; standalone behaviour is unchanged +apart from the journal mode. +""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path +from typing import Union + +# Allow a contended connection to block-and-retry for this many milliseconds +# before raising ``sqlite3.OperationalError: database is locked``. +BUSY_TIMEOUT_MS = 5000 + + +def connect(db_path: Union[str, Path]) -> sqlite3.Connection: + """Open a SQLite connection in WAL mode with a busy timeout. + + Drop-in replacement for ``sqlite3.connect(db_path)``. Callers that need a + ``row_factory`` or other connection attributes should set them on the + returned connection as before. + """ + conn = sqlite3.connect(db_path) + # WAL is harmless (silently stays "memory") on ``:memory:`` databases. + conn.execute("PRAGMA journal_mode=WAL") + conn.execute(f"PRAGMA busy_timeout={BUSY_TIMEOUT_MS}") + return conn diff --git a/taosmd/access_tracker.py b/taosmd/access_tracker.py index 9ebd7a9a..fd838e12 100644 --- a/taosmd/access_tracker.py +++ b/taosmd/access_tracker.py @@ -15,6 +15,8 @@ import time from pathlib import Path +from . import _db + SCHEMA = """ CREATE TABLE IF NOT EXISTS access_log ( memory_key TEXT NOT NULL, @@ -49,7 +51,7 @@ def __init__(self, db_path: str | Path = "data/access-tracker.db"): async def init(self) -> None: Path(self._db_path).parent.mkdir(parents=True, exist_ok=True) - self._conn = sqlite3.connect(self._db_path) + self._conn = _db.connect(self._db_path) self._conn.row_factory = sqlite3.Row self._conn.executescript(SCHEMA) self._conn.commit() diff --git a/taosmd/archive.py b/taosmd/archive.py index 8ac511a8..2ef626c9 100644 --- a/taosmd/archive.py +++ b/taosmd/archive.py @@ -21,6 +21,8 @@ from pathlib import Path from typing import Any +from . import _db + logger = logging.getLogger(__name__) # Event types @@ -95,7 +97,7 @@ def __init__( async def init(self) -> None: self._archive_dir.mkdir(parents=True, exist_ok=True) Path(self._index_path).parent.mkdir(parents=True, exist_ok=True) - self._conn = sqlite3.connect(self._index_path) + self._conn = _db.connect(self._index_path) self._conn.row_factory = sqlite3.Row self._conn.executescript(INDEX_SCHEMA) self._conn.commit() diff --git a/taosmd/browsing_history.py b/taosmd/browsing_history.py index b04a21f6..9bfad5a1 100644 --- a/taosmd/browsing_history.py +++ b/taosmd/browsing_history.py @@ -10,6 +10,8 @@ import time from pathlib import Path +from . import _db + SCHEMA = """ CREATE TABLE IF NOT EXISTS browsing_history ( url TEXT PRIMARY KEY, @@ -34,7 +36,7 @@ def __init__(self, db_path: str | Path = "data/browsing-history.db"): async def init(self) -> None: Path(self._db_path).parent.mkdir(parents=True, exist_ok=True) - self._conn = sqlite3.connect(self._db_path) + self._conn = _db.connect(self._db_path) self._conn.row_factory = sqlite3.Row self._conn.executescript(SCHEMA) self._conn.commit() diff --git a/taosmd/crystallize.py b/taosmd/crystallize.py index c512c736..2d60c44b 100644 --- a/taosmd/crystallize.py +++ b/taosmd/crystallize.py @@ -18,6 +18,8 @@ from pathlib import Path from typing import TYPE_CHECKING +from . import _db + if TYPE_CHECKING: from .knowledge_graph import TemporalKnowledgeGraph @@ -67,7 +69,7 @@ def __init__(self, db_path: str | Path = "data/crystals.db"): async def init(self) -> None: Path(self._db_path).parent.mkdir(parents=True, exist_ok=True) - self._conn = sqlite3.connect(self._db_path) + self._conn = _db.connect(self._db_path) self._conn.row_factory = sqlite3.Row self._conn.executescript(SCHEMA) self._conn.commit() diff --git a/taosmd/knowledge_graph.py b/taosmd/knowledge_graph.py index f56279e8..aba4af7b 100644 --- a/taosmd/knowledge_graph.py +++ b/taosmd/knowledge_graph.py @@ -15,6 +15,8 @@ import time from pathlib import Path +from . import _db + SCHEMA = """ CREATE TABLE IF NOT EXISTS kg_entities ( id TEXT PRIMARY KEY, @@ -80,7 +82,7 @@ def __init__(self, db_path: str | Path = "data/knowledge-graph.db"): async def init(self) -> None: Path(self._db_path).parent.mkdir(parents=True, exist_ok=True) - self._conn = sqlite3.connect(self._db_path) + self._conn = _db.connect(self._db_path) self._conn.row_factory = sqlite3.Row self._conn.executescript(SCHEMA) self._conn.commit() diff --git a/taosmd/pending_decisions.py b/taosmd/pending_decisions.py index 8608ce13..68505bf0 100644 --- a/taosmd/pending_decisions.py +++ b/taosmd/pending_decisions.py @@ -36,6 +36,8 @@ from pathlib import Path from typing import Any +from . import _db + SCHEMA = """ CREATE TABLE IF NOT EXISTS kg_pending_decisions ( @@ -87,7 +89,7 @@ def __init__(self, db_path: str | Path): async def init(self) -> None: Path(self._db_path).parent.mkdir(parents=True, exist_ok=True) - self._conn = sqlite3.connect(self._db_path) + self._conn = _db.connect(self._db_path) self._conn.row_factory = sqlite3.Row self._conn.executescript(SCHEMA) self._conn.commit() diff --git a/taosmd/reflect.py b/taosmd/reflect.py index 6b5b4092..c70106c5 100644 --- a/taosmd/reflect.py +++ b/taosmd/reflect.py @@ -18,6 +18,8 @@ from pathlib import Path from typing import TYPE_CHECKING +from . import _db + if TYPE_CHECKING: from .knowledge_graph import TemporalKnowledgeGraph @@ -134,7 +136,7 @@ def __init__(self, db_path: str | Path = "data/insights.db"): async def init(self) -> None: Path(self._db_path).parent.mkdir(parents=True, exist_ok=True) - self._conn = sqlite3.connect(self._db_path) + self._conn = _db.connect(self._db_path) self._conn.row_factory = sqlite3.Row self._conn.executescript(SCHEMA) self._conn.commit() diff --git a/taosmd/session_catalog.py b/taosmd/session_catalog.py index f5a2da8e..2f6e584a 100644 --- a/taosmd/session_catalog.py +++ b/taosmd/session_catalog.py @@ -28,7 +28,7 @@ import httpx -from taosmd import prompts +from taosmd import _db, prompts logger = logging.getLogger(__name__) @@ -143,7 +143,7 @@ def __init__( async def init(self) -> None: Path(self._db_path).parent.mkdir(parents=True, exist_ok=True) - self._conn = sqlite3.connect(self._db_path) + self._conn = _db.connect(self._db_path) self._conn.row_factory = sqlite3.Row self._conn.executescript(SCHEMA) self._conn.commit() diff --git a/taosmd/taosmd_backend.py b/taosmd/taosmd_backend.py index ce76c55e..b6d58757 100644 --- a/taosmd/taosmd_backend.py +++ b/taosmd/taosmd_backend.py @@ -12,7 +12,7 @@ from pathlib import Path from typing import Any -from . import __version__ +from . import __version__, _db from .backend import MemoryBackend SETTINGS_SCHEMA = """ @@ -82,7 +82,7 @@ def __init__( async def init(self) -> None: """Initialise settings database.""" Path(self._settings_db_path).parent.mkdir(parents=True, exist_ok=True) - self._conn = sqlite3.connect(self._settings_db_path) + self._conn = _db.connect(self._settings_db_path) self._conn.row_factory = sqlite3.Row self._conn.executescript(SETTINGS_SCHEMA + AGENT_CONFIG_SCHEMA) self._conn.commit() @@ -260,7 +260,7 @@ def _ensure_conn(self) -> None: """Ensure the settings DB connection is open (auto-init if needed).""" if self._conn is None: Path(self._settings_db_path).parent.mkdir(parents=True, exist_ok=True) - self._conn = sqlite3.connect(self._settings_db_path) + self._conn = _db.connect(self._settings_db_path) self._conn.row_factory = sqlite3.Row self._conn.executescript(SETTINGS_SCHEMA + AGENT_CONFIG_SCHEMA) self._conn.commit() diff --git a/taosmd/vector_memory.py b/taosmd/vector_memory.py index 231b178a..29208f3a 100644 --- a/taosmd/vector_memory.py +++ b/taosmd/vector_memory.py @@ -15,6 +15,8 @@ import time from pathlib import Path +from . import _db + logger = logging.getLogger(__name__) SCHEMA = """ @@ -90,7 +92,7 @@ def __init__( async def init(self, http_client=None) -> None: Path(self._db_path).parent.mkdir(parents=True, exist_ok=True) - self._conn = sqlite3.connect(self._db_path) + self._conn = _db.connect(self._db_path) self._conn.row_factory = sqlite3.Row self._conn.executescript(SCHEMA) self._conn.commit() From 81139f67334434d60f58766dac4133e0c1e7f76a Mon Sep 17 00:00:00 2001 From: jaylfc Date: Sun, 7 Jun 2026 04:48:49 +0100 Subject: [PATCH 2/2] address kilo review: surface WAL fallback + guard double-init leak --- taosmd/_db.py | 26 +++++++++++++++++++++++--- taosmd/taosmd_backend.py | 3 +++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/taosmd/_db.py b/taosmd/_db.py index f43a748d..3a9f122d 100644 --- a/taosmd/_db.py +++ b/taosmd/_db.py @@ -17,6 +17,7 @@ from __future__ import annotations import sqlite3 +import warnings from pathlib import Path from typing import Union @@ -33,7 +34,26 @@ def connect(db_path: Union[str, Path]) -> sqlite3.Connection: returned connection as before. """ conn = sqlite3.connect(db_path) - # WAL is harmless (silently stays "memory") on ``:memory:`` databases. - conn.execute("PRAGMA journal_mode=WAL") - conn.execute(f"PRAGMA busy_timeout={BUSY_TIMEOUT_MS}") + # ``PRAGMA journal_mode`` echoes the journal mode actually in effect. WAL + # can silently refuse to engage on filesystems without shared-memory/mmap + # support (notably some network mounts), where it falls back to the prior + # rollback journal. ``:memory:`` databases report "memory". We read the + # result so the fallback is observable rather than silent; the connection + # stays fully usable either way, so we deliberately do not raise. + row = conn.execute("PRAGMA journal_mode=WAL").fetchone() + mode = (row[0] if row else "") or "" + # The connection is fully usable whichever journal mode took effect, so we + # do not raise on a fallback. We surface it as a warning instead of letting + # it pass silently; ``:memory:`` legitimately reports "memory". + if mode.lower() not in ("wal", "memory"): + warnings.warn( + f"SQLite WAL mode not enabled for {db_path!r} " + f"(journal_mode={mode!r}); concurrent access may hit " + "'database is locked'.", + RuntimeWarning, + stacklevel=2, + ) + # busy_timeout takes an integer literal; SQLite does not allow bound + # parameters in PRAGMA statements, and the value is an internal constant. + conn.execute(f"PRAGMA busy_timeout={int(BUSY_TIMEOUT_MS)}") return conn diff --git a/taosmd/taosmd_backend.py b/taosmd/taosmd_backend.py index b6d58757..eeefa055 100644 --- a/taosmd/taosmd_backend.py +++ b/taosmd/taosmd_backend.py @@ -81,6 +81,9 @@ def __init__( async def init(self) -> None: """Initialise settings database.""" + # Guard against re-init leaking an already-open connection. + if self._conn is not None: + return Path(self._settings_db_path).parent.mkdir(parents=True, exist_ok=True) self._conn = _db.connect(self._settings_db_path) self._conn.row_factory = sqlite3.Row