fix(sqlite): fall back to journal_mode=DELETE on NFS/SMB/FUSE (fixes /resume on network-mounted HERMES_HOME)

cli.py

@@ -5463,7 +5463,8 @@ def _handle_resume_command(self, cmd_original: str) -> None:
             return
 
         if not self._session_db:
-            _cprint("  Session database not available.")
+            from hermes_state import format_session_db_unavailable
+            _cprint(f"  {format_session_db_unavailable()}")
             return
 
         # Resolve title or ID
@@ -5574,7 +5575,8 @@ def _handle_branch_command(self, cmd_original: str) -> None:
             return
 
         if not self._session_db:
-            _cprint("  Session database not available.")
+            from hermes_state import format_session_db_unavailable
+            _cprint(f"  {format_session_db_unavailable()}")
             return
 
         parts = cmd_original.split(None, 1)
@@ -6850,7 +6852,8 @@ def process_command(self, command: str) -> bool:
                                 self._pending_title = new_title
                                 _cprint(f"  Session title queued: {new_title} (will be saved on first message)")
                     else:
-                        _cprint("  Session database not available.")
+                        from hermes_state import format_session_db_unavailable
+                        _cprint(f"  {format_session_db_unavailable()}")
                 else:
                     _cprint("  Usage: /title <your session title>")
             else:
@@ -6865,7 +6868,8 @@ def process_command(self, command: str) -> bool:
                     else:
                         _cprint("  No title set. Usage: /title <your session title>")
                 else:
-                    _cprint("  Session database not available.")
+                    from hermes_state import format_session_db_unavailable
+                    _cprint(f"  {format_session_db_unavailable()}")
         elif canonical == "new":
             parts = cmd_original.split(maxsplit=1)
             title = parts[1].strip() if len(parts) > 1 else None

gateway/platforms/api_server.py

@@ -312,7 +312,12 @@ def __init__(self, max_size: int = MAX_STORED_RESPONSES, db_path: str = None):
             self._conn = sqlite3.connect(db_path, check_same_thread=False)
         except Exception:
             self._conn = sqlite3.connect(":memory:", check_same_thread=False)
-        self._conn.execute("PRAGMA journal_mode=WAL")
+        # Use shared WAL-fallback helper so response_store.db degrades
+        # gracefully on NFS/SMB/FUSE-mounted HERMES_HOME (same filesystem
+        # issue addressed for state.db/kanban.db — see
+        # hermes_state._WAL_INCOMPAT_MARKERS).
+        from hermes_state import apply_wal_with_fallback
+        apply_wal_with_fallback(self._conn, db_label="response_store.db")
         self._conn.execute(
             """CREATE TABLE IF NOT EXISTS responses (
                 response_id TEXT PRIMARY KEY,

gateway/run.py

@@ -1218,7 +1218,13 @@ def __init__(self, config: Optional[GatewayConfig] = None):
             from hermes_state import SessionDB
             self._session_db = SessionDB()
         except Exception as e:
-            logger.debug("SQLite session store not available: %s", e)
+            # WARNING (not DEBUG) so the failure appears in errors.log — matches
+            # cli.py's handling of the same init path.  Users hitting NFS-mounted
+            # HERMES_HOME silently lost /resume, /title, /history, /branch, and
+            # session search without this.  The underlying cause (usually
+            # "locking protocol" from NFS) is now also captured by
+            # hermes_state.get_last_init_error() for slash-command error strings.
+            logger.warning("SQLite session store not available: %s", e)
 
         # Opportunistic state.db maintenance: prune ended sessions older
         # than sessions.retention_days + optional VACUUM. Tracks last-run
@@ -10374,7 +10380,8 @@ def _telegram_topic_help_text(self) -> str:
     def _disable_telegram_topic_mode_for_chat(self, source: SessionSource) -> str:
         """Cleanly disable topic mode for a chat via /topic off."""
         if not self._session_db:
-            return "Session database not available."
+            from hermes_state import format_session_db_unavailable
+            return format_session_db_unavailable()
         chat_id = str(source.chat_id or "")
         if not chat_id:
             return "Could not determine chat ID."
@@ -10412,7 +10419,8 @@ async def _handle_topic_command(self, event: MessageEvent, args: str = "") -> st
         if source.platform != Platform.TELEGRAM or source.chat_type != "dm":
             return "The /topic command is only available in Telegram private chats."
         if not self._session_db:
-            return "Session database not available."
+            from hermes_state import format_session_db_unavailable
+            return format_session_db_unavailable()
 
         # Authorization: /topic activates multi-session mode and mutates
         # SQLite side tables. Unauthorized senders (not in allowlist) must
@@ -10626,7 +10634,8 @@ async def _handle_title_command(self, event: MessageEvent) -> str:
         session_id = session_entry.session_id
 
         if not self._session_db:
-            return "Session database not available."
+            from hermes_state import format_session_db_unavailable
+            return format_session_db_unavailable()
 
         # Ensure session exists in SQLite DB (it may only exist in session_store
         # if this is the first command in a new session)
@@ -10670,7 +10679,8 @@ async def _handle_title_command(self, event: MessageEvent) -> str:
     async def _handle_resume_command(self, event: MessageEvent) -> str:
         """Handle /resume command — switch to a previously-named session."""
         if not self._session_db:
-            return "Session database not available."
+            from hermes_state import format_session_db_unavailable
+            return format_session_db_unavailable()
 
         source = event.source
         session_key = self._session_key_for_source(source)
@@ -10757,7 +10767,8 @@ async def _handle_branch_command(self, event: MessageEvent) -> str:
         import uuid as _uuid
 
         if not self._session_db:
-            return "Session database not available."
+            from hermes_state import format_session_db_unavailable
+            return format_session_db_unavailable()
 
         source = event.source
         session_key = self._session_key_for_source(source)

hermes_cli/kanban_db.py

@@ -917,7 +917,11 @@ def connect(
     needs_init = resolved not in _INITIALIZED_PATHS
     conn = sqlite3.connect(str(path), isolation_level=None, timeout=30)
     conn.row_factory = sqlite3.Row
-    conn.execute("PRAGMA journal_mode=WAL")
+    # WAL doesn't work on network filesystems (NFS/SMB/FUSE).  Shared helper
+    # falls back to DELETE with one WARNING so kanban stays usable there.
+    # See hermes_state._WAL_INCOMPAT_MARKERS for detection logic.
+    from hermes_state import apply_wal_with_fallback
+    apply_wal_with_fallback(conn, db_label=f"kanban.db ({path.name})")
     conn.execute("PRAGMA synchronous=NORMAL")
     conn.execute("PRAGMA foreign_keys=ON")
     if needs_init:

hermes_state.py

@@ -35,6 +35,153 @@
 
 SCHEMA_VERSION = 11
 
+# ---------------------------------------------------------------------------
+# WAL-compatibility fallback
+# ---------------------------------------------------------------------------
+# SQLite's WAL mode requires shared-memory (mmap) coordination and fcntl
+# byte-range locks that don't reliably work on network filesystems (NFS,
+# SMB/CIFS, some FUSE mounts, WSL1).  Upstream documents this explicitly:
+# https://www.sqlite.org/wal.html#sometimes_queries_return_sqlite_busy_in_wal_mode
+#
+# On those filesystems ``PRAGMA journal_mode=WAL`` raises
+# ``sqlite3.OperationalError: locking protocol`` (SQLITE_PROTOCOL).  If we
+# propagate that, every feature backed by state.db / kanban.db breaks
+# silently — /resume, /title, /history, /branch, kanban dispatcher, etc.
+#
+# Instead, fall back to ``journal_mode=DELETE`` (the pre-WAL default) which
+# works on NFS.  Concurrency drops — concurrent readers are blocked during
+# a write — but the feature works.
+_WAL_INCOMPAT_MARKERS = (
+    "locking protocol",       # SQLITE_PROTOCOL on NFS/SMB
+    "not authorized",         # Some FUSE mounts block WAL pragma outright
+    "disk i/o error",         # Flaky network FS during WAL setup
+)
+
+# Last SessionDB() init error, per-process.  Surfaced in /resume and
+# related slash-command error strings so users know WHY the DB is
+# unavailable instead of getting a bare "Session database not available."
+# Only SessionDB.__init__ writes to this; kanban_db.connect() failures
+# do not update it (by design — kanban failures are reported via their
+# own caller's error handling, not via /resume-style slash commands).
+_last_init_error: Optional[str] = None
+_last_init_error_lock = threading.Lock()
+
+# Paths for which we've already logged a WAL-fallback WARNING.  Without
+# this, kanban_db.connect() (called on every kanban operation — see
+# hermes_cli/kanban_db.py for ~30 call sites) would re-log the same
+# filesystem-incompat warning on every connection, filling errors.log.
+_wal_fallback_warned_paths: set[str] = set()
+_wal_fallback_warned_lock = threading.Lock()
+
+
+def _set_last_init_error(msg: Optional[str]) -> None:
+    """Record (or clear) the most recent state.db init failure.
+
+    Thread-safe via _last_init_error_lock.  Callers pass a message to
+    record a failure or None to clear.  SessionDB.__init__ only calls
+    this to SET on failure — it deliberately does NOT clear on success,
+    because in a multi-threaded caller (e.g. gateway / web_server per-
+    request SessionDB() instantiation), a concurrent successful open
+    racing past a different thread's failure would erase the cause
+    string that thread's /resume handler is about to format.  Explicit
+    clears (e.g. test fixtures) are still supported by passing None.
+    """
+    global _last_init_error
+    with _last_init_error_lock:
+        _last_init_error = msg
+
+
+def get_last_init_error() -> Optional[str]:
+    """Return the most recent state.db init failure, if any.
+
+    Slash-command handlers (``/resume``, ``/title``, ``/history``, ``/branch``)
+    call this to surface the underlying cause in their error messages when
+    ``_session_db is None``.  Returns ``None`` if SessionDB initialized
+    successfully (or hasn't been attempted).
+    """
+    return _last_init_error
+
+
+def format_session_db_unavailable(prefix: str = "Session database not available") -> str:
+    """Format a user-facing 'session DB unavailable' message with cause.
+
+    When ``SessionDB()`` init fails, callers set ``_session_db = None`` and
+    several slash commands (/resume, /title, /history, /branch) previously
+    responded with a bare ``"Session database not available."`` — no
+    indication of WHY.  This helper includes the captured cause (typically
+    ``"locking protocol"`` from NFS/SMB) and points users at the known
+    culprit so they can fix it themselves.
+
+    Example output:
+        Session database not available: locking protocol (state.db may be
+        on NFS/SMB — see https://www.sqlite.org/wal.html).
+    """
+    cause = get_last_init_error()
+    if not cause:
+        return f"{prefix}."
+    hint = ""
+    if any(marker in cause.lower() for marker in _WAL_INCOMPAT_MARKERS):
+        hint = " (state.db may be on NFS/SMB/FUSE — see https://www.sqlite.org/wal.html)"
+    return f"{prefix}: {cause}{hint}."
+
+
+def apply_wal_with_fallback(
+    conn: sqlite3.Connection,
+    *,
+    db_label: str = "state.db",
+) -> str:
+    """Set ``journal_mode=WAL`` on ``conn``, falling back to DELETE on failure.
+
+    Returns the journal mode actually set (``"wal"`` or ``"delete"``).
+
+    On WAL-incompatible filesystems (NFS, SMB, some FUSE), SQLite raises
+    ``OperationalError("locking protocol")`` when setting WAL.  We fall
+    back to DELETE mode — the pre-WAL default, which works on NFS — and
+    log one WARNING explaining why.
+
+    The WARNING is deduplicated per ``db_label``: repeated connections
+    to the same underlying DB (e.g. kanban_db.connect() which is called
+    on every kanban operation) log once per process, not once per call.
+    Different db_labels log independently, so state.db and kanban.db
+    each get one warning on the same NFS mount.
+
+    Shared by :class:`SessionDB` and ``hermes_cli.kanban_db.connect`` so
+    both databases get identical fallback behavior.
+    """
+    try:
+        conn.execute("PRAGMA journal_mode=WAL")
+        return "wal"
+    except sqlite3.OperationalError as exc:
+        msg = str(exc).lower()
+        if not any(marker in msg for marker in _WAL_INCOMPAT_MARKERS):
+            # Unrelated OperationalError — don't silently swallow.
+            raise
+        _log_wal_fallback_once(db_label, exc)
+        conn.execute("PRAGMA journal_mode=DELETE")
+        return "delete"
+
+
+def _log_wal_fallback_once(db_label: str, exc: Exception) -> None:
+    """Log a single WARNING per (process, db_label) about WAL fallback.
+
+    Without this dedup, NFS users running kanban (which opens a fresh
+    connection on every operation — see hermes_cli/kanban_db.py) would
+    fill errors.log with hundreds of identical warnings per hour.
+    """
+    with _wal_fallback_warned_lock:
+        if db_label in _wal_fallback_warned_paths:
+            return
+        _wal_fallback_warned_paths.add(db_label)
+    logger.warning(
+        "%s: WAL journal_mode unsupported on this filesystem (%s) — "
+        "falling back to journal_mode=DELETE (slower rollback-journal "
+        "mode; reduces concurrency but works on NFS/SMB/FUSE). See "
+        "https://www.sqlite.org/wal.html for details. This warning "
+        "fires once per process per database.",
+        db_label,
+        exc,
+    )
+
 SCHEMA_SQL = """
 CREATE TABLE IF NOT EXISTS schema_version (
     version INTEGER NOT NULL
@@ -185,23 +332,40 @@ def __init__(self, db_path: Path = None):
 
         self._lock = threading.Lock()
         self._write_count = 0
-        self._conn = sqlite3.connect(
-            str(self.db_path),
-            check_same_thread=False,
-            # Short timeout — application-level retry with random jitter
-            # handles contention instead of sitting in SQLite's internal
-            # busy handler for up to 30s.
-            timeout=1.0,
-            # Autocommit mode: Python's default isolation_level="" auto-starts
-            # transactions on DML, which conflicts with our explicit
-            # BEGIN IMMEDIATE.  None = we manage transactions ourselves.
-            isolation_level=None,
-        )
-        self._conn.row_factory = sqlite3.Row
-        self._conn.execute("PRAGMA journal_mode=WAL")
-        self._conn.execute("PRAGMA foreign_keys=ON")
+        try:
+            self._conn = sqlite3.connect(
+                str(self.db_path),
+                check_same_thread=False,
+                # Short timeout — application-level retry with random jitter
+                # handles contention instead of sitting in SQLite's internal
+                # busy handler for up to 30s.
+                timeout=1.0,
+                # Autocommit mode: Python's default isolation_level=""
+                # auto-starts transactions on DML, which conflicts with our
+                # explicit BEGIN IMMEDIATE.  None = we manage transactions
+                # ourselves.
+                isolation_level=None,
+            )
+            self._conn.row_factory = sqlite3.Row
+            apply_wal_with_fallback(self._conn, db_label="state.db")
+            self._conn.execute("PRAGMA foreign_keys=ON")
 
-        self._init_schema()
+            self._init_schema()
+        except Exception as exc:
+            # Capture the cause so /resume and friends can surface WHY the
+            # session DB is unavailable instead of a bare "Session database
+            # not available."  Callers that catch this exception keep their
+            # existing ``self._session_db = None`` degradation path.
+            #
+            # Note: we deliberately do NOT clear _last_init_error on the
+            # success path (no else branch).  In multi-threaded callers
+            # (gateway, web_server per-request SessionDB()), a concurrent
+            # successful open racing past this failure would erase the
+            # cause that another thread's /resume is about to format.
+            # Tests that need to reset the state can call
+            # ``hermes_state._set_last_init_error(None)`` explicitly.
+            _set_last_init_error(f"{type(exc).__name__}: {exc}")
+            raise
 
     # ── Core write helper ──
 

plugins/memory/holographic/store.py

@@ -127,7 +127,11 @@ def __init__(
 
     def _init_db(self) -> None:
         """Create tables, indexes, and triggers if they do not exist. Enable WAL mode."""
-        self._conn.execute("PRAGMA journal_mode=WAL")
+        # Use the shared WAL-fallback helper so memory_store.db degrades
+        # gracefully on NFS/SMB/FUSE-mounted HERMES_HOME (same issue as
+        # state.db / kanban.db — see hermes_state._WAL_INCOMPAT_MARKERS).
+        from hermes_state import apply_wal_with_fallback
+        apply_wal_with_fallback(self._conn, db_label="memory_store.db (holographic)")
         self._conn.executescript(_SCHEMA)
         # Migrate: add hrr_vector column if missing (safe for existing databases)
         columns = {row[1] for row in self._conn.execute("PRAGMA table_info(facts)").fetchall()}

run_agent.py

@@ -9869,7 +9869,8 @@ def _invoke_tool(self, function_name: str, function_args: dict, effective_task_i
             )
         elif function_name == "session_search":
             if not self._session_db:
-                return json.dumps({"success": False, "error": "Session database not available."})
+                from hermes_state import format_session_db_unavailable
+                return json.dumps({"success": False, "error": format_session_db_unavailable()})
             from tools.session_search_tool import session_search as _session_search
             return _session_search(
                 query=function_args.get("query", ""),
@@ -10492,7 +10493,8 @@ def _execute_tool_calls_sequential(self, assistant_message, messages: list, effe
                     self._vprint(f"  {_get_cute_tool_message_impl('todo', function_args, tool_duration, result=function_result)}")
             elif function_name == "session_search":
                 if not self._session_db:
-                    function_result = json.dumps({"success": False, "error": "Session database not available."})
+                    from hermes_state import format_session_db_unavailable
+                    function_result = json.dumps({"success": False, "error": format_session_db_unavailable()})
                 else:
                     from tools.session_search_tool import session_search as _session_search
                     function_result = _session_search(