Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions gateway/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -3238,6 +3238,13 @@ def __init__(self, config: Optional[GatewayConfig] = None):
vacuum=bool(_sess_cfg.get("vacuum_after_prune", True)),
sessions_dir=self.config.sessions_dir,
)
# WAL watchdog: force-checkpoint an oversized state.db-wal at
# startup (and hourly, see _start_gateway_housekeeping). Bounds
# the WAL a pinned reader would otherwise let grow to GBs.
if _sess_cfg.get("wal_watchdog", True):
self._session_db._db.wal_watchdog(
max_mb=int(_sess_cfg.get("wal_max_mb", 64)),
)
except Exception as exc:
logger.debug("state.db auto-maintenance skipped: %s", exc)

Expand Down Expand Up @@ -8046,6 +8053,30 @@ async def _session_expiry_watcher(self, interval: int = 300):
except Exception as _e:
logger.debug("SessionStore prune failed: %s", _e)
self._last_session_store_prune_ts = time.time()

# WAL watchdog (hourly): force-checkpoint an oversized
# state.db-wal that a long-lived reader has let grow past
# sessions.wal_max_mb. Complements the startup pass; a giant
# WAL is the malformed-image corruption risk. The checkpoint is
# a synchronous SQLite call, so offload it to a thread — a
# pinned reader can make TRUNCATE block, and this watcher runs
# on the gateway event loop.
_last_wal_ts = getattr(self, "_last_wal_watchdog_ts", 0.0)
if (
self._session_db is not None
and time.time() - _last_wal_ts > 3600.0
):
try:
from hermes_cli.config import load_config as _load_full_config
_sess_cfg = (_load_full_config().get("sessions") or {})
if _sess_cfg.get("wal_watchdog", True):
await asyncio.to_thread(
self._session_db._db.wal_watchdog,
max_mb=int(_sess_cfg.get("wal_max_mb", 64)),
)
except Exception as _e:
logger.debug("WAL watchdog tick failed: %s", _e)
self._last_wal_watchdog_ts = time.time()
except Exception as e:
logger.debug("Session expiry watcher error: %s", e)
# Sleep in small increments so we can stop quickly
Expand Down
14 changes: 14 additions & 0 deletions hermes_cli/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -3059,6 +3059,20 @@ def _ensure_hermes_home_managed(home: Path):
# GBs of disk on heavy users. Opt in only if you have an external
# tool that consumes the JSON files directly.
"write_json_snapshots": False,
# WAL watchdog: bound state.db-wal growth. A long-lived reader can pin
# the WAL so the periodic checkpoint no-ops and the -wal grows without
# bound (seen live at 3.6 GB); a giant WAL + hard shutdown is the
# malformed-image corruption scenario. When true the gateway runs a
# RESTART checkpoint at startup and hourly if -wal exceeds wal_max_mb.
"wal_watchdog": True,
# -wal size (MB) above which the watchdog force-checkpoints.
"wal_max_mb": 64,
# Trigram FTS index for CJK / substring search. The trigram shadow
# tables were the single largest consumer on the live DB (~5 GB serving
# substring search over ~2 GB of text). When false the trigram table +
# triggers are dropped and CJK/substring search falls back to LIKE (no
# feature loss, just a slower scan for those queries). Default true.
"fts_trigram": True,
},

# Contextual first-touch onboarding hints (see agent/onboarding.py).
Expand Down
Loading
Loading