fix(dashboard): return 503, not 500, for a deleted WAL or replaced state.db

hermes_cli/web_routers/_common.py

@@ -160,26 +160,44 @@ def is_redacted_credential_preview(submitted: Any) -> bool:
     "error": "state_db_corrupt",
     "message": "state.db corrupt — run `hermes doctor` (then `hermes doctor --fix` or `hermes sessions repair`).",
 }
+# One payload per persistence-cause bucket (``classify_persistence_error``); ``error`` codes
+# all follow the ``state_db_*`` scheme of the pre-existing ``state_db_corrupt``.
+# Same guidance as the deleted_wal / replaced turn explainers: `doctor --fix` while a holder
+# lives would repair the wrong generation in place, so it is deliberately NOT suggested here.
+DELETED_WAL_DETAIL = {
+    "error": "state_db_deleted_wal",
+    "message": "another Hermes process still holds an old copy of the session database's write-ahead log — "
+               "quit every Hermes process on this profile, run `hermes doctor` (it names the holders), "
+               "then start Hermes again. Do not run `hermes doctor --fix` while they run.",
+}
+STATE_DB_REPLACED_DETAIL = {
+    "error": "state_db_replaced",
+    "message": "state.db was replaced while Hermes was running — stop Hermes, run `hermes doctor`, "
+               "then start it again. Do not run `hermes doctor --fix`, which would repair the wrong file in place.",
+}
+# Every other bucket a malformed image can classify as ("corrupt", "fts_index") is the corrupt payload.
+_STORE_STATUS_DETAIL_BY_CAUSE = {"deleted_wal": DELETED_WAL_DETAIL, "replaced": STATE_DB_REPLACED_DETAIL}
 
 
 @contextlib.contextmanager
 def corrupt_store_as_status(db_path):
-    """Map a corrupt-image ``sqlite3.DatabaseError`` from a state.db read to a 503 status
+    """Map a corrupt-image ``sqlite3.DatabaseError`` or ``StateDbReplacedError`` from a state.db read to a 503 status
     payload, warning once per store per :data:`_CORRUPT_STORE_WARN_INTERVAL_S`.
     Busy/locked and every other error propagate unchanged."""
-    from hermes_state_errors import is_malformed_db_error
+    from hermes_state_errors import StateDbReplacedError, classify_persistence_error, is_malformed_db_error
 
     try:
         yield
-    except sqlite3.DatabaseError as exc:
-        if not is_malformed_db_error(exc):
+    except (sqlite3.DatabaseError, StateDbReplacedError) as exc:
+        if not isinstance(exc, StateDbReplacedError) and not is_malformed_db_error(exc):
             raise
         key, now = str(db_path), time.monotonic()
         last = _corrupt_store_warned_at.get(key)
+        detail = _STORE_STATUS_DETAIL_BY_CAUSE.get(classify_persistence_error(exc), CORRUPT_STORE_DETAIL)
         if last is None or now - last >= _CORRUPT_STORE_WARN_INTERVAL_S:
             _corrupt_store_warned_at[key] = now
-            log.warning("state.db at %s is corrupt (%s); dashboard reads return a status payload until it is "
+            log.warning("state.db at %s is unreadable (%s); dashboard reads return a status payload until it is "
                         "repaired — run `hermes doctor`", db_path, exc)
         else:
-            log.debug("state.db at %s still corrupt: %s", db_path, exc)
-        raise HTTPException(status_code=503, detail={**CORRUPT_STORE_DETAIL, "path": key}) from exc
+            log.debug("state.db at %s still has error: %s", db_path, exc)
+        raise HTTPException(status_code=503, detail={**detail, "path": key}) from exc

hermes_cli/web_routers/sessions.py

@@ -22,9 +22,11 @@
 from hermes_cli.web_server_sessions import _maybe_auto_archive_for_profile, _session_latest_descendant
 from hermes_cli.web_models import (
     BulkDeleteSessions, SessionImport, SessionOwnerBackfill, SessionPrune, SessionRename)
-from hermes_cli.web_routers._common import CORRUPT_STORE_DETAIL, log as _log, destructive_profile, http_failure
+from hermes_cli.web_routers._common import (
+    CORRUPT_STORE_DETAIL, corrupt_store_as_status, log as _log, destructive_profile, http_failure,
+)
 from hermes_state import is_malformed_db_error
-from hermes_state_errors import is_transient_sqlite_error
+from hermes_state_errors import StateDbReplacedError, is_transient_sqlite_error
 from hermes_state_health import STORAGE_CORRUPT, note_storage_error, storage_state
 
 list_router = APIRouter()
@@ -160,6 +162,10 @@ def _resolve_session_id(db, session_id: str) -> Optional[str]:
                 "Sessions cannot be read until it is repaired — run "
                 "`hermes doctor` for diagnosis."),
         ) from exc
+    except StateDbReplacedError:
+        # RuntimeError family, not sqlite3: same 503 payload as the analytics reads (#110054).
+        with corrupt_store_as_status(db.db_path):
+            raise
 
 
 # ``le=100`` on limit: an unbounded limit lets one request drag every session
@@ -250,6 +256,10 @@ def get_sessions(
             raise HTTPException(status_code=500, detail="Internal server error") from exc
         _log.error("GET /api/sessions: state.db at %s is corrupt: %s", db_path, exc)
         raise HTTPException(status_code=503, detail=dict(CORRUPT_STORE_DETAIL)) from exc
+    except StateDbReplacedError:
+        # RuntimeError family, not sqlite3: same 503 payload as the analytics reads (#110054).
+        with corrupt_store_as_status(_session_db_path_for_profile(profile)):
+            raise
     except Exception:
         _log.exception("GET /api/sessions failed")
         raise HTTPException(status_code=500, detail="Internal server error")

tests/hermes_cli/test_web_analytics_corrupt_store.py

@@ -1,5 +1,6 @@
 """A malformed state.db must not turn dashboard analytics polling into a traceback storm (#96591)."""
 import logging
+import re
 import sqlite3
 from pathlib import Path
 
@@ -62,3 +63,58 @@ def test_corrupt_store_polls_return_status_and_warn_once_per_interval(tmp_path,
         r.levelno >= logging.WARNING and r.name.startswith("hermes_cli.web_server")
         for r in caplog.records
     ) == 1
+
+
+def test_corrupt_store_as_status_maps_replaced_store_errors_to_503_without_fix_nudge(tmp_path, monkeypatch):
+    """A retired WAL generation / replaced state.db (RuntimeError subclasses, not sqlite3 errors)
+    must come back as a structured 503 like the corrupt case, and the guidance must never tell the
+    user to run `doctor --fix` while a holder is live (#110054). Busy/locked still propagates."""
+    from fastapi import HTTPException
+    from hermes_state_errors import DeletedWalGenerationError, StateDbReplacedError
+
+    monkeypatch.setattr(_common, "_corrupt_store_warned_at", {})
+    db_path = tmp_path / "state.db"
+    for exc_cls, code in ((DeletedWalGenerationError, "state_db_deleted_wal"), (StateDbReplacedError, "state_db_replaced")):
+        with pytest.raises(HTTPException) as info:
+            with _common.corrupt_store_as_status(db_path):
+                raise exc_cls("retired WAL held by pid 4242")
+        assert info.value.status_code == 503
+        assert info.value.detail["error"] == code
+        assert info.value.detail["path"] == str(db_path)
+        msg = info.value.detail["message"]
+        assert "run `hermes doctor`" in msg
+        # `--fix` may only appear negated — never as the action to take while a holder is live.
+        negated = re.findall(r"(?i)(?:do not|don't|never) run `hermes doctor --fix`", msg)
+        assert len(negated) == msg.count("`hermes doctor --fix`"), msg
+
+    with pytest.raises(sqlite3.OperationalError):
+        with _common.corrupt_store_as_status(db_path):
+            raise sqlite3.OperationalError("database is locked")
+
+    # Sibling route: GET /api/sessions and the session-id resolver used to let the same
+    # RuntimeError family fall through to a generic 500.
+    from hermes_cli.web_routers import sessions
+
+    class _RetiredDb:
+        db_path = str(tmp_path / "state.db")
+
+        def resolve_session_id(self, session_id):
+            raise DeletedWalGenerationError("retired WAL held by pid 4242")
+
+        def list_sessions_rich(self, **kwargs):
+            raise DeletedWalGenerationError("retired WAL held by pid 4242")
+
+        def close(self):
+            pass
+
+    with pytest.raises(HTTPException) as info:
+        sessions._resolve_session_id(_RetiredDb(), "abc")
+    assert (info.value.status_code, info.value.detail["error"]) == (503, "state_db_deleted_wal")
+
+    monkeypatch.setattr(sessions, "_maybe_auto_archive_for_profile", lambda profile: None)
+    monkeypatch.setattr(sessions, "_session_db_path_for_profile", lambda profile: db_path)
+    monkeypatch.setattr(sessions, "_open_session_db_for_profile", lambda profile, read_only: _RetiredDb())
+    app = FastAPI()
+    app.include_router(sessions.list_router)
+    resp = TestClient(app, raise_server_exceptions=False).get("/api/sessions")
+    assert (resp.status_code, resp.json()["detail"]["error"]) == (503, "state_db_deleted_wal")