perf(#4842): freeze the CLI/cron sidebar cache key during streaming (stop the per-poll re-query that pins CPU)

CHANGELOG.md

@@ -3,6 +3,10 @@
 
 ## [Unreleased]
 
+### Fixed
+
+- **`/api/sessions` no longer re-runs the expensive CLI/cron session projection on every poll while a turn is streaming**, a major cause of the multi-second sidebar latency and 100% CPU on cron-heavy installs (#4842, continuing #4672/#4808/#4889). The CLI/cron sidebar projection is cached, but its cache key folded in a state.db content fingerprint (`MAX(rowid) FROM messages`) that advances on every streamed message row — so during a live turn the frontend's ~5s poll always missed the cache and re-ran the full candidate-join + projection (and the lineage-metadata pass), contending for the same SQLite/global lock the streaming worker holds. The route-level session-list cache already froze its key during streaming (#4808), but that freeze never reached this inner CLI-sessions cache. Now, while any turn is streaming, the CLI-sessions cache key folds in the same stable streaming-freeze marker (keyed only on the set of active stream ids) and its TTL widens, so the heavy projection is reused across polls and rebuilt at most once per streaming window instead of once per poll. Structural sidebar mutations (cron completion, new/renamed/archived sessions, attention) clear the cache directly, so nothing user-visible lags under the freeze; idle behavior is unchanged.
+
 ## [v0.51.647] — 2026-06-25 — Release XC (task detail action buttons reappear on mobile PWA)
 
 ### Fixed

api/models.py

@@ -38,6 +38,12 @@
 # sidebar window (#3172).
 CRON_PROJECT_CHIP_LIMIT = 200
 _CLI_SESSIONS_CACHE_TTL_SECONDS = 5.0
+# While a turn is actively streaming, hold the CLI/cron projection longer than
+# one poll interval (mirrors the route-level #4808 hold-down). The frontend
+# polls /api/sessions every ~5s during a stream; without a wider window the
+# CLI cache key advances on every streamed message row (see below) and the
+# expensive state.db CLI/cron projection is re-run on every poll. (#4842)
+_CLI_SESSIONS_CACHE_STREAMING_TTL_SECONDS = 30.0
 _CLI_SESSIONS_CACHE_LOCK = threading.Lock()
 _CLI_SESSIONS_CACHE = {}
 
@@ -4099,6 +4105,17 @@ def _copy_cli_sessions(sessions: list) -> list:
 
 
 def _cli_sessions_cache_ttl_seconds() -> float:
+    # #4842: widen the freshness window while a turn is streaming so the fixed
+    # ~5s streaming poll cadence doesn't force a rebuild on every poll. Paired
+    # with the streaming-freeze cache key (so the key is stable across polls
+    # mid-stream), this bounds the heavy CLI/cron projection to one rebuild per
+    # streaming-TTL window instead of one per poll. Mirrors the route-level
+    # #4808 TTL widening.
+    try:
+        if _cli_sessions_streaming_freeze_marker() is not None:
+            return max(0.0, float(_CLI_SESSIONS_CACHE_STREAMING_TTL_SECONDS))
+    except (TypeError, ValueError):
+        pass
     try:
         return max(0.0, float(_CLI_SESSIONS_CACHE_TTL_SECONDS))
     except (TypeError, ValueError):
@@ -4209,6 +4226,43 @@ def _sqlite_file_stat_cache_key(db_path: Path):
     )
 
 
+def _cli_sessions_streaming_freeze_marker():
+    """Return a stable cache-key marker while any turn is actively streaming.
+
+    The CLI/cron sidebar projection (``_load_cli_sessions_uncached``) is gated by
+    ``_CLI_SESSIONS_CACHE``, whose key folds in ``_sqlite_file_stat_cache_key`` →
+    ``_sqlite_content_fingerprint`` (``MAX(rowid) FROM messages``). During an
+    active chat turn the gateway/CLI writes a message row per streamed delta, so
+    that fingerprint advances on essentially every ``/api/sessions`` poll — busting
+    the CLI cache and re-running the expensive candidate-join + projection (and the
+    lineage-metadata pass) on every poll, while contending for the same SQLite/global
+    lock the streaming worker holds. That is the multi-second ``get_cli_sessions``
+    in #4842 (and #4672/#4808).
+
+    The route-level session-list cache already freezes its own key during streaming
+    (#4808 ``_session_list_cache_streaming_freeze_marker``), but that freeze never
+    reached this *inner* CLI-sessions cache, so the heavy CLI/cron query still
+    re-ran whenever the outer cache validated. This marker mirrors the route-level
+    one: keyed only on the *set* of active stream ids, it is constant while the same
+    turn(s) stream (so the projection is reused across polls) and changes the instant
+    a stream starts/stops (so the just-finished turn's rows are picked up promptly).
+    A streaming session's own CLI/cron title/count is not what this projection
+    returns (the streaming session is overlaid live by the route layer), and any
+    structural mutation invalidates the cache directly via
+    ``clear_cli_sessions_cache``, so nothing user-visible lags under the freeze. (#4842)
+    """
+    try:
+        active = _active_stream_ids()
+    except Exception:
+        return None
+    if not active:
+        return None
+    try:
+        return ("streaming", tuple(sorted(str(x) for x in active)))
+    except Exception:
+        return ("streaming",)
+
+
 def _resolve_cli_sessions_context(source_filter=None):
     # Use the active WebUI profile's HERMES_HOME to find state.db.
     # The active profile is determined by what the user has selected in the UI
@@ -4232,12 +4286,20 @@ def _resolve_cli_sessions_context(source_filter=None):
 
     db_path = hermes_home / 'state.db'
     projects_dir = _default_claude_code_projects_dir()
+    # #4842: while a turn streams, freeze the volatile state.db component of the
+    # key so per-message writes don't bust the CLI cache and re-run the heavy
+    # CLI/cron projection on every poll (mirrors the route-level #4808 freeze).
+    # The wider streaming TTL in get_cli_sessions() still forces a periodic
+    # rebuild so a streaming session's own count stays fresh within that window,
+    # and structural mutations invalidate via clear_cli_sessions_cache().
+    _streaming_marker = _cli_sessions_streaming_freeze_marker()
+    db_state_key = _streaming_marker if _streaming_marker is not None else _sqlite_file_stat_cache_key(db_path)
     cache_key = (
         str(hermes_home),
         str(cli_profile or ''),
         str(db_path),
         str(source_filter or ''),
-        _sqlite_file_stat_cache_key(db_path),
+        db_state_key,
         _path_cache_key(projects_dir),
         _path_stat_cache_key(projects_dir),
         _path_stat_cache_key(SESSION_INDEX_FILE),
@@ -4588,6 +4650,12 @@ def get_cli_sessions(source_filter=None, *, all_profiles: bool = False) -> list:
     source_filter = _normalize_cli_session_source_filter(source_filter)
     if all_profiles:
         contexts, context_cache_key = _all_profiles_cli_contexts()
+        # #4842: freeze the volatile per-profile state.db component while
+        # streaming so a streamed message row in one profile doesn't bust the
+        # all-profiles CLI cache and re-run every profile's heavy projection.
+        _streaming_marker = _cli_sessions_streaming_freeze_marker()
+        if _streaming_marker is not None:
+            context_cache_key = ('streaming-frozen', _streaming_marker)
         cache_key = (
             'all_profiles',
             source_filter or '',

api/routes.py

@@ -205,6 +205,19 @@ def _run() -> None:
 def _on_session_list_changed(profile: str | None = None) -> None:
     """Invalidate in-process /api/sessions cache when sidebar state mutates."""
     _clear_session_list_cache(profile)
+    # #4842: also drop the inner CLI/cron projection cache. While a turn streams
+    # that cache is frozen on a stable streaming marker (so per-token message
+    # writes don't bust it), which means it no longer self-invalidates via the
+    # state.db content fingerprint mid-stream. Structural mutations (cron
+    # completion, new/renamed/archived sessions, attention) DO fire this
+    # listener — and never fire per streamed token — so clearing here restores
+    # prompt freshness for real changes without reintroducing the per-poll
+    # rebuild the freeze removed.
+    try:
+        from api.models import clear_cli_sessions_cache
+        clear_cli_sessions_cache()
+    except Exception:
+        logger.debug("Failed to clear CLI sessions cache on session list change", exc_info=True)
 
 
 try:

tests/test_issue4842_cli_sessions_streaming_freeze.py

@@ -0,0 +1,120 @@
+"""Behavioral test for #4842: the CLI/cron sidebar projection cache must not be
+re-run on every poll while a turn is streaming.
+
+Root cause: ``_CLI_SESSIONS_CACHE`` is keyed (via ``_resolve_cli_sessions_context``
+-> ``_sqlite_file_stat_cache_key`` -> ``_sqlite_content_fingerprint``) on
+``MAX(rowid) FROM messages``. During a live turn the gateway writes a message row
+per streamed delta, so that fingerprint advances on essentially every
+``/api/sessions`` poll, busting the cache and re-running the expensive CLI/cron
+candidate-join + projection (and the lineage-metadata pass) on every poll — the
+multi-second ``get_cli_sessions`` in #4842.
+
+Fix: while any stream is active, the cache key folds in a stable streaming-freeze
+marker (keyed only on the set of active stream ids) instead of the volatile
+content fingerprint, and the cache TTL widens — so the projection is reused across
+polls mid-stream and rebuilt at most once per streaming-TTL window. The instant a
+stream starts/stops the marker changes, so freshly-finished rows surface promptly.
+"""
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from api import models as M
+
+
+def _set_active_streams(monkeypatch, ids):
+    monkeypatch.setattr(M, "_active_stream_ids", lambda: set(ids))
+
+
+def test_freeze_marker_none_when_idle(monkeypatch):
+    _set_active_streams(monkeypatch, [])
+    assert M._cli_sessions_streaming_freeze_marker() is None
+
+
+def test_freeze_marker_stable_for_same_streams(monkeypatch):
+    _set_active_streams(monkeypatch, ["sA", "sB"])
+    m1 = M._cli_sessions_streaming_freeze_marker()
+    m2 = M._cli_sessions_streaming_freeze_marker()
+    assert m1 is not None and m1 == m2
+    # order-independent
+    _set_active_streams(monkeypatch, ["sB", "sA"])
+    assert M._cli_sessions_streaming_freeze_marker() == m1
+
+
+def test_freeze_marker_changes_when_stream_set_changes(monkeypatch):
+    _set_active_streams(monkeypatch, ["sA"])
+    m1 = M._cli_sessions_streaming_freeze_marker()
+    _set_active_streams(monkeypatch, ["sA", "sB"])
+    m2 = M._cli_sessions_streaming_freeze_marker()
+    assert m1 != m2
+    _set_active_streams(monkeypatch, [])
+    assert M._cli_sessions_streaming_freeze_marker() is None
+
+
+def test_cache_key_stable_across_message_writes_while_streaming(monkeypatch, tmp_path):
+    """THE core guarantee: with a stream active, the CLI cache key must NOT change
+    when the state.db content fingerprint advances (a new streamed message row).
+    Before the fix the key folded in the live fingerprint and changed every write."""
+    db = tmp_path / "state.db"
+    db.write_bytes(b"")  # exists; fingerprint reader degrades gracefully
+
+    monkeypatch.setattr(M, "_default_claude_code_projects_dir", lambda: tmp_path / "projects")
+    # Simulate the volatile fingerprint advancing on each streamed message.
+    fp = {"v": 0}
+    monkeypatch.setattr(M, "_sqlite_file_stat_cache_key", lambda p: ("fp", fp["v"]))
+
+    # Active stream -> key should be frozen (independent of fp).
+    _set_active_streams(monkeypatch, ["live-stream-1"])
+    _, _, _, key_a = M._resolve_cli_sessions_context(None)
+    fp["v"] = 1  # a streamed message row landed
+    _, _, _, key_b = M._resolve_cli_sessions_context(None)
+    fp["v"] = 2  # another
+    _, _, _, key_c = M._resolve_cli_sessions_context(None)
+    assert key_a == key_b == key_c, (
+        "CLI cache key changed across message writes while streaming — the heavy "
+        "projection would re-run on every poll (#4842 regression)"
+    )
+
+    # Idle -> key tracks the fingerprint again (so genuine new rows show up).
+    _set_active_streams(monkeypatch, [])
+    fp["v"] = 10
+    _, _, _, key_idle1 = M._resolve_cli_sessions_context(None)
+    fp["v"] = 11
+    _, _, _, key_idle2 = M._resolve_cli_sessions_context(None)
+    assert key_idle1 != key_idle2, (
+        "When idle the CLI cache key must still advance with the content "
+        "fingerprint so newly-committed sessions are not served stale"
+    )
+
+
+def test_streaming_ttl_wider_than_idle(monkeypatch):
+    _set_active_streams(monkeypatch, [])
+    idle = M._cli_sessions_cache_ttl_seconds()
+    _set_active_streams(monkeypatch, ["s1"])
+    streaming = M._cli_sessions_cache_ttl_seconds()
+    assert streaming > idle, (
+        "streaming TTL must exceed idle TTL so the fixed poll cadence does not "
+        "force a rebuild on every poll (#4842)"
+    )
+
+
+def test_structural_change_listener_clears_cli_cache(monkeypatch):
+    """While streaming, the CLI cache is frozen and no longer self-invalidates via
+    the content fingerprint. A structural mutation (cron completion / new / renamed
+    / archived session) must therefore clear the CLI cache directly so the change
+    surfaces promptly instead of lagging up to the streaming TTL. Those structural
+    signals fire the session-list-changed listener; per-token message writes never
+    do — that is exactly what makes the freeze safe."""
+    from api import routes as R
+
+    cleared = {"n": 0}
+    monkeypatch.setattr(M, "clear_cli_sessions_cache", lambda: cleared.__setitem__("n", cleared["n"] + 1))
+    # The route module imports the symbol lazily inside the listener, so patching
+    # api.models.clear_cli_sessions_cache is what the listener resolves.
+    R._on_session_list_changed("default")
+    assert cleared["n"] >= 1, (
+        "_on_session_list_changed must clear the CLI/cron projection cache so a "
+        "structural mutation isn't masked by the streaming freeze (#4842)"
+    )
+