fix(repair): treat SQLITE_BUSY as contention, not corruption, in sqlite_integrity_errors

mempalace/cli.py

@@ -259,14 +259,16 @@ def _ensure_mempalace_files_gitignored(project_dir) -> bool:
     if not (project_path / ".git").exists():
         return False
     gitignore = project_path / ".gitignore"
-    existing = gitignore.read_text() if gitignore.exists() else ""
+    # Force UTF-8: Windows defaults to GBK and chokes on non-ASCII .gitignore
+    # comments, killing auto-init even though the file is valid UTF-8.
+    existing = gitignore.read_text(encoding="utf-8", errors="replace") if gitignore.exists() else ""
     existing_lines = {line.strip() for line in existing.splitlines()}
     missing = [p for p in _MEMPALACE_PROJECT_FILES if p not in existing_lines]
     if not missing:
         return False
     prefix = "" if not existing or existing.endswith("\n") else "\n"
     block = prefix + "\n# MemPalace per-project files (issue #185)\n" + "\n".join(missing) + "\n"
-    with open(gitignore, "a") as f:
+    with open(gitignore, "a", encoding="utf-8") as f:
         f.write(block)
     print(f"  Added {', '.join(missing)} to {gitignore.name}")
     return True

mempalace/layers.py

@@ -18,6 +18,7 @@
 
 import os
 import sys
+from datetime import datetime, timezone
 from pathlib import Path
 from collections import defaultdict
 
@@ -87,7 +88,9 @@ class Layer1:
 
     MAX_DRAWERS = 15  # at most 15 moments in wake-up
     MAX_CHARS = 3200  # hard cap on total L1 text (~800 tokens)
-    MAX_SCAN = 2000  # don't scan more than this for L1 generation
+    MAX_SCAN = 200_000  # safety valve on metadata scan (full corpus expected)
+    RECENCY_WEIGHT = 3.0  # max score boost for a just-filed drawer
+    RECENCY_HALF_LIFE_DAYS = 14.0  # boost halves every N days
 
     def __init__(self, palace_path: str = None, wing: str = None):
         cfg = MempalaceConfig()
@@ -101,37 +104,42 @@ def generate(self) -> str:
         except Exception:
             return "## L1 — No palace found. Run: mempalace mine <dir>"
 
-        # Fetch all drawers in batches to avoid SQLite variable limit (~999)
+        # Scan metadata for the FULL corpus (no documents — cheap), batched
+        # to avoid SQLite variable limit (~999). Documents are fetched later
+        # for only the top MAX_DRAWERS winners.
         _BATCH = 500
-        docs, metas = [], []
+        ids, metas = [], []
         offset = 0
         while True:
-            kwargs = {"include": ["documents", "metadatas"], "limit": _BATCH, "offset": offset}
+            kwargs = {"include": ["metadatas"], "limit": _BATCH, "offset": offset}
             if self.wing:
                 kwargs["where"] = {"wing": self.wing}
             try:
                 batch = col.get(**kwargs)
             except Exception:
                 break
-            batch_docs = batch.get("documents", [])
+            batch_ids = batch.get("ids", [])
             batch_metas = batch.get("metadatas", [])
-            if not batch_docs:
+            if not batch_ids:
                 break
-            docs.extend(batch_docs)
+            ids.extend(batch_ids)
             metas.extend(batch_metas)
-            offset += len(batch_docs)
-            if len(batch_docs) < _BATCH or len(docs) >= self.MAX_SCAN:
+            offset += len(batch_ids)
+            if len(batch_ids) < _BATCH or len(ids) >= self.MAX_SCAN:
                 break
 
-        if not docs:
+        if not ids:
             return "## L1 — No memories yet."
 
-        # Score each drawer: prefer high importance, recent filing
+        # Score each drawer: importance + recency decay (half-life
+        # RECENCY_HALF_LIFE_DAYS). A fresh default-importance drawer can
+        # outrank a stale high-importance one, so the essential story tracks
+        # the present, not the archive.
+        now = datetime.now(timezone.utc)
         scored = []
-        for doc, meta in zip(docs, metas):
+        for did, meta in zip(ids, metas):
             meta = meta or {}
-            doc = doc or ""
-            importance = 3
+            importance = 3.0
             # Try multiple metadata keys that might carry weight info
             for key in ("importance", "emotional_weight", "weight"):
                 val = meta.get(key)
@@ -141,11 +149,45 @@ def generate(self) -> str:
                     except (ValueError, TypeError):
                         pass
                     break
-            scored.append((importance, meta, doc))
+            age_days = None
+            for key in ("authored_at", "created_at", "filed_at"):
+                raw = meta.get(key)
+                if raw:
+                    try:
+                        ts = datetime.fromisoformat(str(raw))
+                        if ts.tzinfo is None:
+                            ts = ts.replace(tzinfo=timezone.utc)
+                        age_days = max((now - ts).total_seconds() / 86400.0, 0.0)
+                    except (ValueError, TypeError):
+                        pass
+                    break
+            recency = (
+                0.0
+                if age_days is None
+                else self.RECENCY_WEIGHT * 0.5 ** (age_days / self.RECENCY_HALF_LIFE_DAYS)
+            )
+            scored.append((importance + recency, did, meta))
 
-        # Sort by importance descending, take top N
+        # Sort by combined score descending, take top N — at most one drawer
+        # per source_file so multi-chunk files don't monopolize L1 — then
+        # fetch documents for just the winners
         scored.sort(key=lambda x: x[0], reverse=True)
-        top = scored[: self.MAX_DRAWERS]
+        winners, seen_sources = [], set()
+        for score, did, meta in scored:
+            source = meta.get("source_file") or did
+            if source in seen_sources:
+                continue
+            seen_sources.add(source)
+            winners.append((score, did, meta))
+            if len(winners) >= self.MAX_DRAWERS:
+                break
+        try:
+            fetched = col.get(ids=[d for _s, d, _m in winners], include=["documents"])
+        except Exception:
+            return "## L1 — No memories yet."
+        doc_by_id = dict(zip(fetched.get("ids", []), fetched.get("documents", [])))
+
+        top = [(score, meta, doc_by_id.get(did) or "") for score, did, meta in winners]
 
         # Group by room for readability
         by_room = defaultdict(list)

mempalace/repair.py

@@ -592,6 +592,14 @@ def sqlite_integrity_errors(palace_path: str) -> list[str]:
 
     try:
         with sqlite3.connect(sqlite_read_uri(sqlite_path), uri=True) as conn:
+            # SQLITE_BUSY from a concurrent writer is contention, not
+            # corruption. Without a busy_timeout, quick_check fails
+            # instantly with "database is locked" whenever another
+            # process holds the write lock (e.g. a long batch write on a
+            # rollback-journal palace), and callers — including the MCP
+            # startup integrity gate (#1818) — misreport a healthy palace
+            # as corrupt. Wait out transient writers before giving up.
+            conn.execute("PRAGMA busy_timeout = 15000")
             rows = conn.execute("PRAGMA quick_check").fetchall()
     except sqlite3.Error as e:
         return [f"PRAGMA quick_check failed: {e}"]

tests/test_layers.py

@@ -71,13 +71,24 @@ def test_layer0_default_path():
 
 
 def _mock_chromadb_for_layer(docs, metas, monkeypatch=None):
-    """Return a mock collection whose get() returns docs/metas."""
+    """Return a mock collection emulating Layer1's two-step access pattern.
+
+    First a paginated metadata-only scan (returns ids + metadatas,
+    honouring limit/offset), then a documents fetch for the winning ids.
+    The old single-shot documents+metadatas shape predates the
+    full-corpus-scan rewrite and made every test see an empty palace.
+    """
     mock_col = MagicMock()
-    # First batch returns data, second batch returns empty (end of pagination)
-    mock_col.get.side_effect = [
-        {"documents": docs, "metadatas": metas},
-        {"documents": [], "metadatas": []},
-    ]
+    all_ids = [f"id{i}" for i in range(len(docs))]
+    doc_by_id = dict(zip(all_ids, docs))
+
+    def fake_get(ids=None, include=None, limit=None, offset=0, where=None):
+        if ids is not None:  # second step: documents for the winners
+            return {"ids": list(ids), "documents": [doc_by_id.get(d) for d in ids]}
+        end = len(all_ids) if limit is None else min(offset + limit, len(all_ids))
+        return {"ids": all_ids[offset:end], "metadatas": metas[offset:end]}
+
+    mock_col.get.side_effect = fake_get
     return mock_col
 
 
@@ -202,12 +213,30 @@ def test_layer1_importance_from_various_keys():
 
 
 def test_layer1_batch_exception_breaks():
-    """If col.get raises on a batch, loop breaks gracefully."""
+    """If col.get raises mid-scan, the loop breaks gracefully.
+
+    L1 must still render from the drawers scanned so far — a transient
+    chroma error partway through the corpus scan should degrade to a
+    shorter essential story, not an empty one. The first page must be
+    exactly one scan-batch long so Layer1 asks for a second page.
+    """
+    batch = 500  # Layer1's _BATCH page size
+    all_ids = [f"id{i}" for i in range(batch)]
+    all_metas = [{"room": "r", "source_file": f"s{i}.txt"} for i in range(batch)]
+    doc_by_id = {d: f"memory {d}" for d in all_ids}
+    scan_calls = []
+
+    def fake_get(ids=None, include=None, limit=None, offset=0, where=None):
+        if ids is not None:  # documents fetch for the winners
+            return {"ids": list(ids), "documents": [doc_by_id[d] for d in ids]}
+        scan_calls.append(offset)
+        if len(scan_calls) > 1:
+            raise RuntimeError("batch error")
+        return {"ids": all_ids[:limit], "metadatas": all_metas[:limit]}
+
     mock_col = MagicMock()
-    mock_col.get.side_effect = [
-        {"documents": ["doc1"], "metadatas": [{"room": "r"}]},
-        RuntimeError("batch error"),
-    ]
+    mock_col.get.side_effect = fake_get
+
     with (
         patch("mempalace.layers.MempalaceConfig") as mock_cfg,
         patch("mempalace.layers._get_collection", return_value=mock_col),
@@ -216,6 +245,7 @@ def test_layer1_batch_exception_breaks():
         layer = Layer1(palace_path="/fake")
         result = layer.generate()
 
+    assert len(scan_calls) == 2  # second page attempted, raised, absorbed
     assert "ESSENTIAL STORY" in result
 
 

tests/test_repair.py

@@ -2,6 +2,7 @@
 
 import os
 import sqlite3
+import threading
 from contextlib import closing
 from unittest.mock import MagicMock, call, patch
 
@@ -1358,6 +1359,45 @@ def test_sqlite_integrity_errors_returns_empty_for_healthy_db(tmp_path):
     assert repair.sqlite_integrity_errors(str(palace)) == []
 
 
+def test_sqlite_integrity_errors_waits_out_transient_writer_lock(tmp_path):
+    """A concurrent writer must read as contention, not corruption.
+
+    Python's sqlite3.connect ships a 5-second default busy timeout, but
+    real peer writes (batch mines, curator passes) routinely hold the
+    write lock longer than that. Before the explicit busy_timeout fix,
+    quick_check then failed with "database is locked" and the MCP startup
+    integrity gate (#1818) reported the palace as corrupt — every client
+    failed loudly (and typically reconnect-stormed) for the entire
+    duration of an otherwise healthy batch write.
+
+    The 7-second hold below is deliberate: over the 5 s default that
+    masked the bug, under the 15 s explicit timeout that fixes it.
+    """
+    palace = tmp_path / "palace"
+    palace.mkdir()
+    db_path = palace / "chroma.sqlite3"
+
+    with sqlite3.connect(db_path) as conn:
+        conn.execute("CREATE TABLE dummy(id INTEGER PRIMARY KEY)")
+        conn.commit()
+
+    locker = sqlite3.connect(db_path, check_same_thread=False)
+    locker.execute("BEGIN EXCLUSIVE")
+
+    def _release():
+        locker.commit()
+        locker.close()
+
+    timer = threading.Timer(7.0, _release)
+    timer.start()
+    try:
+        errors = repair.sqlite_integrity_errors(str(palace))
+    finally:
+        timer.join()
+
+    assert errors == []
+
+
 def test_sqlite_integrity_errors_reports_unreadable_sqlite_file(tmp_path):
     palace = tmp_path / "palace"
     palace.mkdir()