From bacb935bb33bb05ec769c5039fd5691d2ef99b48 Mon Sep 17 00:00:00 2001 From: David F Glidden Date: Sun, 28 Jun 2026 06:19:32 +0200 Subject: [PATCH 1/7] fix(layers): order L1 wake-up by recency so it surfaces the latest moments (#1630) L1's generate() scored drawers by importance/emotional_weight/weight, and the docstring promised "prefer high importance, recent filing". But no ingest path (miner, convo_miner, diary, add_drawer) writes any of those fields, so the sort collapsed to insertion order (oldest first) and recency was never consulted. A scoped `wake-up --wing X` therefore surfaced the *oldest* moments: the opposite of useful. Add filed_at (present on every drawer, ISO-8601, lexically chronological) as the secondary sort key. Importance stays primary for the day a scoring pass populates it; filed_at is the effective signal today, making the "recent filing" half of the promise true with data already present. Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> --- mempalace/layers.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/mempalace/layers.py b/mempalace/layers.py index 6acf8523e1..3b78fd2436 100644 --- a/mempalace/layers.py +++ b/mempalace/layers.py @@ -126,12 +126,21 @@ def generate(self) -> str: if not docs: return "## L1 — No memories yet." - # Score each drawer: prefer high importance, recent filing + # Score each drawer: prefer high importance, then most-recent filing. + # NOTE: the ingest pipeline (miner, convo_miner, diary, add_drawer) + # records provenance metadata — wing/room/source/chunk/filed_at — but + # never an evaluative importance/weight field. So `importance` is + # absent on virtually every drawer and ties at the default, which used + # to collapse the sort to insertion order (oldest first). `filed_at` + # is present on every drawer, so it is the *effective* ordering signal: + # newest first. This keeps importance as the primary key for the day a + # scoring pass populates it, while making the "recent filing" half of + # the promise true today with data we already have. scored = [] for doc, meta in zip(docs, metas): meta = meta or {} doc = doc or "" - importance = 3 + importance = 3.0 # Try multiple metadata keys that might carry weight info for key in ("importance", "emotional_weight", "weight"): val = meta.get(key) @@ -141,11 +150,15 @@ def generate(self) -> str: except (ValueError, TypeError): pass break - scored.append((importance, meta, doc)) - - # Sort by importance descending, take top N - scored.sort(key=lambda x: x[0], reverse=True) - top = scored[: self.MAX_DRAWERS] + # filed_at is an ISO-8601 string; ISO strings sort lexicographically + # in chronological order. Coerce to str so a missing/odd value sorts + # oldest rather than raising during the comparison. + recency = str(meta.get("filed_at") or "") + scored.append((importance, recency, meta, doc)) + + # Sort by importance desc, then recency (filed_at) desc; take top N. + scored.sort(key=lambda x: (x[0], x[1]), reverse=True) + top = [(imp, meta, doc) for imp, _recency, meta, doc in scored[: self.MAX_DRAWERS]] # Group by room for readability by_room = defaultdict(list) From 5cbc37ee9b59c1c5daf5748a555b7bc91ca103b8 Mon Sep 17 00:00:00 2001 From: ALaDingAhmad <166673823+ALaDingAhmad@users.noreply.github.com> Date: Sun, 28 Jun 2026 12:31:19 +0800 Subject: [PATCH 2/7] fix(cli): force UTF-8 when reading/writing .gitignore in init (#1648) On Windows, Path.read_text() and open(path, 'a') use locale encoding (GBK on Chinese-locale systems) before PEP 686 / Python 3.15. A valid UTF-8 .gitignore with non-ASCII comments crashes _ensure_mempalace_files_gitignored() with UnicodeDecodeError, which aborts 'mempalace init' on Windows for any user whose .gitignore contains non-ASCII text. Force encoding='utf-8' on both read and append, with errors='replace' on read as a defensive fallback for legacy mixed-encoding files. Co-authored-by: ALaDingAhmad <16530935@qq.com> Co-authored-by: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> --- mempalace/cli.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mempalace/cli.py b/mempalace/cli.py index 2fbecd506d..3043628631 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -259,14 +259,16 @@ def _ensure_mempalace_files_gitignored(project_dir) -> bool: if not (project_path / ".git").exists(): return False gitignore = project_path / ".gitignore" - existing = gitignore.read_text() if gitignore.exists() else "" + # Force UTF-8: Windows defaults to GBK and chokes on non-ASCII .gitignore + # comments, killing auto-init even though the file is valid UTF-8. + existing = gitignore.read_text(encoding="utf-8", errors="replace") if gitignore.exists() else "" existing_lines = {line.strip() for line in existing.splitlines()} missing = [p for p in _MEMPALACE_PROJECT_FILES if p not in existing_lines] if not missing: return False prefix = "" if not existing or existing.endswith("\n") else "\n" block = prefix + "\n# MemPalace per-project files (issue #185)\n" + "\n".join(missing) + "\n" - with open(gitignore, "a") as f: + with open(gitignore, "a", encoding="utf-8") as f: f.write(block) print(f" Added {', '.join(missing)} to {gitignore.name}") return True From 3dad77de984174b05f997a70894d6b9aab0be2b1 Mon Sep 17 00:00:00 2001 From: elitedevs Date: Mon, 6 Jul 2026 00:35:36 -0400 Subject: [PATCH 3/7] fix(repair): treat SQLITE_BUSY as contention, not corruption, in sqlite_integrity_errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PRAGMA quick_check ran with no explicit busy_timeout, so any peer writer holding the write lock longer than Python's 5 s connect default (batch mines, curator passes on a rollback-journal palace) made quick_check fail with 'database is locked'. Callers treat any error as palace corruption — most damagingly the MCP startup integrity gate (#1818), which then refuses to serve every client for the duration of an otherwise healthy batch write, typically triggering a client reconnect storm. Set busy_timeout = 15000 (same pattern as the sqlite fast paths in mcp_server) so transient writers are waited out. Regression test holds the exclusive lock for 7 s — over the 5 s default that masked the bug, under the 15 s explicit timeout. --- mempalace/repair.py | 8 ++++++++ tests/test_repair.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/mempalace/repair.py b/mempalace/repair.py index 248bec5f29..bc50bcfa92 100644 --- a/mempalace/repair.py +++ b/mempalace/repair.py @@ -592,6 +592,14 @@ def sqlite_integrity_errors(palace_path: str) -> list[str]: try: with sqlite3.connect(sqlite_read_uri(sqlite_path), uri=True) as conn: + # SQLITE_BUSY from a concurrent writer is contention, not + # corruption. Without a busy_timeout, quick_check fails + # instantly with "database is locked" whenever another + # process holds the write lock (e.g. a long batch write on a + # rollback-journal palace), and callers — including the MCP + # startup integrity gate (#1818) — misreport a healthy palace + # as corrupt. Wait out transient writers before giving up. + conn.execute("PRAGMA busy_timeout = 15000") rows = conn.execute("PRAGMA quick_check").fetchall() except sqlite3.Error as e: return [f"PRAGMA quick_check failed: {e}"] diff --git a/tests/test_repair.py b/tests/test_repair.py index ff1d65f74b..0ec4d8e639 100644 --- a/tests/test_repair.py +++ b/tests/test_repair.py @@ -2,6 +2,7 @@ import os import sqlite3 +import threading from contextlib import closing from unittest.mock import MagicMock, call, patch @@ -1358,6 +1359,45 @@ def test_sqlite_integrity_errors_returns_empty_for_healthy_db(tmp_path): assert repair.sqlite_integrity_errors(str(palace)) == [] +def test_sqlite_integrity_errors_waits_out_transient_writer_lock(tmp_path): + """A concurrent writer must read as contention, not corruption. + + Python's sqlite3.connect ships a 5-second default busy timeout, but + real peer writes (batch mines, curator passes) routinely hold the + write lock longer than that. Before the explicit busy_timeout fix, + quick_check then failed with "database is locked" and the MCP startup + integrity gate (#1818) reported the palace as corrupt — every client + failed loudly (and typically reconnect-stormed) for the entire + duration of an otherwise healthy batch write. + + The 7-second hold below is deliberate: over the 5 s default that + masked the bug, under the 15 s explicit timeout that fixes it. + """ + palace = tmp_path / "palace" + palace.mkdir() + db_path = palace / "chroma.sqlite3" + + with sqlite3.connect(db_path) as conn: + conn.execute("CREATE TABLE dummy(id INTEGER PRIMARY KEY)") + conn.commit() + + locker = sqlite3.connect(db_path, check_same_thread=False) + locker.execute("BEGIN EXCLUSIVE") + + def _release(): + locker.commit() + locker.close() + + timer = threading.Timer(7.0, _release) + timer.start() + try: + errors = repair.sqlite_integrity_errors(str(palace)) + finally: + timer.join() + + assert errors == [] + + def test_sqlite_integrity_errors_reports_unreadable_sqlite_file(tmp_path): palace = tmp_path / "palace" palace.mkdir() From 994d76f52af296233c2354bafc64f8efa94eaebf Mon Sep 17 00:00:00 2001 From: elitedevs Date: Tue, 7 Jul 2026 14:39:14 -0400 Subject: [PATCH 4/7] fix(layers): L1 wake-up scans full corpus + recency-decayed scoring Layer1 previously fetched only the first MAX_SCAN=2000 drawers in insertion order and sorted by importance alone, so wake-up permanently showed the oldest high-importance drawers and never surfaced anything filed after the cap. With ~75K drawers the essential story was frozen in spring 2026. - Metadata-only scan of the full corpus (documents fetched only for the top MAX_DRAWERS winners), MAX_SCAN raised to a safety valve - Score = importance + RECENCY_WEIGHT * 0.5^(age_days/14): a fresh default-importance drawer outranks a stale importance-5 one after ~2 half-lives, so L1 tracks the present, not the archive --- mempalace/layers.py | 67 +++++++++++++++++++++++++++++++++------------ 1 file changed, 50 insertions(+), 17 deletions(-) diff --git a/mempalace/layers.py b/mempalace/layers.py index 1d6ceeb50c..91220b6a45 100644 --- a/mempalace/layers.py +++ b/mempalace/layers.py @@ -18,6 +18,7 @@ import os import sys +from datetime import datetime, timezone from pathlib import Path from collections import defaultdict @@ -87,7 +88,9 @@ class Layer1: MAX_DRAWERS = 15 # at most 15 moments in wake-up MAX_CHARS = 3200 # hard cap on total L1 text (~800 tokens) - MAX_SCAN = 2000 # don't scan more than this for L1 generation + MAX_SCAN = 200_000 # safety valve on metadata scan (full corpus expected) + RECENCY_WEIGHT = 2.0 # max score boost for a just-filed drawer + RECENCY_HALF_LIFE_DAYS = 14.0 # boost halves every N days def __init__(self, palace_path: str = None, wing: str = None): cfg = MempalaceConfig() @@ -101,37 +104,42 @@ def generate(self) -> str: except Exception: return "## L1 — No palace found. Run: mempalace mine " - # Fetch all drawers in batches to avoid SQLite variable limit (~999) + # Scan metadata for the FULL corpus (no documents — cheap), batched + # to avoid SQLite variable limit (~999). Documents are fetched later + # for only the top MAX_DRAWERS winners. _BATCH = 500 - docs, metas = [], [] + ids, metas = [], [] offset = 0 while True: - kwargs = {"include": ["documents", "metadatas"], "limit": _BATCH, "offset": offset} + kwargs = {"include": ["metadatas"], "limit": _BATCH, "offset": offset} if self.wing: kwargs["where"] = {"wing": self.wing} try: batch = col.get(**kwargs) except Exception: break - batch_docs = batch.get("documents", []) + batch_ids = batch.get("ids", []) batch_metas = batch.get("metadatas", []) - if not batch_docs: + if not batch_ids: break - docs.extend(batch_docs) + ids.extend(batch_ids) metas.extend(batch_metas) - offset += len(batch_docs) - if len(batch_docs) < _BATCH or len(docs) >= self.MAX_SCAN: + offset += len(batch_ids) + if len(batch_ids) < _BATCH or len(ids) >= self.MAX_SCAN: break - if not docs: + if not ids: return "## L1 — No memories yet." - # Score each drawer: prefer high importance, recent filing + # Score each drawer: importance + recency decay (half-life + # RECENCY_HALF_LIFE_DAYS). A fresh default-importance drawer can + # outrank a stale high-importance one, so the essential story tracks + # the present, not the archive. + now = datetime.now(timezone.utc) scored = [] - for doc, meta in zip(docs, metas): + for did, meta in zip(ids, metas): meta = meta or {} - doc = doc or "" - importance = 3 + importance = 3.0 # Try multiple metadata keys that might carry weight info for key in ("importance", "emotional_weight", "weight"): val = meta.get(key) @@ -141,11 +149,36 @@ def generate(self) -> str: except (ValueError, TypeError): pass break - scored.append((importance, meta, doc)) + age_days = None + for key in ("authored_at", "created_at"): + raw = meta.get(key) + if raw: + try: + ts = datetime.fromisoformat(str(raw)) + if ts.tzinfo is None: + ts = ts.replace(tzinfo=timezone.utc) + age_days = max((now - ts).total_seconds() / 86400.0, 0.0) + except (ValueError, TypeError): + pass + break + recency = ( + 0.0 + if age_days is None + else self.RECENCY_WEIGHT * 0.5 ** (age_days / self.RECENCY_HALF_LIFE_DAYS) + ) + scored.append((importance + recency, did, meta)) - # Sort by importance descending, take top N + # Sort by combined score descending, take top N, then fetch + # documents for just the winners scored.sort(key=lambda x: x[0], reverse=True) - top = scored[: self.MAX_DRAWERS] + winners = scored[: self.MAX_DRAWERS] + try: + fetched = col.get(ids=[d for _s, d, _m in winners], include=["documents"]) + except Exception: + return "## L1 — No memories yet." + doc_by_id = dict(zip(fetched.get("ids", []), fetched.get("documents", []))) + + top = [(score, meta, doc_by_id.get(did) or "") for score, did, meta in winners] # Group by room for readability by_room = defaultdict(list) From ace6284290de242fdbd7daae8b6faa53010a9e33 Mon Sep 17 00:00:00 2001 From: elitedevs Date: Tue, 7 Jul 2026 14:41:04 -0400 Subject: [PATCH 5/7] =?UTF-8?q?fix(layers):=20include=20filed=5Fat=20in=20?= =?UTF-8?q?L1=20recency=20keys=20=E2=80=94=20chroma=20metadata=20uses=20fi?= =?UTF-8?q?led=5Fat,=20not=20authored=5Fat?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mempalace/layers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mempalace/layers.py b/mempalace/layers.py index 91220b6a45..3f326b8a25 100644 --- a/mempalace/layers.py +++ b/mempalace/layers.py @@ -150,7 +150,7 @@ def generate(self) -> str: pass break age_days = None - for key in ("authored_at", "created_at"): + for key in ("authored_at", "created_at", "filed_at"): raw = meta.get(key) if raw: try: From 8b88ef000f56355fe0b25e1bdb8395766486364b Mon Sep 17 00:00:00 2001 From: elitedevs Date: Tue, 7 Jul 2026 14:43:13 -0400 Subject: [PATCH 6/7] fix(layers): L1 recency weight 3.0 + dedupe winners by source_file At weight 2.0 a fresh default-importance drawer scored 5.00, losing by ~0.05 to 2-month-old weight-5 drawers whose residual recency kept them at 5.03-5.07. Weight 3.0 gives today's work a decisive edge for ~10 days while evergreen weight-5 facts still fill remaining slots. Dedupe: multi-chunk files (e.g. 5 chunks of one SKILL.md) were taking 5 of 15 L1 slots; cap at one drawer per source_file. --- mempalace/layers.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/mempalace/layers.py b/mempalace/layers.py index 3f326b8a25..a0c4940f73 100644 --- a/mempalace/layers.py +++ b/mempalace/layers.py @@ -89,7 +89,7 @@ class Layer1: MAX_DRAWERS = 15 # at most 15 moments in wake-up MAX_CHARS = 3200 # hard cap on total L1 text (~800 tokens) MAX_SCAN = 200_000 # safety valve on metadata scan (full corpus expected) - RECENCY_WEIGHT = 2.0 # max score boost for a just-filed drawer + RECENCY_WEIGHT = 3.0 # max score boost for a just-filed drawer RECENCY_HALF_LIFE_DAYS = 14.0 # boost halves every N days def __init__(self, palace_path: str = None, wing: str = None): @@ -168,10 +168,19 @@ def generate(self) -> str: ) scored.append((importance + recency, did, meta)) - # Sort by combined score descending, take top N, then fetch - # documents for just the winners + # Sort by combined score descending, take top N — at most one drawer + # per source_file so multi-chunk files don't monopolize L1 — then + # fetch documents for just the winners scored.sort(key=lambda x: x[0], reverse=True) - winners = scored[: self.MAX_DRAWERS] + winners, seen_sources = [], set() + for score, did, meta in scored: + source = meta.get("source_file") or did + if source in seen_sources: + continue + seen_sources.add(source) + winners.append((score, did, meta)) + if len(winners) >= self.MAX_DRAWERS: + break try: fetched = col.get(ids=[d for _s, d, _m in winners], include=["documents"]) except Exception: From de631286f1dc7072630632deb4be978931a1daf7 Mon Sep 17 00:00:00 2001 From: elitedevs Date: Tue, 14 Jul 2026 10:51:43 -0400 Subject: [PATCH 7/7] test(layers): update L1 mocks for the full-corpus-scan access pattern MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Layer1 rewrite (994d76f..8b88ef0) reads a paginated metadata-only scan keyed on ids, then fetches documents for just the winners. The shared mock still returned the old single-shot documents+metadatas shape with no ids, so generate() saw an empty palace and 7 tests failed without exercising the new scoring at all. Rework _mock_chromadb_for_layer to emulate the real access pattern and rewrite test_layer1_batch_exception_breaks so the first page fills a whole scan batch — the only way the mid-scan exception path can fire. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01XZbwhCTsJk5aLjmrbRVMTL --- tests/test_layers.py | 52 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/tests/test_layers.py b/tests/test_layers.py index d4c54ce7cf..a1f3bf6040 100644 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -71,13 +71,24 @@ def test_layer0_default_path(): def _mock_chromadb_for_layer(docs, metas, monkeypatch=None): - """Return a mock collection whose get() returns docs/metas.""" + """Return a mock collection emulating Layer1's two-step access pattern. + + First a paginated metadata-only scan (returns ids + metadatas, + honouring limit/offset), then a documents fetch for the winning ids. + The old single-shot documents+metadatas shape predates the + full-corpus-scan rewrite and made every test see an empty palace. + """ mock_col = MagicMock() - # First batch returns data, second batch returns empty (end of pagination) - mock_col.get.side_effect = [ - {"documents": docs, "metadatas": metas}, - {"documents": [], "metadatas": []}, - ] + all_ids = [f"id{i}" for i in range(len(docs))] + doc_by_id = dict(zip(all_ids, docs)) + + def fake_get(ids=None, include=None, limit=None, offset=0, where=None): + if ids is not None: # second step: documents for the winners + return {"ids": list(ids), "documents": [doc_by_id.get(d) for d in ids]} + end = len(all_ids) if limit is None else min(offset + limit, len(all_ids)) + return {"ids": all_ids[offset:end], "metadatas": metas[offset:end]} + + mock_col.get.side_effect = fake_get return mock_col @@ -202,12 +213,30 @@ def test_layer1_importance_from_various_keys(): def test_layer1_batch_exception_breaks(): - """If col.get raises on a batch, loop breaks gracefully.""" + """If col.get raises mid-scan, the loop breaks gracefully. + + L1 must still render from the drawers scanned so far — a transient + chroma error partway through the corpus scan should degrade to a + shorter essential story, not an empty one. The first page must be + exactly one scan-batch long so Layer1 asks for a second page. + """ + batch = 500 # Layer1's _BATCH page size + all_ids = [f"id{i}" for i in range(batch)] + all_metas = [{"room": "r", "source_file": f"s{i}.txt"} for i in range(batch)] + doc_by_id = {d: f"memory {d}" for d in all_ids} + scan_calls = [] + + def fake_get(ids=None, include=None, limit=None, offset=0, where=None): + if ids is not None: # documents fetch for the winners + return {"ids": list(ids), "documents": [doc_by_id[d] for d in ids]} + scan_calls.append(offset) + if len(scan_calls) > 1: + raise RuntimeError("batch error") + return {"ids": all_ids[:limit], "metadatas": all_metas[:limit]} + mock_col = MagicMock() - mock_col.get.side_effect = [ - {"documents": ["doc1"], "metadatas": [{"room": "r"}]}, - RuntimeError("batch error"), - ] + mock_col.get.side_effect = fake_get + with ( patch("mempalace.layers.MempalaceConfig") as mock_cfg, patch("mempalace.layers._get_collection", return_value=mock_col), @@ -216,6 +245,7 @@ def test_layer1_batch_exception_breaks(): layer = Layer1(palace_path="/fake") result = layer.generate() + assert len(scan_calls) == 2 # second page attempted, raised, absorbed assert "ESSENTIAL STORY" in result