From 27a158f7a89721cd14fd3dbd6c4ec7c30d08d87e Mon Sep 17 00:00:00 2001 From: jp Date: Tue, 5 May 2026 17:46:31 -0700 Subject: [PATCH 1/2] refactor(mcp): retire mempalace_session_recovery collection + read tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to fix/drop-checkpoint-write-path. With nothing writing to the recovery collection anymore (the hooks moved to verbatim-only on the parent branch), the read paths and the migration that fed the collection are dead code. Delete them. Removed in mempalace/: - palace.py: _SESSION_RECOVERY_COLLECTION constant, get_session_recovery_collection(), _CHECKPOINT_TOPICS tuple - mcp_server.py: _get_session_recovery_collection(), _recovery_collection_cache global, the topic-routing branch in tool_diary_write (now always lands in mempalace_drawers), tool_session_recovery_read() and its TOOLS dict registration - migrate.py: migrate_checkpoints_to_recovery() and the dependent _CHECKPOINT_TOPICS import - cli.py: cmd_repair --mode reorganize handler + the choice flag Removed in tests/: - tests/test_session_recovery.py — entire file (recovery-collection module test suite) - tests/test_migrate.py: TestMigrateCheckpointsToRecovery class - tests/test_mcp_server.py: TestCheckpointRouting and TestSessionRecoveryRead classes Removed in docs/: - website/reference/mcp-tools.md: mempalace_session_recovery_read section (caught by the readme-claims meta-test before deploy) Production data on disks still has the mempalace_session_recovery collection with its 763 archived entries — this PR's code change makes the collection unreachable through any MCP/CLI path, but does NOT delete the on-disk data. A separate one-shot script (scripts/phase2_purge_recovery.py per the spec) handles the collection-level delete after deploy. JP signed off on hard-delete in the spec ack. Net diff: 12 files, −1300 lines / +30 lines. 1540 tests pass, 1 skipped. Ruff lint + format clean. Stacked on jphein/mempalace#3 (which stops new writes); will auto-rebase against main when that merges. Co-Authored-By: Claude Opus 4.7 (1M context) --- mempalace/cli.py | 32 +---- mempalace/mcp_server.py | 195 +----------------------------- mempalace/migrate.py | 93 --------------- mempalace/palace.py | 24 ---- tests/test_mcp_server.py | 209 --------------------------------- tests/test_migrate.py | 114 ------------------ tests/test_session_recovery.py | 59 ---------- website/reference/mcp-tools.md | 17 --- 8 files changed, 7 insertions(+), 736 deletions(-) delete mode 100644 tests/test_session_recovery.py diff --git a/mempalace/cli.py b/mempalace/cli.py index b0b9b2834a..9d1a53231d 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -843,40 +843,16 @@ def cmd_repair_status(args): def cmd_repair(args): - """Rebuild palace vector index, or reorganize derivative drawers.""" + """Rebuild palace vector index.""" import shutil from .backends.chroma import ChromaBackend - from .migrate import ( - confirm_destructive_action, - contains_palace_database, - migrate_checkpoints_to_recovery, - ) + from .migrate import confirm_destructive_action, contains_palace_database from .repair import TruncationDetected, check_extraction_safety palace_path = os.path.abspath( os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path ) - # mode=reorganize: move topic=checkpoint drawers from main → recovery. - # Non-destructive, idempotent. Designed to run on first daemon startup - # post-upgrade to land the checkpoint-collection split (phase D). - if getattr(args, "mode", "rebuild") == "reorganize": - if not os.path.isdir(palace_path) or not contains_palace_database(palace_path): - print(f"\n No palace database found at {palace_path}") - return - print(f"\n{'=' * 55}") - print(" MemPalace Reorganize — checkpoint → session-recovery") - print(f"{'=' * 55}\n") - print(f" Palace: {palace_path}") - moved = migrate_checkpoints_to_recovery(palace_path) - if moved == 0: - print(" Nothing to move — palace is already reorganized.") - else: - print(f" Moved {moved} checkpoint drawer(s) to mempalace_session_recovery.") - print(" mempalace_search now queries content-only.") - print(f"\n{'=' * 55}\n") - return - if getattr(args, "mode", "legacy") == "max-seq-id": from .repair import repair_max_seq_id @@ -1417,13 +1393,11 @@ def main(): ) p_repair.add_argument( "--mode", - choices=["rebuild", "legacy", "reorganize", "max-seq-id"], + choices=["rebuild", "legacy", "max-seq-id"], default="legacy", help=( "rebuild/legacy: full-palace HNSW rebuild via extract + re-upsert (default; " "rebuild and legacy are synonyms). " - "reorganize: move existing topic=checkpoint drawers from the main " - "collection into mempalace_session_recovery (idempotent; safe to re-run). " "max-seq-id: un-poison max_seq_id rows corrupted by the legacy 0.6.x shim." ), ) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 64a1ca4508..e771af83b5 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -67,9 +67,7 @@ hnsw_capacity_status, ) from .query_sanitizer import sanitize_query # noqa: E402 -from .palace import _CHECKPOINT_TOPICS # noqa: E402 from .searcher import search_memories # noqa: E402 -from .palace import _SESSION_RECOVERY_COLLECTION # noqa: E402 from .palace_graph import ( # noqa: E402 traverse, find_tunnels, @@ -115,7 +113,6 @@ def _parse_args(): _client_cache = None _collection_cache = None -_recovery_collection_cache = None _palace_db_inode = 0 # inode of chroma.sqlite3 at cache time _palace_db_mtime = 0.0 # mtime of chroma.sqlite3 at cache time @@ -234,7 +231,6 @@ def _get_client(): global \ _client_cache, \ _collection_cache, \ - _recovery_collection_cache, \ _palace_db_inode, \ _palace_db_mtime, \ _metadata_cache, \ @@ -270,7 +266,6 @@ def _get_client(): _refresh_vector_disabled_flag() _client_cache = ChromaBackend.make_client(_config.palace_path) _collection_cache = None - _recovery_collection_cache = None _metadata_cache = None _metadata_cache_time = 0 _palace_db_inode = current_inode @@ -343,43 +338,6 @@ def _get_collection(create=False): return None -def _get_session_recovery_collection(create=False): - """Return the session-recovery collection, caching between calls. - - Stop-hook checkpoint diary entries route here instead of the main - ``mempalace_drawers`` collection so they don't dominate - ``mempalace_search`` results. Mirrors :func:`_get_collection`'s - shape (same client cache, same ``_pin_hnsw_threads`` retrofit, - same get-then-create / embedding_function= plumbing per #1262 / - #1289 / #1303). - """ - global _recovery_collection_cache - try: - client = _get_client() - if create: - ef = ChromaBackend._resolve_embedding_function() - ef_kwargs = {"embedding_function": ef} if ef is not None else {} - try: - raw = client.get_collection(_SESSION_RECOVERY_COLLECTION, **ef_kwargs) - except _ChromaNotFoundError: - raw = client.create_collection( - _SESSION_RECOVERY_COLLECTION, - metadata={"hnsw:space": "cosine", "hnsw:num_threads": 1}, - **ef_kwargs, - ) - _pin_hnsw_threads(raw) - _recovery_collection_cache = ChromaCollection(raw) - elif _recovery_collection_cache is None: - ef = ChromaBackend._resolve_embedding_function() - ef_kwargs = {"embedding_function": ef} if ef is not None else {} - raw = client.get_collection(_SESSION_RECOVERY_COLLECTION, **ef_kwargs) - _pin_hnsw_threads(raw) - _recovery_collection_cache = ChromaCollection(raw) - return _recovery_collection_cache - except Exception: - return None - - def _no_palace(): return { "error": "No palace found", @@ -1229,11 +1187,9 @@ def tool_diary_write( This is the agent's personal journal — observations, thoughts, what it worked on, what it noticed, what it thinks matters. - When ``topic`` is a checkpoint topic (``checkpoint`` / ``auto-save``) - the entry is routed to the dedicated ``mempalace_session_recovery`` - collection so it doesn't dominate ``mempalace_search`` results. - Pass ``session_id`` to enable filtering checkpoints by session via - ``mempalace_session_recovery_read``. + All entries land in the main ``mempalace_drawers`` collection — the + earlier dedicated checkpoint collection has been retired (verbatim + transcripts already cover the recovery use case). Note: ``agent_name`` is normalized to lowercase before storage so that diary reads are case-insensitive (see #1243). "Claude", @@ -1251,13 +1207,7 @@ def tool_diary_write( else: wing = f"wing_{agent_name.replace(' ', '_')}" room = "diary" - # Stop-hook auto-save checkpoint entries land in the dedicated - # session-recovery collection so they don't dominate vector ranking - # in mempalace_search. Read via mempalace_session_recovery_read. - if topic in _CHECKPOINT_TOPICS: - col = _get_session_recovery_collection(create=True) - else: - col = _get_collection(create=True) + col = _get_collection(create=True) if not col: return _no_palace() @@ -1383,103 +1333,6 @@ def tool_diary_read(agent_name: str, last_n: int = 10, wing: str = ""): return {"error": "Failed to read diary entries"} -def tool_session_recovery_read( - session_id: str = "", - agent: str = "", - since: str = "", - until: str = "", - wing: str = "", - limit: int = 50, -): - """ - Read Stop-hook auto-save checkpoint entries from the dedicated - ``mempalace_session_recovery`` collection. Used for session - recovery, hook auditing, and "what was I doing 2 hours ago" lookup. - - All filters are optional — empty string / zero means "no filter": - - - ``session_id``: only entries written under this Claude Code session - - ``agent``: only entries from this agent (typically ``session-hook``) - - ``since`` / ``until``: ISO date strings, inclusive bounds on filed_at - - ``wing``: only entries from this project wing - - ``limit``: maximum number of entries to return (default 50, max 500) - - Entries are returned sorted by ``filed_at`` descending (newest first). - """ - try: - if agent: - agent = sanitize_name(agent, "agent") - if wing: - wing = sanitize_name(wing) - except ValueError as e: - return {"error": str(e), "entries": [], "total": 0} - - limit = max(1, min(int(limit), 500)) - col = _get_session_recovery_collection() - if not col: - # Recovery collection has never been created — no checkpoints - # have been written yet via the new routing. Return empty rather - # than erroring; caller's most likely interpretation is "no - # session-recovery data exists yet". - return {"entries": [], "total": 0} - - # Build metadata where-clause from non-empty filters. ChromaDB needs - # a single condition or an explicit $and — we assemble accordingly. - conditions = [] - if session_id: - conditions.append({"session_id": session_id}) - if agent: - conditions.append({"agent": agent}) - if wing: - conditions.append({"wing": wing}) - - where = None - if len(conditions) == 1: - where = conditions[0] - elif len(conditions) > 1: - where = {"$and": conditions} - - try: - kwargs = {"include": ["documents", "metadatas"], "limit": 10000} - if where is not None: - kwargs["where"] = where - results = col.get(**kwargs) - except Exception: - logger.exception("session_recovery_read failed") - return {"error": "Failed to read recovery entries", "entries": [], "total": 0} - - if not results.get("ids"): - return {"entries": [], "total": 0} - - entries = [] - for drawer_id, doc, meta in zip(results["ids"], results["documents"], results["metadatas"]): - # Defensive: ChromaDB may return None metadata for legacy / - # partial-write drawers (cf. #999, #1094, #1201). Coerce to {}. - meta = meta or {} - filed_at = meta.get("filed_at", "") or "" - if since and filed_at and filed_at < since: - continue - if until and filed_at and filed_at > until: - continue - entries.append( - { - "drawer_id": drawer_id, - "date": meta.get("date", ""), - "timestamp": filed_at, - "topic": meta.get("topic", ""), - "agent": meta.get("agent", ""), - "wing": meta.get("wing", ""), - "session_id": meta.get("session_id", ""), - "content": doc, - } - ) - - entries.sort(key=lambda x: x["timestamp"], reverse=True) - entries = entries[:limit] - - return {"entries": entries, "total": len(entries)} - - def tool_hook_settings(silent_save: bool = None, desktop_toast: bool = None): """ Get or set hook behavior settings. @@ -2001,46 +1854,6 @@ def tool_reconnect(): }, "handler": tool_diary_read, }, - "mempalace_session_recovery_read": { - "description": ( - "Read Stop-hook auto-save checkpoint entries from the dedicated " - "session-recovery collection. Use for session recovery, hook " - "auditing, or 'what was I doing 2 hours ago' lookup. Filters are " - "all optional; empty string / 0 means 'no filter'. Returns " - "entries newest-first." - ), - "input_schema": { - "type": "object", - "properties": { - "session_id": { - "type": "string", - "description": "Filter by Claude Code session id.", - }, - "agent": { - "type": "string", - "description": "Filter by agent name (typically 'session-hook').", - }, - "since": { - "type": "string", - "description": "ISO datetime string — entries strictly newer than this.", - }, - "until": { - "type": "string", - "description": "ISO datetime string — entries strictly older than this.", - }, - "wing": { - "type": "string", - "description": "Filter by project wing.", - }, - "limit": { - "type": "integer", - "description": "Max entries to return (default 50, max 500).", - }, - }, - "required": [], - }, - "handler": tool_session_recovery_read, - }, "mempalace_hook_settings": { "description": ( "Get or set hook behavior. silent_save: True = save directly " diff --git a/mempalace/migrate.py b/mempalace/migrate.py index 837dad4f7a..76aa054faa 100644 --- a/mempalace/migrate.py +++ b/mempalace/migrate.py @@ -25,8 +25,6 @@ from collections import defaultdict from datetime import datetime -from .palace import _CHECKPOINT_TOPICS - def _restore_stale_palace(palace_path: str, stale_path: str) -> None: """Roll back a failed swap. @@ -285,94 +283,3 @@ def migrate(palace_path: str, dry_run: bool = False, confirm: bool = False): print(f"\n{'=' * 60}\n") return True - - -# --------------------------------------------------------------------------- -# Phase D: move existing topic=checkpoint drawers from the main searchable -# collection into the dedicated session-recovery collection. The main -# collection is the *verbatim* store — chats, tool calls, mined files — -# and should not carry derivative summary entries (Stop-hook auto-save -# checkpoints) that wreck vector ranking. See spec at -# docs/superpowers/specs/2026-04-25-checkpoint-collection-split.md. -# --------------------------------------------------------------------------- - - -def migrate_checkpoints_to_recovery(palace_path: str, batch_size: int = 1000) -> int: - """Move all topic=checkpoint drawers from main → recovery collection. - - Idempotent: re-running on a fully-migrated palace returns 0. Drawer - IDs and metadata are preserved exactly. The original drawer is added - to the recovery collection first, then deleted from main — so a - crash mid-migration leaves a duplicate (recoverable) rather than a - loss. - - Returns the number of drawers moved on this invocation. - """ - from .palace import get_collection, get_session_recovery_collection - - palace_path = os.path.abspath(os.path.expanduser(palace_path)) - if not contains_palace_database(palace_path): - return 0 - - try: - main = get_collection(palace_path, create=False) - except Exception: - # Palace dir exists but main collection isn't readable — nothing to migrate. - return 0 - recovery = get_session_recovery_collection(palace_path, create=True) - - moved_total = 0 - offset = 0 - # Walk the main collection in pages. We deliberately don't use a - # ``where={"topic": {"$in": _CHECKPOINT_TOPICS}}`` clause: the - # ChromaDB 1.5.x filter-planner bug surfaced earlier this week with - # ``$in``/``$nin`` on metadata. Pull batches plain and filter in - # Python. - while True: - try: - batch = main.get( - limit=batch_size, - offset=offset, - include=["documents", "metadatas"], - ) - except Exception: - # Defensive: a chromadb error on the read path stops the - # migration cleanly without corrupting state. Caller can retry. - break - - ids = batch.get("ids") or [] - if not ids: - break - - docs = batch.get("documents") or [] - metas = batch.get("metadatas") or [] - - ids_to_move: list = [] - docs_to_move: list = [] - metas_to_move: list = [] - - for i, doc, meta in zip(ids, docs, metas): - meta = meta or {} - if meta.get("topic") in _CHECKPOINT_TOPICS: - ids_to_move.append(i) - docs_to_move.append(doc) - metas_to_move.append(meta) - - if ids_to_move: - recovery.add( - ids=ids_to_move, - documents=docs_to_move, - metadatas=metas_to_move, - ) - main.delete(ids=ids_to_move) - moved_total += len(ids_to_move) - # The delete shrinks main; the *next* page would skip - # ``len(ids_to_move)`` drawers. Reset offset so we re-page - # over the (now smaller) collection from the same logical - # position — equivalent to the standard "delete-during-walk" - # fixup. - continue - - offset += len(ids) - - return moved_total diff --git a/mempalace/palace.py b/mempalace/palace.py index fd3ee2e8ba..2fd26eb75f 100644 --- a/mempalace/palace.py +++ b/mempalace/palace.py @@ -71,30 +71,6 @@ def get_closets_collection(palace_path: str, create: bool = True): return get_collection(palace_path, collection_name="mempalace_closets", create=create) -# Stop-hook auto-save checkpoint diary entries are routed to this -# dedicated collection so they don't dominate ``mempalace_search`` -# results in the main ``mempalace_drawers`` collection. Read via the -# ``mempalace_session_recovery_read`` MCP tool. See -# ``docs/superpowers/specs/2026-04-25-checkpoint-collection-split.md``. -_SESSION_RECOVERY_COLLECTION = "mempalace_session_recovery" - -# Topic values whose drawers belong in ``_SESSION_RECOVERY_COLLECTION`` -# rather than the searchable main collection. ``checkpoint`` is canonical; -# ``auto-save`` is a legacy synonym from older palace-daemon hook clients. -# Used by write-side routing in ``tool_diary_write`` and by the data -# migration in ``migrate_checkpoints_to_recovery``. -_CHECKPOINT_TOPICS = ("checkpoint", "auto-save") - - -def get_session_recovery_collection(palace_path: str, create: bool = True): - """Get the session-recovery collection — Stop-hook checkpoint storage.""" - return get_collection( - palace_path, - collection_name=_SESSION_RECOVERY_COLLECTION, - create=create, - ) - - CLOSET_CHAR_LIMIT = 1500 # fill closet until ~1500 chars, then start a new one CLOSET_EXTRACT_WINDOW = 5000 # how many chars of source content to scan for entities/topics diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 16e9147dd6..ec9562b8ce 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -1143,212 +1143,3 @@ def _spy_create(self, name, **kwargs): for kwargs in captured["get"]: assert "embedding_function" in kwargs assert kwargs["embedding_function"] is not None - - -class TestCheckpointRouting: - """Phase B — Stop-hook checkpoint diary entries route to the - dedicated ``mempalace_session_recovery`` collection rather than the - searchable ``mempalace_drawers`` collection. See - ``docs/superpowers/specs/2026-04-25-checkpoint-collection-split.md``. - """ - - def test_diary_write_routes_checkpoint_to_recovery(self, monkeypatch, config, palace_path, kg): - _patch_mcp_server(monkeypatch, config, kg) - _client, _col = _get_collection(palace_path, create=True) - del _client - from mempalace.mcp_server import tool_diary_write - from mempalace.palace import ( - get_collection as _get_main, - get_session_recovery_collection, - ) - - result = tool_diary_write( - agent_name="hooks", - entry="CHECKPOINT: 2026-04-25 18:00 — session ended", - topic="checkpoint", - ) - assert result["success"] is True - - main = _get_main(palace_path, create=True) - recovery = get_session_recovery_collection(palace_path, create=True) - assert main.count() == 0 - assert recovery.count() == 1 - - def test_diary_write_routes_auto_save_to_recovery(self, monkeypatch, config, palace_path, kg): - """The legacy ``auto-save`` topic synonym also routes to recovery.""" - _patch_mcp_server(monkeypatch, config, kg) - _client, _col = _get_collection(palace_path, create=True) - del _client - from mempalace.mcp_server import tool_diary_write - from mempalace.palace import ( - get_collection as _get_main, - get_session_recovery_collection, - ) - - tool_diary_write( - agent_name="hooks", - entry="auto-save legacy entry", - topic="auto-save", - ) - - main = _get_main(palace_path, create=True) - recovery = get_session_recovery_collection(palace_path, create=True) - assert main.count() == 0 - assert recovery.count() == 1 - - def test_diary_write_routes_general_to_main(self, monkeypatch, config, palace_path, kg): - """Non-checkpoint topics keep the existing behavior — main collection.""" - _patch_mcp_server(monkeypatch, config, kg) - _client, _col = _get_collection(palace_path, create=True) - del _client - from mempalace.mcp_server import tool_diary_write - from mempalace.palace import ( - get_collection as _get_main, - get_session_recovery_collection, - ) - - tool_diary_write( - agent_name="TestAgent", - entry="A regular journal entry about today's work.", - topic="musings", - ) - - main = _get_main(palace_path, create=True) - recovery = get_session_recovery_collection(palace_path, create=True) - assert main.count() == 1 - assert recovery.count() == 0 - - def test_search_does_not_see_new_checkpoints(self, monkeypatch, config, palace_path, kg): - """Regression: a checkpoint written via ``tool_diary_write`` cannot - surface in ``search_memories`` because it lives in a separate - collection — not merely post-filtered out.""" - _patch_mcp_server(monkeypatch, config, kg) - _client, _col = _get_collection(palace_path, create=True) - del _client - from mempalace.mcp_server import tool_diary_write - from mempalace.searcher import search_memories - - unique = "engram-quasar-fluxor" - tool_diary_write( - agent_name="hooks", - entry=f"CHECKPOINT: 2026-04-25 — {unique}", - topic="checkpoint", - ) - - results = search_memories( - query=unique, - palace_path=palace_path, - n_results=10, - ) - hits = results.get("results", []) if isinstance(results, dict) else [] - assert all("CHECKPOINT:" not in (h.get("content") or "") for h in hits) - - -class TestSessionRecoveryRead: - """Phase C — ``tool_session_recovery_read`` MCP handler reads from the - recovery collection only, with filters for session_id, agent, - since/until, wing, and limit. See - ``docs/superpowers/specs/2026-04-25-checkpoint-collection-split.md``. - """ - - def test_returns_empty_when_no_recovery_data(self, monkeypatch, config, palace_path, kg): - """Empty recovery collection (no checkpoints written) returns empty - results without errors.""" - _patch_mcp_server(monkeypatch, config, kg) - _client, _col = _get_collection(palace_path, create=True) - del _client - from mempalace.mcp_server import tool_session_recovery_read - - result = tool_session_recovery_read() - assert result["entries"] == [] - assert result["total"] == 0 - - def test_filters_by_session_id(self, monkeypatch, config, palace_path, kg): - """Three checkpoints under different session_ids; query for one - returns only that one.""" - _patch_mcp_server(monkeypatch, config, kg) - _client, _col = _get_collection(palace_path, create=True) - del _client - from mempalace.mcp_server import tool_diary_write, tool_session_recovery_read - - tool_diary_write( - agent_name="hooks", - entry="A", - topic="checkpoint", - session_id="alpha", - ) - tool_diary_write( - agent_name="hooks", - entry="B", - topic="checkpoint", - session_id="beta", - ) - tool_diary_write( - agent_name="hooks", - entry="C", - topic="checkpoint", - session_id="gamma", - ) - - result = tool_session_recovery_read(session_id="beta") - assert result["total"] == 1 - assert result["entries"][0]["content"] == "B" - # drawer_id is plumbed through so callers can build links back to - # the underlying drawer (mempalace_get_drawer, citation popovers). - assert result["entries"][0]["drawer_id"] - assert result["entries"][0]["drawer_id"].startswith("diary_") - - def test_filters_by_agent(self, monkeypatch, config, palace_path, kg): - _patch_mcp_server(monkeypatch, config, kg) - _client, _col = _get_collection(palace_path, create=True) - del _client - from mempalace.mcp_server import tool_diary_write, tool_session_recovery_read - - tool_diary_write(agent_name="hooks", entry="from hooks", topic="checkpoint") - tool_diary_write(agent_name="other", entry="from other", topic="checkpoint") - - result = tool_session_recovery_read(agent="hooks") - assert result["total"] == 1 - assert result["entries"][0]["content"] == "from hooks" - - def test_respects_limit(self, monkeypatch, config, palace_path, kg): - _patch_mcp_server(monkeypatch, config, kg) - _client, _col = _get_collection(palace_path, create=True) - del _client - from mempalace.mcp_server import tool_diary_write, tool_session_recovery_read - - for i in range(5): - tool_diary_write(agent_name="hooks", entry=f"checkpoint {i}", topic="checkpoint") - - result = tool_session_recovery_read(limit=3) - assert result["total"] == 3 - - def test_handles_none_metadata(self, monkeypatch, config, palace_path, kg): - """Defensive: tolerates ``None`` entries in the metadatas list at - read time (legacy / partial-write data shape; ChromaDB emits this - even though it rejects writing it). Mirrors the #999 / #1094 / - #1201 family of None-metadata guards.""" - _patch_mcp_server(monkeypatch, config, kg) - - from mempalace import mcp_server - - class _MockCol: - def get(self, **_kwargs): - return { - "ids": ["legacy_1"], - "documents": ["legacy entry"], - "metadatas": [None], # the failure case - } - - monkeypatch.setattr( - mcp_server, - "_get_session_recovery_collection", - lambda create=False: _MockCol(), - ) - - result = mcp_server.tool_session_recovery_read() - assert isinstance(result["entries"], list) - assert result["total"] == 1 - # Entry survives with empty metadata defaults. - assert result["entries"][0]["agent"] == "" - assert result["entries"][0]["session_id"] == "" diff --git a/tests/test_migrate.py b/tests/test_migrate.py index a549d9a748..4701048afd 100644 --- a/tests/test_migrate.py +++ b/tests/test_migrate.py @@ -49,120 +49,6 @@ def test_migrate_aborts_without_confirmation(tmp_path, capsys): mock_rmtree.assert_not_called() -class TestMigrateCheckpointsToRecovery: - """Phase D — move existing topic=checkpoint drawers from the main - collection to the dedicated session-recovery collection. Idempotent; - safe to run multiple times.""" - - def _seed_main(self, palace_path): - from mempalace.palace import get_collection - - main = get_collection(palace_path, create=True) - main.add( - ids=["chk1", "chk2", "auto1", "content1", "content2"], - documents=[ - "CHECKPOINT:2026-04-25 alpha", - "CHECKPOINT:2026-04-25 beta", - "auto-save legacy entry gamma", - "Substantive content about the auth module.", - "Another content drawer about migrations.", - ], - metadatas=[ - {"topic": "checkpoint", "wing": "wing_session-hook"}, - {"topic": "checkpoint", "wing": "wing_session-hook"}, - {"topic": "auto-save", "wing": "wing_session-hook"}, - {"topic": "general", "wing": "project_a"}, - {"topic": "musings", "wing": "project_a"}, - ], - ) - return main - - def test_migrate_moves_checkpoints(self, tmp_path): - from mempalace.migrate import migrate_checkpoints_to_recovery - from mempalace.palace import get_collection, get_session_recovery_collection - - palace_path = str(tmp_path / "palace") - self._seed_main(palace_path) - - moved = migrate_checkpoints_to_recovery(palace_path) - assert moved == 3 - - main = get_collection(palace_path, create=True) - recovery = get_session_recovery_collection(palace_path, create=True) - assert main.count() == 2 - assert recovery.count() == 3 - - def test_migrate_is_idempotent(self, tmp_path): - from mempalace.migrate import migrate_checkpoints_to_recovery - from mempalace.palace import get_collection, get_session_recovery_collection - - palace_path = str(tmp_path / "palace") - self._seed_main(palace_path) - - first = migrate_checkpoints_to_recovery(palace_path) - second = migrate_checkpoints_to_recovery(palace_path) - - assert first == 3 - assert second == 0 - - main = get_collection(palace_path, create=True) - recovery = get_session_recovery_collection(palace_path, create=True) - assert main.count() == 2 - assert recovery.count() == 3 - - def test_migrate_preserves_drawer_ids_and_metadata(self, tmp_path): - from mempalace.migrate import migrate_checkpoints_to_recovery - from mempalace.palace import get_session_recovery_collection - - palace_path = str(tmp_path / "palace") - self._seed_main(palace_path) - - migrate_checkpoints_to_recovery(palace_path) - - recovery = get_session_recovery_collection(palace_path, create=True) - chk1 = recovery.get(ids=["chk1"], include=["documents", "metadatas"]) - assert chk1["ids"] == ["chk1"] - assert chk1["documents"][0] == "CHECKPOINT:2026-04-25 alpha" - assert chk1["metadatas"][0].get("topic") == "checkpoint" - assert chk1["metadatas"][0].get("wing") == "wing_session-hook" - - def test_migrate_handles_legacy_auto_save_topic(self, tmp_path): - from mempalace.migrate import migrate_checkpoints_to_recovery - from mempalace.palace import get_session_recovery_collection - - palace_path = str(tmp_path / "palace") - self._seed_main(palace_path) - - migrate_checkpoints_to_recovery(palace_path) - - recovery = get_session_recovery_collection(palace_path, create=True) - auto = recovery.get(ids=["auto1"], include=["metadatas"]) - assert auto["ids"] == ["auto1"] - assert auto["metadatas"][0].get("topic") == "auto-save" - - def test_migrate_no_checkpoints_returns_zero(self, tmp_path): - from mempalace.migrate import migrate_checkpoints_to_recovery - from mempalace.palace import get_collection - - palace_path = str(tmp_path / "palace") - main = get_collection(palace_path, create=True) - main.add( - ids=["c1"], - documents=["just content"], - metadatas=[{"topic": "general"}], - ) - - moved = migrate_checkpoints_to_recovery(palace_path) - assert moved == 0 - assert main.count() == 1 - - def test_migrate_no_palace_returns_zero(self, tmp_path): - from mempalace.migrate import migrate_checkpoints_to_recovery - - moved = migrate_checkpoints_to_recovery(str(tmp_path / "nope")) - assert moved == 0 - - def test_restore_stale_palace_with_clean_destination(tmp_path): """Rollback when no partial copy exists at palace_path.""" palace_path = tmp_path / "palace" diff --git a/tests/test_session_recovery.py b/tests/test_session_recovery.py deleted file mode 100644 index 6ae242a253..0000000000 --- a/tests/test_session_recovery.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Tests for the session-recovery collection split. - -Stop-hook auto-save checkpoint diary entries are routed to a dedicated -``mempalace_session_recovery`` collection so they don't dominate -``mempalace_search`` results in the main ``mempalace_drawers`` -collection. See: - -- ``docs/superpowers/specs/2026-04-25-checkpoint-collection-split.md`` -- ``docs/superpowers/plans/2026-04-25-checkpoint-collection-split-impl.md`` -""" - -from mempalace.palace import ( - _SESSION_RECOVERY_COLLECTION, - get_collection, - get_session_recovery_collection, -) - - -class TestSessionRecoveryCollection: - """Phase A — scaffolding. No behavior change yet; just the new - collection adapter exists alongside the main collection.""" - - def test_constant_name(self): - assert _SESSION_RECOVERY_COLLECTION == "mempalace_session_recovery" - - def test_creates_with_correct_metadata(self, tmp_path): - """Mirrors ``get_collection``'s shape: cosine space, thread-pin.""" - palace_path = str(tmp_path / "palace") - col = get_session_recovery_collection(palace_path) - assert col.metadata.get("hnsw:space") == "cosine" - assert col.metadata.get("hnsw:num_threads") == 1 - - def test_coexists_with_main_collection(self, tmp_path): - """Both collections live in the same ChromaDB client without - interfering. ChromaDB supports multi-collection per palace - natively; this is a sanity check that we haven't accidentally - wired them to share state.""" - palace_path = str(tmp_path / "palace") - main = get_collection(palace_path, create=True) - recovery = get_session_recovery_collection(palace_path) - - main.add( - ids=["m1"], - documents=["main collection drawer"], - metadatas=[{"topic": "general"}], - ) - recovery.add( - ids=["r1"], - documents=["recovery collection drawer"], - metadatas=[{"topic": "checkpoint"}], - ) - - assert main.count() == 1 - assert recovery.count() == 1 - - # Cross-collection isolation: a recovery ID is not visible - # from the main collection and vice-versa. - assert main.get(ids=["r1"])["ids"] == [] - assert recovery.get(ids=["m1"])["ids"] == [] diff --git a/website/reference/mcp-tools.md b/website/reference/mcp-tools.md index 2c98d250f6..2089a94c5d 100644 --- a/website/reference/mcp-tools.md +++ b/website/reference/mcp-tools.md @@ -347,23 +347,6 @@ Read recent diary entries. --- -### `mempalace_session_recovery_read` - -Read Stop-hook auto-save checkpoint entries from the dedicated session-recovery collection. Use for session recovery, hook auditing, or "what was I doing 2 hours ago" lookup. Checkpoints are stored separately from the main searchable corpus so they don't dominate `mempalace_search` results — read them through this tool instead. - -| Parameter | Type | Required | Description | -|-----------|------|----------|-------------| -| `session_id` | string | No | Filter by Claude Code session id. | -| `agent` | string | No | Filter by agent name (typically `session-hook`). | -| `since` | string | No | ISO datetime — entries strictly newer than this. | -| `until` | string | No | ISO datetime — entries strictly older than this. | -| `wing` | string | No | Filter by project wing. | -| `limit` | integer | No | Max entries to return (default 50, max 500). | - -**Returns:** `{ entries: [{ drawer_id, date, timestamp, topic, agent, wing, session_id, content }], total }` — newest-first. - ---- - ## System Tools ### `mempalace_hook_settings` From 28002b0bf40b2a7ffb48553a2807a74dac402f1a Mon Sep 17 00:00:00 2001 From: jp Date: Tue, 5 May 2026 18:03:08 -0700 Subject: [PATCH 2/2] docs(README): align with verbatim-only retirement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copilot review on jphein/mempalace#5 caught that the README still documented session_recovery_read as current, which would leave users calling a tool that returns "Unknown tool" after this PR ships. Updates the four user-facing sections that described the recovery collection as a current shipping feature: * "What just shipped" — adds 2026-05-05 update note explaining the split was retired, with the generalizable lesson preserved (a side collection without a semantic-search MCP read tool is invisible). * "Verbatim vs derivative" principle — reframes the recovery collection as the failed first attempt rather than a current example; pattern stays valid but each future sibling needs its own read surface. * "Structural retrieval fixes" bullet — describes verbatim-only end-state, links to the new spec, retains historical context for Apr 25 → May 5. * "Deterministic hook saves" bullet — updates to the new ingest-only save path, removes "recovery-collection marker" mention. * "What it looks like in production" code block — updates the systemMessage example to the new "Transcript ingest triggered (wing=...)" shape. * "P8" architectural principle — keeps it but reframes as on-hold with a precondition (read-surface parity) drawn from the cycle. * Fork-ahead tracking table — replaces the old multi-collection-split row with a verbatim-only retirement row pointing at the four PRs; also retires references to session_recovery_read in the drawer_id surfacing row; adds new row for the mining management CLI (#4). 42 README meta-tests pass (test_readme_claims.py). Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 31128230e5..6e8f370e58 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ This fork tracks `upstream/develop` through the 2026-04-27 sync and runs in prod ## What just shipped -On 2026-04-26 the canonical 151K-drawer palace ran an automatic migration on first daemon restart — *"Migrated 667 checkpoint drawer(s) from main → mempalace_session_recovery; mempalace_search now queries content-only."* That move closed a class of failure that recall benchmarks deliberately don't measure: the gap between *finding* the right document and *grounding the model on something useful*. The same Cat 9 A/B that surfaced the failure on 2026-04-25 re-ran post-migration and the predicted convergence held: +On 2026-04-26 the canonical 151K-drawer palace ran an automatic migration on first daemon restart — *"Migrated 667 checkpoint drawer(s) from main → mempalace_session_recovery; mempalace_search now queries content-only."* That move addressed a class of failure that recall benchmarks deliberately don't measure: the gap between *finding* the right document and *grounding the model on something useful*. The same Cat 9 A/B that surfaced the failure on 2026-04-25 re-ran post-migration and the predicted convergence held: | metric | pre-migration | post-migration | |------------------------------|--------------:|---------------:| @@ -20,7 +20,9 @@ On 2026-04-26 the canonical 151K-drawer palace ran an automatic migration on fir | `kind=content` tokens / Q | 3 | **1,267** | | pre vs. post gap | **210×** | **1.3×** | -Both modes now return real content. The structural fix did the work the algorithmic patch (`kind=` filter + over-fetch) couldn't. Empirical detail at [`~/Projects/notebook/data/cat9-postmigrate/REPORT.md`](https://github.com/jphein/notebook/blob/main/data/cat9-postmigrate/REPORT.md); the long-form story behind it lives at [`notebook/essays/2026-04-25-mempalace-lessons.md`](https://github.com/jphein/notebook/blob/main/essays/2026-04-25-mempalace-lessons.md). +Both modes returned real content. The structural fix did the work the algorithmic patch (`kind=` filter + over-fetch) couldn't. Empirical detail at [`~/Projects/notebook/data/cat9-postmigrate/REPORT.md`](https://github.com/jphein/notebook/blob/main/data/cat9-postmigrate/REPORT.md); the long-form story behind it lives at [`notebook/essays/2026-04-25-mempalace-lessons.md`](https://github.com/jphein/notebook/blob/main/essays/2026-04-25-mempalace-lessons.md). + +**Update 2026-05-05 — split retired in favor of verbatim-only.** The recovery-collection split solved the token-tax problem but created a new one: only filter-based reads ever existed for the recovery side, so checkpoints became invisible to `mempalace_search`. Cleaner fix: drop the derivative half entirely. Hooks now write only verbatim transcript chunks, all into `mempalace_drawers`, all directly searchable. The lesson generalizes — *a side collection without a semantic-search MCP read tool is invisible* — and was preserved in the architectural principles section (P8 below). ## The thesis @@ -32,7 +34,7 @@ The unit of memory in MemPalace is the verbatim utterance — chats, tool calls, Most public AI memory systems frame the problem the other way around: ingest raw, transform on write, store the derivative as canonical. Mem0 extracts "memories." Zep and Letta tier and summarize. Cognee builds a knowledge graph. Hindsight retains/recalls/reflects with LLM-extracted facts. In each, the verbatim original is gone — or at best, retrievable only through a layer of inference that already lost nuance. The fork's bet is the inverse: keep verbatim canonical, key derivative layers for their actual access pattern, and treat any derivative store as rebuildable from the verbatim. Derivative layers can then be replaced or re-derived without losing underlying truth. The April-2026 verbatim cohort (Longhand, Celiums, mcp-memory-service, MemPalace) converged on this within ~8 days of each other; the timing is suggestive. -Mixing verbatim and derivative in one corpus is the disease the checkpoint split treats. Recovery checkpoints, transcript-mining outputs, future KG-triple stores, and Haiku-enriched topic docs all want their own homes. The main `mempalace_drawers` collection holds verbatim only; sibling collections (`mempalace_session_recovery` shipped, more proposed) hold derivatives keyed for their actual read pattern. +Mixing verbatim and derivative in one corpus is the failure mode the original checkpoint split tried to treat. The cleaner fix in May 2026 was to drop the derivative half entirely: hooks write only verbatim transcript chunks (auto-mined into `mempalace_drawers`), no separate summaries. Future derivative layers (KG-triple stores, Haiku-enriched topic docs) can still live in sibling collections keyed for their access pattern — but only if and when each one earns its own MCP read tool. Without that read surface, a side collection becomes invisible to search; the recovery-collection cycle (Apr 25 → May 5) made that lesson concrete. This axis is implicit in upstream's [RFC 001](https://github.com/MemPalace/mempalace/pull/743) (`get_collection(palace, collection_name=...)` already supports it) but isn't yet named in the spec. Worth making explicit upstream — multi-collection-by-purpose is the architectural move that future backends should plan for. @@ -105,9 +107,9 @@ The fork is evaluating a Postgres-based backend (pgvector for vector search, Apa Three bands of work, all instances of the principles above. Detail rows in the [appendix](#fork-change-inventory) at the bottom. -- **Structural retrieval fixes (Principle 1, Principle 2).** Multi-collection split moves Stop-hook checkpoints to a dedicated `mempalace_session_recovery` collection — physically absent from `mempalace_search`, queryable via the new `mempalace_session_recovery_read` MCP tool. PreCompact incorporated. Auto-migrates on first daemon restart. The transitional `kind=` filter and over-fetch hack are gone (2026-04-27) — the structural fix made them inert. `drawer_id` surfacing on every search/diary/recovery hit so callers can build citation popovers and follow-ups. +- **Structural retrieval fixes (Principle 1, Principle 2).** Verbatim-only model: hooks no longer write 1KB checkpoint summaries; auto-mined transcript chunks land in `mempalace_drawers` and `mempalace_search` reaches them directly. The earlier dedicated `mempalace_session_recovery` collection (Apr 25–May 5) and its read-only `mempalace_session_recovery_read` MCP tool have been retired (May 5 — see `docs/superpowers/specs/2026-05-05-verbatim-only-design.md`). Net result: one collection, one search path, no kind=filter / over-fetch hack. `drawer_id` surfacing on every search/diary hit so callers can build citation popovers and follow-ups. - **Single-writer architecture (Principle 3).** [palace-daemon](https://github.com/jphein/palace-daemon) is the only process that opens the palace; clients connect over HTTP. ChromaDB 1.5.x's HNSW concurrency hazards (`#974`/`#965`/`#823` family) become structurally impossible. Cold-start integrity sniff-test on segment metadata files prevents `quarantine_stale_hnsw` from destroying healthy indexes during async-flush lag. Cherry-pick of upstream [#1085](https://github.com/MemPalace/mempalace/pull/1085) for 10–30× mining speedup; cherry-pick of upstream-PR-#1094 for boundary-level None-metadata coercion that closes a per-site-guard family. -- **Deterministic hook saves (Principles 1+2+3 compose).** Silent saves bypass auto-memory conflicts entirely — the LLM is no longer in the save path, so `decision: "block"` race conditions and Claude's auto-memory winning over MCP tools both go away. Save marker advances only after confirmed write. `systemMessage` notification surfaces results. PreCompact writes a recovery-collection marker before mining + compaction so context-boundary events leave a queryable timestamp. +- **Deterministic hook saves (Principles 1+2+3 compose).** Silent saves bypass auto-memory conflicts entirely — the LLM is no longer in the save path, so `decision: "block"` race conditions and Claude's auto-memory winning over MCP tools both go away. Verbatim transcript ingest is the entire save path; the save marker advances on each fire and `systemMessage` reports the wing the ingest landed in. PreCompact does the same — sync-mines the transcript before context boundary, no separate marker write. ## Quickstart @@ -126,10 +128,10 @@ For a daemon-fronted deployment (recommended once palace size reaches the multi- ## What it looks like in production -A Stop hook fires every 15 messages in Claude Code, writes directly to `mempalace_session_recovery` via the Python API (no LLM in the loop), and renders a terminal line so the user sees the save land: +A Stop hook fires every 15 messages in Claude Code, triggers verbatim transcript mining via the daemon's `/mine` endpoint (no LLM in the loop), and renders a terminal line so the user sees the ingest land: ```json -{"systemMessage": "✦ 13 memories woven into the palace — investigate, description, symlinkj"} +{"systemMessage": "✦ Transcript ingest triggered (wing=wing_realmwatch)"} ``` `search_memories` (via `mempalace_search` MCP tool) returns results with scope-authoritative context so callers can tell when the vector layer underdelivered: @@ -199,7 +201,7 @@ Reorganized 2026-04-26 around the verbatim-vs-derivative axis. Each item evaluat ### Derivative-store work (the new axis) -- **P8 — Corpus partitioning by purpose** *(architectural)*. The checkpoint collection split is the first instance. `mempalace_session_recovery` for hook-fired audit data; future siblings for transcript-mine outputs (the [#1083](https://github.com/MemPalace/mempalace/issues/1083) family — currently being addressed at the hook layer by [#1199](https://github.com/MemPalace/mempalace/pull/1199), but the collection-level move is the durable fix), KG-triple store ([P4](#p4-anchor)), Haiku-enriched topic docs (companion to P0). Worth flagging in [RFC 001](https://github.com/MemPalace/mempalace/pull/743) so future backends know multi-collection-per-palace is the canonical pattern. +- **P8 — Corpus partitioning by purpose** *(architectural, on hold)*. The recovery-collection split (Apr 25 → May 5, 2026) was the first attempt at this — moved Stop-hook checkpoints to a dedicated `mempalace_session_recovery` collection. Retired May 5: splitting required every read path to query both collections, but the recovery side never got a semantic-search MCP surface, so checkpoints became invisible to `mempalace_search`. The architectural pattern stays valid for future siblings (KG-triple store ([P4](#p4-anchor)), Haiku-enriched topic docs, transcript-mine outputs in the [#1083](https://github.com/MemPalace/mempalace/issues/1083) family), but each new sibling collection has to earn its own read tool before it gets writes. Worth flagging in [RFC 001](https://github.com/MemPalace/mempalace/pull/743) so future backends know that multi-collection-per-palace is the pattern AND that read-surface parity is a precondition. - **P4 — KG auto-population + entity resolution** *(1.5 days)*. Hooks extract `subject/predicate/object` triples on every save using heuristics (no LLM). Triples land in their own store (KG SQLite is already separate, P8-aligned). Normalize entity IDs; alias table + Levenshtein. Triples are *derived* — re-mine if extraction improves; verbatim untouched. *Note: under the [Postgres + pgvector + AGE substrate exploration](#substrate-exploration-postgres--pgvector--apache-age), the graph lives in-database (AGE) rather than in a separate SQLite, which makes this work meaningfully more natural to implement.* - **P5 — Temporal fact validity** *(1 day, depends on P4)*. KG triples get a context slot (SPOC: subject-predicate-object-context). Reference: Zep's [Graphiti](https://github.com/getzep/graphiti). *Same Postgres+AGE caveat as P4 — temporal validity ranges are SQL-native on Postgres in a way they aren't across two engines.* @@ -363,9 +365,9 @@ The canonical source is [`docs/fork-changes.yaml`](docs/fork-changes.yaml); [`FO | Area | Change | Status | Files | |---|---|---|---| -| **Search** | Move Stop-hook auto-save checkpoints to dedicated `mempalace_session_recovery` ChromaDB collection (Principle 1+2). **Phases A–E shipped 2026-04-25 → 2026-04-26**: collection adapter, write routing, new `mempalace_session_recovery_read` MCP tool, migration (idempotent, ID/metadata-preserving), PreCompact incorporation, palace-daemon `lifespan` auto-migrate. Canonical 151K palace migrated 667 checkpoints on 2026-04-26 10:24:09 PDT. Cat 9 A/B re-run shows **632/3 → 974/1267 token convergence**. | PR pending — fork commits [`e266365`](https://github.com/jphein/mempalace/commit/e266365) (A–C) → [`42817d7`](https://github.com/jphein/mempalace/commit/42817d7) (D + PreCompact); palace-daemon [`034023c`](https://github.com/jphein/palace-daemon/commit/034023c) (E); 18 new tests | `palace.py`, `mcp_server.py`, `migrate.py`, `cli.py`, `hooks_cli.py` | -| **Search** | Surface `drawer_id` in `mempalace_search` results, `mempalace_diary_read` entries, and `mempalace_session_recovery_read` payload. ChromaDB primary key was returned but never plumbed into the result-building loop. Defensive zip-with-id-pad for test mocks. | PR pending — fork commit [`9a8bb77`](https://github.com/jphein/mempalace/commit/9a8bb77); upstream [#1219](https://github.com/MemPalace/mempalace/pull/1219) (@pepo72) is the narrower searcher-only equivalent. | `searcher.py`, `mcp_server.py`, `tests/...`, `website/reference/mcp-tools.md` | -| **Reliability** | `hook_precompact` writes a session-recovery checkpoint marker before mining + compaction. Mirrors `hook_stop`'s `_save_diary_direct` call; same routing path (recovery collection, queryable by `session_id`). | Bundled with phase D in [`42817d7`](https://github.com/jphein/mempalace/commit/42817d7) | `mempalace/hooks_cli.py` | +| **Search** | **Verbatim-only retrieval** (May 5). Hooks write only verbatim transcript chunks; the dedicated `mempalace_session_recovery` collection and `mempalace_session_recovery_read` MCP tool are retired. `mempalace_search` reaches all session content directly. Replaces the earlier multi-collection split (Apr 25 → May 5) once it became clear that splitting required every read path to query both collections — never built — so checkpoints became invisible to search. Spec: `docs/superpowers/specs/2026-05-05-verbatim-only-design.md`. | PRs in review — [`#2`](https://github.com/jphein/mempalace/pull/2) (transcript ingest restore), [`#3`](https://github.com/jphein/mempalace/pull/3) (drop checkpoint writes), [`#5`](https://github.com/jphein/mempalace/pull/5) (retire collection); palace-daemon [`#1`](https://github.com/jphein/palace-daemon/pull/1) (path translation) | `hooks_cli.py`, `mcp_server.py`, `palace.py`, `migrate.py`, `cli.py` | +| **Search** | Surface `drawer_id` in `mempalace_search` results and `mempalace_diary_read` entries. ChromaDB primary key was returned but never plumbed into the result-building loop. Defensive zip-with-id-pad for test mocks. | PR pending — fork commit [`9a8bb77`](https://github.com/jphein/mempalace/commit/9a8bb77); upstream [#1219](https://github.com/MemPalace/mempalace/pull/1219) (@pepo72) is the narrower searcher-only equivalent. | `searcher.py`, `mcp_server.py`, `tests/...`, `website/reference/mcp-tools.md` | +| **CLI** | `mempalace mined` lists mined source files grouped by wing × source_file; `mempalace purge --source-file` deletes drawers from a specific file. Closes the "removing manually mined data" half of the mining-management ask. | [`#4`](https://github.com/jphein/mempalace/pull/4) | `cli.py`, `tests/test_cli.py` | | **Performance** | Cherry-picked upstream [#1085](https://github.com/MemPalace/mempalace/pull/1085) (@midweste) — batch ChromaDB inserts in miner. New `_build_drawer()` + `add_drawers()`. Reported 10–30× mining speedup. | Cherry-pick of open #1085 — fork commit [`6be6fff`](https://github.com/jphein/mempalace/commit/6be6fff). Becomes a no-op when #1085 merges. | `mempalace/miner.py` | | **Reliability** | Cherry-picked upstream [#1094](https://github.com/MemPalace/mempalace/pull/1094) — coerce None metadatas at chromadb boundary. Closes the per-site-guard family of None-metadata bugs (#999, #1198, #1201) at one site instead of N. | Cherry-pick of open #1094 — fork commit [`43d728d`](https://github.com/jphein/mempalace/commit/43d728d) | `backends/chroma.py`, `tests/test_backends.py` | | **CLI** | `mempalace purge --wing/--room` via `collection.delete(where=...)`. Earlier nuke-and-rebuild draft predicated on #521's race; @igorls's review traced the stack — race is on the upsert path, not delete-by-where. Simpler version preserves embedding fn, no rmtree window, routes through `ChromaBackend`. | [#1087](https://github.com/MemPalace/mempalace/pull/1087), rewritten 2026-04-26 per review | `cli.py`, `tests/test_cli.py` |