From 4949aab68b71f540514b3a7aad45828fa6366f5a Mon Sep 17 00:00:00 2001 From: eldar702 Date: Sun, 19 Apr 2026 11:13:50 +0300 Subject: [PATCH 01/77] fix: guard None metadata/doc in tool_check_duplicate and Layer1/Layer2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chroma 1.5.x can return ``None`` inside the ``metadatas`` / ``documents`` lists of a query/get result for partially-flushed rows. The codebase already has a systemic None-guard pattern (merged #999, #1013, #1019) but three call sites were still unguarded: * ``mcp_server.tool_check_duplicate`` (``mcp_server.py:487-488``) — ``meta = results["metadatas"][0][i]`` followed by ``meta.get(...)`` raises ``AttributeError: 'NoneType' object has no attribute 'get'``. The broad ``except Exception`` wrapper (line 504) swallows it and returns an uninformative ``"Duplicate check failed"``. * ``layers.Layer1.generate`` (``layers.py:126``) — iterates ``zip(docs, metas)`` and calls ``meta.get(key)`` in the importance loop. A single None metadata blows up the entire wake-up render. * ``layers.Layer2.retrieve`` (``layers.py:224``) — same pattern, same crash path for the on-demand render. Apply the same ``meta = meta or {}`` / ``doc = doc or ""`` idiom used by the merged guards in the search path. Three-line additions, no behaviour change on well-formed results. Tests added: * ``test_check_duplicate_handles_none_metadata`` — mocks the collection query to return ``None`` for one metadata and document, asserts the call does not crash and the sentinel-rendered entry has wing/room "?" and empty content. * ``test_layer1_handles_none_metadata`` / ``_handles_none_document`` * ``test_layer2_handles_none_metadata`` Relationship to other open PRs: * **#1019** guarded ``searcher.py`` loops. This PR extends the same guard to the three call sites #1019 did not touch. * **#979** fixed ``tool_check_duplicate`` negative similarity but left the None-metadata path unguarded. * Does not overlap **#1013** (``Layer3.search_raw``) or **#999**. --- mempalace/layers.py | 4 +++ mempalace/mcp_server.py | 6 ++-- tests/test_layers.py | 69 ++++++++++++++++++++++++++++++++++++++++ tests/test_mcp_server.py | 36 +++++++++++++++++++++ 4 files changed, 113 insertions(+), 2 deletions(-) diff --git a/mempalace/layers.py b/mempalace/layers.py index a0f9b6df0d..b20c65621b 100644 --- a/mempalace/layers.py +++ b/mempalace/layers.py @@ -124,6 +124,8 @@ def generate(self) -> str: # Score each drawer: prefer high importance, recent filing scored = [] for doc, meta in zip(docs, metas): + meta = meta or {} + doc = doc or "" importance = 3 # Try multiple metadata keys that might carry weight info for key in ("importance", "emotional_weight", "weight"): @@ -222,6 +224,8 @@ def retrieve(self, wing: str = None, room: str = None, n_results: int = 10) -> s lines = [f"## L2 — ON-DEMAND ({len(docs)} drawers)"] for doc, meta in zip(docs[:n_results], metas[:n_results]): + meta = meta or {} + doc = doc or "" room_name = meta.get("room", "?") source = Path(meta.get("source_file", "")).name if meta.get("source_file") else "" snippet = doc.strip().replace("\n", " ") diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 06355c46ee..ae1eb717da 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -484,8 +484,10 @@ def tool_check_duplicate(content: str, threshold: float = 0.9): dist = results["distances"][0][i] similarity = round(1 - dist, 3) if similarity >= threshold: - meta = results["metadatas"][0][i] - doc = results["documents"][0][i] + # Chroma 1.5.x can return None for partially-flushed rows; + # coerce to empty sentinels so downstream .get() is safe. + meta = results["metadatas"][0][i] or {} + doc = results["documents"][0][i] or "" duplicates.append( { "id": drawer_id, diff --git a/tests/test_layers.py b/tests/test_layers.py index 575183fc51..d4c54ce7cf 100644 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -655,3 +655,72 @@ def test_memory_stack_status_with_palace(tmp_path): assert result["total_drawers"] == 42 assert result["L0_identity"]["exists"] is True + + +# ── Layer1 / Layer2 None-metadata guards ─────────────────────────────── +# +# Chroma 1.5.x can return ``None`` inside the ``metadatas`` / ``documents`` +# lists for partially-flushed rows. The Layer1.generate() and +# Layer2.retrieve() loops previously called ``meta.get(...)`` without +# coercing, raising ``AttributeError: 'NoneType' object has no attribute +# 'get'`` and blowing up the whole wake-up render. These tests guard that +# the loops tolerate the None entries and render the rest of the result. + + +def test_layer1_handles_none_metadata(): + """Layer1.generate tolerates None entries in the metadatas list.""" + docs = ["important memory", "another memory"] + metas = [{"room": "decisions", "source_file": "a.txt"}, None] + mock_col = _mock_chromadb_for_layer(docs, metas) + + with ( + patch("mempalace.layers.MempalaceConfig") as mock_cfg, + patch("mempalace.layers._get_collection", return_value=mock_col), + ): + mock_cfg.return_value.palace_path = "/fake" + layer = Layer1(palace_path="/fake") + # Should not raise AttributeError on the None entry. + result = layer.generate() + + assert "ESSENTIAL STORY" in result + assert "important memory" in result + + +def test_layer1_handles_none_document(): + """Layer1.generate tolerates None entries in the documents list.""" + docs = ["first doc", None] + metas = [ + {"room": "r", "source_file": "a.txt"}, + {"room": "r", "source_file": "b.txt"}, + ] + mock_col = _mock_chromadb_for_layer(docs, metas) + + with ( + patch("mempalace.layers.MempalaceConfig") as mock_cfg, + patch("mempalace.layers._get_collection", return_value=mock_col), + ): + mock_cfg.return_value.palace_path = "/fake" + layer = Layer1(palace_path="/fake") + result = layer.generate() + + assert result # Render succeeded despite the None document. + + +def test_layer2_handles_none_metadata(): + """Layer2.retrieve tolerates None entries in the metadatas list.""" + mock_col = MagicMock() + mock_col.get.return_value = { + "documents": ["first doc", "second doc"], + "metadatas": [{"room": "r", "source_file": "a.txt"}, None], + } + + with ( + patch("mempalace.layers.MempalaceConfig") as mock_cfg, + patch("mempalace.layers._get_collection", return_value=mock_col), + ): + mock_cfg.return_value.palace_path = "/fake" + layer = Layer2(palace_path="/fake") + # Should not raise AttributeError on the None entry. + result = layer.retrieve() + + assert "L2 — ON-DEMAND" in result diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 899e6a7c79..e376f43efe 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -9,6 +9,7 @@ from datetime import datetime import json import sys +from unittest.mock import MagicMock import pytest @@ -495,6 +496,41 @@ def test_delete_drawer_not_found(self, monkeypatch, config, palace_path, seeded_ result = tool_delete_drawer("nonexistent_drawer") assert result["success"] is False + def test_check_duplicate_handles_none_metadata(self, monkeypatch, config, kg): + """tool_check_duplicate must tolerate None entries in the result lists + that ChromaDB 1.5.x returns for partially-flushed rows. + + Previously ``meta = results["metadatas"][0][i]`` was unguarded and + raised ``AttributeError: 'NoneType' object has no attribute 'get'`` + the moment the first matching drawer came back with None metadata — + surfacing to the MCP client as the uninformative + ``"Duplicate check failed"`` because the broad ``except Exception`` + wrapper swallows the real cause. + """ + _patch_mcp_server(monkeypatch, config, kg) + from mempalace import mcp_server + + mock_col = MagicMock() + mock_col.query.return_value = { + "ids": [["d1", "d2"]], + "distances": [[0.05, 0.05]], + "metadatas": [[{"wing": "w", "room": "r"}, None]], + "documents": [["first doc", None]], + } + monkeypatch.setattr(mcp_server, "_get_collection", lambda: mock_col) + + result = mcp_server.tool_check_duplicate("any content", threshold=0.5) + + # Both entries land in matches (above threshold), None ones rendered + # with sentinel values rather than crashing the whole response. + assert result.get("is_duplicate") is True + assert len(result["matches"]) == 2 + # The None-metadata entry falls back to sentinels. + none_entry = result["matches"][1] + assert none_entry["wing"] == "?" + assert none_entry["room"] == "?" + assert none_entry["content"] == "" + def test_check_duplicate(self, monkeypatch, config, palace_path, seeded_collection, kg): _patch_mcp_server(monkeypatch, config, kg) from mempalace.mcp_server import tool_check_duplicate From 35b033d77ff378b9ff2999bff965d639e0672039 Mon Sep 17 00:00:00 2001 From: alonehobo Date: Tue, 21 Apr 2026 12:33:58 +0500 Subject: [PATCH 02/77] fix(mcp): force UTF-8 on stdio to fix -32000 on non-ASCII payloads On Windows, Python defaults sys.stdin/sys.stdout to the system codepage (e.g. cp1251 on Russian locales, cp1252 on Western European), while MCP JSON-RPC is always UTF-8. Non-ASCII payloads (Cyrillic, CJK, accented European) get mis-decoded before reaching handlers, causing json.loads to fail or tool handlers to receive garbled strings. Both surface to the client as a generic MCP error -32000. Reproduction: 1. On Windows with a non-Latin locale, call mempalace_add_drawer or mempalace_kg_add with Cyrillic/CJK in content or KG object. 2. Server returns: MCP error -32000: Internal tool error. 3. Calling the handler directly from Python works fine -- the bug is purely in the stdio transport. Fix: Reconfigure stdin/stdout to UTF-8 at the start of main(), after _restore_stdout(). Uses errors="replace" defensively so a lone bad byte cannot take down the server. Guarded by hasattr(reconfigure) for exotic stream replacements. This matches the behaviour of PYTHONUTF8=1 / python -X utf8 without requiring users to set an env var. --- mempalace/mcp_server.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 6fe8225e16..48e80312af 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -1689,6 +1689,16 @@ def _restore_stdout(): def main(): _restore_stdout() + # Force UTF-8 on stdio. MCP JSON-RPC is UTF-8, but Python on Windows + # defaults stdin/stdout to the system codepage (e.g. cp1251), which + # corrupts non-ASCII payloads and surfaces as generic -32000 errors on + # Cyrillic/CJK content. See PEP 540. + for stream in (sys.stdin, sys.stdout): + if hasattr(stream, "reconfigure"): + try: + stream.reconfigure(encoding="utf-8", errors="replace") + except (AttributeError, OSError): + pass logger.info("MemPalace MCP Server starting...") while True: try: From c2e053176cf4194fedde2a3361c2785e16426008 Mon Sep 17 00:00:00 2001 From: Sathvik-1007 Date: Thu, 23 Apr 2026 01:40:38 +0530 Subject: [PATCH 03/77] fix: add total count to tool_list_drawers pagination response MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The list_drawers response only included count (current page size) with no total field, making it impossible for callers to know when pagination is exhausted. A page returning count == limit is ambiguous — it could be the last exact-fit page or there could be more results. Add a total field that reports the full number of matching drawers. For unfiltered requests this uses col.count(); for filtered requests (wing/room) it uses a lightweight col.get(include=[]) to count matching IDs without fetching documents. --- mempalace/mcp_server.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 6fe8225e16..e5057afc64 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -746,6 +746,13 @@ def tool_list_drawers(wing: str = None, room: str = None, limit: int = 20, offse kwargs["where"] = where result = col.get(**kwargs) + # Compute total matching drawers for pagination. + if where: + total_result = col.get(where=where, include=[]) + total = len(total_result["ids"]) + else: + total = col.count() + drawers = [] for i, did in enumerate(result["ids"]): meta = result["metadatas"][i] @@ -760,6 +767,7 @@ def tool_list_drawers(wing: str = None, room: str = None, limit: int = 20, offse ) return { "drawers": drawers, + "total": total, "count": len(drawers), "offset": offset, "limit": limit, @@ -1436,7 +1444,7 @@ def tool_reconnect(): "handler": tool_get_drawer, }, "mempalace_list_drawers": { - "description": "List drawers with pagination. Optional wing/room filter. Returns IDs, wings, rooms, and content previews.", + "description": "List drawers with pagination. Optional wing/room filter. Returns IDs, wings, rooms, content previews, and total matching count for pagination.", "input_schema": { "type": "object", "properties": { From 0b8c2c158f1e9edbd14aac1d9b85abfd2bed97e4 Mon Sep 17 00:00:00 2001 From: Arnold Wender Date: Sun, 26 Apr 2026 13:00:27 +0200 Subject: [PATCH 04/77] fix(kg): reject inverted intervals in add_triple (valid_to < valid_from) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A triple with valid_to < valid_from satisfies neither of the temporal filter clauses in query_entity(): valid_from <= as_of AND valid_to >= as_of so the triple is invisible to every query — silently corrupt. Reject at write time with a clear error instead of letting bad data pile up in the SQLite store. The guard only fires when both bounds are present; open intervals (only valid_from or only valid_to) are still accepted, and same-day intervals (valid_from == valid_to, point-in-time facts) are explicitly allowed. --- mempalace/knowledge_graph.py | 9 +++++++++ tests/test_knowledge_graph.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/mempalace/knowledge_graph.py b/mempalace/knowledge_graph.py index 9096ab28f7..30055a130e 100644 --- a/mempalace/knowledge_graph.py +++ b/mempalace/knowledge_graph.py @@ -171,6 +171,15 @@ def add_triple( add_triple("Max", "does", "swimming", valid_from="2025-01-01") add_triple("Alice", "worried_about", "Max injury", valid_from="2026-01", valid_to="2026-02") """ + # Reject inverted intervals: a triple with valid_to < valid_from + # would never satisfy `valid_from <= as_of AND valid_to >= as_of`, + # so it would be invisible to every query — silently corrupt. + if valid_from is not None and valid_to is not None and valid_to < valid_from: + raise ValueError( + f"valid_to={valid_to!r} is before valid_from={valid_from!r}; " + "an inverted interval would be invisible to every KG query" + ) + sub_id = self._entity_id(subject) obj_id = self._entity_id(obj) pred = predicate.lower().replace(" ", "_") diff --git a/tests/test_knowledge_graph.py b/tests/test_knowledge_graph.py index d7d9838fa4..6eeb8d3f5f 100644 --- a/tests/test_knowledge_graph.py +++ b/tests/test_knowledge_graph.py @@ -5,6 +5,8 @@ timeline, stats, and edge cases (duplicate triples, ID collisions). """ +import pytest + class TestEntityOperations: def test_add_entity(self, kg): @@ -45,6 +47,38 @@ def test_invalidated_triple_allows_re_add(self, kg): tid2 = kg.add_triple("Alice", "works_at", "Acme") assert tid1 != tid2 # new triple since old one was closed + def test_add_triple_rejects_inverted_interval(self, kg): + # valid_to before valid_from would never satisfy + # `valid_from <= as_of AND valid_to >= as_of` — silently invisible + # to every query. Reject at write time instead. + with pytest.raises(ValueError, match="before valid_from"): + kg.add_triple( + "Alice", + "worked_at", + "Acme", + valid_from="2026-03-01", + valid_to="2026-02-01", + ) + + def test_add_triple_accepts_equal_dates(self, kg): + # Same-day intervals are valid (point-in-time facts). + tid = kg.add_triple( + "Alice", + "joined", + "Acme", + valid_from="2026-03-15", + valid_to="2026-03-15", + ) + assert tid.startswith("t_alice_joined_acme_") + + def test_add_triple_allows_only_one_bound(self, kg): + # The guard only fires when BOTH bounds are set. + tid1 = kg.add_triple("Alice", "knows", "Bob", valid_from="2026-01-01") + assert tid1.startswith("t_alice_knows_bob_") + kg.invalidate("Alice", "knows", "Bob", ended="2026-02-01") + tid2 = kg.add_triple("Alice", "knew", "Bob", valid_to="2026-03-01") + assert tid2.startswith("t_alice_knew_bob_") + class TestQueries: def test_query_outgoing(self, seeded_kg): From f30fdf2672f340c773d4f0554b0017abb48ed4b9 Mon Sep 17 00:00:00 2001 From: imtylervo Date: Mon, 27 Apr 2026 14:16:20 +1000 Subject: [PATCH 05/77] fix: serialize ChromaCollection writes through palace lock #976 protects `mempalace mine`, but MCP/direct backend writers still call ChromaCollection.add/upsert/update/delete without the palace lock. This moves the lock boundary to the Chroma backend seam so all Chroma writes share the same palace-level serialization, with a re-entrant guard for miner paths that already hold the lock. mine_palace_lock(palace_path) gains a per-thread re-entrant guard (threading.local + pid-tag against fork inheritance) so ChromaCollection write methods can take the lock without self-deadlocking when called from inside miner.mine()'s outer hold. ChromaCollection.__init__ accepts an optional palace_path; when set, add/upsert/update/delete wrap their underlying chromadb call with mine_palace_lock(palace_path). palace_path=None preserves the legacy no-lock behaviour for direct callers and tests. ChromaBackend's get_collection/create_collection pass palace_path through; mcp_server._get_collection forwards _config.palace_path so all MCP write tools inherit the wrapping. Tests: 5 new in tests/test_chroma_collection_lock.py covering opt-in, writer-blocks-during-mine, re-entrant-inside-mine, two-process serialization, and a source-level read-path-not-locked pin. Plus 1 new + 1 rewritten in tests/test_palace_locks.py for the re-entrant semantics. 52 passed in 1.01s including the existing test_backends.py regression suite. Refs #1161. --- mempalace/backends/chroma.py | 54 ++++- mempalace/mcp_server.py | 4 +- mempalace/palace.py | 59 ++++- tests/test_chroma_collection_lock.py | 327 +++++++++++++++++++++++++++ tests/test_palace_locks.py | 70 +++++- 5 files changed, 497 insertions(+), 17 deletions(-) create mode 100644 tests/test_chroma_collection_lock.py diff --git a/mempalace/backends/chroma.py b/mempalace/backends/chroma.py index ad7748f742..d438236651 100644 --- a/mempalace/backends/chroma.py +++ b/mempalace/backends/chroma.py @@ -1,5 +1,6 @@ """ChromaDB-backed MemPalace storage backend (RFC 001 reference implementation).""" +import contextlib import datetime as _dt import logging import os @@ -573,10 +574,43 @@ def _as_list(v: Any) -> list: class ChromaCollection(BaseCollection): - """Thin adapter translating ChromaDB dict returns into typed results.""" + """Thin adapter translating ChromaDB dict returns into typed results. + + When ``palace_path`` is set, all write methods (``add``, ``upsert``, + ``update``, ``delete``) acquire ``mine_palace_lock(palace_path)`` for the + duration of the underlying chromadb call. This serializes MCP and other + direct-backend writers against ``mempalace mine`` and against each other, + closing the race between concurrent writers that triggers ChromaDB's + multi-threaded HNSW corruption (#974/#965). + + The lock is the same primitive used by ``miner.mine()`` so re-entrant + acquisition from inside the mine pipeline (mine -> _mine_body -> + collection.upsert) is short-circuited by the per-thread guard inside + ``mine_palace_lock`` — no self-deadlock. + + ``palace_path=None`` disables the wrapping, preserving the legacy + no-lock behaviour for callers that construct a ``ChromaCollection`` + directly without going through ``ChromaBackend``. + """ - def __init__(self, collection): + def __init__(self, collection, palace_path: Optional[str] = None): self._collection = collection + self._palace_path = palace_path + + @contextlib.contextmanager + def _write_lock(self): + """Acquire ``mine_palace_lock`` for the configured palace, if any. + + No-op (yields immediately) when ``self._palace_path`` is None. + """ + if self._palace_path is None: + yield + return + # Late import — palace.py imports ChromaBackend from this module. + from ..palace import mine_palace_lock + + with mine_palace_lock(self._palace_path): + yield # ------------------------------------------------------------------ # Writes @@ -588,7 +622,8 @@ def add(self, *, documents, ids, metadatas=None, embeddings=None): kwargs["metadatas"] = metadatas if embeddings is not None: kwargs["embeddings"] = embeddings - self._collection.add(**kwargs) + with self._write_lock(): + self._collection.add(**kwargs) def upsert(self, *, documents, ids, metadatas=None, embeddings=None): kwargs: dict[str, Any] = {"documents": documents, "ids": ids} @@ -596,7 +631,8 @@ def upsert(self, *, documents, ids, metadatas=None, embeddings=None): kwargs["metadatas"] = metadatas if embeddings is not None: kwargs["embeddings"] = embeddings - self._collection.upsert(**kwargs) + with self._write_lock(): + self._collection.upsert(**kwargs) def update( self, @@ -615,7 +651,8 @@ def update( kwargs["metadatas"] = metadatas if embeddings is not None: kwargs["embeddings"] = embeddings - self._collection.update(**kwargs) + with self._write_lock(): + self._collection.update(**kwargs) # ------------------------------------------------------------------ # Reads @@ -759,7 +796,8 @@ def delete(self, *, ids=None, where=None): kwargs["ids"] = ids if where is not None: kwargs["where"] = where - self._collection.delete(**kwargs) + with self._write_lock(): + self._collection.delete(**kwargs) def count(self): return self._collection.count() @@ -998,7 +1036,7 @@ def get_collection( else: collection = client.get_collection(collection_name, **ef_kwargs) _pin_hnsw_threads(collection) - return ChromaCollection(collection) + return ChromaCollection(collection, palace_path=palace_path) def close_palace(self, palace) -> None: """Drop cached handles for ``palace``. Accepts ``PalaceRef`` or legacy path str.""" @@ -1045,7 +1083,7 @@ def create_collection( metadata={"hnsw:space": hnsw_space, "hnsw:num_threads": 1}, **ef_kwargs, ) - return ChromaCollection(collection) + return ChromaCollection(collection, palace_path=palace_path) def _normalize_get_collection_args(args, kwargs): diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 9cc454e7f7..7e22227260 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -288,13 +288,13 @@ def _get_collection(create=False): metadata={"hnsw:space": "cosine", "hnsw:num_threads": 1}, ) _pin_hnsw_threads(raw) - _collection_cache = ChromaCollection(raw) + _collection_cache = ChromaCollection(raw, palace_path=_config.palace_path) _metadata_cache = None _metadata_cache_time = 0 elif _collection_cache is None: raw = client.get_collection(_config.collection_name) _pin_hnsw_threads(raw) - _collection_cache = ChromaCollection(raw) + _collection_cache = ChromaCollection(raw, palace_path=_config.palace_path) _metadata_cache = None _metadata_cache_time = 0 return _collection_cache diff --git a/mempalace/palace.py b/mempalace/palace.py index 07efb6a3e3..97f67ff33a 100644 --- a/mempalace/palace.py +++ b/mempalace/palace.py @@ -8,6 +8,7 @@ import hashlib import os import re +import threading from .backends.chroma import ChromaBackend @@ -314,6 +315,47 @@ class MineAlreadyRunning(RuntimeError): """Raised when another `mempalace mine` already holds the per-palace lock.""" +# Per-thread record of palaces this thread already holds the lock for. Used by +# `mine_palace_lock` to short-circuit re-entrant acquisition from the same +# thread (e.g. miner.mine() acquires the outer lock then calls +# ChromaCollection.upsert which now also tries to acquire). Without this guard +# the inner call would block on its own outer flock (Linux fcntl locks are per +# open file description, so a same-thread second open of the lock file is a +# distinct lock and self-deadlocks). +# +# The holder set is tagged with ``pid`` so that a forked child does NOT +# inherit re-entrant credit from its parent: the OS-level flock IS NOT +# inherited as a "we hold it" semantically — the child must reacquire — but +# Python's ``threading.local`` IS inherited across fork. The pid check +# clears stale state so a forked child correctly hits the fcntl path. +_palace_lock_holders = threading.local() + + +def _holder_state(): + """Return the per-thread (pid, keys) record, refreshing after fork.""" + keys = getattr(_palace_lock_holders, "keys", None) + pid = getattr(_palace_lock_holders, "pid", None) + current_pid = os.getpid() + if keys is None or pid != current_pid: + keys = set() + _palace_lock_holders.keys = keys + _palace_lock_holders.pid = current_pid + return keys + + +def _held_by_this_thread(lock_key: str) -> bool: + """Return True if this thread already holds ``mine_palace_lock`` for ``lock_key``.""" + return lock_key in _holder_state() + + +def _mark_held(lock_key: str) -> None: + _holder_state().add(lock_key) + + +def _mark_released(lock_key: str) -> None: + _holder_state().discard(lock_key) + + @contextlib.contextmanager def mine_palace_lock(palace_path: str): """Per-palace non-blocking lock around the full `mine` pipeline. @@ -338,6 +380,12 @@ def mine_palace_lock(palace_path: str): Non-blocking: if another `mine` is already writing to this palace, raise MineAlreadyRunning so the caller can exit cleanly instead of piling up as a waiting worker. + + Re-entrant: if the current thread already holds the lock for the same + palace, the context manager passes through without re-acquiring. This + lets ChromaCollection write methods (which acquire the lock themselves + to protect MCP/direct callers) compose with miner.mine() (which holds + the outer lock for the entire mine pipeline) without self-deadlock. """ lock_dir = os.path.join(os.path.expanduser("~"), ".mempalace", "locks") os.makedirs(lock_dir, exist_ok=True) @@ -346,6 +394,11 @@ def mine_palace_lock(palace_path: str): palace_key = hashlib.sha256(lock_key_source.encode()).hexdigest()[:16] lock_path = os.path.join(lock_dir, f"mine_palace_{palace_key}.lock") + if _held_by_this_thread(palace_key): + # Same thread already holds the lock for this palace — pass through. + yield + return + lf = open(lock_path, "w") acquired = False try: @@ -369,7 +422,11 @@ def mine_palace_lock(palace_path: str): raise MineAlreadyRunning( f"another `mempalace mine` is already running against {resolved}" ) from exc - yield + _mark_held(palace_key) + try: + yield + finally: + _mark_released(palace_key) finally: if acquired: try: diff --git a/tests/test_chroma_collection_lock.py b/tests/test_chroma_collection_lock.py new file mode 100644 index 0000000000..b5d30fbcfe --- /dev/null +++ b/tests/test_chroma_collection_lock.py @@ -0,0 +1,327 @@ +"""Tests for ChromaCollection's palace-write-lock integration. + +Closes the gap left by ``mine_palace_lock`` only protecting the +``mempalace mine`` pipeline: MCP/direct writers that call +``ChromaCollection.add/upsert/update/delete`` must also serialize against +mine and against each other to avoid the multi-threaded HNSW corruption +documented in #974/#965. + +Property tested: + +* ``ChromaCollection(c, palace_path=p)`` wraps every write with + ``mine_palace_lock(p)``. +* Writes raise ``MineAlreadyRunning`` when another holder owns the lock + (instead of silently racing into the underlying chromadb call). +* Re-entrant composition with ``miner.mine()`` does not self-deadlock: + ``with mine_palace_lock(p): col.upsert(...)`` runs to completion. +* ``ChromaCollection(c)`` (no palace_path) preserves legacy no-lock + behaviour for tests/callers that build the adapter directly without + going through ``ChromaBackend``. + +POSIX-only: ``mine_palace_lock`` uses ``fcntl`` on Unix and ``msvcrt`` on +Windows; the contention semantics differ enough that the cross-process +tests are skipped on Windows runners. +""" + +from __future__ import annotations + +import multiprocessing +import os +import time + +import pytest + +from mempalace.backends.chroma import ChromaCollection +from mempalace.palace import MineAlreadyRunning, mine_palace_lock + + +def _get_mp_context(): + """Same start-method picker as test_palace_locks.py.""" + start_method = "spawn" if os.name == "nt" else "fork" + return multiprocessing.get_context(start_method) + + +# --------------------------------------------------------------------------- +# Fakes +# --------------------------------------------------------------------------- + + +class _FakeChromaCollection: + """Records calls; never blocks. Stand-in for chromadb.Collection.""" + + def __init__(self): + self.adds: list[dict] = [] + self.upserts: list[dict] = [] + self.updates: list[dict] = [] + self.deletes: list[dict] = [] + + def add(self, **kwargs): + self.adds.append(kwargs) + + def upsert(self, **kwargs): + self.upserts.append(kwargs) + + def update(self, **kwargs): + self.updates.append(kwargs) + + def delete(self, **kwargs): + self.deletes.append(kwargs) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _hold_lock(palace_path: str, ready_flag: str, release_flag: str) -> int: + """Acquire ``mine_palace_lock``, signal readiness, wait for release. + + Mirrors the helper in ``test_palace_locks.py`` so the contention + semantics match across both test files. + """ + try: + with mine_palace_lock(palace_path): + open(ready_flag, "w").close() + for _ in range(500): + if os.path.exists(release_flag): + return 0 + time.sleep(0.01) + return 0 + except MineAlreadyRunning: + return 1 + + +# --------------------------------------------------------------------------- +# Tests — opt-in lock wiring +# --------------------------------------------------------------------------- + + +def test_palace_path_none_skips_lock(tmp_path, monkeypatch): + """Legacy callers (``ChromaCollection(c)``) keep no-lock behaviour. + + A ``ChromaCollection`` built without ``palace_path`` must not touch the + lock infrastructure at all. This guards against regressions where a + test or third-party caller relies on the historical bare-write path. + """ + monkeypatch.setenv("HOME", str(tmp_path)) + fake = _FakeChromaCollection() + col = ChromaCollection(fake) # no palace_path -> no lock + + # Hold the lock in a child process. Without palace_path, the parent + # write must still succeed (the lock does not gate this caller). + palace = str(tmp_path / "palace") + ready = str(tmp_path / "ready") + release = str(tmp_path / "release") + ctx = _get_mp_context() + holder = ctx.Process(target=_hold_lock, args=(palace, ready, release)) + holder.start() + try: + for _ in range(500): + if os.path.exists(ready): + break + time.sleep(0.01) + assert os.path.exists(ready), "holder failed to acquire lock" + + col.upsert(documents=["doc"], ids=["id-1"]) + assert fake.upserts == [{"documents": ["doc"], "ids": ["id-1"]}] + finally: + open(release, "w").close() + holder.join(timeout=5) + + +def test_writer_blocks_during_mine(tmp_path, monkeypatch): + """A held ``mine_palace_lock`` causes ``ChromaCollection`` writes to raise. + + This is the property that closes the MCP-bypass gap: when a mine is in + flight, MCP/direct writes raise ``MineAlreadyRunning`` rather than + silently entering chromadb's write path concurrent with mine. + """ + monkeypatch.setenv("HOME", str(tmp_path)) + palace = str(tmp_path / "palace") + ready = str(tmp_path / "ready") + release = str(tmp_path / "release") + + ctx = _get_mp_context() + holder = ctx.Process(target=_hold_lock, args=(palace, ready, release)) + holder.start() + try: + for _ in range(500): + if os.path.exists(ready): + break + time.sleep(0.01) + assert os.path.exists(ready), "holder failed to acquire lock" + + fake = _FakeChromaCollection() + col = ChromaCollection(fake, palace_path=palace) + + with pytest.raises(MineAlreadyRunning): + col.upsert(documents=["doc"], ids=["id-1"]) + with pytest.raises(MineAlreadyRunning): + col.add(documents=["doc"], ids=["id-2"]) + with pytest.raises(MineAlreadyRunning): + col.update(ids=["id-3"], documents=["doc"]) + with pytest.raises(MineAlreadyRunning): + col.delete(ids=["id-4"]) + + # The fake must have received NO calls — the lock must gate + # before reaching the underlying chromadb layer. + assert fake.upserts == [] + assert fake.adds == [] + assert fake.updates == [] + assert fake.deletes == [] + finally: + open(release, "w").close() + holder.join(timeout=5) + + +def test_reentrant_inside_mine_passes_through(tmp_path, monkeypatch): + """``ChromaCollection.upsert`` inside ``mine_palace_lock`` does not deadlock. + + ``miner.mine()`` already holds ``mine_palace_lock(palace_path)`` for the + full mine pipeline; ``_mine_body`` then calls + ``collection.upsert(...)``. With the per-thread re-entrant guard in + ``mine_palace_lock``, the inner acquire is a pass-through and the + underlying chromadb call runs immediately. + """ + monkeypatch.setenv("HOME", str(tmp_path)) + palace = str(tmp_path / "palace") + fake = _FakeChromaCollection() + col = ChromaCollection(fake, palace_path=palace) + + with mine_palace_lock(palace): + # If the re-entrant guard were missing, this would self-deadlock on + # the underlying flock. We rely on pytest-timeout (configured in + # pyproject.toml) to enforce this in CI; the assertion just confirms + # the call landed. + col.upsert(documents=["d"], ids=["i"], metadatas=[{"k": "v"}]) + col.add(documents=["d2"], ids=["i2"]) + col.update(ids=["i"], documents=["d-updated"]) + col.delete(ids=["i2"]) + + assert len(fake.upserts) == 1 + assert len(fake.adds) == 1 + assert len(fake.updates) == 1 + assert len(fake.deletes) == 1 + + +class _SlowFakeChromaCollection(_FakeChromaCollection): + """Fake whose write methods hold the caller for ``hold_seconds``. + + Used to keep ``mine_palace_lock`` acquired long enough for a sibling + process to contend deterministically. + """ + + def __init__(self, hold_seconds: float = 0.3): + super().__init__() + self._hold = hold_seconds + + def upsert(self, **kwargs): + time.sleep(self._hold) + super().upsert(**kwargs) + + +def _slow_writer_target(palace_path, tmp_path_str, pid, result_q): + """Subprocess target: try a slow upsert, report ok/busy.""" + os.environ["HOME"] = tmp_path_str + # Fresh import inside child so HOME monkeypatch routes the lock dir. + from mempalace.backends.chroma import ChromaCollection as _CC + from mempalace.palace import MineAlreadyRunning as _MAR + + fake = _SlowFakeChromaCollection(hold_seconds=0.3) + col = _CC(fake, palace_path=palace_path) + try: + col.upsert(documents=[f"d{pid}"], ids=[f"i{pid}"]) + result_q.put(("ok", pid)) + except _MAR: + result_q.put(("busy", pid)) + + +def test_concurrent_writers_serialize(tmp_path, monkeypatch): + """Two processes calling ``ChromaCollection.upsert`` against the same + palace must be serialized: at most one enters chromadb at a time, the + other raises ``MineAlreadyRunning``. + + This is the property that prevents the parallel HNSW insert race that + drives #974/#965 — under concurrent MCP write fan-out, exactly one + writer reaches chromadb and the rest fail loudly instead of corrupting + the index. + + The slow fake holds the lock for 0.3s per writer, large enough for the + second process to contend even on slow CI runners. + """ + monkeypatch.setenv("HOME", str(tmp_path)) + palace = str(tmp_path / "palace") + + ctx = _get_mp_context() + result_q = ctx.Queue() + + p1 = ctx.Process( + target=_slow_writer_target, args=(palace, str(tmp_path), 1, result_q) + ) + p2 = ctx.Process( + target=_slow_writer_target, args=(palace, str(tmp_path), 2, result_q) + ) + p1.start() + # Tiny stagger so p1 wins the race deterministically; without it the + # OS scheduler can pick either, which is also a valid outcome but + # makes the assertion brittle on slow CI. + time.sleep(0.05) + p2.start() + p1.join(timeout=5) + p2.join(timeout=5) + + outcomes = [result_q.get(timeout=1) for _ in range(2)] + statuses = sorted(o[0] for o in outcomes) + assert statuses == ["busy", "ok"], ( + f"expected one ok + one busy, got {outcomes}" + ) + + +def test_read_path_does_not_acquire_lock(tmp_path, monkeypatch): + """``query`` / ``get`` / ``count`` must not be gated by the write lock. + + Read traffic is the dominant workload (semantic search, MCP get, etc.) + and serializing it against mine would tank latency for no correctness + benefit. This test pins that property: with another process holding + the write lock, reads must still complete instantly. + """ + monkeypatch.setenv("HOME", str(tmp_path)) + palace = str(tmp_path / "palace") + ready = str(tmp_path / "ready") + release = str(tmp_path / "release") + + ctx = _get_mp_context() + holder = ctx.Process(target=_hold_lock, args=(palace, ready, release)) + holder.start() + try: + for _ in range(500): + if os.path.exists(ready): + break + time.sleep(0.01) + assert os.path.exists(ready), "holder failed to acquire lock" + + # _FakeChromaCollection doesn't implement query/get/count; we only + # need to confirm the wrapper does not call into mine_palace_lock + # for reads, which we assert by observing the wrapped methods are + # NOT in ChromaCollection's _write_lock path. A direct check via + # source inspection is more honest than mocking the entire chroma + # surface here. + import inspect + + from mempalace.backends.chroma import ChromaCollection as _CC + + for write_attr in ("add", "upsert", "update", "delete"): + src = inspect.getsource(getattr(_CC, write_attr)) + assert "_write_lock" in src, f"{write_attr} should acquire write lock" + + for read_attr in ("query", "get", "count"): + method = getattr(_CC, read_attr, None) + if method is None: + continue + src = inspect.getsource(method) + assert "_write_lock" not in src, ( + f"{read_attr} must NOT acquire the write lock (read path)" + ) + finally: + open(release, "w").close() + holder.join(timeout=5) diff --git a/tests/test_palace_locks.py b/tests/test_palace_locks.py index 601c8941af..39aa50c597 100644 --- a/tests/test_palace_locks.py +++ b/tests/test_palace_locks.py @@ -135,19 +135,77 @@ def test_different_palaces_dont_conflict(tmp_path, monkeypatch): def test_palace_path_is_normalized(tmp_path, monkeypatch): - """Relative and absolute forms of the same path must use the same lock.""" + """Relative and absolute forms of the same path must use the same lock. + + Cross-process variant: a child holds the absolute form, a relative form + in the parent must hash to the same lock key and raise + ``MineAlreadyRunning``. (The same-thread case is now a re-entrant + pass-through by design — see ``test_reentrant_same_thread_passes_through`` + — so we exercise the normalization invariant across a process boundary + where re-entrance does not apply.) + """ monkeypatch.setenv("HOME", str(tmp_path)) monkeypatch.chdir(tmp_path) os.makedirs(tmp_path / "palace", exist_ok=True) absolute = str(tmp_path / "palace") - relative = "palace" + ready = str(tmp_path / "ready") + release = str(tmp_path / "release") - # Hold the lock with the absolute form; attempting to re-acquire with - # the relative form (which resolves to the same absolute path) must fail. - with mine_palace_lock(absolute): + ctx = _get_mp_context() + holder = ctx.Process(target=_hold_lock, args=(absolute, ready, release)) + holder.start() + try: + for _ in range(500): + if os.path.exists(ready): + break + time.sleep(0.01) + assert os.path.exists(ready), "holder failed to acquire lock in time" + + # Parent holds CWD = tmp_path so "palace" is the same on-disk dir as + # the absolute form. The lock key is sha256(realpath+normcase) so the + # two forms must collide. with pytest.raises(MineAlreadyRunning): - with mine_palace_lock(relative): + with mine_palace_lock("palace"): pytest.fail("normalized path collision should have raised") + finally: + open(release, "w").close() + holder.join(timeout=5) + + +def test_reentrant_same_thread_passes_through(tmp_path, monkeypatch): + """Same thread re-acquiring the same palace lock must not deadlock or raise. + + This is the invariant that makes ``ChromaCollection`` write methods (which + take ``mine_palace_lock`` for MCP/direct-writer protection) compose with + ``miner.mine()`` (which already holds the lock for the entire mine + pipeline). Without the per-thread re-entrant guard the inner acquire + would self-deadlock on the outer flock. + """ + monkeypatch.setenv("HOME", str(tmp_path)) + palace = str(tmp_path / "palace") + with mine_palace_lock(palace): + # Re-enter from the same thread — must yield without raising or hanging. + with mine_palace_lock(palace): + pass + # After the inner exits, the outer is still held: confirm via a + # subprocess that tries to acquire and reports back. + ctx = _get_mp_context() + result_q = ctx.Queue() + child = ctx.Process(target=_try_acquire_expect_busy, args=(palace, result_q)) + child.start() + child.join(timeout=5) + assert result_q.get(timeout=1) == "busy", ( + "outer lock should still be held by parent after inner re-entrant exit" + ) + + +def _try_acquire_expect_busy(palace_path, result_q): + """Helper: try to acquire, push 'busy' (raised) or 'free' (acquired) into queue.""" + try: + with mine_palace_lock(palace_path): + result_q.put("free") + except MineAlreadyRunning: + result_q.put("busy") def test_mine_global_lock_is_alias_for_back_compat(tmp_path, monkeypatch): From db28bf1e846592f997835ca42050e43c12b09303 Mon Sep 17 00:00:00 2001 From: sha2fiddy <103975074+sha2fiddy@users.noreply.github.com> Date: Wed, 29 Apr 2026 19:01:54 -0400 Subject: [PATCH 06/77] fix: paginate closet_llm col.get (#1073) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirror the pagination pattern PR #851 landed in miner.py:status(). A single drawers_col.get(limit=total, ...) on palaces larger than SQLite's SQLITE_MAX_VARIABLE_NUMBER (32766) crashes inside chromadb. Fetch drawers in batch_size=5000 chunks, stepping offset until the collection is drained. by_source aggregation semantics are preserved exactly — grouping, wing filter, meta capture all unchanged. Closes #1073. Related: #802, #850, #1016. --- mempalace/closet_llm.py | 33 +++++--- tests/test_closet_llm.py | 176 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 198 insertions(+), 11 deletions(-) diff --git a/mempalace/closet_llm.py b/mempalace/closet_llm.py index c00b735c03..6274f7965d 100644 --- a/mempalace/closet_llm.py +++ b/mempalace/closet_llm.py @@ -221,17 +221,28 @@ def regenerate_closets( print("No drawers in palace.") return {"processed": 0} - all_data = drawers_col.get(limit=total, include=["documents", "metadatas"]) - by_source = {} - for doc_id, doc, meta in zip(all_data["ids"], all_data["documents"], all_data["metadatas"]): - source = meta.get("source_file", "unknown") - w = meta.get("wing", "") - if wing and w != wing: - continue - if source not in by_source: - by_source[source] = {"drawer_ids": [], "content": [], "meta": meta} - by_source[source]["drawer_ids"].append(doc_id) - by_source[source]["content"].append(doc) + # Paginate the fetch — a single get(limit=total, ...) blows through + # SQLite's SQLITE_MAX_VARIABLE_NUMBER (32766) on large palaces and + # crashes inside chromadb (see #802, #850, #1073). + by_source: dict = {} + batch_size = 5000 + offset = 0 + while offset < total: + batch = drawers_col.get(limit=batch_size, offset=offset, include=["documents", "metadatas"]) + ids = batch["ids"] + if not ids: + break + for doc_id, doc, meta in zip(ids, batch["documents"], batch["metadatas"]): + meta = meta or {} + source = meta.get("source_file", "unknown") + w = meta.get("wing", "") + if wing and w != wing: + continue + if source not in by_source: + by_source[source] = {"drawer_ids": [], "content": [], "meta": meta} + by_source[source]["drawer_ids"].append(doc_id) + by_source[source]["content"].append(doc) + offset += len(ids) sources = list(by_source.keys()) if sample > 0: diff --git a/tests/test_closet_llm.py b/tests/test_closet_llm.py index a92e2fa4b2..3a0e84e478 100644 --- a/tests/test_closet_llm.py +++ b/tests/test_closet_llm.py @@ -296,6 +296,182 @@ def fake_urlopen(req, timeout=None): assert meta.get("generated_by", "").startswith("llm:") assert meta.get("normalize_version") == NORMALIZE_VERSION + def test_regen_paginates_drawer_fetch(self, tmp_path): + """Regression for #1073: drawers_col.get must be paginated at + batch_size=5000. A single get(limit=total, ...) on a palace with + more than SQLite's SQLITE_MAX_VARIABLE_NUMBER (32766) drawers + blows up inside chromadb. Matches the miner.status pattern + introduced in #851 (see #802, #850, #1073).""" + from mempalace import closet_llm as closet_llm_mod + + palace = str(tmp_path / "palace") + + # Build a fake collection: 12_000 drawers across 3 source files, + # enough to force 3 batches of batch_size=5000 (5000 + 5000 + 2000). + n_drawers = 12_000 + ids = [f"d{i:05d}" for i in range(n_drawers)] + docs = [f"doc body {i}" for i in range(n_drawers)] + metas = [ + { + "wing": "w", + "room": "r", + "source_file": f"/src/file_{i % 3}.md", + "entities": "", + } + for i in range(n_drawers) + ] + + get_calls: list = [] + + class FakeDrawersCol: + def count(self): + return n_drawers + + def get(self, limit=None, offset=0, include=None, **kwargs): + get_calls.append({"limit": limit, "offset": offset, "include": include}) + end = min(offset + (limit or n_drawers), n_drawers) + return { + "ids": ids[offset:end], + "documents": docs[offset:end], + "metadatas": metas[offset:end], + } + + class FakeClosetsCol: + """Accept the purge + upsert calls the success path makes.""" + + def get(self, *a, **kw): + return {"ids": [], "documents": [], "metadatas": []} + + def delete(self, *a, **kw): + return None + + def upsert(self, *a, **kw): + return None + + fake_drawers = FakeDrawersCol() + fake_closets = FakeClosetsCol() + + def fake_urlopen(req, timeout=None): + return _FakeResp( + { + "choices": [ + {"message": {"content": '{"topics":["t1"],"quotes":[],"summary":""}'}} + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1}, + } + ) + + cfg = LLMConfig(endpoint="http://local/v1", model="m") + + with ( + patch.object(closet_llm_mod, "get_collection", return_value=fake_drawers), + patch.object(closet_llm_mod, "get_closets_collection", return_value=fake_closets), + patch.object(closet_llm_mod, "purge_file_closets", return_value=None), + patch.object(closet_llm_mod, "upsert_closet_lines", return_value=None), + patch("urllib.request.urlopen", side_effect=fake_urlopen), + ): + result = regenerate_closets(palace, cfg=cfg, dry_run=True) + + # Three paginated calls: (limit=5000, offset=0), (5000, 5000), (5000, 10000). + assert len(get_calls) == 3, f"expected 3 batched fetches, got {len(get_calls)}" + for call in get_calls: + assert ( + call["limit"] == 5000 + ), f"batch must be 5000 — got {call['limit']} (would risk SQLITE_MAX_VARIABLE_NUMBER)" + # include must still request both documents and metadatas + assert "documents" in call["include"] + assert "metadatas" in call["include"] + assert [c["offset"] for c in get_calls] == [0, 5000, 10_000] + + # by_source aggregation must be preserved exactly across batches: + # 12_000 drawers, 3 source files → 4_000 drawers each. + # dry_run=True short-circuits LLM calls but still walks by_source. + assert result.get("processed", 0) == 0 # dry_run + # Verify no single call tried to pull more than batch_size. + assert max(c["limit"] for c in get_calls) <= 5000 + + def test_regen_by_source_aggregates_across_batches(self, tmp_path): + """Pagination must not change the by_source grouping — drawers for + the same source_file split across batches still land in one group.""" + from mempalace import closet_llm as closet_llm_mod + + palace = str(tmp_path / "palace") + + # 7_500 drawers, alternating between two source files → forces + # splits across the 5000/2500 boundary. Each source ends up with + # 3_750 drawers after regrouping. + n_drawers = 7_500 + ids = [f"d{i:05d}" for i in range(n_drawers)] + docs = [f"body-{i}" for i in range(n_drawers)] + metas = [ + { + "wing": "w", + "room": "r", + "source_file": f"/src/file_{i % 2}.md", + "entities": "", + } + for i in range(n_drawers) + ] + + captured_sources: dict = {} + + class FakeDrawersCol: + def count(self): + return n_drawers + + def get(self, limit=None, offset=0, include=None, **kwargs): + end = min(offset + (limit or n_drawers), n_drawers) + return { + "ids": ids[offset:end], + "documents": docs[offset:end], + "metadatas": metas[offset:end], + } + + class FakeClosetsCol: + def get(self, *a, **kw): + return {"ids": [], "documents": [], "metadatas": []} + + def delete(self, *a, **kw): + return None + + def upsert(self, *a, **kw): + return None + + # Hook _call_llm to inspect what regenerate_closets aggregated + # per source before the HTTP boundary. + real_call_llm = closet_llm_mod._call_llm + + def spying_call_llm(cfg, source_file, wing, room, content): + captured_sources[source_file] = content + return ( + {"topics": ["t"], "quotes": [], "summary": ""}, + {"prompt_tokens": 1, "completion_tokens": 1}, + ) + + cfg = LLMConfig(endpoint="http://local/v1", model="m") + + with ( + patch.object(closet_llm_mod, "get_collection", return_value=FakeDrawersCol()), + patch.object(closet_llm_mod, "get_closets_collection", return_value=FakeClosetsCol()), + patch.object(closet_llm_mod, "purge_file_closets", return_value=None), + patch.object(closet_llm_mod, "upsert_closet_lines", return_value=None), + patch.object(closet_llm_mod, "_call_llm", side_effect=spying_call_llm), + ): + regenerate_closets(palace, cfg=cfg) + + # Both sources survived the pagination boundary. + assert set(captured_sources.keys()) == {"/src/file_0.md", "/src/file_1.md"} + # Each source accumulated exactly 3_750 drawer bodies, concatenated + # with the "\n\n" separator the regenerate path uses. + for source, content in captured_sources.items(): + assert content.count("\n\n") == 3_749, ( + f"{source}: expected 3_750 chunks joined (3_749 separators), " + f"got {content.count(chr(10) + chr(10)) + 1}" + ) + + # Silence unused-var lint. + assert real_call_llm is not None + def test_regen_uses_basename_not_split_slash(self, tmp_path, monkeypatch): """Regression: the old closet_id base used ``source.split('/')[-1]`` which silently degrades on Windows paths (``C:\\proj\\a.md`` → From 4d98b0524084f7682f2b7df04c77be2e4312c081 Mon Sep 17 00:00:00 2001 From: Arnold Wender Date: Fri, 24 Apr 2026 11:09:16 +0200 Subject: [PATCH 07/77] fix(kg): validate ISO-8601 date formats at MCP boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tool_kg_query (as_of), tool_kg_add (valid_from), and tool_kg_invalidate (ended) accepted any string and forwarded it to SQLite without format validation. Parameterized queries prevent SQL injection, but invalid date strings silently produce empty result sets — callers cannot distinguish "no fact at this time" from "your date format was unrecognized." This is especially painful for natural-language LLM callers that synthesize dates like "March 2026" or "Jan 2025". Add sanitize_iso_date() in config.py alongside the other input validators. It accepts YYYY, YYYY-MM, and YYYY-MM-DD forms; passes through None/empty; and raises ValueError with a field-named message on anything else. Call it from the three kg MCP tool wrappers before values reach the storage layer so the caller gets a clear error instead of a silent miss. Closes #1164 --- mempalace/config.py | 28 ++++++++++++++++ mempalace/mcp_server.py | 4 +++ tests/test_config.py | 70 +++++++++++++++++++++++++++++++++++++++- tests/test_mcp_server.py | 46 ++++++++++++++++++++++++++ 4 files changed, 147 insertions(+), 1 deletion(-) diff --git a/mempalace/config.py b/mempalace/config.py index cacd1f9184..40057795ba 100644 --- a/mempalace/config.py +++ b/mempalace/config.py @@ -81,6 +81,34 @@ def sanitize_kg_value(value: str, field_name: str = "value") -> str: return value +# ISO-8601 date validator for knowledge-graph temporal parameters +# (as_of, valid_from, valid_to, ended). Parameterized queries already +# prevent SQL injection, but unvalidated date strings silently miss +# every row — callers cannot distinguish "no fact at this time" from +# "your date format was unrecognized." Accept YYYY, YYYY-MM, YYYY-MM-DD. +_ISO_DATE_RE = re.compile(r"^\d{4}(?:-(?:0[1-9]|1[0-2])(?:-(?:0[1-9]|[12]\d|3[01]))?)?$") + + +def sanitize_iso_date(value, field_name: str = "date"): + """Validate an ISO-8601 date string, accepting None or empty as-is. + + Accepts ``YYYY``, ``YYYY-MM``, or ``YYYY-MM-DD``. Raises ValueError + on any other non-empty input so the MCP layer can surface a clear + error to the caller instead of silently returning empty results. + """ + if value is None or value == "": + return value + if not isinstance(value, str): + raise ValueError(f"{field_name} must be a string") + value = value.strip() + if not _ISO_DATE_RE.match(value): + raise ValueError( + f"{field_name}={value!r} is not a valid ISO-8601 date " + f"(expected YYYY, YYYY-MM, or YYYY-MM-DD)" + ) + return value + + def sanitize_content(value: str, max_length: int = 100_000) -> str: """Validate drawer/diary content length.""" if not isinstance(value, str) or not value.strip(): diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 43897c8ca3..8aecd05a47 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -55,6 +55,7 @@ sanitize_kg_value, sanitize_name, sanitize_content, + sanitize_iso_date, ) from .version import __version__ # noqa: E402 from .backends.chroma import ( # noqa: E402 @@ -1021,6 +1022,7 @@ def tool_kg_query(entity: str, as_of: str = None, direction: str = "both"): """Query the knowledge graph for an entity's relationships.""" try: entity = sanitize_kg_value(entity, "entity") + as_of = sanitize_iso_date(as_of, "as_of") except ValueError as e: return {"error": str(e)} if direction not in ("outgoing", "incoming", "both"): @@ -1037,6 +1039,7 @@ def tool_kg_add( subject = sanitize_kg_value(subject, "subject") predicate = sanitize_name(predicate, "predicate") object = sanitize_kg_value(object, "object") + valid_from = sanitize_iso_date(valid_from, "valid_from") except ValueError as e: return {"success": False, "error": str(e)} @@ -1062,6 +1065,7 @@ def tool_kg_invalidate(subject: str, predicate: str, object: str, ended: str = N subject = sanitize_kg_value(subject, "subject") predicate = sanitize_name(predicate, "predicate") object = sanitize_kg_value(object, "object") + ended = sanitize_iso_date(ended, "ended") except ValueError as e: return {"success": False, "error": str(e)} _wal_log( diff --git a/tests/test_config.py b/tests/test_config.py index d7707d9829..f5064e221e 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -3,7 +3,13 @@ import tempfile import pytest -from mempalace.config import MempalaceConfig, normalize_wing_name, sanitize_kg_value, sanitize_name +from mempalace.config import ( + MempalaceConfig, + normalize_wing_name, + sanitize_iso_date, + sanitize_kg_value, + sanitize_name, +) def test_default_config(): @@ -212,3 +218,65 @@ def test_kg_value_rejects_null_bytes(): def test_kg_value_rejects_over_length(): with pytest.raises(ValueError): sanitize_kg_value("a" * 129) + + +# --- sanitize_iso_date --- + + +def test_iso_date_accepts_year_only(): + assert sanitize_iso_date("2026") == "2026" + + +def test_iso_date_accepts_year_month(): + assert sanitize_iso_date("2026-03") == "2026-03" + + +def test_iso_date_accepts_full_date(): + assert sanitize_iso_date("2026-03-15") == "2026-03-15" + + +def test_iso_date_passes_through_none(): + assert sanitize_iso_date(None) is None + + +def test_iso_date_passes_through_empty_string(): + assert sanitize_iso_date("") == "" + + +def test_iso_date_strips_whitespace(): + assert sanitize_iso_date(" 2026-03-15 ") == "2026-03-15" + + +def test_iso_date_rejects_natural_language(): + with pytest.raises(ValueError): + sanitize_iso_date("March 2026") + + +def test_iso_date_rejects_abbreviated_month(): + with pytest.raises(ValueError): + sanitize_iso_date("Jan 2025") + + +def test_iso_date_rejects_us_format(): + with pytest.raises(ValueError): + sanitize_iso_date("03/15/2026") + + +def test_iso_date_rejects_invalid_month(): + with pytest.raises(ValueError): + sanitize_iso_date("2026-13") + + +def test_iso_date_rejects_invalid_day(): + with pytest.raises(ValueError): + sanitize_iso_date("2026-02-32") + + +def test_iso_date_rejects_non_string(): + with pytest.raises(ValueError): + sanitize_iso_date(20260315) + + +def test_iso_date_error_names_field(): + with pytest.raises(ValueError, match="valid_from"): + sanitize_iso_date("yesterday", "valid_from") diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 480b6bdbec..1b80f36b91 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -665,6 +665,52 @@ def test_kg_stats(self, monkeypatch, config, palace_path, seeded_kg): result = tool_kg_stats() assert result["entities"] >= 4 + # --- Date validation at the MCP boundary (issue #1164) --- + + def test_kg_add_rejects_invalid_valid_from(self, monkeypatch, config, palace_path, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_kg_add + + result = tool_kg_add( + subject="Alice", + predicate="likes", + object="coffee", + valid_from="Jan 2025", + ) + assert result["success"] is False + assert "valid_from" in result["error"] + assert "ISO-8601" in result["error"] + + def test_kg_query_rejects_invalid_as_of(self, monkeypatch, config, palace_path, seeded_kg): + _patch_mcp_server(monkeypatch, config, seeded_kg) + from mempalace.mcp_server import tool_kg_query + + result = tool_kg_query(entity="Max", as_of="March 2026") + assert "error" in result + assert "as_of" in result["error"] + + def test_kg_invalidate_rejects_invalid_ended(self, monkeypatch, config, palace_path, seeded_kg): + _patch_mcp_server(monkeypatch, config, seeded_kg) + from mempalace.mcp_server import tool_kg_invalidate + + result = tool_kg_invalidate( + subject="Max", + predicate="does", + object="chess", + ended="yesterday", + ) + assert result["success"] is False + assert "ended" in result["error"] + + def test_kg_query_accepts_partial_iso_dates(self, monkeypatch, config, palace_path, seeded_kg): + _patch_mcp_server(monkeypatch, config, seeded_kg) + from mempalace.mcp_server import tool_kg_query + + # YYYY and YYYY-MM are valid ISO-8601 forms — must not be rejected. + for value in ("2026", "2026-03", "2026-03-15"): + result = tool_kg_query(entity="Max", as_of=value) + assert "error" not in result, f"rejected valid date {value!r}: {result}" + # ── Diary Tools ───────────────────────────────────────────────────────── From abe85763d4da974b8652bdfe3654bee3a7534034 Mon Sep 17 00:00:00 2001 From: Arnold Wender Date: Sun, 26 Apr 2026 12:50:43 +0200 Subject: [PATCH 08/77] fix(kg): reject partial ISO dates to avoid silent empty result sets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per qodo-ai review on PR #1167: sanitize_iso_date() previously accepted YYYY and YYYY-MM, but KnowledgeGraph.query_entity() compares valid_from/ valid_to TEXT columns lexicographically against as_of. Lexicographic comparison treats '2026-01-01' as greater than '2026' (because '-' > end-of-string), so partial as_of values silently excluded valid facts — re-introducing the silent-empty-results problem this PR was meant to fix. Tighten _ISO_DATE_RE to require YYYY-MM-DD only. Update docstring and error message accordingly. Invert the two test cases that asserted partials were accepted. --- mempalace/config.py | 18 +++++++++++------- tests/test_config.py | 12 ++++++++---- tests/test_mcp_server.py | 15 +++++++++++---- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/mempalace/config.py b/mempalace/config.py index 40057795ba..2252a49484 100644 --- a/mempalace/config.py +++ b/mempalace/config.py @@ -85,16 +85,21 @@ def sanitize_kg_value(value: str, field_name: str = "value") -> str: # (as_of, valid_from, valid_to, ended). Parameterized queries already # prevent SQL injection, but unvalidated date strings silently miss # every row — callers cannot distinguish "no fact at this time" from -# "your date format was unrecognized." Accept YYYY, YYYY-MM, YYYY-MM-DD. -_ISO_DATE_RE = re.compile(r"^\d{4}(?:-(?:0[1-9]|1[0-2])(?:-(?:0[1-9]|[12]\d|3[01]))?)?$") +# "your date format was unrecognized." Require full YYYY-MM-DD: KG +# queries compare TEXT dates lexicographically, so partials like "2026" +# would re-introduce silent empty results (e.g. "2026-01-01" <= "2026" +# is False), defeating the purpose of validation. +_ISO_DATE_RE = re.compile(r"^\d{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12]\d|3[01])$") def sanitize_iso_date(value, field_name: str = "date"): """Validate an ISO-8601 date string, accepting None or empty as-is. - Accepts ``YYYY``, ``YYYY-MM``, or ``YYYY-MM-DD``. Raises ValueError - on any other non-empty input so the MCP layer can surface a clear - error to the caller instead of silently returning empty results. + Accepts only ``YYYY-MM-DD``. Raises ValueError on any other + non-empty input so the MCP layer can surface a clear error to the + caller instead of silently returning empty results. Partial dates + (``YYYY``, ``YYYY-MM``) are rejected because KG queries compare + TEXT dates lexicographically and would silently exclude valid facts. """ if value is None or value == "": return value @@ -103,8 +108,7 @@ def sanitize_iso_date(value, field_name: str = "date"): value = value.strip() if not _ISO_DATE_RE.match(value): raise ValueError( - f"{field_name}={value!r} is not a valid ISO-8601 date " - f"(expected YYYY, YYYY-MM, or YYYY-MM-DD)" + f"{field_name}={value!r} is not a valid ISO-8601 date " f"(expected YYYY-MM-DD)" ) return value diff --git a/tests/test_config.py b/tests/test_config.py index f5064e221e..204faae421 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -223,12 +223,16 @@ def test_kg_value_rejects_over_length(): # --- sanitize_iso_date --- -def test_iso_date_accepts_year_only(): - assert sanitize_iso_date("2026") == "2026" +def test_iso_date_rejects_year_only(): + # Partial dates re-introduce silent empty result sets via lexicographic + # TEXT comparison in KG queries (e.g. "2026-01-01" <= "2026" is False). + with pytest.raises(ValueError): + sanitize_iso_date("2026") -def test_iso_date_accepts_year_month(): - assert sanitize_iso_date("2026-03") == "2026-03" +def test_iso_date_rejects_year_month(): + with pytest.raises(ValueError): + sanitize_iso_date("2026-03") def test_iso_date_accepts_full_date(): diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 1b80f36b91..136b6f380f 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -702,14 +702,21 @@ def test_kg_invalidate_rejects_invalid_ended(self, monkeypatch, config, palace_p assert result["success"] is False assert "ended" in result["error"] - def test_kg_query_accepts_partial_iso_dates(self, monkeypatch, config, palace_path, seeded_kg): + def test_kg_query_rejects_partial_iso_dates(self, monkeypatch, config, palace_path, seeded_kg): _patch_mcp_server(monkeypatch, config, seeded_kg) from mempalace.mcp_server import tool_kg_query - # YYYY and YYYY-MM are valid ISO-8601 forms — must not be rejected. - for value in ("2026", "2026-03", "2026-03-15"): + # Partial ISO dates are rejected: KG queries compare TEXT dates + # lexicographically, so "2026-01-01" <= "2026" is False, which + # silently excludes facts. Reject at the boundary — only YYYY-MM-DD + # produces correct results. + for value in ("2026", "2026-03"): result = tool_kg_query(entity="Max", as_of=value) - assert "error" not in result, f"rejected valid date {value!r}: {result}" + assert "error" in result, f"accepted partial date {value!r}: {result}" + + # Full ISO-8601 dates still pass. + result = tool_kg_query(entity="Max", as_of="2026-03-15") + assert "error" not in result, f"rejected valid date: {result}" # ── Diary Tools ───────────────────────────────────────────────────────── From c3e1104e75d5310aeb2147c999fb19af8a9b1dcc Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Thu, 30 Apr 2026 09:31:32 -0600 Subject: [PATCH 09/77] fix(chroma): harden HNSW startup preflight --- mempalace/backends/chroma.py | 131 ++++++++++++++-- tests/test_backends.py | 283 ++++++++++++++++++++++++++++++++++- 2 files changed, 395 insertions(+), 19 deletions(-) diff --git a/mempalace/backends/chroma.py b/mempalace/backends/chroma.py index 01ac62771b..646969ba12 100644 --- a/mempalace/backends/chroma.py +++ b/mempalace/backends/chroma.py @@ -3,7 +3,9 @@ import datetime as _dt import logging import os +import pickle import sqlite3 +from numbers import Integral from pathlib import Path from typing import Any, Optional @@ -591,6 +593,97 @@ def _pin_hnsw_threads(collection) -> None: _BLOB_FIX_MARKER = ".blob_seq_ids_migrated" +def _valid_dimensionality(value: object) -> bool: + return isinstance(value, Integral) and not isinstance(value, bool) and int(value) > 0 + + +def _persisted_metadata_fields(obj: object) -> tuple[object, object]: + if isinstance(obj, dict): + return obj.get("dimensionality"), obj.get("id_to_label") + return getattr(obj, "dimensionality", None), getattr(obj, "id_to_label", None) + + +def quarantine_invalid_hnsw_metadata(palace_path: str) -> list[str]: + """Quarantine segment dirs whose ``index_metadata.pickle`` is unreadable or invalid. + + Chroma's persisted HNSW metadata is untrusted disk state. If a segment has + labels but no valid positive dimensionality, current Chroma versions can + accept the pickle and crash later in the Rust loader. We rename the entire + segment out of the way before ``PersistentClient`` opens so Chroma can + rebuild cleanly instead of touching known-bad metadata. + """ + try: + entries = os.listdir(palace_path) + except OSError: + return [] + + moved: list[str] = [] + for name in entries: + if "-" not in name or name.startswith(".") or ".drift-" in name or ".corrupt-" in name: + continue + seg_dir = os.path.join(palace_path, name) + if not os.path.isdir(seg_dir): + continue + + meta_path = os.path.join(seg_dir, "index_metadata.pickle") + if not os.path.isfile(meta_path): + continue + + reason = None + try: + persisted = _SafePersistentDataUnpickler.load(meta_path) + except (EOFError, OSError): + logger.debug( + "Skipping invalid-HNSW quarantine for transient metadata read in %s", + meta_path, + exc_info=True, + ) + continue + except pickle.UnpicklingError as exc: + if "truncated" in str(exc).lower() or "ran out of input" in str(exc).lower(): + logger.debug( + "Skipping invalid-HNSW quarantine for transient metadata read in %s", + meta_path, + exc_info=True, + ) + continue + reason = f"invalid index_metadata.pickle: {exc}" + except Exception as exc: + reason = f"invalid index_metadata.pickle: {exc}" + else: + if not isinstance(persisted, dict) and not ( + hasattr(persisted, "dimensionality") or hasattr(persisted, "id_to_label") + ): + reason = f"unrecognized index_metadata.pickle payload: {type(persisted).__name__}" + else: + dimensionality, id_to_label = _persisted_metadata_fields(persisted) + if id_to_label is not None and not isinstance(id_to_label, dict): + reason = f"invalid id_to_label type {type(id_to_label).__name__}" + else: + has_labels = bool(id_to_label) + if has_labels and not _valid_dimensionality(dimensionality): + reason = ( + "labels present but dimensionality is missing or invalid " + f"({dimensionality!r})" + ) + elif dimensionality is not None and not _valid_dimensionality(dimensionality): + reason = f"invalid dimensionality {dimensionality!r}" + + if reason is None: + continue + + stamp = _dt.datetime.now().strftime("%Y%m%d-%H%M%S") + target = f"{seg_dir}.corrupt-{stamp}" + try: + os.rename(seg_dir, target) + moved.append(target) + logger.warning("Quarantined invalid HNSW metadata in %s: %s", seg_dir, reason) + except OSError: + logger.exception("Failed to quarantine invalid HNSW metadata in %s", seg_dir) + + return moved + + def _fix_blob_seq_ids(palace_path: str) -> None: """Fix ChromaDB 0.6.x -> 1.5.x migration bug: BLOB seq_ids -> INTEGER. @@ -994,6 +1087,12 @@ def _client(self, palace_path: str): if cached is None or inode_changed or mtime_changed or mtime_appeared: _fix_blob_seq_ids(palace_path) + if inode_changed: + ChromaBackend._quarantined_paths.discard(palace_path) + if palace_path not in ChromaBackend._quarantined_paths: + quarantine_invalid_hnsw_metadata(palace_path) + quarantine_stale_hnsw(palace_path) + ChromaBackend._quarantined_paths.add(palace_path) cached = chromadb.PersistentClient(path=palace_path) self._clients[palace_path] = cached # Re-stat after the client constructor runs: chromadb creates @@ -1006,26 +1105,27 @@ def _client(self, palace_path: str): # Public static helpers (legacy; prefer :meth:`get_collection`) # ------------------------------------------------------------------ - # Per-process record of palaces that have already had quarantine_stale_hnsw - # invoked at least once. The proactive drift check is a *cold-start* - # protection — it catches HNSW segments that arrived stale relative to - # ``chroma.sqlite3`` (e.g. cross-machine replication, partial restore, - # crashed-mid-write). Once a long-running process has opened the palace - # cleanly, re-firing on every reconnect is a *runtime thrash*: the - # daemon's own writes bump sqlite mtime but HNSW flushes batch on - # chromadb's internal cadence, so the mtime gap naturally exceeds the - # threshold under steady write load even though nothing is corrupt. + # Per-process record of palaces that have already had the cold-start + # quarantine invoked at least once. The proactive HNSW checks are a + # *cold-start* protection — they catch segments that arrive stale relative + # to ``chroma.sqlite3`` or invalid on disk (e.g. cross-machine replication, + # partial restore, crashed-mid-write). Once a long-running process has + # opened the palace cleanly, re-firing the stale check on every reconnect + # is a *runtime thrash*: the daemon's own writes bump sqlite mtime but HNSW + # flushes batch on chromadb's internal cadence, so the mtime gap naturally + # exceeds the threshold under steady write load even though nothing is + # corrupt. # Real runtime drift is still handled — palace-daemon's ``_auto_repair`` # calls :func:`quarantine_stale_hnsw` directly on observed HNSW errors, # which bypasses this gate. # # Thread-safety: this set is mutated without a lock. Two concurrent # ``make_client()`` calls for the same palace can both pass the - # membership check and both invoke ``quarantine_stale_hnsw``. That's - # safe because the function is idempotent (mtime check + timestamped - # rename of distinct directories), so the worst-case race produces - # one redundant rename attempt that no-ops. Idempotency is the - # safety property; locking would add cost without correctness gain. + # membership check and both invoke the cold-start quarantine. That's + # safe because the functions are idempotent (mtime checks + timestamped + # rename of distinct directories), so the worst-case race produces one + # redundant rename attempt that no-ops. Idempotency is the safety + # property; locking would add cost without correctness gain. _quarantined_paths: set[str] = set() @staticmethod @@ -1036,12 +1136,13 @@ def make_client(palace_path: str): own client cache. New code should obtain a collection through :meth:`get_collection` which manages caching internally. - Quarantines stale HNSW segments **once per palace per process**. See + Quarantines HNSW segments **once per palace per process**. See :attr:`_quarantined_paths` for the rationale (cold-start protection vs. runtime thrash on steady-write daemons). """ _fix_blob_seq_ids(palace_path) if palace_path not in ChromaBackend._quarantined_paths: + quarantine_invalid_hnsw_metadata(palace_path) quarantine_stale_hnsw(palace_path) ChromaBackend._quarantined_paths.add(palace_path) return chromadb.PersistentClient(path=palace_path) diff --git a/tests/test_backends.py b/tests/test_backends.py index 5efa71b319..cbbcdefdf3 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -1,4 +1,5 @@ import os +import pickle import sqlite3 from pathlib import Path @@ -18,6 +19,7 @@ ChromaCollection, _fix_blob_seq_ids, _pin_hnsw_threads, + quarantine_invalid_hnsw_metadata, quarantine_stale_hnsw, ) @@ -708,7 +710,10 @@ def test_make_client_quarantines_only_on_first_call_per_palace(tmp_path, monkeyp """Quarantine fires on first ``make_client()`` for a palace, then is skipped on subsequent calls — prevents runtime thrash where a daemon's own steady writes bump ``chroma.sqlite3`` faster than HNSW flushes, - making the mtime heuristic falsely trigger every reconnect.""" + making the mtime heuristic falsely trigger every reconnect. + + Invalid metadata quarantine shares the same cold-start gate here; the + more aggressive refresh path lives in ``_client()``.""" from mempalace.backends.chroma import ChromaBackend palace_path = str(tmp_path / "palace") @@ -730,9 +735,37 @@ def _spy(path, stale_seconds=300.0): ChromaBackend.make_client(palace_path) ChromaBackend.make_client(palace_path) - assert calls == [ - palace_path - ], "quarantine_stale_hnsw should fire once per palace per process, not on every reconnect" + assert calls == [palace_path], ( + "quarantine_stale_hnsw should fire once per palace per process, not on every reconnect" + ) + + +def test_make_client_gates_invalid_metadata_on_first_call(tmp_path, monkeypatch): + """Invalid metadata quarantine is gated on the first make_client() call.""" + from mempalace.backends.chroma import ChromaBackend + + palace_path = str(tmp_path / "palace") + os.makedirs(palace_path, exist_ok=True) + (Path(palace_path) / "chroma.sqlite3").write_text("") + + monkeypatch.setattr(ChromaBackend, "_quarantined_paths", set()) + + calls: list[str] = [] + + def _invalid(path, *args, **kwargs): + calls.append(path) + return [] + + def _stale(path, stale_seconds=300.0): + return [] + + monkeypatch.setattr("mempalace.backends.chroma.quarantine_invalid_hnsw_metadata", _invalid) + monkeypatch.setattr("mempalace.backends.chroma.quarantine_stale_hnsw", _stale) + + ChromaBackend.make_client(palace_path) + ChromaBackend.make_client(palace_path) + + assert calls == [palace_path] def test_make_client_quarantines_each_palace_independently(tmp_path, monkeypatch): @@ -811,3 +844,245 @@ def test_get_collection_applies_retrofit_on_existing_palace(tmp_path): ) assert wrapper._collection.configuration_json["hnsw"]["num_threads"] == 1 + + +def test_quarantine_invalid_hnsw_metadata_renames_missing_dimensionality(tmp_path): + palace = tmp_path / "palace" + palace.mkdir() + seg = palace / "abcd-1234-5678" + seg.mkdir() + with open(seg / "index_metadata.pickle", "wb") as f: + pickle.dump({"dimensionality": None, "id_to_label": {"a": 1}}, f) + + moved = quarantine_invalid_hnsw_metadata(str(palace)) + + assert len(moved) == 1 + assert ".corrupt-" in moved[0] + assert not seg.exists() + + +def test_quarantine_invalid_hnsw_metadata_allows_uninitialized_segment(tmp_path): + palace = tmp_path / "palace" + palace.mkdir() + seg = palace / "abcd-1234-5678" + seg.mkdir() + with open(seg / "index_metadata.pickle", "wb") as f: + pickle.dump({"dimensionality": None, "id_to_label": {}}, f) + + moved = quarantine_invalid_hnsw_metadata(str(palace)) + + assert moved == [] + assert seg.exists() + + +def test_quarantine_invalid_hnsw_metadata_rejects_non_dict_id_to_label(tmp_path): + palace = tmp_path / "palace" + palace.mkdir() + seg = palace / "abcd-1234-5678" + seg.mkdir() + with open(seg / "index_metadata.pickle", "wb") as f: + pickle.dump({"dimensionality": 8, "id_to_label": ["a", "b"]}, f) + + moved = quarantine_invalid_hnsw_metadata(str(palace)) + + assert len(moved) == 1 + assert ".corrupt-" in moved[0] + assert not seg.exists() + + +def test_quarantine_invalid_hnsw_metadata_rejects_non_schema_payload(tmp_path): + palace = tmp_path / "palace" + palace.mkdir() + seg = palace / "abcd-1234-5678" + seg.mkdir() + with open(seg / "index_metadata.pickle", "wb") as f: + pickle.dump(["not", "a", "metadata", "object"], f) + + moved = quarantine_invalid_hnsw_metadata(str(palace)) + + assert len(moved) == 1 + assert ".corrupt-" in moved[0] + assert not seg.exists() + + +def _dangerous_pickle_payload_executed(): + raise AssertionError("unsafe pickle payload executed") + + +class _DangerousPickle: + def __reduce__(self): + return (_dangerous_pickle_payload_executed, ()) + + +def test_quarantine_invalid_hnsw_metadata_rejects_unsafe_pickle(tmp_path): + palace = tmp_path / "palace" + palace.mkdir() + seg = palace / "abcd-1234-5678" + seg.mkdir() + with open(seg / "index_metadata.pickle", "wb") as f: + pickle.dump(_DangerousPickle(), f) + + moved = quarantine_invalid_hnsw_metadata(str(palace)) + + assert len(moved) == 1 + assert ".corrupt-" in moved[0] + assert not seg.exists() + + +def test_quarantine_invalid_hnsw_metadata_skips_transient_read_errors(tmp_path, monkeypatch): + palace = tmp_path / "palace" + palace.mkdir() + seg = palace / "abcd-1234-5678" + seg.mkdir() + meta = seg / "index_metadata.pickle" + meta.write_bytes(b"partial") + + monkeypatch.setattr( + "mempalace.backends.chroma._SafePersistentDataUnpickler.load", + lambda path: (_ for _ in ()).throw(EOFError("flush in progress")), + ) + + moved = quarantine_invalid_hnsw_metadata(str(palace)) + + assert moved == [] + assert seg.exists() + + +def test_quarantine_invalid_hnsw_metadata_skips_truncated_pickle(tmp_path, monkeypatch): + palace = tmp_path / "palace" + palace.mkdir() + seg = palace / "abcd-1234-5678" + seg.mkdir() + meta = seg / "index_metadata.pickle" + meta.write_bytes(b"partial") + + monkeypatch.setattr( + "mempalace.backends.chroma._SafePersistentDataUnpickler.load", + lambda path: (_ for _ in ()).throw(pickle.UnpicklingError("pickle data was truncated")), + ) + + moved = quarantine_invalid_hnsw_metadata(str(palace)) + + assert moved == [] + assert seg.exists() + + +def test_chroma_backend_preflights_metadata_before_persistent_client(tmp_path, monkeypatch): + palace = tmp_path / "palace" + palace.mkdir() + calls = [] + + def _record(name): + def inner(path, *args, **kwargs): + calls.append((name, path)) + return [] if name != "blob" else None + + return inner + + monkeypatch.setattr("mempalace.backends.chroma._fix_blob_seq_ids", _record("blob")) + monkeypatch.setattr( + "mempalace.backends.chroma.quarantine_invalid_hnsw_metadata", _record("invalid") + ) + monkeypatch.setattr("mempalace.backends.chroma.quarantine_stale_hnsw", _record("stale")) + + class DummyClient: + pass + + monkeypatch.setattr( + "mempalace.backends.chroma.chromadb.PersistentClient", lambda path: DummyClient() + ) + + backend = ChromaBackend() + backend._client(str(palace)) + + assert calls == [ + ("blob", str(palace)), + ("invalid", str(palace)), + ("stale", str(palace)), + ] + + +def test_chroma_backend_stale_quarantine_is_cold_start_only_on_refresh(tmp_path, monkeypatch): + palace = tmp_path / "palace" + palace.mkdir() + (palace / "chroma.sqlite3").write_text("") + calls = [] + + def _record(name): + def inner(path, *args, **kwargs): + calls.append((name, path)) + return [] if name != "blob" else None + + return inner + + monkeypatch.setattr(ChromaBackend, "_quarantined_paths", set()) + monkeypatch.setattr("mempalace.backends.chroma._fix_blob_seq_ids", _record("blob")) + monkeypatch.setattr( + "mempalace.backends.chroma.quarantine_invalid_hnsw_metadata", _record("invalid") + ) + monkeypatch.setattr("mempalace.backends.chroma.quarantine_stale_hnsw", _record("stale")) + + class DummyClient: + pass + + monkeypatch.setattr( + "mempalace.backends.chroma.chromadb.PersistentClient", lambda path: DummyClient() + ) + + backend = ChromaBackend() + stats = iter([(1, 1.0), (1, 1.0), (1, 2.0), (1, 2.0)]) + monkeypatch.setattr(backend, "_db_stat", lambda path: next(stats)) + + backend._client(str(palace)) + backend._client(str(palace)) + + assert calls == [ + ("blob", str(palace)), + ("invalid", str(palace)), + ("stale", str(palace)), + ("blob", str(palace)), + ] + + +def test_chroma_backend_requarantines_after_inode_replacement(tmp_path, monkeypatch): + palace = tmp_path / "palace" + palace.mkdir() + (palace / "chroma.sqlite3").write_text("") + calls = [] + + def _record(name): + def inner(path, *args, **kwargs): + calls.append((name, path)) + return [] if name != "blob" else None + + return inner + + monkeypatch.setattr(ChromaBackend, "_quarantined_paths", set()) + monkeypatch.setattr("mempalace.backends.chroma._fix_blob_seq_ids", _record("blob")) + monkeypatch.setattr( + "mempalace.backends.chroma.quarantine_invalid_hnsw_metadata", _record("invalid") + ) + monkeypatch.setattr("mempalace.backends.chroma.quarantine_stale_hnsw", _record("stale")) + + class DummyClient: + pass + + monkeypatch.setattr( + "mempalace.backends.chroma.chromadb.PersistentClient", lambda path: DummyClient() + ) + + backend = ChromaBackend() + stats = iter([(1, 1.0), (1, 1.0), (2, 2.0), (2, 2.0)]) + monkeypatch.setattr(backend, "_db_stat", lambda path: next(stats)) + + backend._client(str(palace)) + backend._client(str(palace)) + + assert calls == [ + ("blob", str(palace)), + ("invalid", str(palace)), + ("stale", str(palace)), + ("blob", str(palace)), + ("invalid", str(palace)), + ("stale", str(palace)), + ] From 7fa27bd23101f0c7dc4fa33f84be0b1d9710dbe2 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Thu, 30 Apr 2026 09:31:32 -0600 Subject: [PATCH 10/77] fix(repair): rebuild collections through temp staging --- mempalace/repair.py | 159 +++++++++++++++++----- tests/test_repair.py | 314 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 436 insertions(+), 37 deletions(-) diff --git a/mempalace/repair.py b/mempalace/repair.py index 1cd1556975..49d6abef71 100644 --- a/mempalace/repair.py +++ b/mempalace/repair.py @@ -37,10 +37,13 @@ from datetime import datetime from typing import Optional +from chromadb.errors import NotFoundError as ChromaNotFoundError + from .backends.chroma import ChromaBackend, hnsw_capacity_status COLLECTION_NAME = "mempalace_drawers" +REPAIR_TEMP_COLLECTION = f"{COLLECTION_NAME}__repair_tmp" def _get_palace_path(): @@ -83,6 +86,108 @@ def _paginate_ids(col, where=None): return ids +def _extract_drawers(col, total: int, batch_size: int): + all_ids = [] + all_docs = [] + all_metas = [] + offset = 0 + while offset < total: + batch = col.get(limit=batch_size, offset=offset, include=["documents", "metadatas"]) + if not batch["ids"]: + break + all_ids.extend(batch["ids"]) + all_docs.extend(batch["documents"]) + all_metas.extend(batch["metadatas"]) + offset += len(batch["ids"]) + return all_ids, all_docs, all_metas + + +def _verify_collection_count(col, expected: int, label: str) -> None: + actual = col.count() + if actual != expected: + raise RuntimeError(f"{label} count mismatch: expected {expected}, got {actual}") + + +def _is_missing_collection_value_error(exc: ValueError) -> bool: + message = str(exc).lower() + return "does not exist" in message or "not found" in message + + +def _delete_collection_if_exists(backend, palace_path: str, collection_name: str) -> None: + try: + backend.delete_collection(palace_path, collection_name) + except ValueError as exc: + if _is_missing_collection_value_error(exc): + return + raise + except (FileNotFoundError, ChromaNotFoundError): + return + + +class RebuildCollectionError(RuntimeError): + """Raised when temp rebuild fails, carrying whether the live swap happened.""" + + def __init__(self, message: str, *, live_replaced: bool): + super().__init__(message) + self.live_replaced = live_replaced + + +def _rebuild_collection_via_temp( + backend, + palace_path: str, + all_ids, + all_docs, + all_metas, + batch_size: int, + progress=print, +) -> int: + expected = len(all_ids) + temp_name = REPAIR_TEMP_COLLECTION + live_replaced = False + + try: + _delete_collection_if_exists(backend, palace_path, temp_name) + + progress(f" Building temporary collection: {temp_name}") + temp_col = backend.create_collection(palace_path, temp_name) + staged = 0 + for i in range(0, expected, batch_size): + batch_ids = all_ids[i : i + batch_size] + batch_docs = all_docs[i : i + batch_size] + batch_metas = all_metas[i : i + batch_size] + temp_col.upsert(documents=batch_docs, ids=batch_ids, metadatas=batch_metas) + staged += len(batch_ids) + progress(f" Staged {staged}/{expected} drawers...") + _verify_collection_count(temp_col, expected, "temporary rebuild") + + progress(" Rebuilding live collection...") + backend.delete_collection(palace_path, COLLECTION_NAME) + live_replaced = True + new_col = backend.create_collection(palace_path, COLLECTION_NAME) + + rebuilt = 0 + for i in range(0, expected, batch_size): + batch_ids = all_ids[i : i + batch_size] + batch_docs = all_docs[i : i + batch_size] + batch_metas = all_metas[i : i + batch_size] + new_col.upsert(documents=batch_docs, ids=batch_ids, metadatas=batch_metas) + rebuilt += len(batch_ids) + progress(f" Re-filed {rebuilt}/{expected} drawers...") + _verify_collection_count(new_col, expected, "rebuilt live collection") + + try: + _delete_collection_if_exists(backend, palace_path, temp_name) + except Exception: + pass + return rebuilt + except Exception as exc: + try: + _delete_collection_if_exists(backend, palace_path, temp_name) + except Exception: + pass + raise RebuildCollectionError(str(exc), live_replaced=live_replaced) from exc + + def scan_palace(palace_path=None, only_wing=None): """Scan the palace for corrupt/unfetchable IDs. @@ -373,18 +478,7 @@ def rebuild_index(palace_path=None, confirm_truncation_ok: bool = False): # Extract all drawers in batches print("\n Extracting drawers...") batch_size = 5000 - all_ids = [] - all_docs = [] - all_metas = [] - offset = 0 - while offset < total: - batch = col.get(limit=batch_size, offset=offset, include=["documents", "metadatas"]) - if not batch["ids"]: - break - all_ids.extend(batch["ids"]) - all_docs.extend(batch["documents"]) - all_metas.extend(batch["metadatas"]) - offset += len(batch["ids"]) + all_ids, all_docs, all_metas = _extract_drawers(col, total, batch_size) print(f" Extracted {len(all_ids)} drawers") # ── #1208 guard ────────────────────────────────────────────────── @@ -407,28 +501,33 @@ def rebuild_index(palace_path=None, confirm_truncation_ok: bool = False): # Rebuild with correct HNSW settings print(" Rebuilding collection with hnsw:space=cosine...") - backend.delete_collection(palace_path, COLLECTION_NAME) - new_col = backend.create_collection(palace_path, COLLECTION_NAME) - - filed = 0 try: - for i in range(0, len(all_ids), batch_size): - batch_ids = all_ids[i : i + batch_size] - batch_docs = all_docs[i : i + batch_size] - batch_metas = all_metas[i : i + batch_size] - new_col.upsert(documents=batch_docs, ids=batch_ids, metadatas=batch_metas) - filed += len(batch_ids) - print(f" Re-filed {filed}/{len(all_ids)} drawers...") - except Exception as e: + filed = _rebuild_collection_via_temp( + backend, + palace_path, + all_ids, + all_docs, + all_metas, + batch_size, + progress=print, + ) + except RebuildCollectionError as e: print(f"\n ERROR during rebuild: {e}") - print(f" Only {filed}/{len(all_ids)} drawers were re-filed.") - if os.path.exists(backup_path): + print(" Rebuild aborted before completion.") + if e.live_replaced and os.path.exists(backup_path): print(f" Restoring from backup: {backup_path}") - backend.delete_collection(palace_path, COLLECTION_NAME) - shutil.copy2(backup_path, sqlite_path) - print(" Backup restored. Palace is back to pre-repair state.") - else: + try: + _close_chroma_handles(palace_path) + _delete_collection_if_exists(backend, palace_path, COLLECTION_NAME) + shutil.copy2(backup_path, sqlite_path) + print(" Backup restored. Palace is back to pre-repair state.") + except Exception as restore_error: + print(f" Backup restore failed: {restore_error}") + print(f" Manual restore required from: {backup_path}") + elif e.live_replaced: print(" No backup available. Re-mine from source files to recover.") + else: + print(" Live collection was not replaced; leaving the original palace untouched.") raise print(f"\n Repair complete. {filed} drawers rebuilt.") diff --git a/tests/test_repair.py b/tests/test_repair.py index bc770ddfbe..33daad9f7f 100644 --- a/tests/test_repair.py +++ b/tests/test_repair.py @@ -2,7 +2,7 @@ import os import sqlite3 -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch import pytest @@ -229,8 +229,11 @@ def test_rebuild_index_success(mock_backend_cls, mock_shutil, tmp_path): } mock_new_col = MagicMock() + mock_new_col.count.return_value = 2 + mock_temp_col = MagicMock() + mock_temp_col.count.return_value = 2 mock_backend = _install_mock_backend(mock_backend_cls, mock_col) - mock_backend.create_collection.return_value = mock_new_col + mock_backend.create_collection.side_effect = [mock_temp_col, mock_new_col] repair.rebuild_index(palace_path=str(tmp_path)) @@ -239,14 +242,74 @@ def test_rebuild_index_success(mock_backend_cls, mock_shutil, tmp_path): assert "chroma.sqlite3" in str(mock_shutil.copy2.call_args) # Verify: deleted and recreated (cosine is the backend default) - mock_backend.delete_collection.assert_called_once_with(str(tmp_path), "mempalace_drawers") - mock_backend.create_collection.assert_called_once_with(str(tmp_path), "mempalace_drawers") + assert mock_backend.create_collection.call_args_list == [ + call(str(tmp_path), "mempalace_drawers__repair_tmp"), + call(str(tmp_path), "mempalace_drawers"), + ] + assert mock_backend.delete_collection.call_args_list == [ + call(str(tmp_path), "mempalace_drawers__repair_tmp"), + call(str(tmp_path), "mempalace_drawers"), + call(str(tmp_path), "mempalace_drawers__repair_tmp"), + ] # Verify: used upsert not add + mock_temp_col.upsert.assert_called_once() mock_new_col.upsert.assert_called_once() mock_new_col.add.assert_not_called() +@patch("mempalace.repair.shutil") +@patch("mempalace.repair.ChromaBackend") +def test_rebuild_index_ignores_missing_temp_collection_at_start( + mock_backend_cls, mock_shutil, tmp_path +): + sqlite_path = tmp_path / "chroma.sqlite3" + sqlite_path.write_text("fake") + + def _fake_copy2(src, dst): + with open(dst, "w") as handle: + handle.write("backup") + + mock_shutil.copy2.side_effect = _fake_copy2 + + mock_col = MagicMock() + mock_col.count.return_value = 2 + mock_col.get.return_value = { + "ids": ["id1", "id2"], + "documents": ["doc1", "doc2"], + "metadatas": [{"wing": "a"}, {"wing": "b"}], + } + + mock_new_col = MagicMock() + mock_new_col.count.return_value = 2 + mock_temp_col = MagicMock() + mock_temp_col.count.return_value = 2 + mock_backend = _install_mock_backend(mock_backend_cls, mock_col) + mock_backend.create_collection.side_effect = [mock_temp_col, mock_new_col] + mock_backend.delete_collection.side_effect = [ + ValueError("Collection [mempalace_drawers__repair_tmp] does not exist"), + None, + None, + ] + + repair.rebuild_index(palace_path=str(tmp_path)) + + assert mock_shutil.copy2.call_count == 1 + assert mock_backend.delete_collection.call_args_list == [ + call(str(tmp_path), "mempalace_drawers__repair_tmp"), + call(str(tmp_path), "mempalace_drawers"), + call(str(tmp_path), "mempalace_drawers__repair_tmp"), + ] + + +def test_delete_collection_if_exists_reraises_unexpected_value_error(): + mock_backend = MagicMock() + mock_backend.delete_collection.side_effect = ValueError("invalid collection name") + + with pytest.raises(ValueError, match="invalid collection name"): + repair._delete_collection_if_exists(mock_backend, "/palace", "bad/name") + + @patch("mempalace.repair.shutil") @patch("mempalace.repair.ChromaBackend") def test_rebuild_index_error_reading(mock_backend_cls, mock_shutil, tmp_path): @@ -365,19 +428,256 @@ def test_rebuild_index_proceeds_with_override(mock_backend_cls, mock_shutil, tmp }, {"ids": [], "documents": [], "metadatas": []}, ] + mock_temp_col = MagicMock() + mock_temp_col.count.return_value = 10_000 mock_new_col = MagicMock() + mock_new_col.count.return_value = 10_000 mock_backend.get_collection.return_value = mock_col - mock_backend.create_collection.return_value = mock_new_col + mock_backend.create_collection.side_effect = [mock_temp_col, mock_new_col] mock_backend_cls.return_value = mock_backend with patch("mempalace.repair.sqlite_drawer_count", return_value=67_580): repair.rebuild_index(palace_path=str(tmp_path), confirm_truncation_ok=True) - mock_backend.delete_collection.assert_called_once() - mock_backend.create_collection.assert_called_once() + assert mock_backend.delete_collection.call_count == 3 + assert mock_backend.create_collection.call_count == 2 + mock_temp_col.upsert.assert_called() mock_new_col.upsert.assert_called() +@patch("mempalace.repair.shutil") +@patch("mempalace.repair.ChromaBackend") +def test_rebuild_index_stage_failure_leaves_live_collection_untouched( + mock_backend_cls, mock_shutil, tmp_path +): + sqlite_path = tmp_path / "chroma.sqlite3" + sqlite_path.write_text("fake") + + mock_col = MagicMock() + mock_col.count.return_value = 2 + mock_col.get.return_value = { + "ids": ["id1", "id2"], + "documents": ["doc1", "doc2"], + "metadatas": [{"wing": "a"}, {"wing": "b"}], + } + mock_temp_col = MagicMock() + mock_temp_col.count.return_value = 1 + mock_backend = _install_mock_backend(mock_backend_cls, mock_col) + mock_backend.create_collection.return_value = mock_temp_col + + with pytest.raises(repair.RebuildCollectionError) as excinfo: + repair.rebuild_index(palace_path=str(tmp_path)) + + assert excinfo.value.live_replaced is False + assert mock_shutil.copy2.call_count == 1 + assert mock_backend.delete_collection.call_args_list == [ + call(str(tmp_path), "mempalace_drawers__repair_tmp"), + call(str(tmp_path), "mempalace_drawers__repair_tmp"), + ] + + +@patch("mempalace.repair.shutil") +@patch("mempalace.repair.ChromaBackend") +def test_rebuild_index_live_failure_restores_backup(mock_backend_cls, mock_shutil, tmp_path): + sqlite_path = tmp_path / "chroma.sqlite3" + sqlite_path.write_text("fake") + + def _fake_copy2(src, dst): + with open(dst, "w") as handle: + handle.write("backup") + + mock_shutil.copy2.side_effect = _fake_copy2 + + mock_col = MagicMock() + mock_col.count.return_value = 2 + mock_col.get.return_value = { + "ids": ["id1", "id2"], + "documents": ["doc1", "doc2"], + "metadatas": [{"wing": "a"}, {"wing": "b"}], + } + mock_temp_col = MagicMock() + mock_temp_col.count.return_value = 2 + mock_new_col = MagicMock() + mock_new_col.upsert.side_effect = RuntimeError("live upsert failed") + mock_backend = _install_mock_backend(mock_backend_cls, mock_col) + mock_backend.create_collection.side_effect = [mock_temp_col, mock_new_col] + + with pytest.raises(repair.RebuildCollectionError) as excinfo: + repair.rebuild_index(palace_path=str(tmp_path)) + + assert excinfo.value.live_replaced is True + assert mock_shutil.copy2.call_count == 2 + assert mock_backend.delete_collection.call_args_list == [ + call(str(tmp_path), "mempalace_drawers__repair_tmp"), + call(str(tmp_path), "mempalace_drawers"), + call(str(tmp_path), "mempalace_drawers__repair_tmp"), + call(str(tmp_path), "mempalace_drawers"), + ] + + +@patch("mempalace.repair.shutil") +@patch("mempalace.repair.ChromaBackend") +def test_rebuild_index_live_delete_missing_still_restores_backup( + mock_backend_cls, mock_shutil, tmp_path +): + sqlite_path = tmp_path / "chroma.sqlite3" + sqlite_path.write_text("fake") + + def _fake_copy2(src, dst): + with open(dst, "w") as handle: + handle.write("backup") + + mock_shutil.copy2.side_effect = _fake_copy2 + + mock_col = MagicMock() + mock_col.count.return_value = 2 + mock_col.get.return_value = { + "ids": ["id1", "id2"], + "documents": ["doc1", "doc2"], + "metadatas": [{"wing": "a"}, {"wing": "b"}], + } + mock_temp_col = MagicMock() + mock_temp_col.count.return_value = 2 + mock_backend = _install_mock_backend(mock_backend_cls, mock_col) + mock_backend.create_collection.side_effect = [mock_temp_col, RuntimeError("create failed")] + mock_backend.delete_collection.side_effect = [ + None, + None, + None, + repair.ChromaNotFoundError("missing"), + ] + + with pytest.raises(repair.RebuildCollectionError) as excinfo: + repair.rebuild_index(palace_path=str(tmp_path)) + + assert excinfo.value.live_replaced is True + assert mock_shutil.copy2.call_count == 2 + assert mock_backend.delete_collection.call_args_list == [ + call(str(tmp_path), "mempalace_drawers__repair_tmp"), + call(str(tmp_path), "mempalace_drawers"), + call(str(tmp_path), "mempalace_drawers__repair_tmp"), + call(str(tmp_path), "mempalace_drawers"), + ] + + +@patch("mempalace.repair.shutil") +@patch("mempalace.repair.ChromaBackend") +def test_rebuild_index_restore_failure_preserves_original_error( + mock_backend_cls, mock_shutil, tmp_path, capsys +): + sqlite_path = tmp_path / "chroma.sqlite3" + sqlite_path.write_text("fake") + + def _copy2_side_effect(src, dst): + if str(src).endswith(".backup"): + raise PermissionError("locked sqlite") + with open(dst, "w") as handle: + handle.write("backup") + + mock_shutil.copy2.side_effect = _copy2_side_effect + + mock_col = MagicMock() + mock_col.count.return_value = 2 + mock_col.get.return_value = { + "ids": ["id1", "id2"], + "documents": ["doc1", "doc2"], + "metadatas": [{"wing": "a"}, {"wing": "b"}], + } + mock_temp_col = MagicMock() + mock_temp_col.count.return_value = 2 + mock_new_col = MagicMock() + mock_new_col.upsert.side_effect = RuntimeError("live upsert failed") + mock_backend = _install_mock_backend(mock_backend_cls, mock_col) + mock_backend.create_collection.side_effect = [mock_temp_col, mock_new_col] + + with pytest.raises(repair.RebuildCollectionError) as excinfo: + repair.rebuild_index(palace_path=str(tmp_path)) + + out = capsys.readouterr().out + assert "locked sqlite" in out + assert "Manual restore required" in out + assert "live upsert failed" in str(excinfo.value) + + +@patch("mempalace.repair.ChromaBackend") +def test_rebuild_collection_via_temp_keeps_original_error_when_cleanup_fails( + mock_backend_cls, +): + mock_col = MagicMock() + mock_col.count.return_value = 2 + mock_temp_col = MagicMock() + mock_temp_col.count.return_value = 2 + mock_backend = _install_mock_backend(mock_backend_cls, mock_col) + mock_backend.create_collection.side_effect = [mock_temp_col, RuntimeError("live build failed")] + mock_backend.delete_collection.side_effect = [ + None, + None, + RuntimeError("cleanup failed"), + ] + + with pytest.raises(repair.RebuildCollectionError) as excinfo: + repair._rebuild_collection_via_temp( + mock_backend, + "/palace", + ["id1", "id2"], + ["doc1", "doc2"], + [{"wing": "a"}, {"wing": "b"}], + batch_size=5000, + progress=lambda *args, **kwargs: None, + ) + + assert "live build failed" in str(excinfo.value) + assert excinfo.value.live_replaced is True + assert mock_backend.delete_collection.call_args_list == [ + call("/palace", "mempalace_drawers__repair_tmp"), + call("/palace", "mempalace_drawers"), + call("/palace", "mempalace_drawers__repair_tmp"), + ] + + +@patch("mempalace.repair.shutil") +@patch("mempalace.repair.ChromaBackend") +def test_rebuild_index_ignores_temp_cleanup_failure_after_success( + mock_backend_cls, mock_shutil, tmp_path +): + sqlite_path = tmp_path / "chroma.sqlite3" + sqlite_path.write_text("fake") + + def _fake_copy2(src, dst): + with open(dst, "w") as handle: + handle.write("backup") + + mock_shutil.copy2.side_effect = _fake_copy2 + + mock_col = MagicMock() + mock_col.count.return_value = 2 + mock_col.get.return_value = { + "ids": ["id1", "id2"], + "documents": ["doc1", "doc2"], + "metadatas": [{"wing": "a"}, {"wing": "b"}], + } + mock_temp_col = MagicMock() + mock_temp_col.count.return_value = 2 + mock_new_col = MagicMock() + mock_new_col.count.return_value = 2 + mock_backend = _install_mock_backend(mock_backend_cls, mock_col) + mock_backend.create_collection.side_effect = [mock_temp_col, mock_new_col] + mock_backend.delete_collection.side_effect = [ + None, + None, + RuntimeError("cleanup failed"), + ] + + repair.rebuild_index(palace_path=str(tmp_path)) + + assert mock_shutil.copy2.call_count == 1 + assert mock_backend.delete_collection.call_args_list == [ + call(str(tmp_path), "mempalace_drawers__repair_tmp"), + call(str(tmp_path), "mempalace_drawers"), + call(str(tmp_path), "mempalace_drawers__repair_tmp"), + ] + + # ── repair_max_seq_id ───────────────────────────────────────────────── From 2f509b4789d7ae44f8fbf1281df812e098207aa3 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Thu, 30 Apr 2026 09:31:32 -0600 Subject: [PATCH 11/77] fix(cli): restore backup on repair failure --- mempalace/cli.py | 62 +++++++++++++++++++++++++++-------------------- tests/test_cli.py | 50 +++++++++++++++++++++++++++++++++++++- 2 files changed, 85 insertions(+), 27 deletions(-) diff --git a/mempalace/cli.py b/mempalace/cli.py index ca9798b444..27c81d3db3 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -648,7 +648,14 @@ def cmd_repair(args): import shutil from .backends.chroma import ChromaBackend from .migrate import confirm_destructive_action, contains_palace_database - from .repair import TruncationDetected, check_extraction_safety + from .repair import ( + RebuildCollectionError, + TruncationDetected, + _close_chroma_handles, + _extract_drawers, + _rebuild_collection_via_temp, + check_extraction_safety, + ) palace_path = os.path.abspath( os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path @@ -705,18 +712,7 @@ def cmd_repair(args): # Extract all drawers in batches print("\n Extracting drawers...") batch_size = 5000 - all_ids = [] - all_docs = [] - all_metas = [] - offset = 0 - while offset < total: - batch = col.get(limit=batch_size, offset=offset, include=["documents", "metadatas"]) - if not batch["ids"]: - break - all_ids.extend(batch["ids"]) - all_docs.extend(batch["documents"]) - all_metas.extend(batch["metadatas"]) - offset += len(batch["ids"]) + all_ids, all_docs, all_metas = _extract_drawers(col, total, batch_size) print(f" Extracted {len(all_ids)} drawers") # ── #1208 guard ────────────────────────────────────────────────── @@ -736,7 +732,6 @@ def cmd_repair(args): print(e.message) return - # Backup and rebuild palace_path = os.path.normpath(palace_path) backup_path = palace_path + ".backup" if os.path.exists(backup_path): @@ -750,18 +745,33 @@ def cmd_repair(args): print(f" Backing up to {backup_path}...") shutil.copytree(palace_path, backup_path) - print(" Rebuilding collection...") - backend.delete_collection(palace_path, "mempalace_drawers") - new_col = backend.create_collection(palace_path, "mempalace_drawers") - - filed = 0 - for i in range(0, len(all_ids), batch_size): - batch_ids = all_ids[i : i + batch_size] - batch_docs = all_docs[i : i + batch_size] - batch_metas = all_metas[i : i + batch_size] - new_col.add(documents=batch_docs, ids=batch_ids, metadatas=batch_metas) - filed += len(batch_ids) - print(f" Re-filed {filed}/{len(all_ids)} drawers...") + try: + filed = _rebuild_collection_via_temp( + backend, + palace_path, + all_ids, + all_docs, + all_metas, + batch_size, + progress=print, + ) + except RebuildCollectionError as e: + print(f" Repair failed: {e}") + if getattr(e, "live_replaced", False): + print(" Live collection was already replaced; restoring from backup...") + try: + _close_chroma_handles(palace_path) + if os.path.exists(palace_path): + shutil.rmtree(palace_path) + shutil.copytree(backup_path, palace_path) + print(f" Restore complete from backup: {backup_path}") + except Exception as restore_error: + print(f" Automatic restore failed: {restore_error}") + print(" Manual recovery required:") + print(f" 1. Remove or rename the broken directory: {palace_path}") + print(f" 2. Restore the backup directory to: {palace_path}") + print(f" Backup location: {backup_path}") + sys.exit(1) print(f"\n Repair complete. {filed} drawers rebuilt.") print(f" Backup saved at {backup_path}") diff --git a/tests/test_cli.py b/tests/test_cli.py index af7b39d0a6..11845fea3c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,7 +4,7 @@ import shlex import sys from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch import pytest @@ -760,13 +760,61 @@ def test_cmd_repair_success(mock_config_cls, tmp_path, capsys): "documents": ["doc1", "doc2"], "metadatas": [{"wing": "a"}, {"wing": "b"}], } + mock_temp_col = MagicMock() + mock_temp_col.count.return_value = 2 mock_new_col = MagicMock() + mock_new_col.count.return_value = 2 mock_backend = _mock_backend_for(col=mock_col, new_col=mock_new_col) + mock_backend.create_collection.side_effect = [mock_temp_col, mock_new_col] with patch("mempalace.backends.chroma.ChromaBackend", return_value=mock_backend): cmd_repair(args) out = capsys.readouterr().out assert "Repair complete" in out assert "2 drawers rebuilt" in out + assert mock_backend.delete_collection.call_args_list == [ + call(str(palace_dir), "mempalace_drawers__repair_tmp"), + call(str(palace_dir), "mempalace_drawers"), + call(str(palace_dir), "mempalace_drawers__repair_tmp"), + ] + mock_temp_col.upsert.assert_called_once() + mock_new_col.upsert.assert_called_once() + mock_new_col.add.assert_not_called() + + +@patch("mempalace.cli.MempalaceConfig") +def test_cmd_repair_restores_backup_on_live_rebuild_failure(mock_config_cls, tmp_path, capsys): + palace_dir = tmp_path / "palace" + palace_dir.mkdir() + (palace_dir / "chroma.sqlite3").write_text("db") + mock_config_cls.return_value.palace_path = str(palace_dir) + args = argparse.Namespace(palace=None, yes=True) + mock_col = MagicMock() + mock_col.count.return_value = 2 + mock_col.get.return_value = { + "ids": ["id1", "id2"], + "documents": ["doc1", "doc2"], + "metadatas": [{"wing": "a"}, {"wing": "b"}], + } + mock_temp_col = MagicMock() + mock_temp_col.count.return_value = 2 + mock_backend = _mock_backend_for(col=mock_col) + mock_backend.create_collection.side_effect = [mock_temp_col, RuntimeError("live build failed")] + with ( + patch("mempalace.backends.chroma.ChromaBackend", return_value=mock_backend), + patch("mempalace.repair._close_chroma_handles") as mock_close_handles, + ): + with pytest.raises(SystemExit) as excinfo: + cmd_repair(args) + out = capsys.readouterr().out + assert excinfo.value.code == 1 + assert "Repair failed" in out + assert "restoring from backup" in out + mock_close_handles.assert_called_once_with(str(palace_dir)) + assert mock_backend.delete_collection.call_args_list == [ + call(str(palace_dir), "mempalace_drawers__repair_tmp"), + call(str(palace_dir), "mempalace_drawers"), + call(str(palace_dir), "mempalace_drawers__repair_tmp"), + ] @patch("mempalace.cli.MempalaceConfig") From f57f30025f2551efc3d917a368f5ed0dae336c46 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Thu, 30 Apr 2026 09:57:39 -0600 Subject: [PATCH 12/77] fix(repair): close active backend before rollback restore Rollback cleanup was instantiating a fresh ChromaBackend, so the live backend that had opened the PersistentClient could keep file handles alive during restore. Close the active backend instance instead so rollback and CLI recovery can release Windows-safe locks before copying the backup back into place. --- mempalace/cli.py | 2 +- mempalace/repair.py | 14 ++++++++++---- tests/test_cli.py | 7 ++----- tests/test_repair.py | 11 ++++++++--- 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/mempalace/cli.py b/mempalace/cli.py index 27c81d3db3..9a1e8e4590 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -760,7 +760,7 @@ def cmd_repair(args): if getattr(e, "live_replaced", False): print(" Live collection was already replaced; restoring from backup...") try: - _close_chroma_handles(palace_path) + _close_chroma_handles(palace_path, backend=backend) if os.path.exists(palace_path): shutil.rmtree(palace_path) shutil.copytree(backup_path, palace_path) diff --git a/mempalace/repair.py b/mempalace/repair.py index 49d6abef71..0585405e11 100644 --- a/mempalace/repair.py +++ b/mempalace/repair.py @@ -517,7 +517,7 @@ def rebuild_index(palace_path=None, confirm_truncation_ok: bool = False): if e.live_replaced and os.path.exists(backup_path): print(f" Restoring from backup: {backup_path}") try: - _close_chroma_handles(palace_path) + _close_chroma_handles(palace_path, backend=backend) _delete_collection_if_exists(backend, palace_path, COLLECTION_NAME) shutil.copy2(backup_path, sqlite_path) print(" Backup restored. Palace is back to pre-repair state.") @@ -593,12 +593,18 @@ def status(palace_path=None) -> dict: # --------------------------------------------------------------------------- -def _close_chroma_handles(palace_path: str) -> None: - """Drop ChromaBackend + chromadb singleton caches so OS mmap handles release.""" +def _close_chroma_handles(palace_path: str, backend: ChromaBackend | None = None) -> None: + """Drop ChromaBackend + chromadb singleton caches so OS mmap handles release. + + When ``backend`` is provided, close the live instance so rollback/restore + releases the handles it was already using. Otherwise fall back to a + transient backend instance for the max-seq-id repair path. + """ import gc try: - ChromaBackend().close_palace(palace_path) + closer = backend if backend is not None else ChromaBackend() + closer.close_palace(palace_path) except Exception: pass try: diff --git a/tests/test_cli.py b/tests/test_cli.py index 11845fea3c..6572f1d497 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -799,17 +799,14 @@ def test_cmd_repair_restores_backup_on_live_rebuild_failure(mock_config_cls, tmp mock_temp_col.count.return_value = 2 mock_backend = _mock_backend_for(col=mock_col) mock_backend.create_collection.side_effect = [mock_temp_col, RuntimeError("live build failed")] - with ( - patch("mempalace.backends.chroma.ChromaBackend", return_value=mock_backend), - patch("mempalace.repair._close_chroma_handles") as mock_close_handles, - ): + with patch("mempalace.backends.chroma.ChromaBackend", return_value=mock_backend): with pytest.raises(SystemExit) as excinfo: cmd_repair(args) out = capsys.readouterr().out assert excinfo.value.code == 1 assert "Repair failed" in out assert "restoring from backup" in out - mock_close_handles.assert_called_once_with(str(palace_dir)) + mock_backend.close_palace.assert_called_once_with(str(palace_dir)) assert mock_backend.delete_collection.call_args_list == [ call(str(palace_dir), "mempalace_drawers__repair_tmp"), call(str(palace_dir), "mempalace_drawers"), diff --git a/tests/test_repair.py b/tests/test_repair.py index 33daad9f7f..9cd12dd6c8 100644 --- a/tests/test_repair.py +++ b/tests/test_repair.py @@ -499,20 +499,25 @@ def _fake_copy2(src, dst): mock_temp_col.count.return_value = 2 mock_new_col = MagicMock() mock_new_col.upsert.side_effect = RuntimeError("live upsert failed") - mock_backend = _install_mock_backend(mock_backend_cls, mock_col) - mock_backend.create_collection.side_effect = [mock_temp_col, mock_new_col] + active_backend = MagicMock() + active_backend.get_collection.return_value = mock_col + active_backend.create_collection.side_effect = [mock_temp_col, mock_new_col] + helper_backend = MagicMock() + mock_backend_cls.side_effect = [active_backend, helper_backend] with pytest.raises(repair.RebuildCollectionError) as excinfo: repair.rebuild_index(palace_path=str(tmp_path)) assert excinfo.value.live_replaced is True assert mock_shutil.copy2.call_count == 2 - assert mock_backend.delete_collection.call_args_list == [ + assert active_backend.delete_collection.call_args_list == [ call(str(tmp_path), "mempalace_drawers__repair_tmp"), call(str(tmp_path), "mempalace_drawers"), call(str(tmp_path), "mempalace_drawers__repair_tmp"), call(str(tmp_path), "mempalace_drawers"), ] + active_backend.close_palace.assert_called_once_with(str(tmp_path)) + helper_backend.close_palace.assert_not_called() @patch("mempalace.repair.shutil") From 0e32b9643c3d4ceca36b6f610ddbcbbf58ad8dd7 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Thu, 30 Apr 2026 21:47:08 -0600 Subject: [PATCH 13/77] fix: avoid false hnsw divergence fallback --- mempalace/backends/chroma.py | 27 +++--- mempalace/searcher.py | 43 ++++++++-- tests/test_hnsw_capacity.py | 160 ++++++++++++++++++++++++++++++++++- 3 files changed, 204 insertions(+), 26 deletions(-) diff --git a/mempalace/backends/chroma.py b/mempalace/backends/chroma.py index 646969ba12..7ce8fd875a 100644 --- a/mempalace/backends/chroma.py +++ b/mempalace/backends/chroma.py @@ -491,22 +491,17 @@ def hnsw_capacity_status(palace_path: str, collection_name: str = "mempalace_dra divergence_floor = max(_HNSW_DIVERGENCE_FALLBACK_FLOOR, 2 * sync_threshold) if hnsw_count is None: - # No pickle yet — segment hasn't persisted metadata. Could be - # fresh-but-unflushed (normal) or interrupted-mid-flush (bad). - # We can't distinguish without the pickle, so only flag - # divergence when sqlite holds clearly more than two flush - # windows worth — same threshold as the with-pickle path. - if sqlite_count > divergence_floor: - out["status"] = "diverged" - out["diverged"] = True - out["divergence"] = sqlite_count - out["message"] = ( - f"sqlite holds {sqlite_count:,} embeddings but the HNSW segment " - "has never flushed metadata — vector search will return nothing " - "until the segment is rebuilt. Run `mempalace repair`." - ) - else: - out["message"] = "HNSW segment metadata not yet flushed; skipping" + # No pickle yet, so this probe cannot measure HNSW capacity. + # Chroma 1.5.x can have binary HNSW files without a flushed + # metadata pickle; absence of the pickle alone is not proof that + # vector search is unusable or dangerous. Keep the status unknown + # so MCP does not globally disable vectors on an inconclusive + # signal. Corrupt/invalid metadata, when present, is handled by + # quarantine_invalid_hnsw_metadata before Chroma opens. + out["message"] = ( + "HNSW capacity unavailable: metadata has not been flushed; " + "leaving vector search enabled" + ) return out divergence = sqlite_count - hnsw_count diff --git a/mempalace/searcher.py b/mempalace/searcher.py index a14d90dd9d..4ff0f23f24 100644 --- a/mempalace/searcher.py +++ b/mempalace/searcher.py @@ -396,6 +396,31 @@ def _bm25_only_via_sqlite( "hint": "Run: mempalace init && mempalace mine ", } + def _metadata_filter_sql(row_id_expr: str) -> tuple[str, list[str]]: + clauses = [] + params = [] + for key, value in (("wing", wing), ("room", room)): + if not value: + continue + clauses.append( + f""" + AND EXISTS ( + SELECT 1 + FROM embedding_metadata mf + WHERE mf.id = {row_id_expr} + AND mf.key = ? + AND COALESCE( + mf.string_value, + CAST(mf.int_value AS TEXT), + CAST(mf.float_value AS TEXT), + CAST(mf.bool_value AS TEXT) + ) = ? + ) + """ + ) + params.extend([key, value]) + return "".join(clauses), params + try: conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) except sqlite3.Error as e: @@ -408,15 +433,17 @@ def _bm25_only_via_sqlite( candidate_ids: list[int] = [] if tokens: fts_query = " OR ".join(tokens) + filter_sql, filter_params = _metadata_filter_sql("embedding_fulltext_search.rowid") try: rows = conn.execute( - """ + f""" SELECT rowid FROM embedding_fulltext_search WHERE embedding_fulltext_search MATCH ? + {filter_sql} LIMIT ? """, - (fts_query, max_candidates), + (fts_query, *filter_params, max_candidates), ).fetchall() candidate_ids = [r[0] for r in rows] except sqlite3.Error: @@ -434,17 +461,19 @@ def _bm25_only_via_sqlite( # fall back to ordering by primary-key id and finally to an # empty result rather than letting search raise. try: + filter_sql, filter_params = _metadata_filter_sql("e.id") rows = conn.execute( - """ + f""" SELECT e.id FROM embeddings e JOIN segments s ON e.segment_id = s.id JOIN collections c ON s.collection = c.id WHERE c.name = 'mempalace_drawers' + {filter_sql} ORDER BY e.created_at DESC LIMIT ? """, - (max_candidates,), + (*filter_params, max_candidates), ).fetchall() candidate_ids = [r[0] for r in rows] except sqlite3.Error: @@ -453,17 +482,19 @@ def _bm25_only_via_sqlite( exc_info=True, ) try: + filter_sql, filter_params = _metadata_filter_sql("e.id") rows = conn.execute( - """ + f""" SELECT e.id FROM embeddings e JOIN segments s ON e.segment_id = s.id JOIN collections c ON s.collection = c.id WHERE c.name = 'mempalace_drawers' + {filter_sql} ORDER BY e.id DESC LIMIT ? """, - (max_candidates,), + (*filter_params, max_candidates), ).fetchall() candidate_ids = [r[0] for r in rows] except sqlite3.Error: diff --git a/tests/test_hnsw_capacity.py b/tests/test_hnsw_capacity.py index 512fc9c677..912def8bf7 100644 --- a/tests/test_hnsw_capacity.py +++ b/tests/test_hnsw_capacity.py @@ -238,14 +238,39 @@ def test_capacity_status_tolerates_flush_lag(tmp_path): assert info["status"] == "ok" -def test_capacity_status_flags_unflushed_with_large_sqlite(tmp_path): - """No pickle + many sqlite rows is its own divergence signal.""" +def test_capacity_status_does_not_flag_unflushed_with_large_sqlite(tmp_path): + """No pickle + many sqlite rows is inconclusive, not divergence.""" seg = "seg-noflush" _seed_chroma_db(str(tmp_path), sqlite_count=10_000, segment_id=seg) info = hnsw_capacity_status(str(tmp_path), COLLECTION) - assert info["diverged"] is True + assert info["diverged"] is False + assert info["status"] == "unknown" + assert info["divergence"] is None assert info["hnsw_count"] is None - assert "never flushed" in info["message"] + assert "capacity unavailable" in info["message"] + assert "leaving vector search enabled" in info["message"] + + +def test_mcp_probe_does_not_disable_vectors_for_unflushed_metadata(tmp_path, monkeypatch): + """The MCP preflight must not route all searches to BM25 on this signal.""" + from mempalace import mcp_server + + seg = "seg-mcp-noflush" + _seed_chroma_db(str(tmp_path), sqlite_count=10_000, segment_id=seg) + + class _Cfg: + palace_path = str(tmp_path) + + monkeypatch.setattr(mcp_server, "_config", _Cfg()) + monkeypatch.setattr(mcp_server, "_vector_disabled", True) + monkeypatch.setattr(mcp_server, "_vector_disabled_reason", "old divergence") + + mcp_server._refresh_vector_disabled_flag() + + assert mcp_server._vector_disabled is False + assert mcp_server._vector_disabled_reason == "" + assert mcp_server._vector_capacity_status["status"] == "unknown" + assert "leaving vector search enabled" in mcp_server._vector_capacity_status["message"] def test_capacity_status_quiet_for_empty_palace(tmp_path): @@ -372,6 +397,17 @@ def _seed_drawers(palace: str, segment_id: str, drawers: list[tuple[str, dict, s conn.close() +def _set_drawer_created_at(palace: str, timestamps: dict[int, str]) -> None: + db_path = os.path.join(palace, "chroma.sqlite3") + conn = sqlite3.connect(db_path) + try: + for emb_id, created_at in timestamps.items(): + conn.execute("UPDATE embeddings SET created_at = ? WHERE id = ?", (created_at, emb_id)) + conn.commit() + finally: + conn.close() + + @pytest.fixture def palace_with_drawers(tmp_path): seg = "seg-bm25" @@ -417,6 +453,122 @@ def test_bm25_fallback_filters_by_wing(palace_with_drawers): assert all(r["wing"] == "design" for r in out["results"]) +def test_bm25_fallback_applies_wing_before_fts_candidate_limit(tmp_path): + seg = "seg-bm25-fts-limit" + _seed_chroma_db(str(tmp_path), sqlite_count=0, segment_id=seg) + _seed_drawers( + str(tmp_path), + seg, + [ + ( + "shared token outside target wing", + {"wing": "ops", "room": "incidents", "source_file": "/x/ops.md"}, + "d-1", + ), + ( + "shared token inside target wing", + {"wing": "project", "room": "diary", "source_file": "/x/project.md"}, + "d-2", + ), + ], + ) + + out = _bm25_only_via_sqlite("shared token", str(tmp_path), wing="project", max_candidates=1) + + assert out["total_before_filter"] == 1 + assert len(out["results"]) == 1 + assert out["results"][0]["wing"] == "project" + + +def test_bm25_fallback_applies_room_before_fts_candidate_limit(tmp_path): + seg = "seg-bm25-room-limit" + _seed_chroma_db(str(tmp_path), sqlite_count=0, segment_id=seg) + _seed_drawers( + str(tmp_path), + seg, + [ + ( + "shared token wrong room", + {"wing": "project", "room": "scratch", "source_file": "/x/scratch.md"}, + "d-1", + ), + ( + "shared token right room", + {"wing": "project", "room": "diary", "source_file": "/x/diary.md"}, + "d-2", + ), + ], + ) + + out = _bm25_only_via_sqlite( + "shared token", + str(tmp_path), + wing="project", + room="diary", + max_candidates=1, + ) + + assert out["total_before_filter"] == 1 + assert len(out["results"]) == 1 + assert out["results"][0]["wing"] == "project" + assert out["results"][0]["room"] == "diary" + + +def test_bm25_fallback_applies_wing_before_recency_candidate_limit(tmp_path): + seg = "seg-bm25-recency-limit" + _seed_chroma_db(str(tmp_path), sqlite_count=0, segment_id=seg) + _seed_drawers( + str(tmp_path), + seg, + [ + ( + "target drawer for short query", + {"wing": "project", "room": "diary", "source_file": "/x/project.md"}, + "d-1", + ), + ( + "newer drawer outside target wing", + {"wing": "ops", "room": "incidents", "source_file": "/x/ops.md"}, + "d-2", + ), + ], + ) + _set_drawer_created_at( + str(tmp_path), + { + 1: "2026-01-01 00:00:00", + 2: "2026-02-01 00:00:00", + }, + ) + + out = _bm25_only_via_sqlite("a", str(tmp_path), wing="project", max_candidates=1) + + assert out["total_before_filter"] == 1 + assert len(out["results"]) == 1 + assert out["results"][0]["wing"] == "project" + + +def test_bm25_fallback_returns_empty_when_filtered_wing_has_no_candidates(tmp_path): + seg = "seg-bm25-empty-filter" + _seed_chroma_db(str(tmp_path), sqlite_count=0, segment_id=seg) + _seed_drawers( + str(tmp_path), + seg, + [ + ( + "shared token outside target wing", + {"wing": "ops", "room": "incidents", "source_file": "/x/ops.md"}, + "d-1", + ), + ], + ) + + out = _bm25_only_via_sqlite("shared token", str(tmp_path), wing="project", max_candidates=1) + + assert out["total_before_filter"] == 0 + assert out["results"] == [] + + def test_bm25_fallback_no_palace(tmp_path): out = _bm25_only_via_sqlite("anything", str(tmp_path)) assert "error" in out From 8472d553a34b83cb3ba6d50142640a8d2744d80c Mon Sep 17 00:00:00 2001 From: lcatlett Date: Fri, 1 May 2026 19:26:43 -0400 Subject: [PATCH 14/77] fix(hooks): treat absent ~/.mempalace as auto-save off When the user removes ~/.mempalace/ (a strong "do not auto-capture" signal), the next hook fire would silently recreate the entire dir hierarchy and ingest existing transcripts: 1. _log() at hooks_cli.py:148 unconditionally calls STATE_DIR.mkdir(parents=True, exist_ok=True), so the act of writing the hook log line recreated ~/.mempalace/hook_state/ 2. With no config file present, hook_stop_auto_save and hook_precompact_auto_save defaulted to True (no override to read) 3. The full save path then ran, materializing palace/, wal/, knowledge_graph.sqlite3, and N drawers from existing transcripts in ~/.claude/projects/*.jsonl All four entry points (hook_stop, hook_precompact, hook_session_start, and _log itself) now check a new PALACE_ROOT = Path.home() / ".mempalace" constant first and short-circuit (returning {} on stdout, never logging) when the dir is absent. The user-removable directory is now a kill-switch. Five unit tests in tests/test_hooks_cli.py cover: hook_stop / hook_precompact / hook_session_start do not create the dir when absent; _log() does not create it when absent; existing dir proceeds normally (regression). Caught in the wild on a downstream fork: ~146 drawers materialized in under a second after a deliberate `rm -rf ~/.mempalace/`, into a planning session that was explicitly not meant to be captured. --- CHANGELOG.md | 1 + mempalace/hooks_cli.py | 23 +++++++++++++ tests/test_hooks_cli.py | 76 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41dfaacc2c..7a75842aec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Bug Fixes - **MCP server `tool_diary_write` SIGSEGV when default EF provider differs.** `mcp_server._get_collection` bypassed `ChromaBackend.get_collection` and called `client.get_collection` / `client.create_collection` without `embedding_function=`. ChromaDB 1.x persists the EF *identity* (its `name()`) with the collection but not the EF *instance/configuration*, so the MCP server's reopen silently bound chromadb's built-in `DefaultEmbeddingFunction` — its `name()` matches `mempalace.embedding`'s spoofed `"default"` so the identity check passes, but its provider list is chromadb's default rather than the user's resolved device. The miner / Stop hook ingest path routes through the backend helper and binds the configured EF instead. On bleeding-edge interpreters (python 3.14 + chromadb 1.5.x on Apple Silicon) the default provider selection could SIGSEGV the host process on first `col.add()`, killing the MCP stdio server and leaving every subsequent tool call returning `Connection closed` until Claude Code was relaunched. `_get_collection` now reuses `ChromaBackend._resolve_embedding_function()` on the reopen branches that actually open a collection (warm-cache reads stay zero-cost), matching the miner/backend path. (#1299, follow-up to #1262 / #1289) +- **Hooks no longer recreate `~/.mempalace/` after the user removes it.** When `~/.mempalace/` is deleted (a strong "do not auto-capture" signal), the next `Stop`, `PreCompact`, or `SessionStart` hook would silently rebuild the dir hierarchy and ingest existing transcripts: `_log()` called `STATE_DIR.mkdir(parents=True, exist_ok=True)` unconditionally, so the very act of writing `[HH:MM] SESSION START …` recreated `~/.mempalace/hook_state/`; subsequent calls in the save path then materialized `palace/`, `wal/`, `knowledge_graph.sqlite3`, and N drawers from `~/.claude/projects/*.jsonl`. All four entry points (`hook_stop`, `hook_precompact`, `hook_session_start`, and `_log` itself) now check a new module-level `PALACE_ROOT = Path.home() / ".mempalace"` constant first and short-circuit (returning `{}` on stdout, never logging) when the directory is absent. The user-removable directory becomes a kill-switch — `rm -rf ~/.mempalace` is now a stable state. Net: 23 lines added in `mempalace/hooks_cli.py`, 5 unit tests in `tests/test_hooks_cli.py`. (#1305) - **Cross-wing topic tunnels for hyphenated dir names.** `mempalace init` recorded the `topics_by_wing` registry key under the raw directory name (e.g. `mempalace-public`), while `mempalace.yaml`'s `wing` field used the lower-cased + separator-collapsed slug (`mempalace_public`). At mine time the miner read the slug from the yaml and missed the registry, so `_compute_topic_tunnels_for_wing` returned `0` silently. Real-world: any project whose folder contained a hyphen or space lost every topic tunnel. Now both call sites route through a shared `normalize_wing_name()` in `config.py`. (#1194, follow-up to #1180) - **CLI `mempalace search` retrieval quality.** The CLI was using pure ChromaDB cosine distance with no BM25 rerank, so drawers containing every query term but embedding as noise (directory listings, diff output, shell logs) scored `Match: 0.0` alongside genuinely irrelevant results with no way to tell them apart. Wired the CLI through the same `_hybrid_rank` the `mempalace_search` MCP tool already used, and surfaced both `cosine=` and `bm25=` scores in the output so users see which component of the match is firing. MCP search was unaffected; this fixes the human-facing CLI parity gap. - **Legacy-palace distance-metric warning.** CLI search now detects palaces created before `hnsw:space=cosine` was consistently set and prints a one-line notice pointing at `mempalace repair`. Without the warning such palaces silently used L2 distance, under which the similarity display floored every result to `Match: 0.0`. New palaces mined today already set cosine correctly and now have invariant tests pinning that behavior so future refactors can't silently regress it. (#1179) diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index d4f8317232..ca8fb607cd 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -16,6 +16,18 @@ SAVE_INTERVAL = 15 STATE_DIR = Path.home() / ".mempalace" / "hook_state" +PALACE_ROOT = Path.home() / ".mempalace" + + +def _palace_root_exists() -> bool: + """User-removable kill-switch. + + If ~/.mempalace/ does not exist, the user has explicitly cleared it. + All hook side effects (logging, state dir creation, mining, ingestion) + must respect this and short-circuit BEFORE touching disk — including + before logging the short-circuit itself. + """ + return PALACE_ROOT.exists() def _mempalace_python() -> str: @@ -142,6 +154,8 @@ def _count_human_messages(transcript_path: str) -> int: def _log(message: str): """Append to hook state log file.""" + if not PALACE_ROOT.exists(): + return # User removed the palace; do not recreate by logging global _state_dir_initialized try: if not _state_dir_initialized: @@ -550,6 +564,9 @@ def _wing_from_transcript_path(transcript_path: str) -> str: def hook_stop(data: dict, harness: str): """Stop hook: block every N messages for auto-save.""" + if not _palace_root_exists(): + _output({}) + return parsed = _parse_harness_input(data, harness) session_id = parsed["session_id"] stop_hook_active = parsed["stop_hook_active"] @@ -659,6 +676,9 @@ def hook_stop(data: dict, harness: str): def hook_session_start(data: dict, harness: str): """Session start hook: initialize session tracking state.""" + if not _palace_root_exists(): + _output({}) + return parsed = _parse_harness_input(data, harness) session_id = parsed["session_id"] @@ -673,6 +693,9 @@ def hook_session_start(data: dict, harness: str): def hook_precompact(data: dict, harness: str): """Precompact hook: mine transcript synchronously, then allow compaction.""" + if not _palace_root_exists(): + _output({}) + return parsed = _parse_harness_input(data, harness) session_id = parsed["session_id"] transcript_path = parsed["transcript_path"] diff --git a/tests/test_hooks_cli.py b/tests/test_hooks_cli.py index 1ceb530ffb..941288deaa 100644 --- a/tests/test_hooks_cli.py +++ b/tests/test_hooks_cli.py @@ -959,3 +959,79 @@ def test_stop_hook_rejects_injected_stop_hook_active(tmp_path): # The injected value is not "true"/"1"/"yes", so the hook should NOT pass through. # Save must have been attempted. assert mock_save.called + + +# --- Absent palace root: hooks must not recreate ~/.mempalace --- +# +# When the user removes ~/.mempalace (e.g. `rm -rf`), that is the strongest +# possible "do not auto-capture" signal. Hooks must short-circuit BEFORE +# touching disk — including before the log-line that previously triggered +# STATE_DIR.mkdir() on its own. + + +import mempalace.hooks_cli as hooks_cli_mod + + +def _redirect_palace_root(monkeypatch, tmp_path): + """Point PALACE_ROOT and STATE_DIR at a tmp location that does NOT exist.""" + fake_root = tmp_path / "absent-mempalace" + monkeypatch.setattr(hooks_cli_mod, "PALACE_ROOT", fake_root) + monkeypatch.setattr(hooks_cli_mod, "STATE_DIR", fake_root / "hook_state") + monkeypatch.setattr(hooks_cli_mod, "_state_dir_initialized", False) + return fake_root + + +def test_hook_stop_does_not_create_palace_dir_when_absent(tmp_path, monkeypatch): + fake_root = _redirect_palace_root(monkeypatch, tmp_path) + transcript = tmp_path / "t.jsonl" + transcript.write_text("") + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + hook_stop( + {"session_id": "absent", "transcript_path": str(transcript), "stop_hook_active": False}, + "claude-code", + ) + assert json.loads(buf.getvalue() or "{}") == {} + assert not fake_root.exists() + + +def test_hook_precompact_does_not_create_palace_dir_when_absent(tmp_path, monkeypatch): + fake_root = _redirect_palace_root(monkeypatch, tmp_path) + transcript = tmp_path / "t.jsonl" + transcript.write_text("") + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + hook_precompact( + {"session_id": "absent", "transcript_path": str(transcript)}, + "claude-code", + ) + assert json.loads(buf.getvalue() or "{}") == {} + assert not fake_root.exists() + + +def test_hook_session_start_does_not_create_palace_dir_when_absent(tmp_path, monkeypatch): + fake_root = _redirect_palace_root(monkeypatch, tmp_path) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + hook_session_start({"session_id": "absent"}, "claude-code") + assert json.loads(buf.getvalue() or "{}") == {} + assert not fake_root.exists() + + +def test_log_does_not_create_palace_dir_when_absent(tmp_path, monkeypatch): + fake_root = _redirect_palace_root(monkeypatch, tmp_path) + _log("test message") + assert not fake_root.exists() + + +def test_existing_dir_proceeds_normally(tmp_path, monkeypatch): + """Regression: when PALACE_ROOT exists, hooks must proceed (no short-circuit).""" + fake_root = tmp_path / "present-mempalace" + fake_root.mkdir() + monkeypatch.setattr(hooks_cli_mod, "PALACE_ROOT", fake_root) + monkeypatch.setattr(hooks_cli_mod, "STATE_DIR", fake_root / "hook_state") + monkeypatch.setattr(hooks_cli_mod, "_state_dir_initialized", False) + _log("test message") + # _log should have created the state dir under the existing palace root + assert (fake_root / "hook_state").exists() + assert (fake_root / "hook_state" / "hook.log").is_file() From 2d50b214d4c2c05112a3258faa8bef39cf6c07c0 Mon Sep 17 00:00:00 2001 From: lcatlett Date: Sat, 2 May 2026 20:37:47 -0400 Subject: [PATCH 15/77] fix(hooks): use is_dir() for palace root check (review feedback) Both @igorls and the Qodo bot flagged that `_palace_root_exists()` used `Path.exists()`, which returns True for a regular file. A stray file at `~/.mempalace` would let the kill-switch be bypassed and crash later in `STATE_DIR.mkdir()` with NotADirectoryError. Switched to `Path.is_dir()`. Also fold `_log()`'s inline check through `_palace_root_exists()` so both kill-switch sites use the same predicate. New test pins the behavior: a regular file at the palace root path is treated as absent (hook short-circuits, _log does not crash, the stray file is left untouched). --- mempalace/hooks_cli.py | 9 +++++++-- tests/test_hooks_cli.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index ca8fb607cd..8498103581 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -26,8 +26,13 @@ def _palace_root_exists() -> bool: All hook side effects (logging, state dir creation, mining, ingestion) must respect this and short-circuit BEFORE touching disk — including before logging the short-circuit itself. + + Uses ``is_dir()`` rather than ``exists()`` so a stray regular file at + ``~/.mempalace`` (or a broken symlink) is treated as absent — otherwise + the kill-switch would be bypassed and ``STATE_DIR.mkdir()`` would later + crash on ``NotADirectoryError``. """ - return PALACE_ROOT.exists() + return PALACE_ROOT.is_dir() def _mempalace_python() -> str: @@ -154,7 +159,7 @@ def _count_human_messages(transcript_path: str) -> int: def _log(message: str): """Append to hook state log file.""" - if not PALACE_ROOT.exists(): + if not _palace_root_exists(): return # User removed the palace; do not recreate by logging global _state_dir_initialized try: diff --git a/tests/test_hooks_cli.py b/tests/test_hooks_cli.py index 941288deaa..487acf74c1 100644 --- a/tests/test_hooks_cli.py +++ b/tests/test_hooks_cli.py @@ -1035,3 +1035,35 @@ def test_existing_dir_proceeds_normally(tmp_path, monkeypatch): # _log should have created the state dir under the existing palace root assert (fake_root / "hook_state").exists() assert (fake_root / "hook_state" / "hook.log").is_file() + + +def test_regular_file_at_palace_root_treated_as_absent(tmp_path, monkeypatch): + """A regular file at ~/.mempalace must be treated the same as absent. + + ``Path.exists()`` returns True for a regular file, which would let the + kill-switch be bypassed and crash later when ``STATE_DIR.mkdir()`` runs + on ``NotADirectoryError``. ``_palace_root_exists()`` must use + ``is_dir()`` so a stray file (or broken symlink) short-circuits cleanly. + """ + fake_root = tmp_path / "file-not-dir" + fake_root.write_text("oops, this is a file not a directory") + monkeypatch.setattr(hooks_cli_mod, "PALACE_ROOT", fake_root) + monkeypatch.setattr(hooks_cli_mod, "STATE_DIR", fake_root / "hook_state") + monkeypatch.setattr(hooks_cli_mod, "_state_dir_initialized", False) + + # _palace_root_exists() is the source of truth — it must return False. + assert hooks_cli_mod._palace_root_exists() is False + + # Hooks must short-circuit (return {} on stdout) and not touch disk. + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + hook_session_start({"session_id": "file-at-root"}, "claude-code") + assert json.loads(buf.getvalue() or "{}") == {} + + # _log must also short-circuit — it must NOT try to mkdir a path under a + # regular file (which would raise NotADirectoryError). + _log("test message") # would raise if not short-circuited + + # The stray file is left untouched; we never try to convert it. + assert fake_root.is_file() + assert fake_root.read_text() == "oops, this is a file not a directory" From beac5d99547121ca04ff069dbeaac7619fc741a8 Mon Sep 17 00:00:00 2001 From: mvalentsev Date: Fri, 24 Apr 2026 12:46:31 +0500 Subject: [PATCH 16/77] refactor(mcp): replace eager _kg with lazy per-path cache (#1136) Swap the module-level KnowledgeGraph singleton for a lazy, per-path cache keyed by the resolved sqlite path. Import no longer creates a sqlite file as a side effect, and MCP servers started with --palace now route KG calls to the correct tenant when MEMPALACE_PALACE_PATH changes between calls, matching the per-call behavior of _get_client() on the ChromaDB side. Default-path behavior is preserved: without --palace at startup, KG stays on DEFAULT_KG_PATH regardless of env var. The "no --palace but env var set" case is #540's scope and is not changed here. --- mempalace/mcp_server.py | 52 +++++++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index e3e89c6f61..eae048b5f3 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -46,6 +46,7 @@ import json # noqa: E402 import logging # noqa: E402 import hashlib # noqa: E402 +import threading # noqa: E402 import time # noqa: E402 from datetime import date, datetime # noqa: E402 from pathlib import Path # noqa: E402 @@ -78,7 +79,7 @@ follow_tunnels, ) -from .knowledge_graph import KnowledgeGraph # noqa: E402 +from .knowledge_graph import KnowledgeGraph, DEFAULT_KG_PATH # noqa: E402 logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stderr) logger = logging.getLogger("mempalace_mcp") @@ -103,12 +104,39 @@ def _parse_args(): os.environ["MEMPALACE_PALACE_PATH"] = os.path.abspath(_args.palace) _config = MempalaceConfig() -# Only override KG path when --palace is explicitly provided; otherwise use -# KnowledgeGraph's default (~/.mempalace/knowledge_graph.sqlite3). -if _args.palace: - _kg = KnowledgeGraph(db_path=os.path.join(_config.palace_path, "knowledge_graph.sqlite3")) -else: - _kg = KnowledgeGraph() + +# Lazy per-path KG cache. Import no longer creates the sqlite file as a side +# effect (see issue #1136). The path is resolved on each tool call so that a +# multi-tenant host rotating MEMPALACE_PALACE_PATH between calls routes each +# call to the correct KG file, matching the per-call behavior of _get_client() +# on the ChromaDB side. +_kg_by_path: dict[str, KnowledgeGraph] = {} +_kg_cache_lock = threading.Lock() + +# Whether --palace was given at startup. Controls default-path resolution: +# with the flag, KG follows _config.palace_path per call; without it, KG stays +# on DEFAULT_KG_PATH regardless of env var (issue #540's territory, out of +# scope here). +_palace_flag_given: bool = bool(_args.palace) + + +def _resolve_kg_path() -> str: + if _palace_flag_given: + return os.path.join(_config.palace_path, "knowledge_graph.sqlite3") + return DEFAULT_KG_PATH + + +def _get_kg() -> KnowledgeGraph: + path = os.path.abspath(_resolve_kg_path()) + kg = _kg_by_path.get(path) + if kg is not None: + return kg + with _kg_cache_lock: + kg = _kg_by_path.get(path) + if kg is None: + kg = KnowledgeGraph(db_path=path) + _kg_by_path[path] = kg + return kg _client_cache = None @@ -1063,7 +1091,7 @@ def tool_kg_query(entity: str, as_of: str = None, direction: str = "both"): return {"error": str(e)} if direction not in ("outgoing", "incoming", "both"): return {"error": "direction must be 'outgoing', 'incoming', or 'both'"} - results = _kg.query_entity(entity, as_of=as_of, direction=direction) + results = _get_kg().query_entity(entity, as_of=as_of, direction=direction) return {"entity": entity, "as_of": as_of, "facts": results, "count": len(results)} @@ -1108,7 +1136,7 @@ def tool_kg_add( "source_drawer_id": source_drawer_id, }, ) - triple_id = _kg.add_triple( + triple_id = _get_kg().add_triple( subject, predicate, object, @@ -1147,7 +1175,7 @@ def tool_kg_invalidate(subject: str, predicate: str, object: str, ended: str = N "ended": resolved_ended, }, ) - _kg.invalidate(subject, predicate, object, ended=resolved_ended) + _get_kg().invalidate(subject, predicate, object, ended=resolved_ended) return { "success": True, "fact": f"{subject} → {predicate} → {object}", @@ -1162,13 +1190,13 @@ def tool_kg_timeline(entity: str = None): entity = sanitize_kg_value(entity, "entity") except ValueError as e: return {"error": str(e)} - results = _kg.timeline(entity) + results = _get_kg().timeline(entity) return {"entity": entity or "all", "timeline": results, "count": len(results)} def tool_kg_stats(): """Knowledge graph overview: entities, triples, relationship types.""" - return _kg.stats() + return _get_kg().stats() # ==================== AGENT DIARY ==================== From 9e730098e97c173476b9949a5eca2253e56a08a5 Mon Sep 17 00:00:00 2001 From: mvalentsev Date: Fri, 24 Apr 2026 12:48:00 +0500 Subject: [PATCH 17/77] test(mcp): migrate _kg monkeypatches to _get_kg (#1136) Direct module-attribute patching of _kg is obsolete after the lazy cache refactor. Switch test helpers to patch _get_kg instead so the fixture KG replaces the factory rather than a now-missing singleton. - tests/test_mcp_server.py: _patch_mcp_server helper - tests/benchmarks/test_mcp_bench.py: _patch_mcp_config helper - tests/benchmarks/test_memory_profile.py: inline patch in test_tool_status_repeated_calls --- tests/benchmarks/test_mcp_bench.py | 3 ++- tests/benchmarks/test_memory_profile.py | 3 ++- tests/test_mcp_server.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/benchmarks/test_mcp_bench.py b/tests/benchmarks/test_mcp_bench.py index 4e8330b9cc..42e73ecb54 100644 --- a/tests/benchmarks/test_mcp_bench.py +++ b/tests/benchmarks/test_mcp_bench.py @@ -40,8 +40,9 @@ def _patch_mcp_config(monkeypatch, palace_path, tmp_path): import mempalace.mcp_server as mcp_mod + kg = KnowledgeGraph(db_path=str(tmp_path / "kg.sqlite3")) monkeypatch.setattr(mcp_mod, "_config", cfg) - monkeypatch.setattr(mcp_mod, "_kg", KnowledgeGraph(db_path=str(tmp_path / "kg.sqlite3"))) + monkeypatch.setattr(mcp_mod, "_get_kg", lambda: kg) def _get_rss_mb(): diff --git a/tests/benchmarks/test_memory_profile.py b/tests/benchmarks/test_memory_profile.py index b299b2d5eb..047bfaa60e 100644 --- a/tests/benchmarks/test_memory_profile.py +++ b/tests/benchmarks/test_memory_profile.py @@ -84,8 +84,9 @@ def test_tool_status_repeated_calls(self, tmp_path, monkeypatch): cfg = MempalaceConfig(config_dir=str(tmp_path / "cfg")) monkeypatch.setattr(cfg, "_file_config", {"palace_path": palace_path}) + kg = KnowledgeGraph(db_path=str(tmp_path / "kg.sqlite3")) monkeypatch.setattr(mcp_mod, "_config", cfg) - monkeypatch.setattr(mcp_mod, "_kg", KnowledgeGraph(db_path=str(tmp_path / "kg.sqlite3"))) + monkeypatch.setattr(mcp_mod, "_get_kg", lambda: kg) from mempalace.mcp_server import tool_status diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index ec9562b8ce..2ab2eb8e88 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -18,7 +18,7 @@ def _patch_mcp_server(monkeypatch, config, kg): from mempalace import mcp_server monkeypatch.setattr(mcp_server, "_config", config) - monkeypatch.setattr(mcp_server, "_kg", kg) + monkeypatch.setattr(mcp_server, "_get_kg", lambda: kg) def _get_collection(palace_path, create=False): From c69a622a18db0a082bce8729d35c1d6a8d98efdf Mon Sep 17 00:00:00 2001 From: mvalentsev Date: Fri, 24 Apr 2026 12:53:19 +0500 Subject: [PATCH 18/77] test(mcp): add multi-tenant and lazy-init tests for KG (#1136) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestKGLazyCache covers the scenarios behind the lazy per-path refactor: - test_lazy_init_no_import_side_effect: a fresh subprocess import does not create ~/.mempalace/knowledge_graph.sqlite3 (what closed PR #167 was aiming at). - test_get_kg_returns_same_instance: two _get_kg() calls under the same resolved path return the same object, cache has one entry. - test_get_kg_different_paths_different_instances: rotating env var produces distinct KGs. - test_multi_tenant_env_switch: the exact scenario from #1136 — write under path A, query under path B returns empty, switching back to A sees the fact. - test_cache_thread_safe: 16 threads racing _get_kg() end up with one shared instance and one cache entry. --- tests/test_mcp_server.py | 112 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 2ab2eb8e88..86d5878d7e 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -8,6 +8,7 @@ from datetime import datetime import json +import os import sys import pytest @@ -1143,3 +1144,114 @@ def _spy_create(self, name, **kwargs): for kwargs in captured["get"]: assert "embedding_function" in kwargs assert kwargs["embedding_function"] is not None + + +class TestKGLazyCache: + """Lazy per-path KnowledgeGraph cache (issue #1136).""" + + def test_lazy_init_no_import_side_effect(self, tmp_path): + """Importing mcp_server must not create knowledge_graph.sqlite3. + + Runs in a fresh subprocess with HOME pointed at tmp_path so the + assertion targets a clean filesystem, independent of conftest's + session-level HOME patch. + """ + import subprocess + import sys + + kg_file = tmp_path / ".mempalace" / "knowledge_graph.sqlite3" + result = subprocess.run( + [sys.executable, "-c", "import mempalace.mcp_server"], + env={ + "HOME": str(tmp_path), + "USERPROFILE": str(tmp_path), + "PATH": os.environ.get("PATH", ""), + "PYTHONPATH": os.environ.get("PYTHONPATH", ""), + }, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, f"import failed: {result.stderr}" + assert not kg_file.exists(), f"import created sqlite file at {kg_file} as a side effect" + + def test_get_kg_returns_same_instance(self, tmp_path, monkeypatch): + """Two calls with the same resolved path return the same KG.""" + from mempalace import mcp_server + + monkeypatch.setattr(mcp_server, "_kg_by_path", {}) + monkeypatch.setattr(mcp_server, "_palace_flag_given", True) + monkeypatch.setenv("MEMPALACE_PALACE_PATH", str(tmp_path)) + + kg1 = mcp_server._get_kg() + kg2 = mcp_server._get_kg() + assert kg1 is kg2 + assert len(mcp_server._kg_by_path) == 1 + + def test_get_kg_different_paths_different_instances(self, tmp_path, monkeypatch): + """Different palace paths map to different KG instances.""" + from mempalace import mcp_server + + tmp_a = tmp_path / "a" + tmp_b = tmp_path / "b" + tmp_a.mkdir() + tmp_b.mkdir() + + monkeypatch.setattr(mcp_server, "_kg_by_path", {}) + monkeypatch.setattr(mcp_server, "_palace_flag_given", True) + + monkeypatch.setenv("MEMPALACE_PALACE_PATH", str(tmp_a)) + kg_a = mcp_server._get_kg() + monkeypatch.setenv("MEMPALACE_PALACE_PATH", str(tmp_b)) + kg_b = mcp_server._get_kg() + + assert kg_a is not kg_b + assert len(mcp_server._kg_by_path) == 2 + + def test_multi_tenant_env_switch(self, tmp_path, monkeypatch): + """The issue #1136 acceptance scenario. + + Rotating MEMPALACE_PALACE_PATH between MCP tool calls must route + each call to the correct tenant's KG sqlite file. + """ + from mempalace import mcp_server + + tmp_a = tmp_path / "tenant_a" + tmp_b = tmp_path / "tenant_b" + tmp_a.mkdir() + tmp_b.mkdir() + + monkeypatch.setattr(mcp_server, "_kg_by_path", {}) + monkeypatch.setattr(mcp_server, "_palace_flag_given", True) + + monkeypatch.setenv("MEMPALACE_PALACE_PATH", str(tmp_a)) + add_result = mcp_server.tool_kg_add( + subject="alice_secret", + predicate="owns", + object="repo_a", + ) + assert add_result.get("success") is True, add_result + + monkeypatch.setenv("MEMPALACE_PALACE_PATH", str(tmp_b)) + query_b = mcp_server.tool_kg_query(entity="alice_secret") + assert query_b.get("count", 0) == 0, f"tenant B leaked tenant A's fact: {query_b}" + + monkeypatch.setenv("MEMPALACE_PALACE_PATH", str(tmp_a)) + query_a = mcp_server.tool_kg_query(entity="alice_secret") + assert query_a.get("count", 0) >= 1, f"tenant A lost its own fact: {query_a}" + + def test_cache_thread_safe(self, tmp_path, monkeypatch): + """Concurrent _get_kg() for the same path yields one instance.""" + import concurrent.futures + from mempalace import mcp_server + + monkeypatch.setattr(mcp_server, "_kg_by_path", {}) + monkeypatch.setattr(mcp_server, "_palace_flag_given", True) + monkeypatch.setenv("MEMPALACE_PALACE_PATH", str(tmp_path)) + + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as pool: + results = list(pool.map(lambda _: mcp_server._get_kg(), range(16))) + + ids = {id(kg) for kg in results} + assert len(ids) == 1, f"expected 1 unique instance, got {len(ids)}" + assert len(mcp_server._kg_by_path) == 1 From 84f9726a39e65be3f8afba0d14d4183999520c0d Mon Sep 17 00:00:00 2001 From: mvalentsev Date: Fri, 24 Apr 2026 13:03:12 +0500 Subject: [PATCH 19/77] test(mcp): fix Windows subprocess env in KG lazy-init test Passing a stripped env dict without SYSTEMROOT/WINDIR breaks Python bootstrap on Windows (_Py_HashRandomization_Init). Inherit the parent env and strip MEMPAL* vars instead, then override HOME/USERPROFILE to the tmp dir. --- tests/test_mcp_server.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 86d5878d7e..638ac15259 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -1160,14 +1160,12 @@ def test_lazy_init_no_import_side_effect(self, tmp_path): import sys kg_file = tmp_path / ".mempalace" / "knowledge_graph.sqlite3" + env = {k: v for k, v in os.environ.items() if not k.startswith("MEMPAL")} + env["HOME"] = str(tmp_path) + env["USERPROFILE"] = str(tmp_path) result = subprocess.run( [sys.executable, "-c", "import mempalace.mcp_server"], - env={ - "HOME": str(tmp_path), - "USERPROFILE": str(tmp_path), - "PATH": os.environ.get("PATH", ""), - "PYTHONPATH": os.environ.get("PYTHONPATH", ""), - }, + env=env, capture_output=True, text=True, timeout=30, From 19f8a4ff682fe5fdad71c6c5d31a45fac6ce2310 Mon Sep 17 00:00:00 2001 From: mvalentsev Date: Fri, 24 Apr 2026 13:07:34 +0500 Subject: [PATCH 20/77] style(mcp): drop issue-tracker comments from KG cache block Inline comments referencing #1136 and #540 add no information the identifiers do not already convey. PR description carries the context; code stays quiet. --- mempalace/mcp_server.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index eae048b5f3..5f74df6ead 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -105,18 +105,8 @@ def _parse_args(): _config = MempalaceConfig() -# Lazy per-path KG cache. Import no longer creates the sqlite file as a side -# effect (see issue #1136). The path is resolved on each tool call so that a -# multi-tenant host rotating MEMPALACE_PALACE_PATH between calls routes each -# call to the correct KG file, matching the per-call behavior of _get_client() -# on the ChromaDB side. _kg_by_path: dict[str, KnowledgeGraph] = {} _kg_cache_lock = threading.Lock() - -# Whether --palace was given at startup. Controls default-path resolution: -# with the flag, KG follows _config.palace_path per call; without it, KG stays -# on DEFAULT_KG_PATH regardless of env var (issue #540's territory, out of -# scope here). _palace_flag_given: bool = bool(_args.palace) From 0a626580513c09bd3a9f7719e53dc3e4810463f5 Mon Sep 17 00:00:00 2001 From: mvalentsev Date: Sat, 2 May 2026 18:00:36 +0500 Subject: [PATCH 21/77] fix(mcp): drain KG cache on tool_reconnect tool_reconnect cleared ChromaDB caches but left _kg_by_path entries intact. After an external replacement of knowledge_graph.sqlite3 the server kept serving the old open sqlite3.Connection, returning stale results. Now iterate _kg_by_path under _kg_cache_lock, call close() best-effort, and clear the dict so the next tool call reopens the KG from disk. Two new tests in TestKGLazyCache verify cache invalidation and that a failing close() does not block the clear. --- mempalace/mcp_server.py | 14 ++++++++++++-- tests/test_mcp_server.py | 42 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 5f74df6ead..3b3b4e2c86 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -1418,10 +1418,11 @@ def tool_memories_filed_away(): def tool_reconnect(): - """Force the MCP server to drop the cached ChromaDB collection and reconnect. + """Force the MCP server to drop cached ChromaDB + KnowledgeGraph state. Use after external scripts or CLI commands modify the palace database - directly, which can leave the in-memory HNSW index stale. + or replace ``knowledge_graph.sqlite3`` directly, which can leave the + in-memory HNSW index stale or pin a closed-on-disk SQLite connection. """ global \ _client_cache, \ @@ -1439,6 +1440,15 @@ def tool_reconnect(): # still applies after the reconnect. _vector_disabled = False _vector_disabled_reason = "" + # Drain the per-path KnowledgeGraph cache so a replaced sqlite file is + # reopened on the next tool call rather than served from a stale handle. + with _kg_cache_lock: + for kg in _kg_by_path.values(): + try: + kg.close() + except Exception: + pass + _kg_by_path.clear() try: col = _get_collection() if col is None: diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 638ac15259..092b70720b 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -1253,3 +1253,45 @@ def test_cache_thread_safe(self, tmp_path, monkeypatch): ids = {id(kg) for kg in results} assert len(ids) == 1, f"expected 1 unique instance, got {len(ids)}" assert len(mcp_server._kg_by_path) == 1 + + def test_tool_reconnect_drains_kg_cache(self, monkeypatch): + """``tool_reconnect`` must close cached KG instances and clear the dict. + + Without this, an external replacement of ``knowledge_graph.sqlite3`` + leaves the server pinned to a stale ``sqlite3.Connection``. + """ + from mempalace import mcp_server + + class _FakeKG: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + fake_a = _FakeKG() + fake_b = _FakeKG() + monkeypatch.setattr(mcp_server, "_kg_by_path", {"/a": fake_a, "/b": fake_b}) + # Bypass real ChromaDB so the test isolates KG-cache behaviour. + monkeypatch.setattr(mcp_server, "_get_collection", lambda: None) + + mcp_server.tool_reconnect() + + assert fake_a.closed is True + assert fake_b.closed is True + assert mcp_server._kg_by_path == {} + + def test_tool_reconnect_swallows_kg_close_errors(self, monkeypatch): + """A failing ``close()`` on one cached KG must not block cache clearing.""" + from mempalace import mcp_server + + class _BoomKG: + def close(self): + raise RuntimeError("boom") + + monkeypatch.setattr(mcp_server, "_kg_by_path", {"/a": _BoomKG()}) + monkeypatch.setattr(mcp_server, "_get_collection", lambda: None) + + mcp_server.tool_reconnect() + + assert mcp_server._kg_by_path == {} From 45df1a265748200f709a1d737bfb8003061cbb32 Mon Sep 17 00:00:00 2001 From: mvalentsev Date: Wed, 22 Apr 2026 15:59:43 +0500 Subject: [PATCH 22/77] fix(backends/chroma): release SQLite file lock on close_palace/close (#1067) ChromaBackend.close_palace() and close() evicted cached PersistentClients from self._clients without calling client.close(), so chromadb 1.5.x kept the rust-side SQLite file lock until GC. Reopening the same palace path after shutil.rmtree + re-create within one process then failed with SQLITE_READONLY_DBMOVED (SQLite code 1032). Add _close_client() helper with a try/except fallback for older chromadb, and route close_palace(), close(), and the DB-file-missing invalidation branch of _client() through it. The mtime/inode auto-invalidation branch is left as-is: callers there may still hold a live ChromaCollection handle, and closing out from under them clears the rust bindings mid-use. Regression tests cover close_palace reopen-same-path and whole-backend close for multiple palaces. --- mempalace/backends/chroma.py | 28 ++++++++++++++++++--- tests/test_backends.py | 47 ++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/mempalace/backends/chroma.py b/mempalace/backends/chroma.py index d9b99a412b..e7c2e6f3ae 100644 --- a/mempalace/backends/chroma.py +++ b/mempalace/backends/chroma.py @@ -676,6 +676,20 @@ def _as_list(v: Any) -> list: return [v] +def _close_client(client) -> None: + """Call ``PersistentClient.close()`` if available, swallow otherwise. + + chromadb 1.5.x exposes ``Client.close()`` to release rust-side SQLite + file locks; older versions relied on GC. Try/except keeps forward-compat. + """ + if client is None: + return + try: + client.close() + except Exception: + logger.debug("client.close() unavailable or failed", exc_info=True) + + class ChromaCollection(BaseCollection): """Thin adapter translating ChromaDB dict returns into typed results.""" @@ -977,7 +991,7 @@ def _client(self, palace_path: str): db_path = os.path.join(palace_path, "chroma.sqlite3") # DB was present when cache was built but is now missing → invalidate. if cached is not None and not os.path.isfile(db_path): - self._clients.pop(palace_path, None) + _close_client(self._clients.pop(palace_path, None)) self._freshness.pop(palace_path, None) cached = None cached_inode, cached_mtime = 0, 0.0 @@ -1134,14 +1148,22 @@ def get_collection( return ChromaCollection(collection) def close_palace(self, palace) -> None: - """Drop cached handles for ``palace``. Accepts ``PalaceRef`` or legacy path str.""" + """Drop cached handles for ``palace`` and release its SQLite file lock. + + Accepts ``PalaceRef`` or legacy path str. chromadb's rust-side file + lock is held until ``PersistentClient.close()`` is called, so plain + dict eviction would leave the palace path unreopenable and + unremovable in the same process. + """ path = palace.local_path if isinstance(palace, PalaceRef) else palace if path is None: return - self._clients.pop(path, None) + _close_client(self._clients.pop(path, None)) self._freshness.pop(path, None) def close(self) -> None: + for client in self._clients.values(): + _close_client(client) self._clients.clear() self._freshness.clear() self._closed = True diff --git a/tests/test_backends.py b/tests/test_backends.py index 4ddfe1207a..06998c5ae0 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -1,4 +1,5 @@ import os +import shutil import sqlite3 from pathlib import Path @@ -206,6 +207,52 @@ def test_query_empty_preserves_embeddings_outer_shape_when_requested(): assert not_requested.embeddings is None +def test_chroma_close_palace_releases_sqlite_lock_for_reopen(tmp_path): + """close_palace must release chromadb's rust-side SQLite file lock so + a fresh PersistentClient on the same path after shutil.rmtree can + write without hitting SQLITE_READONLY_DBMOVED.""" + backend = ChromaBackend() + palace_path = tmp_path / "palace-a" + ref = PalaceRef(id=str(palace_path), local_path=str(palace_path)) + + col = backend.get_collection(palace=ref, collection_name="mempalace_drawers", create=True) + col.upsert(documents=["hello"], ids=["a"], metadatas=[{"k": "v"}]) + + backend.close_palace(ref) + shutil.rmtree(palace_path) + + col = backend.get_collection(palace=ref, collection_name="mempalace_drawers", create=True) + col.upsert(documents=["world"], ids=["b"], metadatas=[{"k": "v2"}]) + assert col.count() == 1 + + +def test_chroma_close_releases_all_cached_clients(tmp_path): + """close() must release every cached client's SQLite file lock so any + of their palace paths can be reopened by a fresh backend in the same + process.""" + backend = ChromaBackend() + palace_a = tmp_path / "palace-a" + palace_b = tmp_path / "palace-b" + ref_a = PalaceRef(id=str(palace_a), local_path=str(palace_a)) + ref_b = PalaceRef(id=str(palace_b), local_path=str(palace_b)) + + for ref in (ref_a, ref_b): + backend.get_collection(palace=ref, collection_name="mempalace_drawers", create=True).upsert( + documents=["x"], ids=["x"], metadatas=[{"k": "v"}] + ) + + backend.close() + + for path in (palace_a, palace_b): + shutil.rmtree(path) + ref = PalaceRef(id=str(path), local_path=str(path)) + fresh = ChromaBackend() + col = fresh.get_collection(palace=ref, collection_name="mempalace_drawers", create=True) + col.upsert(documents=["y"], ids=["y"], metadatas=[{"k": "v2"}]) + assert col.count() == 1 + fresh.close() + + def test_chroma_cache_invalidates_when_db_file_missing(tmp_path): """A palace rebuild that removes chroma.sqlite3 must drop the stale cache. From 7cee74c8c8c6c31acb8b363788443612e36dac77 Mon Sep 17 00:00:00 2001 From: mvalentsev Date: Thu, 30 Apr 2026 14:49:02 +0500 Subject: [PATCH 23/77] fix(fact-checker): reconfigure stdio to UTF-8 on Windows The `python -m mempalace.fact_checker --stdin` entry point reads non-ASCII text through the system ANSI codepage (cp1252/cp1251/cp950) on Windows, which mojibakes characters before claim-extraction sees them. Reconfigure stdin/stdout/stderr to UTF-8 with `errors="strict"`, wrapped in try/except so a replaced stream (Jupyter, test harness) logs a warning rather than crashing the CLI. Mirrors the same fix shipped for `mcp_server.py:main()` (#400) and `hooks_cli.py:run_hook()` (#1280) -- this is the third and last stdin-reading entry point in the package. --- mempalace/fact_checker.py | 27 +++++++++++++++++ tests/test_fact_checker.py | 60 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/mempalace/fact_checker.py b/mempalace/fact_checker.py index 50e8842f24..c894859972 100644 --- a/mempalace/fact_checker.py +++ b/mempalace/fact_checker.py @@ -303,11 +303,38 @@ def _edit_distance(s1: str, s2: str) -> int: return prev[-1] +def _reconfigure_stdio_utf8_on_windows(): + """Decode --stdin payload as UTF-8 on Windows. + + Without this, Python defaults stdio to the system ANSI codepage + (cp1252/cp1251/cp950 depending on locale), which mojibakes + non-ASCII fact text before pattern parsing sees it. + """ + import sys + + if sys.platform != "win32": + return + for name in ("stdin", "stdout", "stderr"): + stream = getattr(sys, name, None) + reconfigure = getattr(stream, "reconfigure", None) + if reconfigure is None: + continue + try: + reconfigure(encoding="utf-8", errors="strict") + except Exception as exc: + print( + f"WARNING: Could not reconfigure {name} to UTF-8: {exc}", + file=sys.stderr, + ) + + if __name__ == "__main__": import argparse import json import sys + _reconfigure_stdio_utf8_on_windows() + parser = argparse.ArgumentParser( description="Check text against known facts in the MemPalace palace.", epilog="Exits 0 when no issues found, 1 when one or more issues detected.", diff --git a/tests/test_fact_checker.py b/tests/test_fact_checker.py index 5b34a40082..9db370e37b 100644 --- a/tests/test_fact_checker.py +++ b/tests/test_fact_checker.py @@ -286,3 +286,63 @@ def test_exits_nonzero_when_issues_found(self, tmp_path, monkeypatch, capsys): assert "similar_name" in out # Silence unused import warning. _ = (MagicMock, patch, fact_checker) + + def test_reconfigures_stdio_to_utf8_on_windows(self): + """Windows fact_checker --stdin must decode payload as UTF-8. + + Without this, Python defaults stdio to the system ANSI codepage + (cp1252/cp1251/cp950), which mojibakes non-ASCII text before + pattern parsing sees it. + """ + import io + import sys + + from mempalace.fact_checker import _reconfigure_stdio_utf8_on_windows + + class _ReconfigurableStringIO(io.StringIO): + def __init__(self, initial_value=""): + super().__init__(initial_value) + self.reconfigure_calls = [] + + def reconfigure(self, **kwargs): + self.reconfigure_calls.append(kwargs) + + stdin = _ReconfigurableStringIO() + stdout = _ReconfigurableStringIO() + stderr = _ReconfigurableStringIO() + with ( + patch.object(sys, "platform", "win32"), + patch.object(sys, "stdin", stdin), + patch.object(sys, "stdout", stdout), + patch.object(sys, "stderr", stderr), + ): + _reconfigure_stdio_utf8_on_windows() + + expected = {"encoding": "utf-8", "errors": "strict"} + assert stdin.reconfigure_calls == [expected] + assert stdout.reconfigure_calls == [expected] + assert stderr.reconfigure_calls == [expected] + + def test_reconfigure_stdio_is_noop_off_windows(self): + """Linux/macOS already default to UTF-8 stdio -- helper must not touch streams.""" + import io + import sys + + from mempalace.fact_checker import _reconfigure_stdio_utf8_on_windows + + class _ReconfigurableStringIO(io.StringIO): + def __init__(self): + super().__init__() + self.reconfigure_calls = [] + + def reconfigure(self, **kwargs): + self.reconfigure_calls.append(kwargs) + + stdin = _ReconfigurableStringIO() + with ( + patch.object(sys, "platform", "linux"), + patch.object(sys, "stdin", stdin), + ): + _reconfigure_stdio_utf8_on_windows() + + assert stdin.reconfigure_calls == [] From 32f4dfa26d25b8ff243bfd2e636f5e96d8947a83 Mon Sep 17 00:00:00 2001 From: mvalentsev Date: Thu, 30 Apr 2026 15:00:37 +0500 Subject: [PATCH 24/77] fix(cli): reconfigure stdio to UTF-8 on Windows The primary `mempalace` console_script (`cli.py:main()`) reads non-ASCII arguments via piped stdin and writes verbatim drawer text / wing names through `print()`. On Windows, Python defaults stdio to the system ANSI codepage (cp1252/cp1251/cp950), so: - `mempalace search "..." > out.txt` mojibakes any drawer text containing non-Latin characters - `mempalace ... < input.txt` mojibakes piped non-ASCII input Reconfigure stdin/stdout/stderr to UTF-8 (`errors="strict"`) at the top of `main()`, mirroring the helper added in this PR for fact_checker's `__main__` block. Wrapped in try/except so a replaced stream (Jupyter, test harness) logs a warning and continues rather than crashing the CLI. The reconfigure cascades through every `mempalace` subcommand (`init`/`mine`/`search`/`status`/`hook`/etc.) and through the interactive flows that read non-ASCII names via `input()` (onboarding, entity detector, room detector). With this commit the package's three user-facing entry points (`mempalace`, `mempalace-mcp`, and `python -m mempalace.fact_checker`) all reconfigure stdio identically on Windows. --- mempalace/cli.py | 27 ++++++++++++++++++++++++ tests/test_cli.py | 52 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/mempalace/cli.py b/mempalace/cli.py index f2606a4368..7372cd7c31 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -935,7 +935,34 @@ def cmd_compress(args): print(" (dry run -- nothing stored)") +def _reconfigure_stdio_utf8_on_windows(): + """Decode stdio as UTF-8 on Windows for the primary `mempalace` CLI. + + Without this, Python defaults stdio to the system ANSI codepage + (cp1252/cp1251/cp950 depending on locale). That mojibakes non-ASCII + content piped in (`mempalace search ... < query.txt`) or piped out + (`mempalace search "..." > out.txt`) when verbatim drawer text or + wing/room names contain non-Latin characters. + """ + if sys.platform != "win32": + return + for name in ("stdin", "stdout", "stderr"): + stream = getattr(sys, name, None) + reconfigure = getattr(stream, "reconfigure", None) + if reconfigure is None: + continue + try: + reconfigure(encoding="utf-8", errors="strict") + except Exception as exc: + print( + f"WARNING: Could not reconfigure {name} to UTF-8: {exc}", + file=sys.stderr, + ) + + def main(): + _reconfigure_stdio_utf8_on_windows() + version_label = f"MemPalace {__version__}" parser = argparse.ArgumentParser( description="MemPalace — Give your AI a memory. No API key required.", diff --git a/tests/test_cli.py b/tests/test_cli.py index 328b90c0b9..4836d69252 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1042,3 +1042,55 @@ def test_cmd_repair_trailing_slash_does_not_recurse(): palace_path = os.path.expanduser(args.palace).rstrip(os.sep) backup_path = palace_path + ".backup" assert not backup_path.startswith(palace_path + os.sep) + + +# ── stdio reconfigure on Windows ───────────────────────────────────── + + +class _ReconfigurableStringIO: + def __init__(self): + self.reconfigure_calls = [] + + def reconfigure(self, **kwargs): + self.reconfigure_calls.append(kwargs) + + +def test_reconfigures_stdio_to_utf8_on_windows(): + """Windows `mempalace` CLI must decode/encode stdio as UTF-8. + + Without this, piped non-ASCII input (`mempalace search ... < q.txt`) + or piped non-ASCII output (`mempalace search "..." > out.txt`) is + mojibaked through the system ANSI codepage on non-Latin Windows + locales (cp1252/cp1251/cp950). + """ + from mempalace.cli import _reconfigure_stdio_utf8_on_windows + + stdin = _ReconfigurableStringIO() + stdout = _ReconfigurableStringIO() + stderr = _ReconfigurableStringIO() + with ( + patch.object(sys, "platform", "win32"), + patch.object(sys, "stdin", stdin), + patch.object(sys, "stdout", stdout), + patch.object(sys, "stderr", stderr), + ): + _reconfigure_stdio_utf8_on_windows() + + expected = {"encoding": "utf-8", "errors": "strict"} + assert stdin.reconfigure_calls == [expected] + assert stdout.reconfigure_calls == [expected] + assert stderr.reconfigure_calls == [expected] + + +def test_reconfigure_stdio_is_noop_off_windows(): + """Linux/macOS already default to UTF-8 stdio -- helper must not touch streams.""" + from mempalace.cli import _reconfigure_stdio_utf8_on_windows + + stdin = _ReconfigurableStringIO() + with ( + patch.object(sys, "platform", "linux"), + patch.object(sys, "stdin", stdin), + ): + _reconfigure_stdio_utf8_on_windows() + + assert stdin.reconfigure_calls == [] From 03643eb507e4ba81c65d50b519fcfb4dfb3c769f Mon Sep 17 00:00:00 2001 From: mvalentsev Date: Sun, 3 May 2026 21:37:12 +0500 Subject: [PATCH 25/77] fix(cli, fact-checker): per-stream stdio errors policy on Windows Previously all three streams reconfigured to UTF-8 with errors='strict'. That kills 'mempalace search' the moment a drawer carrying a surrogate half (round-tripped from a filename via surrogateescape) hits print(), losing the rest of the result block. Same hazard for warning lines on stderr. Split the policy: stdin -> surrogateescape (malformed bytes from a redirected file survive as lone surrogates instead of crashing the read) stdout -> replace (drawer text with a stray surrogate becomes U+FFFD instead of UnicodeEncodeError mid-print) stderr -> replace (same protection for logger / warning paths) Applied identically in the cli.py and fact_checker.py helpers; the DRY extraction into a shared module is a separate cleanup ask, kept out of this fix to keep the diff narrow. Tests updated for the new per-stream assertion. --- mempalace/cli.py | 20 ++++++++++++++++++-- mempalace/fact_checker.py | 18 ++++++++++++++++-- tests/test_cli.py | 11 +++++++---- tests/test_fact_checker.py | 11 +++++++---- 4 files changed, 48 insertions(+), 12 deletions(-) diff --git a/mempalace/cli.py b/mempalace/cli.py index 7372cd7c31..7052e1f79e 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -943,16 +943,32 @@ def _reconfigure_stdio_utf8_on_windows(): content piped in (`mempalace search ... < query.txt`) or piped out (`mempalace search "..." > out.txt`) when verbatim drawer text or wing/room names contain non-Latin characters. + + Per-stream errors policy: + stdin -- surrogateescape: malformed bytes from a redirected file + survive as lone surrogates instead of crashing the read. + stdout -- replace: ``mempalace search`` prints verbatim drawer + text. A drawer that round-tripped a filename through + surrogateescape can hold a lone surrogate, which would + otherwise raise ``UnicodeEncodeError`` mid-print and + lose the rest of the search result block. + stderr -- replace: same hazard for logger output that quotes + user-supplied path or content. """ if sys.platform != "win32": return - for name in ("stdin", "stdout", "stderr"): + policies = ( + ("stdin", "surrogateescape"), + ("stdout", "replace"), + ("stderr", "replace"), + ) + for name, errors in policies: stream = getattr(sys, name, None) reconfigure = getattr(stream, "reconfigure", None) if reconfigure is None: continue try: - reconfigure(encoding="utf-8", errors="strict") + reconfigure(encoding="utf-8", errors=errors) except Exception as exc: print( f"WARNING: Could not reconfigure {name} to UTF-8: {exc}", diff --git a/mempalace/fact_checker.py b/mempalace/fact_checker.py index c894859972..1844c45df8 100644 --- a/mempalace/fact_checker.py +++ b/mempalace/fact_checker.py @@ -309,18 +309,32 @@ def _reconfigure_stdio_utf8_on_windows(): Without this, Python defaults stdio to the system ANSI codepage (cp1252/cp1251/cp950 depending on locale), which mojibakes non-ASCII fact text before pattern parsing sees it. + + Per-stream errors policy mirrors the primary CLI helper in + ``mempalace/cli.py``: + stdin -- surrogateescape: malformed input bytes survive as lone + surrogates instead of crashing the read. + stdout -- replace: extracted fact text can include surrogate + halves round-tripped from filenames; replace prevents + a UnicodeEncodeError mid-print. + stderr -- replace: same protection for warning lines. """ import sys if sys.platform != "win32": return - for name in ("stdin", "stdout", "stderr"): + policies = ( + ("stdin", "surrogateescape"), + ("stdout", "replace"), + ("stderr", "replace"), + ) + for name, errors in policies: stream = getattr(sys, name, None) reconfigure = getattr(stream, "reconfigure", None) if reconfigure is None: continue try: - reconfigure(encoding="utf-8", errors="strict") + reconfigure(encoding="utf-8", errors=errors) except Exception as exc: print( f"WARNING: Could not reconfigure {name} to UTF-8: {exc}", diff --git a/tests/test_cli.py b/tests/test_cli.py index 4836d69252..6b4b7b3a5c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1076,10 +1076,13 @@ def test_reconfigures_stdio_to_utf8_on_windows(): ): _reconfigure_stdio_utf8_on_windows() - expected = {"encoding": "utf-8", "errors": "strict"} - assert stdin.reconfigure_calls == [expected] - assert stdout.reconfigure_calls == [expected] - assert stderr.reconfigure_calls == [expected] + # Per-stream errors policy: stdin survives bad bytes via + # surrogateescape so a redirected non-UTF-8 file does not crash + # the read; stdout/stderr use replace so a drawer carrying a + # round-tripped surrogate half does not crash mid-print. + assert stdin.reconfigure_calls == [{"encoding": "utf-8", "errors": "surrogateescape"}] + assert stdout.reconfigure_calls == [{"encoding": "utf-8", "errors": "replace"}] + assert stderr.reconfigure_calls == [{"encoding": "utf-8", "errors": "replace"}] def test_reconfigure_stdio_is_noop_off_windows(): diff --git a/tests/test_fact_checker.py b/tests/test_fact_checker.py index 9db370e37b..89d83663ac 100644 --- a/tests/test_fact_checker.py +++ b/tests/test_fact_checker.py @@ -318,10 +318,13 @@ def reconfigure(self, **kwargs): ): _reconfigure_stdio_utf8_on_windows() - expected = {"encoding": "utf-8", "errors": "strict"} - assert stdin.reconfigure_calls == [expected] - assert stdout.reconfigure_calls == [expected] - assert stderr.reconfigure_calls == [expected] + # Per-stream errors policy: stdin uses surrogateescape so a stray + # malformed byte from a redirected file does not crash the read, + # stdout/stderr use replace so an extracted fact carrying a + # surrogate half does not crash mid-print. + assert stdin.reconfigure_calls == [{"encoding": "utf-8", "errors": "surrogateescape"}] + assert stdout.reconfigure_calls == [{"encoding": "utf-8", "errors": "replace"}] + assert stderr.reconfigure_calls == [{"encoding": "utf-8", "errors": "replace"}] def test_reconfigure_stdio_is_noop_off_windows(self): """Linux/macOS already default to UTF-8 stdio -- helper must not touch streams.""" From b8816e0fe2fa857efb984e0aff9e52739fb2af34 Mon Sep 17 00:00:00 2001 From: mvalentsev Date: Sun, 3 May 2026 21:43:51 +0500 Subject: [PATCH 26/77] fix(mcp): retry KG handlers once on concurrent close race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Race scenario: a KG tool handler calls _get_kg() and gets a live KnowledgeGraph; another thread fires tool_reconnect() between that return and the handler's kg.add_triple()/kg.query_entity()/etc call. tool_reconnect drains _kg_by_path and closes the underlying sqlite3.Connection; the handler then raises sqlite3.ProgrammingError: 'Cannot operate on a closed database', which surfaces as a -32000 to the MCP client even though the user just asked for a reconnect. New _call_kg(op) helper wraps each handler's kg call in a one-shot retry: catch exactly sqlite3.ProgrammingError, evict the stale entry (only if the cache slot still points at the closed instance — another thread may have already replaced it), and rerun op against a fresh _get_kg(). Beyond one retry give up so a sustained close-stream surfaces clearly instead of looping. All five KG handlers (tool_kg_query, tool_kg_add, tool_kg_invalidate, tool_kg_timeline, tool_kg_stats) now route through _call_kg. Tests pin the contract: * retries with a fresh KG and returns the second result * non-ProgrammingError exceptions propagate without retry * gives up after exactly one retry on sustained close --- mempalace/mcp_server.py | 61 ++++++++++++++++++++++++------- tests/test_mcp_server.py | 77 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 13 deletions(-) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 3b3b4e2c86..c67619e9f3 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -46,6 +46,7 @@ import json # noqa: E402 import logging # noqa: E402 import hashlib # noqa: E402 +import sqlite3 # noqa: E402 import threading # noqa: E402 import time # noqa: E402 from datetime import date, datetime # noqa: E402 @@ -129,6 +130,38 @@ def _get_kg() -> KnowledgeGraph: return kg +def _call_kg(op): + """Run ``op(kg)`` against the cached KG with one-shot retry on close. + + Race we're guarding against: a handler grabs ``kg = _get_kg()`` and is + about to call ``kg.add_triple(...)`` when ``tool_reconnect`` fires on + another thread, drains ``_kg_by_path``, and closes the underlying + sqlite3.Connection. The handler's call then raises + ``sqlite3.ProgrammingError: Cannot operate on a closed database`` and + bubbles up as a -32000 to the MCP client even though the user just + asked for a reconnect. + + Catch that single class of error, evict the stale entry from the + cache (only if it still points at the closed instance — another + thread may have already replaced it), and try once more with a fresh + KG. Beyond one retry give up: a second close means we're losing a + sustained race we won't win in this loop, and a hung loop is worse + than a clear failure surface. + """ + for attempt in range(2): + kg = _get_kg() + try: + return op(kg) + except sqlite3.ProgrammingError: + if attempt == 0: + path = os.path.abspath(_resolve_kg_path()) + with _kg_cache_lock: + if _kg_by_path.get(path) is kg: + _kg_by_path.pop(path, None) + continue + raise + + _client_cache = None _collection_cache = None _palace_db_inode = 0 # inode of chroma.sqlite3 at cache time @@ -1081,7 +1114,7 @@ def tool_kg_query(entity: str, as_of: str = None, direction: str = "both"): return {"error": str(e)} if direction not in ("outgoing", "incoming", "both"): return {"error": "direction must be 'outgoing', 'incoming', or 'both'"} - results = _get_kg().query_entity(entity, as_of=as_of, direction=direction) + results = _call_kg(lambda kg: kg.query_entity(entity, as_of=as_of, direction=direction)) return {"entity": entity, "as_of": as_of, "facts": results, "count": len(results)} @@ -1126,15 +1159,17 @@ def tool_kg_add( "source_drawer_id": source_drawer_id, }, ) - triple_id = _get_kg().add_triple( - subject, - predicate, - object, - valid_from=valid_from, - valid_to=valid_to, - source_closet=source_closet, - source_file=source_file, - source_drawer_id=source_drawer_id, + triple_id = _call_kg( + lambda kg: kg.add_triple( + subject, + predicate, + object, + valid_from=valid_from, + valid_to=valid_to, + source_closet=source_closet, + source_file=source_file, + source_drawer_id=source_drawer_id, + ) ) return {"success": True, "triple_id": triple_id, "fact": f"{subject} → {predicate} → {object}"} @@ -1165,7 +1200,7 @@ def tool_kg_invalidate(subject: str, predicate: str, object: str, ended: str = N "ended": resolved_ended, }, ) - _get_kg().invalidate(subject, predicate, object, ended=resolved_ended) + _call_kg(lambda kg: kg.invalidate(subject, predicate, object, ended=resolved_ended)) return { "success": True, "fact": f"{subject} → {predicate} → {object}", @@ -1180,13 +1215,13 @@ def tool_kg_timeline(entity: str = None): entity = sanitize_kg_value(entity, "entity") except ValueError as e: return {"error": str(e)} - results = _get_kg().timeline(entity) + results = _call_kg(lambda kg: kg.timeline(entity)) return {"entity": entity or "all", "timeline": results, "count": len(results)} def tool_kg_stats(): """Knowledge graph overview: entities, triples, relationship types.""" - return _get_kg().stats() + return _call_kg(lambda kg: kg.stats()) # ==================== AGENT DIARY ==================== diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 092b70720b..e365afc23c 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -1295,3 +1295,80 @@ def close(self): mcp_server.tool_reconnect() assert mcp_server._kg_by_path == {} + + def test_call_kg_retries_after_concurrent_close(self, monkeypatch): + """A KG closed mid-handler must trigger a one-shot retry with a fresh + instance — not surface a -32000 to the MCP client.""" + import sqlite3 as _sqlite3 + + from mempalace import mcp_server + + path = "/fake/palace/knowledge_graph.sqlite3" + monkeypatch.setattr(mcp_server, "_resolve_kg_path", lambda: path) + + class _ClosedKG: + def query_entity(self, entity, **kwargs): + raise _sqlite3.ProgrammingError("Cannot operate on a closed database") + + class _FreshKG: + def query_entity(self, entity, **kwargs): + return [{"entity": entity}] + + cache = {os.path.abspath(path): _ClosedKG()} + monkeypatch.setattr(mcp_server, "_kg_by_path", cache) + + # Second _get_kg() call (after the cache eviction) constructs a new + # KG. Patch the constructor so we don't open a real sqlite file. + monkeypatch.setattr(mcp_server, "KnowledgeGraph", lambda **_: _FreshKG()) + + result = mcp_server._call_kg(lambda kg: kg.query_entity("Alice")) + assert result == [{"entity": "Alice"}] + # The closed instance must be evicted; the fresh one must be cached. + assert isinstance(cache[os.path.abspath(path)], _FreshKG) + + def test_call_kg_does_not_retry_on_other_errors(self, monkeypatch): + """Non-ProgrammingError exceptions must propagate without retry — + we don't want the retry guard masking real bugs.""" + from mempalace import mcp_server + + path = "/fake/palace/knowledge_graph.sqlite3" + monkeypatch.setattr(mcp_server, "_resolve_kg_path", lambda: path) + + calls = {"count": 0} + + class _FailingKG: + def query_entity(self, entity, **kwargs): + calls["count"] += 1 + raise ValueError("bad input") + + monkeypatch.setattr(mcp_server, "_kg_by_path", {os.path.abspath(path): _FailingKG()}) + monkeypatch.setattr(mcp_server, "KnowledgeGraph", lambda **_: _FailingKG()) + + with pytest.raises(ValueError, match="bad input"): + mcp_server._call_kg(lambda kg: kg.query_entity("Alice")) + assert calls["count"] == 1, "non-ProgrammingError must not trigger retry" + + def test_call_kg_gives_up_after_one_retry(self, monkeypatch): + """If the second attempt also hits a closed DB, give up rather than + loop forever — a sustained close-stream is a different bug.""" + import sqlite3 as _sqlite3 + + from mempalace import mcp_server + + path = "/fake/palace/knowledge_graph.sqlite3" + monkeypatch.setattr(mcp_server, "_resolve_kg_path", lambda: path) + + calls = {"count": 0} + + class _AlwaysClosedKG: + def query_entity(self, entity, **kwargs): + calls["count"] += 1 + raise _sqlite3.ProgrammingError("closed again") + + cache = {} + monkeypatch.setattr(mcp_server, "_kg_by_path", cache) + monkeypatch.setattr(mcp_server, "KnowledgeGraph", lambda **_: _AlwaysClosedKG()) + + with pytest.raises(_sqlite3.ProgrammingError): + mcp_server._call_kg(lambda kg: kg.query_entity("Alice")) + assert calls["count"] == 2, "expected exactly one retry beyond the initial attempt" From 75ad8ae7819ee9eebdf635cc0c3e969b2f9bc73b Mon Sep 17 00:00:00 2001 From: mvalentsev Date: Sun, 3 May 2026 22:04:22 +0500 Subject: [PATCH 27/77] ci: retrigger linux 3.13 (transient onnx download flake) From 285b3b4f2e387c1e8eda865569a2edc400f5c1f1 Mon Sep 17 00:00:00 2001 From: mvalentsev Date: Sun, 3 May 2026 22:25:31 +0500 Subject: [PATCH 28/77] refactor(stdio): extract Windows UTF-8 reconfigure into shared helper Both cli.py and fact_checker.py carried identical 28-line Windows stdio reconfigure helpers; pull the loop into mempalace/_stdio.py so the same machine drives the CLI, the fact_checker --stdin entry point, and the MCP server. The thin per-call-site wrappers stay so existing tests keep importing _reconfigure_stdio_utf8_on_windows from the same module they always have. CLI / fact_checker policy unchanged: stdin=surrogateescape (don't crash on a malformed redirected file), stdout/stderr=replace (don't crash mid-print on a surrogate half round-tripped from a filename). --- mempalace/_stdio.py | 71 +++++++++++++++++++++++++++++++++++++++ mempalace/cli.py | 45 ++++++------------------- mempalace/fact_checker.py | 39 ++++----------------- 3 files changed, 88 insertions(+), 67 deletions(-) create mode 100644 mempalace/_stdio.py diff --git a/mempalace/_stdio.py b/mempalace/_stdio.py new file mode 100644 index 0000000000..13e9509908 --- /dev/null +++ b/mempalace/_stdio.py @@ -0,0 +1,71 @@ +"""Stdio UTF-8 reconfiguration helper for Windows entry points. + +Python on Windows defaults stdio to the system ANSI codepage +(cp1252/cp1251/cp950 depending on locale), which mojibakes UTF-8 input +or output the moment a non-Latin character shows up. Every console +entry point that touches stdio needs to fix this on Windows -- the MCP +server, the CLI, the fact_checker `--stdin` mode -- so the +reconfigure code lives here in one place to keep the per-stream +errors policies aligned across them. + +Per-stream errors policy is caller-chosen: + +* MCP server uses ``strict`` on stdout/stderr because everything written + there is server-controlled JSON-RPC; any encode failure is a real bug + the operator wants loud. +* CLI / fact_checker use ``replace`` on stdout/stderr because they print + verbatim drawer text that may contain surrogate halves round-tripped + from filenames -- ``strict`` would crash mid-print. +* All callers use ``surrogateescape`` on stdin so a malformed byte from + a redirected file or a misbehaving client survives as a lone surrogate + the consumer's parser surfaces, instead of ``UnicodeDecodeError`` + killing the read loop on the first bad byte. +""" + +from __future__ import annotations + +import sys +from typing import Callable, Optional + + +def reconfigure_stdio_utf8_on_windows( + *, + stdin_errors: str = "surrogateescape", + stdout_errors: str = "strict", + stderr_errors: str = "strict", + on_failure: Optional[Callable[[str, BaseException], None]] = None, +) -> None: + """Reconfigure stdio to UTF-8 on Windows. No-op elsewhere. + + Args: + stdin_errors: errors= policy for stdin.reconfigure(). + stdout_errors: errors= policy for stdout.reconfigure(). + stderr_errors: errors= policy for stderr.reconfigure(). + on_failure: optional ``(stream_name, exc) -> None`` callback for + streams whose ``reconfigure`` raises (e.g. Jupyter-replaced + streams that lack the method-shape we expect). Defaults to a + ``WARNING:`` line on the original sys.stderr. + """ + if sys.platform != "win32": + return + + policies = ( + ("stdin", stdin_errors), + ("stdout", stdout_errors), + ("stderr", stderr_errors), + ) + for name, errors in policies: + stream = getattr(sys, name, None) + reconfigure = getattr(stream, "reconfigure", None) + if reconfigure is None: + continue + try: + reconfigure(encoding="utf-8", errors=errors) + except Exception as exc: # noqa: BLE001 -- last-resort guard + if on_failure is not None: + on_failure(name, exc) + else: + print( + f"WARNING: Could not reconfigure {name} to UTF-8: {exc}", + file=sys.stderr, + ) diff --git a/mempalace/cli.py b/mempalace/cli.py index 7052e1f79e..0ab3d0f813 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -938,42 +938,17 @@ def cmd_compress(args): def _reconfigure_stdio_utf8_on_windows(): """Decode stdio as UTF-8 on Windows for the primary `mempalace` CLI. - Without this, Python defaults stdio to the system ANSI codepage - (cp1252/cp1251/cp950 depending on locale). That mojibakes non-ASCII - content piped in (`mempalace search ... < query.txt`) or piped out - (`mempalace search "..." > out.txt`) when verbatim drawer text or - wing/room names contain non-Latin characters. - - Per-stream errors policy: - stdin -- surrogateescape: malformed bytes from a redirected file - survive as lone surrogates instead of crashing the read. - stdout -- replace: ``mempalace search`` prints verbatim drawer - text. A drawer that round-tripped a filename through - surrogateescape can hold a lone surrogate, which would - otherwise raise ``UnicodeEncodeError`` mid-print and - lose the rest of the search result block. - stderr -- replace: same hazard for logger output that quotes - user-supplied path or content. + Thin wrapper around the shared helper in ``mempalace._stdio``. The CLI + overrides stdout/stderr to ``replace`` because ``mempalace search`` + prints verbatim drawer text that may carry surrogate halves + round-tripped from filenames -- ``strict`` would crash mid-print and + lose the rest of the search result block. stdin keeps the default + ``surrogateescape`` so a redirected non-UTF-8 file does not kill the + read on the first bad byte. """ - if sys.platform != "win32": - return - policies = ( - ("stdin", "surrogateescape"), - ("stdout", "replace"), - ("stderr", "replace"), - ) - for name, errors in policies: - stream = getattr(sys, name, None) - reconfigure = getattr(stream, "reconfigure", None) - if reconfigure is None: - continue - try: - reconfigure(encoding="utf-8", errors=errors) - except Exception as exc: - print( - f"WARNING: Could not reconfigure {name} to UTF-8: {exc}", - file=sys.stderr, - ) + from ._stdio import reconfigure_stdio_utf8_on_windows + + reconfigure_stdio_utf8_on_windows(stdout_errors="replace", stderr_errors="replace") def main(): diff --git a/mempalace/fact_checker.py b/mempalace/fact_checker.py index 1844c45df8..403d913605 100644 --- a/mempalace/fact_checker.py +++ b/mempalace/fact_checker.py @@ -306,40 +306,15 @@ def _edit_distance(s1: str, s2: str) -> int: def _reconfigure_stdio_utf8_on_windows(): """Decode --stdin payload as UTF-8 on Windows. - Without this, Python defaults stdio to the system ANSI codepage - (cp1252/cp1251/cp950 depending on locale), which mojibakes - non-ASCII fact text before pattern parsing sees it. - - Per-stream errors policy mirrors the primary CLI helper in - ``mempalace/cli.py``: - stdin -- surrogateescape: malformed input bytes survive as lone - surrogates instead of crashing the read. - stdout -- replace: extracted fact text can include surrogate - halves round-tripped from filenames; replace prevents - a UnicodeEncodeError mid-print. - stderr -- replace: same protection for warning lines. + Thin wrapper around the shared helper in ``mempalace._stdio``. Mirrors + the primary CLI policy: stdout/stderr use ``replace`` because + extracted fact text can include surrogate halves round-tripped from + filenames -- ``strict`` would raise UnicodeEncodeError mid-print. + stdin keeps the default ``surrogateescape``. """ - import sys + from ._stdio import reconfigure_stdio_utf8_on_windows - if sys.platform != "win32": - return - policies = ( - ("stdin", "surrogateescape"), - ("stdout", "replace"), - ("stderr", "replace"), - ) - for name, errors in policies: - stream = getattr(sys, name, None) - reconfigure = getattr(stream, "reconfigure", None) - if reconfigure is None: - continue - try: - reconfigure(encoding="utf-8", errors=errors) - except Exception as exc: - print( - f"WARNING: Could not reconfigure {name} to UTF-8: {exc}", - file=sys.stderr, - ) + reconfigure_stdio_utf8_on_windows(stdout_errors="replace", stderr_errors="replace") if __name__ == "__main__": From f9d939ae1bd5edba6901f0fd3872cff7ab30f790 Mon Sep 17 00:00:00 2001 From: fatkobra <55045047+fatkobra@users.noreply.github.com> Date: Mon, 4 May 2026 06:45:29 +0000 Subject: [PATCH 29/77] fix(storage): quarantine bloated HNSW link payloads --- mempalace/backends/chroma.py | 155 ++++++++++++++++++------------ tests/test_hnsw_payload_health.py | 113 ++++++++++++++++++++++ 2 files changed, 209 insertions(+), 59 deletions(-) create mode 100644 tests/test_hnsw_payload_health.py diff --git a/mempalace/backends/chroma.py b/mempalace/backends/chroma.py index d9b99a412b..02684d223c 100644 --- a/mempalace/backends/chroma.py +++ b/mempalace/backends/chroma.py @@ -29,6 +29,51 @@ _OPTIONAL_OPERATORS = frozenset({"$gt", "$gte", "$lt", "$lte"}) _SUPPORTED_OPERATORS = _REQUIRED_OPERATORS | _OPTIONAL_OPERATORS +# A healthy HNSW payload should keep link_lists.bin proportional to +# data_level0.bin. When link_lists.bin grows orders of magnitude larger than +# data_level0.bin, Chroma/HNSW can segfault while opening the segment even if +# index_metadata.pickle is structurally valid. +# +# The report in #1218 showed ratios above 300x, while healthy snapshots were far below 1x. +# Treat only >10x as corruption so normal flush lag or small segments do not get +# quarantined. +_HNSW_LINK_TO_DATA_MAX_RATIO = 10.0 + + +def _hnsw_link_to_data_ratio(seg_dir: str) -> Optional[float]: + """Return link_lists.bin / data_level0.bin size ratio for a segment. + + ``None`` means the ratio is not meaningful, usually because one file is + missing or data_level0.bin is empty. ``float("inf")`` means the files were + present but could not be statted safely, which should be treated as + suspicious by callers. + """ + + link_path = os.path.join(seg_dir, "link_lists.bin") + data_path = os.path.join(seg_dir, "data_level0.bin") + + if not (os.path.isfile(link_path) and os.path.isfile(data_path)): + return None + + try: + data_size = os.path.getsize(data_path) + link_size = os.path.getsize(link_path) + except OSError: + return float("inf") + + if data_size <= 0: + return None + + return link_size / data_size + + +def _hnsw_payload_appears_sane(seg_dir: str) -> bool: + """Return False when HNSW payload files are structurally implausible.""" + + ratio = _hnsw_link_to_data_ratio(seg_dir) + return ratio is None or ratio <= _HNSW_LINK_TO_DATA_MAX_RATIO + + # HNSW tuning to prevent link_lists.bin bloat on large mines (#344). # # With default params (batch_size=100, sync_threshold=1000, initial capacity @@ -106,6 +151,9 @@ def _segment_appears_healthy(seg_dir: str) -> bool: files and quarantine_stale_hnsw would conservatively rename them out of the way (lazy rebuild on next open recovers). """ + if not _hnsw_payload_appears_sane(seg_dir): + return False + meta_path = os.path.join(seg_dir, "index_metadata.pickle") if not os.path.isfile(meta_path): # No metadata file yet — segment hasn't flushed (fresh / empty). @@ -127,64 +175,35 @@ def _segment_appears_healthy(seg_dir: str) -> bool: def quarantine_stale_hnsw(palace_path: str, stale_seconds: float = 300.0) -> list[str]: - """Rename HNSW segment dirs that are both stale-by-mtime AND fail an - integrity sniff-test. - - Catches the segfault failure mode from #823 (semantic search stale - after ``add_drawer``), observed at neo-cortex-mcp#2 (SIGSEGV on - ``count()`` with chromadb 1.5.5), and acknowledged as by-design at - chroma-core/chroma#2594. Renaming a corrupt segment lets chromadb - rebuild lazily on next open instead of segfaulting. - - Two-stage check: - - 1. **mtime gate.** If ``chroma.sqlite3`` is less than - ``stale_seconds`` newer than the segment's ``data_level0.bin``, - skip — chromadb is in normal write-path territory. - - 2. **Integrity gate** (``_segment_appears_healthy``). Even when the - mtime gap exceeds the threshold, a segment whose - ``index_metadata.pickle`` passes a format sniff-test is healthy: - chromadb 1.5.x flushes HNSW state asynchronously and a clean - shutdown does NOT force-flush, so the on-disk HNSW is *always* - somewhat older than ``chroma.sqlite3``. Production observation - (2026-04-26 disks daemon): three of three segments quarantined - on every cold start, with 538-557s gaps, leaving the 151K-drawer - palace with vector_ranked=0 until rebuild. Renaming a healthy - segment based on mtime alone destroys a valid index — chromadb - creates an empty replacement, orphaning every drawer in sqlite - from vector recall until the operator runs ``mempalace repair - --mode rebuild`` (15+ min on a 151K palace). - - Only segments that pass stage 1 (suspiciously stale) AND fail stage - 2 (metadata file truncated, zero-filled, or absent-with-data) are - renamed to ``.drift-``. The original directory is - renamed, not deleted, so recovery remains possible if the heuristic - misfires. - - The default threshold (5 min) is advisory under daemon-strict; the - integrity gate is what actually distinguishes corruption from flush - lag. The threshold still matters for the cross-machine replication - case (#823), where it bounds how stale a Syncthing-replicated - segment can be before we look harder at it. - - Args: - palace_path: path to the palace directory containing ``chroma.sqlite3`` - stale_seconds: minimum mtime gap to *consider* a segment for quarantine - - Returns: - List of paths that were quarantined (empty if nothing actually - looked corrupt). + """Rename HNSW segment dirs that look unsafe to open. + + This catches two classes of HNSW corruption before ChromaDB opens the + native segment reader: + + 1. stale-by-mtime segments whose ``index_metadata.pickle`` fails the + existing format sniff-test; + 2. structurally impossible HNSW payloads where ``link_lists.bin`` is much + larger than ``data_level0.bin``. + + The second check is intentionally not gated by mtime. A segment with a + 300x link/data ratio is unsafe regardless of whether its mtime is recent; + letting Chroma open it can SIGSEGV before Python fallback code runs. + + The original directory is renamed, not deleted, so recovery remains + possible if the heuristic ever misfires. """ + db_path = os.path.join(palace_path, "chroma.sqlite3") if not os.path.isfile(db_path): return [] + try: sqlite_mtime = os.path.getmtime(db_path) except OSError: return [] moved: list[str] = [] + try: entries = os.listdir(palace_path) except OSError: @@ -193,29 +212,34 @@ def quarantine_stale_hnsw(palace_path: str, stale_seconds: float = 300.0) -> lis for name in entries: if "-" not in name or name.startswith(".") or ".drift-" in name: continue + seg_dir = os.path.join(palace_path, name) if not os.path.isdir(seg_dir): continue + hnsw_bin = os.path.join(seg_dir, "data_level0.bin") if not os.path.isfile(hnsw_bin): continue + try: hnsw_mtime = os.path.getmtime(hnsw_bin) except OSError: continue - if sqlite_mtime - hnsw_mtime < stale_seconds: + + payload_ratio = _hnsw_link_to_data_ratio(seg_dir) + payload_corrupt = payload_ratio is not None and payload_ratio > _HNSW_LINK_TO_DATA_MAX_RATIO + + if not payload_corrupt and sqlite_mtime - hnsw_mtime < stale_seconds: continue - # Stage 2: integrity gate. mtime drift is necessary but not - # sufficient — chromadb's async flush makes drift the steady- - # state condition. A healthy segment metadata file proves - # chromadb can open the segment without segfault; don't - # quarantine a healthy index. - if _segment_appears_healthy(seg_dir): + # Stage 2: integrity gate. Mtime drift alone is not corruption because + # Chroma flushes HNSW asynchronously. A healthy metadata file proves the + # ordinary stale-by-mtime case is just flush lag. + if not payload_corrupt and _segment_appears_healthy(seg_dir): logger.info( "HNSW mtime gap %.0fs on %s exceeds threshold but segment " - "metadata file is intact — flush-lag, not corruption. " - "Leaving in place.", + "metadata and payload size are intact — flush-lag, not " + "corruption. Leaving in place.", sqlite_mtime - hnsw_mtime, seg_dir, ) @@ -223,17 +247,30 @@ def quarantine_stale_hnsw(palace_path: str, stale_seconds: float = 300.0) -> lis stamp = _dt.datetime.now().strftime("%Y%m%d-%H%M%S") target = f"{seg_dir}.drift-{stamp}" + + if payload_corrupt: + reason = ( + f"link_lists.bin/data_level0.bin ratio {payload_ratio:.1f}x " + f"exceeds {_HNSW_LINK_TO_DATA_MAX_RATIO:.1f}x" + ) + else: + reason = ( + f"sqlite {sqlite_mtime - hnsw_mtime:.0f}s newer than HNSW " + "and integrity check failed" + ) + try: os.rename(seg_dir, target) moved.append(target) logger.warning( - "Quarantined corrupt HNSW segment %s (sqlite %.0fs newer than HNSW, integrity check failed); renamed to %s", + "Quarantined corrupt HNSW segment %s (%s); renamed to %s", seg_dir, - sqlite_mtime - hnsw_mtime, + reason, target, ) except OSError: logger.exception("Failed to quarantine corrupt HNSW segment %s", seg_dir) + return moved diff --git a/tests/test_hnsw_payload_health.py b/tests/test_hnsw_payload_health.py new file mode 100644 index 0000000000..0af440a8e9 --- /dev/null +++ b/tests/test_hnsw_payload_health.py @@ -0,0 +1,113 @@ +import os +from pathlib import Path + +from mempalace.backends.chroma import ( + _HNSW_LINK_TO_DATA_MAX_RATIO, + _hnsw_link_to_data_ratio, + _segment_appears_healthy, + quarantine_stale_hnsw, +) + + +def _write_segment( + seg_dir: Path, + *, + data_size: int = 100, + link_size: int = 100, + write_metadata: bool = True, +) -> None: + seg_dir.mkdir(parents=True, exist_ok=True) + (seg_dir / "data_level0.bin").write_bytes(b"\0" * data_size) + (seg_dir / "link_lists.bin").write_bytes(b"\0" * link_size) + + if write_metadata: + # Enough bytes to pass the existing pickle envelope sniff-test: + # starts with pickle protocol marker 0x80 and ends with STOP 0x2e. + (seg_dir / "index_metadata.pickle").write_bytes(b"\x80" + b"x" * 16 + b"\x2e") + + +def test_hnsw_link_to_data_ratio_reports_payload_size_ratio(tmp_path): + seg_dir = tmp_path / "11111111-2222-3333-4444-555555555555" + _write_segment(seg_dir, data_size=100, link_size=250) + + assert _hnsw_link_to_data_ratio(str(seg_dir)) == 2.5 + + +def test_segment_health_rejects_exploded_link_lists_even_with_valid_pickle(tmp_path): + seg_dir = tmp_path / "11111111-2222-3333-4444-555555555555" + _write_segment( + seg_dir, + data_size=100, + link_size=int(100 * (_HNSW_LINK_TO_DATA_MAX_RATIO + 1)), + write_metadata=True, + ) + + assert not _segment_appears_healthy(str(seg_dir)) + + +def test_segment_health_keeps_reasonable_payload_with_valid_pickle(tmp_path): + seg_dir = tmp_path / "11111111-2222-3333-4444-555555555555" + _write_segment( + seg_dir, + data_size=100, + link_size=int(100 * _HNSW_LINK_TO_DATA_MAX_RATIO), + write_metadata=True, + ) + + assert _segment_appears_healthy(str(seg_dir)) + + +def test_quarantine_catches_link_bloat_without_mtime_drift(tmp_path): + palace = tmp_path / "palace" + palace.mkdir() + + db_path = palace / "chroma.sqlite3" + db_path.write_text("sqlite placeholder") + + seg_dir = palace / "11111111-2222-3333-4444-555555555555" + _write_segment( + seg_dir, + data_size=100, + link_size=int(100 * (_HNSW_LINK_TO_DATA_MAX_RATIO + 1)), + write_metadata=True, + ) + + # Make sqlite and HNSW mtimes identical. The old mtime-only gate would + # skip this segment even though the payload is structurally corrupt. + same_time = 1_700_000_000 + os.utime(db_path, (same_time, same_time)) + os.utime(seg_dir / "data_level0.bin", (same_time, same_time)) + + moved = quarantine_stale_hnsw(str(palace), stale_seconds=999_999) + + assert len(moved) == 1 + assert not seg_dir.exists() + + moved_path = Path(moved[0]) + assert moved_path.exists() + assert moved_path.name.startswith("11111111-2222-3333-4444-555555555555.drift-") + + +def test_quarantine_leaves_reasonable_payload_in_place(tmp_path): + palace = tmp_path / "palace" + palace.mkdir() + + db_path = palace / "chroma.sqlite3" + db_path.write_text("sqlite placeholder") + + seg_dir = palace / "11111111-2222-3333-4444-555555555555" + _write_segment( + seg_dir, + data_size=100, + link_size=100, + write_metadata=True, + ) + + same_time = 1_700_000_000 + os.utime(db_path, (same_time, same_time)) + os.utime(seg_dir / "data_level0.bin", (same_time, same_time)) + + moved = quarantine_stale_hnsw(str(palace), stale_seconds=999_999) + + assert moved == [] + assert seg_dir.exists() From 4f36145c2e51aa53c705c67a3f692174230bbcf9 Mon Sep 17 00:00:00 2001 From: Arnold Wender Date: Sun, 26 Apr 2026 13:01:55 +0200 Subject: [PATCH 30/77] fix(entity_registry): atomic write to prevent partial corruption on crash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EntityRegistry.save() called Path.write_text() directly, which truncates the target file and then writes — so a crash mid-write (power loss, OOM, filesystem-full mid-flush) leaves an empty or half-written entity_registry.json. The whole people/projects map is lost; the system falls back to an empty registry on next load. Switch to the standard atomic-write pattern: serialize to a sibling .tmp file in the same directory (so os.replace stays on one filesystem), fsync, chmod 0o600, then os.replace over the target. The replace is atomic on POSIX and Windows, so any crash leaves the previous registry intact instead of a truncated file. Tests cover: no leftover .tmp on success, and previous content preserved when os.replace itself raises mid-save. --- mempalace/entity_registry.py | 15 ++++++++++-- tests/test_entity_registry.py | 46 +++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/mempalace/entity_registry.py b/mempalace/entity_registry.py index 78d8a8b20c..d77b6c1f84 100644 --- a/mempalace/entity_registry.py +++ b/mempalace/entity_registry.py @@ -16,6 +16,7 @@ """ import json +import os import re import urllib.request import urllib.parse @@ -320,11 +321,21 @@ def save(self): self._path.parent.chmod(0o700) except (OSError, NotImplementedError): pass - self._path.write_text(json.dumps(self._data, indent=2), encoding="utf-8") + # Atomic write: serialize to a sibling temp file in the same dir + # (so os.replace stays on one filesystem), fsync, then rename over + # the target. A crash mid-write leaves the previous registry intact + # instead of a half-written file or an empty file from the truncate. + payload = json.dumps(self._data, indent=2) + tmp_path = self._path.with_name(self._path.name + ".tmp") + with open(tmp_path, "w", encoding="utf-8") as f: + f.write(payload) + f.flush() + os.fsync(f.fileno()) try: - self._path.chmod(0o600) + tmp_path.chmod(0o600) except (OSError, NotImplementedError): pass + os.replace(tmp_path, self._path) @staticmethod def _empty() -> dict: diff --git a/tests/test_entity_registry.py b/tests/test_entity_registry.py index c857a071b4..a5f237c983 100644 --- a/tests/test_entity_registry.py +++ b/tests/test_entity_registry.py @@ -2,6 +2,8 @@ from unittest.mock import patch +import pytest + from mempalace.entity_registry import ( COMMON_ENGLISH_WORDS, PERSON_CONTEXT_PATTERNS, @@ -71,6 +73,50 @@ def test_save_creates_file(tmp_path): assert (tmp_path / "entity_registry.json").exists() +def test_save_is_atomic_does_not_leave_tmp(tmp_path): + # Atomic write must not leave the .tmp sidecar file after a successful save. + registry = EntityRegistry.load(config_dir=tmp_path) + registry.save() + leftover = list(tmp_path.glob("entity_registry.json.tmp*")) + assert leftover == [], f"atomic write leaked tmp file(s): {leftover}" + + +def test_save_preserves_previous_on_serialization_failure(tmp_path, monkeypatch): + # If serialization fails mid-write, the previous registry must remain + # intact — this is the whole point of atomic write vs truncating in place. + registry = EntityRegistry.load(config_dir=tmp_path) + registry.seed( + mode="personal", + people=[{"name": "Alice", "relationship": "friend", "context": "personal"}], + projects=[], + ) + registry.save() + target = tmp_path / "entity_registry.json" + original = target.read_text(encoding="utf-8") + + # Force os.replace to raise — simulates filesystem full / permission flip + # AFTER the temp file is written but BEFORE the rename completes. + import os as _os + + real_replace = _os.replace + + def boom(src, dst): + raise OSError("simulated rename failure") + + monkeypatch.setattr(_os, "replace", boom) + with pytest.raises(OSError): + registry.seed( + mode="personal", + people=[{"name": "Bob", "relationship": "friend", "context": "personal"}], + projects=[], + ) + registry.save() + + # Restore os.replace before reading so the assertion can rely on it. + monkeypatch.setattr(_os, "replace", real_replace) + assert target.read_text(encoding="utf-8") == original + + # ── seed ──────────────────────────────────────────────────────────────── From 2e441d17a22f9e14dcb9d8576551ea867e56cda8 Mon Sep 17 00:00:00 2001 From: Arnold Wender Date: Tue, 28 Apr 2026 09:43:27 +0200 Subject: [PATCH 31/77] fix(entity_registry): fsync parent dir after rename for ext4 durability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without this, on ext4 (and similar) filesystems the rename ack does not guarantee durability across power loss — a crash can revert to a state where the temp file is present and the target is at the old version. Suggested by @jphein on #1215. --- mempalace/entity_registry.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/mempalace/entity_registry.py b/mempalace/entity_registry.py index d77b6c1f84..c8ac517249 100644 --- a/mempalace/entity_registry.py +++ b/mempalace/entity_registry.py @@ -336,6 +336,20 @@ def save(self): except (OSError, NotImplementedError): pass os.replace(tmp_path, self._path) + # On ext4 (and similar) the rename's durability across power loss + # requires an additional fsync on the parent directory. Without it, + # the kernel can ack the rename and a crash reverts to the state + # where the temp file is present and the target is at the old version. + try: + dir_fd = os.open(str(self._path.parent), os.O_RDONLY) + try: + os.fsync(dir_fd) + finally: + os.close(dir_fd) + except OSError: + # Windows and some special filesystems reject directory fds — they + # have different durability semantics on rename anyway. + pass @staticmethod def _empty() -> dict: From eff844b1680510c2ab38456bc166bfe42c57effa Mon Sep 17 00:00:00 2001 From: fatkobra <55045047+fatkobra@users.noreply.github.com> Date: Mon, 4 May 2026 10:34:22 +0000 Subject: [PATCH 32/77] fix(storage): quarantine partial HNSW flush without metadata --- mempalace/backends/chroma.py | 39 ++++++++++++++++++-------- tests/test_backends.py | 54 ++++++++++++++++++++++++++++++++++-- 2 files changed, 78 insertions(+), 15 deletions(-) diff --git a/mempalace/backends/chroma.py b/mempalace/backends/chroma.py index d9b99a412b..c611af51aa 100644 --- a/mempalace/backends/chroma.py +++ b/mempalace/backends/chroma.py @@ -54,6 +54,13 @@ "hnsw:sync_threshold": 50_000, } +# Missing index_metadata.pickle is normal only while a segment is still fresh +# or effectively empty. Once data_level0.bin has non-trivial payload, a +# missing metadata pickle means the segment was interrupted after writing HNSW +# data but before writing its metadata. Letting Chroma open that shape can +# segfault or hang in native HNSW code. +_HNSW_MISSING_METADATA_DATA_FLOOR = 1024 + def _validate_where(where: Optional[dict]) -> None: """Scan a where-clause for unknown operators and raise ``UnsupportedFilterError``. @@ -84,16 +91,13 @@ def _segment_appears_healthy(seg_dir: str) -> bool: parsing it. ChromaDB writes that file after a successful HNSW flush; a complete write starts with byte ``0x80`` and ends with byte ``0x2e`` (the protocol/terminator byte sequence chromadb serializes - with). If both bytes are present and the file is non-trivially sized, - chromadb will load the segment cleanly even when its on-disk mtime - trails ``chroma.sqlite3`` — which is the *steady state* under - chromadb 1.5.x's async batched flush, not corruption. + with). - A missing metadata file is treated as "fresh / never-flushed" and - considered healthy. Renaming an empty dir orphans nothing, and a - real corruption case manifests as a present-but-malformed file or a - chromadb load error caught downstream by palace-daemon's - ``_auto_repair`` retry path. + Missing metadata is healthy only while the segment still looks fresh or + empty. If ``data_level0.bin`` already has non-trivial payload but + ``index_metadata.pickle`` is missing, the segment is partially flushed: + Chroma wrote vector data without the metadata it needs to reopen the + HNSW reader safely. Deliberately format-sniffs only; never deserializes. Deserialization can execute arbitrary code, and the byte-sniff is sufficient to @@ -104,13 +108,24 @@ def _segment_appears_healthy(seg_dir: str) -> bool: chromadb writes today; if a future chromadb version emits protocol 0/1 segments, this check would start returning False on healthy files and quarantine_stale_hnsw would conservatively rename them - out of the way (lazy rebuild on next open recovers). + out of the way. """ + meta_path = os.path.join(seg_dir, "index_metadata.pickle") if not os.path.isfile(meta_path): - # No metadata file yet — segment hasn't flushed (fresh / empty). - # Renaming would orphan nothing; consider healthy. + data_path = os.path.join(seg_dir, "data_level0.bin") + try: + if ( + os.path.isfile(data_path) + and os.path.getsize(data_path) > _HNSW_MISSING_METADATA_DATA_FLOOR + ): + return False + except OSError: + return False + + # No metadata and no meaningful vector payload yet: fresh/empty segment. return True + try: size = os.path.getsize(meta_path) # A real chromadb metadata file is at least tens of bytes; a diff --git a/tests/test_backends.py b/tests/test_backends.py index 4ddfe1207a..3a8392b772 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -16,8 +16,10 @@ from mempalace.backends.chroma import ( ChromaBackend, ChromaCollection, + _HNSW_MISSING_METADATA_DATA_FLOOR, _fix_blob_seq_ids, _pin_hnsw_threads, + _segment_appears_healthy, quarantine_stale_hnsw, ) @@ -636,9 +638,9 @@ def test_quarantine_stale_hnsw_leaves_healthy_segment_with_drift_alone(tmp_path) assert seg.exists() -def test_quarantine_stale_hnsw_leaves_segment_without_metadata_alone(tmp_path): - """Segment with no metadata file is treated as fresh / never-flushed - and not quarantined — renaming an empty dir orphans nothing.""" +def test_quarantine_stale_hnsw_leaves_empty_segment_without_metadata_alone(tmp_path): + """Missing metadata is okay only when the segment has no meaningful data yet.""" + now = 1_700_000_000.0 palace, seg = _make_palace_with_segment( tmp_path, @@ -646,11 +648,57 @@ def test_quarantine_stale_hnsw_leaves_segment_without_metadata_alone(tmp_path): sqlite_mtime=now, meta_bytes=None, ) + moved = quarantine_stale_hnsw(str(palace), stale_seconds=3600.0) + assert moved == [] assert seg.exists() +def test_segment_without_metadata_but_with_nontrivial_data_is_unhealthy(tmp_path): + """Data without index_metadata.pickle is a partial flush, not a fresh segment.""" + + seg = tmp_path / "abcd-1234-5678" + seg.mkdir() + (seg / "data_level0.bin").write_bytes(b"\0" * (_HNSW_MISSING_METADATA_DATA_FLOOR + 1)) + + assert not _segment_appears_healthy(str(seg)) + + +def test_segment_without_metadata_and_tiny_data_is_still_treated_as_fresh(tmp_path): + """Tiny data payloads can occur before metadata has flushed; leave them alone.""" + + seg = tmp_path / "abcd-1234-5678" + seg.mkdir() + (seg / "data_level0.bin").write_bytes(b"\0" * _HNSW_MISSING_METADATA_DATA_FLOOR) + + assert _segment_appears_healthy(str(seg)) + + +def test_quarantine_stale_hnsw_renames_missing_metadata_with_nontrivial_data(tmp_path): + """Regression for #1274: missing pickle + non-trivial data must quarantine.""" + + now = 1_700_000_000.0 + palace, seg = _make_palace_with_segment( + tmp_path, + hnsw_mtime=now - 7200, + sqlite_mtime=now, + meta_bytes=None, + ) + (seg / "data_level0.bin").write_bytes(b"\0" * (_HNSW_MISSING_METADATA_DATA_FLOOR + 1)) + os.utime(seg / "data_level0.bin", (now - 7200, now - 7200)) + + moved = quarantine_stale_hnsw(str(palace), stale_seconds=3600.0) + + assert len(moved) == 1 + assert ".drift-" in moved[0] + assert not seg.exists() + + drift_dirs = [p for p in palace.iterdir() if ".drift-" in p.name] + assert len(drift_dirs) == 1 + assert (drift_dirs[0] / "data_level0.bin").exists() + + def test_quarantine_stale_hnsw_renames_truncated_metadata(tmp_path): """Segment with a truncated (under-floor-size) metadata file is quarantined — shape of a partial-flush during process kill.""" From 37e7d394b8cc5ada0508dff62feb54cfb7cb0716 Mon Sep 17 00:00:00 2001 From: fatkobra <55045047+fatkobra@users.noreply.github.com> Date: Tue, 5 May 2026 07:22:10 +0000 Subject: [PATCH 33/77] fix(repair): preflight poisoned max_seq_id --- mempalace/cli.py | 19 +++++++++++-- mempalace/repair.py | 59 ++++++++++++++++++++++++++++++++++++++- tests/test_repair.py | 66 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 4 deletions(-) diff --git a/mempalace/cli.py b/mempalace/cli.py index f2606a4368..103c4ae299 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -654,7 +654,11 @@ def cmd_repair(args): import shutil from .backends.chroma import ChromaBackend from .migrate import confirm_destructive_action, contains_palace_database - from .repair import TruncationDetected, check_extraction_safety + from .repair import ( + TruncationDetected, + check_extraction_safety, + maybe_repair_poisoned_max_seq_id_before_rebuild, + ) palace_path = os.path.abspath( os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path @@ -679,11 +683,20 @@ def cmd_repair(args): print(f"\n No palace found at {palace_path}") return if not contains_palace_database(palace_path): - print(f"\n No palace database found at {db_path}") + print(f"\n No palace database found at {db_path}") + return + + preflight = maybe_repair_poisoned_max_seq_id_before_rebuild( + palace_path, + backup=getattr(args, "backup", True), + dry_run=getattr(args, "dry_run", False), + assume_yes=getattr(args, "yes", False), + ) + if preflight is not None: return print(f"\n{'=' * 55}") - print(" MemPalace Repair") + print(" MemPalace Repair") print(f"{'=' * 55}\n") print(f" Palace: {palace_path}") diff --git a/mempalace/repair.py b/mempalace/repair.py index 1cd1556975..09970debfb 100644 --- a/mempalace/repair.py +++ b/mempalace/repair.py @@ -330,6 +330,56 @@ def sqlite_drawer_count(palace_path: str) -> "int | None": return None +def maybe_repair_poisoned_max_seq_id_before_rebuild( + palace_path: str, + *, + backup: bool = True, + dry_run: bool = False, + assume_yes: bool = False, +) -> "dict | None": + """Run non-destructive max_seq_id repair before a rebuild if needed. + + A poisoned ``max_seq_id`` row can make Chroma believe it has already + consumed every row in ``embeddings_queue``. Writes then report success + because they land in the queue, but they never become visible in + ``embeddings``. + + If this precise corruption is present, do the narrow bookmark repair and + stop instead of continuing into the legacy rebuild path. The rebuild path + extracts only already-visible embeddings and can discard queued writes. + """ + + db_path = os.path.join(palace_path, "chroma.sqlite3") + if not os.path.isfile(db_path): + return None + + try: + poisoned = _detect_poisoned_max_seq_ids(db_path) + except Exception: + return None + + if not poisoned: + return None + + print("\n Detected poisoned max_seq_id rows before repair rebuild.") + print( + " This can make writes report success while embeddings_queue grows " + "and embeddings stay static." + ) + print(" Running the non-destructive max_seq_id repair instead of rebuilding " "the collection.") + print( + " Queued writes remain in chroma.sqlite3 for Chroma to drain after " + "the bookmark is unpoisoned." + ) + + return repair_max_seq_id( + palace_path, + backup=backup, + dry_run=dry_run, + assume_yes=assume_yes, + ) + + def rebuild_index(palace_path=None, confirm_truncation_ok: bool = False): """Rebuild the HNSW index from scratch. @@ -353,7 +403,14 @@ def rebuild_index(palace_path=None, confirm_truncation_ok: bool = False): print(f"\n{'=' * 55}") print(" MemPalace Repair — Index Rebuild") print(f"{'=' * 55}\n") - print(f" Palace: {palace_path}") + print(f" Palace: {palace_path}") + + preflight = maybe_repair_poisoned_max_seq_id_before_rebuild( + palace_path, + assume_yes=True, + ) + if preflight is not None: + return backend = ChromaBackend() try: diff --git a/tests/test_repair.py b/tests/test_repair.py index bc770ddfbe..6f1802f773 100644 --- a/tests/test_repair.py +++ b/tests/test_repair.py @@ -682,3 +682,69 @@ def flaky_detect(*args, **kwargs): # A backup file is still present — caller can roll back from it. leftover = [fn for fn in os.listdir(palace) if "max-seq-id-backup-" in fn] assert leftover + + +def test_max_seq_id_preflight_preserves_embeddings_queue(tmp_path): + """#1295: default repair preflight must not drop queued writes.""" + + palace = str(tmp_path / "palace") + seg = _seed_poisoned_max_seq_id( + palace, + drawers_meta_max=102, + closets_meta_max=11, + ) + db_path = os.path.join(palace, "chroma.sqlite3") + + with sqlite3.connect(db_path) as conn: + conn.executemany( + "INSERT INTO embeddings_queue(seq_id, topic, id) VALUES (?, ?, ?)", + [ + (seq_id, "persistent://default/default/mempalace_drawers", f"queued-{seq_id}") + for seq_id in range(103, 123) + ], + ) + conn.commit() + + result = repair.maybe_repair_poisoned_max_seq_id_before_rebuild( + palace, + assume_yes=True, + ) + + assert result is not None + assert result["segment_repaired"] + + with sqlite3.connect(db_path) as conn: + max_seq_rows = dict(conn.execute("SELECT segment_id, seq_id FROM max_seq_id")) + queue_count = conn.execute("SELECT COUNT(*) FROM embeddings_queue").fetchone()[0] + + assert max_seq_rows[seg["drawers_vec"]] == seg["drawers_meta_max"] + assert max_seq_rows[seg["drawers_meta"]] == seg["drawers_meta_max"] + assert max_seq_rows[seg["closets_vec"]] == seg["closets_meta_max"] + assert max_seq_rows[seg["closets_meta"]] == seg["closets_meta_max"] + + # The old legacy rebuild path can discard queued writes. The preflight + # repair must leave them on disk for Chroma to drain after the bookmark is + # unpoisoned. + assert queue_count == 20 + + +def test_rebuild_index_repairs_poisoned_max_seq_id_before_collection_rebuild(tmp_path, capsys): + """A poisoned bookmark should short-circuit before the legacy rebuild path.""" + + palace = str(tmp_path / "palace") + _seed_poisoned_max_seq_id(palace) + + with patch("mempalace.repair.ChromaBackend") as mock_backend: + repair.rebuild_index(palace) + + out = capsys.readouterr().out + backend = mock_backend.return_value + + # repair_max_seq_id may instantiate ChromaBackend to close cached clients + # after editing sqlite directly. That is safe. The important thing is that + # rebuild_index must not continue into the legacy Chroma collection read / + # count / rebuild path after the max_seq_id preflight handles the issue. + backend.get_collection.assert_not_called() + + assert "Detected poisoned max_seq_id rows" in out + assert "non-destructive max_seq_id repair" in out From bb40a529fde600ba01b49e6ead06820968bba446 Mon Sep 17 00:00:00 2001 From: fatkobra <55045047+fatkobra@users.noreply.github.com> Date: Tue, 5 May 2026 09:01:05 +0000 Subject: [PATCH 34/77] fix(migrate): verify write roundtrip before bailout --- mempalace/migrate.py | 73 +++++++++++++++++++++++++++--- tests/test_migrate.py | 101 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 167 insertions(+), 7 deletions(-) diff --git a/mempalace/migrate.py b/mempalace/migrate.py index 76aa054faa..5b74591b32 100644 --- a/mempalace/migrate.py +++ b/mempalace/migrate.py @@ -22,6 +22,7 @@ import os import shutil import sqlite3 +import uuid from collections import defaultdict from datetime import datetime @@ -155,6 +156,55 @@ def confirm_destructive_action( return True +def _result_ids(result) -> list: + """Return ids from either the backend typed result or raw Chroma dict.""" + + if isinstance(result, dict): + return list(result.get("ids") or []) + + return list(getattr(result, "ids", []) or []) + + +def collection_write_roundtrip_works(col) -> bool: + """Return True only if the collection can upsert, read, and delete. + + Some ChromaDB 0.6.x -> 1.5.x migrated collections remain readable while + writes and deletes silently no-op. A plain ``count()`` probe misses that + failure mode, so migrate must verify an actual write round-trip before + deciding that no rebuild is needed. + """ + + probe_id = f"_mempalace_migrate_probe_{uuid.uuid4().hex}" + probe_doc = "mempalace migrate write round-trip probe" + probe_meta = { + "wing": "_mempalace_probe", + "room": "_mempalace_probe", + "source_file": "mempalace_migrate_probe", + "chunk_index": 0, + } + + try: + col.upsert( + ids=[probe_id], + documents=[probe_doc], + metadatas=[probe_meta], + ) + + after_upsert = col.get(ids=[probe_id], include=[]) + if probe_id not in _result_ids(after_upsert): + return False + + col.delete(ids=[probe_id]) + + after_delete = col.get(ids=[probe_id], include=[]) + if probe_id in _result_ids(after_delete): + return False + + return True + except Exception: + return False + + def migrate(palace_path: str, dry_run: bool = False, confirm: bool = False): """Migrate a palace to the currently installed ChromaDB version.""" from .backends.chroma import ChromaBackend @@ -179,16 +229,27 @@ def migrate(palace_path: str, dry_run: bool = False, confirm: bool = False): print(f" Source: ChromaDB {source_version}") print(f" Target: ChromaDB {target_version}") - # Try reading with current chromadb first + # Try reading and writing with current chromadb first. + # + # A plain count() is not enough: some 0.6.x -> 1.5.x migrated collections + # are readable but silently drop upsert/delete operations. In that state, + # migrate must rebuild from SQLite instead of returning "No migration needed." try: col = ChromaBackend().get_collection(palace_path, "mempalace_drawers") count = col.count() - print(f"\n Palace is already readable by chromadb {target_version}.") - print(f" {count} drawers found. No migration needed.") - return True + + if collection_write_roundtrip_works(col): + print(f"\n Palace is already readable and writable by chromadb {target_version}.") + print(f" {count} drawers found. No migration needed.") + return True + + print( + f"\n Palace is readable by chromadb {target_version}, but write/delete verification failed." + ) + print(" Rebuilding from SQLite to restore native write/delete behavior...") except Exception: - print(f"\n Palace is NOT readable by chromadb {target_version}.") - print(" Extracting from SQLite directly...") + print(f"\n Palace is NOT readable by chromadb {target_version}.") + print(" Extracting from SQLite directly...") # Extract all drawers via raw SQL drawers = extract_drawers_from_sqlite(db_path) diff --git a/tests/test_migrate.py b/tests/test_migrate.py index 4701048afd..ba12ff528b 100644 --- a/tests/test_migrate.py +++ b/tests/test_migrate.py @@ -4,7 +4,7 @@ from types import SimpleNamespace from unittest.mock import MagicMock, patch -from mempalace.migrate import _restore_stale_palace, migrate +from mempalace.migrate import collection_write_roundtrip_works, _restore_stale_palace, migrate def test_migrate_requires_palace_database(tmp_path, capsys): @@ -101,3 +101,102 @@ def test_restore_stale_palace_logs_and_swallows_on_failure(tmp_path, capsys): assert "CRITICAL" in out assert os.fspath(palace_path) in out assert os.fspath(stale_path) in out + + +class _FakeGetResult: + def __init__(self, ids): + self.ids = ids + + +class _WritableFakeCollection: + def __init__(self): + self.ids = set() + self.deleted = [] + + def upsert(self, *, ids, documents, metadatas): + self.ids.update(ids) + + def get(self, *, ids, include=None): + return _FakeGetResult([drawer_id for drawer_id in ids if drawer_id in self.ids]) + + def delete(self, *, ids=None, where=None): + for drawer_id in ids or []: + self.ids.discard(drawer_id) + self.deleted.append(drawer_id) + + +class _SilentWriteDropCollection(_WritableFakeCollection): + def upsert(self, *, ids, documents, metadatas): + return None + + +class _SilentDeleteDropCollection(_WritableFakeCollection): + def delete(self, *, ids=None, where=None): + self.deleted.extend(ids or []) + + +def test_collection_write_roundtrip_works_when_probe_persists_and_deletes(): + col = _WritableFakeCollection() + + assert collection_write_roundtrip_works(col) is True + assert col.ids == set() + assert len(col.deleted) == 1 + + +def test_collection_write_roundtrip_fails_when_upsert_silently_drops(): + col = _SilentWriteDropCollection() + + assert collection_write_roundtrip_works(col) is False + assert col.ids == set() + + +def test_collection_write_roundtrip_fails_when_delete_silently_drops(): + col = _SilentDeleteDropCollection() + + assert collection_write_roundtrip_works(col) is False + assert len(col.ids) == 1 + + +def test_migrate_dry_run_rebuilds_when_collection_is_readable_but_not_writable(tmp_path, capsys): + palace_dir = tmp_path / "palace" + palace_dir.mkdir() + (palace_dir / "chroma.sqlite3").write_text("db") + + fake_col = MagicMock() + fake_col.count.return_value = 102 + + drawers = [ + { + "id": "id1", + "document": "hello", + "metadata": {"wing": "test-wing", "room": "general"}, + } + ] + + with ( + patch("mempalace.migrate.detect_chromadb_version", return_value="1.x"), + patch("mempalace.backends.chroma.ChromaBackend") as mock_backend, + patch( + "mempalace.migrate.collection_write_roundtrip_works", return_value=False + ) as mock_probe, + patch( + "mempalace.migrate.extract_drawers_from_sqlite", return_value=drawers + ) as mock_extract, + ): + mock_backend.backend_version.return_value = "1.5.8" + mock_backend.return_value.get_collection.return_value = fake_col + + result = migrate(str(palace_dir), dry_run=True) + + out = capsys.readouterr().out + + assert result is True + mock_probe.assert_called_once_with(fake_col) + mock_extract.assert_called_once_with( + os.path.join(os.path.abspath(os.fspath(palace_dir)), "chroma.sqlite3") + ) + + assert "readable by chromadb 1.5.8, but write/delete verification failed" in out + assert "Rebuilding from SQLite" in out + assert "Extracted 1 drawers from SQLite" in out + assert "DRY RUN" in out From 6b042982e8ead30e3ca2ed20ede7118b227af5cc Mon Sep 17 00:00:00 2001 From: fatkobra <55045047+fatkobra@users.noreply.github.com> Date: Tue, 5 May 2026 15:51:43 +0000 Subject: [PATCH 35/77] fix(repair): preflight SQLite integrity before rebuild --- mempalace/repair.py | 70 +++++++++++++++++++++++++++++++++++++ tests/test_repair.py | 82 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 147 insertions(+), 5 deletions(-) diff --git a/mempalace/repair.py b/mempalace/repair.py index 1cd1556975..6834a18969 100644 --- a/mempalace/repair.py +++ b/mempalace/repair.py @@ -330,6 +330,71 @@ def sqlite_drawer_count(palace_path: str) -> "int | None": return None +def sqlite_integrity_errors(palace_path: str) -> list[str]: + """Return SQLite quick_check errors for chroma.sqlite3. + + The repair rebuild path eventually calls Chroma's delete_collection(). + If the SQLite layer has corrupt secondary indexes or FTS5 shadow pages, + Chroma can raise an opaque SQLITE_CORRUPT_INDEX / code 779 error before + repair reaches the HNSW rebuild. + + Run a direct SQLite quick_check first so repair can fail with a clear, + actionable message before invoking Chroma's destructive collection-delete + path. + """ + + sqlite_path = os.path.join(palace_path, "chroma.sqlite3") + if not os.path.exists(sqlite_path): + return [] + + try: + with sqlite3.connect(f"file:{sqlite_path}?mode=ro", uri=True) as conn: + rows = conn.execute("PRAGMA quick_check").fetchall() + except sqlite3.Error as e: + return [f"PRAGMA quick_check failed: {e}"] + + errors: list[str] = [] + for row in rows: + if not row: + continue + message = str(row[0]) + if message.lower() != "ok": + errors.append(message) + + return errors + + +def print_sqlite_integrity_abort(palace_path: str, errors: list[str]) -> None: + """Print a clear repair abort message for SQLite-layer corruption.""" + + sqlite_path = os.path.join(palace_path, "chroma.sqlite3") + preview = errors[:5] + + print("\n ABORT: SQLite-layer corruption detected before repair rebuild.") + print(" `mempalace repair` will not call Chroma delete_collection() because") + print(" the SQLite database failed `PRAGMA quick_check`.") + print() + print(f" Database: {sqlite_path}") + print() + print(" quick_check output:") + for message in preview: + print(f" - {message}") + if len(errors) > len(preview): + print(f" ... and {len(errors) - len(preview)} more issue(s)") + print() + print(" This often means derived SQLite structures, such as secondary indexes") + print(" or FTS5 shadow tables, are corrupt while the underlying rows may still") + print(" be recoverable.") + print() + print(" Suggested recovery:") + print(" 1. Stop all MemPalace writers / MCP clients.") + print(" 2. Back up the entire palace directory.") + print(" 3. Recover chroma.sqlite3 offline with sqlite3 `.recover` or `.dump`.") + print(" 4. Recreate the FTS5 virtual table from intact embedding_metadata rows.") + print(" 5. Verify `PRAGMA integrity_check` returns `ok`.") + print(" 6. Re-run `mempalace repair --yes`.") + + def rebuild_index(palace_path=None, confirm_truncation_ok: bool = False): """Rebuild the HNSW index from scratch. @@ -397,6 +462,11 @@ def rebuild_index(palace_path=None, confirm_truncation_ok: bool = False): print(e.message) return + sqlite_errors = sqlite_integrity_errors(palace_path) + if sqlite_errors: + print_sqlite_integrity_abort(palace_path, sqlite_errors) + return + # Back up ONLY the SQLite database, not the bloated HNSW files sqlite_path = os.path.join(palace_path, "chroma.sqlite3") backup_path = sqlite_path + ".backup" diff --git a/tests/test_repair.py b/tests/test_repair.py index bc770ddfbe..ac1761f257 100644 --- a/tests/test_repair.py +++ b/tests/test_repair.py @@ -216,9 +216,11 @@ def test_rebuild_index_empty_palace(mock_backend_cls, mock_shutil, tmp_path): @patch("mempalace.repair.shutil") @patch("mempalace.repair.ChromaBackend") def test_rebuild_index_success(mock_backend_cls, mock_shutil, tmp_path): - # Create a fake sqlite file + # Create a valid sqlite file so the repair preflight can run quick_check. sqlite_path = tmp_path / "chroma.sqlite3" - sqlite_path.write_text("fake") + with sqlite3.connect(sqlite_path) as conn: + conn.execute("CREATE TABLE dummy(id INTEGER PRIMARY KEY)") + conn.commit() mock_col = MagicMock() mock_col.count.return_value = 2 @@ -234,15 +236,15 @@ def test_rebuild_index_success(mock_backend_cls, mock_shutil, tmp_path): repair.rebuild_index(palace_path=str(tmp_path)) - # Verify: backed up sqlite only (not copytree) + # Verify: backed up sqlite only, not copytree. mock_shutil.copy2.assert_called_once() assert "chroma.sqlite3" in str(mock_shutil.copy2.call_args) - # Verify: deleted and recreated (cosine is the backend default) + # Verify: deleted and recreated. mock_backend.delete_collection.assert_called_once_with(str(tmp_path), "mempalace_drawers") mock_backend.create_collection.assert_called_once_with(str(tmp_path), "mempalace_drawers") - # Verify: used upsert not add + # Verify: used upsert, not add. mock_new_col.upsert.assert_called_once() mock_new_col.add.assert_not_called() @@ -682,3 +684,73 @@ def flaky_detect(*args, **kwargs): # A backup file is still present — caller can roll back from it. leftover = [fn for fn in os.listdir(palace) if "max-seq-id-backup-" in fn] assert leftover + + +def test_sqlite_integrity_errors_returns_empty_for_healthy_db(tmp_path): + palace = tmp_path / "palace" + palace.mkdir() + db_path = palace / "chroma.sqlite3" + + with sqlite3.connect(db_path) as conn: + conn.execute("CREATE TABLE dummy(id INTEGER PRIMARY KEY)") + conn.commit() + + assert repair.sqlite_integrity_errors(str(palace)) == [] + + +def test_sqlite_integrity_errors_reports_unreadable_sqlite_file(tmp_path): + palace = tmp_path / "palace" + palace.mkdir() + db_path = palace / "chroma.sqlite3" + db_path.write_bytes(b"not a sqlite database") + + errors = repair.sqlite_integrity_errors(str(palace)) + + assert errors + assert "quick_check failed" in errors[0] + + +@patch("mempalace.repair.shutil") +@patch("mempalace.repair.ChromaBackend") +def test_rebuild_index_aborts_on_sqlite_integrity_errors_before_delete_collection( + mock_backend_cls, + mock_shutil, + tmp_path, + capsys, +): + """Regression for #1362: fail before Chroma delete_collection on sqlite corruption.""" + + sqlite_path = tmp_path / "chroma.sqlite3" + with sqlite3.connect(sqlite_path) as conn: + conn.execute("CREATE TABLE dummy(id INTEGER PRIMARY KEY)") + conn.commit() + + mock_col = MagicMock() + mock_col.count.return_value = 2 + mock_col.get.return_value = { + "ids": ["id1", "id2"], + "documents": ["doc1", "doc2"], + "metadatas": [{"wing": "a"}, {"wing": "b"}], + } + + mock_backend = _install_mock_backend(mock_backend_cls, mock_col) + + with patch( + "mempalace.repair.sqlite_integrity_errors", + return_value=[ + "Page 4 of B-tree 12345: database disk image is malformed", + "Page 8 of B-tree 67890: database disk image is malformed", + ], + ): + repair.rebuild_index(palace_path=str(tmp_path)) + + out = capsys.readouterr().out + + assert "SQLite-layer corruption detected before repair rebuild" in out + assert "PRAGMA quick_check" in out + assert "delete_collection" in out + assert "Page 4 of B-tree" in out + + mock_backend.delete_collection.assert_not_called() + mock_backend.create_collection.assert_not_called() + mock_shutil.copy2.assert_not_called() From 2c0ef2c04e8e1987c98dac87c4a5950f9f86c0f3 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Wed, 6 May 2026 01:38:57 -0300 Subject: [PATCH 36/77] docs(changelog): document v3.3.5 fixes from #1214 #1105 #1215 #1107 #1282 #1167 #1160 Bundled CHANGELOG entries for the seven Tier-1 PRs merged today, including the behavior-change call-out for #1167 (KG date validators now reject non-ISO inputs that previously produced silent empty results). --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3982fe9c7..ba6822a6aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Bug Fixes - **`mempalace_diary_read` silently dropped entries on agent-name case mismatch.** `tool_diary_write` stored the `agent` metadata verbatim after `sanitize_name`, which preserves case, while `tool_diary_read` filtered by exact match. Writing as `"Claude"` and reading as `"claude"` (or vice-versa) returned zero rows. Both endpoints now lowercase `agent_name` immediately after sanitization, so reads are case-insensitive and the default per-agent wing slug is stable across casings. **Behavior change:** entries written prior to this fix under mixed-case agent names will not match the new lowercase filter; run `mempalace repair` if you need to migrate legacy diary metadata. (#1243) +- **Knowledge-graph triples with `valid_to < valid_from` were silently invisible.** `KnowledgeGraph.query_entity()` filters with `valid_from <= as_of AND valid_to >= as_of`, so an inverted interval matches no `as_of` and the row is durably stored but unreachable — a P0 data-integrity foot-gun any caller that mixes up the two date params can hit. `add_triple()` now rejects inverted intervals at write time with a clear `ValueError` naming both bounds. Open intervals (one bound only) and point-in-time facts (`valid_from == valid_to`) remain accepted unchanged. (#1214) +- **`ChromaBackend.close_palace()` / `close()` did not release the SQLite file lock.** Evicted clients sat in `_clients` without `close()`, and chromadb 1.5.x retains the rust-side SQLite lock until GC. Reopening the same palace path after `shutil.rmtree` + recreate within one process failed with `SQLITE_READONLY_DBMOVED` (code 1032). New `_close_client()` helper now calls `PersistentClient.close()` (with a try/except fallback for older chromadb) on `close_palace()`, on whole-backend `close()`, and on the `_client()` invalidation path that detects a missing `chroma.sqlite3`. The mtime/inode auto-invalidation branch is intentionally left alone — callers there may still hold a live `ChromaCollection`. (#1067, #1105) +- **`EntityRegistry.save()` could leave a corrupt or empty `entity_registry.json` on crash.** `Path.write_text()` is not atomic — kernel sees `open('w')` (truncate), `write`, `close`, and any failure between truncate and full-flush (power loss, OOM, FS-full, kill -9) wipes the months-of-mining people/projects map silently (the registry's `load()` swallows `JSONDecodeError`). Save now writes to a sibling `.tmp` in the same directory, `fsync`s, `chmod 0o600`s, then `os.replace()`s into place — atomic on POSIX and Windows. The previous registry stays intact on any crash before the rename returns. (#1215) +- **`mempalace compress` crashed on large palaces.** `regenerate_closets` fetched all closet_llm drawers in a single `col.get()`, which trips `SQLITE_MAX_VARIABLE_NUMBER` on palaces above ~32k drawers. Mirrors the #851 fix in `miner.py`: drawer fetch is now paginated at `batch_size=5000`. Per-source aggregation works across batches, so the LLM regeneration call still groups chunks correctly. (#1073, #1107) +- **CLI and `fact_checker --stdin` mojibaked non-ASCII content on Windows.** Python defaults `sys.stdin`/`stdout`/`stderr` to the system ANSI codepage (cp1252/cp1251/cp950), so `mempalace search > out.txt` and piped fact_checker invocations corrupted Cyrillic / CJK drawer text at the process boundary. New `mempalace/_stdio.py` helper reconfigures all three streams to UTF-8 on `sys.platform == "win32"`, with per-stream `errors` policy: `surrogateescape` on stdin (preserves bad bytes from redirected files for the consumer's parser), `replace` on stdout/stderr (substitutes U+FFFD instead of `UnicodeEncodeError`-ing mid-print). With this, all three user-facing console_scripts (`mcp_server`, `hooks_cli`, `cli`/`fact_checker`) now reconfigure identically on Windows. (#1282) +- **MCP knowledge-graph tools forwarded malformed date strings to SQLite.** `tool_kg_query` (`as_of`), `tool_kg_add` (`valid_from`), and `tool_kg_invalidate` (`ended`) accepted any string and produced empty result sets on natural-language inputs like `"March 2026"` or `"yesterday"` — callers (especially LLM agents) could not distinguish "no fact at this time" from "your date format was unrecognized." New `sanitize_iso_date()` validator in `config.py` accepts `YYYY`, `YYYY-MM`, `YYYY-MM-DD` (and passes through `None`/`""`); all three tools call it before values reach the storage layer. **Behavior change:** previously-silent date typos now raise a clear `ValueError` naming the offending field; full ISO-8601 with time (`YYYY-MM-DDTHH:MM:SS`, timezone offsets) is not yet accepted — file an issue if you have a use case. (#1164, #1167) +- **MCP server's `_kg` was a module-level singleton.** Multi-tenant hosts that rotate `MEMPALACE_PALACE_PATH` between tool calls hit the wrong sqlite file, because the KG was constructed once at import time while the ChromaDB side was already per-call via `_get_client()`. The KG is now resolved per-call through a lazy per-path cache (`_kg_by_path` keyed by `os.path.abspath`, with a double-checked-locking init under `_kg_cache_lock`). `tool_reconnect` drains and `close()`s cached KGs alongside the existing chroma reconnect. A `_call_kg` retry guard catches `sqlite3.ProgrammingError` once after a reconnect race. (#1136, #1160) --- From d1e27b8c42f3892046bb20f950d7b2e04726e230 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Wed, 6 May 2026 01:47:46 -0300 Subject: [PATCH 37/77] style: ruff format new test files (CI lint) --- tests/test_chroma_collection_lock.py | 18 ++++++------------ tests/test_palace_locks.py | 6 +++--- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/tests/test_chroma_collection_lock.py b/tests/test_chroma_collection_lock.py index b5d30fbcfe..536b5e8894 100644 --- a/tests/test_chroma_collection_lock.py +++ b/tests/test_chroma_collection_lock.py @@ -255,12 +255,8 @@ def test_concurrent_writers_serialize(tmp_path, monkeypatch): ctx = _get_mp_context() result_q = ctx.Queue() - p1 = ctx.Process( - target=_slow_writer_target, args=(palace, str(tmp_path), 1, result_q) - ) - p2 = ctx.Process( - target=_slow_writer_target, args=(palace, str(tmp_path), 2, result_q) - ) + p1 = ctx.Process(target=_slow_writer_target, args=(palace, str(tmp_path), 1, result_q)) + p2 = ctx.Process(target=_slow_writer_target, args=(palace, str(tmp_path), 2, result_q)) p1.start() # Tiny stagger so p1 wins the race deterministically; without it the # OS scheduler can pick either, which is also a valid outcome but @@ -272,9 +268,7 @@ def test_concurrent_writers_serialize(tmp_path, monkeypatch): outcomes = [result_q.get(timeout=1) for _ in range(2)] statuses = sorted(o[0] for o in outcomes) - assert statuses == ["busy", "ok"], ( - f"expected one ok + one busy, got {outcomes}" - ) + assert statuses == ["busy", "ok"], f"expected one ok + one busy, got {outcomes}" def test_read_path_does_not_acquire_lock(tmp_path, monkeypatch): @@ -319,9 +313,9 @@ def test_read_path_does_not_acquire_lock(tmp_path, monkeypatch): if method is None: continue src = inspect.getsource(method) - assert "_write_lock" not in src, ( - f"{read_attr} must NOT acquire the write lock (read path)" - ) + assert ( + "_write_lock" not in src + ), f"{read_attr} must NOT acquire the write lock (read path)" finally: open(release, "w").close() holder.join(timeout=5) diff --git a/tests/test_palace_locks.py b/tests/test_palace_locks.py index 39aa50c597..d2397577bb 100644 --- a/tests/test_palace_locks.py +++ b/tests/test_palace_locks.py @@ -194,9 +194,9 @@ def test_reentrant_same_thread_passes_through(tmp_path, monkeypatch): child = ctx.Process(target=_try_acquire_expect_busy, args=(palace, result_q)) child.start() child.join(timeout=5) - assert result_q.get(timeout=1) == "busy", ( - "outer lock should still be held by parent after inner re-entrant exit" - ) + assert ( + result_q.get(timeout=1) == "busy" + ), "outer lock should still be held by parent after inner re-entrant exit" def _try_acquire_expect_busy(palace_path, result_q): From f854da779fef5634e5719a45b59b3c18acb6270b Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Wed, 6 May 2026 01:57:44 -0300 Subject: [PATCH 38/77] fix(lint): hoist hooks_cli_mod import to top of test_hooks_cli (E402) The alias was placed below an explanatory comment block introduced by #1305, which trips ruff E402 (module-level import not at top of file). Moved next to the existing 'from mempalace.hooks_cli import (...)' line. CI lint went red on develop after #1305 merged with the failing check; this re-greens it so subsequent PRs do not inherit the failure. --- tests/test_hooks_cli.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_hooks_cli.py b/tests/test_hooks_cli.py index 487acf74c1..19ecbafe6f 100644 --- a/tests/test_hooks_cli.py +++ b/tests/test_hooks_cli.py @@ -8,6 +8,7 @@ import pytest +import mempalace.hooks_cli as hooks_cli_mod from mempalace.hooks_cli import ( SAVE_INTERVAL, _count_human_messages, @@ -969,9 +970,6 @@ def test_stop_hook_rejects_injected_stop_hook_active(tmp_path): # STATE_DIR.mkdir() on its own. -import mempalace.hooks_cli as hooks_cli_mod - - def _redirect_palace_root(monkeypatch, tmp_path): """Point PALACE_ROOT and STATE_DIR at a tmp location that does NOT exist.""" fake_root = tmp_path / "absent-mempalace" From 733e4353321678ea1a55c093ad1b3d1d5e8c5313 Mon Sep 17 00:00:00 2001 From: Chris Antenesse Date: Sat, 18 Apr 2026 15:08:01 -0500 Subject: [PATCH 39/77] fix(searcher): guard against None metadata/doc in search result loops ChromaDB can return None entries in metadatas/documents lists under partial-flush, mid-delete, upgrade-boundary, and interrupted-mine states. Add `meta = meta or {}` and `doc = doc or ""` guards in the three result loops (search display, closet hybrid, drawer scored) so .get() and .strip() calls never crash on None. Fixes #1007, #1011 --- mempalace/searcher.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mempalace/searcher.py b/mempalace/searcher.py index d6156232f2..16ea4eb75f 100644 --- a/mempalace/searcher.py +++ b/mempalace/searcher.py @@ -340,7 +340,7 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r # `_hybrid_rank`; do the same here so CLI results match what agents # see via `mempalace_search`. hits = [ - {"text": doc, "distance": float(dist), "metadata": meta or {}} + {"text": doc or "", "distance": float(dist), "metadata": meta or {}} for doc, meta, dist in zip(docs, metas, dists) ] hits = _hybrid_rank(hits, query) @@ -809,6 +809,8 @@ def search_memories( _first_or_empty(drawer_results, "metadatas"), _first_or_empty(drawer_results, "distances"), ): + meta = meta or {} + doc = doc or "" # Filter on raw distance before rounding to avoid precision loss. if max_distance > 0.0 and dist > max_distance: continue From 5347c2c71c782d04fc2023504f58dc52c2369187 Mon Sep 17 00:00:00 2001 From: eldar702 Date: Sun, 19 Apr 2026 11:08:45 +0300 Subject: [PATCH 40/77] fix(searcher): clamp effective_distance to valid cosine range [0, 2] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``search_memories`` computes ``effective_dist = dist - boost`` where ``boost`` can be as large as ``CLOSET_RANK_BOOSTS[0] == 0.40`` for a rank-0 closet hit. When the raw drawer distance is small — any near-exact match — the subtraction goes negative. Two downstream effects: 1. Line 418 returns ``round(max(0.0, 1 - effective_dist), 3)`` as ``similarity``. With ``effective_dist = -0.30`` that yields ``similarity = 1.30``, outside the documented ``[0, 1]`` range. The ``max(0.0, ...)`` only prevents negative similarities; it does not cap above 1. 2. Line 427 stores ``_sort_key: effective_dist`` and line 435 sorts ``scored`` ascending by that key. A negative key drops *below* the rest, so the strongest hybrid matches end up sorting after weaker ones — ranking inversion under the exact conditions hybrid retrieval is supposed to serve best. Clamp ``effective_dist`` to the valid cosine-distance range ``[0, 2]``. The boost still wins (closet-backed hit still ranks first), it just no longer flips the order. Test added: mock drawer_col (base dist 0.08 / 0.35 for two sources) + closet_col (rank-0 closet for the 0.08 source) → assert all hits have ``0 <= similarity <= 1`` and ``0 <= effective_distance <= 2``, and that the closet-boosted source still ranks first. Relationship to other PRs: * **#988** clamps the output ``similarity`` alone. That does not fix the sort-key inversion or the invalid ``effective_distance`` in the returned dict. This PR clamps at the arithmetic source so both downstream users of the value stay in range. * Orthogonal to **#979** (``tool_check_duplicate`` negative similarity). --- mempalace/searcher.py | 7 +++++- tests/test_searcher.py | 57 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/mempalace/searcher.py b/mempalace/searcher.py index d6156232f2..e99fc5d319 100644 --- a/mempalace/searcher.py +++ b/mempalace/searcher.py @@ -825,7 +825,12 @@ def search_memories( matched_via = "drawer+closet" closet_preview = c_preview - effective_dist = dist - boost + # Clamp to the valid cosine-distance range [0, 2]. When a strong + # closet boost (up to 0.40) exceeds the raw distance, the subtraction + # can go negative — which (a) yields ``similarity > 1.0`` downstream + # and (b) makes the sort key land *below* ordinary positive distances, + # inverting the ranking so the best hybrid matches sort last. + effective_dist = max(0.0, min(2.0, dist - boost)) entry = { "text": doc, "wing": meta.get("wing", "unknown"), diff --git a/tests/test_searcher.py b/tests/test_searcher.py index 6b85832dc6..ad60641a55 100644 --- a/tests/test_searcher.py +++ b/tests/test_searcher.py @@ -120,6 +120,63 @@ def mock_get_collection(path, create=False): assert none_hit["wing"] == "unknown" assert none_hit["room"] == "unknown" + def test_effective_distance_clamped_to_valid_cosine_range(self): + """A strong closet boost (up to 0.40) applied to a low-distance drawer + can drive ``dist - boost`` negative. That violates the cosine-distance + invariant ``[0, 2]``: the API returns ``similarity > 1.0`` and the + internal ``_sort_key`` sinks below ordinary positive distances, + inverting the ranking so the best hybrid matches sort last. + + With the clamp, ``effective_distance`` stays in ``[0, 2]``, + ``similarity`` stays in ``[0, 1]``, and the sort order is stable. + """ + # Drawer a.md gets a tiny base distance (0.08) — nearly exact match. + # Drawer b.md gets a larger base distance (0.35). + drawers_col = MagicMock() + drawers_col.query.return_value = { + "documents": [["doc-a", "doc-b"]], + "metadatas": [[ + {"source_file": "a.md", "wing": "w", "room": "r", "chunk_index": 0}, + {"source_file": "b.md", "wing": "w", "room": "r", "chunk_index": 0}, + ]], + "distances": [[0.08, 0.35]], + "ids": [["d-a", "d-b"]], + } + # A strong closet at rank 0 points at a.md → boost = 0.40, + # which exceeds a.md's base distance and would go negative without + # the clamp. No closet for b.md. + closets_col = MagicMock() + closets_col.query.return_value = { + "documents": [["closet-preview-a"]], + "metadatas": [[{"source_file": "a.md"}]], + "distances": [[0.2]], # within CLOSET_DISTANCE_CAP (1.5) + "ids": [["c-a"]], + } + + with ( + patch("mempalace.searcher.get_collection", return_value=drawers_col), + patch("mempalace.searcher.get_closets_collection", return_value=closets_col), + ): + result = search_memories("query", "/fake/path", n_results=5) + + hits = result["results"] + assert hits, "should return results" + + # Invariants on every hit. + for h in hits: + assert 0.0 <= h["similarity"] <= 1.0, ( + f"similarity out of range: {h['similarity']} for {h['source_file']}" + ) + assert 0.0 <= h["effective_distance"] <= 2.0, ( + f"effective_distance out of range: {h['effective_distance']} " + f"for {h['source_file']}" + ) + + # With the clamp, the closet-boosted a.md still ranks ahead of b.md — + # the boost still wins, but it no longer flips the ranking. + assert hits[0]["source_file"] == "a.md" + assert hits[0]["matched_via"] == "drawer+closet" + # ── BM25 internals: None / empty document safety ───────────────────── From aac8437979a562ab5b0c14541a22da4996e03250 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Wed, 6 May 2026 01:53:59 -0300 Subject: [PATCH 41/77] style: ruff format tests/test_searcher.py (CI lint) --- tests/test_searcher.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/test_searcher.py b/tests/test_searcher.py index ad60641a55..4f0b4c0da3 100644 --- a/tests/test_searcher.py +++ b/tests/test_searcher.py @@ -135,10 +135,12 @@ def test_effective_distance_clamped_to_valid_cosine_range(self): drawers_col = MagicMock() drawers_col.query.return_value = { "documents": [["doc-a", "doc-b"]], - "metadatas": [[ - {"source_file": "a.md", "wing": "w", "room": "r", "chunk_index": 0}, - {"source_file": "b.md", "wing": "w", "room": "r", "chunk_index": 0}, - ]], + "metadatas": [ + [ + {"source_file": "a.md", "wing": "w", "room": "r", "chunk_index": 0}, + {"source_file": "b.md", "wing": "w", "room": "r", "chunk_index": 0}, + ] + ], "distances": [[0.08, 0.35]], "ids": [["d-a", "d-b"]], } @@ -164,9 +166,9 @@ def test_effective_distance_clamped_to_valid_cosine_range(self): # Invariants on every hit. for h in hits: - assert 0.0 <= h["similarity"] <= 1.0, ( - f"similarity out of range: {h['similarity']} for {h['source_file']}" - ) + assert ( + 0.0 <= h["similarity"] <= 1.0 + ), f"similarity out of range: {h['similarity']} for {h['source_file']}" assert 0.0 <= h["effective_distance"] <= 2.0, ( f"effective_distance out of range: {h['effective_distance']} " f"for {h['source_file']}" From 0fdb480e12caaf894838886ca6db2ad68e3d67b6 Mon Sep 17 00:00:00 2001 From: Oleksii Pylypchuk Date: Sat, 18 Apr 2026 01:55:43 +0300 Subject: [PATCH 42/77] fix(mcp): handle null JSON-RPC request payloads safely When the MCP client sends a malformed or null top-level request, prevent the AttributeError on request.get() by explicitly validating that the request is a dictionary. Returns standard JSON-RPC Error -32600 (Invalid Request) instead of crashing the server. --- mempalace/mcp_server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index ca71f60520..71a96df2da 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -1968,6 +1968,8 @@ def tool_reconnect(): def handle_request(request): + if not isinstance(request, dict): + return {"jsonrpc": "2.0", "error": {"code": -32600, "message": "Invalid Request"}} method = request.get("method") or "" params = request.get("params") or {} req_id = request.get("id") From 55d79dc8cd03d408644b05cb44aa277edaf2eb86 Mon Sep 17 00:00:00 2001 From: Oleksii Pylypchuk Date: Sat, 18 Apr 2026 16:53:53 +0300 Subject: [PATCH 43/77] fix: include null id in JSON-RPC invalid request error responses and add validation tests --- mempalace/mcp_server.py | 2 +- tests/test_mcp_server.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 71a96df2da..227e91a674 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -1969,7 +1969,7 @@ def tool_reconnect(): def handle_request(request): if not isinstance(request, dict): - return {"jsonrpc": "2.0", "error": {"code": -32600, "message": "Invalid Request"}} + return {"jsonrpc": "2.0", "id": None, "error": {"code": -32600, "message": "Invalid Request"}} method = request.get("method") or "" params = request.get("params") or {} req_id = request.get("id") diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index b036afd6a1..0a020d30b6 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -190,6 +190,13 @@ def test_malformed_method_none(self): resp = handle_request({"method": None, "id": 99, "params": {}}) assert resp["error"]["code"] == -32601 + @pytest.mark.parametrize("payload", [None, [], "plain", 42, True]) + def test_handle_request_invalid_payload_returns_jsonrpc_error(self, payload): + from mempalace.mcp_server import handle_request + + resp = handle_request(payload) + assert resp == {"jsonrpc": "2.0", "id": None, "error": {"code": -32600, "message": "Invalid Request"}} + def test_tools_call_dispatches(self, monkeypatch, config, palace_path, seeded_kg): _patch_mcp_server(monkeypatch, config, seeded_kg) from mempalace.mcp_server import handle_request From a85d432b544060795ff57796d713c597122f6293 Mon Sep 17 00:00:00 2001 From: Oleksii Pylypchuk Date: Sat, 18 Apr 2026 22:05:34 +0300 Subject: [PATCH 44/77] feat: add validation for missing name parameter in tools/call requests --- mempalace/mcp_server.py | 6 ++++++ tests/test_mcp_server.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 227e91a674..25439f2204 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -2007,6 +2007,12 @@ def handle_request(request): }, } elif method == "tools/call": + if not isinstance(params, dict) or "name" not in params: + return { + "jsonrpc": "2.0", + "id": req_id, + "error": {"code": -32602, "message": "Invalid params: 'name' is required for tools/call"}, + } tool_name = params.get("name") tool_args = params.get("arguments") or {} if tool_name not in TOOLS: diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 0a020d30b6..fc56b347ea 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -148,6 +148,20 @@ def test_unknown_tool(self): ) assert resp["error"]["code"] == -32601 + def test_tools_call_missing_params(self): + from mempalace.mcp_server import handle_request + + for bad_params in [None, {}, {"arguments": {}}]: + resp = handle_request( + { + "method": "tools/call", + "id": 15, + "params": bad_params, + } + ) + assert resp["error"]["code"] == -32602 + assert "Invalid params" in resp["error"]["message"] + def test_unknown_method(self): from mempalace.mcp_server import handle_request From 869ab3809570fa578e9dd634ec31b0b1817dddc7 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Wed, 6 May 2026 01:54:11 -0300 Subject: [PATCH 45/77] style: ruff format mcp_server.py + test_mcp_server.py (CI lint) --- mempalace/mcp_server.py | 11 +++++++++-- tests/test_mcp_server.py | 6 +++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 25439f2204..30a0beaa0d 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -1969,7 +1969,11 @@ def tool_reconnect(): def handle_request(request): if not isinstance(request, dict): - return {"jsonrpc": "2.0", "id": None, "error": {"code": -32600, "message": "Invalid Request"}} + return { + "jsonrpc": "2.0", + "id": None, + "error": {"code": -32600, "message": "Invalid Request"}, + } method = request.get("method") or "" params = request.get("params") or {} req_id = request.get("id") @@ -2011,7 +2015,10 @@ def handle_request(request): return { "jsonrpc": "2.0", "id": req_id, - "error": {"code": -32602, "message": "Invalid params: 'name' is required for tools/call"}, + "error": { + "code": -32602, + "message": "Invalid params: 'name' is required for tools/call", + }, } tool_name = params.get("name") tool_args = params.get("arguments") or {} diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index fc56b347ea..c073830daa 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -209,7 +209,11 @@ def test_handle_request_invalid_payload_returns_jsonrpc_error(self, payload): from mempalace.mcp_server import handle_request resp = handle_request(payload) - assert resp == {"jsonrpc": "2.0", "id": None, "error": {"code": -32600, "message": "Invalid Request"}} + assert resp == { + "jsonrpc": "2.0", + "id": None, + "error": {"code": -32600, "message": "Invalid Request"}, + } def test_tools_call_dispatches(self, monkeypatch, config, palace_path, seeded_kg): _patch_mcp_server(monkeypatch, config, seeded_kg) From 7b49478ef7049d124ae6f35e55da2f77cadfddf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E7=A5=96=E9=91=AB=28940219=29?= <940219@nd.com.cn> Date: Fri, 1 May 2026 20:46:36 +0800 Subject: [PATCH 46/77] fix: MCP server JSON output ensure_ascii=False for non-ASCII support Without ensure_ascii=False, non-ASCII characters (e.g. Chinese) in tool results and JSON-RPC responses are escaped as \uXXXX, which causes downstream MCP clients to receive escaped text instead of the original characters. This affects all platforms, not just Windows. Co-Authored-By: Claude Opus 4.7 --- mempalace/mcp_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index ca71f60520..f0973d3435 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -2053,7 +2053,7 @@ def handle_request(request): return { "jsonrpc": "2.0", "id": req_id, - "result": {"content": [{"type": "text", "text": json.dumps(result, indent=2)}]}, + "result": {"content": [{"type": "text", "text": json.dumps(result, indent=2, ensure_ascii=False)}]}, } except Exception: logger.exception(f"Tool error in {tool_name}") @@ -2114,7 +2114,7 @@ def main(): request = json.loads(line) response = handle_request(request) if response is not None: - sys.stdout.write(json.dumps(response) + "\n") + sys.stdout.write(json.dumps(response, ensure_ascii=False) + "\n") sys.stdout.flush() except KeyboardInterrupt: break From 74288f1cdd92f6717186865caa1412d43a13fbc7 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Wed, 6 May 2026 01:54:24 -0300 Subject: [PATCH 47/77] style: ruff format mcp_server.py (CI lint) --- mempalace/mcp_server.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index f0973d3435..fa69aa59a8 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -2053,7 +2053,11 @@ def handle_request(request): return { "jsonrpc": "2.0", "id": req_id, - "result": {"content": [{"type": "text", "text": json.dumps(result, indent=2, ensure_ascii=False)}]}, + "result": { + "content": [ + {"type": "text", "text": json.dumps(result, indent=2, ensure_ascii=False)} + ] + }, } except Exception: logger.exception(f"Tool error in {tool_name}") From eef053d75093645faae6348c08b17d1f6d733a5e Mon Sep 17 00:00:00 2001 From: bobo-xxx <111567133+bobo-xxx@users.noreply.github.com> Date: Sat, 18 Apr 2026 12:40:58 +0800 Subject: [PATCH 48/77] fix(mcp_server): clamp similarity to [0,1] to avoid negative values --- mempalace/mcp_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index ca71f60520..dbe44975e5 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -740,7 +740,7 @@ def tool_check_duplicate(content: str, threshold: float = 0.9): if results["ids"] and results["ids"][0]: for i, drawer_id in enumerate(results["ids"][0]): dist = results["distances"][0][i] - similarity = round(1 - dist, 3) + similarity = round(max(0.0, 1 - dist), 3) if similarity >= threshold: # Chroma 1.5.x can return None for partially-flushed rows; # coerce to empty sentinels so downstream .get() is safe. From f2bed9284fccaec51dd78266f9113db41a9e966f Mon Sep 17 00:00:00 2001 From: bobo-xxx <111567133+bobo-xxx@users.noreply.github.com> Date: Sat, 18 Apr 2026 12:41:12 +0800 Subject: [PATCH 49/77] fix(layers): clamp similarity to [0,1] to avoid negative values --- mempalace/layers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mempalace/layers.py b/mempalace/layers.py index b20c65621b..d549afe72e 100644 --- a/mempalace/layers.py +++ b/mempalace/layers.py @@ -287,7 +287,7 @@ def search(self, query: str, wing: str = None, room: str = None, n_results: int for i, (doc, meta, dist) in enumerate(zip(docs, metas, dists), 1): meta = meta or {} doc = doc or "" - similarity = round(1 - dist, 3) + similarity = round(max(0.0, 1 - dist), 3) wing_name = meta.get("wing", "?") room_name = meta.get("room", "?") source = Path(meta.get("source_file", "")).name if meta.get("source_file") else "" From b68485dfd4673848bcd44064a0c8764d89f6b48a Mon Sep 17 00:00:00 2001 From: Anthony Clendenen Date: Thu, 23 Apr 2026 13:33:28 -0700 Subject: [PATCH 50/77] fix(closet_llm): reject non-http(s) endpoints LLMConfig accepted any URL scheme from LLM_ENDPOINT / --endpoint, so a misconfigured endpoint such as file:///etc/passwd would be passed straight to urllib.request.urlopen. Validate the scheme at construction time and raise ValueError on anything other than http/https, preserving the "privacy by architecture" guarantee. Co-Authored-By: Claude Opus 4.7 (1M context) --- mempalace/closet_llm.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mempalace/closet_llm.py b/mempalace/closet_llm.py index 6274f7965d..50000c8076 100644 --- a/mempalace/closet_llm.py +++ b/mempalace/closet_llm.py @@ -40,6 +40,7 @@ import os import re import time +import urllib.parse import urllib.request import urllib.error from datetime import datetime @@ -101,6 +102,14 @@ def __init__( self.endpoint = (endpoint or os.environ.get("LLM_ENDPOINT", "")).rstrip("/") self.key = key or os.environ.get("LLM_KEY", "") self.model = model or os.environ.get("LLM_MODEL", "") + if self.endpoint: + # Privacy-by-architecture: reject file:// and other non-HTTP schemes + # so a misconfigured endpoint cannot exfiltrate local files. + scheme = urllib.parse.urlparse(self.endpoint).scheme.lower() + if scheme not in ("http", "https"): + raise ValueError( + f"LLM_ENDPOINT must use http:// or https:// (got scheme {scheme!r})" + ) def missing(self) -> list: missing = [] From ca5899e361a1bc8823145f6d1efad22f22639409 Mon Sep 17 00:00:00 2001 From: Anthony Clendenen Date: Thu, 23 Apr 2026 13:33:38 -0700 Subject: [PATCH 51/77] refactor: fix ruff bugbear and silent-except findings - B904: chain OSError/collection errors with "raise ... from e" in normalize.py and searcher.py so the original traceback is preserved. - B007: rename unused loop variables to _name in dedup, dialect, layers, and room_detector_local. - S110/S112: replace bare "try/except/pass" and "try/except/continue" with logger.debug(..., exc_info=True) in mcp_server, searcher, palace, palace_graph, miner, convo_miner, and fact_checker so background failures are observable without changing behaviour. A module-level logger ("mempalace_mcp", matching mcp_server/searcher) is added to the five files that didn't already have one. Configured ruff checks (E/F/W/C901) and ruff --select B, S110, S112 all pass. Co-Authored-By: Claude Opus 4.7 (1M context) --- mempalace/convo_miner.py | 5 ++++- mempalace/dedup.py | 2 +- mempalace/dialect.py | 2 +- mempalace/fact_checker.py | 4 ++++ mempalace/layers.py | 2 +- mempalace/mcp_server.py | 4 ++-- mempalace/miner.py | 5 ++++- mempalace/normalize.py | 4 ++-- mempalace/palace.py | 7 +++++-- mempalace/palace_graph.py | 2 +- mempalace/room_detector_local.py | 2 +- mempalace/searcher.py | 10 ++++++---- 12 files changed, 32 insertions(+), 17 deletions(-) diff --git a/mempalace/convo_miner.py b/mempalace/convo_miner.py index 2cf57e4488..915b4d1d05 100644 --- a/mempalace/convo_miner.py +++ b/mempalace/convo_miner.py @@ -11,6 +11,7 @@ import os import sys import hashlib +import logging from pathlib import Path from datetime import datetime from collections import defaultdict @@ -24,6 +25,8 @@ mine_lock, ) +logger = logging.getLogger("mempalace_mcp") + # Cached hall keywords — avoids re-reading config per drawer _HALL_KEYWORDS_CACHE = None @@ -331,7 +334,7 @@ def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extr try: collection.delete(where={"source_file": source_file}) except Exception: - pass + logger.debug("Stale-drawer purge failed for %s", source_file, exc_info=True) # Batch chunks into bounded upserts so large transcripts keep most of # the embedding speedup without one huge Chroma/SQLite request. Keep diff --git a/mempalace/dedup.py b/mempalace/dedup.py index 6b1bac106b..5e57aff956 100644 --- a/mempalace/dedup.py +++ b/mempalace/dedup.py @@ -89,7 +89,7 @@ def dedup_source_group(col, drawer_ids, threshold=DEFAULT_THRESHOLD, dry_run=Tru kept = [] to_delete = [] - for did, doc, meta in items: + for did, doc, _meta in items: if not doc or len(doc) < 20: to_delete.append(did) continue diff --git a/mempalace/dialect.py b/mempalace/dialect.py index b72c52c77c..e6e214c8e1 100644 --- a/mempalace/dialect.py +++ b/mempalace/dialect.py @@ -873,7 +873,7 @@ def generate_layer1( for date_key in sorted(by_date.keys()): lines.append(f"=MOMENTS[{date_key}]=") - for z, fnum in by_date[date_key]: + for z, _fnum in by_date[date_key]: entities = [] for p in z.get("people", []): code = self.encode_entity(p) diff --git a/mempalace/fact_checker.py b/mempalace/fact_checker.py index 403d913605..8f1c3ba133 100644 --- a/mempalace/fact_checker.py +++ b/mempalace/fact_checker.py @@ -27,6 +27,7 @@ from __future__ import annotations +import logging import os import re from datetime import datetime, timezone @@ -35,6 +36,8 @@ # ~/.mempalace/known_entities.json on every check_text call. from .miner import _load_known_entities_raw +logger = logging.getLogger("mempalace_mcp") + # Narrow detection patterns — parse "X is Y's Z" and "X's Z is Y". # Names are captured greedily as word sequences (letters + optional @@ -214,6 +217,7 @@ def _check_kg_contradictions(text: str, palace_path: str) -> list: try: facts = kg.query_entity(subject, direction="outgoing") except Exception: + logger.debug("KG lookup failed for subject %r", subject, exc_info=True) continue if not facts: continue diff --git a/mempalace/layers.py b/mempalace/layers.py index d549afe72e..b92890aa26 100644 --- a/mempalace/layers.py +++ b/mempalace/layers.py @@ -157,7 +157,7 @@ def generate(self) -> str: lines.append(room_line) total_len += len(room_line) - for imp, meta, doc in entries: + for _imp, meta, doc in entries: source = Path(meta.get("source_file", "")).name if meta.get("source_file") else "" # Truncate doc to keep L1 compact diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 46982bbdfa..58f9ba9767 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -900,7 +900,7 @@ def tool_add_drawer( if existing and existing["ids"]: return {"success": True, "reason": "already_exists", "drawer_id": drawer_id} except Exception: - pass + logger.debug("Idempotency pre-check failed for %s", drawer_id, exc_info=True) try: col.upsert( @@ -1418,7 +1418,7 @@ def tool_hook_settings(silent_save: bool = None, desktop_toast: bool = None): try: config = MempalaceConfig() except Exception: - pass + logger.debug("Could not re-read config after update", exc_info=True) result = { "success": True, diff --git a/mempalace/miner.py b/mempalace/miner.py index ba0c630631..88734c9bb6 100644 --- a/mempalace/miner.py +++ b/mempalace/miner.py @@ -12,6 +12,7 @@ import shlex import hashlib import fnmatch +import logging from pathlib import Path from datetime import datetime from collections import defaultdict @@ -31,6 +32,8 @@ upsert_closet_lines, ) +logger = logging.getLogger("mempalace_mcp") + READABLE_EXTENSIONS = { ".txt", ".md", @@ -842,7 +845,7 @@ def process_file( try: collection.delete(where={"source_file": source_file}) except Exception: - pass + logger.debug("Stale-drawer purge failed for %s", source_file, exc_info=True) # Batch chunks into bounded upserts so the embedding model sees many # chunks per forward pass without building one huge Chroma/SQLite diff --git a/mempalace/normalize.py b/mempalace/normalize.py index 4252afa4dc..ca62cca3df 100644 --- a/mempalace/normalize.py +++ b/mempalace/normalize.py @@ -118,14 +118,14 @@ def normalize(filepath: str) -> str: try: file_size = os.path.getsize(filepath) except OSError as e: - raise IOError(f"Could not read {filepath}: {e}") + raise IOError(f"Could not read {filepath}: {e}") from e if file_size > 500 * 1024 * 1024: # 500 MB safety limit raise IOError(f"File too large ({file_size // (1024 * 1024)} MB): {filepath}") try: with open(filepath, "r", encoding="utf-8", errors="replace") as f: content = f.read() except OSError as e: - raise IOError(f"Could not read {filepath}: {e}") + raise IOError(f"Could not read {filepath}: {e}") from e if not content.strip(): return content diff --git a/mempalace/palace.py b/mempalace/palace.py index 97f67ff33a..e5f6411855 100644 --- a/mempalace/palace.py +++ b/mempalace/palace.py @@ -6,12 +6,15 @@ import contextlib import hashlib +import logging import os import re import threading from .backends.chroma import ChromaBackend +logger = logging.getLogger("mempalace_mcp") + SKIP_DIRS = { ".git", "node_modules", @@ -229,7 +232,7 @@ def purge_file_closets(closets_col, source_file: str) -> None: try: closets_col.delete(where={"source_file": source_file}) except Exception: - pass + logger.debug("Closet purge failed for %s", source_file, exc_info=True) def upsert_closet_lines(closets_col, closet_id_base, lines, metadata): @@ -307,7 +310,7 @@ def mine_lock(source_file: str): fcntl.flock(lf, fcntl.LOCK_UN) except Exception: - pass + logger.debug("Mine-lock release failed", exc_info=True) lf.close() diff --git a/mempalace/palace_graph.py b/mempalace/palace_graph.py index 3296cd5c3e..0fff763fd1 100644 --- a/mempalace/palace_graph.py +++ b/mempalace/palace_graph.py @@ -575,7 +575,7 @@ def follow_tunnels(wing: str, room: str, col=None, config=None): if did and did in drawer_map: c["drawer_preview"] = drawer_map[did][:300] except Exception: - pass + logger.debug("Drawer preview hydration failed", exc_info=True) return connections diff --git a/mempalace/room_detector_local.py b/mempalace/room_detector_local.py index 31d5b058d9..8e3fc20611 100644 --- a/mempalace/room_detector_local.py +++ b/mempalace/room_detector_local.py @@ -202,7 +202,7 @@ def detect_rooms_from_files(project_dir: str) -> list: SKIP_DIRS = {".git", "node_modules", "__pycache__", ".venv", "venv", "dist", "build"} - for root, dirs, filenames in os.walk(project_path): + for _root, dirs, filenames in os.walk(project_path): dirs[:] = [d for d in dirs if d not in SKIP_DIRS] for filename in filenames: name_lower = filename.lower().replace("-", "_").replace(" ", "_") diff --git a/mempalace/searcher.py b/mempalace/searcher.py index ddddc46e04..536610e089 100644 --- a/mempalace/searcher.py +++ b/mempalace/searcher.py @@ -245,7 +245,7 @@ def _expand_with_neighbors(drawers_col, matched_doc: str, matched_meta: dict, ra all_meta = drawers_col.get(where={"source_file": src}, include=["metadatas"]) total_drawers = len(all_meta.ids) if all_meta.ids else None except Exception: - pass + logger.debug("total_drawers lookup failed for %s", src, exc_info=True) return { "text": combined_text, @@ -297,10 +297,10 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r """ try: col = get_collection(palace_path, create=False) - except Exception: + except Exception as e: print(f"\n No palace found at {palace_path}") print(" Run: mempalace init then mempalace mine ") - raise SearchError(f"No palace found at {palace_path}") + raise SearchError(f"No palace found at {palace_path}") from e # Alert the user if this palace predates hnsw:space=cosine being set on # creation — their similarity scores will be junk until they run repair. @@ -795,7 +795,8 @@ def search_memories( if source and source not in closet_boost_by_source: closet_boost_by_source[source] = (rank, cdist, cdoc[:200]) except Exception: - pass # no closets yet — hybrid degrades to pure drawer search + # No closets yet — hybrid degrades to pure drawer search. + logger.debug("Closet collection unavailable; using drawer-only search", exc_info=True) # Rank-based boost. The ordinal signal ("which closet matched best") is # more reliable than absolute distance on narrative content, where @@ -877,6 +878,7 @@ def search_memories( include=["documents", "metadatas"], ) except Exception: + logger.debug("Neighbor fetch failed for %s", full_source, exc_info=True) continue docs = source_drawers.documents metas_ = source_drawers.metadatas From a7c4ed24d7fd222cafb25af915fa3cef87e92b27 Mon Sep 17 00:00:00 2001 From: Brian potter Date: Sat, 2 May 2026 00:25:46 -0500 Subject: [PATCH 52/77] fix(repair): add --mode from-sqlite to recover palaces with corrupt HNSW (#1308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both `--mode legacy` and the inline `cli.cmd_repair` rebuild path call `Collection.count()` as their first read — the same call that raises `chromadb.errors.InternalError: Failed to apply logs to the hnsw segment writer` on the corruption class reported in #1308. Repair would print "Cannot recover — palace may need to be re-mined from source files" even though the underlying SQLite tables were fully intact. The new `--mode from-sqlite` reads `(id, document, metadata)` rows directly from `chroma.sqlite3` via `segments` → `embeddings` → `embedding_metadata` joins, never opens a chromadb client against the corrupt palace, and re-upserts everything into a fresh palace. - `--source PATH` extracts from a corrupt palace already moved aside - `--archive-existing` handles the in-place case by renaming the existing palace to `.pre-rebuild-` first - Partial-rebuild failures raise `RebuildPartialError` with the archive path so users can recover; CLI exits non-zero - In-place mode calls `SharedSystemClient.clear_system_cache()` to drop chromadb's process-wide System registry (cross-palace use does not, to limit blast radius for library callers) - Source validation runs before any destructive moves Verified end-to-end recovering a 52,300-row real-world corrupt palace. Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 1 + mempalace/cli.py | 68 +++++++- mempalace/repair.py | 358 ++++++++++++++++++++++++++++++++++++++++++- tests/test_repair.py | 291 +++++++++++++++++++++++++++++++++++ 4 files changed, 714 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3011aabecb..1972c03b09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - **CLI and `fact_checker --stdin` mojibaked non-ASCII content on Windows.** Python defaults `sys.stdin`/`stdout`/`stderr` to the system ANSI codepage (cp1252/cp1251/cp950), so `mempalace search > out.txt` and piped fact_checker invocations corrupted Cyrillic / CJK drawer text at the process boundary. New `mempalace/_stdio.py` helper reconfigures all three streams to UTF-8 on `sys.platform == "win32"`, with per-stream `errors` policy: `surrogateescape` on stdin (preserves bad bytes from redirected files for the consumer's parser), `replace` on stdout/stderr (substitutes U+FFFD instead of `UnicodeEncodeError`-ing mid-print). With this, all three user-facing console_scripts (`mcp_server`, `hooks_cli`, `cli`/`fact_checker`) now reconfigure identically on Windows. (#1282) - **MCP knowledge-graph tools forwarded malformed date strings to SQLite.** `tool_kg_query` (`as_of`), `tool_kg_add` (`valid_from`), and `tool_kg_invalidate` (`ended`) accepted any string and produced empty result sets on natural-language inputs like `"March 2026"` or `"yesterday"` — callers (especially LLM agents) could not distinguish "no fact at this time" from "your date format was unrecognized." New `sanitize_iso_date()` validator in `config.py` accepts `YYYY`, `YYYY-MM`, `YYYY-MM-DD` (and passes through `None`/`""`); all three tools call it before values reach the storage layer. **Behavior change:** previously-silent date typos now raise a clear `ValueError` naming the offending field; full ISO-8601 with time (`YYYY-MM-DDTHH:MM:SS`, timezone offsets) is not yet accepted — file an issue if you have a use case. (#1164, #1167) - **MCP server's `_kg` was a module-level singleton.** Multi-tenant hosts that rotate `MEMPALACE_PALACE_PATH` between tool calls hit the wrong sqlite file, because the KG was constructed once at import time while the ChromaDB side was already per-call via `_get_client()`. The KG is now resolved per-call through a lazy per-path cache (`_kg_by_path` keyed by `os.path.abspath`, with a double-checked-locking init under `_kg_cache_lock`). `tool_reconnect` drains and `close()`s cached KGs alongside the existing chroma reconnect. A `_call_kg` retry guard catches `sqlite3.ProgrammingError` once after a reconnect race. (#1136, #1160) +- **`mempalace repair` can now recover palaces whose HNSW segment writer is stuck on `apply_logs`.** Both the existing `--mode legacy` rebuild and the inline `cli.cmd_repair` path call `Collection.count()` as their first read — exactly the call that raises `chromadb.errors.InternalError: Failed to apply logs to the hnsw segment writer` on the corruption class introduced upstream and reported in #1308. Repair would print `Cannot recover — palace may need to be re-mined from source files` even though the underlying SQLite tables were fully intact (the corruption lives in the on-disk index files, not the data layer). New `--mode from-sqlite` reads `(id, document, metadata)` rows directly from `chroma.sqlite3` via a `segments` → `embeddings` → `embedding_metadata` join, never opens a chromadb client against the corrupt palace, and re-upserts everything into a fresh palace at `--palace`. `--source PATH` extracts from a corrupt palace already moved aside; `--archive-existing` handles the in-place case by renaming the existing palace to `.pre-rebuild-` before reading from it. Documents are re-embedded under the user's configured embedding function (the original HNSW vectors live in the corrupt `data_level0.bin` and cannot be recovered, but the embedding model is deterministic so search results remain semantically equivalent). Verified end-to-end on a 52,300-row real-world corrupt palace. (#1308) --- diff --git a/mempalace/cli.py b/mempalace/cli.py index 0ab3d0f813..468b76515a 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -673,6 +673,48 @@ def cmd_repair(args): ) return + if getattr(args, "mode", "legacy") == "from-sqlite": + from .migrate import confirm_destructive_action + from .repair import RebuildPartialError, rebuild_from_sqlite + + source_path = getattr(args, "source", None) + source_path = ( + os.path.abspath(os.path.expanduser(source_path)) if source_path else palace_path + ) + archive_existing = getattr(args, "archive_existing", False) + + # Gate any path that touches the user's existing palace dir + # behind confirm_destructive_action. The legacy mode already + # gates; from-sqlite needs the same protection because: + # (a) --archive-existing renames the existing palace, + # (b) --source PATH writes into --palace dir which the user + # may not realize is also a palace. + # No prompt when source != dest AND dest does not exist (pure + # extract-into-fresh-dir case is non-destructive to existing + # palaces). + is_destructive_to_dest = source_path == palace_path or os.path.exists(palace_path) + if is_destructive_to_dest and not confirm_destructive_action( + "Rebuild from SQLite", palace_path, assume_yes=getattr(args, "yes", False) + ): + return + + try: + rebuild_from_sqlite( + source_palace=source_path, + dest_palace=palace_path, + archive_existing_dest=archive_existing, + ) + except RebuildPartialError as exc: + # The error itself was already printed by rebuild_from_sqlite + # with recovery instructions; surface a non-zero exit so + # scripts and CI gates see the failure. + print( + "\n Rebuild partial — see message above. " + f"Failed in collection: {exc.failed_collection}" + ) + sys.exit(1) + return + db_path = os.path.join(palace_path, "chroma.sqlite3") if not os.path.isdir(palace_path): @@ -1213,11 +1255,31 @@ def main(): ) p_repair.add_argument( "--mode", - choices=["legacy", "max-seq-id"], + choices=["legacy", "max-seq-id", "from-sqlite"], default="legacy", help=( - "legacy: full-palace rebuild (default). " - "max-seq-id: un-poison max_seq_id rows corrupted by the legacy 0.6.x shim." + "legacy: full-palace rebuild via the chromadb client (default). " + "max-seq-id: un-poison max_seq_id rows corrupted by the legacy 0.6.x shim. " + "from-sqlite: rebuild by reading rows directly from chroma.sqlite3, " + "bypassing the chromadb client. Use when legacy mode bails because the " + "chromadb client cannot open the collection." + ), + ) + p_repair.add_argument( + "--source", + default=None, + help=( + "Source palace path for --mode from-sqlite (defaults to --palace). " + "Use when extracting from an archived corrupt palace into a new location." + ), + ) + p_repair.add_argument( + "--archive-existing", + action="store_true", + help=( + "For --mode from-sqlite when --source equals --palace: rename the " + "existing palace to .pre-rebuild- before " + "rebuilding so the corrupt copy is preserved." ), ) p_repair.add_argument( diff --git a/mempalace/repair.py b/mempalace/repair.py index 1cd1556975..7e98f0f1d6 100644 --- a/mempalace/repair.py +++ b/mempalace/repair.py @@ -34,14 +34,21 @@ import shutil import sqlite3 import time +from collections import defaultdict from datetime import datetime -from typing import Optional +from typing import Iterator, Optional from .backends.chroma import ChromaBackend, hnsw_capacity_status COLLECTION_NAME = "mempalace_drawers" +# Collections rebuilt by ``rebuild_from_sqlite``. Order matters for the +# upsert pass — drawers carry the bulk of the data, closets are the AAAK +# index layer and reference drawer IDs by string in their documents (no +# foreign-key validation, so ordering is informational, not load-bearing). +RECOVERABLE_COLLECTIONS = ("mempalace_drawers", "mempalace_closets") + def _get_palace_path(): """Resolve palace path from config.""" @@ -436,6 +443,355 @@ def rebuild_index(palace_path=None, confirm_truncation_ok: bool = False): print(f"\n{'=' * 55}\n") +class RebuildPartialError(Exception): + """Raised when ``rebuild_from_sqlite`` fails partway through upserts. + + Carries enough state for the user (or CLI) to recover: the + per-collection counts that succeeded, the collection that failed, + the dest path holding the partial palace, and the archive path + (when an in-place rebuild had moved the original aside). Re-raises + the underlying chromadb error as ``__cause__``. + """ + + def __init__( + self, + message: str, + *, + partial_counts: dict[str, int], + failed_collection: str, + dest_palace: str, + archive_path: Optional[str], + ): + super().__init__(message) + self.message = message + self.partial_counts = partial_counts + self.failed_collection = failed_collection + self.dest_palace = dest_palace + self.archive_path = archive_path + + +def _rebuild_one_collection( + *, + backend: ChromaBackend, + source_palace: str, + dest_palace: str, + collection_name: str, + batch_size: int, + archive_path: Optional[str], + counts_so_far: dict[str, int], +) -> int: + """Stream rows for one collection from SQLite and upsert into a + freshly-created collection at ``dest_palace``. Returns rows + upserted. Raises :class:`RebuildPartialError` (with the underlying + chromadb exception as ``__cause__``) on any upsert failure so the + caller can stop the loop and print recovery instructions instead of + silently shipping a partial palace. + """ + col = backend.create_collection(dest_palace, collection_name) + + ids: list[str] = [] + docs: list[str] = [] + metas: list[dict] = [] + upserted = 0 + + def _flush() -> int: + nonlocal upserted + if not ids: + return upserted + col.upsert(ids=list(ids), documents=list(docs), metadatas=list(metas)) + upserted += len(ids) + print(f" upserted {upserted}") + ids.clear() + docs.clear() + metas.clear() + return upserted + + try: + for emb_id, doc, meta in extract_via_sqlite(source_palace, collection_name): + ids.append(emb_id) + docs.append(doc or "") + # chromadb 1.5.x rejects None entries in the metadatas list + # but accepts empty dicts. Mempalace drawers always carry at + # least wing/room, so this branch is defensive — corruption + # in embedding_metadata could yield an emb_id with no rows. + metas.append(meta if meta else {}) + if len(ids) >= batch_size: + _flush() + _flush() + except Exception as exc: # noqa: BLE001 — chromadb raises many shapes + partial = dict(counts_so_far) + partial[collection_name] = upserted + msg_parts = [ + f"Upsert failed in collection {collection_name!r} after {upserted} rows: {exc!r}", + f"Partial palace left at: {dest_palace}", + ] + if archive_path is not None: + msg_parts.append(f"Original palace archived at: {archive_path}") + msg_parts.append( + " Recover by removing the partial dest and re-running with " + f"--source {archive_path}" + ) + else: + msg_parts.append(" Source palace is unchanged. Remove the partial dest and re-run.") + message = "\n ".join(msg_parts) + print(f"\n ERROR: {message}") + raise RebuildPartialError( + message, + partial_counts=partial, + failed_collection=collection_name, + dest_palace=dest_palace, + archive_path=archive_path, + ) from exc + + return upserted + + +def extract_via_sqlite(palace_path: str, collection_name: str) -> Iterator[tuple[str, str, dict]]: + """Yield ``(embedding_id, document, metadata)`` for every row in + ``collection_name``'s metadata segment by reading ``chroma.sqlite3`` + directly. + + Bypasses the chromadb client entirely — never opens a + ``PersistentClient``, never imports hnswlib, never invokes the + HNSW segment writer. This is the recovery path for palaces where + ``Collection.count()`` / ``Collection.get()`` raise ``InternalError`` + because the compactor cannot apply WAL logs to the HNSW segment + (#1308). The drawer rows are still on disk in + ``embeddings`` + ``embedding_metadata``; the corruption lives in the + on-disk index files, not the SQLite tables. + + Resolution rule for chromadb's typed metadata columns: each + ``embedding_metadata`` row stores its value in exactly one of + ``string_value`` / ``int_value`` / ``float_value`` / ``bool_value``; + we pick the first non-NULL column in that order. Rows where every + typed column is NULL are dropped (chromadb never writes that shape). + The ``chroma:document`` key is removed from the metadata dict and + returned as the document; this matches how chromadb itself stores + ``add(documents=...)``. + + Silent on missing palace, missing ``chroma.sqlite3``, or unknown + collection name — yields nothing. Callers that need to distinguish + "empty collection" from "collection not present" should query + :func:`sqlite_drawer_count` first. + """ + sqlite_path = os.path.join(palace_path, "chroma.sqlite3") + if not os.path.isfile(sqlite_path): + return + + conn = sqlite3.connect(f"file:{sqlite_path}?mode=ro", uri=True) + try: + seg_row = conn.execute( + """ + SELECT s.id FROM segments s + JOIN collections c ON s.collection = c.id + WHERE c.name = ? AND s.scope = 'METADATA' + """, + (collection_name,), + ).fetchone() + if not seg_row: + return + segment_id = seg_row[0] + + per_id: dict[str, dict] = defaultdict(dict) + order: list[str] = [] + for emb_id, key, sv, iv, fv, bv in conn.execute( + """ + SELECT e.embedding_id, em.key, em.string_value, em.int_value, + em.float_value, em.bool_value + FROM embedding_metadata em + JOIN embeddings e ON em.id = e.id + WHERE e.segment_id = ? + ORDER BY em.id + """, + (segment_id,), + ): + if emb_id not in per_id: + order.append(emb_id) + if sv is not None: + per_id[emb_id][key] = sv + elif iv is not None: + per_id[emb_id][key] = iv + elif fv is not None: + per_id[emb_id][key] = fv + elif bv is not None: + per_id[emb_id][key] = bool(bv) + + for emb_id in order: + kv = per_id[emb_id] + doc = kv.pop("chroma:document", "") + yield emb_id, doc, kv + finally: + conn.close() + + +def rebuild_from_sqlite( + source_palace: str, + dest_palace: str, + *, + archive_existing_dest: bool = False, + batch_size: int = 1000, +) -> dict[str, int]: + """Rebuild a palace by reading drawers from ``source_palace``'s + ``chroma.sqlite3`` and upserting them into a fresh palace at + ``dest_palace``. + + Recovery path for the #1308 failure mode: the chromadb client raises + ``InternalError: Failed to apply logs to the hnsw segment writer`` + on every operation that touches the index (``count``, ``get``, + ``query``), but the underlying SQLite tables are intact. Both the + legacy ``rebuild_index`` and the inline ``cli.cmd_repair`` path call + ``Collection.count()`` as their first read — exactly the call that + fails — so neither can recover this class of corruption. This + function bypasses the chromadb read path entirely via + :func:`extract_via_sqlite`. + + Re-embeds documents at upsert time using the configured embedding + function; the original HNSW vectors are not preserved (they live in + the corrupt ``data_level0.bin`` / ``link_lists.bin``, not in + SQLite). Acceptable for a corruption-recovery flow because the + embedding model is deterministic — same model + same document text + yields semantically equivalent search results. + + ``archive_existing_dest`` controls behavior when ``dest_palace`` + already exists: + + * ``False`` (default) — refuse with a clear message. Callers must + manually move the existing palace aside first. + * ``True`` — rename ``dest_palace`` to + ``.pre-rebuild-`` and read from there + instead. Used by the in-place CLI flow where ``--source`` defaults + to the same path as ``--palace``. + + Returns a ``{collection_name: row_count}`` dict so callers (CLI, + tests) can verify the per-collection rebuild count without parsing + stdout. Returns ``{}`` on validation failures (missing source, + refusing to overwrite). Raises :class:`RebuildPartialError` if a + chromadb upsert fails partway through; the dest palace is left in + place so the user can inspect what landed, and the in-place archive + (when applicable) is reported in the error so the user can re-run + against it. + + .. warning:: + + In-place mode (``source_palace == dest_palace`` with + ``archive_existing_dest=True``) calls + ``chromadb.api.client.SharedSystemClient.clear_system_cache()`` to + drop chromadb's process-wide System registry — required because + an existing cached System built against the original palace will + refuse ``create_collection`` after the dir is renamed (chromadb + still thinks the collections exist). This invalidates any + PersistentClient instances held elsewhere in the same process for + *any* palace, not just this one. Do not call this function from + inside a long-running mempalace process (MCP server, daemon) + while other callers hold live ``PersistentClient`` references — + use the CLI in a separate process instead. Cross-palace use + (``source != dest``) does not touch the cache. + + Note on metadata fidelity: the resolution rule + (``string_value`` → ``int_value`` → ``float_value`` → ``bool_value``) + matches the precedent in :mod:`mempalace.migrate`. ChromaDB 0.4.x + occasionally wrote booleans as ``int_value=0/1``; those will + round-trip as ``int`` rather than ``bool`` after this rebuild. This + is a known divergence and matches the existing migrate-path + behavior. + """ + source_palace = os.path.abspath(os.path.expanduser(source_palace)) + dest_palace = os.path.abspath(os.path.expanduser(dest_palace)) + + src_db = os.path.join(source_palace, "chroma.sqlite3") + + in_place = source_palace == dest_palace + + print(f"\n{'=' * 55}") + print(" MemPalace Repair — Rebuild from SQLite") + print(f"{'=' * 55}\n") + print(f" Source: {source_palace}") + print(f" Dest: {dest_palace}") + + # Validate source BEFORE any destructive moves. An earlier draft + # archived the dest first and surfaced the missing-chroma.sqlite3 + # error after — leaving the user with a renamed dir to manually undo + # when the archive itself was empty. Validate first so a user error + # (--source pointing at a non-palace dir) bails cleanly. + if in_place: + if not archive_existing_dest: + print( + "\n Source and dest are the same path. Pass " + "archive_existing_dest=True (CLI: --archive-existing) to move " + "the existing palace aside, or pass a different source_palace= " + "(CLI: --source)." + ) + return {} + if not os.path.isfile(src_db): + print(f"\n Source palace has no chroma.sqlite3 at {src_db}") + return {} + else: + if not os.path.isfile(src_db): + print(f"\n Source palace has no chroma.sqlite3 at {src_db}") + return {} + if os.path.exists(dest_palace): + print( + f"\n Refusing to rebuild into existing path: {dest_palace}\n" + " Move it aside, pass a different dest, or set " + "archive_existing_dest=True if rebuilding in place " + "(source_palace == dest_palace)." + ) + return {} + + archive_path: Optional[str] = None + if in_place: + ts = datetime.now().strftime("%Y%m%d-%H%M%S") + archive_path = f"{dest_palace}.pre-rebuild-{ts}" + print(f" Archiving {dest_palace} → {archive_path}") + shutil.move(dest_palace, archive_path) + source_palace = archive_path + src_db = os.path.join(source_palace, "chroma.sqlite3") + + # In-place only: drop chromadb's process-wide System registry so + # the new client at dest_palace builds a fresh System. Without + # this, ``create_collection`` raises "Collection already exists" + # because the cached System still holds the pre-rename schema. + # Cross-palace mode does not need this and would needlessly + # invalidate other callers' clients (see docstring warning). + try: + from chromadb.api.client import SharedSystemClient + + SharedSystemClient.clear_system_cache() + except Exception as exc: # noqa: BLE001 + print( + f" Warning: could not clear chromadb system cache ({exc!r}); " + "in-place rebuild may fail with 'Collection already exists'." + ) + + os.makedirs(dest_palace, exist_ok=True) + + backend = ChromaBackend() + counts: dict[str, int] = {} + + for cname in RECOVERABLE_COLLECTIONS: + print(f"\n [{cname}]") + upserted = _rebuild_one_collection( + backend=backend, + source_palace=source_palace, + dest_palace=dest_palace, + collection_name=cname, + batch_size=batch_size, + archive_path=archive_path, + counts_so_far=counts, + ) + counts[cname] = upserted + if upserted == 0: + print(f" no rows found for {cname} in source palace") + else: + print(f" done: {upserted} rows in {cname}") + + print(f"\n Rebuild complete. {sum(counts.values())} total rows.") + if archive_path is not None: + print(f" Original palace archived at: {archive_path}") + print(f"{'=' * 55}\n") + return counts + + def status(palace_path=None) -> dict: """Read-only health check: compare sqlite vs HNSW element counts. diff --git a/tests/test_repair.py b/tests/test_repair.py index bc770ddfbe..35e6a44c83 100644 --- a/tests/test_repair.py +++ b/tests/test_repair.py @@ -682,3 +682,294 @@ def flaky_detect(*args, **kwargs): # A backup file is still present — caller can roll back from it. leftover = [fn for fn in os.listdir(palace) if "max-seq-id-backup-" in fn] assert leftover + + +# ── extract_via_sqlite + rebuild_from_sqlite (#1308) ────────────────── +# +# These tests build real chromadb palaces in tmp_path rather than mocking +# the SQLite layer. The bug class they guard against is "extraction sees +# different rows than chromadb stored" — the only honest check is to let +# chromadb actually write rows and then read them back via the SQLite +# bypass. Mocking the SQLite cursor would defeat the test. + + +def _seed_palace(palace_path, collection_name, rows): + """Build a real chromadb palace at ``palace_path`` and add ``rows``. + + ``rows`` is a list of ``(id, document, metadata)`` tuples. Returns + the populated collection so callers can assert on the writer's view + of state before the SQLite read. + """ + from mempalace.backends.chroma import ChromaBackend + + backend = ChromaBackend() + col = backend.create_collection(str(palace_path), collection_name) + col.upsert( + ids=[r[0] for r in rows], + documents=[r[1] for r in rows], + metadatas=[r[2] for r in rows], + ) + return col + + +def test_extract_via_sqlite_returns_all_rows_with_metadata(tmp_path): + """Round-trip: a chromadb palace with N upserted rows returns those + same N rows when read via the SQLite bypass. + + Catches: anyone who breaks the segments/embeddings/embedding_metadata + JOIN, swaps the metadata vs vector segment, or changes how the + document is stored under the ``chroma:document`` key. + """ + rows = [ + (f"drawer_{i:03d}", f"document body {i}", {"wing": "test_wing", "room": f"r{i % 3}"}) + for i in range(25) + ] + _seed_palace(tmp_path, "mempalace_drawers", rows) + + extracted = list(repair.extract_via_sqlite(str(tmp_path), "mempalace_drawers")) + + assert len(extracted) == 25 + by_id = {emb_id: (doc, meta) for emb_id, doc, meta in extracted} + assert set(by_id) == {r[0] for r in rows} + for emb_id, doc, meta in rows: + got_doc, got_meta = by_id[emb_id] + assert got_doc == doc, f"document mangled for {emb_id}" + assert got_meta == meta, f"metadata mangled for {emb_id}: {got_meta!r}" + + +def test_extract_via_sqlite_preserves_typed_metadata(tmp_path): + """Chromadb stores int / float / bool / string in distinct typed + columns. Extraction must round-trip the original type, not coerce + everything to string. + + Catches: a regression where the SELECT order changes and ints come + back as None, or where the column-resolution rule prefers the wrong + column. + """ + rows = [ + ( + "drawer_typed", + "doc", + { + "wing": "w", + "chunk_index": 7, # int + "score": 0.42, # float + "is_active": True, # bool + }, + ), + ] + _seed_palace(tmp_path, "mempalace_drawers", rows) + + extracted = list(repair.extract_via_sqlite(str(tmp_path), "mempalace_drawers")) + assert len(extracted) == 1 + _, _, meta = extracted[0] + + assert meta["chunk_index"] == 7 and isinstance(meta["chunk_index"], int) + assert meta["score"] == 0.42 and isinstance(meta["score"], float) + assert meta["is_active"] is True + assert meta["wing"] == "w" + + +def test_extract_via_sqlite_unknown_collection_yields_nothing(tmp_path): + """Asking for a collection that isn't in the palace must return an + empty iterator, not silently fall back to another collection's + metadata segment. Seeds two real collections and queries for a third + name so a regression that drops the WHERE c.name=? filter would leak + rows from the seeded collections rather than passing. + """ + _seed_palace(tmp_path, "mempalace_drawers", [("d1", "doc", {"wing": "w"})]) + _seed_palace(tmp_path, "mempalace_closets", [("c1", "abbrev", {"wing": "w"})]) + assert list(repair.extract_via_sqlite(str(tmp_path), "not_a_real_collection")) == [] + + +def test_extract_via_sqlite_missing_palace_yields_nothing(tmp_path): + """No chroma.sqlite3 → empty iterator, no exception. Callers depend + on this when probing speculatively.""" + empty = tmp_path / "no_palace_here" + empty.mkdir() + assert list(repair.extract_via_sqlite(str(empty), "mempalace_drawers")) == [] + + +def test_rebuild_from_sqlite_roundtrips_via_real_chromadb(tmp_path): + """End-to-end: seed source palace, rebuild into a fresh dest, then + open dest with a fresh ChromaBackend and verify ``count()`` and + metadata filters return the original rows. Also asserts a closet + document round-trips so a future regression that re-embeds with the + wrong EF or swaps drawer/closet content would fail here. + + This is the single most important regression guard. If + ``rebuild_from_sqlite`` silently drops rows or mangles metadata, no + other test in this file would catch it because they all stop at the + extraction layer. + """ + from mempalace.backends.chroma import ChromaBackend + + source = tmp_path / "source" + dest = tmp_path / "dest" + + rows = [ + (f"drawer_{i:03d}", f"body {i}", {"wing": "alpha" if i % 2 else "beta", "room": "r0"}) + for i in range(40) + ] + _seed_palace(source, "mempalace_drawers", rows) + _seed_palace( + source, + "mempalace_closets", + [("closet_x", "abbrev pointer →drawer_001", {"wing": "alpha"})], + ) + + counts = repair.rebuild_from_sqlite(str(source), str(dest)) + assert counts == {"mempalace_drawers": 40, "mempalace_closets": 1} + + backend = ChromaBackend() + drawers = backend.get_collection(str(dest), "mempalace_drawers") + assert drawers.count() == 40 + alpha = drawers.get(where={"wing": "alpha"}) + assert len(alpha["ids"]) == 20 + + # Spot-check that document text round-trips for one specific drawer + # — protects against a regression where extraction or upsert order + # silently swaps document bodies between IDs. + one = drawers.get(ids=["drawer_007"], include=["documents", "metadatas"]) + assert one["documents"] == ["body 7"] + assert one["metadatas"][0]["wing"] == "alpha" + + # Closets: the AAAK index layer. Re-embedded with the same EF so a + # known closet ID and its document body must come back intact. + closets = backend.get_collection(str(dest), "mempalace_closets") + assert closets.count() == 1 + closet_row = closets.get(ids=["closet_x"], include=["documents", "metadatas"]) + assert closet_row["documents"] == ["abbrev pointer →drawer_001"] + assert closet_row["metadatas"][0] == {"wing": "alpha"} + + +def test_rebuild_from_sqlite_refuses_existing_dest(tmp_path): + """Refuse to write into a directory that already exists when source + and dest differ. Without this, an unattended re-run would silently + interleave a partial rebuild with whatever's already at dest. + """ + source = tmp_path / "source" + dest = tmp_path / "dest" + _seed_palace(source, "mempalace_drawers", [("d1", "doc", {"wing": "w"})]) + dest.mkdir() + # Drop a marker file so we can prove the dir wasn't touched. + (dest / "marker.txt").write_text("preexisting") + + counts = repair.rebuild_from_sqlite(str(source), str(dest)) + assert counts == {} + assert (dest / "marker.txt").read_text() == "preexisting" + assert not (dest / "chroma.sqlite3").exists() + + +def test_rebuild_from_sqlite_in_place_archives_when_opted_in(tmp_path): + """In-place rebuild (source == dest) with ``archive_existing_dest=True`` + must move the original aside to ``.pre-rebuild-`` and read + from the archive — the original drawer rows must survive in the new + palace, AND the archive itself must still contain the original rows. + + Catches: a refactor that moves the original out but then reads from + the now-empty original location, producing an empty rebuild; also + catches a swap that empties the archive after reading. + """ + palace = tmp_path / "palace" + rows = [(f"d{i}", f"body {i}", {"wing": "w", "room": "r"}) for i in range(15)] + _seed_palace(palace, "mempalace_drawers", rows) + + counts = repair.rebuild_from_sqlite(str(palace), str(palace), archive_existing_dest=True) + assert counts["mempalace_drawers"] == 15 + + archives = [p for p in tmp_path.iterdir() if p.name.startswith("palace.pre-rebuild-")] + assert len(archives) == 1 + assert (archives[0] / "chroma.sqlite3").exists() + # Archive must still hold the same row count via the SQLite bypass — + # proves the archive wasn't silently truncated as a side effect. + archived_rows = list(repair.extract_via_sqlite(str(archives[0]), "mempalace_drawers")) + assert len(archived_rows) == 15 + + from mempalace.backends.chroma import ChromaBackend + + rebuilt = ChromaBackend().get_collection(str(palace), "mempalace_drawers") + assert rebuilt.count() == 15 + + +def test_rebuild_from_sqlite_in_place_refuses_without_archive_flag(tmp_path): + """Source == dest without archive flag must abort untouched. The + most catastrophic possible regression of this code path is silently + deleting the only copy of the user's data.""" + palace = tmp_path / "palace" + _seed_palace(palace, "mempalace_drawers", [("d1", "doc", {"wing": "w"})]) + sqlite_before = (palace / "chroma.sqlite3").stat().st_size + + counts = repair.rebuild_from_sqlite(str(palace), str(palace)) + assert counts == {} + # Same file, untouched. + assert (palace / "chroma.sqlite3").stat().st_size == sqlite_before + archives = [p for p in tmp_path.iterdir() if "pre-rebuild" in p.name] + assert archives == [] + + +def test_rebuild_from_sqlite_source_missing_chroma_db(tmp_path): + """Source dir exists but has no chroma.sqlite3 → returns empty, + leaves dest untouched.""" + source = tmp_path / "source" + source.mkdir() + (source / "stray_file").write_text("not a palace") + dest = tmp_path / "dest" + + counts = repair.rebuild_from_sqlite(str(source), str(dest)) + assert counts == {} + assert not dest.exists() + + +def test_rebuild_from_sqlite_in_place_validates_source_before_archiving(tmp_path): + """In-place + archive_existing_dest=True with a dir that lacks + chroma.sqlite3 must NOT rename the dir before bailing. An earlier + revision archived first and validated second, leaving the user with + a renamed empty dir to manually undo. Catches that ordering bug. + """ + palace = tmp_path / "palace" + palace.mkdir() + (palace / "marker.txt").write_text("not a real palace") + + counts = repair.rebuild_from_sqlite(str(palace), str(palace), archive_existing_dest=True) + assert counts == {} + # No archive created — original dir still in place with its marker. + assert palace.exists() + assert (palace / "marker.txt").read_text() == "not a real palace" + archives = [p for p in tmp_path.iterdir() if "pre-rebuild" in p.name] + assert archives == [] + + +def test_rebuild_from_sqlite_raises_on_upsert_failure(tmp_path, monkeypatch): + """Mid-batch upsert failure must raise ``RebuildPartialError`` and + surface the failed collection + archive path so the user can recover. + Without this, an unattended script gets exit-code-zero on a partial + rebuild and the user discovers the data loss only when search starts + returning fewer hits. + """ + palace = tmp_path / "palace" + rows = [(f"d{i}", f"body {i}", {"wing": "w", "room": "r"}) for i in range(5)] + _seed_palace(palace, "mempalace_drawers", rows) + + # Make the very first upsert raise so we don't depend on batch + # boundary behavior. Patching ChromaCollection.upsert (the wrapper + # mempalace's backend returns) keeps the failure path realistic. + # ``monkeypatch`` is pytest's built-in fixture that auto-restores + # the original attribute when the test exits, so we don't need to + # undo this manually. + from mempalace.backends.chroma import ChromaCollection + + def boom(self, **kwargs): + raise RuntimeError("simulated chromadb upsert failure") + + monkeypatch.setattr(ChromaCollection, "upsert", boom) + + with pytest.raises(repair.RebuildPartialError) as excinfo: + repair.rebuild_from_sqlite(str(palace), str(palace), archive_existing_dest=True) + + err = excinfo.value + assert err.failed_collection == "mempalace_drawers" + assert err.partial_counts.get("mempalace_drawers") == 0 + assert err.archive_path is not None + assert os.path.isfile(os.path.join(err.archive_path, "chroma.sqlite3")) + assert err.dest_palace == os.path.abspath(str(palace)) From cb6bfd523149f976ae10264c80a8dd4ff733b25c Mon Sep 17 00:00:00 2001 From: Brian potter Date: Sat, 2 May 2026 12:11:52 -0500 Subject: [PATCH 53/77] chore: gitignore .envrc for direnv users Co-Authored-By: Claude Opus 4.7 (1M context) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1619ba8702..ba8ec102cb 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ venv/ # ChromaDB local data *.sqlite3-journal +.envrc From d92c741084c7a3d276ecd31c483931ff468bdd35 Mon Sep 17 00:00:00 2001 From: Brian potter Date: Sat, 2 May 2026 12:12:08 -0500 Subject: [PATCH 54/77] fix(repair): address PR #1310 review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five small hardening fixes for the from-sqlite rebuild path, all from mjc's review on #1310: - repair.py: drawers collection name now resolves from MempalaceConfig().collection_name via _drawers_collection_name() (closets stays fixed by design — AAAK index references drawer IDs by string). Lines up with the broader configured-collection work in #1312 so that PR can rebase cleanly on top. - repair.py: create_collection() moved inside the try block in _rebuild_one_collection so a Chroma "Collection already exists" failure surfaces as RebuildPartialError with archive_path, not an unstructured exception that strands the user without recovery instructions. - repair.py: rebuild_from_sqlite wraps backend lifetime in try/finally with backend.close() so PersistentClient handles to dest_palace are released on every exit path. The from-sqlite path post-dates #1285's lifecycle hardening of the legacy rebuild, so this needed its own cleanup. - cli.py: cmd_repair (from-sqlite mode) now exits non-zero when rebuild_from_sqlite returns {} (validation refusal sentinel), so unattended scripts/CI distinguish "invalid inputs" from a successful rebuild that legitimately found zero rows. - tests/test_repair.py: test_extract_via_sqlite_returns_all_rows_with_metadata now asserts every backing segment is scope='METADATA', locking in the segment-layout assumption against future regressions that point the JOIN at the VECTOR segment. New test coverage: - test_rebuild_from_sqlite_honors_configured_drawer_collection_name - test_cmd_repair_from_sqlite_validation_refusal_exits_nonzero - test_cmd_repair_from_sqlite_success_does_not_exit Co-Authored-By: Claude Opus 4.7 (1M context) --- mempalace/cli.py | 11 +++- mempalace/repair.py | 121 ++++++++++++++++++++++++++++++++----------- tests/test_cli.py | 61 ++++++++++++++++++++++ tests/test_repair.py | 89 +++++++++++++++++++++++++++++++ 4 files changed, 250 insertions(+), 32 deletions(-) diff --git a/mempalace/cli.py b/mempalace/cli.py index 468b76515a..95915eb27f 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -699,7 +699,7 @@ def cmd_repair(args): return try: - rebuild_from_sqlite( + counts = rebuild_from_sqlite( source_palace=source_path, dest_palace=palace_path, archive_existing_dest=archive_existing, @@ -713,6 +713,15 @@ def cmd_repair(args): f"Failed in collection: {exc.failed_collection}" ) sys.exit(1) + # An empty counts dict is rebuild_from_sqlite's documented signal + # for a validation refusal (missing source, existing dest, + # in-place without --archive-existing). The library already + # printed an actionable message; exit non-zero so unattended + # scripts/CI distinguish "invalid inputs" from a successful + # rebuild that legitimately found zero rows (which still returns + # a populated dict with 0-valued counts). + if not counts: + sys.exit(1) return db_path = os.path.join(palace_path, "chroma.sqlite3") diff --git a/mempalace/repair.py b/mempalace/repair.py index 7e98f0f1d6..90fb0cfabc 100644 --- a/mempalace/repair.py +++ b/mempalace/repair.py @@ -43,11 +43,46 @@ COLLECTION_NAME = "mempalace_drawers" -# Collections rebuilt by ``rebuild_from_sqlite``. Order matters for the -# upsert pass — drawers carry the bulk of the data, closets are the AAAK -# index layer and reference drawer IDs by string in their documents (no -# foreign-key validation, so ordering is informational, not load-bearing). -RECOVERABLE_COLLECTIONS = ("mempalace_drawers", "mempalace_closets") +# The closets collection (AAAK index layer) is intentionally fixed — +# closets reference drawer IDs by string and live alongside drawers in the +# same palace; renaming the closets collection per-deployment would break +# cross-palace AAAK lookups. Drawer collection name comes from config +# (see ``_recoverable_collections``). +CLOSETS_COLLECTION_NAME = "mempalace_closets" + + +def _drawers_collection_name() -> str: + """Resolve the drawers collection name from user config, falling back + to the module default ``COLLECTION_NAME`` if config is unreadable. + + Recovery flows must honor ``MempalaceConfig().collection_name`` so a + user with a non-default drawer collection (e.g. multi-palace setups) + rebuilds the right rows. Closets remain fixed — see + ``CLOSETS_COLLECTION_NAME``. + """ + try: + from .config import MempalaceConfig + + return MempalaceConfig().collection_name or COLLECTION_NAME + except Exception: + return COLLECTION_NAME + + +def _recoverable_collections() -> tuple[str, ...]: + """Collections rebuilt by ``rebuild_from_sqlite``, in upsert order. + + Drawers first (bulk data), then closets (AAAK index layer that + references drawer IDs by string in their documents — no + foreign-key validation, so ordering is informational, not + load-bearing). + """ + return (_drawers_collection_name(), CLOSETS_COLLECTION_NAME) + + +# Back-compat alias for callers that imported the constant. New code +# should call ``_recoverable_collections()`` so config changes are picked +# up at call time. +RECOVERABLE_COLLECTIONS = (COLLECTION_NAME, CLOSETS_COLLECTION_NAME) def _get_palace_path(): @@ -487,12 +522,11 @@ def _rebuild_one_collection( caller can stop the loop and print recovery instructions instead of silently shipping a partial palace. """ - col = backend.create_collection(dest_palace, collection_name) - ids: list[str] = [] docs: list[str] = [] metas: list[dict] = [] upserted = 0 + col = None def _flush() -> int: nonlocal upserted @@ -507,6 +541,14 @@ def _flush() -> int: return upserted try: + # ``create_collection`` lives inside the try so a Chroma-side + # "Collection already exists" failure (which can happen when the + # process-wide System cache still holds a pre-archive schema) is + # reported as a structured ``RebuildPartialError`` carrying + # ``archive_path`` — instead of an unstructured exception that + # strands the user without recovery instructions. + col = backend.create_collection(dest_palace, collection_name) + for emb_id, doc, meta in extract_via_sqlite(source_palace, collection_name): ids.append(emb_id) docs.append(doc or "") @@ -664,8 +706,14 @@ def rebuild_from_sqlite( Returns a ``{collection_name: row_count}`` dict so callers (CLI, tests) can verify the per-collection rebuild count without parsing - stdout. Returns ``{}`` on validation failures (missing source, - refusing to overwrite). Raises :class:`RebuildPartialError` if a + stdout. A successful rebuild always returns a dict with one key per + recoverable collection (values may be ``0`` when a collection is + legitimately empty in the source). The empty dict ``{}`` is reserved + for validation refusals (missing source DB, refusing to overwrite an + existing dest, in-place mode without ``archive_existing_dest``); CLI + callers should treat ``{}`` as an error and exit non-zero so CI and + scripts can distinguish "invalid inputs" from "successful recovery + that found zero rows." Raises :class:`RebuildPartialError` if a chromadb upsert fails partway through; the dest palace is left in place so the user can inspect what landed, and the in-place archive (when applicable) is reported in the error so the user can re-run @@ -765,31 +813,42 @@ def rebuild_from_sqlite( os.makedirs(dest_palace, exist_ok=True) + # Backend lifetime is wrapped in try/finally so the dest palace's + # PersistentClient handle (opened lazily inside ``create_collection`` + # / ``get_collection``) is released on every exit path: success, + # ``RebuildPartialError``, or any unexpected exception. Without this, + # a long-running process that calls ``rebuild_from_sqlite`` would + # leak SQLite/HNSW file handles into Chroma's ``SharedSystemClient`` + # cache, surfacing later as "Collection already exists" on the next + # in-place rebuild or as a Windows file-lock failure on cleanup + # (cf. #1285's lifecycle hardening for the legacy rebuild path). backend = ChromaBackend() counts: dict[str, int] = {} + try: + for cname in _recoverable_collections(): + print(f"\n [{cname}]") + upserted = _rebuild_one_collection( + backend=backend, + source_palace=source_palace, + dest_palace=dest_palace, + collection_name=cname, + batch_size=batch_size, + archive_path=archive_path, + counts_so_far=counts, + ) + counts[cname] = upserted + if upserted == 0: + print(f" no rows found for {cname} in source palace") + else: + print(f" done: {upserted} rows in {cname}") - for cname in RECOVERABLE_COLLECTIONS: - print(f"\n [{cname}]") - upserted = _rebuild_one_collection( - backend=backend, - source_palace=source_palace, - dest_palace=dest_palace, - collection_name=cname, - batch_size=batch_size, - archive_path=archive_path, - counts_so_far=counts, - ) - counts[cname] = upserted - if upserted == 0: - print(f" no rows found for {cname} in source palace") - else: - print(f" done: {upserted} rows in {cname}") - - print(f"\n Rebuild complete. {sum(counts.values())} total rows.") - if archive_path is not None: - print(f" Original palace archived at: {archive_path}") - print(f"{'=' * 55}\n") - return counts + print(f"\n Rebuild complete. {sum(counts.values())} total rows.") + if archive_path is not None: + print(f" Original palace archived at: {archive_path}") + print(f"{'=' * 55}\n") + return counts + finally: + backend.close() def status(palace_path=None) -> dict: diff --git a/tests/test_cli.py b/tests/test_cli.py index 6b4b7b3a5c..71ca63d004 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1097,3 +1097,64 @@ def test_reconfigure_stdio_is_noop_off_windows(): _reconfigure_stdio_utf8_on_windows() assert stdin.reconfigure_calls == [] + + +# ── cmd_repair: from-sqlite mode exit codes ────────────────────────── + + +@patch("mempalace.cli.MempalaceConfig") +def test_cmd_repair_from_sqlite_validation_refusal_exits_nonzero(mock_config_cls, tmp_path, capsys): + """When ``rebuild_from_sqlite`` returns ``{}`` for a validation + refusal (missing source DB, in-place without --archive-existing, + refusing to overwrite an existing dest), the CLI must surface a + non-zero exit so unattended scripts and CI distinguish "invalid + inputs" from "successful recovery that found zero rows." + + Catches: a regression where the CLI treats the validation-refusal + sentinel as success, leaving CI green on a no-op repair that should + have alerted an operator. + """ + palace_dir = tmp_path / "palace" + palace_dir.mkdir() + mock_config_cls.return_value.palace_path = str(palace_dir) + + args = argparse.Namespace( + palace=str(palace_dir), + mode="from-sqlite", + source=None, + archive_existing=False, + yes=True, + ) + with patch("mempalace.repair.rebuild_from_sqlite", return_value={}): + with pytest.raises(SystemExit) as excinfo: + cmd_repair(args) + assert excinfo.value.code == 1 + + +@patch("mempalace.cli.MempalaceConfig") +def test_cmd_repair_from_sqlite_success_does_not_exit(mock_config_cls, tmp_path): + """A successful from-sqlite rebuild — even one that finds zero rows + in a legitimately empty source palace — must NOT call ``sys.exit``. + A populated counts dict (with ``0`` values) is the success signal; + only the empty dict ``{}`` is reserved for validation refusal. + + Catches: a regression where ``if not counts`` is replaced by + ``if not sum(counts.values())`` or similar, conflating "empty source" + with "validation refused" and breaking idempotent recovery scripts. + """ + palace_dir = tmp_path / "palace" + palace_dir.mkdir() + mock_config_cls.return_value.palace_path = str(palace_dir) + + args = argparse.Namespace( + palace=str(palace_dir), + mode="from-sqlite", + source=None, + archive_existing=False, + yes=True, + ) + # Zero rows but per-collection keys present → success, no exit. + fake_counts = {"mempalace_drawers": 0, "mempalace_closets": 0} + with patch("mempalace.repair.rebuild_from_sqlite", return_value=fake_counts): + # Should return cleanly; no SystemExit raised. + cmd_repair(args) diff --git a/tests/test_repair.py b/tests/test_repair.py index 35e6a44c83..8ca72fb70d 100644 --- a/tests/test_repair.py +++ b/tests/test_repair.py @@ -719,6 +719,13 @@ def test_extract_via_sqlite_returns_all_rows_with_metadata(tmp_path): Catches: anyone who breaks the segments/embeddings/embedding_metadata JOIN, swaps the metadata vs vector segment, or changes how the document is stored under the ``chroma:document`` key. + + Also asserts every embedding row underlying the extraction lives in + a ``segments.scope = 'METADATA'`` segment. Document + metadata rows + are stored under METADATA in Chroma's segment layout while HNSW + files live under ``VECTOR``; locking that assumption in here means a + future refactor that accidentally points the JOIN at ``VECTOR`` + fails this test instead of silently regressing the recovery path. """ rows = [ (f"drawer_{i:03d}", f"document body {i}", {"wing": "test_wing", "room": f"r{i % 3}"}) @@ -736,6 +743,35 @@ def test_extract_via_sqlite_returns_all_rows_with_metadata(tmp_path): assert got_doc == doc, f"document mangled for {emb_id}" assert got_meta == meta, f"metadata mangled for {emb_id}: {got_meta!r}" + # Lock the segment-scope assumption directly against Chroma's on-disk + # layout so a future change that points the extraction JOIN at the + # VECTOR segment cannot pass this test. Query each extracted row's + # backing segment scope via the same SQLite tables ``extract_via_sqlite`` + # reads from. + sqlite_path = os.path.join(str(tmp_path), "chroma.sqlite3") + conn = sqlite3.connect(f"file:{sqlite_path}?mode=ro", uri=True) + try: + scopes = { + scope + for (scope,) in conn.execute( + """ + SELECT DISTINCT s.scope + FROM embeddings e + JOIN segments s ON e.segment_id = s.id + JOIN collections c ON s.collection = c.id + WHERE c.name = ? AND e.embedding_id IN ({}) + """.format(",".join("?" * len(extracted))), + ("mempalace_drawers", *(emb_id for emb_id, _, _ in extracted)), + ) + } + finally: + conn.close() + assert scopes == {"METADATA"}, ( + f"extraction is reading from segments scoped {scopes!r}; only " + "'METADATA' should back the document/metadata rows. If Chroma's " + "segment layout changed, update extract_via_sqlite's WHERE clause." + ) + def test_extract_via_sqlite_preserves_typed_metadata(tmp_path): """Chromadb stores int / float / bool / string in distinct typed @@ -973,3 +1009,56 @@ def boom(self, **kwargs): assert err.archive_path is not None assert os.path.isfile(os.path.join(err.archive_path, "chroma.sqlite3")) assert err.dest_palace == os.path.abspath(str(palace)) + + +def test_rebuild_from_sqlite_honors_configured_drawer_collection_name(tmp_path, monkeypatch): + """A user with a non-default drawers collection name (set via + ``MempalaceConfig().collection_name``) must have THAT collection + rebuilt — not the hardcoded ``mempalace_drawers``. + + Catches: a regression where the recovery path silently rebuilds the + default-name collection on a custom-named palace, leaving the user's + actual data unrebuilt while reporting "rebuild complete." This is + the failure mode reviewer mjc flagged on PR #1310 as needing to line + up with the configured-collection-name work in #1312. Closets stay + fixed (``mempalace_closets``) by design — the AAAK index references + drawer IDs by string and is not per-deployment configurable. + + Strategy: monkeypatch the lazy resolver so the test is hermetic and + does not depend on the global config file or env state. + """ + from mempalace.backends.chroma import ChromaBackend + + custom_drawers = "custom_drawers_xyz" + monkeypatch.setattr(repair, "_drawers_collection_name", lambda: custom_drawers) + + source = tmp_path / "source" + dest = tmp_path / "dest" + + drawer_rows = [(f"d{i}", f"body {i}", {"wing": "alpha"}) for i in range(3)] + closet_rows = [("closet_a", "abbrev →d0", {"wing": "alpha"})] + _seed_palace(source, custom_drawers, drawer_rows) + _seed_palace(source, "mempalace_closets", closet_rows) + + counts = repair.rebuild_from_sqlite(str(source), str(dest)) + + # Rebuilt under the custom name, not under the default "mempalace_drawers". + assert counts == {custom_drawers: 3, "mempalace_closets": 1} + + backend = ChromaBackend() + rebuilt_drawers = backend.get_collection(str(dest), custom_drawers) + assert rebuilt_drawers.count() == 3 + + # Default-name collection must NOT exist in dest — proves we did not + # silently fall back to the hardcoded name during rebuild. + try: + rebuilt_default = backend.get_collection(str(dest), "mempalace_drawers") + # If get_collection returns without raising, count() should be 0 + # (chromadb may auto-create on get with some EFs); a non-zero + # count would mean we wrote rows to the wrong collection. + assert rebuilt_default.count() == 0, ( + "rebuild leaked rows into the default-name collection on a " + "custom-name palace — recovery wrote to the wrong collection." + ) + except Exception: + pass # Expected: collection wasn't created. From e334e257bf4bb634cf4f31b4087f84f8b467938b Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Wed, 6 May 2026 04:52:18 -0300 Subject: [PATCH 55/77] fix(mcp): retry _get_collection once on transient failure (#1286) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A transient chromadb exception inside `_get_collection` was swallowed by the bare `except Exception: return None`, leaving every subsequent tool call hitting the same poisoned cache silently. The fix wraps the body in a `for attempt in range(2)` loop: on attempt 0 failure, log via `logger.exception(...)` and clear `_client_cache` / `_collection_cache` / `_metadata_cache` so the next iteration forces `_get_client()` to rebuild from scratch — that path now re-runs `quarantine_stale_hnsw` (per #1322), so the second attempt heals the common stale-handle case automatically. If both attempts fail, return `None` (matches the prior contract for permanent failures). Two new tests in `tests/test_mcp_server.py::TestCacheInvalidation`: - `test_get_collection_retries_once_on_exception` — first attempt raises via a monkeypatched `_get_client`, second attempt succeeds; assert the caller gets the collection back, not None. - `test_get_collection_returns_none_after_two_failures` — both attempts fail, assert we exhaust the loop and return None (no infinite retry). Surgical extraction from PR #1286, which carried the same fix idea (plus a fork-sync bundle that couldn't be merged); credit to the original author below. Co-authored-by: Jeffrey Hein --- mempalace/mcp_server.py | 148 +++++++++++++++++++++++---------------- tests/test_mcp_server.py | 65 +++++++++++++++++ 2 files changed, 152 insertions(+), 61 deletions(-) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 58f9ba9767..bbb9c937a0 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -326,68 +326,94 @@ def _get_client(): def _get_collection(create=False): - """Return the ChromaDB collection, caching the client between calls.""" - global _collection_cache, _metadata_cache, _metadata_cache_time - try: - client = _get_client() - # ChromaDB 1.x persists the EF *identity* (its ``name()``) with the - # collection but not the EF *instance/configuration*. So a reader or - # writer that omits ``embedding_function=`` silently gets chromadb's - # built-in ``DefaultEmbeddingFunction`` — its ``name()`` matches the - # one we spoof in ``mempalace.embedding`` (both report ``"default"``, - # the identity check passes), but the *provider list* is chromadb's - # default rather than the user's resolved device. On bleeding-edge - # interpreters (#1299: python 3.14 + chromadb 1.5.x on Apple Silicon) - # that default provider selection can SIGSEGV the host process on - # first ``col.add()``. The miner / Stop hook ingest path avoids this - # because it routes through ``ChromaBackend.get_collection``, which - # resolves the EF via ``ChromaBackend._resolve_embedding_function``; - # the MCP server bypassed that abstraction. Resolve the EF inside the - # branches that actually open a collection so warm-cache reads stay - # zero-cost. Reuse the backend helper so the two call sites can't - # drift on logging or fallback semantics. - if create: - ef = ChromaBackend._resolve_embedding_function() - ef_kwargs = {"embedding_function": ef} if ef is not None else {} - # hnsw:num_threads=1 disables ChromaDB's multi-threaded ParallelFor - # HNSW insert path, which has a race in repairConnectionsForUpdate / - # addPoint (see issues #974, #965). Set via metadata on fresh - # collections and re-applied via _pin_hnsw_threads() for legacy - # palaces whose collections were created before this fix (the - # runtime config does not persist cross-process in chromadb 1.5.x, - # so the retrofit runs every time _get_collection opens a cache). - # - # ChromaDB 1.5.x's Rust binding SIGSEGVs when get_or_create_collection - # is called with metadata that differs from what's stored. The split - # below skips the metadata-comparison codepath for existing - # collections, mirroring the backend-layer fix from #1262. - try: + """Return the ChromaDB collection, caching the client between calls. + + On failure, log the exception and retry once after clearing the client + and collection caches. Tools were silently returning ``None`` when a + cached client/collection went stale — typically after the chromadb + rust bindings invalidated a handle following an out-of-band write — + leaving the LLM with no diagnostic and no recovery path. The retry + forces ``_get_client()`` to rebuild from scratch (which re-runs + ``quarantine_stale_hnsw`` per #1322), so the second attempt heals the + common stale-handle / stale-HNSW case automatically. + """ + global _client_cache, _collection_cache, _metadata_cache, _metadata_cache_time + for attempt in range(2): + try: + client = _get_client() + # ChromaDB 1.x persists the EF *identity* (its ``name()``) with the + # collection but not the EF *instance/configuration*. So a reader or + # writer that omits ``embedding_function=`` silently gets chromadb's + # built-in ``DefaultEmbeddingFunction`` — its ``name()`` matches the + # one we spoof in ``mempalace.embedding`` (both report ``"default"``, + # the identity check passes), but the *provider list* is chromadb's + # default rather than the user's resolved device. On bleeding-edge + # interpreters (#1299: python 3.14 + chromadb 1.5.x on Apple Silicon) + # that default provider selection can SIGSEGV the host process on + # first ``col.add()``. The miner / Stop hook ingest path avoids this + # because it routes through ``ChromaBackend.get_collection``, which + # resolves the EF via ``ChromaBackend._resolve_embedding_function``; + # the MCP server bypassed that abstraction. Resolve the EF inside the + # branches that actually open a collection so warm-cache reads stay + # zero-cost. Reuse the backend helper so the two call sites can't + # drift on logging or fallback semantics. + if create: + ef = ChromaBackend._resolve_embedding_function() + ef_kwargs = {"embedding_function": ef} if ef is not None else {} + # hnsw:num_threads=1 disables ChromaDB's multi-threaded ParallelFor + # HNSW insert path, which has a race in repairConnectionsForUpdate / + # addPoint (see issues #974, #965). Set via metadata on fresh + # collections and re-applied via _pin_hnsw_threads() for legacy + # palaces whose collections were created before this fix (the + # runtime config does not persist cross-process in chromadb 1.5.x, + # so the retrofit runs every time _get_collection opens a cache). + # + # ChromaDB 1.5.x's Rust binding SIGSEGVs when get_or_create_collection + # is called with metadata that differs from what's stored. The split + # below skips the metadata-comparison codepath for existing + # collections, mirroring the backend-layer fix from #1262. + try: + raw = client.get_collection(_config.collection_name, **ef_kwargs) + except _ChromaNotFoundError: + raw = client.create_collection( + _config.collection_name, + metadata={ + "hnsw:space": "cosine", + "hnsw:num_threads": 1, + **_HNSW_BLOAT_GUARD, + }, + **ef_kwargs, + ) + _pin_hnsw_threads(raw) + _collection_cache = ChromaCollection(raw, palace_path=_config.palace_path) + _metadata_cache = None + _metadata_cache_time = 0 + elif _collection_cache is None: + ef = ChromaBackend._resolve_embedding_function() + ef_kwargs = {"embedding_function": ef} if ef is not None else {} raw = client.get_collection(_config.collection_name, **ef_kwargs) - except _ChromaNotFoundError: - raw = client.create_collection( - _config.collection_name, - metadata={ - "hnsw:space": "cosine", - "hnsw:num_threads": 1, - **_HNSW_BLOAT_GUARD, - }, - **ef_kwargs, - ) - _pin_hnsw_threads(raw) - _collection_cache = ChromaCollection(raw, palace_path=_config.palace_path) - _metadata_cache = None - _metadata_cache_time = 0 - elif _collection_cache is None: - ef = ChromaBackend._resolve_embedding_function() - ef_kwargs = {"embedding_function": ef} if ef is not None else {} - raw = client.get_collection(_config.collection_name, **ef_kwargs) - _pin_hnsw_threads(raw) - _collection_cache = ChromaCollection(raw, palace_path=_config.palace_path) - _metadata_cache = None - _metadata_cache_time = 0 - return _collection_cache - except Exception: - return None + _pin_hnsw_threads(raw) + _collection_cache = ChromaCollection(raw, palace_path=_config.palace_path) + _metadata_cache = None + _metadata_cache_time = 0 + return _collection_cache + except Exception: + logger.exception( + "_get_collection attempt %d/2 failed (palace=%s, create=%s)", + attempt + 1, + _config.palace_path, + create, + ) + if attempt == 0: + # Reset all caches so the next attempt forces _get_client() + # to rebuild the chromadb client from scratch — that path + # re-runs quarantine_stale_hnsw (#1322) and reopens the + # collection cleanly, healing the common stale-handle case. + _client_cache = None + _collection_cache = None + _metadata_cache = None + _metadata_cache_time = 0 + return None def _no_palace(): diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index c073830daa..ae20bf32e3 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -1259,6 +1259,71 @@ def _spy_create(self, name, **kwargs): assert "embedding_function" in kwargs assert kwargs["embedding_function"] is not None + def test_get_collection_retries_once_on_exception(self, monkeypatch, config, palace_path, kg): + """Regression: a transient failure inside _get_collection must trigger + one retry after clearing the client/collection caches, not silently + return None. + + Before this fix, a stale chromadb handle (e.g. the rust bindings + invalidating after an out-of-band write) would raise inside the + single ``try`` block, get swallowed by ``except Exception: return + None``, and every subsequent tool call would hit the same poisoned + cache returning None. The retry forces ``_get_client()`` to rebuild + the client (which re-runs ``quarantine_stale_hnsw`` per #1322), so + the second attempt heals the common stale-handle case. + """ + _patch_mcp_server(monkeypatch, config, kg) + _client, _col = _get_collection(palace_path, create=True) + del _client + from mempalace import mcp_server + + # Force a cold cache so the first call goes through the open path. + mcp_server._client_cache = None + mcp_server._collection_cache = None + + real_get_client = mcp_server._get_client + attempts = {"count": 0} + + def flaky_get_client(): + attempts["count"] += 1 + if attempts["count"] == 1: + raise RuntimeError("simulated transient chromadb failure") + return real_get_client() + + monkeypatch.setattr(mcp_server, "_get_client", flaky_get_client) + + col = mcp_server._get_collection() + + # Both attempts ran and the second succeeded. + assert attempts["count"] == 2 + assert col is not None + + def test_get_collection_returns_none_after_two_failures( + self, monkeypatch, config, palace_path, kg + ): + """If both attempts fail, return None (matches the prior contract for + permanent failures — only the transient case is now self-healing).""" + _patch_mcp_server(monkeypatch, config, kg) + _client, _col = _get_collection(palace_path, create=True) + del _client + from mempalace import mcp_server + + mcp_server._client_cache = None + mcp_server._collection_cache = None + + attempts = {"count": 0} + + def always_fails(): + attempts["count"] += 1 + raise RuntimeError("permanent chromadb failure") + + monkeypatch.setattr(mcp_server, "_get_client", always_fails) + + col = mcp_server._get_collection() + + assert attempts["count"] == 2 + assert col is None + class TestKGLazyCache: """Lazy per-path KnowledgeGraph cache (issue #1136).""" From bddba59ae3ffe1fab6a281b7090f9945287a98d6 Mon Sep 17 00:00:00 2001 From: MillaJ <232237854+milla-jovovich@users.noreply.github.com> Date: Wed, 6 May 2026 12:35:01 -0700 Subject: [PATCH 56/77] docs: add 30-day expiry callout + ship 4 auto-save tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a brief [!IMPORTANT] callout at the top of the README pointing users to the urgent announcement at #1388. Claude Code auto-deletes local JSONL transcripts after 30 days; users without the auto-save hooks wired are losing transcript data off the rolling window. Ships 4 small standalone tools at tools/: - backup_claude_jsonls.sh — rsync ~/.claude/projects/ to a safe folder - render_jsonl.py — convert JSONL transcripts to readable text - find_orphan_claude_jsonls.sh — scan backup locations for orphan Claude Code transcripts (multi-line shape detection + topic preview) - save.md — Claude Code slash command for manual /save into MemPalace Tools verified by independent agent against v3.3.4 source. Read-only on user data. POSIX bash + Python stdlib only. --- README.md | 4 + tools/backup_claude_jsonls.sh | 39 ++++++++++ tools/find_orphan_claude_jsonls.sh | 115 +++++++++++++++++++++++++++++ tools/render_jsonl.py | 71 ++++++++++++++++++ tools/save.md | 26 +++++++ 5 files changed, 255 insertions(+) create mode 100755 tools/backup_claude_jsonls.sh create mode 100755 tools/find_orphan_claude_jsonls.sh create mode 100755 tools/render_jsonl.py create mode 100644 tools/save.md diff --git a/README.md b/README.md index 8157fcae66..d82bcd271d 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,10 @@ > domain — including `mempalace.tech` — is an impostor and may distribute > malware. Details and timeline: [docs/HISTORY.md](docs/HISTORY.md). +> [!IMPORTANT] +> **🚨 Claude Code sessions expire in 30 days w/out auto-save hooks wired!** **[Read this →](https://github.com/MemPalace/mempalace/discussions/1388)** + +
MemPalace diff --git a/tools/backup_claude_jsonls.sh b/tools/backup_claude_jsonls.sh new file mode 100755 index 0000000000..f252de0ec3 --- /dev/null +++ b/tools/backup_claude_jsonls.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# backup_claude_jsonls.sh +# +# Claude Code stores every conversation as a JSONL transcript at +# ~/.claude/projects//.jsonl +# Anthropic auto-deletes those files after 30 DAYS: +# https://docs.claude.com/en/docs/claude-code/data-usage +# +# This script copies them, read-only, into ~/Documents/Claude_JSONL_Backup/ +# so the 30-day clock no longer applies. Re-run any time — rsync is incremental. +# It NEVER deletes, modifies, or touches files inside ~/.claude/. + +set -eu + +SRC="${HOME}/.claude/projects/" +DST="${HOME}/Documents/Claude_JSONL_Backup/" + +[ -d "$SRC" ] || { echo "ERROR: $SRC does not exist."; exit 1; } +mkdir -p "$DST" + +echo "Backing up $SRC -> $DST" +rsync -a --times "$SRC" "$DST" + +src_count=$(find "$SRC" -type f -name '*.jsonl' | wc -l | tr -d ' ') +dst_count=$(find "$DST" -type f -name '*.jsonl' | wc -l | tr -d ' ') +oldest=$(find "$DST" -type f -name '*.jsonl' -exec stat -f '%Sm %N' -t '%Y-%m-%d' {} \; 2>/dev/null \ + || find "$DST" -type f -name '*.jsonl' -printf '%TY-%Tm-%Td %p\n' 2>/dev/null) +oldest_date=$(echo "$oldest" | sort | head -n 1 | awk '{print $1}') +newest_date=$(echo "$oldest" | sort | tail -n 1 | awk '{print $1}') + +echo "Source JSONL count : $src_count" +echo "Backup JSONL count : $dst_count" +echo "Oldest backup file : ${oldest_date:-n/a}" +echo "Newest backup file : ${newest_date:-n/a}" + +if [ "$src_count" -ne "$dst_count" ]; then + echo "FAIL: count mismatch ($src_count vs $dst_count)"; exit 2 +fi +echo "OK: backup verified." diff --git a/tools/find_orphan_claude_jsonls.sh b/tools/find_orphan_claude_jsonls.sh new file mode 100755 index 0000000000..43523f50ec --- /dev/null +++ b/tools/find_orphan_claude_jsonls.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# find_orphan_claude_jsonls.sh — v3 (multi-line shape + verb-aware preview) +# ----------------------------------------------------------------------------- +# Finds Claude Code conversation transcripts (.jsonl) that may have survived in +# backup/sync locations. Claude Code stores transcripts at +# ~/.claude/projects//.jsonl and auto-deletes them locally +# after 30 days. If your machine syncs to iCloud, Dropbox, Google Drive, +# OneDrive, Time Machine, or you copied transcripts elsewhere manually, those +# copies still exist. This script finds them and shows a topic preview from +# the first substantive user message — strips leading filler interjections +# ("ok so", "oh", "well", "hey") so previews surface the actual content. +# +# Read-only. Safe to re-run. +# ----------------------------------------------------------------------------- +set -eu + +LOCATIONS=( + "$HOME/Library/Mobile Documents" "$HOME/Dropbox" "$HOME/Google Drive" + "$HOME/OneDrive" "$HOME/Documents" "$HOME/Desktop" "/Volumes" +) + +TMP="$(mktemp)"; trap 'rm -f "$TMP" "$TMP.s"' EXIT + +printf "Scanning backup locations" >&2 +for loc in "${LOCATIONS[@]}"; do + [ -d "$loc" ] || continue + printf "." >&2 + while IFS= read -r -d '' f; do + # Combined: shape detection (multi-line) + verb-aware topic preview + if preview="$(python3 - "$f" 2>/dev/null <<'PYEOF' +import json, sys, re + +# Single-word/short greetings — message gets skipped entirely if it is just one of these +GREETINGS = {'hi','hey','hello','thanks','thank you','ok','okay','yes','no', + 'sure','cool','great','good','done','yep','nope','perfect','copy'} + +# Leading filler — interjections that get STRIPPED from the start of a message +# before the preview is taken. Iterative — handles "ok so well, then..." → "then..." +LEADING_FILLER = re.compile( + r'^(?:ok(?:ay)?|so|oh|well|anyway|btw|hmm+|um+|uh+|hey|hi|hello|right|' + r'yes|no|sure|cool|great|good|listen|look|wait|actually|alright|gotcha|' + r'yeah|yep|nope|nah)\b[\s,!.?:;-]*', + re.IGNORECASE +) + +path = sys.argv[1] +shape_ok = False +preview = "" +try: + with open(path, 'r', errors='replace') as fh: + for i, line in enumerate(fh): + if i >= 30: break + try: + d = json.loads(line) + except Exception: + continue + if not isinstance(d, dict): continue + # Shape check — accept if any line in first 30 has session fields + if not shape_ok and 'sessionId' in d and 'timestamp' in d and 'message' in d: + shape_ok = True + # Preview — first user message after stripping leading filler + if not preview: + role = d.get('type', '') or d.get('message', {}).get('role', '') + if role == 'user': + content = d.get('message', {}).get('content', '') + if isinstance(content, list): + text = ' '.join( + c.get('text', '') for c in content + if isinstance(c, dict) and c.get('type') == 'text' + ) + elif isinstance(content, str): + text = content + else: + text = '' + text = re.sub(r'\s+', ' ', text).strip() + # Skip messages that are pure greetings + if text.lower() in GREETINGS: + continue + # Iteratively strip leading filler tokens until stable + prev_text = None + while prev_text != text: + prev_text = text + text = LEADING_FILLER.sub('', text).strip() + # Skip if what remains is too short + if len(text) < 20: + continue + preview = text[:80] + ('...' if len(text) > 80 else '') + if shape_ok and preview: break +except Exception: + pass +if shape_ok: + print(preview if preview else "(no preview — first 30 lines were greetings or short)") + sys.exit(0) +sys.exit(1) +PYEOF +)"; then + mtime="$(stat -f '%Sm' -t '%Y-%m-%d' "$f" 2>/dev/null || stat -c '%y' "$f" 2>/dev/null | cut -d' ' -f1)" + size="$(stat -f '%z' "$f" 2>/dev/null || stat -c '%s' "$f" 2>/dev/null)" + printf '%s\t%s\t%s\t%s\n' "$mtime" "$size" "$f" "$preview" >>"$TMP" + fi + done < <(find "$loc" -type f -name '*.jsonl' -print0 2>/dev/null) +done +printf "\n" >&2 + +count=$(wc -l <"$TMP" | tr -d ' ') +if [ "$count" -eq 0 ]; then + echo "No orphan Claude Code transcripts found in scanned backup locations." + exit 0 +fi +sort -k1,1 "$TMP" >"$TMP.s" +oldest="$(head -n 1 "$TMP.s" | cut -f1)" +newest="$(tail -n 1 "$TMP.s" | cut -f1)" +echo "Found $count orphan Claude Code transcript(s). Oldest: $oldest Newest: $newest" +echo "----------------------------------------------------------------------" +awk -F'\t' '{ printf "%s %10s %s\n \"%s\"\n\n", $1, $2, $3, $4 }' "$TMP.s" diff --git a/tools/render_jsonl.py b/tools/render_jsonl.py new file mode 100755 index 0000000000..3d74c005f0 --- /dev/null +++ b/tools/render_jsonl.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""render_jsonl.py — turn one Claude Code JSONL transcript into readable text. + +Claude Code stores conversations at ~/.claude/projects//.jsonl and +Anthropic auto-deletes them after 30 days +(https://docs.claude.com/en/docs/claude-code/data-usage). This script renders a +JSONL into a clean .txt so you can keep / read / share it without the tooling. + +Usage: + python3 render_jsonl.py [output.txt] + +Stdlib only. Python 3.9+. Read-only on the input. +""" +import json, sys +from pathlib import Path + +def extract_text(content): + if isinstance(content, str): + return content.strip() + if isinstance(content, list): + parts = [] + for blk in content: + if isinstance(blk, dict) and blk.get("type") == "text": + t = (blk.get("text") or "").strip() + if t: + parts.append(t) + return "\n".join(parts) + return "" + +def main(): + if len(sys.argv) < 2: + print(__doc__); sys.exit(1) + src = Path(sys.argv[1]) + if not src.is_file(): + print(f"ERROR: not a file: {src}"); sys.exit(1) + out = open(sys.argv[2], "w", encoding="utf-8") if len(sys.argv) > 2 else sys.stdout + + turns, stamps = [], [] + for raw in src.read_text(encoding="utf-8", errors="replace").splitlines(): + if not raw.strip(): + continue + try: + obj = json.loads(raw) + except json.JSONDecodeError: + continue + role = obj.get("type") or (obj.get("message") or {}).get("role") + if role not in ("user", "assistant"): + continue + msg = obj.get("message") or obj + text = extract_text(msg.get("content")) + if not text: + continue + ts = obj.get("timestamp") or "" + if ts: stamps.append(ts) + turns.append((ts, role, text)) + + header = [ + f"# Claude Code transcript: {src}", + f"# Total turns: {len(turns)}", + f"# Date range : {min(stamps) if stamps else 'n/a'} -> {max(stamps) if stamps else 'n/a'}", + "#" + "-" * 70, "", + ] + out.write("\n".join(header)) + for ts, role, text in turns: + out.write(f"\n[{ts}] {role.upper()}\n{text}\n\n{'-'*72}\n") + if out is not sys.stdout: + out.close() + print(f"Wrote {len(turns)} turns to {sys.argv[2]}") + +if __name__ == "__main__": + main() diff --git a/tools/save.md b/tools/save.md new file mode 100644 index 0000000000..914156b411 --- /dev/null +++ b/tools/save.md @@ -0,0 +1,26 @@ +--- +description: Save the current Claude Code session into MemPalace. Idempotent — won't dupe. +--- + +# /save + +Save the current Claude Code session into MemPalace. Run this when you +want a checkpoint. Safe to run repeatedly — drawer IDs are content-hashed +so re-running on the same session overwrites in place, no duplicates. + +Behavior: + +1. Find the current session's JSONL transcript path (Claude Code passes + it via the conversation context — look for `~/.claude/projects/` paths). +2. Run via bash: + + ``` + mempalace mine "" --mode convos --wing claude_imports + ``` + +3. If the user supplied an argument after `/save`, use it as the wing name + instead of `claude_imports` (e.g. `/save my_research` → + `--wing my_research`). +4. Report back: how many drawers were filed, into which wing/room. + +Requires `mempalace` to be installed (`pip install mempalace`). From 921ff5a6faf753130ee5b6e9666daf6eec2bfc65 Mon Sep 17 00:00:00 2001 From: MillaJ <232237854+milla-jovovich@users.noreply.github.com> Date: Wed, 6 May 2026 15:39:08 -0700 Subject: [PATCH 57/77] fix(tools/render_jsonl): split chained statements per ruff 0.4.x Addresses CI lint feedback on PR #1391. No behavior change. - Split `import json, sys` into separate lines (E401) - Split chained `print(...); sys.exit(1)` into two lines (E702, two occurrences) - Split inline `if ts: stamps.append(ts)` into two lines (E701) Verified: `ruff check tools/render_jsonl.py` reports "All checks passed!" Tool still renders correctly (3 turns from a real JSONL test, identical output to pre-fix). --- tools/render_jsonl.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/render_jsonl.py b/tools/render_jsonl.py index 3d74c005f0..3372ee1576 100755 --- a/tools/render_jsonl.py +++ b/tools/render_jsonl.py @@ -11,7 +11,8 @@ Stdlib only. Python 3.9+. Read-only on the input. """ -import json, sys +import json +import sys from pathlib import Path def extract_text(content): @@ -29,10 +30,12 @@ def extract_text(content): def main(): if len(sys.argv) < 2: - print(__doc__); sys.exit(1) + print(__doc__) + sys.exit(1) src = Path(sys.argv[1]) if not src.is_file(): - print(f"ERROR: not a file: {src}"); sys.exit(1) + print(f"ERROR: not a file: {src}") + sys.exit(1) out = open(sys.argv[2], "w", encoding="utf-8") if len(sys.argv) > 2 else sys.stdout turns, stamps = [], [] @@ -51,7 +54,8 @@ def main(): if not text: continue ts = obj.get("timestamp") or "" - if ts: stamps.append(ts) + if ts: + stamps.append(ts) turns.append((ts, role, text)) header = [ From 7c679ba6250fd8cc57af24a22ce9e19b67b20429 Mon Sep 17 00:00:00 2001 From: MillaJ <232237854+milla-jovovich@users.noreply.github.com> Date: Wed, 6 May 2026 16:12:34 -0700 Subject: [PATCH 58/77] fix(tools/render_jsonl): apply ruff format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Earlier commit fixed ruff lint but missed the formatter check. This applies `ruff format` — adds standard PEP8 blank lines between functions, splits one inline list. No behavior change. Verified: both `ruff format --check` and `ruff check` pass cleanly. Tool still renders correctly. --- tools/render_jsonl.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/render_jsonl.py b/tools/render_jsonl.py index 3372ee1576..2bec0dab2a 100755 --- a/tools/render_jsonl.py +++ b/tools/render_jsonl.py @@ -11,10 +11,12 @@ Stdlib only. Python 3.9+. Read-only on the input. """ + import json import sys from pathlib import Path + def extract_text(content): if isinstance(content, str): return content.strip() @@ -28,6 +30,7 @@ def extract_text(content): return "\n".join(parts) return "" + def main(): if len(sys.argv) < 2: print(__doc__) @@ -62,7 +65,8 @@ def main(): f"# Claude Code transcript: {src}", f"# Total turns: {len(turns)}", f"# Date range : {min(stamps) if stamps else 'n/a'} -> {max(stamps) if stamps else 'n/a'}", - "#" + "-" * 70, "", + "#" + "-" * 70, + "", ] out.write("\n".join(header)) for ts, role, text in turns: @@ -71,5 +75,6 @@ def main(): out.close() print(f"Wrote {len(turns)} turns to {sys.argv[2]}") + if __name__ == "__main__": main() From 670aba974f7fd537bc48bab63c90599972bb304c Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Thu, 7 May 2026 07:37:25 -0300 Subject: [PATCH 59/77] test(repair): close ChromaBackend in _seed_palace to release Windows file locks The helper opened a chromadb PersistentClient via ChromaBackend and never closed it, leaving rust-side SQLite/HNSW file locks alive after the helper returned. On Windows that blocks the in-place archive rename inside rebuild_from_sqlite with WinError 32 on data_level0.bin, causing test_rebuild_from_sqlite_in_place_archives_when_opted_in and test_rebuild_from_sqlite_raises_on_upsert_failure to fail in the test-windows CI job. No test consumes the returned collection, so closing the backend in a try/finally is safe and drops the return. --- tests/test_repair.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/test_repair.py b/tests/test_repair.py index 8ca72fb70d..0277d9dbc9 100644 --- a/tests/test_repair.py +++ b/tests/test_repair.py @@ -696,20 +696,24 @@ def flaky_detect(*args, **kwargs): def _seed_palace(palace_path, collection_name, rows): """Build a real chromadb palace at ``palace_path`` and add ``rows``. - ``rows`` is a list of ``(id, document, metadata)`` tuples. Returns - the populated collection so callers can assert on the writer's view - of state before the SQLite read. + ``rows`` is a list of ``(id, document, metadata)`` tuples. """ from mempalace.backends.chroma import ChromaBackend backend = ChromaBackend() - col = backend.create_collection(str(palace_path), collection_name) - col.upsert( - ids=[r[0] for r in rows], - documents=[r[1] for r in rows], - metadatas=[r[2] for r in rows], - ) - return col + try: + col = backend.create_collection(str(palace_path), collection_name) + col.upsert( + ids=[r[0] for r in rows], + documents=[r[1] for r in rows], + metadatas=[r[2] for r in rows], + ) + finally: + # Release chromadb's rust-side SQLite/HNSW file locks before the + # caller proceeds. Without this, an in-place rebuild on Windows + # fails with WinError 32 on data_level0.bin during the archive + # rename (cf. PR #1310 test-windows job). + backend.close() def test_extract_via_sqlite_returns_all_rows_with_metadata(tmp_path): From 7cf9b1758222a525e9285f979844797d92c1d59e Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Thu, 7 May 2026 07:53:28 -0300 Subject: [PATCH 60/77] fix(repair): quote ChromaBackend annotation for Python 3.9 compatibility `backend: ChromaBackend | None = None` evaluates the X | None union eagerly at function-definition time, which Python 3.9 rejects with TypeError: unsupported operand type(s) for |: 'ABCMeta' and 'NoneType' since the new union syntax is 3.10+. Quoting matches the existing forward-reference style in repair.py (sqlite_drawer_count, etc.) and defers evaluation, restoring 3.9 compatibility. --- mempalace/repair.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mempalace/repair.py b/mempalace/repair.py index 0585405e11..d7af0d95a1 100644 --- a/mempalace/repair.py +++ b/mempalace/repair.py @@ -593,7 +593,7 @@ def status(palace_path=None) -> dict: # --------------------------------------------------------------------------- -def _close_chroma_handles(palace_path: str, backend: ChromaBackend | None = None) -> None: +def _close_chroma_handles(palace_path: str, backend: "ChromaBackend | None" = None) -> None: """Drop ChromaBackend + chromadb singleton caches so OS mmap handles release. When ``backend`` is provided, close the live instance so rollback/restore From 5488e7bb2286e5d5770d68c2719acefe92efba41 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Thu, 7 May 2026 08:56:41 -0300 Subject: [PATCH 61/77] fix(miner): harden Windows mine against ONNX bad_alloc + silent partial exits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three small changes that together address the failure modes in #1296: 1. Add pnpm-lock.yaml and yarn.lock to SKIP_FILENAMES, mirroring the existing package-lock.json rule. A 24K-line pnpm-lock.yaml produced ~1124 chunks in one batch and tripped onnxruntime bad_alloc on Windows; pnpm/yarn lockfiles are no more useful to mine than npm's. 2. Skip any file that produces more than MAX_CHUNKS_PER_FILE (500) chunks, with a clear log line. Catches the broader class — generated CSV/JSON, build artifacts, etc. — that the named-file SKIP list will never fully cover. The cap is conservative (500 chunks * 800 chars ≈ 400 KB of source) so legitimate hand-written content still mines. 3. Print a partial-progress summary on any exception in _mine_impl, not just KeyboardInterrupt, then re-raise. Without this, an arbitrary exception (ONNX bad_alloc, chromadb HNSW error, OS fault) propagates silently — the operator sees only the last progress line and assumes the mine succeeded. The new path mirrors the KeyboardInterrupt summary (files_processed, drawers_filed, last_file) plus the exception type and message, then re-raises so the original traceback surfaces and the exit code is non-zero. Tests cover: SKIP_FILENAMES contents, the chunk-cap path returning (0, room) with no upserts, and the new mine-aborted summary surfacing both the partial counters and the exception class. --- mempalace/miner.py | 34 ++++++++++++++++++++ tests/test_miner.py | 77 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/mempalace/miner.py b/mempalace/miner.py index 88734c9bb6..6aeddd4609 100644 --- a/mempalace/miner.py +++ b/mempalace/miner.py @@ -66,6 +66,8 @@ "mempal.yml", ".gitignore", "package-lock.json", + "pnpm-lock.yaml", + "yarn.lock", } CHUNK_SIZE = 800 # chars per drawer @@ -73,6 +75,13 @@ MIN_CHUNK_SIZE = 50 # skip tiny chunks DRAWER_UPSERT_BATCH_SIZE = 1000 MAX_FILE_SIZE = 500 * 1024 * 1024 # 500 MB — skip files larger than this. +# A single file producing more chunks than this is almost always a generated +# artifact (CSV/JSON dump, lockfile not in SKIP_FILENAMES, etc.). Embedding +# thousands of chunks from one file in one batch has triggered ONNX runtime +# `bad allocation` errors on Windows (#1296). The cap is conservative: a +# 500-chunk file at CHUNK_SIZE=800 is ~400 KB of source, which covers most +# legitimate hand-written content while bounding the worst-case batch. +MAX_CHUNKS_PER_FILE = 500 # Long Claude Code sessions and large transcript exports routinely exceed # 10 MB. The cap exists as a defensive rail against pathological binary # files, not as a limit on legitimate text. Per-drawer size is bounded @@ -825,6 +834,13 @@ def process_file( room = detect_room(filepath, content, rooms, project_path) chunks = chunk_text(content, source_file) + if len(chunks) > MAX_CHUNKS_PER_FILE: + print( + f" ! [skip] {filepath.name[:50]:50} produced {len(chunks)} chunks " + f"(> {MAX_CHUNKS_PER_FILE}); add to SKIP_FILENAMES or .gitignore" + ) + return 0, room + if dry_run: print(f" [DRY RUN] {filepath.name} -> room:{room} ({len(chunks)} drawers)") return len(chunks), room @@ -1167,6 +1183,24 @@ def _mine_impl( "already-filed drawers are\n upserted idempotently and will not duplicate.\n" ) sys.exit(130) + except Exception as exc: + # Without this, an arbitrary exception (ONNX bad_alloc, chromadb HNSW + # error, OS fault) propagates and the process exits with no completion + # banner — the operator sees only the final progress line and assumes + # the mine succeeded (#1296). Print the partial-progress summary the + # way we do for KeyboardInterrupt, then re-raise so the original + # traceback still surfaces and the exit code is non-zero. + print("\n\n Mine aborted by exception.") + print(f" files_processed: {files_processed}/{len(files)}") + print(f" drawers_filed: {total_drawers}") + print(f" last_file: {last_file or ''}") + print(f" error: {type(exc).__name__}: {exc}") + print( + f"\n Re-run `mempalace mine {shlex.quote(project_dir)}` after addressing " + "the cause — already-filed\n drawers are upserted idempotently and will " + "not duplicate.\n" + ) + raise finally: # Clean up the hooks-side PID lock if it points at us. Stale # entries already pass _pid_alive() == False on POSIX, but diff --git a/tests/test_miner.py b/tests/test_miner.py index 10124eefd3..10dd33d32c 100644 --- a/tests/test_miner.py +++ b/tests/test_miner.py @@ -699,6 +699,83 @@ def fake_process_file(*args, **kwargs): assert f"mempalace mine {shlex.quote(str(project_root))}" in out +def test_skip_filenames_includes_lockfiles(): + """pnpm-lock.yaml and yarn.lock must be skipped alongside package-lock.json + so a Windows mine over a typical JS monorepo doesn't OOM the ONNX embedder + on a 24K-line lockfile (#1296).""" + from mempalace import miner + + assert "package-lock.json" in miner.SKIP_FILENAMES + assert "pnpm-lock.yaml" in miner.SKIP_FILENAMES + assert "yarn.lock" in miner.SKIP_FILENAMES + + +def test_process_file_skips_when_chunks_exceed_max(tmp_path, monkeypatch): + """A file producing more than MAX_CHUNKS_PER_FILE chunks must be skipped + with a clear message and zero upserts. Generated artifacts (CSVs, lock + files not in SKIP_FILENAMES) hit this — the cap is what prevents ONNX + bad_alloc on Windows when the embedder is asked to swallow thousands of + chunks in one batch (#1296).""" + from unittest.mock import MagicMock + + from mempalace import miner + + monkeypatch.setattr(miner, "MAX_CHUNKS_PER_FILE", 5) + over_cap = [{"content": f"chunk {i}", "chunk_index": i} for i in range(7)] + monkeypatch.setattr(miner, "chunk_text", lambda content, source_file: over_cap) + + source = tmp_path / "huge.csv" + source.write_text("col1,col2\n" + "x,y\n" * 500, encoding="utf-8") + col = MagicMock() + col.get.return_value = {"ids": []} + + drawers, room = miner.process_file( + source, + tmp_path, + col, + "wing", + [{"name": "general", "description": "General"}], + "agent", + False, + ) + + assert drawers == 0 + col.upsert.assert_not_called() + + +def test_mine_arbitrary_exception_prints_summary_and_reraises(tmp_path, capsys): + """A non-KeyboardInterrupt exception mid-mine must surface a summary + banner before propagating, so users don't see a silent exit-0 with no + completion message (#1296 Failure 2). Re-raise preserves the traceback + and yields a non-zero exit code.""" + import pytest + from unittest.mock import patch + + project_root = tmp_path / "proj" + project_root.mkdir() + _make_minable_project(project_root, n_files=4) + palace_path = project_root / "palace" + + call_count = {"n": 0} + + def fake_process_file(*args, **kwargs): + call_count["n"] += 1 + if call_count["n"] == 2: + raise RuntimeError("simulated ONNX bad_alloc") + return (1, "general") + + with patch("mempalace.miner.process_file", side_effect=fake_process_file): + with pytest.raises(RuntimeError, match="simulated ONNX bad_alloc"): + mine(str(project_root), str(palace_path)) + + out = capsys.readouterr().out + assert "Mine aborted by exception." in out + assert "files_processed: 1/" in out + assert "drawers_filed:" in out + assert "RuntimeError: simulated ONNX bad_alloc" in out + assert "upserted idempotently" in out + + def test_mine_cleans_up_pid_file_on_interrupt(tmp_path): """Our own PID entry in mine.pid is removed in the finally clause.""" import pytest From ec6d2dde0170387804730d57f13ea17db6c7c5f8 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Sat, 2 May 2026 00:16:29 -0600 Subject: [PATCH 62/77] fix: use configured collection in recovery paths --- mempalace/cli.py | 8 +++- mempalace/config.py | 8 ++++ mempalace/mcp_server.py | 61 ++++++++++++++++++++---- mempalace/palace.py | 7 ++- mempalace/repair.py | 59 ++++++++++++++++-------- mempalace/searcher.py | 49 +++++++++++++------- tests/test_backends.py | 23 ++++++++++ tests/test_cli.py | 44 ++++++++++++++++++ tests/test_hnsw_capacity.py | 2 + tests/test_mcp_server.py | 45 ++++++++++++++++-- tests/test_repair.py | 92 +++++++++++++++++++++++++++++++++++++ tests/test_searcher.py | 26 +++++++++-- 12 files changed, 370 insertions(+), 54 deletions(-) diff --git a/mempalace/cli.py b/mempalace/cli.py index 740de96f62..ac00283a1f 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -663,8 +663,10 @@ def cmd_repair(args): check_extraction_safety, ) + config = MempalaceConfig() + collection_name = config.collection_name palace_path = os.path.abspath( - os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path + os.path.expanduser(args.palace) if args.palace else config.palace_path ) if getattr(args, "mode", "legacy") == "max-seq-id": @@ -749,7 +751,7 @@ def cmd_repair(args): # Try to read existing drawers try: - col = backend.get_collection(palace_path, "mempalace_drawers") + col = backend.get_collection(palace_path, collection_name) total = col.count() print(f" Drawers found: {total}") except Exception as e: @@ -784,6 +786,7 @@ def cmd_repair(args): palace_path, len(all_ids), confirm_truncation_ok=getattr(args, "confirm_truncation_ok", False), + collection_name=collection_name, ) except TruncationDetected as e: print(e.message) @@ -810,6 +813,7 @@ def cmd_repair(args): all_docs, all_metas, batch_size, + collection_name=collection_name, progress=print, ) except RebuildCollectionError as e: diff --git a/mempalace/config.py b/mempalace/config.py index 2252a49484..fd32a1709f 100644 --- a/mempalace/config.py +++ b/mempalace/config.py @@ -7,6 +7,7 @@ import json import os import re +from functools import lru_cache from pathlib import Path @@ -127,6 +128,13 @@ def sanitize_content(value: str, max_length: int = 100_000) -> str: DEFAULT_PALACE_PATH = os.path.expanduser("~/.mempalace/palace") DEFAULT_COLLECTION_NAME = "mempalace_drawers" + +@lru_cache(maxsize=1) +def get_configured_collection_name() -> str: + """Return the configured drawer collection name without repeated config-file reads.""" + return MempalaceConfig().collection_name + + DEFAULT_TOPIC_WINGS = [ "emotions", "consciousness", diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index bbb9c937a0..521cb078d2 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -193,7 +193,7 @@ def _refresh_vector_disabled_flag() -> None: """ global _vector_disabled, _vector_disabled_reason, _vector_capacity_status try: - info = hnsw_capacity_status(_config.palace_path, "mempalace_drawers") + info = hnsw_capacity_status(_config.palace_path, _config.collection_name) except Exception: logger.debug("HNSW capacity probe raised", exc_info=True) return @@ -490,6 +490,7 @@ def _tool_status_via_sqlite() -> dict: db_path = os.path.join(_config.palace_path, "chroma.sqlite3") if not os.path.isfile(db_path): return _no_palace() + collection_name = _config.collection_name wings: dict = {} rooms: dict = {} @@ -503,8 +504,9 @@ def _tool_status_via_sqlite() -> dict: FROM embeddings e JOIN segments s ON e.segment_id = s.id JOIN collections c ON s.collection = c.id - WHERE c.name = 'mempalace_drawers' - """ + WHERE c.name = ? + """, + (collection_name,), ).fetchone() total = int(row[0]) if row and row[0] is not None else 0 for key, target in (("wing", wings), ("room", rooms)): @@ -515,12 +517,12 @@ def _tool_status_via_sqlite() -> dict: JOIN embeddings e ON em.id = e.id JOIN segments s ON e.segment_id = s.id JOIN collections c ON s.collection = c.id - WHERE c.name = 'mempalace_drawers' + WHERE c.name = ? AND em.key = ? AND em.string_value IS NOT NULL GROUP BY em.string_value """, - (key,), + (collection_name, key), ): target[value] = count finally: @@ -720,6 +722,7 @@ def tool_search( n_results=limit, max_distance=dist, vector_disabled=_vector_disabled, + collection_name=_config.collection_name, ) if _vector_disabled: result["vector_disabled"] = True @@ -922,8 +925,8 @@ def tool_add_drawer( # Idempotency: if the deterministic ID already exists, return success as a no-op. try: - existing = col.get(ids=[drawer_id]) - if existing and existing["ids"]: + existing = col.get(ids=[drawer_id], include=[]) + if existing.ids: return {"success": True, "reason": "already_exists", "drawer_id": drawer_id} except Exception: logger.debug("Idempotency pre-check failed for %s", drawer_id, exc_info=True) @@ -943,6 +946,12 @@ def tool_add_drawer( } ], ) + inserted = col.get(ids=[drawer_id], include=[]) + if not inserted.ids: + raise RuntimeError( + "Drawer write was acknowledged but the new ID is not readable. " + "The palace index may be stale; run reconnect or repair." + ) _metadata_cache = None logger.info(f"Filed drawer: {drawer_id} → {wing}/{room}") return {"success": True, "drawer_id": drawer_id, "wing": wing, "room": room} @@ -1506,6 +1515,30 @@ def tool_reconnect(): _palace_db_mtime, \ _vector_disabled, \ _vector_disabled_reason + from . import palace as palace_module + + close_errors = [] + try: + palace_module._DEFAULT_BACKEND.close_palace(_config.palace_path) + except Exception as exc: + logger.debug("Failed to close shared palace backend during reconnect", exc_info=True) + close_errors.append(f"backend close_palace failed: {exc}") + try: + from chromadb.api.client import SharedSystemClient + + clear_system_cache = getattr(SharedSystemClient, "clear_system_cache", None) + if callable(clear_system_cache): + clear_system_cache() + else: + logger.debug( + "SharedSystemClient.clear_system_cache is unavailable; skipping shared Chroma cache clear during reconnect" + ) + except Exception as exc: + logger.debug( + "Failed to clear Chroma shared system cache during reconnect", + exc_info=True, + ) + close_errors.append(f"shared Chroma cache clear failed: {exc}") _client_cache = None _collection_cache = None _palace_db_inode = 0 @@ -1527,12 +1560,24 @@ def tool_reconnect(): try: col = _get_collection() if col is None: - return { + result = { "success": False, "message": "No palace found after reconnect", "drawers": 0, "vector_disabled": _vector_disabled, } + if close_errors: + result["error"] = "; ".join(close_errors) + return result + if close_errors: + return { + "success": False, + "message": "Reconnect reopened the palace but failed to fully reset cached handles", + "drawers": col.count(), + "vector_disabled": _vector_disabled, + "vector_disabled_reason": _vector_disabled_reason, + "error": "; ".join(close_errors), + } return { "success": True, "message": "Reconnected to palace", diff --git a/mempalace/palace.py b/mempalace/palace.py index e5f6411855..dee5c8fbc6 100644 --- a/mempalace/palace.py +++ b/mempalace/palace.py @@ -10,6 +10,7 @@ import os import re import threading +from typing import Optional from .backends.chroma import ChromaBackend @@ -56,10 +57,14 @@ def get_collection( palace_path: str, - collection_name: str = "mempalace_drawers", + collection_name: Optional[str] = None, create: bool = True, ): """Get the palace collection through the backend layer.""" + if collection_name is None: + from .config import get_configured_collection_name + + collection_name = get_configured_collection_name() return _DEFAULT_BACKEND.get_collection( palace_path, collection_name=collection_name, diff --git a/mempalace/repair.py b/mempalace/repair.py index 34d165c2a0..b47bcd6a0c 100644 --- a/mempalace/repair.py +++ b/mempalace/repair.py @@ -181,10 +181,12 @@ def _rebuild_collection_via_temp( all_docs, all_metas, batch_size: int, + collection_name: Optional[str] = None, progress=print, ) -> int: expected = len(all_ids) - temp_name = REPAIR_TEMP_COLLECTION + collection_name = collection_name or _drawers_collection_name() + temp_name = f"{collection_name}__repair_tmp" live_replaced = False try: @@ -203,9 +205,9 @@ def _rebuild_collection_via_temp( _verify_collection_count(temp_col, expected, "temporary rebuild") progress(" Rebuilding live collection...") - backend.delete_collection(palace_path, COLLECTION_NAME) + backend.delete_collection(palace_path, collection_name) live_replaced = True - new_col = backend.create_collection(palace_path, COLLECTION_NAME) + new_col = backend.create_collection(palace_path, collection_name) rebuilt = 0 for i in range(0, expected, batch_size): @@ -230,7 +232,7 @@ def _rebuild_collection_via_temp( raise RebuildCollectionError(str(exc), live_replaced=live_replaced) from exc -def scan_palace(palace_path=None, only_wing=None): +def scan_palace(palace_path=None, only_wing=None, collection_name: Optional[str] = None): """Scan the palace for corrupt/unfetchable IDs. Probes in batches of 100, falls back to per-ID on failure. @@ -239,14 +241,15 @@ def scan_palace(palace_path=None, only_wing=None): Returns (good_set, bad_set). """ palace_path = palace_path or _get_palace_path() + collection_name = collection_name or _drawers_collection_name() print(f"\n Palace: {palace_path}") print(" Loading...") - col = ChromaBackend().get_collection(palace_path, COLLECTION_NAME) + col = ChromaBackend().get_collection(palace_path, collection_name) where = {"wing": only_wing} if only_wing else None total = col.count() - print(f" Collection: {COLLECTION_NAME}, total: {total:,}") + print(f" Collection: {collection_name}, total: {total:,}") if only_wing: print(f" Scanning wing: {only_wing}") @@ -307,9 +310,10 @@ def scan_palace(palace_path=None, only_wing=None): return good_set, bad_set -def prune_corrupt(palace_path=None, confirm=False): +def prune_corrupt(palace_path=None, confirm=False, collection_name: Optional[str] = None): """Delete corrupt IDs listed in corrupt_ids.txt.""" palace_path = palace_path or _get_palace_path() + collection_name = collection_name or _drawers_collection_name() bad_file = os.path.join(palace_path, "corrupt_ids.txt") if not os.path.exists(bad_file): @@ -325,7 +329,7 @@ def prune_corrupt(palace_path=None, confirm=False): print(" Re-run with --confirm to actually delete.") return - col = ChromaBackend().get_collection(palace_path, COLLECTION_NAME) + col = ChromaBackend().get_collection(palace_path, collection_name) before = col.count() print(f" Collection size before: {before:,}") @@ -379,7 +383,10 @@ def __init__(self, message: str, sqlite_count: "int | None", extracted: int): def check_extraction_safety( - palace_path: str, extracted: int, confirm_truncation_ok: bool = False + palace_path: str, + extracted: int, + confirm_truncation_ok: bool = False, + collection_name: Optional[str] = None, ) -> None: """Cross-check that ``extracted`` matches the SQLite ground truth. @@ -401,7 +408,8 @@ def check_extraction_safety( if confirm_truncation_ok: return - sqlite_count = sqlite_drawer_count(palace_path) + collection_name = collection_name or _drawers_collection_name() + sqlite_count = sqlite_drawer_count(palace_path, collection_name) cap_signal = extracted == CHROMADB_DEFAULT_GET_LIMIT if sqlite_count is not None and sqlite_count > extracted: @@ -437,7 +445,7 @@ def check_extraction_safety( raise TruncationDetected(message, sqlite_count, extracted) -def sqlite_drawer_count(palace_path: str) -> "int | None": +def sqlite_drawer_count(palace_path: str, collection_name: Optional[str] = None) -> "int | None": """Count rows in ``chroma.sqlite3.embeddings`` for the drawers collection. Used as an independent ground-truth check against the chromadb @@ -449,6 +457,7 @@ def sqlite_drawer_count(palace_path: str) -> "int | None": drift, missing tables, locked file). Callers treat ``None`` as "unknown" and fall back to the cap-detection check. """ + collection_name = collection_name or _drawers_collection_name() sqlite_path = os.path.join(palace_path, "chroma.sqlite3") if not os.path.exists(sqlite_path): return None @@ -465,7 +474,7 @@ def sqlite_drawer_count(palace_path: str) -> "int | None": JOIN collections c ON s.collection = c.id WHERE c.name = ? """, - (COLLECTION_NAME,), + (collection_name,), ).fetchone() return int(row[0]) if row and row[0] is not None else None finally: @@ -477,7 +486,11 @@ def sqlite_drawer_count(palace_path: str) -> "int | None": return None -def rebuild_index(palace_path=None, confirm_truncation_ok: bool = False): +def rebuild_index( + palace_path=None, + confirm_truncation_ok: bool = False, + collection_name: Optional[str] = None, +): """Rebuild the HNSW index from scratch. 1. Extract all drawers via ChromaDB get() @@ -492,6 +505,7 @@ def rebuild_index(palace_path=None, confirm_truncation_ok: bool = False): (typically only a concern for palaces sized at exactly 10 000 rows). """ palace_path = palace_path or _get_palace_path() + collection_name = collection_name or _drawers_collection_name() if not os.path.isdir(palace_path): print(f"\n No palace found at {palace_path}") @@ -504,7 +518,7 @@ def rebuild_index(palace_path=None, confirm_truncation_ok: bool = False): backend = ChromaBackend() try: - col = backend.get_collection(palace_path, COLLECTION_NAME) + col = backend.get_collection(palace_path, collection_name) total = col.count() except Exception as e: print(f" Error reading palace: {e}") @@ -528,7 +542,12 @@ def rebuild_index(palace_path=None, confirm_truncation_ok: bool = False): # short of the SQLite ground truth (or when extraction == chromadb # default get() cap and the SQLite check couldn't run). try: - check_extraction_safety(palace_path, len(all_ids), confirm_truncation_ok) + check_extraction_safety( + palace_path, + len(all_ids), + confirm_truncation_ok, + collection_name=collection_name, + ) except TruncationDetected as e: print(e.message) return @@ -551,6 +570,7 @@ def rebuild_index(palace_path=None, confirm_truncation_ok: bool = False): all_docs, all_metas, batch_size, + collection_name=collection_name, progress=print, ) except RebuildCollectionError as e: @@ -560,7 +580,7 @@ def rebuild_index(palace_path=None, confirm_truncation_ok: bool = False): print(f" Restoring from backup: {backup_path}") try: _close_chroma_handles(palace_path, backend=backend) - _delete_collection_if_exists(backend, palace_path, COLLECTION_NAME) + _delete_collection_if_exists(backend, palace_path, collection_name) shutil.copy2(backup_path, sqlite_path) print(" Backup restored. Palace is back to pre-repair state.") except Exception as restore_error: @@ -950,7 +970,7 @@ def rebuild_from_sqlite( backend.close() -def status(palace_path=None) -> dict: +def status(palace_path=None, collection_name: Optional[str] = None) -> dict: """Read-only health check: compare sqlite vs HNSW element counts. Catches the #1222 failure mode where chromadb's HNSW segment freezes @@ -968,6 +988,7 @@ def status(palace_path=None) -> dict: ``status="unknown"`` when no palace exists at the given path. """ palace_path = palace_path or _get_palace_path() + collection_name = collection_name or _drawers_collection_name() print(f"\n{'=' * 55}") print(" MemPalace Repair — Status") print(f"{'=' * 55}\n") @@ -977,8 +998,8 @@ def status(palace_path=None) -> dict: print(" No palace found.\n") return {"status": "unknown", "message": "no palace at path"} - drawers = hnsw_capacity_status(palace_path, "mempalace_drawers") - closets = hnsw_capacity_status(palace_path, "mempalace_closets") + drawers = hnsw_capacity_status(palace_path, collection_name) + closets = hnsw_capacity_status(palace_path, CLOSETS_COLLECTION_NAME) for label, info in (("drawers", drawers), ("closets", closets)): print(f"\n [{label}]") diff --git a/mempalace/searcher.py b/mempalace/searcher.py index b318d99678..f644fda973 100644 --- a/mempalace/searcher.py +++ b/mempalace/searcher.py @@ -382,6 +382,7 @@ def _bm25_only_via_sqlite( n_results: int = 5, max_candidates: int = 500, _include_internal: bool = False, + collection_name: str = None, ) -> dict: """BM25-only search reading drawers directly from chroma.sqlite3. @@ -405,6 +406,10 @@ def _bm25_only_via_sqlite( "error": "No palace found", "hint": "Run: mempalace init && mempalace mine ", } + if collection_name is None: + from .config import get_configured_collection_name + + collection_name = get_configured_collection_name() def _metadata_filter_sql(row_id_expr: str) -> tuple[str, list[str]]: clauses = [] @@ -441,35 +446,43 @@ def _metadata_filter_sql(row_id_expr: str) -> tuple[str, list[str]]: # shorter than 3 chars (trigram tokenizer can't match them). tokens = [t for t in _tokenize(query) if len(t) >= 3] candidate_ids: list[int] = [] + use_recency_fallback = not tokens if tokens: fts_query = " OR ".join(tokens) filter_sql, filter_params = _metadata_filter_sql("embedding_fulltext_search.rowid") try: rows = conn.execute( f""" - SELECT rowid + SELECT embedding_fulltext_search.rowid FROM embedding_fulltext_search + JOIN embeddings e ON e.id = embedding_fulltext_search.rowid + JOIN segments s ON e.segment_id = s.id + JOIN collections c ON s.collection = c.id WHERE embedding_fulltext_search MATCH ? + AND c.name = ? {filter_sql} LIMIT ? """, - (fts_query, *filter_params, max_candidates), + (fts_query, collection_name, *filter_params, max_candidates), ).fetchall() candidate_ids = [r[0] for r in rows] except sqlite3.Error: # FTS5 tokenizer mismatch or syntax error — fall through # to the recency-window selector below. logger.debug("FTS5 MATCH failed; using recency fallback", exc_info=True) - - if not candidate_ids: - # No FTS hits (or no usable tokens) — pull the most recent - # rows for the drawers segment so we can BM25-rank something - # rather than return empty-handed. Wrapped in try/except - # because the schema may differ on legacy palaces (older - # chromadb without ``created_at``, missing ``segments`` - # rows after partial restore, etc.); on schema mismatch we - # fall back to ordering by primary-key id and finally to an - # empty result rather than letting search raise. + use_recency_fallback = True + + if not candidate_ids and use_recency_fallback: + # No usable FTS tokens, or FTS itself failed — pull the most + # recent rows for the drawers segment so we can BM25-rank + # something rather than return empty-handed. A clean FTS miss + # must stay empty, especially after wing/room filtering, because + # recency fallback would return unrelated scoped drawers. + # Wrapped in try/except because the schema may differ on legacy + # palaces (older chromadb without ``created_at``, missing + # ``segments`` rows after partial restore, etc.); on schema + # mismatch we fall back to ordering by primary-key id and finally + # to an empty result rather than letting search raise. try: filter_sql, filter_params = _metadata_filter_sql("e.id") rows = conn.execute( @@ -478,12 +491,12 @@ def _metadata_filter_sql(row_id_expr: str) -> tuple[str, list[str]]: FROM embeddings e JOIN segments s ON e.segment_id = s.id JOIN collections c ON s.collection = c.id - WHERE c.name = 'mempalace_drawers' + WHERE c.name = ? {filter_sql} ORDER BY e.created_at DESC LIMIT ? """, - (*filter_params, max_candidates), + (collection_name, *filter_params, max_candidates), ).fetchall() candidate_ids = [r[0] for r in rows] except sqlite3.Error: @@ -499,12 +512,12 @@ def _metadata_filter_sql(row_id_expr: str) -> tuple[str, list[str]]: FROM embeddings e JOIN segments s ON e.segment_id = s.id JOIN collections c ON s.collection = c.id - WHERE c.name = 'mempalace_drawers' + WHERE c.name = ? {filter_sql} ORDER BY e.id DESC LIMIT ? """, - (*filter_params, max_candidates), + (collection_name, *filter_params, max_candidates), ).fetchall() candidate_ids = [r[0] for r in rows] except sqlite3.Error: @@ -720,6 +733,7 @@ def search_memories( max_distance: float = 0.0, vector_disabled: bool = False, candidate_strategy: str = "vector", + collection_name: str = None, ) -> dict: """Programmatic search — returns a dict instead of printing. @@ -770,10 +784,11 @@ def search_memories( wing=wing, room=room, n_results=n_results, + collection_name=collection_name, ) try: - drawers_col = get_collection(palace_path, create=False) + drawers_col = get_collection(palace_path, collection_name=collection_name, create=False) except Exception as e: logger.error("No palace found at %s: %s", palace_path, e) return { diff --git a/tests/test_backends.py b/tests/test_backends.py index 4cd9480f12..06625fc0b6 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -1194,3 +1194,26 @@ class DummyClient: ("invalid", str(palace)), ("stale", str(palace)), ] + + +def test_palace_get_collection_uses_configured_collection_name(monkeypatch): + from mempalace import palace + + captured = {} + + def fake_get_collection(palace_path, collection_name=None, create=False): + captured["palace_path"] = palace_path + captured["collection_name"] = collection_name + captured["create"] = create + return object() + + monkeypatch.setattr(palace._DEFAULT_BACKEND, "get_collection", fake_get_collection) + monkeypatch.setattr("mempalace.config.get_configured_collection_name", lambda: "custom_drawers") + + palace.get_collection("/palace", create=False) + + assert captured == { + "palace_path": "/palace", + "collection_name": "custom_drawers", + "create": False, + } diff --git a/tests/test_cli.py b/tests/test_cli.py index de0066431d..0b61b0c783 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -776,6 +776,7 @@ def test_cmd_repair_error_reading(mock_config_cls, tmp_path, capsys): palace_dir.mkdir() (palace_dir / "chroma.sqlite3").write_text("db") mock_config_cls.return_value.palace_path = str(palace_dir) + mock_config_cls.return_value.collection_name = "mempalace_drawers" args = argparse.Namespace(palace=None) mock_backend = MagicMock() mock_backend.get_collection.side_effect = Exception("corrupt db") @@ -791,6 +792,7 @@ def test_cmd_repair_zero_drawers(mock_config_cls, tmp_path, capsys): palace_dir.mkdir() (palace_dir / "chroma.sqlite3").write_text("db") mock_config_cls.return_value.palace_path = str(palace_dir) + mock_config_cls.return_value.collection_name = "mempalace_drawers" args = argparse.Namespace(palace=None) mock_col = MagicMock() mock_col.count.return_value = 0 @@ -807,6 +809,7 @@ def test_cmd_repair_success(mock_config_cls, tmp_path, capsys): palace_dir.mkdir() (palace_dir / "chroma.sqlite3").write_text("db") mock_config_cls.return_value.palace_path = str(palace_dir) + mock_config_cls.return_value.collection_name = "mempalace_drawers" args = argparse.Namespace(palace=None, yes=True) mock_col = MagicMock() mock_col.count.return_value = 2 @@ -836,12 +839,52 @@ def test_cmd_repair_success(mock_config_cls, tmp_path, capsys): mock_new_col.add.assert_not_called() +@patch("mempalace.cli.MempalaceConfig") +def test_cmd_repair_uses_configured_collection(mock_config_cls, tmp_path, capsys): + palace_dir = tmp_path / "palace" + palace_dir.mkdir() + (palace_dir / "chroma.sqlite3").write_text("db") + mock_config_cls.return_value.palace_path = str(palace_dir) + mock_config_cls.return_value.collection_name = "custom_drawers" + args = argparse.Namespace(palace=None, yes=True) + mock_col = MagicMock() + mock_col.count.return_value = 2 + mock_col.get.return_value = { + "ids": ["id1", "id2"], + "documents": ["doc1", "doc2"], + "metadatas": [{"wing": "a"}, {"wing": "b"}], + } + mock_temp_col = MagicMock() + mock_temp_col.count.return_value = 2 + mock_new_col = MagicMock() + mock_new_col.count.return_value = 2 + mock_backend = _mock_backend_for(col=mock_col, new_col=mock_new_col) + mock_backend.create_collection.side_effect = [mock_temp_col, mock_new_col] + + with patch("mempalace.backends.chroma.ChromaBackend", return_value=mock_backend): + cmd_repair(args) + + out = capsys.readouterr().out + assert "Repair complete" in out + mock_backend.get_collection.assert_called_once_with(str(palace_dir), "custom_drawers") + assert mock_backend.create_collection.call_args_list == [ + call(str(palace_dir), "custom_drawers__repair_tmp"), + call(str(palace_dir), "custom_drawers"), + ] + assert mock_backend.delete_collection.call_args_list == [ + call(str(palace_dir), "custom_drawers__repair_tmp"), + call(str(palace_dir), "custom_drawers"), + call(str(palace_dir), "custom_drawers__repair_tmp"), + ] + + @patch("mempalace.cli.MempalaceConfig") def test_cmd_repair_restores_backup_on_live_rebuild_failure(mock_config_cls, tmp_path, capsys): palace_dir = tmp_path / "palace" palace_dir.mkdir() (palace_dir / "chroma.sqlite3").write_text("db") mock_config_cls.return_value.palace_path = str(palace_dir) + mock_config_cls.return_value.collection_name = "mempalace_drawers" args = argparse.Namespace(palace=None, yes=True) mock_col = MagicMock() mock_col.count.return_value = 2 @@ -875,6 +918,7 @@ def test_cmd_repair_aborts_without_confirmation(mock_config_cls, tmp_path, capsy palace_dir.mkdir() (palace_dir / "chroma.sqlite3").write_text("db") mock_config_cls.return_value.palace_path = str(palace_dir) + mock_config_cls.return_value.collection_name = "mempalace_drawers" args = argparse.Namespace(palace=None) mock_col = MagicMock() mock_col.count.return_value = 1 diff --git a/tests/test_hnsw_capacity.py b/tests/test_hnsw_capacity.py index 912def8bf7..53775b096b 100644 --- a/tests/test_hnsw_capacity.py +++ b/tests/test_hnsw_capacity.py @@ -260,6 +260,7 @@ def test_mcp_probe_does_not_disable_vectors_for_unflushed_metadata(tmp_path, mon class _Cfg: palace_path = str(tmp_path) + collection_name = "mempalace_drawers" monkeypatch.setattr(mcp_server, "_config", _Cfg()) monkeypatch.setattr(mcp_server, "_vector_disabled", True) @@ -625,6 +626,7 @@ def test_tool_status_via_sqlite_returns_breakdown(palace_with_drawers, monkeypat # MempalaceConfig. class _Cfg: palace_path = str(palace_with_drawers) + collection_name = "mempalace_drawers" monkeypatch.setattr(mcp_server, "_config", _Cfg()) monkeypatch.setattr(mcp_server, "_vector_disabled", True) diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index ae20bf32e3..613cc235cb 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -484,6 +484,26 @@ def test_add_drawer_duplicate_detection(self, monkeypatch, config, palace_path, assert result2["success"] is True assert result2["reason"] == "already_exists" + def test_add_drawer_fails_when_readback_misses(self, monkeypatch, config, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace import mcp_server + + class _FakeGetResult: + ids = [] + + class _FakeCol: + def get(self, **kwargs): + return _FakeGetResult() + + def upsert(self, **kwargs): + return None + + monkeypatch.setattr(mcp_server, "_get_collection", lambda create=False: _FakeCol()) + + result = mcp_server.tool_add_drawer("w", "r", "content") + assert result["success"] is False + assert "not readable" in result["error"] + def test_add_drawer_shared_header_no_collision(self, monkeypatch, config, palace_path, kg): """Documents sharing a >100-char header must get distinct IDs (full-content hash).""" _patch_mcp_server(monkeypatch, config, kg) @@ -503,9 +523,9 @@ def test_add_drawer_shared_header_no_collision(self, monkeypatch, config, palace assert result1["success"] is True assert result2["success"] is True - assert ( - result1["drawer_id"] != result2["drawer_id"] - ), "Documents with shared header but different content must have distinct drawer IDs" + assert result1["drawer_id"] != result2["drawer_id"], ( + "Documents with shared header but different content must have distinct drawer IDs" + ) def test_delete_drawer(self, monkeypatch, config, palace_path, seeded_collection, kg): _patch_mcp_server(monkeypatch, config, kg) @@ -1158,6 +1178,25 @@ def test_reconnect_reports_success(self, monkeypatch, config, palace_path, kg): assert "Reconnected" in result["message"] assert isinstance(result["drawers"], int) + def test_reconnect_closes_shared_backend(self, monkeypatch, config, kg): + _patch_mcp_server(monkeypatch, config, kg) + from unittest.mock import MagicMock + + from mempalace import mcp_server, palace + + close_palace = MagicMock() + monkeypatch.setattr(palace._DEFAULT_BACKEND, "close_palace", close_palace) + + class _FakeCol: + def count(self): + return 7 + + monkeypatch.setattr(mcp_server, "_get_collection", lambda create=False: _FakeCol()) + + result = mcp_server.tool_reconnect() + assert result["success"] is True + close_palace.assert_called_once_with(config.palace_path) + def test_get_collection_create_true_avoids_get_or_create_on_reopen( self, monkeypatch, config, palace_path, kg ): diff --git a/tests/test_repair.py b/tests/test_repair.py index a60836a07b..8e9f95b371 100644 --- a/tests/test_repair.py +++ b/tests/test_repair.py @@ -28,6 +28,16 @@ def test_get_palace_path_fallback(): assert ".mempalace" in result +def test_get_collection_name_from_config(): + from mempalace.config import get_configured_collection_name + + get_configured_collection_name.cache_clear() + with patch("mempalace.config.MempalaceConfig") as mock_config_cls: + mock_config_cls.return_value.collection_name = "custom_drawers" + assert repair._drawers_collection_name() == "custom_drawers" + get_configured_collection_name.cache_clear() + + # ── _paginate_ids ───────────────────────────────────────────────────── @@ -330,6 +340,21 @@ def test_check_extraction_safety_passes_when_counts_match(tmp_path): repair.check_extraction_safety(str(tmp_path), 500) +def test_check_extraction_safety_uses_configured_collection(tmp_path): + with patch("mempalace.repair.sqlite_drawer_count", return_value=500) as count: + repair.check_extraction_safety(str(tmp_path), 500, collection_name="custom_drawers") + count.assert_called_once_with(str(tmp_path), "custom_drawers") + + +def test_check_extraction_safety_default_uses_configured_collection(tmp_path): + with ( + patch("mempalace.repair._drawers_collection_name", return_value="custom_drawers"), + patch("mempalace.repair.sqlite_drawer_count", return_value=500) as count, + ): + repair.check_extraction_safety(str(tmp_path), 500) + count.assert_called_once_with(str(tmp_path), "custom_drawers") + + def test_check_extraction_safety_passes_when_sqlite_unreadable_and_under_cap(tmp_path): """SQLite check fails (None) but extraction is well under the cap → safe.""" with patch("mempalace.repair.sqlite_drawer_count", return_value=None): @@ -384,6 +409,73 @@ def test_sqlite_drawer_count_returns_none_on_unreadable_schema(tmp_path): assert repair.sqlite_drawer_count(str(tmp_path)) is None +@patch("mempalace.repair.shutil") +@patch("mempalace.repair.ChromaBackend") +def test_rebuild_index_default_uses_configured_collection(mock_backend_cls, mock_shutil, tmp_path): + sqlite_path = tmp_path / "chroma.sqlite3" + sqlite_path.write_text("fake") + mock_col = MagicMock() + mock_col.count.return_value = 2 + mock_col.get.return_value = { + "ids": ["id1", "id2"], + "documents": ["doc1", "doc2"], + "metadatas": [{"wing": "a"}, {"wing": "b"}], + } + mock_temp_col = MagicMock() + mock_temp_col.count.return_value = 2 + mock_new_col = MagicMock() + mock_new_col.count.return_value = 2 + mock_backend = _install_mock_backend(mock_backend_cls, mock_col) + mock_backend.create_collection.side_effect = [mock_temp_col, mock_new_col] + + with ( + patch("mempalace.repair._drawers_collection_name", return_value="custom_drawers"), + patch("mempalace.repair.sqlite_drawer_count", return_value=2) as count, + ): + repair.rebuild_index(palace_path=str(tmp_path)) + + mock_backend.get_collection.assert_called_once_with(str(tmp_path), "custom_drawers") + count.assert_called_once_with(str(tmp_path), "custom_drawers") + assert mock_backend.create_collection.call_args_list == [ + call(str(tmp_path), "custom_drawers__repair_tmp"), + call(str(tmp_path), "custom_drawers"), + ] + assert mock_backend.delete_collection.call_args_list == [ + call(str(tmp_path), "custom_drawers__repair_tmp"), + call(str(tmp_path), "custom_drawers"), + call(str(tmp_path), "custom_drawers__repair_tmp"), + ] + + +def test_status_default_uses_configured_drawer_collection(tmp_path): + with ( + patch("mempalace.repair._drawers_collection_name", return_value="custom_drawers"), + patch("mempalace.repair.hnsw_capacity_status") as capacity_status, + ): + capacity_status.side_effect = [ + { + "sqlite_count": 1, + "hnsw_count": 1, + "divergence": 0, + "diverged": False, + "status": "ok", + "message": "", + }, + { + "sqlite_count": 0, + "hnsw_count": 0, + "divergence": 0, + "diverged": False, + "status": "ok", + "message": "", + }, + ] + repair.status(palace_path=str(tmp_path)) + + assert capacity_status.call_args_list[0].args == (str(tmp_path), "custom_drawers") + assert capacity_status.call_args_list[1].args == (str(tmp_path), "mempalace_closets") + + @patch("mempalace.repair.shutil") @patch("mempalace.repair.ChromaBackend") def test_rebuild_index_aborts_on_truncation_signal(mock_backend_cls, mock_shutil, tmp_path): diff --git a/tests/test_searcher.py b/tests/test_searcher.py index 4f0b4c0da3..60bef9ff1b 100644 --- a/tests/test_searcher.py +++ b/tests/test_searcher.py @@ -84,6 +84,24 @@ def test_search_memories_query_error(self): assert "error" in result assert "query failed" in result["error"] + def test_search_memories_vector_path_uses_explicit_collection_name(self): + mock_col = MagicMock() + mock_col.query.return_value = { + "documents": [[]], + "metadatas": [[]], + "distances": [[]], + "ids": [[]], + } + + with patch("mempalace.searcher.get_collection", return_value=mock_col) as get_collection: + search_memories("test", "/fake/path", collection_name="custom_drawers") + + get_collection.assert_called_once_with( + "/fake/path", + collection_name="custom_drawers", + create=False, + ) + def test_search_memories_filters_in_result(self, palace_path, seeded_collection): result = search_memories("test", palace_path, wing="project", room="backend") assert result["filters"]["wing"] == "project" @@ -102,7 +120,7 @@ def test_search_memories_handles_none_metadata(self): "ids": [["d1", "d2"]], } - def mock_get_collection(path, create=False): + def mock_get_collection(path, collection_name=None, create=False): # First call: drawers. Second call: closets — raise so hybrid # degrades to pure drawer search (the catch block covers it). if not hasattr(mock_get_collection, "_called"): @@ -309,9 +327,9 @@ def test_search_applies_bm25_hybrid_rerank(self, capsys): captured = capsys.readouterr() first_block, _, _ = captured.out.partition("[2]") # Lexical match must rank first - assert ( - "b.md" in first_block - ), f"expected lexical match 'b.md' at rank 1, got:\n{captured.out}" + assert "b.md" in first_block, ( + f"expected lexical match 'b.md' at rank 1, got:\n{captured.out}" + ) # Non-zero bm25 reported assert "bm25=" in first_block assert "bm25=0.0" not in first_block From e9aee194335a446787f2e2a007ff28e11269f385 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Thu, 7 May 2026 09:10:22 -0300 Subject: [PATCH 63/77] fix(tests): apply ruff format after rebase resolution The collection_name plumbing rebase produced a few unformatted blocks in test_mcp_server.py and test_searcher.py; bringing them in line with the 0.4.x CI pin so test-windows / lint stay green. --- tests/test_mcp_server.py | 6 +++--- tests/test_searcher.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 613cc235cb..1f4719223d 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -523,9 +523,9 @@ def test_add_drawer_shared_header_no_collision(self, monkeypatch, config, palace assert result1["success"] is True assert result2["success"] is True - assert result1["drawer_id"] != result2["drawer_id"], ( - "Documents with shared header but different content must have distinct drawer IDs" - ) + assert ( + result1["drawer_id"] != result2["drawer_id"] + ), "Documents with shared header but different content must have distinct drawer IDs" def test_delete_drawer(self, monkeypatch, config, palace_path, seeded_collection, kg): _patch_mcp_server(monkeypatch, config, kg) diff --git a/tests/test_searcher.py b/tests/test_searcher.py index 60bef9ff1b..f4d46a0a61 100644 --- a/tests/test_searcher.py +++ b/tests/test_searcher.py @@ -327,9 +327,9 @@ def test_search_applies_bm25_hybrid_rerank(self, capsys): captured = capsys.readouterr() first_block, _, _ = captured.out.partition("[2]") # Lexical match must rank first - assert "b.md" in first_block, ( - f"expected lexical match 'b.md' at rank 1, got:\n{captured.out}" - ) + assert ( + "b.md" in first_block + ), f"expected lexical match 'b.md' at rank 1, got:\n{captured.out}" # Non-zero bm25 reported assert "bm25=" in first_block assert "bm25=0.0" not in first_block From 5134a635ed5cfe819ccc479f58b256bd3e19af45 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Thu, 7 May 2026 11:52:58 -0300 Subject: [PATCH 64/77] fix(repair): run SQLite integrity preflight before chromadb open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #1364 added the SQLite quick_check preflight to rebuild_index, but placed it AFTER backend.get_collection(...). On a SQLite-corrupt palace, chromadb's rust binding raises pyo3_runtime.PanicException — which is not a regular Exception subclass — so it propagates past the existing `except Exception` handlers and the user sees a 30-line stack trace instead of the friendly abort message #1364 was designed to deliver. Reproduced with `mempalace repair --yes` against a palace whose chroma.sqlite3 has 4 mangled pages: pre-fix, panic; post-fix, the clean abort message and exit code 1. Two changes: - mempalace/cli.py cmd_repair: run sqlite_integrity_errors() right after the basic palace-existence check, BEFORE the max_seq_id preflight (which itself opens sqlite3) and BEFORE backend = ChromaBackend(). Exit non-zero so unattended scripts and CI gates see the failure. - mempalace/repair.py rebuild_index: same move at the function level for direct callers (tests, MCP) that bypass cmd_repair. The new test test_rebuild_index_runs_sqlite_preflight_before_chromadb_open uses a real chromadb-built palace (no ChromaBackend mock) plus a real corrupt SQLite (16 KB of mangled pages) so the ordering is exercised end-to-end. The previously-shipping test for the abort path mocked both the backend and sqlite_integrity_errors, which is why the ordering bug shipped CI-green. Six existing test_cli.py cmd_repair tests used `(palace_dir / "chroma.sqlite3").write_text("db")` to fake the SQLite file. The new preflight correctly fails quick_check on those 2-byte stubs, so the tests now create empty real SQLite DBs the same way the test_repair.py fixtures already do. --- mempalace/cli.py | 14 ++++++++++++ mempalace/repair.py | 16 +++++++++----- tests/test_cli.py | 13 +++++------ tests/test_repair.py | 51 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 11 deletions(-) diff --git a/mempalace/cli.py b/mempalace/cli.py index 14db5a88be..6a531e7ef0 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -662,6 +662,8 @@ def cmd_repair(args): _rebuild_collection_via_temp, check_extraction_safety, maybe_repair_poisoned_max_seq_id_before_rebuild, + print_sqlite_integrity_abort, + sqlite_integrity_errors, ) config = MempalaceConfig() @@ -743,6 +745,18 @@ def cmd_repair(args): print(f"\n No palace database found at {db_path}") return + # Run the SQLite integrity preflight before any chromadb client open. + # ChromaDB's rust binding raises pyo3_runtime.PanicException on a + # malformed page, which is not a regular Exception subclass and + # propagates past the try/except below — the user gets a 30-line + # stack trace instead of the friendly abort message. Run quick_check + # here so we can surface the clear recovery instructions and exit + # cleanly before chromadb's compactor touches the disk. + sqlite_errors = sqlite_integrity_errors(palace_path) + if sqlite_errors: + print_sqlite_integrity_abort(palace_path, sqlite_errors) + sys.exit(1) + preflight = maybe_repair_poisoned_max_seq_id_before_rebuild( palace_path, backup=getattr(args, "backup", True), diff --git a/mempalace/repair.py b/mempalace/repair.py index 6e170ef772..dd4c46ac65 100644 --- a/mempalace/repair.py +++ b/mempalace/repair.py @@ -633,6 +633,17 @@ def rebuild_index( print(f"{'=' * 55}\n") print(f" Palace: {palace_path}") + # Run the SQLite integrity preflight before any chromadb client open. + # ChromaDB's rust binding raises pyo3_runtime.PanicException (which is + # not a regular Exception subclass) on a malformed page, propagating + # past the try/except around get_collection below. Catching the + # corruption here lets us surface the clear recovery instructions and + # exit cleanly before chromadb's compactor touches the disk. + sqlite_errors = sqlite_integrity_errors(palace_path) + if sqlite_errors: + print_sqlite_integrity_abort(palace_path, sqlite_errors) + return + preflight = maybe_repair_poisoned_max_seq_id_before_rebuild( palace_path, assume_yes=True, @@ -676,11 +687,6 @@ def rebuild_index( print(e.message) return - sqlite_errors = sqlite_integrity_errors(palace_path) - if sqlite_errors: - print_sqlite_integrity_abort(palace_path, sqlite_errors) - return - # Back up ONLY the SQLite database, not the bloated HNSW files sqlite_path = os.path.join(palace_path, "chroma.sqlite3") backup_path = sqlite_path + ".backup" diff --git a/tests/test_cli.py b/tests/test_cli.py index 0b61b0c783..fa5680d8ed 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,6 +2,7 @@ import argparse import shlex +import sqlite3 import sys from pathlib import Path from unittest.mock import MagicMock, call, patch @@ -774,7 +775,7 @@ def test_cmd_repair_requires_palace_database(mock_config_cls, tmp_path, capsys): def test_cmd_repair_error_reading(mock_config_cls, tmp_path, capsys): palace_dir = tmp_path / "palace" palace_dir.mkdir() - (palace_dir / "chroma.sqlite3").write_text("db") + sqlite3.connect(str(palace_dir / "chroma.sqlite3")).close() mock_config_cls.return_value.palace_path = str(palace_dir) mock_config_cls.return_value.collection_name = "mempalace_drawers" args = argparse.Namespace(palace=None) @@ -790,7 +791,7 @@ def test_cmd_repair_error_reading(mock_config_cls, tmp_path, capsys): def test_cmd_repair_zero_drawers(mock_config_cls, tmp_path, capsys): palace_dir = tmp_path / "palace" palace_dir.mkdir() - (palace_dir / "chroma.sqlite3").write_text("db") + sqlite3.connect(str(palace_dir / "chroma.sqlite3")).close() mock_config_cls.return_value.palace_path = str(palace_dir) mock_config_cls.return_value.collection_name = "mempalace_drawers" args = argparse.Namespace(palace=None) @@ -807,7 +808,7 @@ def test_cmd_repair_zero_drawers(mock_config_cls, tmp_path, capsys): def test_cmd_repair_success(mock_config_cls, tmp_path, capsys): palace_dir = tmp_path / "palace" palace_dir.mkdir() - (palace_dir / "chroma.sqlite3").write_text("db") + sqlite3.connect(str(palace_dir / "chroma.sqlite3")).close() mock_config_cls.return_value.palace_path = str(palace_dir) mock_config_cls.return_value.collection_name = "mempalace_drawers" args = argparse.Namespace(palace=None, yes=True) @@ -843,7 +844,7 @@ def test_cmd_repair_success(mock_config_cls, tmp_path, capsys): def test_cmd_repair_uses_configured_collection(mock_config_cls, tmp_path, capsys): palace_dir = tmp_path / "palace" palace_dir.mkdir() - (palace_dir / "chroma.sqlite3").write_text("db") + sqlite3.connect(str(palace_dir / "chroma.sqlite3")).close() mock_config_cls.return_value.palace_path = str(palace_dir) mock_config_cls.return_value.collection_name = "custom_drawers" args = argparse.Namespace(palace=None, yes=True) @@ -882,7 +883,7 @@ def test_cmd_repair_uses_configured_collection(mock_config_cls, tmp_path, capsys def test_cmd_repair_restores_backup_on_live_rebuild_failure(mock_config_cls, tmp_path, capsys): palace_dir = tmp_path / "palace" palace_dir.mkdir() - (palace_dir / "chroma.sqlite3").write_text("db") + sqlite3.connect(str(palace_dir / "chroma.sqlite3")).close() mock_config_cls.return_value.palace_path = str(palace_dir) mock_config_cls.return_value.collection_name = "mempalace_drawers" args = argparse.Namespace(palace=None, yes=True) @@ -916,7 +917,7 @@ def test_cmd_repair_restores_backup_on_live_rebuild_failure(mock_config_cls, tmp def test_cmd_repair_aborts_without_confirmation(mock_config_cls, tmp_path, capsys): palace_dir = tmp_path / "palace" palace_dir.mkdir() - (palace_dir / "chroma.sqlite3").write_text("db") + sqlite3.connect(str(palace_dir / "chroma.sqlite3")).close() mock_config_cls.return_value.palace_path = str(palace_dir) mock_config_cls.return_value.collection_name = "mempalace_drawers" args = argparse.Namespace(palace=None) diff --git a/tests/test_repair.py b/tests/test_repair.py index dda83ec428..264561fa7f 100644 --- a/tests/test_repair.py +++ b/tests/test_repair.py @@ -1153,6 +1153,57 @@ def test_rebuild_index_aborts_on_sqlite_integrity_errors_before_delete_collectio mock_shutil.copy2.assert_not_called() +def test_rebuild_index_runs_sqlite_preflight_before_chromadb_open(tmp_path, capsys): + """The SQLite integrity preflight must run BEFORE backend.get_collection. + + chromadb's rust binding raises pyo3_runtime.PanicException (which is not + a regular Exception subclass) on a malformed page, so any get_collection + call against a corrupt SQLite propagates past `except Exception` handlers + and produces a 30-line stack trace instead of the friendly abort message. + Regression test for the ordering bug where the preflight was placed after + the chromadb client open and therefore never reached on the cases it was + designed to catch (#1364 follow-up). + """ + palace = tmp_path / "palace" + palace.mkdir() + + # Build a real chromadb palace with one drawer so chroma.sqlite3 exists + # at full schema size, then mangle several middle pages so PRAGMA + # quick_check fails with "disk image is malformed". This matches the + # production failure mode users hit in #1362 / #1364. + from mempalace.backends.chroma import ChromaBackend + + backend = ChromaBackend() + try: + col = backend.create_collection(str(palace), "mempalace_drawers") + col.upsert( + ids=["d1"], + documents=["doc"], + metadatas=[{"wing": "w", "room": "r"}], + ) + finally: + backend.close() + + sqlite_path = palace / "chroma.sqlite3" + pre_size = sqlite_path.stat().st_size + assert pre_size > 16384, "need a multi-page sqlite db to mangle" + + with open(sqlite_path, "r+b") as f: + f.seek(40960) # page 10 + f.write(b"\xde\xad\xbe\xef" * 4096) # 16 KB of garbage + + # No chromadb mocks: rebuild_index must reach sqlite_integrity_errors + # before any code path that opens a chromadb client. If the preflight + # comes too late, the test fails with pyo3_runtime.PanicException + # instead of returning cleanly. + repair.rebuild_index(palace_path=str(palace)) + + out = capsys.readouterr().out + assert "SQLite-layer corruption detected before repair rebuild" in out + assert "PRAGMA quick_check" in out + assert "disk image is malformed" in out + + def test_max_seq_id_preflight_preserves_embeddings_queue(tmp_path): """#1295: default repair preflight must not drop queued writes.""" From 7b151039c9479b15df9c27ada667e7f919acbe62 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Thu, 7 May 2026 12:07:54 -0300 Subject: [PATCH 65/77] test(repair): page-align corruption offset in preflight regression test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address Copilot review on #1403: the test seeked unconditionally to offset 40960 with only `pre_size > 16384` as a guard. If pre_size sat between 16384 and 40960 + 16384 = 57344 (e.g., on a chromadb version that allocated fewer pages on init, or a future schema change), the seek would extend the file with zero-padding and the original pages would stay intact — quick_check would still pass on the (untouched) real data, and the regression guard would silently skip detecting a preflight-ordering regression. Compute the offset from pre_size, page-aligned, with explicit asserts that the file is large enough to mangle 4 pages without truncating the header or extending past EOF. --- tests/test_repair.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/test_repair.py b/tests/test_repair.py index 264561fa7f..37651bafe1 100644 --- a/tests/test_repair.py +++ b/tests/test_repair.py @@ -1186,11 +1186,31 @@ def test_rebuild_index_runs_sqlite_preflight_before_chromadb_open(tmp_path, caps sqlite_path = palace / "chroma.sqlite3" pre_size = sqlite_path.stat().st_size - assert pre_size > 16384, "need a multi-page sqlite db to mangle" + + # Compute a page-aligned corruption offset that's always inside the + # existing file. SQLite uses 4 KB pages by default; we mangle 4 pages + # somewhere in the middle, skipping at least the first 2 pages + # (header + root) so the file still opens. Without clamping to the + # actual file size, a seek past EOF on r+b mode would silently + # extend the file with zero-padding and leave the original pages + # intact — quick_check would still pass, and the regression guard + # would skip the bug. + PAGE = 4096 + CORRUPT_BYTES = 16384 # 4 pages + HEADER_GUARD = PAGE * 2 # leave header + root pages intact + assert ( + pre_size >= HEADER_GUARD + CORRUPT_BYTES + ), f"sqlite db too small to mangle without truncating: {pre_size} bytes" + # Round (pre_size - CORRUPT_BYTES) down to a page boundary so we + # mangle whole pages. Cap at offset 40960 (page 10) for stable + # diagnostics across SQLite versions that may grow the file. + max_offset = (pre_size - CORRUPT_BYTES) & ~(PAGE - 1) + corrupt_offset = min(40960, max_offset) + assert corrupt_offset >= HEADER_GUARD, f"corruption offset {corrupt_offset} too close to header" with open(sqlite_path, "r+b") as f: - f.seek(40960) # page 10 - f.write(b"\xde\xad\xbe\xef" * 4096) # 16 KB of garbage + f.seek(corrupt_offset) + f.write(b"\xde\xad\xbe\xef" * (CORRUPT_BYTES // 4)) # No chromadb mocks: rebuild_index must reach sqlite_integrity_errors # before any code path that opens a chromadb client. If the preflight From 2a0ed0cb8f8bf7458be7ce5dc494b62ba0c39510 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Thu, 7 May 2026 12:38:39 -0300 Subject: [PATCH 66/77] fix(closet_llm): retry _call_llm on JSONDecodeError instead of bailing The retry loop already backs off on HTTP 429/503 and rate-limit-shaped exceptions, but JSONDecodeError exited on the first failure. Local LLM runtimes occasionally produce malformed JSON (truncated streams, partial chunks under load), and the retry was effectively dead for that path. Mirror the 429/503 branch: sleep with exponential backoff and continue through all 3 attempts, only returning None after the final failure. Closes #1155 --- mempalace/closet_llm.py | 3 +++ tests/test_closet_llm.py | 26 +++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/mempalace/closet_llm.py b/mempalace/closet_llm.py index 50000c8076..a85d5178b9 100644 --- a/mempalace/closet_llm.py +++ b/mempalace/closet_llm.py @@ -169,6 +169,9 @@ def _call_llm(cfg: LLMConfig, source_file: str, wing: str, room: str, content: s parsed = json.loads(text) return parsed, payload.get("usage") except json.JSONDecodeError: + if attempt < 2: + time.sleep(2**attempt) + continue return None, None except urllib.error.HTTPError as e: # 429 / 503 = retry with backoff diff --git a/tests/test_closet_llm.py b/tests/test_closet_llm.py index 3a0e84e478..0255ee8e18 100644 --- a/tests/test_closet_llm.py +++ b/tests/test_closet_llm.py @@ -196,9 +196,33 @@ def fake_urlopen(req, timeout=None): } ) - with patch("urllib.request.urlopen", side_effect=fake_urlopen): + with ( + patch("urllib.request.urlopen", side_effect=fake_urlopen), + patch("mempalace.closet_llm.time.sleep"), + ): + parsed, usage = _call_llm(cfg, "/tmp/x", "w", "r", "c") + assert parsed is None + + def test_retries_on_json_decode_error(self): + cfg = self._make_cfg() + call_count = {"n": 0} + + def fake_urlopen(req, timeout=None): + call_count["n"] += 1 + return _FakeResp( + { + "choices": [{"message": {"content": "not json at all"}}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1}, + } + ) + + with ( + patch("urllib.request.urlopen", side_effect=fake_urlopen), + patch("mempalace.closet_llm.time.sleep"), + ): parsed, usage = _call_llm(cfg, "/tmp/x", "w", "r", "c") assert parsed is None + assert call_count["n"] == 3 # ── regenerate_closets error paths ─────────────────────────────────────── From 40e2c8b056b6954629da4dc9841a7a2d44360dab Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Thu, 7 May 2026 12:40:26 -0300 Subject: [PATCH 67/77] fix(exporter): refuse symlinks at export targets A symlink pre-placed at the export output_dir or any wing subdirectory would redirect markdown writes to wherever the symlink points. The miner already rejects symlinked inputs via Path.is_symlink(); the exporter should apply the same caution to outputs. Add _reject_symlink() helper and call it before makedirs on both output_dir and each wing_dir. Refusal raises ValueError with a clear message rather than silently falling through. Closes #1156 --- mempalace/exporter.py | 16 ++++++++++++++++ tests/test_exporter.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/mempalace/exporter.py b/mempalace/exporter.py index 4b903b093c..a19181bb3b 100644 --- a/mempalace/exporter.py +++ b/mempalace/exporter.py @@ -26,6 +26,20 @@ def _safe_path_component(name: str) -> str: return name or "unknown" +def _reject_symlink(path: str, label: str) -> None: + """Refuse to write into a path that is itself a symlink. + + Defense-in-depth: a pre-placed symlink at the export target would + redirect writes to wherever it points (e.g., system directories). + Mirrors the miner's input-side caution. + """ + if os.path.islink(path): + raise ValueError( + f"refusing to export: {label} is a symbolic link ({path!r}). " + f"Remove the symlink or choose a different output path." + ) + + def export_palace(palace_path: str, output_dir: str, format: str = "markdown") -> dict: """Export all palace drawers as markdown files organized by wing/room. @@ -48,6 +62,7 @@ def export_palace(palace_path: str, output_dir: str, format: str = "markdown") - print(" Palace is empty — nothing to export.") return {"wings": 0, "rooms": 0, "drawers": 0} + _reject_symlink(output_dir, "output_dir") os.makedirs(output_dir, exist_ok=True) try: os.chmod(output_dir, 0o700) @@ -89,6 +104,7 @@ def export_palace(palace_path: str, output_dir: str, format: str = "markdown") - safe_wing = _safe_path_component(wing) wing_dir = os.path.join(output_dir, safe_wing) if wing_dir not in created_wing_dirs: + _reject_symlink(wing_dir, f"wing directory {safe_wing!r}") os.makedirs(wing_dir, exist_ok=True) try: os.chmod(wing_dir, 0o700) diff --git a/tests/test_exporter.py b/tests/test_exporter.py index 0597ec1c2f..e4d4ee724a 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -134,3 +134,45 @@ def test_export_empty_palace(): assert stats == {"wings": 0, "rooms": 0, "drawers": 0} finally: shutil.rmtree(tmpdir, ignore_errors=True) + + +def test_export_refuses_symlinked_output_dir(): + """A symlink at the output path must not be followed (defense-in-depth).""" + import pytest + + tmpdir = tempfile.mkdtemp() + try: + palace_path = _setup_palace(tmpdir) + decoy_target = os.path.join(tmpdir, "decoy_target") + os.makedirs(decoy_target) + output_dir = os.path.join(tmpdir, "export") + os.symlink(decoy_target, output_dir) + + with pytest.raises(ValueError, match="symbolic link"): + export_palace(palace_path, output_dir) + + # Decoy target must remain empty — nothing followed the symlink. + assert os.listdir(decoy_target) == [] + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def test_export_refuses_symlinked_wing_dir(): + """A symlink pre-placed at a wing subdirectory must also be refused.""" + import pytest + + tmpdir = tempfile.mkdtemp() + try: + palace_path = _setup_palace(tmpdir) + decoy_target = os.path.join(tmpdir, "decoy_target") + os.makedirs(decoy_target) + output_dir = os.path.join(tmpdir, "export") + os.makedirs(output_dir) + os.symlink(decoy_target, os.path.join(output_dir, "alpha")) + + with pytest.raises(ValueError, match="symbolic link"): + export_palace(palace_path, output_dir) + + assert os.listdir(decoy_target) == [] + finally: + shutil.rmtree(tmpdir, ignore_errors=True) From 0d1c1fbcaab751a1e7f8d992debaf0c5976904de Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Thu, 7 May 2026 12:42:02 -0300 Subject: [PATCH 68/77] fix(diary): detect same-size edits via content hash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The skip-if-unchanged check compared byte length only, so any in-place edit preserving total length (typo fix "teh"→"the", word swap) was silently dropped — a verbatim-storage violation: the user's actual words never reached the palace. Switch the gate to sha256(text). State entries gain a "content_hash" field; the legacy size-only path is preserved when prev_hash is missing so a post-upgrade run does not re-ingest every untouched diary. Closes #925 --- mempalace/diary_ingest.py | 19 +++++++++++++++---- tests/test_closets.py | 20 ++++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/mempalace/diary_ingest.py b/mempalace/diary_ingest.py index 503f0c0827..7939dda9f2 100644 --- a/mempalace/diary_ingest.py +++ b/mempalace/diary_ingest.py @@ -120,12 +120,22 @@ def ingest_diaries( continue date_str = date_match.group(1) - # Skip if content hasn't changed + # Skip if content hasn't changed. Hash-based — size alone false-negatives + # on same-length edits (e.g. "teh" → "the"), silently dropping real edits. state_key = f"{wing}|{diary_path.name}" - prev_size = state.get(state_key, {}).get("size", 0) + prev_entry = state.get(state_key, {}) + prev_hash = prev_entry.get("content_hash") + prev_size = prev_entry.get("size", 0) curr_size = len(text) - if curr_size == prev_size and not force: - continue + curr_hash = hashlib.sha256(text.encode("utf-8")).hexdigest() + if not force: + if prev_hash is not None: + if curr_hash == prev_hash: + continue + elif curr_size == prev_size and prev_size > 0: + # Legacy state without content_hash: keep size-based skip so a + # post-upgrade run doesn't re-ingest every untouched diary. + continue now_iso = datetime.now(timezone.utc).isoformat() drawer_id = _diary_drawer_id(wing, date_str) @@ -184,6 +194,7 @@ def ingest_diaries( state[state_key] = { "size": curr_size, + "content_hash": curr_hash, "entry_count": len(entries), "ingested_at": now_iso, } diff --git a/tests/test_closets.py b/tests/test_closets.py index 976086dce2..bd996e19bc 100644 --- a/tests/test_closets.py +++ b/tests/test_closets.py @@ -605,6 +605,26 @@ def test_ingest_skips_unchanged_on_second_run(self, tmp_path): result = ingest_diaries(str(diary_dir), str(palace_dir)) assert result["days_updated"] == 0 + def test_ingest_detects_same_size_content_edit(self, tmp_path): + # Regression #925: the prior skip-check compared byte length only, so + # any in-place edit preserving total length (e.g. typo fix "teh"→"the") + # was silently dropped. Content-hash check must catch it. + diary_dir = tmp_path / "diaries" + diary_dir.mkdir() + diary_file = diary_dir / "2026-04-13.md" + original = "# 2026-04-13\n\n## 10:00 — Test\n\nThe quick brown fox jumps over.\n" + edited = "# 2026-04-13\n\n## 10:00 — Test\n\nTeh quick brown fox jumps over.\n" + assert len(original) == len(edited), "test setup: edited content must be same length" + diary_file.write_text(original) + palace_dir = tmp_path / "palace" + + from mempalace.diary_ingest import ingest_diaries + + ingest_diaries(str(diary_dir), str(palace_dir), force=True) + diary_file.write_text(edited) + result = ingest_diaries(str(diary_dir), str(palace_dir)) + assert result["days_updated"] == 1, "same-size content edit must trigger re-ingest" + def test_state_file_lives_outside_diary_dir(self, tmp_path): # Regression: the original implementation wrote # ``.diary_ingest_state.json`` *inside* the user's diary directory, From 8e21b5abd48a97830bce4fb3c28584d6e9b2caa5 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Thu, 7 May 2026 12:49:27 -0300 Subject: [PATCH 69/77] test(closet_llm): use _ for unused return values per copilot review --- tests/test_closet_llm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_closet_llm.py b/tests/test_closet_llm.py index 0255ee8e18..905d28b777 100644 --- a/tests/test_closet_llm.py +++ b/tests/test_closet_llm.py @@ -200,7 +200,7 @@ def fake_urlopen(req, timeout=None): patch("urllib.request.urlopen", side_effect=fake_urlopen), patch("mempalace.closet_llm.time.sleep"), ): - parsed, usage = _call_llm(cfg, "/tmp/x", "w", "r", "c") + parsed, _ = _call_llm(cfg, "/tmp/x", "w", "r", "c") assert parsed is None def test_retries_on_json_decode_error(self): @@ -220,7 +220,7 @@ def fake_urlopen(req, timeout=None): patch("urllib.request.urlopen", side_effect=fake_urlopen), patch("mempalace.closet_llm.time.sleep"), ): - parsed, usage = _call_llm(cfg, "/tmp/x", "w", "r", "c") + parsed, _ = _call_llm(cfg, "/tmp/x", "w", "r", "c") assert parsed is None assert call_count["n"] == 3 From 75452380a896959b79a25eb99061db8c7d95a001 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Thu, 7 May 2026 12:51:47 -0300 Subject: [PATCH 70/77] fix(exporter): refuse symlinks at file targets and skip tests on Windows Address Copilot review on #1156: - Per-file symlink check via new _safe_open_for_write() helper. Uses O_NOFOLLOW on POSIX (close TOCTOU window between islink check and open) and falls back to islink + open on Windows. Applied to room files and index.md, mirroring the existing dir-level check. - Tests now wrap os.symlink() in _try_symlink_or_skip() so Windows without Developer Mode and restricted CI sandboxes skip rather than hard-fail. Added two regression tests for the file-level cases (room file, index.md). --- mempalace/exporter.py | 29 +++++++++++++++++-- tests/test_exporter.py | 63 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 88 insertions(+), 4 deletions(-) diff --git a/mempalace/exporter.py b/mempalace/exporter.py index a19181bb3b..2b874c6e7d 100644 --- a/mempalace/exporter.py +++ b/mempalace/exporter.py @@ -11,6 +11,7 @@ regardless of palace size. """ +import errno import os import re from collections import defaultdict @@ -40,6 +41,30 @@ def _reject_symlink(path: str, label: str) -> None: ) +def _safe_open_for_write(path: str, mode: str, encoding: str = "utf-8"): + """Open a file for writing, refusing to follow a symlink at the target path. + + On POSIX (O_NOFOLLOW available) the open itself fails with ELOOP if path is + a symlink — closing the TOCTOU window between an islink check and the open. + On platforms without O_NOFOLLOW (Windows), pre-checks ``os.path.islink``, + which is narrower than no check at all. + """ + o_nofollow = getattr(os, "O_NOFOLLOW", 0) + if o_nofollow: + flags = os.O_WRONLY | os.O_CREAT | o_nofollow + flags |= os.O_APPEND if "a" in mode else os.O_TRUNC + try: + fd = os.open(path, flags, 0o600) + except OSError as e: + if e.errno == errno.ELOOP: + raise ValueError(f"refusing to write: {path!r} is a symbolic link.") from None + raise + return os.fdopen(fd, mode, encoding=encoding) + if os.path.islink(path): + raise ValueError(f"refusing to write: {path!r} is a symbolic link.") + return open(path, mode, encoding=encoding) + + def export_palace(palace_path: str, output_dir: str, format: str = "markdown") -> dict: """Export all palace drawers as markdown files organized by wing/room. @@ -118,7 +143,7 @@ def export_palace(palace_path: str, output_dir: str, format: str = "markdown") - key = (wing, room) is_new = key not in opened_rooms - with open(room_path, "a" if not is_new else "w", encoding="utf-8") as f: + with _safe_open_for_write(room_path, "a" if not is_new else "w") as f: if is_new: f.write(f"# {wing} / {room}\n\n") opened_rooms.add(key) @@ -168,7 +193,7 @@ def export_palace(palace_path: str, output_dir: str, format: str = "markdown") - index_lines.append("") index_path = os.path.join(output_dir, "index.md") - with open(index_path, "w", encoding="utf-8") as f: + with _safe_open_for_write(index_path, "w") as f: f.write("\n".join(index_lines)) stats = { diff --git a/tests/test_exporter.py b/tests/test_exporter.py index e4d4ee724a..6709339e21 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -136,6 +136,22 @@ def test_export_empty_palace(): shutil.rmtree(tmpdir, ignore_errors=True) +def _try_symlink_or_skip(target: str, link: str): + """Create a symlink, skipping the test if the runtime forbids it. + + Windows without Developer Mode/admin and some restricted CI sandboxes + refuse os.symlink with OSError or NotImplementedError. The exporter + hardening is meaningful only where symlinks can be created at all, so + skipping is preferable to a hard failure. + """ + import pytest + + try: + os.symlink(target, link) + except (OSError, NotImplementedError) as e: + pytest.skip(f"symlink creation not supported in this environment: {e}") + + def test_export_refuses_symlinked_output_dir(): """A symlink at the output path must not be followed (defense-in-depth).""" import pytest @@ -146,7 +162,7 @@ def test_export_refuses_symlinked_output_dir(): decoy_target = os.path.join(tmpdir, "decoy_target") os.makedirs(decoy_target) output_dir = os.path.join(tmpdir, "export") - os.symlink(decoy_target, output_dir) + _try_symlink_or_skip(decoy_target, output_dir) with pytest.raises(ValueError, match="symbolic link"): export_palace(palace_path, output_dir) @@ -168,7 +184,7 @@ def test_export_refuses_symlinked_wing_dir(): os.makedirs(decoy_target) output_dir = os.path.join(tmpdir, "export") os.makedirs(output_dir) - os.symlink(decoy_target, os.path.join(output_dir, "alpha")) + _try_symlink_or_skip(decoy_target, os.path.join(output_dir, "alpha")) with pytest.raises(ValueError, match="symbolic link"): export_palace(palace_path, output_dir) @@ -176,3 +192,46 @@ def test_export_refuses_symlinked_wing_dir(): assert os.listdir(decoy_target) == [] finally: shutil.rmtree(tmpdir, ignore_errors=True) + + +def test_export_refuses_symlinked_room_file(): + """A symlink pre-placed at a room file path must not be followed.""" + import pytest + + tmpdir = tempfile.mkdtemp() + try: + palace_path = _setup_palace(tmpdir) + decoy_target = os.path.join(tmpdir, "decoy_target.md") + Path(decoy_target).write_text("untouched\n", encoding="utf-8") + output_dir = os.path.join(tmpdir, "export") + os.makedirs(os.path.join(output_dir, "alpha")) + _try_symlink_or_skip(decoy_target, os.path.join(output_dir, "alpha", "backend.md")) + + with pytest.raises(ValueError, match="symbolic link"): + export_palace(palace_path, output_dir) + + # Decoy file must remain unchanged — open did not follow the symlink. + assert Path(decoy_target).read_text(encoding="utf-8") == "untouched\n" + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def test_export_refuses_symlinked_index_file(): + """A symlink pre-placed at output_dir/index.md must not be followed.""" + import pytest + + tmpdir = tempfile.mkdtemp() + try: + palace_path = _setup_palace(tmpdir) + decoy_target = os.path.join(tmpdir, "decoy_index.md") + Path(decoy_target).write_text("untouched\n", encoding="utf-8") + output_dir = os.path.join(tmpdir, "export") + os.makedirs(output_dir) + _try_symlink_or_skip(decoy_target, os.path.join(output_dir, "index.md")) + + with pytest.raises(ValueError, match="symbolic link"): + export_palace(palace_path, output_dir) + + assert Path(decoy_target).read_text(encoding="utf-8") == "untouched\n" + finally: + shutil.rmtree(tmpdir, ignore_errors=True) From 2ff6283b32f2ed5825122a1e3284cee2fee8b85c Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Thu, 7 May 2026 12:54:09 -0300 Subject: [PATCH 71/77] fix(diary): rebuild closets on hash change + backfill legacy state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address Copilot review on #925: - Full closet rebuild whenever the content hash differs from prior state, not only on entry-count growth. Without this, an in-place edit (same entry count, different body) updated the drawer but left the closet/search index stale — defeats the verbatim guarantee at the search layer even if the drawer is correct. - Legacy size-only skip path now records the computed content_hash back into state so subsequent runs use the strict hash check instead of remaining on the size-only path indefinitely. - Test updates: typo direction in the regression test now matches the comment (typo "Teh" → fix "The"), assertion now also checks the closet collection reflects the edit, and a new test exercises the legacy-state backfill path. --- mempalace/diary_ingest.py | 19 ++++++++---- tests/test_closets.py | 64 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 73 insertions(+), 10 deletions(-) diff --git a/mempalace/diary_ingest.py b/mempalace/diary_ingest.py index 7939dda9f2..e6ffe42d9f 100644 --- a/mempalace/diary_ingest.py +++ b/mempalace/diary_ingest.py @@ -133,10 +133,16 @@ def ingest_diaries( if curr_hash == prev_hash: continue elif curr_size == prev_size and prev_size > 0: - # Legacy state without content_hash: keep size-based skip so a - # post-upgrade run doesn't re-ingest every untouched diary. + # Legacy state without content_hash: keep size-based skip but + # backfill the hash so future runs use the strict check. + state[state_key] = {**prev_entry, "content_hash": curr_hash} continue + # An in-place edit (same entry count, different content) means existing + # closets are stale. Force a full rebuild whenever the hash changes, + # not only on entry-count growth. + content_changed = prev_hash is not None and curr_hash != prev_hash + now_iso = datetime.now(timezone.utc).isoformat() drawer_id = _diary_drawer_id(wing, date_str) entities = _extract_entities_for_metadata(text) @@ -163,7 +169,8 @@ def ingest_diaries( entries = _split_entries(text) prev_entry_count = state.get(state_key, {}).get("entry_count", 0) - new_entries = entries if force else entries[prev_entry_count:] + full_rebuild = force or content_changed + new_entries = entries if full_rebuild else entries[prev_entry_count:] if new_entries: all_lines = [] @@ -185,9 +192,9 @@ def ingest_diaries( } if entities: closet_meta["entities"] = entities - # On a force rebuild, wipe any leftover numbered closets - # from a longer prior run before re-writing. - if force: + # On any full rebuild (force or detected content edit), + # wipe leftover closets from a prior run before re-writing. + if full_rebuild: purge_file_closets(closets_col, source_file) n = upsert_closet_lines(closets_col, closet_id_base, all_lines, closet_meta) closets_created += n diff --git a/tests/test_closets.py b/tests/test_closets.py index bd996e19bc..37a78f4aa4 100644 --- a/tests/test_closets.py +++ b/tests/test_closets.py @@ -23,6 +23,7 @@ cross-diary collisions, force=True purges leftover closets. """ +import hashlib import json import multiprocessing import os @@ -607,13 +608,16 @@ def test_ingest_skips_unchanged_on_second_run(self, tmp_path): def test_ingest_detects_same_size_content_edit(self, tmp_path): # Regression #925: the prior skip-check compared byte length only, so - # any in-place edit preserving total length (e.g. typo fix "teh"→"the") - # was silently dropped. Content-hash check must catch it. + # any in-place edit preserving total length (typo fix "teh"→"the", + # word swap, character reorder) was silently dropped. Content-hash + # check must catch the change AND rebuild the searchable closet so + # the index does not stay stale while the drawer updates. diary_dir = tmp_path / "diaries" diary_dir.mkdir() diary_file = diary_dir / "2026-04-13.md" - original = "# 2026-04-13\n\n## 10:00 — Test\n\nThe quick brown fox jumps over.\n" - edited = "# 2026-04-13\n\n## 10:00 — Test\n\nTeh quick brown fox jumps over.\n" + # Original has the typo "Teh"; the edit fixes it to "The" — same length. + original = "# 2026-04-13\n\n## 10:00 — Test\n\nTeh elaborate jakarta postgres bug.\n" + edited = "# 2026-04-13\n\n## 10:00 — Test\n\nThe elaborate jakarta postgres bug.\n" assert len(original) == len(edited), "test setup: edited content must be same length" diary_file.write_text(original) palace_dir = tmp_path / "palace" @@ -625,6 +629,58 @@ def test_ingest_detects_same_size_content_edit(self, tmp_path): result = ingest_diaries(str(diary_dir), str(palace_dir)) assert result["days_updated"] == 1, "same-size content edit must trigger re-ingest" + # Drawer must hold the corrected text. + drawers = get_collection(str(palace_dir)).get(where={"source_file": str(diary_file)}) + joined_drawers = "\n".join(drawers["documents"]) + assert "The elaborate" in joined_drawers + assert "Teh elaborate" not in joined_drawers, "drawer still holds pre-edit content" + + # And the closet (search index) must reflect the edit too — not just the + # drawer. Otherwise searches would surface stale text. + closets = get_closets_collection(str(palace_dir)).get( + where={"source_file": str(diary_file)} + ) + joined_closets = "\n".join(closets["documents"]) + assert "Teh elaborate" not in joined_closets, "closet index still holds stale content" + + def test_legacy_state_backfills_content_hash(self, tmp_path): + # Upgraded users can carry legacy state entries without ``content_hash``. + # Same-size skip is preserved for that one run, but the hash must be + # recorded so the strict check engages on subsequent runs. + diary_dir = tmp_path / "diaries" + diary_dir.mkdir() + diary_file = diary_dir / "2026-04-13.md" + text = "# 2026-04-13\n\n## 10:00 — Test\n\nUnchanged body content here.\n" + diary_file.write_text(text) + palace_dir = tmp_path / "palace" + + from mempalace.diary_ingest import _state_file_for, ingest_diaries + + # Simulate a legacy state file: only size + entry_count, no content_hash. + state_file = _state_file_for(str(palace_dir), diary_dir.resolve()) + state_file.parent.mkdir(parents=True, exist_ok=True) + state_file.write_text( + json.dumps( + { + f"diary|{diary_file.name}": { + "size": len(text), + "entry_count": 1, + "ingested_at": "2026-04-12T00:00:00+00:00", + } + } + ) + ) + + # Run with no force — size matches, so this should skip ingest. + result = ingest_diaries(str(diary_dir), str(palace_dir)) + assert result["days_updated"] == 0 + + # Hash must have been backfilled into state for the next run's strict check. + persisted = json.loads(state_file.read_text()) + entry = persisted[f"diary|{diary_file.name}"] + assert "content_hash" in entry, "legacy skip path must record the hash" + assert entry["content_hash"] == hashlib.sha256(text.encode("utf-8")).hexdigest() + def test_state_file_lives_outside_diary_dir(self, tmp_path): # Regression: the original implementation wrote # ``.diary_ingest_state.json`` *inside* the user's diary directory, From 26bc3d4f912fd27ede8a4618e8e8b8a54438a953 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Thu, 7 May 2026 17:41:19 -0300 Subject: [PATCH 72/77] test(diary): write fixture with explicit utf-8 to fix Windows hash assert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_legacy_state_backfills_content_hash failed on test-windows because Path.write_text without an encoding uses the system locale (cp1252 on Windows). The em dash was written as 0x97, then read back by diary_ingest as UTF-8 with errors=replace — round-trip produced different bytes than the in-Python literal, so the assertion comparing the persisted hash to sha256(text.encode(utf-8)) diverged. Pin the write side to encoding=utf-8 so the on-disk bytes match what diary_ingest decodes. No production change. --- tests/test_closets.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_closets.py b/tests/test_closets.py index 37a78f4aa4..e016d83e07 100644 --- a/tests/test_closets.py +++ b/tests/test_closets.py @@ -650,8 +650,11 @@ def test_legacy_state_backfills_content_hash(self, tmp_path): diary_dir = tmp_path / "diaries" diary_dir.mkdir() diary_file = diary_dir / "2026-04-13.md" + # Write explicit UTF-8 so the round-trip matches how diary_ingest reads. + # Windows' default text-mode encoding is cp1252; without this the em + # dash would round-trip lossy and the hash assertion below would fail. text = "# 2026-04-13\n\n## 10:00 — Test\n\nUnchanged body content here.\n" - diary_file.write_text(text) + diary_file.write_text(text, encoding="utf-8") palace_dir = tmp_path / "palace" from mempalace.diary_ingest import _state_file_for, ingest_diaries From dfb85cc687638d898e091384190f45d5fc7d1e7f Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 7 May 2026 18:28:09 -0700 Subject: [PATCH 73/77] docs(claude): update last-sync reference to 2026-05-07 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records the upstream/develop sync from 1888b67 (2026-05-03) → ea36a00 (2026-05-07, fork merge 1dbffe0). Tip is the v3.3.5-prep develop branch. --- CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8f59d01f83..b6ac64aba0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ JP's fork of [milla-jovovich/mempalace](https://github.com/milla-jovovich/mempalace) — a local AI memory system using ChromaDB for verbatim storage and semantic search. - **Fork**: `jphein/mempalace` (origin) / `milla-jovovich/mempalace` (upstream) -- **Version**: upstream shipped v3.3.2 on 2026-04-21 (includes our #681/#1000/#1023) and v3.3.3 on 2026-04-24 (includes our #659/#1021). Main merged upstream/develop through 2026-05-03 (commit `1888b67`) so fork runs post-v3.3.3 code; upstream's `chore/release-3.3.4-prep` is in flight. +- **Version**: upstream shipped v3.3.2 on 2026-04-21 (includes our #681/#1000/#1023) and v3.3.3 on 2026-04-24 (includes our #659/#1021). Main merged upstream/develop through 2026-05-07 (commit `ea36a00`, fork merge `1dbffe0`) so fork runs the v3.3.5-prep develop tip; upstream's #1377 retry-once (our co-authored extract) is in tree. - **Python**: venv at `./venv/`, editable install with dev deps - **Palace data**: `~/.mempalace/palace` (ChromaDB) + `~/.mempalace/config.json` From 1975207c99193b8c8a088e57b96ad11b0fa5665d Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 7 May 2026 20:26:41 -0700 Subject: [PATCH 74/77] docs(claude): backfill missing commit refs + retire moot row 15 hedge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves the 4 pre-existing failures from `scripts/check-docs.sh` check 5 (fork-only YAML commits referenced in CLAUDE.md): - Row 21: add `f9f5cc4` (kind= filter add) and `7ba28dc` (retire) to the existing add→delete narrative. - Row 23: add commit ref `42817d7` to the phase D 2026-04-26 update. - Row 31: add commit ref `8252025` for the fork-side `scripts/deploy.sh` (was listed as needs-generalization but commit hash was missing). Row 15 hedge cleanup: the "tracked as a follow-up" sentence about `_get_session_recovery_collection` SIGSEGV exposure is moot — that function no longer exists anywhere in the tree (verified via grep); the recovery collection was retired entirely in row 21's deletion and the verbatim-only shift (row 32). Replaced with a forward reference instead of a tracking item. Verified: - `scripts/check-docs.sh` now passes all 5 checks (was 4 failures) - `pytest -q` still 1,733 passed, 1 skipped — docs-only change --- CLAUDE.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index b6ac64aba0..3ebcb87882 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -44,16 +44,16 @@ Ruff for linting (`ruff check`), line length 100, target Python 3.9. 12. **feat: search warnings + sqlite BM25 top-up** — `search_memories()` returns `warnings: [...]` and `available_in_scope: N` whenever the vector path underdelivers (sparse HNSW after repair, `#951` filter-planner failure, drift). Fallback promotes BM25-ranked sqlite candidates tagged `matched_via: "sqlite_bm25_fallback"`. Closes the "silent 0-hit when data is in sqlite" failure mode. CLI `search()` delegates to `search_memories()` so both paths share the fallback. 13. ~~**fix: stop_hook_active guard**~~ — **merged upstream via #1021 on 2026-04-22.** No longer fork-ahead. 14. ~~**fix: `_output()` stdout routing**~~ — **merged upstream via #1021 on 2026-04-22.** No longer fork-ahead. -15. ~~**fix: `_get_client()` get-then-create guard**~~ — **merged upstream via #1262 (Legion345) on 2026-05-01 + the MCP-server-side companion #1289 (igorls) on 2026-05-01 + the `embedding_function=` plumbing in #1303 (igorls) on 2026-05-01**. All three landed in the develop sync on 2026-05-03. The fork-only `_get_session_recovery_collection` (introduced in row 23) still uses the older `get_or_create_collection` pattern; theoretical SIGSEGV exposure on legacy recovery collections only — tracked as a follow-up. No longer fork-ahead in the canonical paths. +15. ~~**fix: `_get_client()` get-then-create guard**~~ — **merged upstream via #1262 (Legion345) on 2026-05-01 + the MCP-server-side companion #1289 (igorls) on 2026-05-01 + the `embedding_function=` plumbing in #1303 (igorls) on 2026-05-01**. All three landed in the develop sync on 2026-05-03. The earlier hedge about `_get_session_recovery_collection` is now moot — the recovery collection was retired entirely in the verbatim-only shift (row 21 deletion / row 32). No longer fork-ahead. 16. **perf: `miner.status()` paginated `col.get()`** — upstream's single `col.get(limit=total)` hits SQLite's max-variable limit on palaces with many thousands of drawers; fork paginates in 10 K-drawer batches. 17. **feat: configurable chunking parameters** — `chunk_size`, `chunk_overlap`, `min_chunk_size` exposed via `MempalaceConfig` properties (defaults 800 / 100 / 50). **Update 2026-05-03 (commit `6ce37c0`):** the three keys are intentionally NOT written to `config.json` by `cfg.init()`. Earlier wording above was wrong — writing the miner.py defaults as materialized values into config.json broke `convo_miner.py:427-431`'s "user has tuned this" detection (`_file_config.get("min_chunk_size") is None` ⇔ "untuned"), which silently overrode convo_miner's stricter 30-char floor and dropped legitimate short conversation exchanges on any user who'd ever run `mempalace init`. JP's pre-row-17 config.json never had these keys so he never saw it; surfaced by a pytest fixture leak (`tests/conftest.py:21-27` HOME redirect → polluting test writes default config in session-tmp `~/.mempalace/` → next test reads `min_chunk_size: 50` from there). Fix is module-default-only; properties at `config.py:204-216` already supply the right fallback values via `.get(key, default)`. Same fix pushed to #1024's PR branch as `df9187c`. 18. ~~**fix: PID file guard prevents stacking mine processes**~~ — **merged upstream via #1023 in v3.3.2.** Includes the Windows `os.kill` → `OpenProcess` cross-platform fix. No longer fork-ahead. 19. **fix: `.claude-plugin/` venv-aware Python resolution** — hooks (`mempal-stop-hook.sh`, `mempal-precompact-hook.sh`) and `.mcp.json` resolve Python in this order: `MEMPALACE_PYTHON` env → `$PLUGIN_ROOT/venv/bin/python3` → system `python3`. Upstream's `5fe0c1c` + `be9214a` (fatkobra) and `9f5b8f5` (Pim) regressed to PATH-only lookups and bare `"mempalace-mcp"` command, which break editable dev installs where `mempalace`/`mempalace-mcp` only live in the repo venv. Documented here so future `upstream/develop` merges surface the conflict rather than silently re-regress. Attempted via #1115 on 2026-04-22; withdrew 2026-04-23 as premature pending #1069 arbitration — CI correctly caught the #942 PATH-only contract violation. Re-submit after bensig's direction on #1069. 20. ~~**fix: `_tokenize` None-document guard**~~ — **merged upstream via #1198 on 2026-04-26.** No longer fork-ahead. -21. ~~**feat: `kind` filter on `search_memories` excludes Stop-hook checkpoints by default**~~ — **deleted 2026-04-27 as transitional/inert.** The structural split (Phases A–E, see row 23) moved all checkpoints to `mempalace_session_recovery`; production has 0 checkpoints in `mempalace_drawers`, so the filter was filtering nothing. Removed `_CHECKPOINT_TOPICS` from `searcher.py`, `_is_checkpoint_drawer`, `_apply_kind_text_filter`, the `max(n*20, 100)` over-fetch hack (back to `n_results * 3`), and the `kind=` parameter on `search_memories` / `mempalace_search` / palace-daemon `/search` & `/context`. Write-side `_CHECKPOINT_TOPICS` (topic→collection routing in `tool_diary_write`) lives in `palace.py` now alongside `_SESSION_RECOVERY_COLLECTION`. `TestCheckpointFilter` (9 tests) deleted. +21. ~~**feat: `kind` filter on `search_memories` excludes Stop-hook checkpoints by default**~~ (originally added in commit `f9f5cc4`) — **deleted 2026-04-27 in commit `7ba28dc` as transitional/inert.** The structural split (Phases A–E, see row 23) moved all checkpoints to `mempalace_session_recovery`; production has 0 checkpoints in `mempalace_drawers`, so the filter was filtering nothing. Removed `_CHECKPOINT_TOPICS` from `searcher.py`, `_is_checkpoint_drawer`, `_apply_kind_text_filter`, the `max(n*20, 100)` over-fetch hack (back to `n_results * 3`), and the `kind=` parameter on `search_memories` / `mempalace_search` / palace-daemon `/search` & `/context`. Write-side `_CHECKPOINT_TOPICS` (topic→collection routing in `tool_diary_write`) lives in `palace.py` now alongside `_SESSION_RECOVERY_COLLECTION`. `TestCheckpointFilter` (9 tests) deleted. 22. ~~**fix: `palace_graph.build_graph` skips None metadata**~~ — **merged upstream via #1201 on 2026-04-26.** No longer fork-ahead. -23. **feat: checkpoint collection split — phases A–C** (commit `e266365`, 2026-04-25) — Promoted from "future work" to "necessary" by 2026-04-25 Cat 9 A/B (`kind=all` 632 tokens/Q vs `kind=content` 3 tokens/Q on the canonical 151K palace; over-fetch=100 inadequate, structural fix non-optional). **Phase A:** new `_SESSION_RECOVERY_COLLECTION` constant + `get_session_recovery_collection()` in `palace.py` (mirrors `get_collection`'s shape — cosine, num_threads=1). **Phase B:** `tool_diary_write` routes `topic in _CHECKPOINT_TOPICS` to the dedicated `mempalace_session_recovery` collection, everything else stays in `mempalace_drawers`; new `_get_session_recovery_collection()` in `mcp_server.py` with parallel cache. **Phase C:** new `tool_session_recovery_read` MCP handler reads recovery collection only with optional filters `session_id`, `agent`, `since`, `until`, `wing`, `limit`; `session_id` added as optional metadata field on `tool_diary_write` so the new tool can filter by Claude Code session. Registered in `TOOLS` dict, documented in `website/reference/mcp-tools.md`. 12 new tests across `tests/test_session_recovery.py` + `TestCheckpointRouting` + `TestSessionRecoveryRead`. Design + plan at `docs/superpowers/specs/2026-04-25-checkpoint-collection-split.md` and `docs/superpowers/plans/2026-04-25-checkpoint-collection-split-impl.md`. **Phases D (data migration of ~640 existing checkpoints out of main collection) and E (palace-daemon `lifespan` auto-migrate + `mempalace repair --mode reorganize`) deferred** — multi-day work, gated on a separate go-ahead. Once D lands and the canonical-palace re-run shows the predicted `kind=all` ≈ `kind=content` token convergence, the `kind=` post-filter and over-fetch hack become deletable. **Update 2026-04-26:** phase D shipped — `migrate_checkpoints_to_recovery()` in `mempalace/migrate.py`, idempotent walk that moves topic in `_CHECKPOINT_TOPICS` drawers from main → recovery while preserving IDs and metadata. Wired into `mempalace repair --mode reorganize` (CLI dispatch in `cli.py` chooses between `rebuild` (HNSW from sqlite) and `reorganize` (this new path)). PreCompact hook also incorporated — `hook_precompact` now writes a recovery marker via `_save_diary_direct` mirroring Stop, so a context-compaction event leaves a queryable timestamp in the recovery collection. 6 new migration tests in `test_migrate.py::TestMigrateCheckpointsToRecovery`. **Phase E shipped** in palace-daemon commit [`034023c`](https://github.com/jphein/palace-daemon/commit/034023c) on 2026-04-26 — `lifespan` calls `migrate_checkpoints_to_recovery()` in an executor on startup, gated behind `PALACE_AUTO_MIGRATE_CHECKPOINTS=1` (default on), with `ImportError` fallthrough so upstream-shaped installs without `mempalace.migrate` still start cleanly. Canonical 151K palace migrated 667 checkpoints on 2026-04-26 10:24:09 PDT. **Cleanup phase shipped 2026-04-27** — empirical check on production showed 0 checkpoints in `mempalace_drawers` (763 in `mempalace_session_recovery`), so the kind= filter was provably inert. Deleted in row 21 above. +23. **feat: checkpoint collection split — phases A–C** (commit `e266365`, 2026-04-25) — Promoted from "future work" to "necessary" by 2026-04-25 Cat 9 A/B (`kind=all` 632 tokens/Q vs `kind=content` 3 tokens/Q on the canonical 151K palace; over-fetch=100 inadequate, structural fix non-optional). **Phase A:** new `_SESSION_RECOVERY_COLLECTION` constant + `get_session_recovery_collection()` in `palace.py` (mirrors `get_collection`'s shape — cosine, num_threads=1). **Phase B:** `tool_diary_write` routes `topic in _CHECKPOINT_TOPICS` to the dedicated `mempalace_session_recovery` collection, everything else stays in `mempalace_drawers`; new `_get_session_recovery_collection()` in `mcp_server.py` with parallel cache. **Phase C:** new `tool_session_recovery_read` MCP handler reads recovery collection only with optional filters `session_id`, `agent`, `since`, `until`, `wing`, `limit`; `session_id` added as optional metadata field on `tool_diary_write` so the new tool can filter by Claude Code session. Registered in `TOOLS` dict, documented in `website/reference/mcp-tools.md`. 12 new tests across `tests/test_session_recovery.py` + `TestCheckpointRouting` + `TestSessionRecoveryRead`. Design + plan at `docs/superpowers/specs/2026-04-25-checkpoint-collection-split.md` and `docs/superpowers/plans/2026-04-25-checkpoint-collection-split-impl.md`. **Phases D (data migration of ~640 existing checkpoints out of main collection) and E (palace-daemon `lifespan` auto-migrate + `mempalace repair --mode reorganize`) deferred** — multi-day work, gated on a separate go-ahead. Once D lands and the canonical-palace re-run shows the predicted `kind=all` ≈ `kind=content` token convergence, the `kind=` post-filter and over-fetch hack become deletable. **Update 2026-04-26 (commit `42817d7`):** phase D shipped — `migrate_checkpoints_to_recovery()` in `mempalace/migrate.py`, idempotent walk that moves topic in `_CHECKPOINT_TOPICS` drawers from main → recovery while preserving IDs and metadata. Wired into `mempalace repair --mode reorganize` (CLI dispatch in `cli.py` chooses between `rebuild` (HNSW from sqlite) and `reorganize` (this new path)). PreCompact hook also incorporated — `hook_precompact` now writes a recovery marker via `_save_diary_direct` mirroring Stop, so a context-compaction event leaves a queryable timestamp in the recovery collection. 6 new migration tests in `test_migrate.py::TestMigrateCheckpointsToRecovery`. **Phase E shipped** in palace-daemon commit [`034023c`](https://github.com/jphein/palace-daemon/commit/034023c) on 2026-04-26 — `lifespan` calls `migrate_checkpoints_to_recovery()` in an executor on startup, gated behind `PALACE_AUTO_MIGRATE_CHECKPOINTS=1` (default on), with `ImportError` fallthrough so upstream-shaped installs without `mempalace.migrate` still start cleanly. Canonical 151K palace migrated 667 checkpoints on 2026-04-26 10:24:09 PDT. **Cleanup phase shipped 2026-04-27** — empirical check on production showed 0 checkpoints in `mempalace_drawers` (763 in `mempalace_session_recovery`), so the kind= filter was provably inert. Deleted in row 21 above. 27. **perf: batch ChromaDB inserts in miner (cherry-pick of upstream #1085)** (commit `6be6fff`, 2026-04-26) — Cherry-picked @midweste's [#1085](https://github.com/MemPalace/mempalace/pull/1085) "batch ChromaDB inserts in miner — 10-30x faster mining". Upstream PR #1085 is still **OPEN** as of 2026-04-26 (created 2026-04-21, base=develop, not yet merged) — verified via `gh pr view 1085 --repo MemPalace/mempalace`. We cherry-picked the commit ahead of merge so the fork can use it now; this row clears when #1085 merges into develop and we next sync. We don't file a competing fork-side PR — the proposal is @midweste's. New `_build_drawer()` helper builds id+document+metadata in one shot; new `add_drawers()` batch-insert function takes the full chunk list and sub-batches at `DRAWER_UPSERT_BATCH_SIZE` (one chromadb upsert + one ONNX embedding forward-pass per sub-batch instead of per-chunk). `process_file` now calls `add_drawers` directly. Hoists `datetime.now()` and `os.path.getmtime()` to file-level (2 syscalls per file instead of 2N). **Conflict resolution:** fork already had a fork-only `_build_drawer_metadata` + an outer batch loop in `process_file`; upstream's clean structure supersedes both. Kept fork's `DRAWER_UPSERT_BATCH_SIZE=1000` (more conservative than upstream's 5000 for embedding-pass memory headroom); aliased upstream's `CHROMA_BATCH_LIMIT` to point at it so any code/test referencing either name sees the same value. 74/74 miner+convo_miner tests pass; full suite 1366/1366. Becomes a no-op when #1085 merges into upstream develop and we next sync develop→main. @@ -69,7 +69,7 @@ Ruff for linting (`ruff check`), line length 100, target Python 3.9. 30. **fix: scrub embedded API key from `.claude-plugin/` plugin manifests** (commits `8119149` → `9f91e18`, 2026-04-27) — Two of our committed plugin manifests (`.claude-plugin/.mcp.json`, `.claude-plugin/hooks/hooks.json`) shipped with my real (rotated) API key embedded as a literal in the manifest's `env` block, plus my homelab daemon URL. First commit reverted both to upstream-shape (no env block, in-process MCP); second commit restored daemon-routing on `.mcp.json` (URL + path) but **without** the embedded credential — `PALACE_API_KEY` now inherits at runtime from `~/.claude/settings.local.json`'s `env` block (which Claude Code passes to spawned MCP servers and hooks). Net: my fork-main carries the daemon-routed config matching production deployment; the literal credential lives one place only (gitignored `settings.local.json`). The literal is still in commit history at `c09582c` and earlier — destructive-history-rewrite is not worth the cost on an already-rotated key. Companion to palace-daemon [PR #12](https://github.com/rboarescu/palace-daemon/pull/12) which fixes the same class of embedded-default in `clients/palace-mode`. -31. **palace-daemon upstream PR push** (2026-04-27) — Filed seven small/medium PRs against `rboarescu/palace-daemon` covering most of the fork-ahead daemon work: [#7](https://github.com/rboarescu/palace-daemon/pull/7) `limit=` honored fix, [#8](https://github.com/rboarescu/palace-daemon/pull/8) `_canonical_topic` synonym rewrite, [#9](https://github.com/rboarescu/palace-daemon/pull/9) `verify-routes.sh` smoke test, [#10](https://github.com/rboarescu/palace-daemon/pull/10) portable-path fix in dispatcher (real bug — broke for everyone except me), [#11](https://github.com/rboarescu/palace-daemon/pull/11) `event-log-frame.md` architectural reference doc, [#12](https://github.com/rboarescu/palace-daemon/pull/12) strip embedded API-key/URL defaults from `palace-mode` (security hygiene), [#13](https://github.com/rboarescu/palace-daemon/pull/13) `GET /graph` endpoint with design doc. Pending follow-ups: `GET /viz` (depends on #13), `clients/mempal-fast.py` refactor; needs-generalization items: `palace-mode install` subcommand, `auto-repair-if-empty.sh`, `deploy.sh`. Daemon README has the full open-PR + needs-generalization breakdown. +31. **palace-daemon upstream PR push** (2026-04-27) — Filed seven small/medium PRs against `rboarescu/palace-daemon` covering most of the fork-ahead daemon work: [#7](https://github.com/rboarescu/palace-daemon/pull/7) `limit=` honored fix, [#8](https://github.com/rboarescu/palace-daemon/pull/8) `_canonical_topic` synonym rewrite, [#9](https://github.com/rboarescu/palace-daemon/pull/9) `verify-routes.sh` smoke test, [#10](https://github.com/rboarescu/palace-daemon/pull/10) portable-path fix in dispatcher (real bug — broke for everyone except me), [#11](https://github.com/rboarescu/palace-daemon/pull/11) `event-log-frame.md` architectural reference doc, [#12](https://github.com/rboarescu/palace-daemon/pull/12) strip embedded API-key/URL defaults from `palace-mode` (security hygiene), [#13](https://github.com/rboarescu/palace-daemon/pull/13) `GET /graph` endpoint with design doc. Pending follow-ups: `GET /viz` (depends on #13), `clients/mempal-fast.py` refactor; needs-generalization items: `palace-mode install` subcommand, `auto-repair-if-empty.sh`, `deploy.sh` (fork-side `scripts/deploy.sh` shipped in commit `8252025` — Syncthing-aware redeploy, not yet generalized for upstream). Daemon README has the full open-PR + needs-generalization breakdown. 34. **feat: daemon routing for CLI `status` / `search` / `mine`** (commit `22ef562`, 2026-05-07) — Companion to row 33; closes the last desktop-side path that opened a local chromadb client. Adds `_daemon_strict()`, `_call_daemon_tool()`, `_post_daemon_mine_cli()` helpers in `cli.py` mirroring `hooks_cli.py`'s gate; gates `cmd_status`, `cmd_search`, `cmd_mine`. Read paths route via JSON-RPC `tools/call` against the daemon's `/mcp`; mine routes via `/mine` (same endpoint hooks_cli already uses). Output is formatted to match the local `miner.status` and `searcher.search` printers — same human-readable shape, with the daemon URL surfaced in the header so the reader knows which view they're looking at. `--palace ` always overrides routing (explicit path means the user asked for THAT palace, not the canonical one). Local-only commands (`init`, `repair`, `export`, `sweep`, `purge`, `mined`, `wakeup`) stay local because they need on-host filesystem access (HNSW rebuild, palace dump, sweeper deduplication state). When `mempalace-data/` is archived those commands will fail with "no palace found" until pointed elsewhere — that's the right "your data is at the daemon, not local" signpost. 14 new tests in `tests/test_cli_daemon.py`. Live smoke against `disks.jphe.in:8085`: `mempalace status` returns 160,351 drawers, `mempalace search` returns properly-formatted hits. Suite 1591 passed. From 0639b77079d6ceb3596194f7c5f11592d65dc959 Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 7 May 2026 20:29:41 -0700 Subject: [PATCH 75/77] fix(lint): remove unused pytest import in test_mcp_server_daemon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ruff F401: `pytest` imported but unused. The file uses no `pytest.` calls — fixtures (tmp_path, monkeypatch) are pytest's implicit injections, not pytest module references. Pre-existing on main since 41359ba (2026-05-07, row 33's daemon routing tests). Surfaced now because PR #15 is the first PR- triggered CI run that touches the lint job since the import landed. Same fix applies to PR #14's lint failure once #15 merges into main and #14 rebases. 15/15 daemon tests still pass. --- tests/test_mcp_server_daemon.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_mcp_server_daemon.py b/tests/test_mcp_server_daemon.py index 511152abdf..d91852760d 100644 --- a/tests/test_mcp_server_daemon.py +++ b/tests/test_mcp_server_daemon.py @@ -16,8 +16,6 @@ import json from unittest.mock import patch -import pytest - # ── _daemon_strict gate ───────────────────────────────────────────────── From fe2992be927ca4165debebbd0f8f86f350eeaf13 Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 7 May 2026 20:32:23 -0700 Subject: [PATCH 76/77] style: ruff format scripts/chunk_strategy_ablation.py + tests/test_cli_daemon.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI's `ruff format --check .` flagged these two fork-only files as unformatted. Same pre-existing class as the F401 in `0639b77` — local pytest doesn't run ruff, so format drift was invisible until CI surfaced it. No semantic changes; pure whitespace/quote-style reformatting. --- scripts/chunk_strategy_ablation.py | 186 +++++++++++++++++++---------- tests/test_cli_daemon.py | 4 +- 2 files changed, 126 insertions(+), 64 deletions(-) diff --git a/scripts/chunk_strategy_ablation.py b/scripts/chunk_strategy_ablation.py index 5beed3103c..2688e2784d 100755 --- a/scripts/chunk_strategy_ablation.py +++ b/scripts/chunk_strategy_ablation.py @@ -28,6 +28,7 @@ across the three runs. The output is intended to be pasted into an upstream RFC issue or a fork-side ADR. """ + from __future__ import annotations import ast @@ -165,9 +166,11 @@ def _chunk_python_ast( lines = content.splitlines(keepends=True) cuts = sorted( - {n.lineno - 1 for n in tree.body if isinstance( - n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) - )} + { + n.lineno - 1 + for n in tree.body + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) + } ) if not cuts: return _chunk_paragraph_aware( @@ -218,12 +221,16 @@ def _A(content, source_file): def _B(content, source_file): if source_file.lower().endswith((".md", ".markdown", ".rst")): - return _chunk_markdown_heading_aware(content, source_file, chunk_size, overlap, min_size) + return _chunk_markdown_heading_aware( + content, source_file, chunk_size, overlap, min_size + ) return _chunk_paragraph_aware(content, source_file, chunk_size, overlap, min_size) def _C(content, source_file): if source_file.lower().endswith((".md", ".markdown", ".rst")): - return _chunk_markdown_heading_aware(content, source_file, chunk_size, overlap, min_size) + return _chunk_markdown_heading_aware( + content, source_file, chunk_size, overlap, min_size + ) if source_file.lower().endswith(".py"): return _chunk_python_ast(content, source_file, chunk_size, overlap, min_size) return _chunk_paragraph_aware(content, source_file, chunk_size, overlap, min_size) @@ -246,53 +253,104 @@ def build_strategies(chunk_size: int) -> List[Tuple[str, Callable]]: PROBES: List[Tuple[str, str, str]] = [ - ("How does the stop hook save diary entries?", "hooks_cli.py", - "primary save path lives in hooks_cli; mcp_server's tool_diary_write is the API surface"), - ("ChromaDB HNSW segment quarantine for stale indexes", "backends/chroma.py", - "quarantine_stale_hnsw lives in the chroma backend"), - ("BM25 tokenization with None-document safety", "searcher.py", - "_tokenize and _bm25_scores are searcher-internal"), - ("How is a wing name normalized — hyphens vs underscores?", "config.py", - "normalize_wing_name lives at module top of config.py"), - ("Validate chunk_size to prevent the infinite loop hazard", "miner.py", - "chunk_text guard or its analog"), - ("Pre-PreCompact transcript auto-mining via daemon /mine endpoint", "hooks_cli.py", - "_post_daemon_mine lives in hooks_cli; daemon side is palace-daemon repo"), - ("Topic-routing branch routing checkpoint diary writes to a separate collection", "palace.py", - "_CHECKPOINT_TOPICS / get_session_recovery_collection (note: removed in fork; baseline corpus has it)"), - ("Knowledge-graph entity validation that allows commas and parentheses", "config.py", - "sanitize_kg_value"), - ("Closet-boost rank constants used to re-order vector hits", "searcher.py", - "CLOSET_RANK_BOOSTS and the rank loop"), - ("Hybrid rank combining BM25 with cosine similarity", "searcher.py", - "_hybrid_rank"), - ("Wing assignment from Claude Code transcript path regex", "convo_miner.py", - "_wing_from_transcript_path"), - ("Detect HNSW capacity divergence and quarantine the segment", "backends/chroma.py", - "quarantine_stale_hnsw + cold-start gate"), - ("Spellcheck user prose before writing to drawers", "spellcheck.py", - "spellcheck_user_text is the entry point"), - ("Slack JSON export normalization with positional speaker roles", "normalize.py", - "_try_slack_json + provenance footer"), - ("Build the where filter for wing/room scoping in chromadb queries", "searcher.py", - "build_where_filter — small but probe a known target"), + ( + "How does the stop hook save diary entries?", + "hooks_cli.py", + "primary save path lives in hooks_cli; mcp_server's tool_diary_write is the API surface", + ), + ( + "ChromaDB HNSW segment quarantine for stale indexes", + "backends/chroma.py", + "quarantine_stale_hnsw lives in the chroma backend", + ), + ( + "BM25 tokenization with None-document safety", + "searcher.py", + "_tokenize and _bm25_scores are searcher-internal", + ), + ( + "How is a wing name normalized — hyphens vs underscores?", + "config.py", + "normalize_wing_name lives at module top of config.py", + ), + ( + "Validate chunk_size to prevent the infinite loop hazard", + "miner.py", + "chunk_text guard or its analog", + ), + ( + "Pre-PreCompact transcript auto-mining via daemon /mine endpoint", + "hooks_cli.py", + "_post_daemon_mine lives in hooks_cli; daemon side is palace-daemon repo", + ), + ( + "Topic-routing branch routing checkpoint diary writes to a separate collection", + "palace.py", + "_CHECKPOINT_TOPICS / get_session_recovery_collection (note: removed in fork; baseline corpus has it)", + ), + ( + "Knowledge-graph entity validation that allows commas and parentheses", + "config.py", + "sanitize_kg_value", + ), + ( + "Closet-boost rank constants used to re-order vector hits", + "searcher.py", + "CLOSET_RANK_BOOSTS and the rank loop", + ), + ("Hybrid rank combining BM25 with cosine similarity", "searcher.py", "_hybrid_rank"), + ( + "Wing assignment from Claude Code transcript path regex", + "convo_miner.py", + "_wing_from_transcript_path", + ), + ( + "Detect HNSW capacity divergence and quarantine the segment", + "backends/chroma.py", + "quarantine_stale_hnsw + cold-start gate", + ), + ( + "Spellcheck user prose before writing to drawers", + "spellcheck.py", + "spellcheck_user_text is the entry point", + ), + ( + "Slack JSON export normalization with positional speaker roles", + "normalize.py", + "_try_slack_json + provenance footer", + ), + ( + "Build the where filter for wing/room scoping in chromadb queries", + "searcher.py", + "build_where_filter — small but probe a known target", + ), # ── Markdown-targeted probes (only meaningful on a corpus that # includes the repo's docs, not the package-only mine) ──────────── - ("Why did we move checkpoints into a separate recovery collection?", - "2026-04-25-checkpoint-collection-split.md", - "Phase A-E split design doc; only in spec, not code"), - ("Verbatim-only Phase 2 architecture and migration plan", - "2026-05-05-verbatim-only-design.md", - "verbatim-only spec — drops checkpoint summaries, retires recovery collection"), - ("Pre-release grep checklist for mempalace-mcp entry point alignment", - "RELEASING.md", - "fork-side release doc filed as upstream #1142"), - ("Fork-ahead row inventory and upstream PR tracking table", - "CLAUDE.md", - "row inventory at fork's CLAUDE.md, not in any .py"), - ("Hook silent_save vs block-mode behavior — why one beats the other", - "CLAUDE.md", - "Hook Save Architecture section in CLAUDE.md"), + ( + "Why did we move checkpoints into a separate recovery collection?", + "2026-04-25-checkpoint-collection-split.md", + "Phase A-E split design doc; only in spec, not code", + ), + ( + "Verbatim-only Phase 2 architecture and migration plan", + "2026-05-05-verbatim-only-design.md", + "verbatim-only spec — drops checkpoint summaries, retires recovery collection", + ), + ( + "Pre-release grep checklist for mempalace-mcp entry point alignment", + "RELEASING.md", + "fork-side release doc filed as upstream #1142", + ), + ( + "Fork-ahead row inventory and upstream PR tracking table", + "CLAUDE.md", + "row inventory at fork's CLAUDE.md, not in any .py", + ), + ( + "Hook silent_save vs block-mode behavior — why one beats the other", + "CLAUDE.md", + "Hook Save Architecture section in CLAUDE.md", + ), ] @@ -314,6 +372,7 @@ def _build_palace(corpus_dir: Path, palace_dir: Path, strategy_fn: Callable) -> # MempalaceConfig — set MEMPALACE_PALACE_PATH so it points at our # temp palace. import os + os.environ["MEMPALACE_PALACE_PATH"] = str(palace_dir) # Force a fresh MempalaceConfig read. cfg = MempalaceConfig() @@ -325,6 +384,7 @@ def _build_palace(corpus_dir: Path, palace_dir: Path, strategy_fn: Callable) -> files=files, ) from mempalace.palace import get_collection + col = get_collection(str(palace_dir)) return col.count() finally: @@ -334,8 +394,10 @@ def _build_palace(corpus_dir: Path, palace_dir: Path, strategy_fn: Callable) -> def _query_palace(palace_dir: Path, query: str, n_results: int = 10) -> list: """Return list of (rank, source_file_basename, similarity) tuples.""" import os + os.environ["MEMPALACE_PALACE_PATH"] = str(palace_dir) from mempalace.searcher import search_memories + result = search_memories(query, str(palace_dir), n_results=n_results) out = [] for rank, hit in enumerate(result.get("results") or [], start=1): @@ -366,9 +428,7 @@ def _recall_at_k(hits: list, expected: str, k: int) -> int: def main(argv: List[str] | None = None) -> int: - parser = argparse.ArgumentParser( - description="Three-way A/B/C chunking strategy ablation." - ) + parser = argparse.ArgumentParser(description="Three-way A/B/C chunking strategy ablation.") parser.add_argument( "--corpus", default=str(_REPO_ROOT / "mempalace"), @@ -452,13 +512,15 @@ def main(argv: List[str] | None = None) -> int: rrs.append(rr) recall_at_5 += r5 recall_at_10 += r10 - per_probe.append({ - "query": query, - "expected": expected, - "rank": rank, - "rr": round(rr, 4), - "top3": [b for _r, b, _ in hits[:3]], - }) + per_probe.append( + { + "query": query, + "expected": expected, + "rank": rank, + "rr": round(rr, 4), + "top3": [b for _r, b, _ in hits[:3]], + } + ) mrr = sum(rrs) / len(rrs) r5_pct = 100 * recall_at_5 / len(PROBES) r10_pct = 100 * recall_at_10 / len(PROBES) @@ -501,7 +563,9 @@ def main(argv: List[str] | None = None) -> int: for name, _ in strategies: r = summary["strategies"][name]["probes"][i]["rank"] ranks.append(str(r) if r is not None else "-") - print(f" [{','.join(f'{x:>3}' for x in ranks)}] expects={Path(expected).name:<22} q={q[:55]}") + print( + f" [{','.join(f'{x:>3}' for x in ranks)}] expects={Path(expected).name:<22} q={q[:55]}" + ) if args.out: Path(args.out).write_text(json.dumps(summary, indent=2)) diff --git a/tests/test_cli_daemon.py b/tests/test_cli_daemon.py index 8095ff6804..94054f721a 100644 --- a/tests/test_cli_daemon.py +++ b/tests/test_cli_daemon.py @@ -242,9 +242,7 @@ def test_local_path_when_daemon_unset(self): with patch.dict("os.environ", {}, clear=True): with patch("mempalace.cli.MempalaceConfig") as mock_cfg: mock_cfg.return_value.palace_path = "/local/palace" - args = argparse.Namespace( - query="x", wing=None, room=None, results=5, palace=None - ) + args = argparse.Namespace(query="x", wing=None, room=None, results=5, palace=None) with patch("mempalace.searcher.search") as mock_search: cli.cmd_search(args) mock_search.assert_called_once() From f40224b2c7e78f0144221147557312b1190153be Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 7 May 2026 20:39:42 -0700 Subject: [PATCH 77/77] fix(tests): make 2 fork-side tests cross-platform on Windows Pre-existing windows-only failures since rows 32 and 34 landed (2026-05-05 and 2026-05-07). Surfaced now because PR #15 is the first PR-triggered Windows CI run that exercised these. 1. tests/test_cli_daemon.py::test_routes_projects_mode_to_daemon `endswith("/home/u/proj")` failed because Path.expanduser() .resolve() produces "D:\\home\\u\\proj" on Windows. Normalize separators before the suffix check. 2. tests/test_normalize.py::test_config_reads_from_hooks_block monkeypatch.setenv("HOME", ...) doesn't redirect Path.home() on Windows (Path.home consults USERPROFILE on Windows, HOME on POSIX). Set both so the test redirect works on both platforms. Both files PASS on Linux (no regression). Same fix would apply to PR #14 once #15 merges into main. --- tests/test_cli_daemon.py | 5 ++++- tests/test_normalize.py | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_cli_daemon.py b/tests/test_cli_daemon.py index 94054f721a..26b45e5f13 100644 --- a/tests/test_cli_daemon.py +++ b/tests/test_cli_daemon.py @@ -289,7 +289,10 @@ def fake_urlopen(req, timeout=None): assert ex.value.code == 0 assert captured["body"]["mode"] == "projects" - assert captured["body"]["dir"].endswith("/home/u/proj") + # Normalize separators so the assertion holds on Windows, where + # Path.expanduser().resolve() returns a backslash-prefixed path + # like "D:\\home\\u\\proj". + assert captured["body"]["dir"].replace("\\", "/").endswith("/home/u/proj") def test_routes_convos_mode_to_daemon(self): from mempalace import cli diff --git a/tests/test_normalize.py b/tests/test_normalize.py index 6a374fd81a..e6d07cb7dd 100644 --- a/tests/test_normalize.py +++ b/tests/test_normalize.py @@ -1551,7 +1551,10 @@ def test_config_default_off(self): def test_config_reads_from_hooks_block(self, tmp_path, monkeypatch): from mempalace.config import MempalaceConfig + # Path.home() consults HOME on POSIX and USERPROFILE on Windows; + # set both so the redirect works on both platforms. monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) cfg_dir = tmp_path / ".mempalace" cfg_dir.mkdir() (cfg_dir / "config.json").write_text(json.dumps({"hooks": {"verbatim_mode": True}}))