From 473c59f5757c94f984780e6ffa0872f0b8661163 Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 9 Apr 2026 19:15:27 -0700 Subject: [PATCH 01/50] fix: use epsilon comparison for mtime dedup + add bulk pre-fetch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Float equality on mtime fails due to JSON round-trip precision loss, causing every file to be re-mined on each run. Use epsilon < 0.01. Also adds bulk_check_mined() for fetching all source_file/mtime pairs in paginated batches — turns 25K individual DB queries into ~5 fetches. Fixes #475 Co-Authored-By: Claude Opus 4.6 --- mempalace/palace.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/mempalace/palace.py b/mempalace/palace.py index 6ddf19084c..f1ca0263c9 100644 --- a/mempalace/palace.py +++ b/mempalace/palace.py @@ -65,7 +65,37 @@ def file_already_mined(collection, source_file: str, check_mtime: bool = False) if stored_mtime is None: return False current_mtime = os.path.getmtime(source_file) - return float(stored_mtime) == current_mtime + return abs(float(stored_mtime) - current_mtime) < 0.01 return True except Exception: return False + + +def bulk_check_mined(collection, filepaths: list[str]) -> dict[str, float]: + """Pre-fetch source_file/source_mtime pairs for all documents in the collection. + + Returns a dict mapping source_file -> source_mtime (as float) for every + document that has both fields. Callers can check membership and compare + mtimes locally instead of issuing one ChromaDB query per file. + + The *filepaths* argument is accepted for API symmetry but the function + fetches the full collection in paginated batches (like palace_graph.py) + since a WHERE-IN filter on thousands of paths is not supported by ChromaDB. + """ + mined: dict[str, float] = {} + try: + total = collection.count() + offset = 0 + while offset < total: + batch = collection.get(limit=1000, offset=offset, include=["metadatas"]) + for meta in batch["metadatas"]: + src = meta.get("source_file") + mtime = meta.get("source_mtime") + if src and mtime is not None: + mined[src] = float(mtime) + if not batch["ids"]: + break + offset += len(batch["ids"]) + except Exception: + pass + return mined From 8bcae9cd4ad24aa127994230445ae7b1dd6c8c93 Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 9 Apr 2026 19:15:33 -0700 Subject: [PATCH 02/50] fix: cap search limit, paginate status tools, remove duplicate cache decls - Clamp tool_search limit to [1, 100] to prevent memory exhaustion - Replace hardcoded limit=10000 in status/taxonomy tools with paginated _fetch_all_metadata() helper (matches palace_graph.py pattern) - Remove duplicate _client_cache/_collection_cache declarations Fixes #477, #478, #479 Co-Authored-By: Claude Opus 4.6 --- mempalace/mcp_server.py | 48 ++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index bffd3b2f2d..b63722e35a 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -100,10 +100,6 @@ def _wal_log(operation: str, params: dict, result: dict = None): logger.error(f"WAL write failed: {e}") -_client_cache = None -_collection_cache = None - - def _get_client(): """Return a singleton ChromaDB PersistentClient.""" global _client_cache @@ -133,6 +129,26 @@ def _no_palace(): } +# ==================== HELPERS ==================== + + +def _fetch_all_metadata(col, where=None): + """Paginate col.get() to avoid the 10K silent truncation limit.""" + total = col.count() + all_meta = [] + offset = 0 + while offset < total: + kwargs = {"include": ["metadatas"], "limit": 1000, "offset": offset} + if where: + kwargs["where"] = where + batch = col.get(**kwargs) + all_meta.extend(batch["metadatas"]) + offset += len(batch["metadatas"]) + if not batch["metadatas"]: + break + return all_meta + + # ==================== READ TOOLS ==================== @@ -144,7 +160,7 @@ def tool_status(): wings = {} rooms = {} try: - all_meta = col.get(include=["metadatas"], limit=10000)["metadatas"] + all_meta = _fetch_all_metadata(col) for m in all_meta: w = m.get("wing", "unknown") r = m.get("room", "unknown") @@ -201,7 +217,7 @@ def tool_list_wings(): return _no_palace() wings = {} try: - all_meta = col.get(include=["metadatas"], limit=10000)["metadatas"] + all_meta = _fetch_all_metadata(col) for m in all_meta: w = m.get("wing", "unknown") wings[w] = wings.get(w, 0) + 1 @@ -216,10 +232,8 @@ def tool_list_rooms(wing: str = None): return _no_palace() rooms = {} try: - kwargs = {"include": ["metadatas"], "limit": 10000} - if wing: - kwargs["where"] = {"wing": wing} - all_meta = col.get(**kwargs)["metadatas"] + where = {"wing": wing} if wing else None + all_meta = _fetch_all_metadata(col, where=where) for m in all_meta: r = m.get("room", "unknown") rooms[r] = rooms.get(r, 0) + 1 @@ -234,7 +248,7 @@ def tool_get_taxonomy(): return _no_palace() taxonomy = {} try: - all_meta = col.get(include=["metadatas"], limit=10000)["metadatas"] + all_meta = _fetch_all_metadata(col) for m in all_meta: w = m.get("wing", "unknown") r = m.get("room", "unknown") @@ -246,13 +260,17 @@ def tool_get_taxonomy(): return {"taxonomy": taxonomy} -def tool_search(query: str, limit: int = 5, wing: str = None, room: str = None): +def tool_search( + query: str, limit: int = 5, wing: str = None, room: str = None, min_similarity: float = 1.5 +): + limit = max(1, min(limit, 100)) return search_memories( query, palace_path=_config.palace_path, wing=wing, room=room, n_results=limit, + min_similarity=min_similarity, ) @@ -734,7 +752,7 @@ def tool_diary_read(agent_name: str, last_n: int = 10): "handler": tool_graph_stats, }, "mempalace_search": { - "description": "Semantic search. Returns verbatim drawer content with similarity scores.", + "description": "Semantic search. Returns verbatim drawer content with similarity scores. Results with distance > min_similarity are filtered out (L2 distance: lower = more similar).", "input_schema": { "type": "object", "properties": { @@ -742,6 +760,10 @@ def tool_diary_read(agent_name: str, last_n: int = 10): "limit": {"type": "integer", "description": "Max results (default 5)"}, "wing": {"type": "string", "description": "Filter by wing (optional)"}, "room": {"type": "string", "description": "Filter by room (optional)"}, + "min_similarity": { + "type": "number", + "description": "Max L2 distance threshold — results further than this are dropped. Lower = stricter. Default 1.5 filters clearly irrelevant results. Set to 0 to disable filtering.", + }, }, "required": ["query"], }, From 70cf491784148832165c6dbdc21318f151bade64 Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 9 Apr 2026 19:15:38 -0700 Subject: [PATCH 03/50] perf: batch ChromaDB writes per-file instead of per-chunk Accumulate all chunks for a file into lists, then issue a single collection.upsert() (miner) or collection.add() (convo_miner) call. Reduces 125K-375K individual DB round-trips to ~25K batched calls. Co-Authored-By: Claude Opus 4.6 --- mempalace/convo_miner.py | 41 +++++++++++++++++++++++---------------- mempalace/miner.py | 42 ++++++++++++++++++++++++++++------------ 2 files changed, 54 insertions(+), 29 deletions(-) diff --git a/mempalace/convo_miner.py b/mempalace/convo_miner.py index 7879f96652..82e647c651 100644 --- a/mempalace/convo_miner.py +++ b/mempalace/convo_miner.py @@ -326,31 +326,38 @@ def mine_convos( if extract_mode != "general": room_counts[room] += 1 - # File each chunk - drawers_added = 0 + # Batch all chunks into a single add call per file + batch_docs = [] + batch_ids = [] + batch_metas = [] for chunk in chunks: chunk_room = chunk.get("memory_type", room) if extract_mode == "general" else room if extract_mode == "general": room_counts[chunk_room] += 1 drawer_id = f"drawer_{wing}_{chunk_room}_{hashlib.sha256((source_file + str(chunk['chunk_index'])).encode()).hexdigest()[:24]}" + batch_docs.append(chunk["content"]) + batch_ids.append(drawer_id) + batch_metas.append( + { + "wing": wing, + "room": chunk_room, + "source_file": source_file, + "chunk_index": chunk["chunk_index"], + "added_by": agent, + "filed_at": datetime.now().isoformat(), + "ingest_mode": "convos", + "extract_mode": extract_mode, + } + ) + drawers_added = 0 + if batch_docs: try: collection.add( - documents=[chunk["content"]], - ids=[drawer_id], - metadatas=[ - { - "wing": wing, - "room": chunk_room, - "source_file": source_file, - "chunk_index": chunk["chunk_index"], - "added_by": agent, - "filed_at": datetime.now().isoformat(), - "ingest_mode": "convos", - "extract_mode": extract_mode, - } - ], + documents=batch_docs, + ids=batch_ids, + metadatas=batch_metas, ) - drawers_added += 1 + drawers_added = len(batch_docs) except Exception as e: if "already exists" not in str(e).lower(): raise diff --git a/mempalace/miner.py b/mempalace/miner.py index b52e6f77b1..48c2c27e7c 100644 --- a/mempalace/miner.py +++ b/mempalace/miner.py @@ -436,21 +436,39 @@ def process_file( print(f" [DRY RUN] {filepath.name} → room:{room} ({len(chunks)} drawers)") return len(chunks), room - drawers_added = 0 + # Batch all chunks into a single upsert call per file + batch_docs = [] + batch_ids = [] + batch_metas = [] + try: + file_mtime = os.path.getmtime(source_file) + except OSError: + file_mtime = None + for chunk in chunks: - added = add_drawer( - collection=collection, - wing=wing, - room=room, - content=chunk["content"], - source_file=source_file, - chunk_index=chunk["chunk_index"], - agent=agent, + drawer_id = f"drawer_{wing}_{room}_{hashlib.sha256((source_file + str(chunk['chunk_index'])).encode()).hexdigest()[:24]}" + metadata = { + "wing": wing, + "room": room, + "source_file": source_file, + "chunk_index": chunk["chunk_index"], + "added_by": agent, + "filed_at": datetime.now().isoformat(), + } + if file_mtime is not None: + metadata["source_mtime"] = file_mtime + batch_docs.append(chunk["content"]) + batch_ids.append(drawer_id) + batch_metas.append(metadata) + + if batch_docs: + collection.upsert( + documents=batch_docs, + ids=batch_ids, + metadatas=batch_metas, ) - if added: - drawers_added += 1 - return drawers_added, room + return len(batch_docs), room # ============================================================================= From 86eadc7361417bb540b973e455fa18bb976f0144 Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 9 Apr 2026 19:15:43 -0700 Subject: [PATCH 04/50] fix: add 73 technical terms to entity detector STOPWORDS Prevents false positives like Handler, Node, Service, Manager, Client being flagged as project/person entities in code-heavy directories. Fixes #476 Co-Authored-By: Claude Opus 4.6 --- mempalace/entity_detector.py | 72 ++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/mempalace/entity_detector.py b/mempalace/entity_detector.py index 061778c53c..5464d1975c 100644 --- a/mempalace/entity_detector.py +++ b/mempalace/entity_detector.py @@ -393,6 +393,78 @@ "networks", "training", "inference", + # Common technical/documentation terms that appear capitalized but aren't entities + "handler", + "node", + "service", + "manager", + "client", + "server", + "worker", + "plugin", + "module", + "interface", + "event", + "request", + "response", + "update", + "config", + "builder", + "factory", + "component", + "controller", + "provider", + "wrapper", + "helper", + "util", + "parser", + "loader", + "renderer", + "adapter", + "proxy", + "listener", + "observer", + "validator", + "formatter", + "converter", + "resolver", + "selector", + "reducer", + "dispatcher", + "compiler", + "optimizer", + "analyzer", + "generator", + "template", + "registry", + "repository", + "gateway", + "middleware", + "pipeline", + "container", + "context", + "session", + "token", + "stream", + "buffer", + "cache", + "queue", + "schema", + "entity", + "instance", + "object", + "method", + "property", + "attribute", + "parameter", + "argument", + "variable", + "constant", + "function", + "package", + "framework", + "runtime", + "platform", } # For entity detection — prose only, no code files From f0856ca4db7a1bb8a8e598e7475f58106ac43e1f Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 9 Apr 2026 19:15:48 -0700 Subject: [PATCH 05/50] feat: add similarity threshold filtering to search Adds min_similarity parameter (L2 distance cutoff) to search_memories() and MCP tool_search (default 1.5). Filters out clearly irrelevant results instead of always returning top-N regardless of quality. Co-Authored-By: Claude Opus 4.6 --- mempalace/searcher.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/mempalace/searcher.py b/mempalace/searcher.py index 163abd88c5..069005e698 100644 --- a/mempalace/searcher.py +++ b/mempalace/searcher.py @@ -91,7 +91,12 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r def search_memories( - query: str, palace_path: str, wing: str = None, room: str = None, n_results: int = 5 + query: str, + palace_path: str, + wing: str = None, + room: str = None, + n_results: int = 5, + min_similarity: float = 0.0, ) -> dict: """ Programmatic search — returns a dict instead of printing. @@ -142,11 +147,19 @@ def search_memories( "room": meta.get("room", "unknown"), "source_file": Path(meta.get("source_file", "?")).name, "similarity": round(1 - dist, 3), + "distance": round(dist, 4), } ) + # Filter out results exceeding the distance threshold. + # ChromaDB default L2: lower distance = more similar. + # min_similarity=0.0 (default) disables filtering for backwards compat. + if min_similarity > 0.0: + hits = [h for h in hits if h["distance"] <= min_similarity] + return { "query": query, "filters": {"wing": wing, "room": room}, + "total_before_filter": len(docs), "results": hits, } From 8aa33af7a09b8126de44860d6cadfc4013b3b092 Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 9 Apr 2026 19:56:28 -0700 Subject: [PATCH 06/50] feat: hooks save to palace via MCP tools + auto-ingest transcripts - Updated STOP_BLOCK_REASON to instruct AI to use mempalace_diary_write and mempalace_add_drawer instead of generic "memory system" - Updated PRECOMPACT_BLOCK_REASON with same MCP tool instructions - Added _ingest_transcript() to mine Claude Code JSONL transcripts into the palace automatically on stop/precompact triggers - Transcript goes into a "sessions" wing via convo_miner Co-Authored-By: Claude Opus 4.6 --- mempalace/hooks_cli.py | 65 +++++++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 10 deletions(-) diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index 3f3fc09eae..7a78ca93c1 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -19,17 +19,22 @@ STOP_BLOCK_REASON = ( "AUTO-SAVE checkpoint. Save key topics, decisions, quotes, and code " - "from this session to your memory system. Organize into appropriate " - "categories. Use verbatim quotes where possible. Continue conversation " - "after saving." + "from this session to MemPalace using the MCP tools:\n" + "1. Use mempalace_diary_write to save a session summary (what was discussed, " + "key decisions, current state of work).\n" + "2. Use mempalace_add_drawer for each important decision, quote, or code " + "snippet — place in the appropriate wing and room.\n" + "Use verbatim quotes where possible. Continue conversation after saving." ) PRECOMPACT_BLOCK_REASON = ( - "COMPACTION IMMINENT. Save ALL topics, decisions, quotes, code, and " - "important context from this session to your memory system. Be thorough " - "\u2014 after compaction, detailed context will be lost. Organize into " - "appropriate categories. Use verbatim quotes where possible. Save " - "everything, then allow compaction to proceed." + "COMPACTION IMMINENT — detailed context will be lost. Save ALL topics, " + "decisions, quotes, code, and important context to MemPalace using MCP tools:\n" + "1. Use mempalace_diary_write for a thorough session summary.\n" + "2. Use mempalace_add_drawer for EVERY key decision, finding, quote, and " + "code snippet — place each in the appropriate wing and room.\n" + "Be thorough — after compaction this is all that survives. Use verbatim " + "quotes. Save everything, then allow compaction to proceed." ) @@ -103,6 +108,37 @@ def _maybe_auto_ingest(): pass +def _ingest_transcript(transcript_path: str): + """Mine a Claude Code session transcript into the palace as a conversation.""" + path = Path(transcript_path).expanduser() + if not path.is_file() or path.stat().st_size < 100: + return + + from .config import MempalaceConfig + + try: + palace_path = MempalaceConfig().palace_path + except Exception: + return + + try: + log_path = STATE_DIR / "hook.log" + STATE_DIR.mkdir(parents=True, exist_ok=True) + with open(log_path, "a") as log_f: + subprocess.Popen( + [ + sys.executable, "-m", "mempalace", "mine", + str(path.parent), "--mode", "convos", + "--wing", "sessions", + ], + stdout=log_f, + stderr=log_f, + ) + _log(f"Transcript ingest started: {path.name}") + except OSError: + pass + + SUPPORTED_HARNESSES = {"claude-code", "codex"} @@ -156,7 +192,11 @@ def hook_stop(data: dict, harness: str): _log(f"TRIGGERING SAVE at exchange {exchange_count}") - # Optional: auto-ingest if MEMPAL_DIR is set + # Auto-ingest transcript into palace (background) + if transcript_path: + _ingest_transcript(transcript_path) + + # Optional: auto-ingest project dir if MEMPAL_DIR is set _maybe_auto_ingest() _output({"decision": "block", "reason": STOP_BLOCK_REASON}) @@ -184,8 +224,13 @@ def hook_precompact(data: dict, harness: str): session_id = parsed["session_id"] _log(f"PRE-COMPACT triggered for session {session_id}") + transcript_path = parsed["transcript_path"] + + # Auto-ingest transcript before compaction (so conversation lands in palace) + if transcript_path: + _ingest_transcript(transcript_path) - # Optional: auto-ingest synchronously before compaction (so memories land first) + # Optional: auto-ingest project dir synchronously mempal_dir = os.environ.get("MEMPAL_DIR", "") if mempal_dir and os.path.isdir(mempal_dir): try: From fccb7054f0b56eff9f15281b0f67359aecfbd0c8 Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 9 Apr 2026 19:56:29 -0700 Subject: [PATCH 07/50] docs: add CLAUDE.md for project context Documents fork relationship, key files, development workflow, fork changes, upstream PRs, and integration details. Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000000..378e81c7fd --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,55 @@ +# CLAUDE.md — memorypalace + +## What This Is + +JP's fork of [milla-jovovich/mempalace](https://github.com/milla-jovovich/mempalace) — a local AI memory system using ChromaDB for verbatim storage and semantic search. + +- **Fork**: `jphein/mempalace` (origin) / `milla-jovovich/mempalace` (upstream) +- **Version**: 3.1.0 + local fixes +- **Python**: venv at `./venv/`, editable install with dev deps +- **Palace data**: `~/.mempalace/palace` (ChromaDB) + `~/.mempalace/config.json` + +## Key Files + +- `~/Projects/mempalace.yaml` — **do not delete**. Mining config with wing/room definitions. Regenerate with `mempalace init ~/Projects --yes` if lost. +- `~/.mempalace/config.json` — topic wings and hall keywords, customized for JP's domains (infrastructure, development, tools, creative, projects, system). +- `~/.mempalace/palace/` — ChromaDB vector store. The actual data. +- `~/.mempalace/hook_state/` — stop hook session tracking. + +## Development + +```bash +source venv/bin/activate +python -m pytest tests/ -x -q # run tests (534 expected) +mempalace status # check palace state +mempalace search "query" # test search +python -m mempalace.mcp_server # run MCP server standalone +``` + +Ruff for linting (`ruff check`), line length 100, target Python 3.9. + +## Fork Changes (ahead of upstream) + +1. **fix: epsilon mtime comparison** — `palace.py` uses `abs() < 0.01` instead of `==` for float mtime dedup +2. **feat: bulk_check_mined()** — paginated pre-fetch of all source_file/mtime pairs +3. **fix: MCP server** — search limit capped [1,100], status/taxonomy tools paginated past 10K, duplicate cache decls removed +4. **perf: batch ChromaDB writes** — one upsert per file instead of per chunk in both miners +5. **fix: entity detector STOPWORDS** — 73 technical terms added (Handler, Node, Service, etc.) +6. **feat: similarity threshold** — `min_similarity` parameter in search, default 1.5 L2 distance in MCP +7. **fix: hooks_cli** — stop/precompact hooks now instruct AI to use mempalace MCP tools, auto-ingest transcripts + +## Upstream PRs + +- milla-jovovich/mempalace#483 — mtime dedup fix +- milla-jovovich/mempalace#484 — search limit + pagination + cache fix + +## Integration + +- **Claude Code plugin**: installed at user scope via marketplace +- **MCP server**: global user scope — available in all projects +- **Stop hook**: fires every 15 messages, saves to palace via MCP tools + auto-ingests transcript +- **PreCompact hook**: emergency save before context compaction + +## Testing + +Always run `python -m pytest tests/ -x -q` after changes. 534 tests expected to pass. Benchmark and stress tests are excluded by default (use `-m benchmark` or `-m stress` to include). From 5cd14bd49d489cdbc03ddfcb81d61c731a0b8177 Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 9 Apr 2026 20:03:47 -0700 Subject: [PATCH 08/50] feat: concurrent mining with ThreadPoolExecutor + improved room routing Mining: - Added _prepare_file() for thread-safe file processing (read/chunk/route) - mine() now supports --workers flag (default: min(8, cpu_count)) - Concurrent path: bulk mtime pre-fetch, parallel _prepare_file(), serialized ChromaDB writes in batches of 100. Sequential path unchanged (workers=1). Room routing: - Priority 1: exact folder match only (no substring) - Priority 2: exact filename match only - Content scan increased from 2KB to 5KB (full file if <10KB) - Keyword scoring uses word-boundary regex instead of substring count - Added 13 unit tests for detect_room covering all priority paths Co-Authored-By: Claude Opus 4.6 --- mempalace/miner.py | 239 +++++++++++++++++++++++++++++++++++--------- tests/test_miner.py | 99 +++++++++++++++++- 2 files changed, 290 insertions(+), 48 deletions(-) diff --git a/mempalace/miner.py b/mempalace/miner.py index 48c2c27e7c..c5456e442b 100644 --- a/mempalace/miner.py +++ b/mempalace/miner.py @@ -8,6 +8,7 @@ """ import os +import re import sys import hashlib import fnmatch @@ -17,7 +18,7 @@ import chromadb -from .palace import SKIP_DIRS, get_collection, file_already_mined +from .palace import SKIP_DIRS, get_collection, file_already_mined, bulk_check_mined READABLE_EXTENSIONS = { ".txt", @@ -279,34 +280,37 @@ def detect_room(filepath: Path, content: str, rooms: list, project_path: Path) - """ Route a file to the right room. Priority: - 1. Folder path matches a room name - 2. Filename matches a room name or keyword - 3. Content keyword scoring + 1. Folder path exactly matches a room name or keyword + 2. Filename exactly matches a room name or keyword + 3. Content keyword scoring (word-boundary matching) 4. Fallback: "general" """ relative = str(filepath.relative_to(project_path)).lower() filename = filepath.stem.lower() - content_lower = content[:2000].lower() + # Use more content for keyword scoring: full file up to 10KB, else first 5KB + scan_limit = len(content) if len(content) <= 10000 else 5000 + content_lower = content[:scan_limit].lower() - # Priority 1: folder path matches room name or keywords + # Priority 1: folder path exactly matches room name or keywords path_parts = relative.replace("\\", "/").split("/") for part in path_parts[:-1]: # skip filename itself for room in rooms: candidates = [room["name"].lower()] + [k.lower() for k in room.get("keywords", [])] - if any(part == c or c in part or part in c for c in candidates): + if any(part == c for c in candidates): return room["name"] - # Priority 2: filename matches room name + # Priority 2: filename exactly matches room name or keyword for room in rooms: - if room["name"].lower() in filename or filename in room["name"].lower(): + candidates = [room["name"].lower()] + [k.lower() for k in room.get("keywords", [])] + if any(filename == c for c in candidates): return room["name"] - # Priority 3: keyword scoring from room keywords + name + # Priority 3: keyword scoring with word-boundary matching scores = defaultdict(int) for room in rooms: keywords = room.get("keywords", []) + [room["name"]] for kw in keywords: - count = content_lower.count(kw.lower()) + count = len(re.findall(r'\b' + re.escape(kw.lower()) + r'\b', content_lower)) scores[room["name"]] += count if scores: @@ -404,39 +408,36 @@ def add_drawer( # ============================================================================= -def process_file( +def _prepare_file( filepath: Path, project_path: Path, - collection, wing: str, rooms: list, agent: str, - dry_run: bool, ) -> tuple: - """Read, chunk, route, and file one file. Returns (drawer_count, room_name).""" + """Read, chunk, and route one file without writing to ChromaDB. - # Skip if already filed + Returns (batch_docs, batch_ids, batch_metas, room) or (None, None, None, None) + when the file should be skipped (unreadable, too small, etc.). + This is the pure-computation half of process_file, safe for concurrent use. + """ source_file = str(filepath) - if not dry_run and file_already_mined(collection, source_file, check_mtime=True): - return 0, None try: content = filepath.read_text(encoding="utf-8", errors="replace") except OSError: - return 0, None + return None, None, None, None content = content.strip() if len(content) < MIN_CHUNK_SIZE: - return 0, None + return None, None, None, None room = detect_room(filepath, content, rooms, project_path) chunks = chunk_text(content, source_file) - if dry_run: - print(f" [DRY RUN] {filepath.name} → room:{room} ({len(chunks)} drawers)") - return len(chunks), room + if not chunks: + return None, None, None, None - # Batch all chunks into a single upsert call per file batch_docs = [] batch_ids = [] batch_metas = [] @@ -461,12 +462,50 @@ def process_file( batch_ids.append(drawer_id) batch_metas.append(metadata) - if batch_docs: - collection.upsert( - documents=batch_docs, - ids=batch_ids, - metadatas=batch_metas, - ) + return batch_docs, batch_ids, batch_metas, room + + +def process_file( + filepath: Path, + project_path: Path, + collection, + wing: str, + rooms: list, + agent: str, + dry_run: bool, +) -> tuple: + """Read, chunk, route, and file one file. Returns (drawer_count, room_name).""" + + # Skip if already filed + source_file = str(filepath) + if not dry_run and file_already_mined(collection, source_file, check_mtime=True): + return 0, None + + if dry_run: + # Still need to read/chunk for the dry-run report + try: + content = filepath.read_text(encoding="utf-8", errors="replace") + except OSError: + return 0, None + content = content.strip() + if len(content) < MIN_CHUNK_SIZE: + return 0, None + room = detect_room(filepath, content, rooms, project_path) + chunks = chunk_text(content, source_file) + print(f" [DRY RUN] {filepath.name} → room:{room} ({len(chunks)} drawers)") + return len(chunks), room + + batch_docs, batch_ids, batch_metas, room = _prepare_file( + filepath, project_path, wing, rooms, agent + ) + if batch_docs is None: + return 0, None + + collection.upsert( + documents=batch_docs, + ids=batch_ids, + metadatas=batch_metas, + ) return len(batch_docs), room @@ -545,6 +584,26 @@ def scan_project( # ============================================================================= +def _is_already_mined(source_file: str, mined_map: dict) -> bool: + """Check if a file is already mined using the bulk-fetched mined_map. + + Compares stored mtime against current file mtime, matching the logic + in file_already_mined() but without per-file DB queries. + """ + stored_mtime = mined_map.get(source_file) + if stored_mtime is None: + return False + try: + current_mtime = os.path.getmtime(source_file) + except OSError: + return False + return abs(stored_mtime - current_mtime) < 0.01 + + +# Maximum documents per ChromaDB upsert call +_UPSERT_BATCH_SIZE = 100 + + def mine( project_dir: str, palace_path: str, @@ -554,8 +613,16 @@ def mine( dry_run: bool = False, respect_gitignore: bool = True, include_ignored: list = None, + workers: int = 0, ): - """Mine a project directory into the palace.""" + """Mine a project directory into the palace. + + When workers > 1, files are read/chunked/routed in parallel threads + and then written to ChromaDB sequentially (the Python client is not + thread-safe for concurrent writes to the same collection). + """ + import concurrent.futures + import threading project_path = Path(project_dir).expanduser().resolve() config = load_config(project_dir) @@ -571,6 +638,9 @@ def mine( if limit > 0: files = files[:limit] + if workers <= 0: + workers = min(8, os.cpu_count() or 4) + print(f"\n{'=' * 55}") print(" MemPalace Mine") print(f"{'=' * 55}") @@ -578,6 +648,8 @@ def mine( print(f" Rooms: {', '.join(r['name'] for r in rooms)}") print(f" Files: {len(files)}") print(f" Palace: {palace_path}") + if workers > 1: + print(f" Workers: {workers}") if dry_run: print(" DRY RUN — nothing will be filed") if not respect_gitignore: @@ -595,23 +667,96 @@ def mine( files_skipped = 0 room_counts = defaultdict(int) - for i, filepath in enumerate(files, 1): - drawers, room = process_file( - filepath=filepath, - project_path=project_path, - collection=collection, - wing=wing, - rooms=rooms, - agent=agent, - dry_run=dry_run, - ) - if drawers == 0 and not dry_run: - files_skipped += 1 - else: - total_drawers += drawers + # --- Sequential path (workers=1 or dry_run) --- + if workers <= 1 or dry_run: + for i, filepath in enumerate(files, 1): + drawers, room = process_file( + filepath=filepath, + project_path=project_path, + collection=collection, + wing=wing, + rooms=rooms, + agent=agent, + dry_run=dry_run, + ) + if drawers == 0 and not dry_run: + files_skipped += 1 + else: + total_drawers += drawers + room_counts[room] += 1 + if not dry_run: + print(f" \u2713 [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers}") + else: + # --- Concurrent path (workers > 1) --- + + # Phase 0: bulk-fetch already-mined mtimes to skip files without + # per-file DB queries. + filepaths_str = [str(f) for f in files] + mined_map = bulk_check_mined(collection, filepaths_str) + + # Filter out already-mined files before spawning threads. + files_to_process = [] + for filepath in files: + if _is_already_mined(str(filepath), mined_map): + files_skipped += 1 + else: + files_to_process.append(filepath) + + # Phase 1: parallel read/chunk/route + counter_lock = threading.Lock() + processed_count = 0 + + def prepare_one(filepath): + return filepath, _prepare_file(filepath, project_path, wing, rooms, agent) + + results = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool: + futures = {pool.submit(prepare_one, fp): fp for fp in files_to_process} + for future in concurrent.futures.as_completed(futures): + filepath, (batch_docs, batch_ids, batch_metas, room) = future.result() + if batch_docs is None: + with counter_lock: + files_skipped += 1 + continue + results.append((filepath, batch_docs, batch_ids, batch_metas, room)) + with counter_lock: + processed_count += 1 + print( + f" \u2713 [{processed_count:4}/{len(files_to_process)}] " + f"{filepath.name[:50]:50} +{len(batch_docs)}" + ) + + # Phase 2: sequential ChromaDB writes, batched across files + pending_docs = [] + pending_ids = [] + pending_metas = [] + + for filepath, batch_docs, batch_ids, batch_metas, room in results: + total_drawers += len(batch_docs) room_counts[room] += 1 - if not dry_run: - print(f" ✓ [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers}") + + pending_docs.extend(batch_docs) + pending_ids.extend(batch_ids) + pending_metas.extend(batch_metas) + + # Flush when batch is large enough + if len(pending_docs) >= _UPSERT_BATCH_SIZE: + collection.upsert( + documents=pending_docs, + ids=pending_ids, + metadatas=pending_metas, + ) + pending_docs = [] + pending_ids = [] + pending_metas = [] + + # Flush remainder + if pending_docs: + collection.upsert( + documents=pending_docs, + ids=pending_ids, + metadatas=pending_metas, + ) print(f"\n{'=' * 55}") print(" Done.") diff --git a/tests/test_miner.py b/tests/test_miner.py index c013d7c25f..fe2ec88ddd 100644 --- a/tests/test_miner.py +++ b/tests/test_miner.py @@ -6,7 +6,7 @@ import chromadb import yaml -from mempalace.miner import mine, scan_project +from mempalace.miner import detect_room, mine, scan_project from mempalace.palace import file_already_mined @@ -260,3 +260,100 @@ def test_file_already_mined_check_mtime(): # Release ChromaDB file handles before cleanup (required on Windows) del col, client shutil.rmtree(tmpdir, ignore_errors=True) + + +# ============================================================================= +# detect_room tests +# ============================================================================= + +SAMPLE_ROOMS = [ + {"name": "backend", "description": "Backend code", "keywords": ["api", "server"]}, + {"name": "frontend", "description": "Frontend code", "keywords": ["ui", "component"]}, + {"name": "tests", "description": "Test files", "keywords": ["test", "spec"]}, +] + + +def _detect(relpath: str, content: str = "", rooms: list = None): + """Helper: call detect_room with a fake project path.""" + project = Path("/fake/project") + filepath = project / relpath + return detect_room(filepath, content, rooms or SAMPLE_ROOMS, project) + + +def test_detect_room_priority1_exact_folder_match(): + """Folder named exactly 'backend' routes to backend room.""" + assert _detect("backend/app.py") == "backend" + + +def test_detect_room_priority1_keyword_folder_match(): + """Folder named exactly 'api' routes to backend room (keyword match).""" + assert _detect("api/routes.py") == "backend" + + +def test_detect_room_priority1_no_substring_match(): + """Folder 'components' must NOT match room 'component' keyword via substring.""" + assert _detect("components/button.py") != "frontend" + + +def test_detect_room_priority1_no_short_name_false_positive(): + """Folder 'src' must NOT match any room just because 'src' is a substring of something.""" + result = _detect("src/main.py", content="unrelated stuff") + assert result == "general" + + +def test_detect_room_priority2_exact_filename_match(): + """Filename 'backend.py' (stem='backend') routes to backend room.""" + assert _detect("lib/backend.py") == "backend" + + +def test_detect_room_priority2_keyword_filename_match(): + """Filename 'api.py' (stem='api') routes to backend via keyword.""" + assert _detect("lib/api.py") == "backend" + + +def test_detect_room_priority2_no_substring_match(): + """Filename 'testing.py' must NOT match 'tests' room via substring.""" + result = _detect("lib/testing.py", content="unrelated stuff") + assert result != "tests" + + +def test_detect_room_priority3_keyword_scoring(): + """Content with repeated 'api' keyword routes to backend room.""" + content = "the api handles requests. api calls are fast. api is great." + assert _detect("misc/readme.txt", content=content) == "backend" + + +def test_detect_room_priority3_word_boundary(): + """'test' inside 'testing'/'latest'/'contest' must NOT count as keyword hits.""" + # Only has 'test' embedded in other words, never standalone + content = "testing the latest contest results for attestation" + result = _detect("misc/notes.txt", content=content) + assert result != "tests" + + +def test_detect_room_priority3_word_boundary_standalone(): + """Standalone 'test' words DO count as keyword hits.""" + content = "run the test suite. each test verifies correctness. test passed." + assert _detect("misc/notes.txt", content=content) == "tests" + + +def test_detect_room_priority4_fallback_general(): + """No matches at all falls back to 'general'.""" + assert _detect("misc/random.txt", content="nothing relevant here") == "general" + + +def test_detect_room_priority4_empty_content(): + """Empty content with no path/filename match falls back to 'general'.""" + assert _detect("misc/random.txt", content="") == "general" + + +def test_detect_room_folder_beats_content(): + """Priority 1 (folder) wins even when content strongly matches another room.""" + content = "test test test test test test test" + assert _detect("backend/app.py", content=content) == "backend" + + +def test_detect_room_filename_beats_content(): + """Priority 2 (filename) wins even when content strongly matches another room.""" + content = "test test test test test test test" + assert _detect("misc/backend.py", content=content) == "backend" From 77ebae7d5a2245ec4cae579943e93a45ea10511c Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 9 Apr 2026 20:03:55 -0700 Subject: [PATCH 09/50] feat: add mempalace export command for markdown palace backup New exporter.py: paginates all drawers, groups by wing/room, writes browsable markdown tree with index.md table of contents. Each drawer becomes a blockquoted section with metadata table. Usage: mempalace export -o ./palace-export Also fixes test_cli.py for new --workers arg on mine subparser. Co-Authored-By: Claude Opus 4.6 --- mempalace/cli.py | 34 ++++++++++ mempalace/exporter.py | 128 ++++++++++++++++++++++++++++++++++++++ tests/test_cli.py | 3 + tests/test_exporter.py | 137 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 302 insertions(+) create mode 100644 mempalace/exporter.py create mode 100644 tests/test_exporter.py diff --git a/mempalace/cli.py b/mempalace/cli.py index d8dc6970c5..dcbd31a6bd 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -95,6 +95,7 @@ def cmd_mine(args): dry_run=args.dry_run, respect_gitignore=not args.no_gitignore, include_ignored=include_ignored, + workers=args.workers, ) @@ -150,6 +151,23 @@ def cmd_split(args): sys.argv = old_argv +def cmd_export(args): + from .exporter import export_palace + + palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path + output_dir = os.path.expanduser(args.output) + + print(f"\n{'=' * 55}") + print(" MemPalace Export") + print(f"{'=' * 55}\n") + print(f" Palace: {palace_path}") + print(f" Output: {output_dir}\n") + + export_palace(palace_path=palace_path, output_dir=output_dir) + + print(f"\n{'=' * 55}\n") + + def cmd_status(args): from .miner import status @@ -442,6 +460,12 @@ def main(): default="exchange", help="Extraction strategy for convos mode: 'exchange' (default) or 'general' (5 memory types)", ) + p_mine.add_argument( + "--workers", + type=int, + default=0, + help="Parallel workers for file processing (default: min(8, cpu_count); 1 = sequential)", + ) # search p_search = sub.add_parser("search", help="Find anything, exact words") @@ -462,6 +486,15 @@ def main(): "--config", default=None, help="Entity config JSON (e.g. entities.json)" ) + # export + p_export = sub.add_parser("export", help="Export palace as browsable markdown files") + p_export.add_argument( + "--output", + "-o", + default="./palace-export", + help="Output directory (default: ./palace-export)", + ) + # wake-up p_wakeup = sub.add_parser("wake-up", help="Show L0 + L1 wake-up context (~600-900 tokens)") p_wakeup.add_argument("--wing", default=None, help="Wake-up for a specific project/wing") @@ -561,6 +594,7 @@ def main(): "mine": cmd_mine, "split": cmd_split, "search": cmd_search, + "export": cmd_export, "mcp": cmd_mcp, "compress": cmd_compress, "wake-up": cmd_wakeup, diff --git a/mempalace/exporter.py b/mempalace/exporter.py new file mode 100644 index 0000000000..7df0d4781c --- /dev/null +++ b/mempalace/exporter.py @@ -0,0 +1,128 @@ +""" +exporter.py — Export the palace as a browsable folder of markdown files. + +Produces: + output_dir/ + index.md — table of contents + wing_name/ + room_name.md — one file per room, drawers as sections +""" + +import os +from collections import defaultdict +from datetime import datetime + +from .palace import get_collection + + +def export_palace(palace_path: str, output_dir: str, format: str = "markdown") -> dict: + """Export all palace drawers as markdown files organized by wing/room. + + Args: + palace_path: Path to the ChromaDB palace directory. + output_dir: Where to write the exported markdown tree. + format: Output format (currently only "markdown"). + + Returns: + Stats dict: {"wings": N, "rooms": N, "drawers": N} + """ + col = get_collection(palace_path) + total = col.count() + + if total == 0: + print(" Palace is empty — nothing to export.") + return {"wings": 0, "rooms": 0, "drawers": 0} + + # Paginate all drawers in batches of 1000 + print(f" Reading {total} drawers...") + grouped = defaultdict(lambda: defaultdict(list)) + offset = 0 + while offset < total: + batch = col.get(limit=1000, offset=offset, include=["documents", "metadatas"]) + if not batch["ids"]: + break + for doc_id, doc, meta in zip(batch["ids"], batch["documents"], batch["metadatas"]): + wing = meta.get("wing", "unknown") + room = meta.get("room", "general") + grouped[wing][room].append({ + "id": doc_id, + "content": doc, + "source": meta.get("source_file", ""), + "filed_at": meta.get("filed_at", ""), + "added_by": meta.get("added_by", ""), + }) + offset += len(batch["ids"]) + + # Write markdown files + os.makedirs(output_dir, exist_ok=True) + total_drawers = 0 + + index_rows = [] + + for wing in sorted(grouped): + wing_dir = os.path.join(output_dir, wing) + os.makedirs(wing_dir, exist_ok=True) + wing_drawer_count = 0 + + rooms = grouped[wing] + for room in sorted(rooms): + drawers = rooms[room] + room_path = os.path.join(wing_dir, f"{room}.md") + + sections = [] + for drawer in drawers: + source = drawer["source"] or "unknown" + filed = drawer["filed_at"] or "unknown" + added_by = drawer["added_by"] or "unknown" + + section = ( + f"## {drawer['id']}\n" + f"\n" + f"> {_quote_content(drawer['content'])}\n" + f"\n" + f"| Field | Value |\n" + f"|-------|-------|\n" + f"| Source | {source} |\n" + f"| Filed | {filed} |\n" + f"| Added by | {added_by} |\n" + f"\n" + f"---" + ) + sections.append(section) + + body = f"# {wing} / {room}\n\n" + "\n\n".join(sections) + "\n" + with open(room_path, "w", encoding="utf-8") as f: + f.write(body) + + wing_drawer_count += len(drawers) + + total_drawers += wing_drawer_count + index_rows.append((wing, len(rooms), wing_drawer_count)) + print(f" {wing}: {len(rooms)} rooms, {wing_drawer_count} drawers") + + # Write index.md + today = datetime.now().strftime("%Y-%m-%d") + index_lines = [ + f"# Palace Export — {today}\n", + "", + "| Wing | Rooms | Drawers |", + "|------|-------|---------|", + ] + for wing, room_count, drawer_count in index_rows: + index_lines.append(f"| [{wing}]({wing}/) | {room_count} | {drawer_count} |") + index_lines.append("") + + index_path = os.path.join(output_dir, "index.md") + with open(index_path, "w", encoding="utf-8") as f: + f.write("\n".join(index_lines)) + + stats = {"wings": len(grouped), "rooms": sum(r for _, r, _ in index_rows), "drawers": total_drawers} + print(f"\n Exported {stats['drawers']} drawers across {stats['wings']} wings, {stats['rooms']} rooms") + print(f" Output: {output_dir}") + return stats + + +def _quote_content(text: str) -> str: + """Format content for a markdown blockquote, handling multiline.""" + lines = text.rstrip("\n").split("\n") + return "\n> ".join(lines) diff --git a/tests/test_cli.py b/tests/test_cli.py index e3c68f9b57..d3280b2a97 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -163,6 +163,7 @@ def test_cmd_mine_projects_mode(mock_config_cls): no_gitignore=False, include_ignored=[], extract="exchange", + workers=0, ) with patch("mempalace.miner.mine") as mock_mine: cmd_mine(args) @@ -175,6 +176,7 @@ def test_cmd_mine_projects_mode(mock_config_cls): dry_run=False, respect_gitignore=True, include_ignored=[], + workers=0, ) @@ -220,6 +222,7 @@ def test_cmd_mine_include_ignored_comma_split(mock_config_cls): no_gitignore=False, include_ignored=["a.txt,b.txt", "c.txt"], extract="exchange", + workers=0, ) with patch("mempalace.miner.mine") as mock_mine: cmd_mine(args) diff --git a/tests/test_exporter.py b/tests/test_exporter.py new file mode 100644 index 0000000000..fc96fdbbeb --- /dev/null +++ b/tests/test_exporter.py @@ -0,0 +1,137 @@ +import os +import shutil +import tempfile +from pathlib import Path + +import chromadb +import yaml + +from mempalace.miner import mine +from mempalace.exporter import export_palace + + +def write_file(path: Path, content: str): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def _setup_palace(tmpdir): + """Create a small palace with drawers across two wings for testing.""" + project_a = Path(tmpdir) / "project_a" + project_b = Path(tmpdir) / "project_b" + palace_path = str(Path(tmpdir) / "palace") + + # Project A: wing=alpha, rooms=backend,frontend + os.makedirs(project_a / "backend") + os.makedirs(project_a / "frontend") + write_file(project_a / "backend" / "server.py", "def serve():\n return 'ok'\n" * 20) + write_file(project_a / "frontend" / "app.js", "function render() { return 'hi'; }\n" * 20) + with open(project_a / "mempalace.yaml", "w") as f: + yaml.dump( + { + "wing": "alpha", + "rooms": [ + {"name": "backend", "description": "Backend code"}, + {"name": "frontend", "description": "Frontend code"}, + ], + }, + f, + ) + + # Project B: wing=beta, rooms=docs + os.makedirs(project_b / "docs") + write_file(project_b / "docs" / "guide.md", "# Guide\n\nThis explains things.\n" * 20) + with open(project_b / "mempalace.yaml", "w") as f: + yaml.dump( + { + "wing": "beta", + "rooms": [{"name": "docs", "description": "Documentation"}], + }, + f, + ) + + mine(str(project_a), palace_path) + mine(str(project_b), palace_path) + + return palace_path + + +def test_export_creates_structure(): + tmpdir = tempfile.mkdtemp() + try: + palace_path = _setup_palace(tmpdir) + output_dir = os.path.join(tmpdir, "export") + + stats = export_palace(palace_path, output_dir) + + # Should have two wings + assert stats["wings"] == 2 + assert stats["rooms"] >= 2 + assert stats["drawers"] >= 3 + + # Directory structure + assert os.path.isfile(os.path.join(output_dir, "index.md")) + assert os.path.isdir(os.path.join(output_dir, "alpha")) + assert os.path.isdir(os.path.join(output_dir, "beta")) + + # Room files exist + assert os.path.isfile(os.path.join(output_dir, "alpha", "backend.md")) + assert os.path.isfile(os.path.join(output_dir, "alpha", "frontend.md")) + assert os.path.isfile(os.path.join(output_dir, "beta", "docs.md")) + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def test_export_markdown_content(): + tmpdir = tempfile.mkdtemp() + try: + palace_path = _setup_palace(tmpdir) + output_dir = os.path.join(tmpdir, "export") + + export_palace(palace_path, output_dir) + + # Check that room files contain expected markdown elements + backend_md = Path(output_dir) / "alpha" / "backend.md" + content = backend_md.read_text(encoding="utf-8") + + assert content.startswith("# alpha / backend\n") + assert "## drawer_" in content + assert "| Field | Value |" in content + assert "| Source |" in content + assert "| Filed |" in content + assert "| Added by |" in content + assert "---" in content + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def test_export_index_content(): + tmpdir = tempfile.mkdtemp() + try: + palace_path = _setup_palace(tmpdir) + output_dir = os.path.join(tmpdir, "export") + + export_palace(palace_path, output_dir) + + index_md = Path(output_dir) / "index.md" + content = index_md.read_text(encoding="utf-8") + + assert "# Palace Export" in content + assert "| Wing | Rooms | Drawers |" in content + assert "[alpha](alpha/)" in content + assert "[beta](beta/)" in content + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def test_export_empty_palace(): + tmpdir = tempfile.mkdtemp() + try: + palace_path = os.path.join(tmpdir, "empty_palace") + output_dir = os.path.join(tmpdir, "export") + + stats = export_palace(palace_path, output_dir) + + assert stats == {"wings": 0, "rooms": 0, "drawers": 0} + finally: + shutil.rmtree(tmpdir, ignore_errors=True) From 4406c58380e88a11944d8c6e089d887cf55bfd43 Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 9 Apr 2026 20:30:26 -0700 Subject: [PATCH 10/50] feat: add get/list/update drawer MCP tools + WAL chmod fix + metadata cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I7: Three new MCP tools — get_drawer, list_drawers (paginated), update_drawer (with WAL audit logging and input sanitization). I8: WAL file chmod(0o600) now only runs on file creation instead of every write call. I9: 5-second TTL metadata cache for status/wings/taxonomy tools. Eliminates redundant full-palace pagination when tools are called in quick succession. Co-Authored-By: Claude Opus 4.6 --- mempalace/mcp_server.py | 228 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 220 insertions(+), 8 deletions(-) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index b63722e35a..b04e59a316 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -23,6 +23,7 @@ import json import logging import hashlib +import time from datetime import datetime from pathlib import Path @@ -90,12 +91,14 @@ def _wal_log(operation: str, params: dict, result: dict = None): "result": result, } try: + created = not _WAL_FILE.exists() with open(_WAL_FILE, "a", encoding="utf-8") as f: f.write(json.dumps(entry, default=str) + "\n") - try: - _WAL_FILE.chmod(0o600) - except (OSError, NotImplementedError): - pass + if created: + try: + _WAL_FILE.chmod(0o600) + except (OSError, NotImplementedError): + pass except Exception as e: logger.error(f"WAL write failed: {e}") @@ -110,13 +113,17 @@ def _get_client(): def _get_collection(create=False): """Return the ChromaDB collection, caching the client between calls.""" - global _collection_cache + global _collection_cache, _metadata_cache, _metadata_cache_time try: client = _get_client() if create: _collection_cache = client.get_or_create_collection(_config.collection_name) + _metadata_cache = None + _metadata_cache_time = 0 elif _collection_cache is None: _collection_cache = client.get_collection(_config.collection_name) + _metadata_cache = None + _metadata_cache_time = 0 return _collection_cache except Exception: return None @@ -149,6 +156,28 @@ def _fetch_all_metadata(col, where=None): return all_meta +_metadata_cache = None +_metadata_cache_time = 0 +_METADATA_CACHE_TTL = 5.0 # seconds + + +def _get_cached_metadata(col, where=None): + """Return cached metadata if fresh, else fetch and cache.""" + global _metadata_cache, _metadata_cache_time + now = time.time() + if ( + where is None + and _metadata_cache is not None + and (now - _metadata_cache_time) < _METADATA_CACHE_TTL + ): + return _metadata_cache + result = _fetch_all_metadata(col, where=where) + if where is None: + _metadata_cache = result + _metadata_cache_time = now + return result + + # ==================== READ TOOLS ==================== @@ -160,7 +189,7 @@ def tool_status(): wings = {} rooms = {} try: - all_meta = _fetch_all_metadata(col) + all_meta = _get_cached_metadata(col) for m in all_meta: w = m.get("wing", "unknown") r = m.get("room", "unknown") @@ -217,7 +246,7 @@ def tool_list_wings(): return _no_palace() wings = {} try: - all_meta = _fetch_all_metadata(col) + all_meta = _get_cached_metadata(col) for m in all_meta: w = m.get("wing", "unknown") wings[w] = wings.get(w, 0) + 1 @@ -248,7 +277,7 @@ def tool_get_taxonomy(): return _no_palace() taxonomy = {} try: - all_meta = _fetch_all_metadata(col) + all_meta = _get_cached_metadata(col) for m in all_meta: w = m.get("wing", "unknown") r = m.get("room", "unknown") @@ -428,6 +457,136 @@ def tool_delete_drawer(drawer_id: str): return {"success": False, "error": str(e)} +def tool_get_drawer(drawer_id: str): + """Fetch a single drawer by ID. Returns full content and metadata.""" + col = _get_collection() + if not col: + return _no_palace() + try: + result = col.get(ids=[drawer_id], include=["documents", "metadatas"]) + if not result["ids"]: + return {"error": f"Drawer not found: {drawer_id}"} + meta = result["metadatas"][0] + doc = result["documents"][0] + return { + "drawer_id": drawer_id, + "content": doc, + "wing": meta.get("wing", ""), + "room": meta.get("room", ""), + "metadata": meta, + } + except Exception as e: + return {"error": str(e)} + + +def tool_list_drawers(wing: str = None, room: str = None, limit: int = 20, offset: int = 0): + """List drawers with pagination. Optional wing/room filter.""" + limit = max(1, min(limit, 100)) + col = _get_collection() + if not col: + return _no_palace() + try: + where = None + conditions = [] + if wing: + conditions.append({"wing": wing}) + if room: + conditions.append({"room": room}) + if len(conditions) == 1: + where = conditions[0] + elif len(conditions) > 1: + where = {"$and": conditions} + + kwargs = {"include": ["documents", "metadatas"], "limit": limit, "offset": offset} + if where: + kwargs["where"] = where + result = col.get(**kwargs) + + drawers = [] + for i, did in enumerate(result["ids"]): + meta = result["metadatas"][i] + doc = result["documents"][i] + drawers.append( + { + "drawer_id": did, + "wing": meta.get("wing", ""), + "room": meta.get("room", ""), + "content_preview": doc[:200] + "..." if len(doc) > 200 else doc, + } + ) + return { + "drawers": drawers, + "count": len(drawers), + "offset": offset, + "limit": limit, + } + except Exception as e: + return {"error": str(e)} + + +def tool_update_drawer(drawer_id: str, content: str = None, wing: str = None, room: str = None): + """Update an existing drawer's content and/or metadata.""" + col = _get_collection() + if not col: + return _no_palace() + try: + existing = col.get(ids=[drawer_id], include=["documents", "metadatas"]) + if not existing["ids"]: + return {"success": False, "error": f"Drawer not found: {drawer_id}"} + + old_meta = existing["metadatas"][0] + old_doc = existing["documents"][0] + + # Sanitize inputs + new_doc = old_doc + if content is not None: + try: + new_doc = sanitize_content(content) + except ValueError as e: + return {"success": False, "error": str(e)} + + new_meta = dict(old_meta) + if wing is not None: + try: + new_meta["wing"] = sanitize_name(wing, "wing") + except ValueError as e: + return {"success": False, "error": str(e)} + if room is not None: + try: + new_meta["room"] = sanitize_name(room, "room") + except ValueError as e: + return {"success": False, "error": str(e)} + + _wal_log( + "update_drawer", + { + "drawer_id": drawer_id, + "old_wing": old_meta.get("wing", ""), + "old_room": old_meta.get("room", ""), + "new_wing": new_meta.get("wing", ""), + "new_room": new_meta.get("room", ""), + "content_changed": content is not None, + "content_preview": new_doc[:200] if content is not None else None, + }, + ) + + update_kwargs = {"ids": [drawer_id]} + if content is not None: + update_kwargs["documents"] = [new_doc] + update_kwargs["metadatas"] = [new_meta] + col.update(**update_kwargs) + + logger.info(f"Updated drawer: {drawer_id}") + return { + "success": True, + "drawer_id": drawer_id, + "wing": new_meta.get("wing", ""), + "room": new_meta.get("room", ""), + } + except Exception as e: + return {"success": False, "error": str(e)} + + # ==================== KNOWLEDGE GRAPH ==================== @@ -816,6 +975,59 @@ def tool_diary_read(agent_name: str, last_n: int = 10): }, "handler": tool_delete_drawer, }, + "mempalace_get_drawer": { + "description": "Fetch a single drawer by ID — returns full content and metadata.", + "input_schema": { + "type": "object", + "properties": { + "drawer_id": {"type": "string", "description": "ID of the drawer to fetch"}, + }, + "required": ["drawer_id"], + }, + "handler": tool_get_drawer, + }, + "mempalace_list_drawers": { + "description": "List drawers with pagination. Optional wing/room filter. Returns IDs, wings, rooms, and content previews.", + "input_schema": { + "type": "object", + "properties": { + "wing": {"type": "string", "description": "Filter by wing (optional)"}, + "room": {"type": "string", "description": "Filter by room (optional)"}, + "limit": { + "type": "integer", + "description": "Max results per page (default 20, max 100)", + }, + "offset": { + "type": "integer", + "description": "Offset for pagination (default 0)", + }, + }, + }, + "handler": tool_list_drawers, + }, + "mempalace_update_drawer": { + "description": "Update an existing drawer's content and/or metadata (wing, room). Fetches existing drawer first; returns error if not found.", + "input_schema": { + "type": "object", + "properties": { + "drawer_id": {"type": "string", "description": "ID of the drawer to update"}, + "content": { + "type": "string", + "description": "New content (optional — omit to keep existing)", + }, + "wing": { + "type": "string", + "description": "New wing (optional — omit to keep existing)", + }, + "room": { + "type": "string", + "description": "New room (optional — omit to keep existing)", + }, + }, + "required": ["drawer_id"], + }, + "handler": tool_update_drawer, + }, "mempalace_diary_write": { "description": "Write to your personal agent diary in AAAK format. Your observations, thoughts, what you worked on, what matters. Each agent has their own diary with full history. Write in AAAK for compression — e.g. 'SESSION:2026-04-04|built.palace.graph+diary.tools|ALC.req:agent.diaries.in.aaak|★★★'. Use entity codes from the AAAK spec.", "input_schema": { From d63ffd45f49ce4ec30b0912e7255e8be250eb8fd Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 9 Apr 2026 20:30:34 -0700 Subject: [PATCH 11/50] fix: configurable chunks, Layer1 scan cap, search filter dedup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I6: Chunk size/overlap/min now configurable via ~/.mempalace/config.json instead of hardcoded constants. Wired through mine() → process_file() → chunk_text(). I11: Layer1.generate() capped at MAX_SCAN=2000 drawers (was unbounded). Reduces wake-up from 250+ ChromaDB round-trips to 4 max. I12: Extracted _build_where_filter() helper in searcher.py, replaced 5 duplicate where-filter blocks across searcher.py and layers.py. Co-Authored-By: Claude Opus 4.6 --- mempalace/config.py | 18 +++++++++ mempalace/layers.py | 28 +++----------- mempalace/miner.py | 85 +++++++++++++++++++++++++++++++++++++------ mempalace/searcher.py | 29 +++++++-------- 4 files changed, 110 insertions(+), 50 deletions(-) diff --git a/mempalace/config.py b/mempalace/config.py index fcfb2c8afe..e20e22ad86 100644 --- a/mempalace/config.py +++ b/mempalace/config.py @@ -173,6 +173,21 @@ def hall_keywords(self): """Mapping of hall names to keyword lists.""" return self._file_config.get("hall_keywords", DEFAULT_HALL_KEYWORDS) + @property + def chunk_size(self): + """Characters per drawer chunk.""" + return self._file_config.get("chunk_size", 800) + + @property + def chunk_overlap(self): + """Overlap between adjacent chunks.""" + return self._file_config.get("chunk_overlap", 100) + + @property + def min_chunk_size(self): + """Minimum chunk size — skip smaller chunks.""" + return self._file_config.get("min_chunk_size", 50) + def init(self): """Create config directory and write default config.json if it doesn't exist.""" self._config_dir.mkdir(parents=True, exist_ok=True) @@ -187,6 +202,9 @@ def init(self): "collection_name": DEFAULT_COLLECTION_NAME, "topic_wings": DEFAULT_TOPIC_WINGS, "hall_keywords": DEFAULT_HALL_KEYWORDS, + "chunk_size": 800, + "chunk_overlap": 100, + "min_chunk_size": 50, } with open(self._config_file, "w") as f: json.dump(default_config, f, indent=2) diff --git a/mempalace/layers.py b/mempalace/layers.py index 6abb99bc9b..4fd5fc0b92 100644 --- a/mempalace/layers.py +++ b/mempalace/layers.py @@ -24,6 +24,7 @@ import chromadb from .config import MempalaceConfig +from .searcher import _build_where_filter # --------------------------------------------------------------------------- @@ -82,6 +83,7 @@ class Layer1: MAX_DRAWERS = 15 # at most 15 moments in wake-up MAX_CHARS = 3200 # hard cap on total L1 text (~800 tokens) + MAX_SCAN = 2000 # don't scan more than this for L1 generation def __init__(self, palace_path: str = None, wing: str = None): cfg = MempalaceConfig() @@ -115,7 +117,7 @@ def generate(self) -> str: docs.extend(batch_docs) metas.extend(batch_metas) offset += len(batch_docs) - if len(batch_docs) < _BATCH: + if len(batch_docs) < _BATCH or len(docs) >= self.MAX_SCAN: break if not docs: @@ -201,13 +203,7 @@ def retrieve(self, wing: str = None, room: str = None, n_results: int = 10) -> s except Exception: return "No palace found." - where = {} - if wing and room: - where = {"$and": [{"wing": wing}, {"room": room}]} - elif wing: - where = {"wing": wing} - elif room: - where = {"room": room} + where = _build_where_filter(wing, room) kwargs = {"include": ["documents", "metadatas"], "limit": n_results} if where: @@ -265,13 +261,7 @@ def search(self, query: str, wing: str = None, room: str = None, n_results: int except Exception: return "No palace found." - where = {} - if wing and room: - where = {"$and": [{"wing": wing}, {"room": room}]} - elif wing: - where = {"wing": wing} - elif room: - where = {"room": room} + where = _build_where_filter(wing, room) kwargs = { "query_texts": [query], @@ -321,13 +311,7 @@ def search_raw( except Exception: return [] - where = {} - if wing and room: - where = {"$and": [{"wing": wing}, {"room": room}]} - elif wing: - where = {"wing": wing} - elif room: - where = {"room": room} + where = _build_where_filter(wing, room) kwargs = { "query_texts": [query], diff --git a/mempalace/miner.py b/mempalace/miner.py index c5456e442b..b2c97a7a98 100644 --- a/mempalace/miner.py +++ b/mempalace/miner.py @@ -326,12 +326,27 @@ def detect_room(filepath: Path, content: str, rooms: list, project_path: Path) - # ============================================================================= -def chunk_text(content: str, source_file: str) -> list: +def chunk_text( + content: str, + source_file: str, + chunk_size: int = None, + chunk_overlap: int = None, + min_chunk_size: int = None, +) -> list: """ Split content into drawer-sized chunks. Tries to split on paragraph/line boundaries. Returns list of {"content": str, "chunk_index": int} + + Optional params override module-level defaults when provided. """ + if chunk_size is None: + chunk_size = CHUNK_SIZE + if chunk_overlap is None: + chunk_overlap = CHUNK_OVERLAP + if min_chunk_size is None: + min_chunk_size = MIN_CHUNK_SIZE + # Clean up content = content.strip() if not content: @@ -342,20 +357,20 @@ def chunk_text(content: str, source_file: str) -> list: chunk_index = 0 while start < len(content): - end = min(start + CHUNK_SIZE, len(content)) + end = min(start + chunk_size, len(content)) # Try to break at paragraph boundary if end < len(content): newline_pos = content.rfind("\n\n", start, end) - if newline_pos > start + CHUNK_SIZE // 2: + if newline_pos > start + chunk_size // 2: end = newline_pos else: newline_pos = content.rfind("\n", start, end) - if newline_pos > start + CHUNK_SIZE // 2: + if newline_pos > start + chunk_size // 2: end = newline_pos chunk = content[start:end].strip() - if len(chunk) >= MIN_CHUNK_SIZE: + if len(chunk) >= min_chunk_size: chunks.append( { "content": chunk, @@ -364,7 +379,7 @@ def chunk_text(content: str, source_file: str) -> list: ) chunk_index += 1 - start = end - CHUNK_OVERLAP if end < len(content) else end + start = end - chunk_overlap if end < len(content) else end return chunks @@ -414,6 +429,9 @@ def _prepare_file( wing: str, rooms: list, agent: str, + chunk_size: int = None, + chunk_overlap: int = None, + min_chunk_size: int = None, ) -> tuple: """Read, chunk, and route one file without writing to ChromaDB. @@ -421,6 +439,7 @@ def _prepare_file( when the file should be skipped (unreadable, too small, etc.). This is the pure-computation half of process_file, safe for concurrent use. """ + effective_min = min_chunk_size if min_chunk_size is not None else MIN_CHUNK_SIZE source_file = str(filepath) try: @@ -429,11 +448,17 @@ def _prepare_file( return None, None, None, None content = content.strip() - if len(content) < MIN_CHUNK_SIZE: + if len(content) < effective_min: return None, None, None, None room = detect_room(filepath, content, rooms, project_path) - chunks = chunk_text(content, source_file) + chunks = chunk_text( + content, + source_file, + chunk_size=chunk_size, + chunk_overlap=chunk_overlap, + min_chunk_size=min_chunk_size, + ) if not chunks: return None, None, None, None @@ -473,8 +498,12 @@ def process_file( rooms: list, agent: str, dry_run: bool, + chunk_size: int = None, + chunk_overlap: int = None, + min_chunk_size: int = None, ) -> tuple: """Read, chunk, route, and file one file. Returns (drawer_count, room_name).""" + effective_min = min_chunk_size if min_chunk_size is not None else MIN_CHUNK_SIZE # Skip if already filed source_file = str(filepath) @@ -488,15 +517,28 @@ def process_file( except OSError: return 0, None content = content.strip() - if len(content) < MIN_CHUNK_SIZE: + if len(content) < effective_min: return 0, None room = detect_room(filepath, content, rooms, project_path) - chunks = chunk_text(content, source_file) + chunks = chunk_text( + content, + source_file, + chunk_size=chunk_size, + chunk_overlap=chunk_overlap, + min_chunk_size=min_chunk_size, + ) print(f" [DRY RUN] {filepath.name} → room:{room} ({len(chunks)} drawers)") return len(chunks), room batch_docs, batch_ids, batch_metas, room = _prepare_file( - filepath, project_path, wing, rooms, agent + filepath, + project_path, + wing, + rooms, + agent, + chunk_size=chunk_size, + chunk_overlap=chunk_overlap, + min_chunk_size=min_chunk_size, ) if batch_docs is None: return 0, None @@ -624,8 +666,15 @@ def mine( import concurrent.futures import threading + from .config import MempalaceConfig + project_path = Path(project_dir).expanduser().resolve() config = load_config(project_dir) + palace_config = MempalaceConfig() + + cfg_chunk_size = palace_config.chunk_size + cfg_chunk_overlap = palace_config.chunk_overlap + cfg_min_chunk_size = palace_config.min_chunk_size wing = wing_override or config["wing"] rooms = config.get("rooms", [{"name": "general", "description": "All project files"}]) @@ -678,6 +727,9 @@ def mine( rooms=rooms, agent=agent, dry_run=dry_run, + chunk_size=cfg_chunk_size, + chunk_overlap=cfg_chunk_overlap, + min_chunk_size=cfg_min_chunk_size, ) if drawers == 0 and not dry_run: files_skipped += 1 @@ -707,7 +759,16 @@ def mine( processed_count = 0 def prepare_one(filepath): - return filepath, _prepare_file(filepath, project_path, wing, rooms, agent) + return filepath, _prepare_file( + filepath, + project_path, + wing, + rooms, + agent, + chunk_size=cfg_chunk_size, + chunk_overlap=cfg_chunk_overlap, + min_chunk_size=cfg_min_chunk_size, + ) results = [] with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool: diff --git a/mempalace/searcher.py b/mempalace/searcher.py index 069005e698..3ab5632c77 100644 --- a/mempalace/searcher.py +++ b/mempalace/searcher.py @@ -18,6 +18,17 @@ class SearchError(Exception): """Raised when search cannot proceed (e.g. no palace found).""" +def _build_where_filter(wing: str = None, room: str = None) -> dict: + """Build ChromaDB where filter for wing/room filtering.""" + if wing and room: + return {"$and": [{"wing": wing}, {"room": room}]} + elif wing: + return {"wing": wing} + elif room: + return {"room": room} + return {} + + def search(query: str, palace_path: str, wing: str = None, room: str = None, n_results: int = 5): """ Search the palace. Returns verbatim drawer content. @@ -31,14 +42,7 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r print(" Run: mempalace init then mempalace mine ") raise SearchError(f"No palace found at {palace_path}") - # Build where filter - where = {} - if wing and room: - where = {"$and": [{"wing": wing}, {"room": room}]} - elif wing: - where = {"wing": wing} - elif room: - where = {"room": room} + where = _build_where_filter(wing, room) try: kwargs = { @@ -112,14 +116,7 @@ def search_memories( "hint": "Run: mempalace init && mempalace mine ", } - # Build where filter - where = {} - if wing and room: - where = {"$and": [{"wing": wing}, {"room": room}]} - elif wing: - where = {"wing": wing} - elif room: - where = {"room": room} + where = _build_where_filter(wing, room) try: kwargs = { From 4a12748e47f35c082c4f015c13951a54e4d2ff40 Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 9 Apr 2026 20:30:41 -0700 Subject: [PATCH 12/50] fix: chunk_text tests, KG direction default, plugin version sync I10: 10 unit tests for chunk_text() covering boundaries, overlap, indices, empty/whitespace input, content preservation. I13: KG query_entity default direction aligned from "outgoing" to "both" to match the MCP schema default. I14: Plugin versions synced to 3.1.0 in both .claude-plugin/ and .codex-plugin/. Co-Authored-By: Claude Opus 4.6 --- .claude-plugin/plugin.json | 2 +- .codex-plugin/plugin.json | 2 +- mempalace/knowledge_graph.py | 2 +- tests/test_miner.py | 106 ++++++++++++++++++++++++++++++++++- 4 files changed, 108 insertions(+), 4 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index fa05a15ab9..f153c55ecb 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "mempalace", - "version": "3.0.14", + "version": "3.1.0", "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 19 MCP tools, auto-save hooks, and guided setup.", "author": { "name": "milla-jovovich" diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 5784847186..1c8e4c9bf7 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "mempalace", - "version": "3.0.14", + "version": "3.1.0", "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 19 MCP tools, auto-save hooks, and guided setup.", "author": { "name": "milla-jovovich" diff --git a/mempalace/knowledge_graph.py b/mempalace/knowledge_graph.py index b094f06f7c..2199628823 100644 --- a/mempalace/knowledge_graph.py +++ b/mempalace/knowledge_graph.py @@ -193,7 +193,7 @@ def invalidate(self, subject: str, predicate: str, obj: str, ended: str = None): # ── Query operations ────────────────────────────────────────────────── - def query_entity(self, name: str, as_of: str = None, direction: str = "outgoing"): + def query_entity(self, name: str, as_of: str = None, direction: str = "both"): """ Get all relationships for an entity. diff --git a/tests/test_miner.py b/tests/test_miner.py index fe2ec88ddd..fb6bcc3946 100644 --- a/tests/test_miner.py +++ b/tests/test_miner.py @@ -6,7 +6,15 @@ import chromadb import yaml -from mempalace.miner import detect_room, mine, scan_project +from mempalace.miner import ( + CHUNK_OVERLAP, + CHUNK_SIZE, + MIN_CHUNK_SIZE, + chunk_text, + detect_room, + mine, + scan_project, +) from mempalace.palace import file_already_mined @@ -357,3 +365,99 @@ def test_detect_room_filename_beats_content(): """Priority 2 (filename) wins even when content strongly matches another room.""" content = "test test test test test test test" assert _detect("misc/backend.py", content=content) == "backend" + + +# ============================================================================= +# chunk_text tests +# ============================================================================= + + +def test_chunk_text_short_content(): + """Content shorter than CHUNK_SIZE produces a single chunk.""" + content = "a" * (CHUNK_SIZE - 1) + chunks = chunk_text(content, "/fake/file.py") + assert len(chunks) == 1 + assert chunks[0]["content"] == content + assert chunks[0]["chunk_index"] == 0 + + +def test_chunk_text_exact_chunk_size(): + """Content exactly CHUNK_SIZE long produces a single chunk.""" + content = "a" * CHUNK_SIZE + chunks = chunk_text(content, "/fake/file.py") + assert len(chunks) == 1 + assert chunks[0]["content"] == content + + +def test_chunk_text_two_chunks(): + """Content slightly over CHUNK_SIZE produces two chunks with overlap.""" + content = "a" * (CHUNK_SIZE + 1) + chunks = chunk_text(content, "/fake/file.py") + assert len(chunks) == 2 + + +def test_chunk_text_overlap_content(): + """The second chunk starts at CHUNK_SIZE - CHUNK_OVERLAP (overlap region).""" + # Use digits so each position is unique and verifiable + content = "".join(str(i % 10) for i in range(CHUNK_SIZE + 200)) + chunks = chunk_text(content, "/fake/file.py") + assert len(chunks) >= 2 + # The overlap means the second chunk's content starts from the overlap region + expected_start = CHUNK_SIZE - CHUNK_OVERLAP + assert chunks[1]["content"].startswith(content[expected_start : expected_start + 10]) + + +def test_chunk_text_chunk_indices(): + """chunk_index values increment sequentially starting from 0.""" + content = "a" * (CHUNK_SIZE * 3) + chunks = chunk_text(content, "/fake/file.py") + assert len(chunks) >= 3 + for i, chunk in enumerate(chunks): + assert chunk["chunk_index"] == i + + +def test_chunk_text_empty_content(): + """Empty string returns an empty list.""" + chunks = chunk_text("", "/fake/file.py") + assert chunks == [] + + +def test_chunk_text_below_min_size(): + """Content below MIN_CHUNK_SIZE returns an empty list.""" + content = "a" * (MIN_CHUNK_SIZE - 1) + chunks = chunk_text(content, "/fake/file.py") + assert chunks == [] + + +def test_chunk_text_whitespace_only(): + """Whitespace-only content returns an empty list (stripped to empty).""" + chunks = chunk_text(" \n\n\t \n ", "/fake/file.py") + assert chunks == [] + +def test_chunk_text_many_chunks(): + """Very long content (10x CHUNK_SIZE) produces the correct number of chunks.""" + content = "a" * (CHUNK_SIZE * 10) + chunks = chunk_text(content, "/fake/file.py") + # With overlap, each chunk after the first starts CHUNK_SIZE - CHUNK_OVERLAP ahead. + # So we need ceil((total_len - CHUNK_OVERLAP) / (CHUNK_SIZE - CHUNK_OVERLAP)) chunks, + # but the exact count depends on boundary logic. Just verify it's reasonable. + total_len = len(content) + step = CHUNK_SIZE - CHUNK_OVERLAP + expected_min = total_len // CHUNK_SIZE # at least this many + expected_max = (total_len // step) + 1 # at most this many + assert expected_min <= len(chunks) <= expected_max + + +def test_chunk_text_preserves_content(): + """All original content is covered by the union of chunks (nothing lost).""" + # Use only non-whitespace so chunk stripping doesn't drop characters at boundaries + content = "abcdefghij" * (CHUNK_SIZE * 3 // 10) + chunks = chunk_text(content, "/fake/file.py") + assert len(chunks) >= 2 + # Every character in the original must appear in at least one chunk + all_chunk_text = "".join(c["content"] for c in chunks) + for ch_pos, ch in enumerate(content): + assert ch in all_chunk_text, f"Character '{ch}' at position {ch_pos} not in any chunk" + # Stronger: the joined chunks should contain more characters than the original + # (due to overlap), confirming nothing is dropped + assert len(all_chunk_text) >= len(content) From 83c50ac8fa63e4b1ea6c891b0a41a8aebedb19ea Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 9 Apr 2026 21:10:16 -0700 Subject: [PATCH 13/50] =?UTF-8?q?fix:=20PR=20review=20feedback=20=E2=80=94?= =?UTF-8?q?=20error=20handling,=20streaming=20export,=20L2=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address web3guru888's review feedback across PRs #492 and #493: - palace.py: remove unused filepaths param from bulk_check_mined(), replace bare except with logger.warning for partial fetch visibility - miner.py: wrap future.result() in try/except so one file failure doesn't abort the entire concurrent mining run - exporter.py: stream drawers in batches instead of loading entire palace into memory — keeps memory bounded for large palaces - searcher.py: document min_similarity as L2 distance (not cosine) with typical range guidance in docstring Co-Authored-By: Claude Opus 4.6 --- mempalace/exporter.py | 111 ++++++++++++++++++++++++------------------ mempalace/miner.py | 15 ++++-- mempalace/palace.py | 11 +++-- mempalace/searcher.py | 15 +++++- 4 files changed, 95 insertions(+), 57 deletions(-) diff --git a/mempalace/exporter.py b/mempalace/exporter.py index 7df0d4781c..7dfd7af0c8 100644 --- a/mempalace/exporter.py +++ b/mempalace/exporter.py @@ -6,6 +6,9 @@ index.md — table of contents wing_name/ room_name.md — one file per room, drawers as sections + +Streams drawers in paginated batches so memory usage stays bounded +regardless of palace size. """ import os @@ -18,6 +21,10 @@ def export_palace(palace_path: str, output_dir: str, format: str = "markdown") -> dict: """Export all palace drawers as markdown files organized by wing/room. + Streams drawers in batches of 1000 and writes each wing/room file + incrementally, keeping memory usage proportional to batch size rather + than total palace size. + Args: palace_path: Path to the ChromaDB palace directory. output_dir: Where to write the exported markdown tree. @@ -33,70 +40,78 @@ def export_palace(palace_path: str, output_dir: str, format: str = "markdown") - print(" Palace is empty — nothing to export.") return {"wings": 0, "rooms": 0, "drawers": 0} - # Paginate all drawers in batches of 1000 - print(f" Reading {total} drawers...") - grouped = defaultdict(lambda: defaultdict(list)) + os.makedirs(output_dir, exist_ok=True) + + # Track which room files have been opened (so we can append vs overwrite) + opened_rooms: set[tuple[str, str]] = set() + # Track stats per wing: {wing: {room: count}} + wing_stats: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int)) + total_drawers = 0 + + print(f" Streaming {total} drawers...") offset = 0 while offset < total: batch = col.get(limit=1000, offset=offset, include=["documents", "metadatas"]) if not batch["ids"]: break + + # Group this batch by wing/room so we do one file write per room per batch + batch_grouped: dict[str, dict[str, list]] = defaultdict(lambda: defaultdict(list)) for doc_id, doc, meta in zip(batch["ids"], batch["documents"], batch["metadatas"]): wing = meta.get("wing", "unknown") room = meta.get("room", "general") - grouped[wing][room].append({ + batch_grouped[wing][room].append({ "id": doc_id, "content": doc, "source": meta.get("source_file", ""), "filed_at": meta.get("filed_at", ""), "added_by": meta.get("added_by", ""), }) - offset += len(batch["ids"]) - # Write markdown files - os.makedirs(output_dir, exist_ok=True) - total_drawers = 0 + # Write/append each room file + for wing, rooms in batch_grouped.items(): + wing_dir = os.path.join(output_dir, wing) + os.makedirs(wing_dir, exist_ok=True) + + for room, drawers in rooms.items(): + room_path = os.path.join(wing_dir, f"{room}.md") + key = (wing, room) + is_new = key not in opened_rooms + + with open(room_path, "a" if not is_new else "w", encoding="utf-8") as f: + if is_new: + f.write(f"# {wing} / {room}\n\n") + opened_rooms.add(key) + + for drawer in drawers: + source = drawer["source"] or "unknown" + filed = drawer["filed_at"] or "unknown" + added_by = drawer["added_by"] or "unknown" + + f.write( + f"## {drawer['id']}\n" + f"\n" + f"> {_quote_content(drawer['content'])}\n" + f"\n" + f"| Field | Value |\n" + f"|-------|-------|\n" + f"| Source | {source} |\n" + f"| Filed | {filed} |\n" + f"| Added by | {added_by} |\n" + f"\n" + f"---\n\n" + ) + + wing_stats[wing][room] += len(drawers) + total_drawers += len(drawers) - index_rows = [] + offset += len(batch["ids"]) - for wing in sorted(grouped): - wing_dir = os.path.join(output_dir, wing) - os.makedirs(wing_dir, exist_ok=True) - wing_drawer_count = 0 - - rooms = grouped[wing] - for room in sorted(rooms): - drawers = rooms[room] - room_path = os.path.join(wing_dir, f"{room}.md") - - sections = [] - for drawer in drawers: - source = drawer["source"] or "unknown" - filed = drawer["filed_at"] or "unknown" - added_by = drawer["added_by"] or "unknown" - - section = ( - f"## {drawer['id']}\n" - f"\n" - f"> {_quote_content(drawer['content'])}\n" - f"\n" - f"| Field | Value |\n" - f"|-------|-------|\n" - f"| Source | {source} |\n" - f"| Filed | {filed} |\n" - f"| Added by | {added_by} |\n" - f"\n" - f"---" - ) - sections.append(section) - - body = f"# {wing} / {room}\n\n" + "\n\n".join(sections) + "\n" - with open(room_path, "w", encoding="utf-8") as f: - f.write(body) - - wing_drawer_count += len(drawers) - - total_drawers += wing_drawer_count + # Build and print stats + index_rows = [] + for wing in sorted(wing_stats): + rooms = wing_stats[wing] + wing_drawer_count = sum(rooms.values()) index_rows.append((wing, len(rooms), wing_drawer_count)) print(f" {wing}: {len(rooms)} rooms, {wing_drawer_count} drawers") @@ -116,7 +131,7 @@ def export_palace(palace_path: str, output_dir: str, format: str = "markdown") - with open(index_path, "w", encoding="utf-8") as f: f.write("\n".join(index_lines)) - stats = {"wings": len(grouped), "rooms": sum(r for _, r, _ in index_rows), "drawers": total_drawers} + stats = {"wings": len(wing_stats), "rooms": sum(r for _, r, _ in index_rows), "drawers": total_drawers} print(f"\n Exported {stats['drawers']} drawers across {stats['wings']} wings, {stats['rooms']} rooms") print(f" Output: {output_dir}") return stats diff --git a/mempalace/miner.py b/mempalace/miner.py index b2c97a7a98..4f3afe05d4 100644 --- a/mempalace/miner.py +++ b/mempalace/miner.py @@ -7,6 +7,7 @@ Stores verbatim chunks as drawers. No summaries. Ever. """ +import logging import os import re import sys @@ -16,6 +17,8 @@ from datetime import datetime from collections import defaultdict +logger = logging.getLogger(__name__) + import chromadb from .palace import SKIP_DIRS, get_collection, file_already_mined, bulk_check_mined @@ -743,8 +746,7 @@ def mine( # Phase 0: bulk-fetch already-mined mtimes to skip files without # per-file DB queries. - filepaths_str = [str(f) for f in files] - mined_map = bulk_check_mined(collection, filepaths_str) + mined_map = bulk_check_mined(collection) # Filter out already-mined files before spawning threads. files_to_process = [] @@ -774,7 +776,14 @@ def prepare_one(filepath): with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool: futures = {pool.submit(prepare_one, fp): fp for fp in files_to_process} for future in concurrent.futures.as_completed(futures): - filepath, (batch_docs, batch_ids, batch_metas, room) = future.result() + try: + filepath, (batch_docs, batch_ids, batch_metas, room) = future.result() + except Exception as exc: + failed_path = futures[future] + logger.warning("Skipping %s: %s", failed_path, exc) + with counter_lock: + files_skipped += 1 + continue if batch_docs is None: with counter_lock: files_skipped += 1 diff --git a/mempalace/palace.py b/mempalace/palace.py index f1ca0263c9..537200d053 100644 --- a/mempalace/palace.py +++ b/mempalace/palace.py @@ -4,9 +4,13 @@ Consolidates ChromaDB access patterns used by both miners and the MCP server. """ +import logging import os + import chromadb +logger = logging.getLogger(__name__) + SKIP_DIRS = { ".git", "node_modules", @@ -71,15 +75,14 @@ def file_already_mined(collection, source_file: str, check_mtime: bool = False) return False -def bulk_check_mined(collection, filepaths: list[str]) -> dict[str, float]: +def bulk_check_mined(collection) -> dict[str, float]: """Pre-fetch source_file/source_mtime pairs for all documents in the collection. Returns a dict mapping source_file -> source_mtime (as float) for every document that has both fields. Callers can check membership and compare mtimes locally instead of issuing one ChromaDB query per file. - The *filepaths* argument is accepted for API symmetry but the function - fetches the full collection in paginated batches (like palace_graph.py) + Fetches the full collection in paginated batches (like palace_graph.py) since a WHERE-IN filter on thousands of paths is not supported by ChromaDB. """ mined: dict[str, float] = {} @@ -97,5 +100,5 @@ def bulk_check_mined(collection, filepaths: list[str]) -> dict[str, float]: break offset += len(batch["ids"]) except Exception: - pass + logger.warning("bulk_check_mined: partial fetch, %d files loaded", len(mined)) return mined diff --git a/mempalace/searcher.py b/mempalace/searcher.py index 3ab5632c77..a2f33379c2 100644 --- a/mempalace/searcher.py +++ b/mempalace/searcher.py @@ -102,9 +102,20 @@ def search_memories( n_results: int = 5, min_similarity: float = 0.0, ) -> dict: - """ - Programmatic search — returns a dict instead of printing. + """Programmatic search — returns a dict instead of printing. + Used by the MCP server and other callers that need data. + + Args: + query: Natural language search query. + palace_path: Path to the ChromaDB palace directory. + wing: Optional wing filter. + room: Optional room filter. + n_results: Max results to return. + min_similarity: Max L2 (Euclidean) distance threshold. ChromaDB uses + L2 distance by default — 0 = identical, larger = less similar. + Results with distance > this value are filtered out. A value of + 0.0 disables filtering. Typical useful range: 0.5–1.5. """ try: client = chromadb.PersistentClient(path=palace_path) From 8f565c772c6ab245f065137dea629d4f03383dee Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 9 Apr 2026 21:16:39 -0700 Subject: [PATCH 14/50] fix: address remaining PR review feedback (16 items) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename _build_where_filter → build_where_filter (public cross-module API) - Add float() cast + TypeError/ValueError handling in _is_already_mined - Add chunk_overlap validation (must be >= 0 and < chunk_size) - Batch convo_miner adds to 100 docs per call (avoid SQLite limits) - Stream miner writes as futures complete (bounded memory) - Remove unused palace_path in hooks_cli - Remove unused chromadb import in test_exporter - Sanitize wing/room as path components in exporter (prevent traversal) - Filter on raw distance before rounding in searcher - Clamp negative offset in tool_list_drawers - No-op early return + cache invalidation in tool_update_drawer - Add min/max schema bounds for search limit and list_drawers limit/offset - Update CLAUDE.md test count (534 → 562) - Improve chunk coverage test with position-unique tokens Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 2 +- mempalace/convo_miner.py | 23 ++++++++------- mempalace/exporter.py | 14 +++++++-- mempalace/hooks_cli.py | 2 +- mempalace/layers.py | 8 +++--- mempalace/mcp_server.py | 14 ++++++++- mempalace/miner.py | 62 +++++++++++++++++++++------------------- mempalace/searcher.py | 15 ++++------ tests/test_exporter.py | 1 - tests/test_miner.py | 15 +++++----- 10 files changed, 90 insertions(+), 66 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 378e81c7fd..31d95bd2f3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,7 +20,7 @@ JP's fork of [milla-jovovich/mempalace](https://github.com/milla-jovovich/mempal ```bash source venv/bin/activate -python -m pytest tests/ -x -q # run tests (534 expected) +python -m pytest tests/ -x -q # run tests (562 expected) mempalace status # check palace state mempalace search "query" # test search python -m mempalace.mcp_server # run MCP server standalone diff --git a/mempalace/convo_miner.py b/mempalace/convo_miner.py index 82e647c651..d54e0c3bbd 100644 --- a/mempalace/convo_miner.py +++ b/mempalace/convo_miner.py @@ -350,17 +350,20 @@ def mine_convos( } ) drawers_added = 0 + _ADD_BATCH_SIZE = 100 if batch_docs: - try: - collection.add( - documents=batch_docs, - ids=batch_ids, - metadatas=batch_metas, - ) - drawers_added = len(batch_docs) - except Exception as e: - if "already exists" not in str(e).lower(): - raise + for batch_start in range(0, len(batch_docs), _ADD_BATCH_SIZE): + batch_end = batch_start + _ADD_BATCH_SIZE + try: + collection.add( + documents=batch_docs[batch_start:batch_end], + ids=batch_ids[batch_start:batch_end], + metadatas=batch_metas[batch_start:batch_end], + ) + drawers_added += len(batch_docs[batch_start:batch_end]) + except Exception as e: + if "already exists" not in str(e).lower(): + raise total_drawers += drawers_added print(f" ✓ [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers_added}") diff --git a/mempalace/exporter.py b/mempalace/exporter.py index 7dfd7af0c8..c08215f307 100644 --- a/mempalace/exporter.py +++ b/mempalace/exporter.py @@ -12,12 +12,20 @@ """ import os +import re from collections import defaultdict from datetime import datetime from .palace import get_collection +def _safe_path_component(name: str) -> str: + """Sanitize a string for use as a directory/file name component.""" + name = re.sub(r'[/\\:*?"<>|]', '_', name) + name = name.strip('. ') + return name or 'unknown' + + def export_palace(palace_path: str, output_dir: str, format: str = "markdown") -> dict: """Export all palace drawers as markdown files organized by wing/room. @@ -70,11 +78,13 @@ def export_palace(palace_path: str, output_dir: str, format: str = "markdown") - # Write/append each room file for wing, rooms in batch_grouped.items(): - wing_dir = os.path.join(output_dir, wing) + safe_wing = _safe_path_component(wing) + wing_dir = os.path.join(output_dir, safe_wing) os.makedirs(wing_dir, exist_ok=True) for room, drawers in rooms.items(): - room_path = os.path.join(wing_dir, f"{room}.md") + safe_room = _safe_path_component(room) + room_path = os.path.join(wing_dir, f"{safe_room}.md") key = (wing, room) is_new = key not in opened_rooms diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index 7a78ca93c1..8709e2f080 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -117,7 +117,7 @@ def _ingest_transcript(transcript_path: str): from .config import MempalaceConfig try: - palace_path = MempalaceConfig().palace_path + MempalaceConfig() # validate config loads except Exception: return diff --git a/mempalace/layers.py b/mempalace/layers.py index 4fd5fc0b92..b14f00fe0c 100644 --- a/mempalace/layers.py +++ b/mempalace/layers.py @@ -24,7 +24,7 @@ import chromadb from .config import MempalaceConfig -from .searcher import _build_where_filter +from .searcher import build_where_filter # --------------------------------------------------------------------------- @@ -203,7 +203,7 @@ def retrieve(self, wing: str = None, room: str = None, n_results: int = 10) -> s except Exception: return "No palace found." - where = _build_where_filter(wing, room) + where = build_where_filter(wing, room) kwargs = {"include": ["documents", "metadatas"], "limit": n_results} if where: @@ -261,7 +261,7 @@ def search(self, query: str, wing: str = None, room: str = None, n_results: int except Exception: return "No palace found." - where = _build_where_filter(wing, room) + where = build_where_filter(wing, room) kwargs = { "query_texts": [query], @@ -311,7 +311,7 @@ def search_raw( except Exception: return [] - where = _build_where_filter(wing, room) + where = build_where_filter(wing, room) kwargs = { "query_texts": [query], diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index b04e59a316..bd75d2adef 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -482,6 +482,7 @@ def tool_get_drawer(drawer_id: str): def tool_list_drawers(wing: str = None, room: str = None, limit: int = 20, offset: int = 0): """List drawers with pagination. Optional wing/room filter.""" limit = max(1, min(limit, 100)) + offset = max(0, offset) col = _get_collection() if not col: return _no_palace() @@ -526,6 +527,11 @@ def tool_list_drawers(wing: str = None, room: str = None, limit: int = 20, offse def tool_update_drawer(drawer_id: str, content: str = None, wing: str = None, room: str = None): """Update an existing drawer's content and/or metadata.""" + global _metadata_cache + + if content is None and wing is None and room is None: + return {"success": True, "drawer_id": drawer_id, "noop": True} + col = _get_collection() if not col: return _no_palace() @@ -576,6 +582,9 @@ def tool_update_drawer(drawer_id: str, content: str = None, wing: str = None, ro update_kwargs["metadatas"] = [new_meta] col.update(**update_kwargs) + # Invalidate metadata cache so status/taxonomy reflect changes + _metadata_cache = None + logger.info(f"Updated drawer: {drawer_id}") return { "success": True, @@ -916,7 +925,7 @@ def tool_diary_read(agent_name: str, last_n: int = 10): "type": "object", "properties": { "query": {"type": "string", "description": "What to search for"}, - "limit": {"type": "integer", "description": "Max results (default 5)"}, + "limit": {"type": "integer", "description": "Max results (default 5)", "minimum": 1, "maximum": 100}, "wing": {"type": "string", "description": "Filter by wing (optional)"}, "room": {"type": "string", "description": "Filter by room (optional)"}, "min_similarity": { @@ -996,10 +1005,13 @@ def tool_diary_read(agent_name: str, last_n: int = 10): "limit": { "type": "integer", "description": "Max results per page (default 20, max 100)", + "minimum": 1, + "maximum": 100, }, "offset": { "type": "integer", "description": "Offset for pagination (default 0)", + "minimum": 0, }, }, }, diff --git a/mempalace/miner.py b/mempalace/miner.py index 4f3afe05d4..ac7169ab3b 100644 --- a/mempalace/miner.py +++ b/mempalace/miner.py @@ -350,6 +350,11 @@ def chunk_text( if min_chunk_size is None: min_chunk_size = MIN_CHUNK_SIZE + if chunk_overlap < 0 or chunk_overlap >= chunk_size: + raise ValueError( + f"chunk_overlap ({chunk_overlap}) must be >= 0 and < chunk_size ({chunk_size})" + ) + # Clean up content = content.strip() if not content: @@ -632,17 +637,17 @@ def scan_project( def _is_already_mined(source_file: str, mined_map: dict) -> bool: """Check if a file is already mined using the bulk-fetched mined_map. - Compares stored mtime against current file mtime, matching the logic - in file_already_mined() but without per-file DB queries. + Compares stored mtime against current file mtime using epsilon tolerance, + matching the logic in file_already_mined() but without per-file DB queries. """ stored_mtime = mined_map.get(source_file) if stored_mtime is None: return False try: current_mtime = os.path.getmtime(source_file) - except OSError: + return abs(float(stored_mtime) - current_mtime) < 0.01 + except (OSError, TypeError, ValueError): return False - return abs(stored_mtime - current_mtime) < 0.01 # Maximum documents per ChromaDB upsert call @@ -772,7 +777,11 @@ def prepare_one(filepath): min_chunk_size=cfg_min_chunk_size, ) - results = [] + # Phase 1 read/chunk + Phase 2 write as futures complete (stream to DB) + pending_docs = [] + pending_ids = [] + pending_metas = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool: futures = {pool.submit(prepare_one, fp): fp for fp in files_to_process} for future in concurrent.futures.as_completed(futures): @@ -788,7 +797,24 @@ def prepare_one(filepath): with counter_lock: files_skipped += 1 continue - results.append((filepath, batch_docs, batch_ids, batch_metas, room)) + + total_drawers += len(batch_docs) + room_counts[room] += 1 + pending_docs.extend(batch_docs) + pending_ids.extend(batch_ids) + pending_metas.extend(batch_metas) + + # Flush when batch is large enough + if len(pending_docs) >= _UPSERT_BATCH_SIZE: + collection.upsert( + documents=pending_docs, + ids=pending_ids, + metadatas=pending_metas, + ) + pending_docs = [] + pending_ids = [] + pending_metas = [] + with counter_lock: processed_count += 1 print( @@ -796,30 +822,6 @@ def prepare_one(filepath): f"{filepath.name[:50]:50} +{len(batch_docs)}" ) - # Phase 2: sequential ChromaDB writes, batched across files - pending_docs = [] - pending_ids = [] - pending_metas = [] - - for filepath, batch_docs, batch_ids, batch_metas, room in results: - total_drawers += len(batch_docs) - room_counts[room] += 1 - - pending_docs.extend(batch_docs) - pending_ids.extend(batch_ids) - pending_metas.extend(batch_metas) - - # Flush when batch is large enough - if len(pending_docs) >= _UPSERT_BATCH_SIZE: - collection.upsert( - documents=pending_docs, - ids=pending_ids, - metadatas=pending_metas, - ) - pending_docs = [] - pending_ids = [] - pending_metas = [] - # Flush remainder if pending_docs: collection.upsert( diff --git a/mempalace/searcher.py b/mempalace/searcher.py index a2f33379c2..40c5ad24f3 100644 --- a/mempalace/searcher.py +++ b/mempalace/searcher.py @@ -18,7 +18,7 @@ class SearchError(Exception): """Raised when search cannot proceed (e.g. no palace found).""" -def _build_where_filter(wing: str = None, room: str = None) -> dict: +def build_where_filter(wing: str = None, room: str = None) -> dict: """Build ChromaDB where filter for wing/room filtering.""" if wing and room: return {"$and": [{"wing": wing}, {"room": room}]} @@ -42,7 +42,7 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r print(" Run: mempalace init then mempalace mine ") raise SearchError(f"No palace found at {palace_path}") - where = _build_where_filter(wing, room) + where = build_where_filter(wing, room) try: kwargs = { @@ -127,7 +127,7 @@ def search_memories( "hint": "Run: mempalace init && mempalace mine ", } - where = _build_where_filter(wing, room) + where = build_where_filter(wing, room) try: kwargs = { @@ -148,6 +148,9 @@ def search_memories( hits = [] for doc, meta, dist in zip(docs, metas, dists): + # Filter on raw distance before rounding to avoid precision loss + if min_similarity > 0.0 and dist > min_similarity: + continue hits.append( { "text": doc, @@ -159,12 +162,6 @@ def search_memories( } ) - # Filter out results exceeding the distance threshold. - # ChromaDB default L2: lower distance = more similar. - # min_similarity=0.0 (default) disables filtering for backwards compat. - if min_similarity > 0.0: - hits = [h for h in hits if h["distance"] <= min_similarity] - return { "query": query, "filters": {"wing": wing, "room": room}, diff --git a/tests/test_exporter.py b/tests/test_exporter.py index fc96fdbbeb..0597ec1c2f 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -3,7 +3,6 @@ import tempfile from pathlib import Path -import chromadb import yaml from mempalace.miner import mine diff --git a/tests/test_miner.py b/tests/test_miner.py index fb6bcc3946..aa6c4df4e9 100644 --- a/tests/test_miner.py +++ b/tests/test_miner.py @@ -450,14 +450,15 @@ def test_chunk_text_many_chunks(): def test_chunk_text_preserves_content(): """All original content is covered by the union of chunks (nothing lost).""" - # Use only non-whitespace so chunk stripping doesn't drop characters at boundaries - content = "abcdefghij" * (CHUNK_SIZE * 3 // 10) + # Use position-unique tokens so we can verify each segment appears in a chunk + tokens = [f"[T{i:04d}]" for i in range(300)] + content = " ".join(tokens) chunks = chunk_text(content, "/fake/file.py") assert len(chunks) >= 2 - # Every character in the original must appear in at least one chunk + # Every unique token must appear in at least one chunk all_chunk_text = "".join(c["content"] for c in chunks) - for ch_pos, ch in enumerate(content): - assert ch in all_chunk_text, f"Character '{ch}' at position {ch_pos} not in any chunk" - # Stronger: the joined chunks should contain more characters than the original - # (due to overlap), confirming nothing is dropped + for token in tokens: + assert token in all_chunk_text, f"Token '{token}' not found in any chunk" + # The joined chunks should contain at least as many characters as the original + # (overlap means more, confirming nothing is dropped) assert len(all_chunk_text) >= len(content) From 8bd289a2d43aa27d552b7a7b76549bbd58e72165 Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 9 Apr 2026 21:19:48 -0700 Subject: [PATCH 15/50] fix: use venv Python in hook scripts for cross-project compatibility Stop and precompact hooks used bare `python3` which resolves to system Python in sessions outside the memorypalace project directory, causing `No module named mempalace` errors. Now uses the venv's Python with fallback to system python3. Co-Authored-By: Claude Opus 4.6 --- .claude-plugin/hooks/mempal-precompact-hook.sh | 7 ++++++- .claude-plugin/hooks/mempal-stop-hook.sh | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.claude-plugin/hooks/mempal-precompact-hook.sh b/.claude-plugin/hooks/mempal-precompact-hook.sh index 0ac46ddc4b..8e31371eb2 100644 --- a/.claude-plugin/hooks/mempal-precompact-hook.sh +++ b/.claude-plugin/hooks/mempal-precompact-hook.sh @@ -1,5 +1,10 @@ #!/bin/bash # MemPalace PreCompact Hook — thin wrapper calling Python CLI # All logic lives in mempalace.hooks_cli for cross-harness extensibility +# Uses the mempalace venv Python so this works from any project directory. +MEMPALACE_PYTHON="${HOME}/Projects/memorypalace/venv/bin/python3" +if [ ! -x "$MEMPALACE_PYTHON" ]; then + MEMPALACE_PYTHON="python3" # fallback to system python +fi INPUT=$(cat) -echo "$INPUT" | python3 -m mempalace hook run --hook precompact --harness claude-code +echo "$INPUT" | "$MEMPALACE_PYTHON" -m mempalace hook run --hook precompact --harness claude-code diff --git a/.claude-plugin/hooks/mempal-stop-hook.sh b/.claude-plugin/hooks/mempal-stop-hook.sh index cba3284961..d88bfab6fe 100644 --- a/.claude-plugin/hooks/mempal-stop-hook.sh +++ b/.claude-plugin/hooks/mempal-stop-hook.sh @@ -1,5 +1,10 @@ #!/bin/bash # MemPalace Stop Hook — thin wrapper calling Python CLI # All logic lives in mempalace.hooks_cli for cross-harness extensibility +# Uses the mempalace venv Python so this works from any project directory. +MEMPALACE_PYTHON="${HOME}/Projects/memorypalace/venv/bin/python3" +if [ ! -x "$MEMPALACE_PYTHON" ]; then + MEMPALACE_PYTHON="python3" # fallback to system python +fi INPUT=$(cat) -echo "$INPUT" | python3 -m mempalace hook run --hook stop --harness claude-code +echo "$INPUT" | "$MEMPALACE_PYTHON" -m mempalace hook run --hook stop --harness claude-code From d16ac4b0dd8a13919e6f3426149a3c06c64dfebe Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 9 Apr 2026 21:21:03 -0700 Subject: [PATCH 16/50] fix: portable Python resolution in hook scripts Replace hardcoded venv path with a resolution chain: 1. MEMPALACE_PYTHON env var (user override) 2. Plugin root's own venv (development installs) 3. System python3 (pip/pipx installs) Co-Authored-By: Claude Opus 4.6 --- .../hooks/mempal-precompact-hook.sh | 20 ++++++++++++++----- .claude-plugin/hooks/mempal-stop-hook.sh | 20 ++++++++++++++----- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/.claude-plugin/hooks/mempal-precompact-hook.sh b/.claude-plugin/hooks/mempal-precompact-hook.sh index 8e31371eb2..1d4b7bfca9 100644 --- a/.claude-plugin/hooks/mempal-precompact-hook.sh +++ b/.claude-plugin/hooks/mempal-precompact-hook.sh @@ -1,10 +1,20 @@ #!/bin/bash # MemPalace PreCompact Hook — thin wrapper calling Python CLI # All logic lives in mempalace.hooks_cli for cross-harness extensibility -# Uses the mempalace venv Python so this works from any project directory. -MEMPALACE_PYTHON="${HOME}/Projects/memorypalace/venv/bin/python3" -if [ ! -x "$MEMPALACE_PYTHON" ]; then - MEMPALACE_PYTHON="python3" # fallback to system python +# +# Python resolution order: +# 1. MEMPALACE_PYTHON env var (user override) +# 2. Plugin root's venv (development installs) +# 3. System python3 (pip install --user / pipx) +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PLUGIN_ROOT="$(dirname "$SCRIPT_DIR")" + +if [ -n "$MEMPALACE_PYTHON" ] && [ -x "$MEMPALACE_PYTHON" ]; then + PYTHON="$MEMPALACE_PYTHON" +elif [ -x "$PLUGIN_ROOT/venv/bin/python3" ]; then + PYTHON="$PLUGIN_ROOT/venv/bin/python3" +else + PYTHON="python3" fi INPUT=$(cat) -echo "$INPUT" | "$MEMPALACE_PYTHON" -m mempalace hook run --hook precompact --harness claude-code +echo "$INPUT" | "$PYTHON" -m mempalace hook run --hook precompact --harness claude-code diff --git a/.claude-plugin/hooks/mempal-stop-hook.sh b/.claude-plugin/hooks/mempal-stop-hook.sh index d88bfab6fe..2149c2175f 100644 --- a/.claude-plugin/hooks/mempal-stop-hook.sh +++ b/.claude-plugin/hooks/mempal-stop-hook.sh @@ -1,10 +1,20 @@ #!/bin/bash # MemPalace Stop Hook — thin wrapper calling Python CLI # All logic lives in mempalace.hooks_cli for cross-harness extensibility -# Uses the mempalace venv Python so this works from any project directory. -MEMPALACE_PYTHON="${HOME}/Projects/memorypalace/venv/bin/python3" -if [ ! -x "$MEMPALACE_PYTHON" ]; then - MEMPALACE_PYTHON="python3" # fallback to system python +# +# Python resolution order: +# 1. MEMPALACE_PYTHON env var (user override) +# 2. Plugin root's venv (development installs) +# 3. System python3 (pip install --user / pipx) +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PLUGIN_ROOT="$(dirname "$SCRIPT_DIR")" + +if [ -n "$MEMPALACE_PYTHON" ] && [ -x "$MEMPALACE_PYTHON" ]; then + PYTHON="$MEMPALACE_PYTHON" +elif [ -x "$PLUGIN_ROOT/venv/bin/python3" ]; then + PYTHON="$PLUGIN_ROOT/venv/bin/python3" +else + PYTHON="python3" fi INPUT=$(cat) -echo "$INPUT" | "$MEMPALACE_PYTHON" -m mempalace hook run --hook stop --harness claude-code +echo "$INPUT" | "$PYTHON" -m mempalace hook run --hook stop --harness claude-code From ad82c2d245441d53522d7ea4cb808453ac2e797d Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 9 Apr 2026 21:31:34 -0700 Subject: [PATCH 17/50] test: add MCP tool tests for get/list/update_drawer Covers basic CRUD, filtering, pagination, negative offset clamping, not-found errors, and no-op update detection. Addresses review comment on PR #493 requesting coverage for the new drawer tools. Co-Authored-By: Claude Opus 4.6 --- tests/test_mcp_server.py | 91 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 96fe80cd07..ecc7dff01a 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -321,6 +321,97 @@ def test_check_duplicate(self, monkeypatch, config, palace_path, seeded_collecti ) assert result["is_duplicate"] is False + def test_get_drawer(self, monkeypatch, config, palace_path, seeded_collection, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_get_drawer + + result = tool_get_drawer("drawer_proj_backend_aaa") + assert result["drawer_id"] == "drawer_proj_backend_aaa" + assert result["wing"] == "project" + assert result["room"] == "backend" + assert "JWT tokens" in result["content"] + + def test_get_drawer_not_found(self, monkeypatch, config, palace_path, seeded_collection, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_get_drawer + + result = tool_get_drawer("nonexistent_drawer") + assert "error" in result + + def test_list_drawers(self, monkeypatch, config, palace_path, seeded_collection, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_list_drawers + + result = tool_list_drawers() + assert result["count"] == 4 + assert len(result["drawers"]) == 4 + + def test_list_drawers_with_wing_filter(self, monkeypatch, config, palace_path, seeded_collection, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_list_drawers + + result = tool_list_drawers(wing="project") + assert result["count"] == 3 + assert all(d["wing"] == "project" for d in result["drawers"]) + + def test_list_drawers_with_room_filter(self, monkeypatch, config, palace_path, seeded_collection, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_list_drawers + + result = tool_list_drawers(wing="project", room="backend") + assert result["count"] == 2 + assert all(d["room"] == "backend" for d in result["drawers"]) + + def test_list_drawers_pagination(self, monkeypatch, config, palace_path, seeded_collection, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_list_drawers + + result = tool_list_drawers(limit=2, offset=0) + assert result["count"] == 2 + assert result["limit"] == 2 + assert result["offset"] == 0 + + def test_list_drawers_negative_offset_clamped(self, monkeypatch, config, palace_path, seeded_collection, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_list_drawers + + result = tool_list_drawers(offset=-5) + assert result["offset"] == 0 + + def test_update_drawer_content(self, monkeypatch, config, palace_path, seeded_collection, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_update_drawer, tool_get_drawer + + result = tool_update_drawer("drawer_proj_backend_aaa", content="Updated content about auth.") + assert result["success"] is True + + fetched = tool_get_drawer("drawer_proj_backend_aaa") + assert fetched["content"] == "Updated content about auth." + + def test_update_drawer_wing_and_room(self, monkeypatch, config, palace_path, seeded_collection, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_update_drawer + + result = tool_update_drawer("drawer_proj_backend_aaa", wing="new_wing", room="new_room") + assert result["success"] is True + assert result["wing"] == "new_wing" + assert result["room"] == "new_room" + + def test_update_drawer_not_found(self, monkeypatch, config, palace_path, seeded_collection, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_update_drawer + + result = tool_update_drawer("nonexistent_drawer", content="hello") + assert result["success"] is False + + def test_update_drawer_noop(self, monkeypatch, config, palace_path, seeded_collection, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_update_drawer + + result = tool_update_drawer("drawer_proj_backend_aaa") + assert result["success"] is True + assert result.get("noop") is True + # ── KG Tools ──────────────────────────────────────────────────────────── From d8c423b95db875fdd0024f8cc6bef6ebdb285134 Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 9 Apr 2026 21:35:17 -0700 Subject: [PATCH 18/50] docs: update expected test count to 573 Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 31d95bd2f3..bff04b8423 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,7 +20,7 @@ JP's fork of [milla-jovovich/mempalace](https://github.com/milla-jovovich/mempal ```bash source venv/bin/activate -python -m pytest tests/ -x -q # run tests (562 expected) +python -m pytest tests/ -x -q # run tests (573 expected) mempalace status # check palace state mempalace search "query" # test search python -m mempalace.mcp_server # run MCP server standalone From 6dc88910f8b39c96fd0aee1b284de7fa066b6267 Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 9 Apr 2026 21:36:47 -0700 Subject: [PATCH 19/50] refactor: extract _MAX_RESULTS constant for search/list limit cap Single source of truth for the limit ceiling (100) so operators can adjust without hunting through multiple clamp sites. Co-Authored-By: Claude Opus 4.6 --- mempalace/mcp_server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index bd75d2adef..68193e0b47 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -159,6 +159,7 @@ def _fetch_all_metadata(col, where=None): _metadata_cache = None _metadata_cache_time = 0 _METADATA_CACHE_TTL = 5.0 # seconds +_MAX_RESULTS = 100 # upper bound for search/list limit params def _get_cached_metadata(col, where=None): @@ -292,7 +293,7 @@ def tool_get_taxonomy(): def tool_search( query: str, limit: int = 5, wing: str = None, room: str = None, min_similarity: float = 1.5 ): - limit = max(1, min(limit, 100)) + limit = max(1, min(limit, _MAX_RESULTS)) return search_memories( query, palace_path=_config.palace_path, @@ -481,7 +482,7 @@ def tool_get_drawer(drawer_id: str): def tool_list_drawers(wing: str = None, room: str = None, limit: int = 20, offset: int = 0): """List drawers with pagination. Optional wing/room filter.""" - limit = max(1, min(limit, 100)) + limit = max(1, min(limit, _MAX_RESULTS)) offset = max(0, offset) col = _get_collection() if not col: From 2da4a651191e4a09d1253a58a016254d5474edec Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 10:03:08 -0700 Subject: [PATCH 20/50] docs: update expected test count to 573 Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index bff04b8423..35b5ee941f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -52,4 +52,4 @@ Ruff for linting (`ruff check`), line length 100, target Python 3.9. ## Testing -Always run `python -m pytest tests/ -x -q` after changes. 534 tests expected to pass. Benchmark and stress tests are excluded by default (use `-m benchmark` or `-m stress` to include). +Always run `python -m pytest tests/ -x -q` after changes. 573 tests expected to pass. Benchmark and stress tests are excluded by default (use `-m benchmark` or `-m stress` to include). From 0259154421351a7c0593d2bf53a8d05737ef2897 Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 10:12:50 -0700 Subject: [PATCH 21/50] =?UTF-8?q?feat:=20silent=20stop=20hook=20=E2=80=94?= =?UTF-8?q?=20save=20directly=20via=20Python=20API=20instead=20of=20blocki?= =?UTF-8?q?ng=20MCP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop hook no longer blocks Claude with MCP tool call instructions every 15 messages. Instead it saves a diary checkpoint directly via the Python API and shows a single-line terminal notification + desktop toast. Fixes milla-jovovich/mempalace#554 Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 2 +- mempalace/hooks_cli.py | 95 +++++++++++++++++++++++++++++++----- tests/test_hooks_cli.py | 103 ++++++++++++++++++++++++++++------------ 3 files changed, 155 insertions(+), 45 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 35b5ee941f..066e8e4579 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -52,4 +52,4 @@ Ruff for linting (`ruff check`), line length 100, target Python 3.9. ## Testing -Always run `python -m pytest tests/ -x -q` after changes. 573 tests expected to pass. Benchmark and stress tests are excluded by default (use `-m benchmark` or `-m stress` to include). +Always run `python -m pytest tests/ -x -q` after changes. 576 tests expected to pass. Benchmark and stress tests are excluded by default (use `-m benchmark` or `-m stress` to include). diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index 8709e2f080..65d7924b9d 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -17,15 +17,7 @@ SAVE_INTERVAL = 15 STATE_DIR = Path.home() / ".mempalace" / "hook_state" -STOP_BLOCK_REASON = ( - "AUTO-SAVE checkpoint. Save key topics, decisions, quotes, and code " - "from this session to MemPalace using the MCP tools:\n" - "1. Use mempalace_diary_write to save a session summary (what was discussed, " - "key decisions, current state of work).\n" - "2. Use mempalace_add_drawer for each important decision, quote, or code " - "snippet — place in the appropriate wing and room.\n" - "Use verbatim quotes where possible. Continue conversation after saving." -) +_RECENT_MSG_COUNT = 30 # how many recent user messages to summarize PRECOMPACT_BLOCK_REASON = ( "COMPACTION IMMINENT — detailed context will be lost. Save ALL topics, " @@ -92,6 +84,19 @@ def _output(data: dict): print(json.dumps(data, indent=2, ensure_ascii=False)) +def _notify(body: str, title: str = "MemPalace"): + """Send a desktop toast + short terminal line. Fails silently.""" + print(f"\033[38;5;141m\u2726 {title}\033[0m \033[2m{body}\033[0m", file=sys.stderr) + try: + subprocess.Popen( + ["notify-send", "--app-name=MemPalace", "--icon=brain", title, body], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + except OSError: + pass + + def _maybe_auto_ingest(): """If MEMPAL_DIR is set and exists, run mempalace mine in background.""" mempal_dir = os.environ.get("MEMPAL_DIR", "") @@ -108,6 +113,70 @@ def _maybe_auto_ingest(): pass +def _extract_recent_messages(transcript_path: str, count: int = _RECENT_MSG_COUNT) -> list[str]: + """Extract the last N user messages from a JSONL transcript.""" + path = Path(transcript_path).expanduser() + if not path.is_file(): + return [] + messages = [] + try: + with open(path, encoding="utf-8", errors="replace") as f: + for line in f: + try: + entry = json.loads(line) + msg = entry.get("message", {}) + if not isinstance(msg, dict) or msg.get("role") != "user": + continue + content = msg.get("content", "") + if isinstance(content, list): + content = " ".join( + b.get("text", "") for b in content if isinstance(b, dict) + ) + if not isinstance(content, str) or not content.strip(): + continue + if "" in content or "" in content: + continue + # Truncate long messages + text = content.strip()[:200] + messages.append(text) + except (json.JSONDecodeError, AttributeError): + pass + except OSError: + return [] + return messages[-count:] + + +def _save_diary_direct(transcript_path: str, session_id: str): + """Write a diary checkpoint directly via Python API (no MCP calls).""" + messages = _extract_recent_messages(transcript_path) + if not messages: + _log("No recent messages to save") + return + + # Build a compressed diary entry from recent conversation + now = datetime.now() + topics = "|".join(m[:80] for m in messages[-10:]) + entry = ( + f"CHECKPOINT:{now.strftime('%Y-%m-%d')}|session:{session_id}" + f"|msgs:{len(messages)}|recent:{topics}" + ) + + try: + from .mcp_server import tool_diary_write + result = tool_diary_write( + agent_name="session-hook", + entry=entry, + topic="checkpoint", + ) + if result.get("success"): + _log(f"Diary checkpoint saved: {result.get('entry_id', '?')}") + _notify(f"Checkpoint saved \u2014 {len(messages)} messages archived") + else: + _log(f"Diary checkpoint failed: {result.get('error', 'unknown')}") + except Exception as e: + _log(f"Diary checkpoint error: {e}") + + def _ingest_transcript(transcript_path: str): """Mine a Claude Code session transcript into the palace as a conversation.""" path = Path(transcript_path).expanduser() @@ -192,16 +261,16 @@ def hook_stop(data: dict, harness: str): _log(f"TRIGGERING SAVE at exchange {exchange_count}") - # Auto-ingest transcript into palace (background) + # Save diary checkpoint directly (no MCP, no terminal clutter) if transcript_path: + _save_diary_direct(transcript_path, session_id) _ingest_transcript(transcript_path) # Optional: auto-ingest project dir if MEMPAL_DIR is set _maybe_auto_ingest() - _output({"decision": "block", "reason": STOP_BLOCK_REASON}) - else: - _output({}) + # Never block — saving happens silently above + _output({}) def hook_session_start(data: dict, harness: str): diff --git a/tests/test_hooks_cli.py b/tests/test_hooks_cli.py index 5a1870e02f..5848691611 100644 --- a/tests/test_hooks_cli.py +++ b/tests/test_hooks_cli.py @@ -8,9 +8,9 @@ from mempalace.hooks_cli import ( SAVE_INTERVAL, - STOP_BLOCK_REASON, PRECOMPACT_BLOCK_REASON, _count_human_messages, + _extract_recent_messages, _log, _maybe_auto_ingest, _parse_harness_input, @@ -105,6 +105,40 @@ def test_count_malformed_json_lines(tmp_path): assert _count_human_messages(str(transcript)) == 1 +# --- _extract_recent_messages --- + + +def test_extract_recent_messages_basic(tmp_path): + transcript = tmp_path / "t.jsonl" + _write_transcript( + transcript, + [{"message": {"role": "user", "content": f"msg {i}"}} for i in range(5)], + ) + msgs = _extract_recent_messages(str(transcript), count=3) + assert len(msgs) == 3 + assert msgs[0] == "msg 2" + assert msgs[2] == "msg 4" + + +def test_extract_recent_messages_skips_commands(tmp_path): + transcript = tmp_path / "t.jsonl" + _write_transcript( + transcript, + [ + {"message": {"role": "user", "content": "real msg"}}, + {"message": {"role": "user", "content": "status"}}, + {"message": {"role": "user", "content": "hook"}}, + ], + ) + msgs = _extract_recent_messages(str(transcript)) + assert len(msgs) == 1 + assert msgs[0] == "real msg" + + +def test_extract_recent_messages_missing_file(): + assert _extract_recent_messages("/nonexistent.jsonl") == [] + + # --- hook_stop --- @@ -157,19 +191,21 @@ def test_stop_hook_passthrough_below_interval(tmp_path): assert result == {} -def test_stop_hook_blocks_at_interval(tmp_path): +def test_stop_hook_saves_silently_at_interval(tmp_path): transcript = tmp_path / "t.jsonl" _write_transcript( transcript, [{"message": {"role": "user", "content": f"msg {i}"}} for i in range(SAVE_INTERVAL)], ) - result = _capture_hook_output( - hook_stop, - {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}, - state_dir=tmp_path, - ) - assert result["decision"] == "block" - assert result["reason"] == STOP_BLOCK_REASON + with patch("mempalace.hooks_cli._save_diary_direct") as mock_save: + result = _capture_hook_output( + hook_stop, + {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}, + state_dir=tmp_path, + ) + # Never blocks — saves directly and passes through + assert result == {} + mock_save.assert_called_once_with(str(transcript), "test") def test_stop_hook_tracks_save_point(tmp_path): @@ -180,13 +216,16 @@ def test_stop_hook_tracks_save_point(tmp_path): ) data = {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)} - # First call blocks - result = _capture_hook_output(hook_stop, data, state_dir=tmp_path) - assert result["decision"] == "block" + # First call saves silently + with patch("mempalace.hooks_cli._save_diary_direct"): + result = _capture_hook_output(hook_stop, data, state_dir=tmp_path) + assert result == {} - # Second call with same count passes through (already saved) - result = _capture_hook_output(hook_stop, data, state_dir=tmp_path) + # Second call with same count skips save (already saved) + with patch("mempalace.hooks_cli._save_diary_direct") as mock_save: + result = _capture_hook_output(hook_stop, data, state_dir=tmp_path) assert result == {} + mock_save.assert_not_called() # --- hook_session_start --- @@ -295,12 +334,13 @@ def test_stop_hook_oserror_on_last_save_read(tmp_path): ) # Write invalid content to last save file (tmp_path / "test_last_save").write_text("not_a_number") - result = _capture_hook_output( - hook_stop, - {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}, - state_dir=tmp_path, - ) - assert result["decision"] == "block" + with patch("mempalace.hooks_cli._save_diary_direct"): + result = _capture_hook_output( + hook_stop, + {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}, + state_dir=tmp_path, + ) + assert result == {} def test_stop_hook_oserror_on_write(tmp_path): @@ -315,17 +355,18 @@ def bad_write_text(*args, **kwargs): raise OSError("disk full") with patch("mempalace.hooks_cli.STATE_DIR", tmp_path): - with patch.object(Path, "write_text", bad_write_text): - result = _capture_hook_output( - hook_stop, - { - "session_id": "test", - "stop_hook_active": False, - "transcript_path": str(transcript), - }, - state_dir=tmp_path, - ) - assert result["decision"] == "block" + with patch("mempalace.hooks_cli._save_diary_direct"): + with patch.object(Path, "write_text", bad_write_text): + result = _capture_hook_output( + hook_stop, + { + "session_id": "test", + "stop_hook_active": False, + "transcript_path": str(transcript), + }, + state_dir=tmp_path, + ) + assert result == {} # --- hook_precompact with MEMPAL_DIR --- From 1efa3ba90be617770f7aac0827cee972910d2359 Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 10:20:56 -0700 Subject: [PATCH 22/50] feat: configurable hook settings via MCP tool + config Add hooks.silent_save and hooks.desktop_toast to config.json, readable via new mempalace_hook_settings MCP tool (get/set). Stop hook checks config to decide between silent direct save vs legacy blocking MCP. Restore STOP_BLOCK_REASON for legacy mode. Toast is opt-in via config. Co-Authored-By: Claude Opus 4.6 --- mempalace/config.py | 21 +++++++++++++ mempalace/hooks_cli.py | 70 ++++++++++++++++++++++++++++------------- mempalace/mcp_server.py | 68 +++++++++++++++++++++++++++++++++++++++ tests/test_hooks_cli.py | 2 +- 4 files changed, 138 insertions(+), 23 deletions(-) diff --git a/mempalace/config.py b/mempalace/config.py index e20e22ad86..81d8f2223d 100644 --- a/mempalace/config.py +++ b/mempalace/config.py @@ -173,6 +173,27 @@ def hall_keywords(self): """Mapping of hall names to keyword lists.""" return self._file_config.get("hall_keywords", DEFAULT_HALL_KEYWORDS) + @property + def hook_silent_save(self): + """Whether the stop hook saves directly (True) or blocks for MCP calls (False).""" + return self._file_config.get("hooks", {}).get("silent_save", True) + + @property + def hook_desktop_toast(self): + """Whether the stop hook shows a desktop notification via notify-send.""" + return self._file_config.get("hooks", {}).get("desktop_toast", False) + + def set_hook_setting(self, key: str, value: bool): + """Update a hook setting and write config to disk.""" + if "hooks" not in self._file_config: + self._file_config["hooks"] = {} + self._file_config["hooks"][key] = value + try: + with open(self._config_file, "w") as f: + json.dump(self._file_config, f, indent=2) + except OSError: + pass + @property def chunk_size(self): """Characters per drawer chunk.""" diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index 65d7924b9d..8fd3c8971a 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -19,6 +19,16 @@ _RECENT_MSG_COUNT = 30 # how many recent user messages to summarize +STOP_BLOCK_REASON = ( + "AUTO-SAVE checkpoint. Save key topics, decisions, quotes, and code " + "from this session to MemPalace using the MCP tools:\n" + "1. Use mempalace_diary_write to save a session summary (what was discussed, " + "key decisions, current state of work).\n" + "2. Use mempalace_add_drawer for each important decision, quote, or code " + "snippet — place in the appropriate wing and room.\n" + "Use verbatim quotes where possible. Continue conversation after saving." +) + PRECOMPACT_BLOCK_REASON = ( "COMPACTION IMMINENT — detailed context will be lost. Save ALL topics, " "decisions, quotes, code, and important context to MemPalace using MCP tools:\n" @@ -84,17 +94,18 @@ def _output(data: dict): print(json.dumps(data, indent=2, ensure_ascii=False)) -def _notify(body: str, title: str = "MemPalace"): - """Send a desktop toast + short terminal line. Fails silently.""" +def _notify(body: str, title: str = "MemPalace", toast: bool = False): + """Send a terminal line and optionally a desktop toast. Fails silently.""" print(f"\033[38;5;141m\u2726 {title}\033[0m \033[2m{body}\033[0m", file=sys.stderr) - try: - subprocess.Popen( - ["notify-send", "--app-name=MemPalace", "--icon=brain", title, body], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - except OSError: - pass + if toast: + try: + subprocess.Popen( + ["notify-send", "--app-name=MemPalace", "--icon=brain", title, body], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + except OSError: + pass def _maybe_auto_ingest(): @@ -146,7 +157,7 @@ def _extract_recent_messages(transcript_path: str, count: int = _RECENT_MSG_COUN return messages[-count:] -def _save_diary_direct(transcript_path: str, session_id: str): +def _save_diary_direct(transcript_path: str, session_id: str, toast: bool = False): """Write a diary checkpoint directly via Python API (no MCP calls).""" messages = _extract_recent_messages(transcript_path) if not messages: @@ -170,7 +181,7 @@ def _save_diary_direct(transcript_path: str, session_id: str): ) if result.get("success"): _log(f"Diary checkpoint saved: {result.get('entry_id', '?')}") - _notify(f"Checkpoint saved \u2014 {len(messages)} messages archived") + _notify(f"Checkpoint saved \u2014 {len(messages)} messages archived", toast=toast) else: _log(f"Diary checkpoint failed: {result.get('error', 'unknown')}") except Exception as e: @@ -261,16 +272,31 @@ def hook_stop(data: dict, harness: str): _log(f"TRIGGERING SAVE at exchange {exchange_count}") - # Save diary checkpoint directly (no MCP, no terminal clutter) - if transcript_path: - _save_diary_direct(transcript_path, session_id) - _ingest_transcript(transcript_path) - - # Optional: auto-ingest project dir if MEMPAL_DIR is set - _maybe_auto_ingest() - - # Never block — saving happens silently above - _output({}) + # Read hook settings from config + from .config import MempalaceConfig + try: + config = MempalaceConfig() + silent = config.hook_silent_save + toast = config.hook_desktop_toast + except Exception: + silent = True + toast = False + + if silent: + # Save directly via Python API — no MCP calls, no terminal clutter + if transcript_path: + _save_diary_direct(transcript_path, session_id, toast=toast) + _ingest_transcript(transcript_path) + _maybe_auto_ingest() + _output({}) + else: + # Legacy: block and ask Claude to save via MCP tools + if transcript_path: + _ingest_transcript(transcript_path) + _maybe_auto_ingest() + _output({"decision": "block", "reason": STOP_BLOCK_REASON}) + else: + _output({}) def hook_session_start(data: dict, harness: str): diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 68193e0b47..da083a1f6c 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -772,6 +772,53 @@ def tool_diary_read(agent_name: str, last_n: int = 10): return {"error": str(e)} +# ==================== SETTINGS TOOLS ==================== + + +def tool_hook_settings(silent_save: bool = None, desktop_toast: bool = None): + """ + Get or set hook behavior settings. + + - silent_save: True = stop hook saves directly (no MCP clutter), + False = legacy blocking MCP calls. Default: True. + - desktop_toast: True = show notify-send desktop toast on save, + False = terminal-only notification. Default: False. + + Call with no arguments to see current settings. + """ + from .config import MempalaceConfig + + try: + config = MempalaceConfig() + except Exception as e: + return {"success": False, "error": str(e)} + + changed = [] + if silent_save is not None: + config.set_hook_setting("silent_save", silent_save) + changed.append(f"silent_save → {silent_save}") + if desktop_toast is not None: + config.set_hook_setting("desktop_toast", desktop_toast) + changed.append(f"desktop_toast → {desktop_toast}") + + # Re-read to return current state + try: + config = MempalaceConfig() + except Exception: + pass + + result = { + "success": True, + "settings": { + "silent_save": config.hook_silent_save, + "desktop_toast": config.hook_desktop_toast, + }, + } + if changed: + result["updated"] = changed + return result + + # ==================== MCP PROTOCOL ==================== TOOLS = { @@ -1081,6 +1128,27 @@ def tool_diary_read(agent_name: str, last_n: int = 10): }, "handler": tool_diary_read, }, + "mempalace_hook_settings": { + "description": ( + "Get or set hook behavior. silent_save: True = save directly " + "(no MCP clutter), False = legacy blocking. desktop_toast: " + "True = show desktop notification. Call with no args to view." + ), + "input_schema": { + "type": "object", + "properties": { + "silent_save": { + "type": "boolean", + "description": "True = silent direct save, False = blocking MCP calls", + }, + "desktop_toast": { + "type": "boolean", + "description": "True = show desktop toast via notify-send", + }, + }, + }, + "handler": tool_hook_settings, + }, } diff --git a/tests/test_hooks_cli.py b/tests/test_hooks_cli.py index 5848691611..6187984100 100644 --- a/tests/test_hooks_cli.py +++ b/tests/test_hooks_cli.py @@ -205,7 +205,7 @@ def test_stop_hook_saves_silently_at_interval(tmp_path): ) # Never blocks — saves directly and passes through assert result == {} - mock_save.assert_called_once_with(str(transcript), "test") + mock_save.assert_called_once_with(str(transcript), "test", toast=False) def test_stop_hook_tracks_save_point(tmp_path): From e9e06c8a27c0e09af849a51fbdd8c5198557792c Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 10:25:25 -0700 Subject: [PATCH 23/50] fix: use short block reason for terminal visibility instead of stderr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit stderr from hook subprocesses doesn't reach the Claude Code terminal. Block with a one-liner notification after the direct save completes — save already happened, Claude just continues. Co-Authored-By: Claude Opus 4.6 --- mempalace/hooks_cli.py | 49 +++++++++++++++++++++++++---------------- tests/test_hooks_cli.py | 24 +++++++++++--------- 2 files changed, 43 insertions(+), 30 deletions(-) diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index 8fd3c8971a..ae8a04174d 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -94,18 +94,16 @@ def _output(data: dict): print(json.dumps(data, indent=2, ensure_ascii=False)) -def _notify(body: str, title: str = "MemPalace", toast: bool = False): - """Send a terminal line and optionally a desktop toast. Fails silently.""" - print(f"\033[38;5;141m\u2726 {title}\033[0m \033[2m{body}\033[0m", file=sys.stderr) - if toast: - try: - subprocess.Popen( - ["notify-send", "--app-name=MemPalace", "--icon=brain", title, body], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - except OSError: - pass +def _desktop_toast(body: str, title: str = "MemPalace"): + """Send a desktop notification via notify-send. Fails silently.""" + try: + subprocess.Popen( + ["notify-send", "--app-name=MemPalace", "--icon=brain", title, body], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + except OSError: + pass def _maybe_auto_ingest(): @@ -157,12 +155,15 @@ def _extract_recent_messages(transcript_path: str, count: int = _RECENT_MSG_COUN return messages[-count:] -def _save_diary_direct(transcript_path: str, session_id: str, toast: bool = False): - """Write a diary checkpoint directly via Python API (no MCP calls).""" +def _save_diary_direct(transcript_path: str, session_id: str, toast: bool = False) -> int: + """Write a diary checkpoint directly via Python API (no MCP calls). + + Returns the number of messages archived. + """ messages = _extract_recent_messages(transcript_path) if not messages: _log("No recent messages to save") - return + return 0 # Build a compressed diary entry from recent conversation now = datetime.now() @@ -181,11 +182,13 @@ def _save_diary_direct(transcript_path: str, session_id: str, toast: bool = Fals ) if result.get("success"): _log(f"Diary checkpoint saved: {result.get('entry_id', '?')}") - _notify(f"Checkpoint saved \u2014 {len(messages)} messages archived", toast=toast) + if toast: + _desktop_toast(f"Checkpoint saved \u2014 {len(messages)} messages archived") else: _log(f"Diary checkpoint failed: {result.get('error', 'unknown')}") except Exception as e: _log(f"Diary checkpoint error: {e}") + return len(messages) def _ingest_transcript(transcript_path: str): @@ -283,12 +286,20 @@ def hook_stop(data: dict, harness: str): toast = False if silent: - # Save directly via Python API — no MCP calls, no terminal clutter + # Save directly via Python API — no MCP calls + msg_count = 0 if transcript_path: - _save_diary_direct(transcript_path, session_id, toast=toast) + msg_count = _save_diary_direct(transcript_path, session_id, toast=toast) _ingest_transcript(transcript_path) _maybe_auto_ingest() - _output({}) + # Block with short notification so it appears in terminal + _output({ + "decision": "block", + "reason": ( + f"\u2726 MemPalace checkpoint saved — {msg_count} messages archived. " + "Continue what you were doing." + ), + }) else: # Legacy: block and ask Claude to save via MCP tools if transcript_path: diff --git a/tests/test_hooks_cli.py b/tests/test_hooks_cli.py index 6187984100..47d19b98da 100644 --- a/tests/test_hooks_cli.py +++ b/tests/test_hooks_cli.py @@ -197,14 +197,16 @@ def test_stop_hook_saves_silently_at_interval(tmp_path): transcript, [{"message": {"role": "user", "content": f"msg {i}"}} for i in range(SAVE_INTERVAL)], ) - with patch("mempalace.hooks_cli._save_diary_direct") as mock_save: + with patch("mempalace.hooks_cli._save_diary_direct", return_value=15) as mock_save: result = _capture_hook_output( hook_stop, {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}, state_dir=tmp_path, ) - # Never blocks — saves directly and passes through - assert result == {} + # Blocks with short notification (save already happened) + assert result["decision"] == "block" + assert "\u2726" in result["reason"] + assert "Continue" in result["reason"] mock_save.assert_called_once_with(str(transcript), "test", toast=False) @@ -216,12 +218,12 @@ def test_stop_hook_tracks_save_point(tmp_path): ) data = {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)} - # First call saves silently - with patch("mempalace.hooks_cli._save_diary_direct"): + # First call saves and blocks with notification + with patch("mempalace.hooks_cli._save_diary_direct", return_value=15): result = _capture_hook_output(hook_stop, data, state_dir=tmp_path) - assert result == {} + assert result["decision"] == "block" - # Second call with same count skips save (already saved) + # Second call with same count passes through (already saved) with patch("mempalace.hooks_cli._save_diary_direct") as mock_save: result = _capture_hook_output(hook_stop, data, state_dir=tmp_path) assert result == {} @@ -334,13 +336,13 @@ def test_stop_hook_oserror_on_last_save_read(tmp_path): ) # Write invalid content to last save file (tmp_path / "test_last_save").write_text("not_a_number") - with patch("mempalace.hooks_cli._save_diary_direct"): + with patch("mempalace.hooks_cli._save_diary_direct", return_value=15): result = _capture_hook_output( hook_stop, {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}, state_dir=tmp_path, ) - assert result == {} + assert result["decision"] == "block" def test_stop_hook_oserror_on_write(tmp_path): @@ -355,7 +357,7 @@ def bad_write_text(*args, **kwargs): raise OSError("disk full") with patch("mempalace.hooks_cli.STATE_DIR", tmp_path): - with patch("mempalace.hooks_cli._save_diary_direct"): + with patch("mempalace.hooks_cli._save_diary_direct", return_value=15): with patch.object(Path, "write_text", bad_write_text): result = _capture_hook_output( hook_stop, @@ -366,7 +368,7 @@ def bad_write_text(*args, **kwargs): }, state_dir=tmp_path, ) - assert result == {} + assert result["decision"] == "block" # --- hook_precompact with MEMPAL_DIR --- From 5ebfcd4a345495a9f2b67747510989aeab14242e Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 10:28:12 -0700 Subject: [PATCH 24/50] =?UTF-8?q?fix:=20make=20silent=20stop=20hook=20full?= =?UTF-8?q?y=20silent=20=E2=80=94=20no=20block,=20no=20error=20label?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude Code shows all hook blocks as "Stop hook error:" with no info level available. Return {} for truly invisible saves. Co-Authored-By: Claude Opus 4.6 --- mempalace/hooks_cli.py | 9 +-------- tests/test_hooks_cli.py | 14 ++++++-------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index ae8a04174d..4f784eb69f 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -292,14 +292,7 @@ def hook_stop(data: dict, harness: str): msg_count = _save_diary_direct(transcript_path, session_id, toast=toast) _ingest_transcript(transcript_path) _maybe_auto_ingest() - # Block with short notification so it appears in terminal - _output({ - "decision": "block", - "reason": ( - f"\u2726 MemPalace checkpoint saved — {msg_count} messages archived. " - "Continue what you were doing." - ), - }) + _output({}) else: # Legacy: block and ask Claude to save via MCP tools if transcript_path: diff --git a/tests/test_hooks_cli.py b/tests/test_hooks_cli.py index 47d19b98da..91f2196e81 100644 --- a/tests/test_hooks_cli.py +++ b/tests/test_hooks_cli.py @@ -203,10 +203,8 @@ def test_stop_hook_saves_silently_at_interval(tmp_path): {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}, state_dir=tmp_path, ) - # Blocks with short notification (save already happened) - assert result["decision"] == "block" - assert "\u2726" in result["reason"] - assert "Continue" in result["reason"] + # Saves silently — no block, no terminal clutter + assert result == {} mock_save.assert_called_once_with(str(transcript), "test", toast=False) @@ -218,10 +216,10 @@ def test_stop_hook_tracks_save_point(tmp_path): ) data = {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)} - # First call saves and blocks with notification + # First call saves silently with patch("mempalace.hooks_cli._save_diary_direct", return_value=15): result = _capture_hook_output(hook_stop, data, state_dir=tmp_path) - assert result["decision"] == "block" + assert result == {} # Second call with same count passes through (already saved) with patch("mempalace.hooks_cli._save_diary_direct") as mock_save: @@ -342,7 +340,7 @@ def test_stop_hook_oserror_on_last_save_read(tmp_path): {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}, state_dir=tmp_path, ) - assert result["decision"] == "block" + assert result == {} def test_stop_hook_oserror_on_write(tmp_path): @@ -368,7 +366,7 @@ def bad_write_text(*args, **kwargs): }, state_dir=tmp_path, ) - assert result["decision"] == "block" + assert result == {} # --- hook_precompact with MEMPAL_DIR --- From 930b6d8891ff0d3692727042a16310592e6279cc Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 10:34:52 -0700 Subject: [PATCH 25/50] feat: checkpoint ack MCP tool for minimal terminal confirmation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hook saves directly, then blocks asking Claude to call mempalace_checkpoint_ack — a zero-param tool returning one line like "✦ Journal entry filed — 30 messages tucked into drawers". Replaces both the verbose MCP diary/drawer calls and the invisible silent mode with a single clean terminal line. Co-Authored-By: Claude Opus 4.6 --- mempalace/hooks_cli.py | 19 +++++++++++++++---- mempalace/mcp_server.py | 19 +++++++++++++++++++ tests/test_hooks_cli.py | 13 +++++++------ 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index 4f784eb69f..3cfeb8fef8 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -182,6 +182,15 @@ def _save_diary_direct(transcript_path: str, session_id: str, toast: bool = Fals ) if result.get("success"): _log(f"Diary checkpoint saved: {result.get('entry_id', '?')}") + # Write state for ack tool to read + try: + ack_file = STATE_DIR / "last_checkpoint" + ack_file.write_text( + json.dumps({"msgs": len(messages), "ts": now.isoformat()}), + encoding="utf-8", + ) + except OSError: + pass if toast: _desktop_toast(f"Checkpoint saved \u2014 {len(messages)} messages archived") else: @@ -286,13 +295,15 @@ def hook_stop(data: dict, harness: str): toast = False if silent: - # Save directly via Python API — no MCP calls - msg_count = 0 + # Save directly via Python API, then ask Claude to call ack tool if transcript_path: - msg_count = _save_diary_direct(transcript_path, session_id, toast=toast) + _save_diary_direct(transcript_path, session_id, toast=toast) _ingest_transcript(transcript_path) _maybe_auto_ingest() - _output({}) + _output({ + "decision": "block", + "reason": "Call mempalace_checkpoint_ack now. Do not say anything else.", + }) else: # Legacy: block and ask Claude to save via MCP tools if transcript_path: diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index da083a1f6c..5c9604772c 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -819,6 +819,20 @@ def tool_hook_settings(silent_save: bool = None, desktop_toast: bool = None): return result +def tool_checkpoint_ack(): + """Acknowledge the latest silent checkpoint. Returns a short summary.""" + state_dir = Path.home() / ".mempalace" / "hook_state" + ack_file = state_dir / "last_checkpoint" + if not ack_file.is_file(): + return "\u2726 Palace quiet \u2014 no recent journal entry" + try: + data = json.loads(ack_file.read_text(encoding="utf-8")) + ack_file.unlink(missing_ok=True) + return f"\u2726 Journal entry filed \u2014 {data.get('msgs', '?')} messages tucked into drawers" + except (json.JSONDecodeError, OSError): + return "\u2726 Journal entry filed in the palace" + + # ==================== MCP PROTOCOL ==================== TOOLS = { @@ -1149,6 +1163,11 @@ def tool_hook_settings(silent_save: bool = None, desktop_toast: bool = None): }, "handler": tool_hook_settings, }, + "mempalace_checkpoint_ack": { + "description": "Acknowledge a palace checkpoint. Call when the stop hook asks you to.", + "input_schema": {"type": "object", "properties": {}}, + "handler": tool_checkpoint_ack, + }, } diff --git a/tests/test_hooks_cli.py b/tests/test_hooks_cli.py index 91f2196e81..2cb83ee471 100644 --- a/tests/test_hooks_cli.py +++ b/tests/test_hooks_cli.py @@ -203,8 +203,9 @@ def test_stop_hook_saves_silently_at_interval(tmp_path): {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}, state_dir=tmp_path, ) - # Saves silently — no block, no terminal clutter - assert result == {} + # Blocks with ack instruction (save already happened) + assert result["decision"] == "block" + assert "checkpoint_ack" in result["reason"] mock_save.assert_called_once_with(str(transcript), "test", toast=False) @@ -216,10 +217,10 @@ def test_stop_hook_tracks_save_point(tmp_path): ) data = {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)} - # First call saves silently + # First call saves and blocks with ack instruction with patch("mempalace.hooks_cli._save_diary_direct", return_value=15): result = _capture_hook_output(hook_stop, data, state_dir=tmp_path) - assert result == {} + assert result["decision"] == "block" # Second call with same count passes through (already saved) with patch("mempalace.hooks_cli._save_diary_direct") as mock_save: @@ -340,7 +341,7 @@ def test_stop_hook_oserror_on_last_save_read(tmp_path): {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}, state_dir=tmp_path, ) - assert result == {} + assert result["decision"] == "block" def test_stop_hook_oserror_on_write(tmp_path): @@ -366,7 +367,7 @@ def bad_write_text(*args, **kwargs): }, state_dir=tmp_path, ) - assert result == {} + assert result["decision"] == "block" # --- hook_precompact with MEMPAL_DIR --- From 995aed3c714f8d77a68c550e39ba98296077d73f Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 10:39:22 -0700 Subject: [PATCH 26/50] =?UTF-8?q?fix:=20fully=20silent=20stop=20hook=20?= =?UTF-8?q?=E2=80=94=20no=20block,=20no=20error=20label?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude Code labels all hook blocks as "Stop hook error:" with no way to customize. Go fully silent instead — save happens invisibly. Co-Authored-By: Claude Opus 4.6 --- mempalace/hooks_cli.py | 5 +---- tests/test_hooks_cli.py | 13 ++++++------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index 3cfeb8fef8..03dfbece47 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -300,10 +300,7 @@ def hook_stop(data: dict, harness: str): _save_diary_direct(transcript_path, session_id, toast=toast) _ingest_transcript(transcript_path) _maybe_auto_ingest() - _output({ - "decision": "block", - "reason": "Call mempalace_checkpoint_ack now. Do not say anything else.", - }) + _output({}) else: # Legacy: block and ask Claude to save via MCP tools if transcript_path: diff --git a/tests/test_hooks_cli.py b/tests/test_hooks_cli.py index 2cb83ee471..26d8ee759b 100644 --- a/tests/test_hooks_cli.py +++ b/tests/test_hooks_cli.py @@ -203,9 +203,8 @@ def test_stop_hook_saves_silently_at_interval(tmp_path): {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}, state_dir=tmp_path, ) - # Blocks with ack instruction (save already happened) - assert result["decision"] == "block" - assert "checkpoint_ack" in result["reason"] + # Saves silently — no block + assert result == {} mock_save.assert_called_once_with(str(transcript), "test", toast=False) @@ -217,10 +216,10 @@ def test_stop_hook_tracks_save_point(tmp_path): ) data = {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)} - # First call saves and blocks with ack instruction + # First call saves silently with patch("mempalace.hooks_cli._save_diary_direct", return_value=15): result = _capture_hook_output(hook_stop, data, state_dir=tmp_path) - assert result["decision"] == "block" + assert result == {} # Second call with same count passes through (already saved) with patch("mempalace.hooks_cli._save_diary_direct") as mock_save: @@ -341,7 +340,7 @@ def test_stop_hook_oserror_on_last_save_read(tmp_path): {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}, state_dir=tmp_path, ) - assert result["decision"] == "block" + assert result == {} def test_stop_hook_oserror_on_write(tmp_path): @@ -367,7 +366,7 @@ def bad_write_text(*args, **kwargs): }, state_dir=tmp_path, ) - assert result["decision"] == "block" + assert result == {} # --- hook_precompact with MEMPAL_DIR --- From 22094840c9cf061f8ba234254fb61a641e9f1e59 Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 11:11:32 -0700 Subject: [PATCH 27/50] feat: systemMessage notification for stop hook checkpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop hook now outputs {"systemMessage": "✦ N messages filed away"} which Claude Code renders as a visible one-line terminal notification — no MCP tool call needed. Also renames checkpoint_ack → memories_filed_away and fixes MCP server to silently ignore all notifications/ methods per spec. Co-Authored-By: Claude Opus 4.6 --- mempalace/hooks_cli.py | 10 +++++++--- mempalace/mcp_server.py | 24 ++++++++++++++++-------- tests/test_hooks_cli.py | 12 ++++++------ 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index 03dfbece47..685c3bc26c 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -295,12 +295,16 @@ def hook_stop(data: dict, harness: str): toast = False if silent: - # Save directly via Python API, then ask Claude to call ack tool + # Save directly via Python API — systemMessage renders in terminal + saved = 0 if transcript_path: - _save_diary_direct(transcript_path, session_id, toast=toast) + saved = _save_diary_direct(transcript_path, session_id, toast=toast) _ingest_transcript(transcript_path) _maybe_auto_ingest() - _output({}) + if saved > 0: + _output({"systemMessage": f"\u2726 {saved} messages filed away"}) + else: + _output({}) else: # Legacy: block and ask Claude to save via MCP tools if transcript_path: diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 5c9604772c..8572be534f 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -819,18 +819,22 @@ def tool_hook_settings(silent_save: bool = None, desktop_toast: bool = None): return result -def tool_checkpoint_ack(): +def tool_memories_filed_away(): """Acknowledge the latest silent checkpoint. Returns a short summary.""" state_dir = Path.home() / ".mempalace" / "hook_state" ack_file = state_dir / "last_checkpoint" if not ack_file.is_file(): - return "\u2726 Palace quiet \u2014 no recent journal entry" + return {"palace": "quiet", "message": "No recent journal entry"} try: data = json.loads(ack_file.read_text(encoding="utf-8")) ack_file.unlink(missing_ok=True) - return f"\u2726 Journal entry filed \u2014 {data.get('msgs', '?')} messages tucked into drawers" + msgs = data.get("msgs", "?") + return { + "message": f"\u2726 {msgs} messages tucked into drawers", + "timestamp": data.get("ts", ""), + } except (json.JSONDecodeError, OSError): - return "\u2726 Journal entry filed in the palace" + return {"message": "\u2726 Journal entry filed in the palace"} # ==================== MCP PROTOCOL ==================== @@ -1163,10 +1167,10 @@ def tool_checkpoint_ack(): }, "handler": tool_hook_settings, }, - "mempalace_checkpoint_ack": { - "description": "Acknowledge a palace checkpoint. Call when the stop hook asks you to.", + "memories_filed_away": { + "description": "Check if a recent palace checkpoint was saved. Returns message count and timestamp.", "input_schema": {"type": "object", "properties": {}}, - "handler": tool_checkpoint_ack, + "handler": tool_memories_filed_away, }, } @@ -1200,7 +1204,8 @@ def handle_request(request): "serverInfo": {"name": "mempalace", "version": __version__}, }, } - elif method == "notifications/initialized": + elif method.startswith("notifications/"): + # Notifications (no id) never get a response per JSON-RPC spec return None elif method == "tools/list": return { @@ -1248,6 +1253,9 @@ def handle_request(request): "error": {"code": -32000, "message": "Internal tool error"}, } + # Notifications (missing id) must never get a response + if req_id is None: + return None return { "jsonrpc": "2.0", "id": req_id, diff --git a/tests/test_hooks_cli.py b/tests/test_hooks_cli.py index 26d8ee759b..5e408b1d74 100644 --- a/tests/test_hooks_cli.py +++ b/tests/test_hooks_cli.py @@ -203,8 +203,8 @@ def test_stop_hook_saves_silently_at_interval(tmp_path): {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}, state_dir=tmp_path, ) - # Saves silently — no block - assert result == {} + # Saves silently — systemMessage notification, no block + assert result == {"systemMessage": "\u2726 15 messages filed away"} mock_save.assert_called_once_with(str(transcript), "test", toast=False) @@ -216,10 +216,10 @@ def test_stop_hook_tracks_save_point(tmp_path): ) data = {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)} - # First call saves silently + # First call saves silently with systemMessage notification with patch("mempalace.hooks_cli._save_diary_direct", return_value=15): result = _capture_hook_output(hook_stop, data, state_dir=tmp_path) - assert result == {} + assert result == {"systemMessage": "\u2726 15 messages filed away"} # Second call with same count passes through (already saved) with patch("mempalace.hooks_cli._save_diary_direct") as mock_save: @@ -340,7 +340,7 @@ def test_stop_hook_oserror_on_last_save_read(tmp_path): {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}, state_dir=tmp_path, ) - assert result == {} + assert result == {"systemMessage": "\u2726 15 messages filed away"} def test_stop_hook_oserror_on_write(tmp_path): @@ -366,7 +366,7 @@ def bad_write_text(*args, **kwargs): }, state_dir=tmp_path, ) - assert result == {} + assert result == {"systemMessage": "\u2726 15 messages filed away"} # --- hook_precompact with MEMPAL_DIR --- From 363ef374b2d0f93a0af56aef180ee9ceccf81347 Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 11:14:31 -0700 Subject: [PATCH 28/50] fix: advance save marker only after successful checkpoint Copilot review caught that hook_stop() updated the last-save marker before _save_diary_direct() ran. If save failed, the marker would still advance and skip the next checkpoint. Move marker write after save confirms success. Also updates CLAUDE.md test count and hook docs. Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 6 +++--- mempalace/hooks_cli.py | 15 +++++++++------ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 066e8e4579..b32265a7c1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,7 +20,7 @@ JP's fork of [milla-jovovich/mempalace](https://github.com/milla-jovovich/mempal ```bash source venv/bin/activate -python -m pytest tests/ -x -q # run tests (573 expected) +python -m pytest tests/ -x -q # run tests (576 expected) mempalace status # check palace state mempalace search "query" # test search python -m mempalace.mcp_server # run MCP server standalone @@ -36,7 +36,7 @@ Ruff for linting (`ruff check`), line length 100, target Python 3.9. 4. **perf: batch ChromaDB writes** — one upsert per file instead of per chunk in both miners 5. **fix: entity detector STOPWORDS** — 73 technical terms added (Handler, Node, Service, etc.) 6. **feat: similarity threshold** — `min_similarity` parameter in search, default 1.5 L2 distance in MCP -7. **fix: hooks_cli** — stop/precompact hooks now instruct AI to use mempalace MCP tools, auto-ingest transcripts +7. **feat: hooks_cli** — stop hook saves directly via Python API with systemMessage notification, precompact blocks for AI-driven save, auto-ingest transcripts ## Upstream PRs @@ -47,7 +47,7 @@ Ruff for linting (`ruff check`), line length 100, target Python 3.9. - **Claude Code plugin**: installed at user scope via marketplace - **MCP server**: global user scope — available in all projects -- **Stop hook**: fires every 15 messages, saves to palace via MCP tools + auto-ingests transcript +- **Stop hook**: fires every 15 messages, saves directly via Python API + systemMessage notification + auto-ingests transcript - **PreCompact hook**: emergency save before context compaction ## Testing diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index 685c3bc26c..a9b83adefa 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -276,12 +276,6 @@ def hook_stop(data: dict, harness: str): _log(f"Session {session_id}: {exchange_count} exchanges, {since_last} since last save") if since_last >= SAVE_INTERVAL and exchange_count > 0: - # Update last save point - try: - last_save_file.write_text(str(exchange_count), encoding="utf-8") - except OSError: - pass - _log(f"TRIGGERING SAVE at exchange {exchange_count}") # Read hook settings from config @@ -301,12 +295,21 @@ def hook_stop(data: dict, harness: str): saved = _save_diary_direct(transcript_path, session_id, toast=toast) _ingest_transcript(transcript_path) _maybe_auto_ingest() + # Only advance save marker after successful save if saved > 0: + try: + last_save_file.write_text(str(exchange_count), encoding="utf-8") + except OSError: + pass _output({"systemMessage": f"\u2726 {saved} messages filed away"}) else: _output({}) else: # Legacy: block and ask Claude to save via MCP tools + try: + last_save_file.write_text(str(exchange_count), encoding="utf-8") + except OSError: + pass if transcript_path: _ingest_transcript(transcript_path) _maybe_auto_ingest() From 094677b53f426dd0ede87d6785ac9ab72b043cf2 Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 11:20:02 -0700 Subject: [PATCH 29/50] feat: palace-themed notification with conversation themes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop hook now extracts topic keywords from recent messages and displays them in the notification: "✦ 10 memories woven into the palace — hooks, notifications, MCP". Stopword filtering keeps only distinctive terms. Co-Authored-By: Claude Opus 4.6 --- mempalace/hooks_cli.py | 54 +++++++++++++++++++++++++++++++++++------ tests/test_hooks_cli.py | 24 +++++++++++------- 2 files changed, 61 insertions(+), 17 deletions(-) diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index a9b83adefa..615390ba35 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -155,15 +155,44 @@ def _extract_recent_messages(transcript_path: str, count: int = _RECENT_MSG_COUN return messages[-count:] -def _save_diary_direct(transcript_path: str, session_id: str, toast: bool = False) -> int: +_THEME_STOPWORDS = frozenset( + "the a an and or but in on at to for of is it i me my you your we our " + "this that with from by was were be been are not no yes can do did don't " + "will would should could have has had let's let just also like so if then " + "ok okay sure yeah hey hi here there what when where how why which some " + "all any each every about into out up down over after before between " + "get got make made need want use used using check look see run try " + "know think right now still already really very much more most too " + "file files code one two new first last next thing things way well".split() +) + + +def _extract_themes(messages: list[str], max_themes: int = 3) -> list[str]: + """Pull 2-3 distinctive topic words from recent messages.""" + from collections import Counter + words: Counter[str] = Counter() + for msg in messages: + for word in msg.lower().split(): + # Strip punctuation, keep words 4+ chars + clean = word.strip(".,;:!?\"'`()[]{}#<>/\\-_=+@$%^&*~") + if len(clean) >= 4 and clean not in _THEME_STOPWORDS and clean.isalpha(): + words[clean] += 1 + return [w for w, _ in words.most_common(max_themes)] + + +def _save_diary_direct( + transcript_path: str, session_id: str, toast: bool = False, +) -> dict: """Write a diary checkpoint directly via Python API (no MCP calls). - Returns the number of messages archived. + Returns {"count": N, "themes": [...]} on success, {"count": 0} on failure. """ messages = _extract_recent_messages(transcript_path) if not messages: _log("No recent messages to save") - return 0 + return {"count": 0} + + themes = _extract_themes(messages) # Build a compressed diary entry from recent conversation now = datetime.now() @@ -193,11 +222,12 @@ def _save_diary_direct(transcript_path: str, session_id: str, toast: bool = Fals pass if toast: _desktop_toast(f"Checkpoint saved \u2014 {len(messages)} messages archived") + return {"count": len(messages), "themes": themes} else: _log(f"Diary checkpoint failed: {result.get('error', 'unknown')}") except Exception as e: _log(f"Diary checkpoint error: {e}") - return len(messages) + return {"count": 0} def _ingest_transcript(transcript_path: str): @@ -290,18 +320,26 @@ def hook_stop(data: dict, harness: str): if silent: # Save directly via Python API — systemMessage renders in terminal - saved = 0 + result = {"count": 0} if transcript_path: - saved = _save_diary_direct(transcript_path, session_id, toast=toast) + result = _save_diary_direct(transcript_path, session_id, toast=toast) _ingest_transcript(transcript_path) _maybe_auto_ingest() # Only advance save marker after successful save - if saved > 0: + count = result.get("count", 0) + if count > 0: try: last_save_file.write_text(str(exchange_count), encoding="utf-8") except OSError: pass - _output({"systemMessage": f"\u2726 {saved} messages filed away"}) + themes = result.get("themes", []) + if themes: + tag = " \u2014 " + ", ".join(themes) + else: + tag = "" + _output({ + "systemMessage": f"\u2726 {count} memories woven into the palace{tag}", + }) else: _output({}) else: diff --git a/tests/test_hooks_cli.py b/tests/test_hooks_cli.py index 5e408b1d74..c6c5655755 100644 --- a/tests/test_hooks_cli.py +++ b/tests/test_hooks_cli.py @@ -197,14 +197,16 @@ def test_stop_hook_saves_silently_at_interval(tmp_path): transcript, [{"message": {"role": "user", "content": f"msg {i}"}} for i in range(SAVE_INTERVAL)], ) - with patch("mempalace.hooks_cli._save_diary_direct", return_value=15) as mock_save: + save_result = {"count": 15, "themes": ["hooks", "notifications"]} + with patch("mempalace.hooks_cli._save_diary_direct", return_value=save_result) as mock_save: result = _capture_hook_output( hook_stop, {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}, state_dir=tmp_path, ) - # Saves silently — systemMessage notification, no block - assert result == {"systemMessage": "\u2726 15 messages filed away"} + # Saves silently — systemMessage notification with themes, no block + assert result["systemMessage"].startswith("\u2726 15 memories woven into the palace") + assert "hooks" in result["systemMessage"] mock_save.assert_called_once_with(str(transcript), "test", toast=False) @@ -217,9 +219,10 @@ def test_stop_hook_tracks_save_point(tmp_path): data = {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)} # First call saves silently with systemMessage notification - with patch("mempalace.hooks_cli._save_diary_direct", return_value=15): + save_result = {"count": 15, "themes": ["hooks"]} + with patch("mempalace.hooks_cli._save_diary_direct", return_value=save_result): result = _capture_hook_output(hook_stop, data, state_dir=tmp_path) - assert result == {"systemMessage": "\u2726 15 messages filed away"} + assert "systemMessage" in result # Second call with same count passes through (already saved) with patch("mempalace.hooks_cli._save_diary_direct") as mock_save: @@ -334,13 +337,15 @@ def test_stop_hook_oserror_on_last_save_read(tmp_path): ) # Write invalid content to last save file (tmp_path / "test_last_save").write_text("not_a_number") - with patch("mempalace.hooks_cli._save_diary_direct", return_value=15): + save_result = {"count": 15, "themes": ["testing"]} + with patch("mempalace.hooks_cli._save_diary_direct", return_value=save_result): result = _capture_hook_output( hook_stop, {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}, state_dir=tmp_path, ) - assert result == {"systemMessage": "\u2726 15 messages filed away"} + assert "systemMessage" in result + assert "15 memories" in result["systemMessage"] def test_stop_hook_oserror_on_write(tmp_path): @@ -354,8 +359,9 @@ def test_stop_hook_oserror_on_write(tmp_path): def bad_write_text(*args, **kwargs): raise OSError("disk full") + save_result = {"count": 15, "themes": []} with patch("mempalace.hooks_cli.STATE_DIR", tmp_path): - with patch("mempalace.hooks_cli._save_diary_direct", return_value=15): + with patch("mempalace.hooks_cli._save_diary_direct", return_value=save_result): with patch.object(Path, "write_text", bad_write_text): result = _capture_hook_output( hook_stop, @@ -366,7 +372,7 @@ def bad_write_text(*args, **kwargs): }, state_dir=tmp_path, ) - assert result == {"systemMessage": "\u2726 15 messages filed away"} + assert "systemMessage" in result # --- hook_precompact with MEMPAL_DIR --- From 362a8e1fb7273544b432dd8a436aee699acb2c26 Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 11:31:30 -0700 Subject: [PATCH 30/50] =?UTF-8?q?fix:=20address=20PR=20review=20feedback?= =?UTF-8?q?=20=E2=80=94=20rename,=20tests,=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename min_similarity → max_distance (searcher + MCP schema), keep backwards compat alias in MCP tool handler - Fix ingest comment accuracy (async/best-effort, not guaranteed) - Add notification protocol tests (all notifications/* return None, unknown methods without id return None) - 578 tests passing Co-Authored-By: Claude Opus 4.6 --- mempalace/hooks_cli.py | 3 ++- mempalace/mcp_server.py | 11 +++++++---- mempalace/searcher.py | 6 +++--- tests/test_mcp_server.py | 20 ++++++++++++++++++++ 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index 615390ba35..57c0bbf08d 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -378,7 +378,8 @@ def hook_precompact(data: dict, harness: str): _log(f"PRE-COMPACT triggered for session {session_id}") transcript_path = parsed["transcript_path"] - # Auto-ingest transcript before compaction (so conversation lands in palace) + # Best-effort background ingest — spawns async subprocess, not guaranteed + # to complete before compaction but gives the palace a head start if transcript_path: _ingest_transcript(transcript_path) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 8572be534f..61644f2862 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -291,16 +291,19 @@ def tool_get_taxonomy(): def tool_search( - query: str, limit: int = 5, wing: str = None, room: str = None, min_similarity: float = 1.5 + query: str, limit: int = 5, wing: str = None, room: str = None, + max_distance: float = 1.5, min_similarity: float = None, ): limit = max(1, min(limit, _MAX_RESULTS)) + # Backwards compat: accept old name + dist = min_similarity if min_similarity is not None else max_distance return search_memories( query, palace_path=_config.palace_path, wing=wing, room=room, n_results=limit, - min_similarity=min_similarity, + max_distance=dist, ) @@ -986,7 +989,7 @@ def tool_memories_filed_away(): "handler": tool_graph_stats, }, "mempalace_search": { - "description": "Semantic search. Returns verbatim drawer content with similarity scores. Results with distance > min_similarity are filtered out (L2 distance: lower = more similar).", + "description": "Semantic search. Returns verbatim drawer content with similarity scores. Results with L2 distance > max_distance are filtered out (lower = more similar).", "input_schema": { "type": "object", "properties": { @@ -994,7 +997,7 @@ def tool_memories_filed_away(): "limit": {"type": "integer", "description": "Max results (default 5)", "minimum": 1, "maximum": 100}, "wing": {"type": "string", "description": "Filter by wing (optional)"}, "room": {"type": "string", "description": "Filter by room (optional)"}, - "min_similarity": { + "max_distance": { "type": "number", "description": "Max L2 distance threshold — results further than this are dropped. Lower = stricter. Default 1.5 filters clearly irrelevant results. Set to 0 to disable filtering.", }, diff --git a/mempalace/searcher.py b/mempalace/searcher.py index 40c5ad24f3..1b3bbcabbd 100644 --- a/mempalace/searcher.py +++ b/mempalace/searcher.py @@ -100,7 +100,7 @@ def search_memories( wing: str = None, room: str = None, n_results: int = 5, - min_similarity: float = 0.0, + max_distance: float = 0.0, ) -> dict: """Programmatic search — returns a dict instead of printing. @@ -112,7 +112,7 @@ def search_memories( wing: Optional wing filter. room: Optional room filter. n_results: Max results to return. - min_similarity: Max L2 (Euclidean) distance threshold. ChromaDB uses + max_distance: Max L2 (Euclidean) distance threshold. ChromaDB uses L2 distance by default — 0 = identical, larger = less similar. Results with distance > this value are filtered out. A value of 0.0 disables filtering. Typical useful range: 0.5–1.5. @@ -149,7 +149,7 @@ def search_memories( hits = [] for doc, meta, dist in zip(docs, metas, dists): # Filter on raw distance before rounding to avoid precision loss - if min_similarity > 0.0 and dist > min_similarity: + if max_distance > 0.0 and dist > max_distance: continue hits.append( { diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index ecc7dff01a..d777ec39ac 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -138,6 +138,26 @@ def test_unknown_method(self): resp = handle_request({"method": "unknown/method", "id": 4, "params": {}}) assert resp["error"]["code"] == -32601 + def test_any_notification_returns_none(self): + """All notifications/* methods should return None (no response).""" + from mempalace.mcp_server import handle_request + + for method in [ + "notifications/initialized", + "notifications/cancelled", + "notifications/progress", + "notifications/roots/list_changed", + ]: + resp = handle_request({"method": method, "params": {}}) + assert resp is None, f"{method} should return None" + + def test_unknown_method_no_id_returns_none(self): + """Messages without id (notifications) must never get a response.""" + from mempalace.mcp_server import handle_request + + resp = handle_request({"method": "unknown/thing", "params": {}}) + assert resp is None + def test_tools_call_dispatches(self, monkeypatch, config, palace_path, seeded_kg): _patch_mcp_server(monkeypatch, config, seeded_kg) from mempalace.mcp_server import handle_request From 44545bd7142d8304a81921a8bb419e051d0f396e Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 11:51:05 -0700 Subject: [PATCH 31/50] fix: harden MCP server from GPT 5.4 Pro review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Guard method.startswith() against None/missing method key - Rename tool to mempalace_memories_filed_away (restore namespace prefix) - Normalize return shapes: all branches return {status, message, count, timestamp} - Clean up corrupt last_checkpoint file on JSONDecodeError - Document legacy save marker as best-effort in hook_stop - Update CLAUDE.md fork item 6 for max_distance rename - Add backwards-compat test for min_similarity shim - Add tests for malformed method (None, missing, None+id) - Test count: 576 → 580 Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 4 ++-- mempalace/hooks_cli.py | 4 +++- mempalace/mcp_server.py | 15 +++++++++------ tests/test_mcp_server.py | 30 ++++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index b32265a7c1..ff1084c9ec 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,7 +20,7 @@ JP's fork of [milla-jovovich/mempalace](https://github.com/milla-jovovich/mempal ```bash source venv/bin/activate -python -m pytest tests/ -x -q # run tests (576 expected) +python -m pytest tests/ -x -q # run tests (580 expected) mempalace status # check palace state mempalace search "query" # test search python -m mempalace.mcp_server # run MCP server standalone @@ -35,7 +35,7 @@ Ruff for linting (`ruff check`), line length 100, target Python 3.9. 3. **fix: MCP server** — search limit capped [1,100], status/taxonomy tools paginated past 10K, duplicate cache decls removed 4. **perf: batch ChromaDB writes** — one upsert per file instead of per chunk in both miners 5. **fix: entity detector STOPWORDS** — 73 technical terms added (Handler, Node, Service, etc.) -6. **feat: similarity threshold** — `min_similarity` parameter in search, default 1.5 L2 distance in MCP +6. **feat: similarity threshold** — `max_distance` parameter in search (renamed from `min_similarity`), default 1.5 L2 distance in MCP 7. **feat: hooks_cli** — stop hook saves directly via Python API with systemMessage notification, precompact blocks for AI-driven save, auto-ingest transcripts ## Upstream PRs diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index 57c0bbf08d..056b3ae763 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -343,7 +343,9 @@ def hook_stop(data: dict, harness: str): else: _output({}) else: - # Legacy: block and ask Claude to save via MCP tools + # Legacy: block and ask Claude to save via MCP tools. + # Marker advances before confirmed save — best-effort; if Claude + # fails to save, the checkpoint is lost but won't retry endlessly. try: last_save_file.write_text(str(exchange_count), encoding="utf-8") except OSError: diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 61644f2862..cd99c570ea 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -827,17 +827,20 @@ def tool_memories_filed_away(): state_dir = Path.home() / ".mempalace" / "hook_state" ack_file = state_dir / "last_checkpoint" if not ack_file.is_file(): - return {"palace": "quiet", "message": "No recent journal entry"} + return {"status": "quiet", "message": "No recent journal entry", "count": 0, "timestamp": None} try: data = json.loads(ack_file.read_text(encoding="utf-8")) ack_file.unlink(missing_ok=True) - msgs = data.get("msgs", "?") + msgs = data.get("msgs", 0) return { + "status": "ok", "message": f"\u2726 {msgs} messages tucked into drawers", - "timestamp": data.get("ts", ""), + "count": msgs, + "timestamp": data.get("ts", None), } except (json.JSONDecodeError, OSError): - return {"message": "\u2726 Journal entry filed in the palace"} + ack_file.unlink(missing_ok=True) + return {"status": "error", "message": "\u2726 Journal entry filed in the palace", "count": 0, "timestamp": None} # ==================== MCP PROTOCOL ==================== @@ -1170,7 +1173,7 @@ def tool_memories_filed_away(): }, "handler": tool_hook_settings, }, - "memories_filed_away": { + "mempalace_memories_filed_away": { "description": "Check if a recent palace checkpoint was saved. Returns message count and timestamp.", "input_schema": {"type": "object", "properties": {}}, "handler": tool_memories_filed_away, @@ -1187,7 +1190,7 @@ def tool_memories_filed_away(): def handle_request(request): - method = request.get("method", "") + method = request.get("method") or "" params = request.get("params", {}) req_id = request.get("id") diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index d777ec39ac..f7d84b0dca 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -158,6 +158,22 @@ def test_unknown_method_no_id_returns_none(self): resp = handle_request({"method": "unknown/thing", "params": {}}) assert resp is None + def test_malformed_method_none(self): + """method=None or missing should not crash.""" + from mempalace.mcp_server import handle_request + + # Explicit None + resp = handle_request({"method": None, "params": {}}) + assert resp is None # no id → no response + + # Missing method entirely + resp = handle_request({"params": {}}) + assert resp is None + + # method=None with id → should return error, not crash + resp = handle_request({"method": None, "id": 99, "params": {}}) + assert resp["error"]["code"] == -32601 + def test_tools_call_dispatches(self, monkeypatch, config, palace_path, seeded_kg): _patch_mcp_server(monkeypatch, config, seeded_kg) from mempalace.mcp_server import handle_request @@ -272,6 +288,20 @@ def test_search_with_room_filter(self, monkeypatch, config, palace_path, seeded_ result = tool_search(query="database", room="backend") assert all(r["room"] == "backend" for r in result["results"]) + def test_search_min_similarity_backwards_compat(self, monkeypatch, config, palace_path, seeded_collection, kg): + """Old min_similarity param still works via backwards-compat shim.""" + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_search + + # Old name should work + result = tool_search(query="JWT", min_similarity=1.5) + assert "results" in result + + # Old name takes precedence when both provided + result_strict = tool_search(query="JWT", max_distance=999.0, min_similarity=0.01) + result_loose = tool_search(query="JWT", max_distance=0.01, min_similarity=999.0) + assert len(result_strict["results"]) <= len(result_loose["results"]) + # ── Write Tools ───────────────────────────────────────────────────────── From fd0b73dc19cc10fcfb9ef41e0788771f3567e30a Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 11:57:59 -0700 Subject: [PATCH 32/50] fix: resolve venv python for background ingest subprocesses Hook subprocesses used sys.executable which may be the system python when Claude Code invokes hooks outside the venv. Now resolves the venv's bin/python from __file__ so chromadb and other deps are found. Co-Authored-By: Claude Opus 4.6 --- mempalace/hooks_cli.py | 26 +++++++++++++++++++++++--- tests/test_hooks_cli.py | 16 ++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index 056b3ae763..10bdec96f4 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -17,6 +17,26 @@ SAVE_INTERVAL = 15 STATE_DIR = Path.home() / ".mempalace" / "hook_state" + +def _mempalace_python() -> str: + """Return the python interpreter that has mempalace installed. + + When hooks are invoked by Claude Code, sys.executable may be the system + python which lacks chromadb and other deps. Walk up from this file to + find the venv's python, falling back to sys.executable. + """ + # This file lives at /lib/pythonX.Y/site-packages/mempalace/hooks_cli.py + # or /mempalace/hooks_cli.py (editable install). + # In either case, the venv bin/python sits alongside the venv's lib/. + venv_bin = Path(__file__).resolve().parents[3] / "bin" / "python" + if venv_bin.is_file(): + return str(venv_bin) + # Editable install: project root has venv/ dir + project_venv = Path(__file__).resolve().parents[1] / "venv" / "bin" / "python" + if project_venv.is_file(): + return str(project_venv) + return sys.executable + _RECENT_MSG_COUNT = 30 # how many recent user messages to summarize STOP_BLOCK_REASON = ( @@ -114,7 +134,7 @@ def _maybe_auto_ingest(): log_path = STATE_DIR / "hook.log" with open(log_path, "a") as log_f: subprocess.Popen( - [sys.executable, "-m", "mempalace", "mine", mempal_dir], + [_mempalace_python(), "-m", "mempalace", "mine", mempal_dir], stdout=log_f, stderr=log_f, ) @@ -249,7 +269,7 @@ def _ingest_transcript(transcript_path: str): with open(log_path, "a") as log_f: subprocess.Popen( [ - sys.executable, "-m", "mempalace", "mine", + _mempalace_python(), "-m", "mempalace", "mine", str(path.parent), "--mode", "convos", "--wing", "sessions", ], @@ -392,7 +412,7 @@ def hook_precompact(data: dict, harness: str): log_path = STATE_DIR / "hook.log" with open(log_path, "a") as log_f: subprocess.run( - [sys.executable, "-m", "mempalace", "mine", mempal_dir], + [_mempalace_python(), "-m", "mempalace", "mine", mempal_dir], stdout=log_f, stderr=log_f, timeout=60, diff --git a/tests/test_hooks_cli.py b/tests/test_hooks_cli.py index c6c5655755..22ecb3162c 100644 --- a/tests/test_hooks_cli.py +++ b/tests/test_hooks_cli.py @@ -13,6 +13,7 @@ _extract_recent_messages, _log, _maybe_auto_ingest, + _mempalace_python, _parse_harness_input, _sanitize_session_id, hook_stop, @@ -22,6 +23,21 @@ ) +# --- _mempalace_python --- + + +def test_mempalace_python_returns_string(): + result = _mempalace_python() + assert isinstance(result, str) + assert "python" in result + + +def test_mempalace_python_finds_venv(): + """Should resolve to a venv python, not bare sys.executable.""" + result = _mempalace_python() + assert "venv" in result or "site-packages" in result or result.endswith("python") + + # --- _sanitize_session_id --- From 8020bdd2a3c6b54580a754187679955e68e58548 Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 12:05:37 -0700 Subject: [PATCH 33/50] fix: cache invalidation in add/delete drawer + doc comments - tool_add_drawer and tool_delete_drawer now invalidate _metadata_cache so status/taxonomy reflect changes immediately (review item 3) - Document English-only stopword limitation in _extract_themes - Document venv/ assumption in _mempalace_python editable-install path Co-Authored-By: Claude Opus 4.6 --- mempalace/hooks_cli.py | 7 +++++-- mempalace/mcp_server.py | 4 ++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index 7e2135ca88..59c4ac4958 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -31,7 +31,7 @@ def _mempalace_python() -> str: venv_bin = Path(__file__).resolve().parents[3] / "bin" / "python" if venv_bin.is_file(): return str(venv_bin) - # Editable install: project root has venv/ dir + # Editable install: assumes project root has a venv/ sibling to mempalace/ project_venv = Path(__file__).resolve().parents[1] / "venv" / "bin" / "python" if project_venv.is_file(): return str(project_venv) @@ -196,7 +196,10 @@ def _extract_recent_messages(transcript_path: str, count: int = _RECENT_MSG_COUN def _extract_themes(messages: list[str], max_themes: int = 3) -> list[str]: - """Pull 2-3 distinctive topic words from recent messages.""" + """Pull 2-3 distinctive topic words from recent messages. + + Note: stopword list is English-only; non-English corpora will produce noisy themes. + """ from collections import Counter words: Counter[str] = Counter() for msg in messages: diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index cd99c570ea..e188daeb44 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -378,6 +378,7 @@ def tool_add_drawer( wing: str, room: str, content: str, source_file: str = None, added_by: str = "mcp" ): """File verbatim content into a wing/room. Checks for duplicates first.""" + global _metadata_cache try: wing = sanitize_name(wing, "wing") room = sanitize_name(room, "room") @@ -426,6 +427,7 @@ def tool_add_drawer( } ], ) + _metadata_cache = None logger.info(f"Filed drawer: {drawer_id} → {wing}/{room}") return {"success": True, "drawer_id": drawer_id, "wing": wing, "room": room} except Exception as e: @@ -434,6 +436,7 @@ def tool_add_drawer( def tool_delete_drawer(drawer_id: str): """Delete a single drawer by ID.""" + global _metadata_cache col = _get_collection() if not col: return _no_palace() @@ -455,6 +458,7 @@ def tool_delete_drawer(drawer_id: str): try: col.delete(ids=[drawer_id]) + _metadata_cache = None logger.info(f"Deleted drawer: {drawer_id}") return {"success": True, "drawer_id": drawer_id} except Exception as e: From 8ed7987993f55e8bb00ac4b741f3044e825be7b1 Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 12:08:26 -0700 Subject: [PATCH 34/50] =?UTF-8?q?fix:=20address=20Copilot=20review=20?= =?UTF-8?q?=E2=80=94=20env=20var=20override,=20similarity=20clamp,=20utf-8?= =?UTF-8?q?=20config?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _mempalace_python: honor MEMPALACE_PYTHON env var as first priority - searcher: clamp similarity to [0, 1] so L2 distances > 1 don't go negative - config: use encoding="utf-8" + ensure_ascii=False for config writes Co-Authored-By: Claude Opus 4.6 --- mempalace/config.py | 4 ++-- mempalace/hooks_cli.py | 12 +++++++++--- mempalace/searcher.py | 4 ++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/mempalace/config.py b/mempalace/config.py index 81d8f2223d..ec5c7ea553 100644 --- a/mempalace/config.py +++ b/mempalace/config.py @@ -189,8 +189,8 @@ def set_hook_setting(self, key: str, value: bool): self._file_config["hooks"] = {} self._file_config["hooks"][key] = value try: - with open(self._config_file, "w") as f: - json.dump(self._file_config, f, indent=2) + with open(self._config_file, "w", encoding="utf-8") as f: + json.dump(self._file_config, f, indent=2, ensure_ascii=False) except OSError: pass diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index 59c4ac4958..d2efa13584 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -22,12 +22,18 @@ def _mempalace_python() -> str: """Return the python interpreter that has mempalace installed. When hooks are invoked by Claude Code, sys.executable may be the system - python which lacks chromadb and other deps. Walk up from this file to - find the venv's python, falling back to sys.executable. + python which lacks chromadb and other deps. Resolution order: + 1. MEMPALACE_PYTHON env var (explicit override) + 2. Venv python from package install path + 3. Editable install: venv/ sibling to mempalace/ + 4. sys.executable fallback """ + # Honor explicit override (used by shell hook wrappers) + env_python = os.environ.get("MEMPALACE_PYTHON", "") + if env_python and os.path.isfile(env_python) and os.access(env_python, os.X_OK): + return env_python # This file lives at /lib/pythonX.Y/site-packages/mempalace/hooks_cli.py # or /mempalace/hooks_cli.py (editable install). - # In either case, the venv bin/python sits alongside the venv's lib/. venv_bin = Path(__file__).resolve().parents[3] / "bin" / "python" if venv_bin.is_file(): return str(venv_bin) diff --git a/mempalace/searcher.py b/mempalace/searcher.py index 1b3bbcabbd..91e388b1ba 100644 --- a/mempalace/searcher.py +++ b/mempalace/searcher.py @@ -76,7 +76,7 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r print(f"{'=' * 60}\n") for i, (doc, meta, dist) in enumerate(zip(docs, metas, dists), 1): - similarity = round(1 - dist, 3) + similarity = round(max(0.0, 1 - dist), 3) source = Path(meta.get("source_file", "?")).name wing_name = meta.get("wing", "?") room_name = meta.get("room", "?") @@ -157,7 +157,7 @@ def search_memories( "wing": meta.get("wing", "unknown"), "room": meta.get("room", "unknown"), "source_file": Path(meta.get("source_file", "?")).name, - "similarity": round(1 - dist, 3), + "similarity": round(max(0.0, 1 - dist), 3), "distance": round(dist, 4), } ) From 9e64fe054dacc4e0afd5d5002c42c7ee2e01175e Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 14:26:39 -0700 Subject: [PATCH 35/50] feat: junk file filter, purge command, status pagination fix - Add SKIP_PATTERNS and JUNK_FILE_SIZE (500KB) to miner to exclude minified JS/CSS, bundles, source maps, lockfiles, and large generated files from palace mining - Add `mempalace purge --wing/--room` CLI command for batch deletion with confirmation prompt and batched ChromaDB deletes - Fix status display: use col.count() + paginated metadata fetch instead of hard-capped 10K get(), add thousands separators to output - Update CLAUDE.md test count to 615 Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 4 +-- mempalace/cli.py | 69 ++++++++++++++++++++++++++++++++++++++++++++++ mempalace/miner.py | 46 +++++++++++++++++++++++++------ 3 files changed, 109 insertions(+), 10 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ff1084c9ec..0bb9dd51ed 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,7 +20,7 @@ JP's fork of [milla-jovovich/mempalace](https://github.com/milla-jovovich/mempal ```bash source venv/bin/activate -python -m pytest tests/ -x -q # run tests (580 expected) +python -m pytest tests/ -x -q # run tests (615 expected) mempalace status # check palace state mempalace search "query" # test search python -m mempalace.mcp_server # run MCP server standalone @@ -52,4 +52,4 @@ Ruff for linting (`ruff check`), line length 100, target Python 3.9. ## Testing -Always run `python -m pytest tests/ -x -q` after changes. 576 tests expected to pass. Benchmark and stress tests are excluded by default (use `-m benchmark` or `-m stress` to include). +Always run `python -m pytest tests/ -x -q` after changes. 615 tests expected to pass. Benchmark and stress tests are excluded by default (use `-m benchmark` or `-m stress` to include). diff --git a/mempalace/cli.py b/mempalace/cli.py index b569409cf9..024e3341fc 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -176,6 +176,69 @@ def cmd_migrate(args): migrate(palace_path=palace_path, dry_run=args.dry_run) +def cmd_purge(args): + """Delete drawers by wing and/or room.""" + import chromadb + + palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path + try: + client = chromadb.PersistentClient(path=palace_path) + col = client.get_collection("mempalace_drawers") + except Exception: + print(f"\n No palace found at {palace_path}") + return + + where = {} + if args.wing and args.room: + where = {"$and": [{"wing": args.wing}, {"room": args.room}]} + elif args.wing: + where = {"wing": args.wing} + elif args.room: + where = {"room": args.room} + else: + print(" Error: specify --wing and/or --room") + return + + # ChromaDB doesn't have a count(where=...), so paginate to count + match_count = 0 + offset = 0 + while True: + batch = col.get(limit=10000, offset=offset, where=where, include=[]) + if not batch["ids"]: + break + match_count += len(batch["ids"]) + offset += len(batch["ids"]) + + if match_count == 0: + label = f"wing={args.wing}" if args.wing else "" + if args.room: + label = f"{label} room={args.room}" if label else f"room={args.room}" + print(f"\n No drawers found matching {label}\n") + return + + label = f"wing={args.wing}" if args.wing else "" + if args.room: + label = f"{label} room={args.room}" if label else f"room={args.room}" + print(f"\n Found {match_count:,} drawers matching {label}") + + if not args.yes: + confirm = input(f" Delete {match_count:,} drawers? [y/N] ").strip().lower() + if confirm not in ("y", "yes"): + print(" Aborted.") + return + + deleted = 0 + while True: + batch = col.get(limit=10000, where=where, include=[]) + if not batch["ids"]: + break + col.delete(ids=batch["ids"]) + deleted += len(batch["ids"]) + print(f" Deleted {deleted:,} / {match_count:,}...", flush=True) + + print(f"\n Purged {deleted:,} drawers. Remaining: {col.count():,}\n") + + def cmd_status(args): from .miner import status @@ -583,6 +646,11 @@ def main(): help="Show what would be migrated without changing anything", ) + p_purge = sub.add_parser("purge", help="Delete drawers by wing and/or room") + p_purge.add_argument("--wing", help="Wing to purge") + p_purge.add_argument("--room", help="Room to purge") + p_purge.add_argument("--yes", "-y", action="store_true", help="Skip confirmation prompt") + sub.add_parser("status", help="Show what's been filed") args = parser.parse_args() @@ -619,6 +687,7 @@ def main(): "wake-up": cmd_wakeup, "repair": cmd_repair, "migrate": cmd_migrate, + "purge": cmd_purge, "status": cmd_status, } dispatch[args.command](args) diff --git a/mempalace/miner.py b/mempalace/miner.py index 0cfe43155b..a230616cb7 100644 --- a/mempalace/miner.py +++ b/mempalace/miner.py @@ -55,6 +55,22 @@ "package-lock.json", } +# Patterns for files that are technically text but useless for semantic search. +# Matched against the filename (case-insensitive). +SKIP_PATTERNS = [ + ".min.js", # minified JS (jquery.min.js, etc.) + ".min.css", # minified CSS + ".bundle.js", # bundled JS + ".chunk.js", # webpack chunks + ".map", # source maps + "-lock.json", # lockfiles (yarn.lock handled by extension) + ".lock", # lockfiles +] + +# Files larger than this are likely dumps/generated — skip them even if under MAX_FILE_SIZE. +# This catches database dumps, large SQL exports, huge JSON fixtures, etc. +JUNK_FILE_SIZE = 500 * 1024 # 500 KB — most useful source files are well under this + CHUNK_SIZE = 800 # chars per drawer CHUNK_OVERLAP = 100 # overlap between chunks MIN_CHUNK_SIZE = 50 # skip tiny chunks @@ -623,6 +639,11 @@ def scan_project( continue if filepath.suffix.lower() not in READABLE_EXTENSIONS and not exact_force_include: continue + # Skip minified/bundled/lock files — text but useless for recall + if not force_include: + lower_name = filename.lower() + if any(lower_name.endswith(pat) for pat in SKIP_PATTERNS): + continue if respect_gitignore and active_matchers and not force_include: if is_gitignored(filepath, active_matchers, is_dir=False): continue @@ -631,7 +652,11 @@ def scan_project( continue # Skip files exceeding size limit try: - if filepath.stat().st_size > MAX_FILE_SIZE: + fsize = filepath.stat().st_size + if fsize > MAX_FILE_SIZE: + continue + # Skip suspiciously large text files (SQL dumps, generated JSON, etc.) + if not force_include and fsize > JUNK_FILE_SIZE: continue except OSError: continue @@ -867,20 +892,25 @@ def status(palace_path: str): print(" Run: mempalace init then mempalace mine ") return - # Count by wing and room - r = col.get(limit=10000, include=["metadatas"]) - metas = r["metadatas"] + total = col.count() + # Paginate all metadata to get accurate wing/room counts wing_rooms = defaultdict(lambda: defaultdict(int)) - for m in metas: - wing_rooms[m.get("wing", "?")][m.get("room", "?")] += 1 + offset = 0 + while offset < total: + r = col.get(limit=10000, offset=offset, include=["metadatas"]) + if not r["metadatas"]: + break + for m in r["metadatas"]: + wing_rooms[m.get("wing", "?")][m.get("room", "?")] += 1 + offset += len(r["metadatas"]) print(f"\n{'=' * 55}") - print(f" MemPalace Status — {len(metas)} drawers") + print(f" MemPalace Status — {total:,} drawers") print(f"{'=' * 55}\n") for wing, rooms in sorted(wing_rooms.items()): print(f" WING: {wing}") for room, count in sorted(rooms.items(), key=lambda x: x[1], reverse=True): - print(f" ROOM: {room:20} {count:5} drawers") + print(f" ROOM: {room:20} {count:>8,} drawers") print() print(f"{'=' * 55}\n") From 1ab6b60a5fce8733868c2e1d8d7cf3749875c157 Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 14:34:49 -0700 Subject: [PATCH 36/50] fix: repair command nukes palace dir, MCP server detects db replacement - repair: delete entire palace directory and create fresh PersistentClient instead of same-process delete_collection/create_collection which left corrupted HNSW ghost entries. Adds index verification query after rebuild. - mcp_server: track chroma.sqlite3 inode in _get_client() to detect palace rebuilds on disk. Automatically reconnects when the database file changes, eliminating stale cache after repair/nuke operations. Co-Authored-By: Claude Opus 4.6 --- mempalace/cli.py | 48 +++++++++++++++++++++++++++++++---------- mempalace/mcp_server.py | 21 +++++++++++++++--- 2 files changed, 55 insertions(+), 14 deletions(-) diff --git a/mempalace/cli.py b/mempalace/cli.py index 024e3341fc..36f9c803fd 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -247,7 +247,14 @@ def cmd_status(args): def cmd_repair(args): - """Rebuild palace vector index from SQLite metadata.""" + """Rebuild palace vector index by nuking and recreating the database. + + ChromaDB's HNSW index can become corrupted after bulk deletes (ghost + entries cause segfaults on query). A same-process delete_collection + + create_collection is NOT sufficient — the PersistentClient reuses + corrupted state. This command extracts all drawers, deletes the entire + palace directory, creates a fresh PersistentClient, and re-inserts. + """ import chromadb import shutil @@ -286,13 +293,18 @@ def cmd_repair(args): offset = 0 while offset < total: batch = col.get(limit=batch_size, offset=offset, include=["documents", "metadatas"]) + if not batch["ids"]: + break all_ids.extend(batch["ids"]) all_docs.extend(batch["documents"]) all_metas.extend(batch["metadatas"]) - offset += batch_size + offset += len(batch["ids"]) print(f" Extracted {len(all_ids)} drawers") - # Backup and rebuild + # Release the old client before nuking the directory + del col, client + + # Backup the entire palace directory palace_path = palace_path.rstrip(os.sep) backup_path = palace_path + ".backup" if os.path.exists(backup_path): @@ -300,19 +312,33 @@ def cmd_repair(args): print(f" Backing up to {backup_path}...") shutil.copytree(palace_path, backup_path) - print(" Rebuilding collection...") - client.delete_collection("mempalace_drawers") - new_col = client.create_collection("mempalace_drawers") + # Nuke and recreate — fresh PersistentClient gets a clean HNSW index + print(" Rebuilding from scratch...") + shutil.rmtree(palace_path) + os.makedirs(palace_path, mode=0o700) + + new_client = chromadb.PersistentClient(path=palace_path) + new_col = new_client.create_collection("mempalace_drawers") filed = 0 for i in range(0, len(all_ids), batch_size): - batch_ids = all_ids[i : i + batch_size] - batch_docs = all_docs[i : i + batch_size] - batch_metas = all_metas[i : i + batch_size] - new_col.add(documents=batch_docs, ids=batch_ids, metadatas=batch_metas) - filed += len(batch_ids) + end = min(i + batch_size, len(all_ids)) + new_col.add( + documents=all_docs[i:end], + ids=all_ids[i:end], + metadatas=all_metas[i:end], + ) + filed += end - i print(f" Re-filed {filed}/{len(all_ids)} drawers...") + # Verify the new index works + try: + new_col.query(query_texts=["test"], n_results=1, include=["documents"]) + print(" Index verification: OK") + except Exception as e: + print(f" Index verification FAILED: {e}") + print(f" Restore from backup: mv {backup_path} {palace_path}") + print(f"\n Repair complete. {filed} drawers rebuilt.") print(f" Backup saved at {backup_path}") print(f"\n{'=' * 55}\n") diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index e188daeb44..b7ffbd7093 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -66,6 +66,7 @@ def _parse_args(): _client_cache = None _collection_cache = None +_palace_db_inode = 0 # inode of chroma.sqlite3 at cache time # ==================== WRITE-AHEAD LOG ==================== @@ -104,10 +105,24 @@ def _wal_log(operation: str, params: dict, result: dict = None): def _get_client(): - """Return a singleton ChromaDB PersistentClient.""" - global _client_cache - if _client_cache is None: + """Return a ChromaDB PersistentClient, reconnecting if the database changed on disk. + + Detects palace rebuilds (repair/nuke) by checking the inode of + chroma.sqlite3. A full rebuild replaces the file, changing the inode. + """ + global _client_cache, _collection_cache, _palace_db_inode, _metadata_cache, _metadata_cache_time + db_path = os.path.join(_config.palace_path, "chroma.sqlite3") + try: + current_inode = os.stat(db_path).st_ino + except OSError: + current_inode = 0 + + if _client_cache is None or (current_inode and current_inode != _palace_db_inode): _client_cache = chromadb.PersistentClient(path=_config.palace_path) + _collection_cache = None + _metadata_cache = None + _metadata_cache_time = 0 + _palace_db_inode = current_inode return _client_cache From f20bdb1f1e14e81831ca69d5c7f81ee1897411ce Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 14:44:05 -0700 Subject: [PATCH 37/50] =?UTF-8?q?fix:=20upstream=20bug=20fixes=20=E2=80=94?= =?UTF-8?q?=20emotion=20regex,=20unicode=20checkmark,=20KG=20path,=20skill?= =?UTF-8?q?=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove overly broad `\*[^*]+\*` regex from EMOTION_MARKERS that matched all markdown bold/italic, routing 66% of technical content to emotional room (#536) - Replace Unicode checkmark (U+2713) with ASCII '+' in convo_miner and split_mega_files progress output — crashes Windows cp1251/cp1252 (#535) - Fix KG path mismatch: MCP server now always uses palace_path for knowledge_graph.sqlite3 instead of diverging default path (#538) - Fix SKILL.md: mempalace --version doesn't exist, use mempalace status (#534) - Fix init instructions: add --yes flag for agent-friendly non-interactive init (#534) Co-Authored-By: Claude Opus 4.6 --- .claude-plugin/skills/mempalace/SKILL.md | 2 +- mempalace/convo_miner.py | 2 +- mempalace/general_extractor.py | 1 - mempalace/instructions/init.md | 2 +- mempalace/mcp_server.py | 5 +---- mempalace/split_mega_files.py | 2 +- 6 files changed, 5 insertions(+), 9 deletions(-) diff --git a/.claude-plugin/skills/mempalace/SKILL.md b/.claude-plugin/skills/mempalace/SKILL.md index ae60fca9e1..34ee584af0 100644 --- a/.claude-plugin/skills/mempalace/SKILL.md +++ b/.claude-plugin/skills/mempalace/SKILL.md @@ -13,7 +13,7 @@ A searchable memory palace for AI — mine projects and conversations, then sear Ensure `mempalace` is installed: ```bash -mempalace --version +mempalace status ``` If not installed: diff --git a/mempalace/convo_miner.py b/mempalace/convo_miner.py index c84f2eee60..b46adfcb77 100644 --- a/mempalace/convo_miner.py +++ b/mempalace/convo_miner.py @@ -362,7 +362,7 @@ def mine_convos( drawers_added += len(batch_docs[batch_start:batch_end]) total_drawers += drawers_added - print(f" ✓ [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers_added}") + print(f" + [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers_added}") print(f"\n{'=' * 55}") print(" Done.") diff --git a/mempalace/general_extractor.py b/mempalace/general_extractor.py index e849d7cf13..0a05cf46ca 100644 --- a/mempalace/general_extractor.py +++ b/mempalace/general_extractor.py @@ -157,7 +157,6 @@ r"i need", r"never told anyone", r"nobody knows", - r"\*[^*]+\*", ] ALL_MARKERS = { diff --git a/mempalace/instructions/init.md b/mempalace/instructions/init.md index 40fe8fcaaa..40f0c20dd7 100644 --- a/mempalace/instructions/init.md +++ b/mempalace/instructions/init.md @@ -41,7 +41,7 @@ before continuing. ## Step 5: Initialize the palace -Run `mempalace init ` where `` is the directory from Step 4. +Run `mempalace init --yes ` where `` is the directory from Step 4. If this fails, report the error and stop. diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index b7ffbd7093..1461715636 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -58,10 +58,7 @@ def _parse_args(): os.environ["MEMPALACE_PALACE_PATH"] = os.path.abspath(_args.palace) _config = MempalaceConfig() -if _args.palace: - _kg = KnowledgeGraph(db_path=os.path.join(_config.palace_path, "knowledge_graph.sqlite3")) -else: - _kg = KnowledgeGraph() +_kg = KnowledgeGraph(db_path=os.path.join(_config.palace_path, "knowledge_graph.sqlite3")) _client_cache = None diff --git a/mempalace/split_mega_files.py b/mempalace/split_mega_files.py index 24b59569c8..8552627a8e 100644 --- a/mempalace/split_mega_files.py +++ b/mempalace/split_mega_files.py @@ -224,7 +224,7 @@ def split_file(filepath, output_dir, dry_run=False): print(f" [{i + 1}/{len(boundaries) - 1}] {name} ({len(chunk)} lines)") else: out_path.write_text("".join(chunk), encoding="utf-8") - print(f" ✓ {name} ({len(chunk)} lines)") + print(f" + {name} ({len(chunk)} lines)") written.append(out_path) From 3dd85b66efb22950d45e5def79e08b9a56b56e68 Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 14:48:51 -0700 Subject: [PATCH 38/50] =?UTF-8?q?fix:=20cherry-pick=20upstream=20bug=20fix?= =?UTF-8?q?es=20=E2=80=94=20cosine=20distance,=20WAL=20rotation,=20compres?= =?UTF-8?q?s=20keys?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Default new collections to cosine distance (hnsw:space=cosine) instead of L2 — fixes negative similarity scores (#568) - Filter unexpected MCP tool args before dispatch — prevents TypeError crash from extra params like top_k (#572) - Rotate WAL at 10 MB with one backup to prevent unbounded growth (#573) - Fix cmd_compress KeyError: align dict keys with compression_stats() return values (#569) - Fix spellcheck _load_known_names: read from "people" key, use dict keys as canonical names (#570) - Repair command uses cosine distance for rebuilt collection Co-Authored-By: Claude Opus 4.6 --- mempalace/cli.py | 12 +++++++----- mempalace/mcp_server.py | 16 +++++++++++++++- mempalace/palace.py | 4 +++- mempalace/spellcheck.py | 4 ++-- tests/test_cli.py | 16 ++++++++-------- tests/test_spellcheck_extra.py | 6 +++--- 6 files changed, 38 insertions(+), 20 deletions(-) diff --git a/mempalace/cli.py b/mempalace/cli.py index 36f9c803fd..ad975cc1cd 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -318,7 +318,9 @@ def cmd_repair(args): os.makedirs(palace_path, mode=0o700) new_client = chromadb.PersistentClient(path=palace_path) - new_col = new_client.create_collection("mempalace_drawers") + new_col = new_client.create_collection( + "mempalace_drawers", metadata={"hnsw:space": "cosine"} + ) filed = 0 for i in range(0, len(all_ids), batch_size): @@ -456,7 +458,7 @@ def cmd_compress(args): stats = dialect.compression_stats(doc, compressed) total_original += stats["original_chars"] - total_compressed += stats["compressed_chars"] + total_compressed += stats["summary_chars"] compressed_entries.append((doc_id, compressed, meta, stats)) @@ -466,7 +468,7 @@ def cmd_compress(args): source = Path(meta.get("source_file", "?")).name print(f" [{wing_name}/{room_name}] {source}") print( - f" {stats['original_tokens']}t -> {stats['compressed_tokens']}t ({stats['ratio']:.1f}x)" + f" {stats['original_tokens_est']}t -> {stats['summary_tokens_est']}t ({stats['size_ratio']:.1f}x)" ) print(f" {compressed}") print() @@ -477,8 +479,8 @@ def cmd_compress(args): comp_col = client.get_or_create_collection("mempalace_compressed") for doc_id, compressed, meta, stats in compressed_entries: comp_meta = dict(meta) - comp_meta["compression_ratio"] = round(stats["ratio"], 1) - comp_meta["original_tokens"] = stats["original_tokens"] + comp_meta["compression_ratio"] = round(stats["size_ratio"], 1) + comp_meta["original_tokens"] = stats["original_tokens_est"] comp_col.upsert( ids=[doc_id], documents=[compressed], diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 1461715636..5e35aab478 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -89,6 +89,15 @@ def _wal_log(operation: str, params: dict, result: dict = None): "result": result, } try: + # Rotate WAL at 10 MB to prevent unbounded growth + _WAL_MAX_BYTES = 10 * 1024 * 1024 + if _WAL_FILE.exists() and _WAL_FILE.stat().st_size > _WAL_MAX_BYTES: + backup = _WAL_FILE.with_suffix(".jsonl.1") + try: + _WAL_FILE.replace(backup) + backup.chmod(0o600) + except OSError: + pass created = not _WAL_FILE.exists() with open(_WAL_FILE, "a", encoding="utf-8") as f: f.write(json.dumps(entry, default=str) + "\n") @@ -129,7 +138,9 @@ def _get_collection(create=False): try: client = _get_client() if create: - _collection_cache = client.get_or_create_collection(_config.collection_name) + _collection_cache = client.get_or_create_collection( + _config.collection_name, metadata={"hnsw:space": "cosine"} + ) _metadata_cache = None _metadata_cache_time = 0 elif _collection_cache is None: @@ -1253,6 +1264,9 @@ def handle_request(request): # MCP JSON transport may deliver integers as floats or strings; # ChromaDB and Python slicing require native int. schema_props = TOOLS[tool_name]["input_schema"].get("properties", {}) + # Filter to declared params only — clients may send extras (e.g. top_k) + valid_keys = set(schema_props.keys()) + tool_args = {k: v for k, v in tool_args.items() if k in valid_keys} for key, value in list(tool_args.items()): prop_schema = schema_props.get(key, {}) declared_type = prop_schema.get("type") diff --git a/mempalace/palace.py b/mempalace/palace.py index 537200d053..1b89de1ec4 100644 --- a/mempalace/palace.py +++ b/mempalace/palace.py @@ -49,7 +49,9 @@ def get_collection(palace_path: str, collection_name: str = "mempalace_drawers") try: return client.get_collection(collection_name) except Exception: - return client.create_collection(collection_name) + return client.create_collection( + collection_name, metadata={"hnsw:space": "cosine"} + ) def file_already_mined(collection, source_file: str, check_mtime: bool = False) -> bool: diff --git a/mempalace/spellcheck.py b/mempalace/spellcheck.py index fe8da38cdb..0368d33e8a 100644 --- a/mempalace/spellcheck.py +++ b/mempalace/spellcheck.py @@ -119,8 +119,8 @@ def _load_known_names() -> set: reg = EntityRegistry.load() names = set() - for entity in reg._data.get("entities", {}).values(): - names.add(entity.get("canonical", "").lower()) + for name, entity in reg._data.get("people", {}).items(): + names.add(name.lower()) for alias in entity.get("aliases", []): names.add(alias.lower()) return names diff --git a/tests/test_cli.py b/tests/test_cli.py index d3280b2a97..2d6b064209 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -549,10 +549,10 @@ def test_cmd_compress_dry_run(mock_config_cls, capsys): mock_dialect.compress.return_value = "compressed" mock_dialect.compression_stats.return_value = { "original_chars": 100, - "compressed_chars": 30, - "original_tokens": 25, - "compressed_tokens": 8, - "ratio": 3.3, + "summary_chars": 30, + "original_tokens_est": 25, + "summary_tokens_est": 8, + "size_ratio": 3.3, } mock_dialect_mod = _make_mock_dialect_module(mock_dialect) @@ -622,10 +622,10 @@ def test_cmd_compress_stores_results(mock_config_cls, capsys): mock_dialect.compress.return_value = "compressed" mock_dialect.compression_stats.return_value = { "original_chars": 100, - "compressed_chars": 30, - "original_tokens": 25, - "compressed_tokens": 8, - "ratio": 3.3, + "summary_chars": 30, + "original_tokens_est": 25, + "summary_tokens_est": 8, + "size_ratio": 3.3, } mock_dialect_mod = _make_mock_dialect_module(mock_dialect) diff --git a/tests/test_spellcheck_extra.py b/tests/test_spellcheck_extra.py index 567cb01cc8..b44d9ff6b1 100644 --- a/tests/test_spellcheck_extra.py +++ b/tests/test_spellcheck_extra.py @@ -12,9 +12,9 @@ class TestLoadKnownNames: def test_returns_names_from_registry(self): mock_reg = MagicMock() mock_reg._data = { - "entities": { - "e1": {"canonical": "Alice", "aliases": ["ali"]}, - "e2": {"canonical": "Bob", "aliases": []}, + "people": { + "Alice": {"source": "onboarding", "aliases": ["ali"]}, + "Bob": {"source": "onboarding", "aliases": []}, } } with patch("mempalace.entity_registry.EntityRegistry") as MockER: From 173ace89147722f3ab10fc0c10f7dfc3f2f0a1f3 Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 15:13:28 -0700 Subject: [PATCH 39/50] fix: purge rebuilds index instead of in-place delete, inode comment - Purge now extracts drawers to keep, nukes the palace directory, and rebuilds with a fresh PersistentClient + cosine HNSW index. This prevents ghost entries from ChromaDB's in-place collection.delete() which caused segfaults on subsequent queries/inserts. - Add FAT/exFAT inode caveat comment on _get_client() detection - Clarify purge --room help: without --wing, purges across ALL wings Co-Authored-By: Claude Opus 4.6 --- mempalace/cli.py | 78 ++++++++++++++++++++++++++++++++--------- mempalace/mcp_server.py | 4 ++- 2 files changed, 65 insertions(+), 17 deletions(-) diff --git a/mempalace/cli.py b/mempalace/cli.py index ad975cc1cd..8fbcffc008 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -177,8 +177,17 @@ def cmd_migrate(args): def cmd_purge(args): - """Delete drawers by wing and/or room.""" + """Delete drawers by wing and/or room. + + Extracts the drawers to *keep*, nukes the palace directory, and + re-inserts them into a fresh ChromaDB. This avoids HNSW ghost entries + that ChromaDB's in-place ``collection.delete()`` leaves behind, which + cause segfaults on subsequent queries or inserts. + + Note: ``--room`` without ``--wing`` purges that room across ALL wings. + """ import chromadb + import shutil palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path try: @@ -199,17 +208,19 @@ def cmd_purge(args): print(" Error: specify --wing and/or --room") return - # ChromaDB doesn't have a count(where=...), so paginate to count - match_count = 0 + total = col.count() + + # Count matching drawers + match_ids = set() offset = 0 while True: batch = col.get(limit=10000, offset=offset, where=where, include=[]) if not batch["ids"]: break - match_count += len(batch["ids"]) + match_ids.update(batch["ids"]) offset += len(batch["ids"]) - if match_count == 0: + if not match_ids: label = f"wing={args.wing}" if args.wing else "" if args.room: label = f"{label} room={args.room}" if label else f"room={args.room}" @@ -219,24 +230,59 @@ def cmd_purge(args): label = f"wing={args.wing}" if args.wing else "" if args.room: label = f"{label} room={args.room}" if label else f"room={args.room}" - print(f"\n Found {match_count:,} drawers matching {label}") + keep_count = total - len(match_ids) + print(f"\n Found {len(match_ids):,} drawers matching {label}") + print(f" Will keep {keep_count:,} drawers, rebuild index") if not args.yes: - confirm = input(f" Delete {match_count:,} drawers? [y/N] ").strip().lower() + confirm = input(f" Purge {len(match_ids):,} drawers? [y/N] ").strip().lower() if confirm not in ("y", "yes"): print(" Aborted.") return - deleted = 0 - while True: - batch = col.get(limit=10000, where=where, include=[]) + # Extract drawers to keep (everything NOT matching the filter) + print(" Extracting drawers to keep...") + keep_ids, keep_docs, keep_metas = [], [], [] + offset = 0 + batch_size = 5000 + while offset < total: + batch = col.get(limit=batch_size, offset=offset, include=["documents", "metadatas"]) if not batch["ids"]: break - col.delete(ids=batch["ids"]) - deleted += len(batch["ids"]) - print(f" Deleted {deleted:,} / {match_count:,}...", flush=True) + for i, doc_id in enumerate(batch["ids"]): + if doc_id not in match_ids: + keep_ids.append(doc_id) + keep_docs.append(batch["documents"][i]) + keep_metas.append(batch["metadatas"][i]) + offset += len(batch["ids"]) + print(f" Extracted {len(keep_ids):,} drawers to keep") + + # Release client before nuking + del col, client + + # Nuke and rebuild with clean HNSW index + palace_path = palace_path.rstrip(os.sep) + print(" Rebuilding palace...") + shutil.rmtree(palace_path) + os.makedirs(palace_path, mode=0o700) + + new_client = chromadb.PersistentClient(path=palace_path) + new_col = new_client.create_collection( + "mempalace_drawers", metadata={"hnsw:space": "cosine"} + ) + + filed = 0 + for i in range(0, len(keep_ids), batch_size): + end = min(i + batch_size, len(keep_ids)) + new_col.add( + documents=keep_docs[i:end], + ids=keep_ids[i:end], + metadatas=keep_metas[i:end], + ) + filed += end - i + print(f" Re-filed {filed:,} / {len(keep_ids):,}...", flush=True) - print(f"\n Purged {deleted:,} drawers. Remaining: {col.count():,}\n") + print(f"\n Purged {len(match_ids):,} drawers. Remaining: {new_col.count():,}\n") def cmd_status(args): @@ -674,9 +720,9 @@ def main(): help="Show what would be migrated without changing anything", ) - p_purge = sub.add_parser("purge", help="Delete drawers by wing and/or room") + p_purge = sub.add_parser("purge", help="Delete drawers by wing and/or room (rebuilds index)") p_purge.add_argument("--wing", help="Wing to purge") - p_purge.add_argument("--room", help="Room to purge") + p_purge.add_argument("--room", help="Room to purge (without --wing, purges across ALL wings)") p_purge.add_argument("--yes", "-y", action="store_true", help="Skip confirmation prompt") sub.add_parser("status", help="Show what's been filed") diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 5e35aab478..fd1d886248 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -113,8 +113,10 @@ def _wal_log(operation: str, params: dict, result: dict = None): def _get_client(): """Return a ChromaDB PersistentClient, reconnecting if the database changed on disk. - Detects palace rebuilds (repair/nuke) by checking the inode of + Detects palace rebuilds (repair/nuke/purge) by checking the inode of chroma.sqlite3. A full rebuild replaces the file, changing the inode. + Note: FAT/exFAT may return 0 for st_ino — the ``current_inode != 0`` + guard skips reconnect detection on those filesystems (safe fallback). """ global _client_cache, _collection_cache, _palace_db_inode, _metadata_cache, _metadata_cache_time db_path = os.path.join(_config.palace_path, "chroma.sqlite3") From fe5cd56aa5ee380685f82e80a3ad98b5c70ab15e Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 17:31:24 -0700 Subject: [PATCH 40/50] feat: add --version flag to CLI (from upstream PR #559) Co-Authored-By: Claude Opus 4.6 --- .claude-plugin/skills/mempalace/SKILL.md | 2 +- mempalace/cli.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.claude-plugin/skills/mempalace/SKILL.md b/.claude-plugin/skills/mempalace/SKILL.md index 34ee584af0..ae60fca9e1 100644 --- a/.claude-plugin/skills/mempalace/SKILL.md +++ b/.claude-plugin/skills/mempalace/SKILL.md @@ -13,7 +13,7 @@ A searchable memory palace for AI — mine projects and conversations, then sear Ensure `mempalace` is installed: ```bash -mempalace status +mempalace --version ``` If not installed: diff --git a/mempalace/cli.py b/mempalace/cli.py index 8fbcffc008..4f352fef98 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -34,6 +34,7 @@ from pathlib import Path from .config import MempalaceConfig +from .version import __version__ def cmd_init(args): @@ -554,6 +555,9 @@ def main(): formatter_class=argparse.RawDescriptionHelpFormatter, epilog=__doc__, ) + parser.add_argument( + "--version", action="version", version=f"mempalace {__version__}" + ) parser.add_argument( "--palace", default=None, From d4eee8288a05bd3102db225e470b5b2fd7f4d7fe Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 17:54:32 -0700 Subject: [PATCH 41/50] docs: design spec for tool output capture in conversation mining Co-Authored-By: Claude Opus 4.6 --- .../2026-04-10-tool-output-mining-design.md | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 docs/superpowers/specs/2026-04-10-tool-output-mining-design.md diff --git a/docs/superpowers/specs/2026-04-10-tool-output-mining-design.md b/docs/superpowers/specs/2026-04-10-tool-output-mining-design.md new file mode 100644 index 0000000000..5adf58d58c --- /dev/null +++ b/docs/superpowers/specs/2026-04-10-tool-output-mining-design.md @@ -0,0 +1,89 @@ +# Tool Output Capture in Conversation Mining + +**Date**: 2026-04-10 +**Status**: Approved +**Scope**: `mempalace/normalize.py` — Claude Code JSONL only + +## Problem + +`_extract_content()` in `normalize.py` only extracts `type: "text"` blocks from Claude Code JSONL transcripts. Tool use blocks (847 per typical session) and tool result blocks (847 matching) are silently dropped during normalization. + +This loses unique findings that exist nowhere else in the codebase or palace: +- Bash command output (firmware probes, build errors, test results, runtime behavior) +- External API responses +- Runtime diagnostics and system state + +## Decision: Selective Capture by Tool Type + +Not all tool output has equal value. File contents from `Read` are already mined as project drawers. Git diffs from `Edit` are in version history. But Bash output contains unique runtime findings that are irreproducible from code alone. + +Strategy: **capture aggressively for Bash, breadcrumb-only for everything else.** + +## What Changes + +All changes are in `normalize.py`, within `_try_claude_code_jsonl()` and `_extract_content()`. No new modules. + +### Tool Use Formatting + +Tool invocations are formatted inline with assistant text: + +| Tool | Format | +|------|--------| +| Bash | `[Bash] ` (command truncated at 200 chars) | +| Read | `[Read :-]` | +| Grep | `[Grep] in ` | +| Edit | `[Edit ]` | +| Write | `[Write ]` | +| Other | `[ToolName] ` | + +### Tool Result Extraction Strategies + +| Tool | Strategy | Rationale | +|------|----------|-----------| +| Bash | First 20 lines + last 20 lines, gap marker if middle truncated | Unique findings live here; errors appear at tail | +| Read | Omitted (path in tool_use is sufficient) | Content already mined as project files | +| Grep/Glob | Query + matched file list, cap 20 matches | Matches are the finding; context is reproducible | +| Edit/Write | Omitted (path in tool_use is sufficient) | Actual diff is in git history | +| Other (MCP, etc.) | First 2KB, truncate with `... [truncated, N chars]` | Safe default for unknown tools | + +### Inline Formatting Example + +``` +Let me check the firmware version. +[Bash] lsusb | grep -i razer +→ Bus 002 Device 005: ID 1532:0e05 Razer USA, Ltd Razer Kiyo Pro +Then I ran the XU probe... +``` + +- Tool results are prefixed with `→ ` +- Bash head+tail gap marker: `→ ... [N lines omitted] ...` +- Truncation marker: `→ ... [truncated, N chars]` + +### Tool Use → Tool Result Matching + +Claude Code JSONL links tool calls via `tool_use_id` (on `tool_result` blocks) matching `id` (on `tool_use` blocks). Within `_try_claude_code_jsonl()`, a dict maps `{tool_use_id: tool_name}` as blocks are processed, so when a `tool_result` is encountered, the correct extraction strategy is applied. + +## What Doesn't Change + +- `convo_miner.py` — chunker sees normal transcript text, no changes needed +- `_messages_to_transcript()` — unchanged +- Other format parsers (Codex, ChatGPT, Slack, Claude.ai) — untouched +- `thinking` blocks — still ignored (redacted/empty in JSONL, only signature remains) +- `image` blocks — still ignored (binary, can't text-mine) + +## Testing + +New test cases added to existing normalize test file: +- Mock JSONL with tool_use/tool_result content blocks +- Verify Bash head+tail strategy (short output, long output with gap) +- Verify Read produces path-only breadcrumb +- Verify Grep produces query + match list +- Verify Edit/Write produce path-only breadcrumb +- Verify fallback truncation for unknown tools +- Verify tool_use → tool_result ID matching across message boundaries +- Verify existing text-only extraction is unaffected + +## Future Work + +- Codex JSONL tool output (when needed) +- Copilot CLI / Gemini CLI parsers (no parsers exist yet; tool output handling built in from the start) From b7d2bc52d810a7050c6535ce57d7c56046bb656f Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 18:06:28 -0700 Subject: [PATCH 42/50] docs: implementation plan for tool output mining Co-Authored-By: Claude Opus 4.6 --- .../plans/2026-04-10-tool-output-mining.md | 693 ++++++++++++++++++ 1 file changed, 693 insertions(+) create mode 100644 docs/superpowers/plans/2026-04-10-tool-output-mining.md diff --git a/docs/superpowers/plans/2026-04-10-tool-output-mining.md b/docs/superpowers/plans/2026-04-10-tool-output-mining.md new file mode 100644 index 0000000000..86d9172aea --- /dev/null +++ b/docs/superpowers/plans/2026-04-10-tool-output-mining.md @@ -0,0 +1,693 @@ +# Tool Output Mining Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Capture tool use and tool result blocks from Claude Code JSONL transcripts so Bash output, search results, and command context are mined into the palace instead of silently dropped. + +**Architecture:** Enhance `_extract_content()` and `_try_claude_code_jsonl()` in `normalize.py` to recognize `tool_use` and `tool_result` content blocks. Tool-specific formatting strategies (head+tail for Bash, path-only for Read, etc.) are applied during normalization. The chunker and rest of the pipeline are untouched. + +**Tech Stack:** Python, json, existing normalize.py module + +**Spec:** `docs/superpowers/specs/2026-04-10-tool-output-mining-design.md` + +--- + +### Task 1: Add tool_use formatting helper + +**Files:** +- Modify: `mempalace/normalize.py` (add `_format_tool_use` after `_extract_content` at ~line 288) +- Test: `tests/test_normalize.py` + +- [ ] **Step 1: Write the failing tests** + +Add to `tests/test_normalize.py` after the existing `_extract_content` tests: + +```python +from mempalace.normalize import _format_tool_use + + +def test_format_tool_use_bash(): + block = {"type": "tool_use", "id": "t1", "name": "Bash", + "input": {"command": "lsusb | grep razer", "description": "Check USB"}} + result = _format_tool_use(block) + assert result == "[Bash] lsusb | grep razer" + + +def test_format_tool_use_bash_truncates_long_command(): + block = {"type": "tool_use", "id": "t1", "name": "Bash", + "input": {"command": "x" * 300}} + result = _format_tool_use(block) + assert len(result) <= len("[Bash] ") + 200 + len("...") + assert result.endswith("...") + + +def test_format_tool_use_read(): + block = {"type": "tool_use", "id": "t1", "name": "Read", + "input": {"file_path": "/home/jp/file.py"}} + result = _format_tool_use(block) + assert result == "[Read /home/jp/file.py]" + + +def test_format_tool_use_read_with_range(): + block = {"type": "tool_use", "id": "t1", "name": "Read", + "input": {"file_path": "/home/jp/file.py", "offset": 10, "limit": 50}} + result = _format_tool_use(block) + assert result == "[Read /home/jp/file.py:10-60]" + + +def test_format_tool_use_grep(): + block = {"type": "tool_use", "id": "t1", "name": "Grep", + "input": {"pattern": "firmware", "path": "/home/jp/proj"}} + result = _format_tool_use(block) + assert result == "[Grep] firmware in /home/jp/proj" + + +def test_format_tool_use_grep_with_glob(): + block = {"type": "tool_use", "id": "t1", "name": "Grep", + "input": {"pattern": "TODO", "glob": "*.py"}} + result = _format_tool_use(block) + assert result == "[Grep] TODO in *.py" + + +def test_format_tool_use_glob(): + block = {"type": "tool_use", "id": "t1", "name": "Glob", + "input": {"pattern": "/home/jp/proj/**/*.py"}} + result = _format_tool_use(block) + assert result == "[Glob] /home/jp/proj/**/*.py" + + +def test_format_tool_use_edit(): + block = {"type": "tool_use", "id": "t1", "name": "Edit", + "input": {"file_path": "/home/jp/file.py", "old_string": "x", "new_string": "y"}} + result = _format_tool_use(block) + assert result == "[Edit /home/jp/file.py]" + + +def test_format_tool_use_write(): + block = {"type": "tool_use", "id": "t1", "name": "Write", + "input": {"file_path": "/home/jp/file.py", "content": "..."}} + result = _format_tool_use(block) + assert result == "[Write /home/jp/file.py]" + + +def test_format_tool_use_unknown_tool(): + block = {"type": "tool_use", "id": "t1", "name": "mcp__mempalace__search", + "input": {"query": "firmware probe", "limit": 5}} + result = _format_tool_use(block) + assert result.startswith("[mcp__mempalace__search]") + assert "firmware probe" in result + + +def test_format_tool_use_unknown_tool_truncates(): + block = {"type": "tool_use", "id": "t1", "name": "SomeTool", + "input": {"data": "x" * 300}} + result = _format_tool_use(block) + assert result.endswith("...") + assert len(result) <= len("[SomeTool] ") + 200 + len("...") +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `python -m pytest tests/test_normalize.py::test_format_tool_use_bash -v` +Expected: FAIL with `ImportError: cannot import name '_format_tool_use'` + +- [ ] **Step 3: Implement `_format_tool_use`** + +Add after `_extract_content` (around line 288) in `mempalace/normalize.py`: + +```python +def _format_tool_use(block: dict) -> str: + """Format a tool_use block into a human-readable one-liner.""" + name = block.get("name", "Unknown") + inp = block.get("input", {}) + + if name == "Bash": + cmd = inp.get("command", "") + if len(cmd) > 200: + cmd = cmd[:200] + "..." + return f"[Bash] {cmd}" + + if name == "Read": + path = inp.get("file_path", "?") + offset = inp.get("offset") + limit = inp.get("limit") + if offset is not None and limit is not None: + return f"[Read {path}:{offset}-{offset + limit}]" + return f"[Read {path}]" + + if name == "Grep": + pattern = inp.get("pattern", "") + target = inp.get("path") or inp.get("glob") or "" + return f"[Grep] {pattern} in {target}" + + if name == "Glob": + pattern = inp.get("pattern", "") + return f"[Glob] {pattern}" + + if name in ("Edit", "Write"): + path = inp.get("file_path", "?") + return f"[{name} {path}]" + + # Unknown tool — serialize input, truncate + summary = json.dumps(inp, separators=(",", ":")) + if len(summary) > 200: + summary = summary[:200] + "..." + return f"[{name}] {summary}" +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `python -m pytest tests/test_normalize.py -k "test_format_tool_use" -v` +Expected: all 12 tests PASS + +- [ ] **Step 5: Commit** + +```bash +git add mempalace/normalize.py tests/test_normalize.py +git commit -m "feat: add _format_tool_use for Claude Code JSONL tool blocks" +``` + +--- + +### Task 2: Add tool_result formatting helper + +**Files:** +- Modify: `mempalace/normalize.py` (add `_format_tool_result` after `_format_tool_use`) +- Test: `tests/test_normalize.py` + +- [ ] **Step 1: Write the failing tests** + +```python +from mempalace.normalize import _format_tool_result + + +def test_format_tool_result_bash_short(): + """Short Bash output is preserved in full.""" + content = "Bus 002 Device 005: ID 1532:0e05 Razer Kiyo Pro" + result = _format_tool_result(content, "Bash") + assert result == "→ Bus 002 Device 005: ID 1532:0e05 Razer Kiyo Pro" + + +def test_format_tool_result_bash_head_tail(): + """Long Bash output gets head+tail with gap marker.""" + lines = [f"line {i}" for i in range(60)] + content = "\n".join(lines) + result = _format_tool_result(content, "Bash") + assert "line 0" in result + assert "line 19" in result + assert "line 40" in result + assert "line 59" in result + assert "20 lines omitted" in result + # Lines 20-39 should be gone + assert "line 20\n" not in result + + +def test_format_tool_result_bash_exactly_40_lines(): + """Bash output at exactly 40 lines is not truncated.""" + lines = [f"line {i}" for i in range(40)] + content = "\n".join(lines) + result = _format_tool_result(content, "Bash") + assert "omitted" not in result + assert "line 0" in result + assert "line 39" in result + + +def test_format_tool_result_read_omitted(): + """Read results are omitted (content already in palace from project mining).""" + result = _format_tool_result("lots of file content here...", "Read") + assert result == "" + + +def test_format_tool_result_edit_omitted(): + """Edit results are omitted (diff is in git).""" + result = _format_tool_result("file updated", "Edit") + assert result == "" + + +def test_format_tool_result_write_omitted(): + """Write results are omitted.""" + result = _format_tool_result("file created", "Write") + assert result == "" + + +def test_format_tool_result_grep_short(): + """Short Grep output is kept.""" + content = "src/foo.py\nsrc/bar.py\nsrc/baz.py" + result = _format_tool_result(content, "Grep") + assert "→ src/foo.py" in result + assert "→ src/baz.py" in result + + +def test_format_tool_result_grep_caps_at_20(): + """Grep output beyond 20 lines is truncated.""" + lines = [f"match_{i}.py" for i in range(30)] + content = "\n".join(lines) + result = _format_tool_result(content, "Grep") + assert "match_19.py" in result + assert "match_20.py" not in result + assert "10 more matches" in result + + +def test_format_tool_result_glob_caps_at_20(): + """Glob output beyond 20 lines is truncated.""" + lines = [f"/path/file_{i}.py" for i in range(25)] + content = "\n".join(lines) + result = _format_tool_result(content, "Glob") + assert "file_19.py" in result + assert "file_20.py" not in result + assert "5 more matches" in result + + +def test_format_tool_result_unknown_short(): + """Unknown tool with short output is kept.""" + result = _format_tool_result("some output", "mcp__mempalace__search") + assert result == "→ some output" + + +def test_format_tool_result_unknown_truncates(): + """Unknown tool output over 2KB is truncated.""" + content = "x" * 3000 + result = _format_tool_result(content, "SomeTool") + assert result.endswith("... [truncated, 3000 chars]") + assert len(result) < 2200 + + +def test_format_tool_result_list_content(): + """tool_result content can be a list of text blocks.""" + content = [{"type": "text", "text": "result line 1"}, {"type": "text", "text": "result line 2"}] + result = _format_tool_result(content, "Bash") + assert "result line 1" in result + assert "result line 2" in result + + +def test_format_tool_result_empty(): + """Empty result returns empty string.""" + result = _format_tool_result("", "Bash") + assert result == "" +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `python -m pytest tests/test_normalize.py::test_format_tool_result_bash_short -v` +Expected: FAIL with `ImportError: cannot import name '_format_tool_result'` + +- [ ] **Step 3: Implement `_format_tool_result`** + +Add after `_format_tool_use` in `mempalace/normalize.py`: + +```python +_TOOL_RESULT_MAX_LINES_BASH = 20 # head and tail line count +_TOOL_RESULT_MAX_MATCHES = 20 # Grep/Glob cap +_TOOL_RESULT_MAX_BYTES = 2048 # fallback cap for unknown tools + + +def _format_tool_result(content, tool_name: str) -> str: + """Format a tool_result based on the originating tool's type. + + Args: + content: Result text (str) or list of content blocks (list of dicts). + tool_name: Name of the tool that produced this result. + + Returns: + Formatted string prefixed with ``→ ``, or empty string if omitted. + """ + # Normalize list-of-blocks to plain text + if isinstance(content, list): + parts = [] + for item in content: + if isinstance(item, dict) and item.get("type") == "text": + parts.append(item.get("text", "")) + elif isinstance(item, str): + parts.append(item) + text = "\n".join(parts) + else: + text = str(content) if content else "" + + text = text.strip() + if not text: + return "" + + # Read/Edit/Write — omit result (content is in palace or git) + if tool_name in ("Read", "Edit", "Write"): + return "" + + lines = text.split("\n") + + # Bash — head + tail + if tool_name == "Bash": + n = _TOOL_RESULT_MAX_LINES_BASH + if len(lines) <= n * 2: + return "→ " + "\n→ ".join(lines) + head = lines[:n] + tail = lines[-n:] + omitted = len(lines) - 2 * n + return ( + "→ " + "\n→ ".join(head) + + f"\n→ ... [{omitted} lines omitted] ..." + + "\n→ " + "\n→ ".join(tail) + ) + + # Grep/Glob — cap matches + if tool_name in ("Grep", "Glob"): + cap = _TOOL_RESULT_MAX_MATCHES + if len(lines) <= cap: + return "→ " + "\n→ ".join(lines) + kept = lines[:cap] + remaining = len(lines) - cap + return "→ " + "\n→ ".join(kept) + f"\n→ ... [{remaining} more matches]" + + # Unknown — byte cap + if len(text) > _TOOL_RESULT_MAX_BYTES: + return "→ " + text[:_TOOL_RESULT_MAX_BYTES] + f"... [truncated, {len(text)} chars]" + return "→ " + text +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `python -m pytest tests/test_normalize.py -k "test_format_tool_result" -v` +Expected: all 14 tests PASS + +- [ ] **Step 5: Commit** + +```bash +git add mempalace/normalize.py tests/test_normalize.py +git commit -m "feat: add _format_tool_result with per-tool strategies" +``` + +--- + +### Task 3: Wire tool blocks into `_extract_content` and `_try_claude_code_jsonl` + +**Files:** +- Modify: `mempalace/normalize.py` — `_extract_content` and `_try_claude_code_jsonl` +- Test: `tests/test_normalize.py` + +- [ ] **Step 1: Write the failing integration tests** + +```python +def test_extract_content_with_tool_use(): + """_extract_content includes formatted tool_use blocks.""" + content = [ + {"type": "text", "text": "Let me check."}, + {"type": "tool_use", "id": "t1", "name": "Bash", + "input": {"command": "lsusb"}}, + ] + result = _extract_content(content) + assert "Let me check." in result + assert "[Bash] lsusb" in result + + +def test_extract_content_with_tool_result(): + """_extract_content includes formatted tool_result blocks (needs tool_use_map).""" + content = [ + {"type": "tool_result", "tool_use_id": "t1", "content": "some output"}, + ] + result = _extract_content(content, tool_use_map={"t1": "Bash"}) + assert "→ some output" in result + + +def test_extract_content_tool_result_without_map_uses_fallback(): + """tool_result without a map entry uses fallback strategy.""" + content = [ + {"type": "tool_result", "tool_use_id": "t1", "content": "some output"}, + ] + result = _extract_content(content) + assert "→ some output" in result + + +def test_claude_code_jsonl_captures_tool_output(): + """Full integration: tool_use + tool_result appear in normalized transcript.""" + lines = [ + json.dumps({"type": "human", "message": {"content": "Check the camera"}}), + json.dumps({"type": "assistant", "message": {"content": [ + {"type": "text", "text": "Let me check."}, + {"type": "tool_use", "id": "t1", "name": "Bash", + "input": {"command": "lsusb | grep razer"}}, + ]}}), + json.dumps({"type": "human", "message": {"content": [ + {"type": "tool_result", "tool_use_id": "t1", + "content": "Bus 002 Device 005: ID 1532:0e05 Razer Kiyo Pro"}, + ]}}), + json.dumps({"type": "assistant", "message": {"content": "Found it."}}), + ] + result = _try_claude_code_jsonl("\n".join(lines)) + assert result is not None + assert "> Check the camera" in result + assert "[Bash] lsusb | grep razer" in result + assert "→ Bus 002 Device 005" in result + assert "Found it." in result + + +def test_claude_code_jsonl_read_result_omitted(): + """Read tool results are omitted but the path breadcrumb is kept.""" + lines = [ + json.dumps({"type": "human", "message": {"content": "Show me the file"}}), + json.dumps({"type": "assistant", "message": {"content": [ + {"type": "text", "text": "Reading it."}, + {"type": "tool_use", "id": "t1", "name": "Read", + "input": {"file_path": "/home/jp/file.py"}}, + ]}}), + json.dumps({"type": "human", "message": {"content": [ + {"type": "tool_result", "tool_use_id": "t1", + "content": "entire file contents here that should not appear"}, + ]}}), + json.dumps({"type": "assistant", "message": {"content": "Here it is."}}), + ] + result = _try_claude_code_jsonl("\n".join(lines)) + assert result is not None + assert "[Read /home/jp/file.py]" in result + assert "entire file contents here" not in result + + +def test_claude_code_jsonl_tool_only_user_message_not_counted(): + """A user message containing ONLY tool_results (no text) should not + be added as a separate user turn with '>'.""" + lines = [ + json.dumps({"type": "human", "message": {"content": "Do it"}}), + json.dumps({"type": "assistant", "message": {"content": [ + {"type": "text", "text": "Running."}, + {"type": "tool_use", "id": "t1", "name": "Bash", + "input": {"command": "echo hi"}}, + ]}}), + json.dumps({"type": "human", "message": {"content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": "hi"}, + ]}}), + json.dumps({"type": "assistant", "message": {"content": "Done."}}), + ] + result = _try_claude_code_jsonl("\n".join(lines)) + assert result is not None + # Only one user turn marker — the original "Do it" + user_turns = [l for l in result.split("\n") if l.strip().startswith(">")] + assert len(user_turns) == 1 + assert "> Do it" in result + + +def test_extract_content_text_only_backward_compat(): + """Text-only content blocks still work (backward compat).""" + content = [ + {"type": "text", "text": "Hello"}, + {"type": "text", "text": "World"}, + ] + result = _extract_content(content) + assert "Hello" in result + assert "World" in result + + +def test_extract_content_string_unchanged(): + """Plain string content still works.""" + result = _extract_content("just a string") + assert result == "just a string" + + +def test_claude_code_jsonl_thinking_blocks_ignored(): + """Thinking blocks are still ignored.""" + lines = [ + json.dumps({"type": "human", "message": {"content": "Q"}}), + json.dumps({"type": "assistant", "message": {"content": [ + {"type": "thinking", "thinking": "", "signature": "abc"}, + {"type": "text", "text": "A"}, + ]}}), + ] + result = _try_claude_code_jsonl("\n".join(lines)) + assert result is not None + assert "thinking" not in result.lower() + assert "signature" not in result + assert "A" in result +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `python -m pytest tests/test_normalize.py::test_claude_code_jsonl_captures_tool_output -v` +Expected: FAIL — tool output not in result + +- [ ] **Step 3: Modify `_extract_content` to handle tool blocks** + +Update the function signature and list-handling branch in `mempalace/normalize.py`: + +```python +def _extract_content(content, tool_use_map: dict = None) -> str: + """Pull text from content — handles str, list of blocks, or dict. + + Args: + content: Message content — string, list of content blocks, or dict. + tool_use_map: Optional mapping of tool_use_id → tool_name, used to + select the right formatting strategy for tool_result blocks. + """ + if isinstance(content, str): + return content.strip() + if isinstance(content, list): + parts = [] + for item in content: + if isinstance(item, str): + parts.append(item) + elif isinstance(item, dict): + block_type = item.get("type") + if block_type == "text": + parts.append(item.get("text", "")) + elif block_type == "tool_use": + parts.append(_format_tool_use(item)) + elif block_type == "tool_result": + tid = item.get("tool_use_id", "") + tname = (tool_use_map or {}).get(tid, "Unknown") + result_content = item.get("content", "") + formatted = _format_tool_result(result_content, tname) + if formatted: + parts.append(formatted) + return "\n".join(p for p in parts if p).strip() + if isinstance(content, dict): + return content.get("text", "").strip() + return "" +``` + +Note: the join changes from `" ".join(parts)` to `"\n".join(p for p in parts if p)` — tool blocks need newline separation, not space. + +- [ ] **Step 4: Modify `_try_claude_code_jsonl` to track tool IDs and handle tool-only messages** + +Replace the function in `mempalace/normalize.py`: + +```python +def _try_claude_code_jsonl(content: str) -> Optional[str]: + """Claude Code JSONL sessions.""" + lines = [line.strip() for line in content.strip().split("\n") if line.strip()] + messages = [] + tool_use_map = {} # tool_use_id → tool_name + + for line in lines: + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(entry, dict): + continue + msg_type = entry.get("type", "") + message = entry.get("message", {}) + msg_content = message.get("content", "") + + # Build tool_use_map from assistant messages + if msg_type == "assistant" and isinstance(msg_content, list): + for block in msg_content: + if isinstance(block, dict) and block.get("type") == "tool_use": + tool_use_map[block.get("id", "")] = block.get("name", "Unknown") + + if msg_type in ("human", "user"): + # Check if this message is tool_results only (no user text) + is_tool_only = ( + isinstance(msg_content, list) + and all( + isinstance(b, dict) and b.get("type") == "tool_result" + for b in msg_content + ) + ) + text = _extract_content(msg_content, tool_use_map=tool_use_map) + if text: + if is_tool_only and messages and messages[-1][0] == "assistant": + # Append tool results to the previous assistant message + prev_role, prev_text = messages[-1] + messages[-1] = (prev_role, prev_text + "\n" + text) + elif not is_tool_only: + messages.append(("user", text)) + elif msg_type == "assistant": + text = _extract_content(msg_content, tool_use_map=tool_use_map) + if text: + # If previous message is also assistant (multi-turn tool loop), + # merge into the same assistant turn + if messages and messages[-1][0] == "assistant": + prev_role, prev_text = messages[-1] + messages[-1] = (prev_role, prev_text + "\n" + text) + else: + messages.append(("assistant", text)) + + if len(messages) >= 2: + return _messages_to_transcript(messages) + return None +``` + +Key changes: +1. `tool_use_map` dict built as we scan assistant messages +2. Tool-result-only user messages are merged into the previous assistant turn (not added as `> ` user turns) +3. Consecutive assistant messages are merged (handles tool loops: assistant→tool_result→assistant) +4. `_extract_content` receives `tool_use_map` for result formatting + +- [ ] **Step 5: Run all tests** + +Run: `python -m pytest tests/test_normalize.py -v` +Expected: all tests PASS (existing + new) + +- [ ] **Step 6: Commit** + +```bash +git add mempalace/normalize.py tests/test_normalize.py +git commit -m "feat: wire tool_use/tool_result into Claude Code JSONL normalization" +``` + +--- + +### Task 4: Run full test suite and verify no regressions + +**Files:** +- None modified — verification only + +- [ ] **Step 1: Run full test suite** + +Run: `python -m pytest tests/ -x -q` +Expected: 615+ tests pass, 0 failures + +- [ ] **Step 2: Test with a real JSONL transcript** + +```bash +python -c " +from mempalace.normalize import normalize +result = normalize('/home/jp/.claude/projects/-home-jp-Projects-kiyo-xhci-fix/f5a0d9cf-38fa-43c0-9684-790c4145b695.jsonl') +lines = result.split('\n') +tool_lines = [l for l in lines if l.startswith('[Bash]') or l.startswith('→') or l.startswith('[Read') or l.startswith('[Grep]')] +print(f'Total lines: {len(lines)}') +print(f'Tool lines: {len(tool_lines)}') +print() +print('Sample tool output:') +for l in tool_lines[:10]: + print(l[:120]) +" +``` + +Expected: tool_lines count > 0, showing `[Bash]` commands and `→` result prefixes + +- [ ] **Step 3: Commit (if any test fixes needed)** + +```bash +git add -p # only if fixes were needed +git commit -m "fix: test adjustments for tool output mining" +``` + +--- + +### Task 5: Final commit and push + +- [ ] **Step 1: Push to origin** + +```bash +git push origin main +``` + +- [ ] **Step 2: Update PR #562 with comment about the new feature** + +Comment on the PR noting tool output capture was added. From 374da401bf91278704bf59a57cffb15bee200685 Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 18:09:01 -0700 Subject: [PATCH 43/50] feat: add _format_tool_use for Claude Code JSONL tool blocks --- mempalace/normalize.py | 39 +++++++++++++++++++ tests/test_normalize.py | 84 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) diff --git a/mempalace/normalize.py b/mempalace/normalize.py index a89450010f..6afbf19a1a 100644 --- a/mempalace/normalize.py +++ b/mempalace/normalize.py @@ -287,6 +287,45 @@ def _extract_content(content) -> str: return "" +def _format_tool_use(block: dict) -> str: + """Format a tool_use block into a human-readable one-liner.""" + name = block.get("name", "Unknown") + inp = block.get("input", {}) + + if name == "Bash": + cmd = inp.get("command", "") + if len(cmd) > 200: + cmd = cmd[:200] + "..." + return f"[Bash] {cmd}" + + if name == "Read": + path = inp.get("file_path", "?") + offset = inp.get("offset") + limit = inp.get("limit") + if offset is not None and limit is not None: + return f"[Read {path}:{offset}-{offset + limit}]" + return f"[Read {path}]" + + if name == "Grep": + pattern = inp.get("pattern", "") + target = inp.get("path") or inp.get("glob") or "" + return f"[Grep] {pattern} in {target}" + + if name == "Glob": + pattern = inp.get("pattern", "") + return f"[Glob] {pattern}" + + if name in ("Edit", "Write"): + path = inp.get("file_path", "?") + return f"[{name} {path}]" + + # Unknown tool — serialize input, truncate + summary = json.dumps(inp, separators=(",", ":")) + if len(summary) > 200: + summary = summary[:200] + "..." + return f"[{name}] {summary}" + + def _messages_to_transcript(messages: list, spellcheck: bool = True) -> str: """Convert [(role, text), ...] to transcript format with > markers.""" if spellcheck: diff --git a/tests/test_normalize.py b/tests/test_normalize.py index 959668f5a9..c74fa88212 100644 --- a/tests/test_normalize.py +++ b/tests/test_normalize.py @@ -3,6 +3,7 @@ from mempalace.normalize import ( _extract_content, + _format_tool_use, _messages_to_transcript, _try_chatgpt_json, _try_claude_ai_json, @@ -102,6 +103,89 @@ def test_extract_content_mixed_list(): assert _extract_content(blocks) == "plain block" +# ── _format_tool_use ────────────────────────────────────────────────── + + +def test_format_tool_use_bash(): + block = {"type": "tool_use", "id": "t1", "name": "Bash", + "input": {"command": "lsusb | grep razer", "description": "Check USB"}} + result = _format_tool_use(block) + assert result == "[Bash] lsusb | grep razer" + + +def test_format_tool_use_bash_truncates_long_command(): + block = {"type": "tool_use", "id": "t1", "name": "Bash", + "input": {"command": "x" * 300}} + result = _format_tool_use(block) + assert len(result) <= len("[Bash] ") + 200 + len("...") + assert result.endswith("...") + + +def test_format_tool_use_read(): + block = {"type": "tool_use", "id": "t1", "name": "Read", + "input": {"file_path": "/home/jp/file.py"}} + result = _format_tool_use(block) + assert result == "[Read /home/jp/file.py]" + + +def test_format_tool_use_read_with_range(): + block = {"type": "tool_use", "id": "t1", "name": "Read", + "input": {"file_path": "/home/jp/file.py", "offset": 10, "limit": 50}} + result = _format_tool_use(block) + assert result == "[Read /home/jp/file.py:10-60]" + + +def test_format_tool_use_grep(): + block = {"type": "tool_use", "id": "t1", "name": "Grep", + "input": {"pattern": "firmware", "path": "/home/jp/proj"}} + result = _format_tool_use(block) + assert result == "[Grep] firmware in /home/jp/proj" + + +def test_format_tool_use_grep_with_glob(): + block = {"type": "tool_use", "id": "t1", "name": "Grep", + "input": {"pattern": "TODO", "glob": "*.py"}} + result = _format_tool_use(block) + assert result == "[Grep] TODO in *.py" + + +def test_format_tool_use_glob(): + block = {"type": "tool_use", "id": "t1", "name": "Glob", + "input": {"pattern": "/home/jp/proj/**/*.py"}} + result = _format_tool_use(block) + assert result == "[Glob] /home/jp/proj/**/*.py" + + +def test_format_tool_use_edit(): + block = {"type": "tool_use", "id": "t1", "name": "Edit", + "input": {"file_path": "/home/jp/file.py", "old_string": "x", "new_string": "y"}} + result = _format_tool_use(block) + assert result == "[Edit /home/jp/file.py]" + + +def test_format_tool_use_write(): + block = {"type": "tool_use", "id": "t1", "name": "Write", + "input": {"file_path": "/home/jp/file.py", "content": "..."}} + result = _format_tool_use(block) + assert result == "[Write /home/jp/file.py]" + + +def test_format_tool_use_unknown_tool(): + block = {"type": "tool_use", "id": "t1", "name": "mcp__mempalace__search", + "input": {"query": "firmware probe", "limit": 5}} + result = _format_tool_use(block) + assert result.startswith("[mcp__mempalace__search]") + assert "firmware probe" in result + + +def test_format_tool_use_unknown_tool_truncates(): + block = {"type": "tool_use", "id": "t1", "name": "SomeTool", + "input": {"data": "x" * 300}} + result = _format_tool_use(block) + assert result.endswith("...") + assert len(result) <= len("[SomeTool] ") + 200 + len("...") + + # ── _try_claude_code_jsonl ───────────────────────────────────────────── From 9775dd1196fc651d0a0c38167198214d1b56d92a Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 18:09:57 -0700 Subject: [PATCH 44/50] feat: add _format_tool_result with per-tool strategies Co-Authored-By: Claude Opus 4.6 --- mempalace/normalize.py | 66 ++++++++++++++++++++++++ tests/test_normalize.py | 109 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+) diff --git a/mempalace/normalize.py b/mempalace/normalize.py index 6afbf19a1a..0272ce0de4 100644 --- a/mempalace/normalize.py +++ b/mempalace/normalize.py @@ -326,6 +326,72 @@ def _format_tool_use(block: dict) -> str: return f"[{name}] {summary}" +_TOOL_RESULT_MAX_LINES_BASH = 20 # head and tail line count +_TOOL_RESULT_MAX_MATCHES = 20 # Grep/Glob cap +_TOOL_RESULT_MAX_BYTES = 2048 # fallback cap for unknown tools + + +def _format_tool_result(content, tool_name: str) -> str: + """Format a tool_result based on the originating tool's type. + + Args: + content: Result text (str) or list of content blocks (list of dicts). + tool_name: Name of the tool that produced this result. + + Returns: + Formatted string prefixed with ``→ ``, or empty string if omitted. + """ + # Normalize list-of-blocks to plain text + if isinstance(content, list): + parts = [] + for item in content: + if isinstance(item, dict) and item.get("type") == "text": + parts.append(item.get("text", "")) + elif isinstance(item, str): + parts.append(item) + text = "\n".join(parts) + else: + text = str(content) if content else "" + + text = text.strip() + if not text: + return "" + + # Read/Edit/Write — omit result (content is in palace or git) + if tool_name in ("Read", "Edit", "Write"): + return "" + + lines = text.split("\n") + + # Bash — head + tail + if tool_name == "Bash": + n = _TOOL_RESULT_MAX_LINES_BASH + if len(lines) <= n * 2: + return "→ " + "\n→ ".join(lines) + head = lines[:n] + tail = lines[-n:] + omitted = len(lines) - 2 * n + return ( + "→ " + "\n→ ".join(head) + + f"\n→ ... [{omitted} lines omitted] ..." + + "\n→ " + "\n→ ".join(tail) + ) + + # Grep/Glob — cap matches + if tool_name in ("Grep", "Glob"): + cap = _TOOL_RESULT_MAX_MATCHES + if len(lines) <= cap: + return "→ " + "\n→ ".join(lines) + kept = lines[:cap] + remaining = len(lines) - cap + return "→ " + "\n→ ".join(kept) + f"\n→ ... [{remaining} more matches]" + + # Unknown — byte cap + if len(text) > _TOOL_RESULT_MAX_BYTES: + return "→ " + text[:_TOOL_RESULT_MAX_BYTES] + f"... [truncated, {len(text)} chars]" + return "→ " + text + + def _messages_to_transcript(messages: list, spellcheck: bool = True) -> str: """Convert [(role, text), ...] to transcript format with > markers.""" if spellcheck: diff --git a/tests/test_normalize.py b/tests/test_normalize.py index c74fa88212..a5db48471c 100644 --- a/tests/test_normalize.py +++ b/tests/test_normalize.py @@ -3,6 +3,7 @@ from mempalace.normalize import ( _extract_content, + _format_tool_result, _format_tool_use, _messages_to_transcript, _try_chatgpt_json, @@ -186,6 +187,114 @@ def test_format_tool_use_unknown_tool_truncates(): assert len(result) <= len("[SomeTool] ") + 200 + len("...") +# ── _format_tool_result ────────────────────────────────────────────── + + +def test_format_tool_result_bash_short(): + """Short Bash output is preserved in full.""" + content = "Bus 002 Device 005: ID 1532:0e05 Razer Kiyo Pro" + result = _format_tool_result(content, "Bash") + assert result == "→ Bus 002 Device 005: ID 1532:0e05 Razer Kiyo Pro" + + +def test_format_tool_result_bash_head_tail(): + """Long Bash output gets head+tail with gap marker.""" + lines = [f"line {i}" for i in range(60)] + content = "\n".join(lines) + result = _format_tool_result(content, "Bash") + assert "line 0" in result + assert "line 19" in result + assert "line 40" in result + assert "line 59" in result + assert "20 lines omitted" in result + # Lines 20-39 should be gone + assert "line 20\n" not in result + + +def test_format_tool_result_bash_exactly_40_lines(): + """Bash output at exactly 40 lines is not truncated.""" + lines = [f"line {i}" for i in range(40)] + content = "\n".join(lines) + result = _format_tool_result(content, "Bash") + assert "omitted" not in result + assert "line 0" in result + assert "line 39" in result + + +def test_format_tool_result_read_omitted(): + """Read results are omitted (content already in palace from project mining).""" + result = _format_tool_result("lots of file content here...", "Read") + assert result == "" + + +def test_format_tool_result_edit_omitted(): + """Edit results are omitted (diff is in git).""" + result = _format_tool_result("file updated", "Edit") + assert result == "" + + +def test_format_tool_result_write_omitted(): + """Write results are omitted.""" + result = _format_tool_result("file created", "Write") + assert result == "" + + +def test_format_tool_result_grep_short(): + """Short Grep output is kept.""" + content = "src/foo.py\nsrc/bar.py\nsrc/baz.py" + result = _format_tool_result(content, "Grep") + assert "→ src/foo.py" in result + assert "→ src/baz.py" in result + + +def test_format_tool_result_grep_caps_at_20(): + """Grep output beyond 20 lines is truncated.""" + lines = [f"match_{i}.py" for i in range(30)] + content = "\n".join(lines) + result = _format_tool_result(content, "Grep") + assert "match_19.py" in result + assert "match_20.py" not in result + assert "10 more matches" in result + + +def test_format_tool_result_glob_caps_at_20(): + """Glob output beyond 20 lines is truncated.""" + lines = [f"/path/file_{i}.py" for i in range(25)] + content = "\n".join(lines) + result = _format_tool_result(content, "Glob") + assert "file_19.py" in result + assert "file_20.py" not in result + assert "5 more matches" in result + + +def test_format_tool_result_unknown_short(): + """Unknown tool with short output is kept.""" + result = _format_tool_result("some output", "mcp__mempalace__search") + assert result == "→ some output" + + +def test_format_tool_result_unknown_truncates(): + """Unknown tool output over 2KB is truncated.""" + content = "x" * 3000 + result = _format_tool_result(content, "SomeTool") + assert result.endswith("... [truncated, 3000 chars]") + assert len(result) < 2200 + + +def test_format_tool_result_list_content(): + """tool_result content can be a list of text blocks.""" + content = [{"type": "text", "text": "result line 1"}, {"type": "text", "text": "result line 2"}] + result = _format_tool_result(content, "Bash") + assert "result line 1" in result + assert "result line 2" in result + + +def test_format_tool_result_empty(): + """Empty result returns empty string.""" + result = _format_tool_result("", "Bash") + assert result == "" + + # ── _try_claude_code_jsonl ───────────────────────────────────────────── From 66f8195c4cb2c8ffc7999d1708687ce117ad72bb Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 18:12:40 -0700 Subject: [PATCH 45/50] feat: wire tool_use/tool_result into Claude Code JSONL normalization _extract_content now handles tool_use and tool_result content blocks via _format_tool_use and _format_tool_result helpers. Join changed from space to newline for proper tool block separation. _try_claude_code_jsonl tracks tool_use_map, merges tool-result-only user messages into the previous assistant turn, and merges consecutive assistant messages from multi-turn tool loops. Co-Authored-By: Claude Opus 4.6 --- mempalace/normalize.py | 65 ++++++++++++++++--- tests/test_normalize.py | 137 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 191 insertions(+), 11 deletions(-) diff --git a/mempalace/normalize.py b/mempalace/normalize.py index 0272ce0de4..9810332702 100644 --- a/mempalace/normalize.py +++ b/mempalace/normalize.py @@ -83,6 +83,8 @@ def _try_claude_code_jsonl(content: str) -> Optional[str]: """Claude Code JSONL sessions.""" lines = [line.strip() for line in content.strip().split("\n") if line.strip()] messages = [] + tool_use_map = {} # tool_use_id → tool_name + for line in lines: try: entry = json.loads(line) @@ -92,14 +94,42 @@ def _try_claude_code_jsonl(content: str) -> Optional[str]: continue msg_type = entry.get("type", "") message = entry.get("message", {}) + msg_content = message.get("content", "") + + # Build tool_use_map from assistant messages + if msg_type == "assistant" and isinstance(msg_content, list): + for block in msg_content: + if isinstance(block, dict) and block.get("type") == "tool_use": + tool_use_map[block.get("id", "")] = block.get("name", "Unknown") + if msg_type in ("human", "user"): - text = _extract_content(message.get("content", "")) + # Check if this message is tool_results only (no user text) + is_tool_only = ( + isinstance(msg_content, list) + and all( + isinstance(b, dict) and b.get("type") == "tool_result" + for b in msg_content + ) + ) + text = _extract_content(msg_content, tool_use_map=tool_use_map) if text: - messages.append(("user", text)) + if is_tool_only and messages and messages[-1][0] == "assistant": + # Append tool results to the previous assistant message + prev_role, prev_text = messages[-1] + messages[-1] = (prev_role, prev_text + "\n" + text) + elif not is_tool_only: + messages.append(("user", text)) elif msg_type == "assistant": - text = _extract_content(message.get("content", "")) + text = _extract_content(msg_content, tool_use_map=tool_use_map) if text: - messages.append(("assistant", text)) + # If previous message is also assistant (multi-turn tool loop), + # merge into the same assistant turn + if messages and messages[-1][0] == "assistant": + prev_role, prev_text = messages[-1] + messages[-1] = (prev_role, prev_text + "\n" + text) + else: + messages.append(("assistant", text)) + if len(messages) >= 2: return _messages_to_transcript(messages) return None @@ -270,8 +300,14 @@ def _try_slack_json(data) -> Optional[str]: return None -def _extract_content(content) -> str: - """Pull text from content — handles str, list of blocks, or dict.""" +def _extract_content(content, tool_use_map: dict = None) -> str: + """Pull text from content — handles str, list of blocks, or dict. + + Args: + content: Message content — string, list of content blocks, or dict. + tool_use_map: Optional mapping of tool_use_id → tool_name, used to + select the right formatting strategy for tool_result blocks. + """ if isinstance(content, str): return content.strip() if isinstance(content, list): @@ -279,9 +315,20 @@ def _extract_content(content) -> str: for item in content: if isinstance(item, str): parts.append(item) - elif isinstance(item, dict) and item.get("type") == "text": - parts.append(item.get("text", "")) - return " ".join(parts).strip() + elif isinstance(item, dict): + block_type = item.get("type") + if block_type == "text": + parts.append(item.get("text", "")) + elif block_type == "tool_use": + parts.append(_format_tool_use(item)) + elif block_type == "tool_result": + tid = item.get("tool_use_id", "") + tname = (tool_use_map or {}).get(tid, "Unknown") + result_content = item.get("content", "") + formatted = _format_tool_result(result_content, tname) + if formatted: + parts.append(formatted) + return "\n".join(p for p in parts if p).strip() if isinstance(content, dict): return content.get("text", "").strip() return "" diff --git a/tests/test_normalize.py b/tests/test_normalize.py index a5db48471c..559117cf9c 100644 --- a/tests/test_normalize.py +++ b/tests/test_normalize.py @@ -83,7 +83,7 @@ def test_extract_content_string(): def test_extract_content_list_of_strings(): - assert _extract_content(["hello", "world"]) == "hello world" + assert _extract_content(["hello", "world"]) == "hello\nworld" def test_extract_content_list_of_blocks(): @@ -101,7 +101,7 @@ def test_extract_content_none(): def test_extract_content_mixed_list(): blocks = ["plain", {"type": "text", "text": "block"}] - assert _extract_content(blocks) == "plain block" + assert _extract_content(blocks) == "plain\nblock" # ── _format_tool_use ────────────────────────────────────────────────── @@ -694,6 +694,139 @@ def test_messages_to_transcript_assistant_first(): assert "> Q" in result +# ── Tool block integration (Task 3) ─────────────────────────────────── + + +def test_extract_content_with_tool_use(): + """_extract_content includes formatted tool_use blocks.""" + content = [ + {"type": "text", "text": "Let me check."}, + {"type": "tool_use", "id": "t1", "name": "Bash", + "input": {"command": "lsusb"}}, + ] + result = _extract_content(content) + assert "Let me check." in result + assert "[Bash] lsusb" in result + + +def test_extract_content_with_tool_result(): + """_extract_content includes formatted tool_result blocks (needs tool_use_map).""" + content = [ + {"type": "tool_result", "tool_use_id": "t1", "content": "some output"}, + ] + result = _extract_content(content, tool_use_map={"t1": "Bash"}) + assert "→ some output" in result + + +def test_extract_content_tool_result_without_map_uses_fallback(): + """tool_result without a map entry uses fallback strategy.""" + content = [ + {"type": "tool_result", "tool_use_id": "t1", "content": "some output"}, + ] + result = _extract_content(content) + assert "→ some output" in result + + +def test_claude_code_jsonl_captures_tool_output(): + """Full integration: tool_use + tool_result appear in normalized transcript.""" + lines = [ + json.dumps({"type": "human", "message": {"content": "Check the camera"}}), + json.dumps({"type": "assistant", "message": {"content": [ + {"type": "text", "text": "Let me check."}, + {"type": "tool_use", "id": "t1", "name": "Bash", + "input": {"command": "lsusb | grep razer"}}, + ]}}), + json.dumps({"type": "human", "message": {"content": [ + {"type": "tool_result", "tool_use_id": "t1", + "content": "Bus 002 Device 005: ID 1532:0e05 Razer Kiyo Pro"}, + ]}}), + json.dumps({"type": "assistant", "message": {"content": "Found it."}}), + ] + result = _try_claude_code_jsonl("\n".join(lines)) + assert result is not None + assert "> Check the camera" in result + assert "[Bash] lsusb | grep razer" in result + assert "→ Bus 002 Device 005" in result + assert "Found it." in result + + +def test_claude_code_jsonl_read_result_omitted(): + """Read tool results are omitted but the path breadcrumb is kept.""" + lines = [ + json.dumps({"type": "human", "message": {"content": "Show me the file"}}), + json.dumps({"type": "assistant", "message": {"content": [ + {"type": "text", "text": "Reading it."}, + {"type": "tool_use", "id": "t1", "name": "Read", + "input": {"file_path": "/home/jp/file.py"}}, + ]}}), + json.dumps({"type": "human", "message": {"content": [ + {"type": "tool_result", "tool_use_id": "t1", + "content": "entire file contents here that should not appear"}, + ]}}), + json.dumps({"type": "assistant", "message": {"content": "Here it is."}}), + ] + result = _try_claude_code_jsonl("\n".join(lines)) + assert result is not None + assert "[Read /home/jp/file.py]" in result + assert "entire file contents here" not in result + + +def test_claude_code_jsonl_tool_only_user_message_not_counted(): + """A user message containing ONLY tool_results (no text) should not + be added as a separate user turn with '>'.""" + lines = [ + json.dumps({"type": "human", "message": {"content": "Do it"}}), + json.dumps({"type": "assistant", "message": {"content": [ + {"type": "text", "text": "Running."}, + {"type": "tool_use", "id": "t1", "name": "Bash", + "input": {"command": "echo hi"}}, + ]}}), + json.dumps({"type": "human", "message": {"content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": "hi"}, + ]}}), + json.dumps({"type": "assistant", "message": {"content": "Done."}}), + ] + result = _try_claude_code_jsonl("\n".join(lines)) + assert result is not None + # Only one user turn marker — the original "Do it" + user_turns = [l for l in result.split("\n") if l.strip().startswith(">")] + assert len(user_turns) == 1 + assert "> Do it" in result + + +def test_extract_content_text_only_backward_compat(): + """Text-only content blocks still work (backward compat).""" + content = [ + {"type": "text", "text": "Hello"}, + {"type": "text", "text": "World"}, + ] + result = _extract_content(content) + assert "Hello" in result + assert "World" in result + + +def test_extract_content_string_unchanged(): + """Plain string content still works.""" + result = _extract_content("just a string") + assert result == "just a string" + + +def test_claude_code_jsonl_thinking_blocks_ignored(): + """Thinking blocks are still ignored.""" + lines = [ + json.dumps({"type": "human", "message": {"content": "Q"}}), + json.dumps({"type": "assistant", "message": {"content": [ + {"type": "thinking", "thinking": "", "signature": "abc"}, + {"type": "text", "text": "A"}, + ]}}), + ] + result = _try_claude_code_jsonl("\n".join(lines)) + assert result is not None + assert "thinking" not in result.lower() + assert "signature" not in result + assert "A" in result + + def test_normalize_rejects_large_file(): """Files over 500 MB should raise IOError before reading.""" with patch("mempalace.normalize.os.path.getsize", return_value=600 * 1024 * 1024): From 321e77f6b034f1bd9f4376de725b5e5d87604542 Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 18:30:19 -0700 Subject: [PATCH 46/50] docs: add file handle and idempotency comments per review Co-Authored-By: Claude Opus 4.6 --- mempalace/cli.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/mempalace/cli.py b/mempalace/cli.py index 4f352fef98..345bbe113d 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -186,6 +186,10 @@ def cmd_purge(args): cause segfaults on subsequent queries or inserts. Note: ``--room`` without ``--wing`` purges that room across ALL wings. + + Not idempotent — running purge twice on the same criteria will fail the + second time if the first run completed (nothing left to match). The + backup directory is preserved for recovery. """ import chromadb import shutil @@ -258,7 +262,8 @@ def cmd_purge(args): offset += len(batch["ids"]) print(f" Extracted {len(keep_ids):,} drawers to keep") - # Release client before nuking + # Release client before nuking — ChromaDB holds open file handles + # (WAL journal, HNSW mmap) that block rmtree on Windows and some Linux FS. del col, client # Nuke and rebuild with clean HNSW index @@ -348,7 +353,9 @@ def cmd_repair(args): offset += len(batch["ids"]) print(f" Extracted {len(all_ids)} drawers") - # Release the old client before nuking the directory + # Release the old client before nuking the directory — ChromaDB holds + # open file handles (WAL journal, HNSW mmap) that block rmtree on Windows + # and some Linux FS. del col, client # Backup the entire palace directory From b9e2a09c0b7ef1a7102d15f0ea2b1a03626471d1 Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 18:40:41 -0700 Subject: [PATCH 47/50] fix: dry-run room=None crash (#586), precompact hook sanitization (#589) Co-Authored-By: Claude Opus 4.6 --- hooks/mempal_precompact_hook.sh | 8 +++++++- mempalace/miner.py | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/hooks/mempal_precompact_hook.sh b/hooks/mempal_precompact_hook.sh index 550a813be1..7cff44beb6 100755 --- a/hooks/mempal_precompact_hook.sh +++ b/hooks/mempal_precompact_hook.sh @@ -57,7 +57,13 @@ MEMPAL_DIR="" # Read JSON input from stdin INPUT=$(cat) -SESSION_ID=$(echo "$INPUT" | python3 -c "import sys,json; print(json.load(sys.stdin).get('session_id','unknown'))" 2>/dev/null) +SESSION_ID=$(echo "$INPUT" | python3 -c " +import sys, json, re +data = json.load(sys.stdin) +sid = data.get('session_id', 'unknown') +safe = lambda s: re.sub(r'[^a-zA-Z0-9_/.\-~]', '', str(s)) +print(safe(sid)) +" 2>/dev/null) echo "[$(date '+%H:%M:%S')] PRE-COMPACT triggered for session $SESSION_ID" >> "$STATE_DIR/hook.log" diff --git a/mempalace/miner.py b/mempalace/miner.py index a230616cb7..7d53a83cd6 100644 --- a/mempalace/miner.py +++ b/mempalace/miner.py @@ -778,7 +778,7 @@ def mine( files_skipped += 1 else: total_drawers += drawers - room_counts[room] += 1 + room_counts[room or "general"] += 1 if not dry_run: print(f" \u2713 [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers}") else: @@ -834,7 +834,7 @@ def prepare_one(filepath): continue total_drawers += len(batch_docs) - room_counts[room] += 1 + room_counts[room or "general"] += 1 pending_docs.extend(batch_docs) pending_ids.extend(batch_ids) pending_metas.extend(batch_metas) From 70dc3dfc7825c040b95539449eff0d42b194148e Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 18:47:40 -0700 Subject: [PATCH 48/50] feat: upgrade chromadb pin from 0.6.x to >=1.5.4 (#581) Fixes ARM64 HNSW segfaults, Python 3.13/3.14 compat. Auto-migrates existing databases, zero API breakage. 648 tests pass, 50K palace verified. Co-Authored-By: Claude Opus 4.6 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cd47f98872..6302dc89f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Utilities", ] dependencies = [ - "chromadb>=0.5.0,<0.7", + "chromadb>=1.5.4,<2", "pyyaml>=6.0,<7", ] From 17518c3462564ca8f347caab9df3962ce07ccabf Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 19:43:18 -0700 Subject: [PATCH 49/50] feat: hooks auto-mine transcript for tool output capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both save and precompact hooks now auto-mine the JSONL transcript directly into the palace before blocking the AI. This captures raw tool output (Bash results, search findings, build errors) that the AI would otherwise summarize away during its save cycle. - Add MP_PYTHON auto-detection (MEMPAL_PYTHON env → repo venv → system) - Add inline normalize → chunk → upsert pipeline to both hooks - Skip file_already_mined — transcript grows, upsert is idempotent - Update block reason messages to explicitly request verbatim tool output - Precompact hook: parse transcript_path from input, fallback to session_id lookup - Update README with two-layer capture docs and MEMPAL_PYTHON config Co-Authored-By: Claude Opus 4.6 --- hooks/README.md | 42 +++++++++++------ hooks/mempal_precompact_hook.sh | 81 ++++++++++++++++++++++++++++++++- hooks/mempal_save_hook.sh | 60 +++++++++++++++++++++++- 3 files changed, 164 insertions(+), 19 deletions(-) diff --git a/hooks/README.md b/hooks/README.md index d5380ef617..3c1c922609 100644 --- a/hooks/README.md +++ b/hooks/README.md @@ -6,10 +6,10 @@ These hook scripts make MemPalace save automatically. No manual "save" commands | Hook | When It Fires | What Happens | |------|--------------|-------------| -| **Save Hook** | Every 15 human messages | Blocks the AI, tells it to save key topics/decisions/quotes to the palace | -| **PreCompact Hook** | Right before context compaction | Emergency save — forces the AI to save EVERYTHING before losing context | +| **Save Hook** | Every 15 human messages | Auto-mines transcript (tool output included), then blocks the AI to save topics/decisions/quotes | +| **PreCompact Hook** | Right before context compaction | Auto-mines transcript, then emergency save — forces the AI to save EVERYTHING before losing context | -The AI does the actual filing — it knows the conversation context, so it classifies memories into the right wings/halls/closets. The hooks just tell it WHEN to save. +**Two-layer capture:** Hooks auto-mine the JSONL transcript directly into the palace (capturing raw tool output — Bash results, search findings, build errors). They also block the AI with a reason message telling it to save verbatim tool output and key context. Belt and suspenders — tool output gets stored even if the AI summarizes instead of quoting. ## Install — Claude Code @@ -68,6 +68,7 @@ Edit `mempal_save_hook.sh` to change: - **`SAVE_INTERVAL=15`** — How many human messages between saves. Lower = more frequent saves, higher = less interruption. - **`STATE_DIR`** — Where hook state is stored (defaults to `~/.mempalace/hook_state/`) - **`MEMPAL_DIR`** — Optional. Set to a conversations directory to auto-run `mempalace mine ` on each save trigger. Leave blank (default) to let the AI handle saving via the block reason message. +- **`MEMPAL_PYTHON`** — Optional env var. Python interpreter with mempalace + chromadb installed. Auto-detects: `MEMPAL_PYTHON` env var → repo `venv/bin/python3` → system `python3`. Set this if your venv is in a non-standard location. ### mempalace CLI @@ -91,15 +92,19 @@ User sends message → AI responds → Claude Code fires Stop hook ↓ ┌─── < 15 since last save ──→ echo "{}" (let AI stop) │ - └─── ≥ 15 since last save ──→ {"decision": "block", "reason": "save..."} - ↓ - AI saves to palace - ↓ - AI tries to stop again - ↓ - stop_hook_active = true - ↓ - Hook sees flag → echo "{}" (let it through) + └─── ≥ 15 since last save + ↓ + Auto-mine transcript → palace (tool output captured) + ↓ + {"decision": "block", "reason": "save tool output verbatim..."} + ↓ + AI saves to palace (topics, decisions, quotes) + ↓ + AI tries to stop again + ↓ + stop_hook_active = true + ↓ + Hook sees flag → echo "{}" (let it through) ``` The `stop_hook_active` flag prevents infinite loops: block once → AI saves → tries to stop → flag is true → we let it through. @@ -109,14 +114,18 @@ The `stop_hook_active` flag prevents infinite loops: block once → AI saves → ``` Context window getting full → Claude Code fires PreCompact ↓ - Hook ALWAYS blocks + Find transcript (from input or session_id lookup) + ↓ + Auto-mine transcript → palace (tool output captured) + ↓ + {"decision": "block", "reason": "save tool output verbatim..."} ↓ AI saves everything ↓ Compaction proceeds ``` -No counting needed — compaction always warrants a save. +No counting needed — compaction always warrants a save. The auto-mine captures raw tool output before the AI gets a chance to summarize it away. ## Debugging @@ -135,4 +144,7 @@ Example output: ## Cost -**Zero extra tokens.** The hooks are bash scripts that run locally. They don't call any API. The only "cost" is the AI spending a few seconds organizing memories at each checkpoint — and it's doing that with context it already has loaded. +**Zero extra API tokens.** The hooks are bash scripts that run locally. They don't call any API. The auto-mining uses the local ChromaDB instance. The only "cost" is: +- ~1-13 seconds for transcript mining (depending on session length) +- The AI spending a few seconds organizing memories at each checkpoint — with context it already has loaded +- ChromaDB disk space for the mined chunks (~1KB per exchange pair) diff --git a/hooks/mempal_precompact_hook.sh b/hooks/mempal_precompact_hook.sh index 7cff44beb6..2a873b7d27 100755 --- a/hooks/mempal_precompact_hook.sh +++ b/hooks/mempal_precompact_hook.sh @@ -54,6 +54,16 @@ mkdir -p "$STATE_DIR" # Leave empty to skip auto-ingest (AI handles saving via the block reason). MEMPAL_DIR="" +# Python interpreter with mempalace + chromadb installed. +# Auto-detects: MEMPAL_PYTHON env var → repo venv → system python3 +if [ -n "$MEMPAL_PYTHON" ]; then + MP_PYTHON="$MEMPAL_PYTHON" +elif [ -f "$(dirname "$(dirname "${BASH_SOURCE[0]}")")/venv/bin/python3" ]; then + MP_PYTHON="$(dirname "$(dirname "${BASH_SOURCE[0]}")")/venv/bin/python3" +else + MP_PYTHON="python3" +fi + # Read JSON input from stdin INPUT=$(cat) @@ -67,17 +77,84 @@ print(safe(sid)) echo "[$(date '+%H:%M:%S')] PRE-COMPACT triggered for session $SESSION_ID" >> "$STATE_DIR/hook.log" +# Also parse transcript_path if present in the input +TRANSCRIPT_PATH=$(echo "$INPUT" | python3 -c " +import sys, json, re +data = json.load(sys.stdin) +tp = data.get('transcript_path', '') +safe = lambda s: re.sub(r'[^a-zA-Z0-9_/.\-~]', '', str(s)) +print(safe(tp)) +" 2>/dev/null) +TRANSCRIPT_PATH="${TRANSCRIPT_PATH/#\~/$HOME}" + +# If no transcript_path in input, find it by session_id +if [ -z "$TRANSCRIPT_PATH" ] || [ ! -f "$TRANSCRIPT_PATH" ]; then + if [ -n "$SESSION_ID" ] && [ "$SESSION_ID" != "unknown" ]; then + FOUND=$(find "$HOME/.claude/projects" -name "${SESSION_ID}.jsonl" -type f 2>/dev/null | head -1) + if [ -n "$FOUND" ]; then + TRANSCRIPT_PATH="$FOUND" + fi + fi +fi + +# Auto-mine the transcript — captures tool output before compaction loses it +if [ -f "$TRANSCRIPT_PATH" ]; then + echo "[$(date '+%H:%M:%S')] Mining transcript: $TRANSCRIPT_PATH" >> "$STATE_DIR/hook.log" + "$MP_PYTHON" - "$TRANSCRIPT_PATH" <<'PYMINE' +import sys +try: + import hashlib + from datetime import datetime + from mempalace.normalize import normalize + from mempalace.convo_miner import chunk_exchanges, detect_convo_room + from mempalace.palace import get_collection + from mempalace.config import MempalaceConfig + palace = MempalaceConfig().palace_path + content = normalize(sys.argv[1]) + if content and len(content.strip()) >= 50: + collection = get_collection(palace) + source = sys.argv[1] + # No file_already_mined check — transcript grows during session. + # upsert is idempotent: same chunk_index → same ID → overwrite. + chunks = chunk_exchanges(content) + if chunks: + room = detect_convo_room(content) or "session" + wing = "conversations" + docs, ids, metas = [], [], [] + for chunk in chunks: + cid = hashlib.sha256( + (source + str(chunk["chunk_index"])).encode() + ).hexdigest()[:24] + docs.append(chunk["content"]) + ids.append(f"drawer_{wing}_{room}_{cid}") + metas.append({ + "wing": wing, "room": room, "source_file": source, + "chunk_index": chunk["chunk_index"], + "added_by": "hook", "filed_at": datetime.now().isoformat(), + "ingest_mode": "convos", "extract_mode": "exchange", + }) + for i in range(0, len(docs), 100): + collection.upsert( + documents=docs[i:i+100], ids=ids[i:i+100], + metadatas=metas[i:i+100], + ) +except Exception: + pass # Hook must never crash the AI +PYMINE + >> "$STATE_DIR/hook.log" 2>&1 +fi + # Optional: run mempalace ingest synchronously so memories land before compaction if [ -n "$MEMPAL_DIR" ] && [ -d "$MEMPAL_DIR" ]; then SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_DIR="$(dirname "$SCRIPT_DIR")" - python3 -m mempalace mine "$MEMPAL_DIR" >> "$STATE_DIR/hook.log" 2>&1 + "$MP_PYTHON" -m mempalace mine "$MEMPAL_DIR" >> "$STATE_DIR/hook.log" 2>&1 fi # Always block — compaction = save everything cat << 'HOOKJSON' { "decision": "block", - "reason": "COMPACTION IMMINENT. Save ALL topics, decisions, quotes, code, and important context from this session to your memory system. Be thorough — after compaction, detailed context will be lost. Organize into appropriate categories. Use verbatim quotes where possible. Save everything, then allow compaction to proceed." + "reason": "COMPACTION IMMINENT — Save EVERYTHING before context is compressed. CRITICAL: Save tool output VERBATIM — Bash command results, probe findings, USB descriptors, firmware bytes, search results, build output, error messages. These are lost on compaction and exist nowhere else. Also save all topics, decisions, quotes, code, and important context. Be thorough — after compaction, detailed context will be lost. Organize into appropriate categories. Save everything, then allow compaction to proceed." } HOOKJSON diff --git a/hooks/mempal_save_hook.sh b/hooks/mempal_save_hook.sh index a0e4681fca..243c4b7f50 100755 --- a/hooks/mempal_save_hook.sh +++ b/hooks/mempal_save_hook.sh @@ -61,6 +61,16 @@ mkdir -p "$STATE_DIR" # Leave empty to skip auto-ingest (AI handles saving via the block reason). MEMPAL_DIR="" +# Python interpreter with mempalace + chromadb installed. +# Auto-detects: MEMPAL_PYTHON env var → repo venv → system python3 +if [ -n "$MEMPAL_PYTHON" ]; then + MP_PYTHON="$MEMPAL_PYTHON" +elif [ -f "$(dirname "$(dirname "${BASH_SOURCE[0]}")")/venv/bin/python3" ]; then + MP_PYTHON="$(dirname "$(dirname "${BASH_SOURCE[0]}")")/venv/bin/python3" +else + MP_PYTHON="python3" +fi + # Read JSON input from stdin INPUT=$(cat) @@ -137,7 +147,53 @@ if [ "$SINCE_LAST" -ge "$SAVE_INTERVAL" ] && [ "$EXCHANGE_COUNT" -gt 0 ]; then if [ -n "$MEMPAL_DIR" ] && [ -d "$MEMPAL_DIR" ]; then SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_DIR="$(dirname "$SCRIPT_DIR")" - python3 -m mempalace mine "$MEMPAL_DIR" >> "$STATE_DIR/hook.log" 2>&1 & + "$MP_PYTHON" -m mempalace mine "$MEMPAL_DIR" >> "$STATE_DIR/hook.log" 2>&1 & + fi + + # Auto-mine the transcript — captures tool output that the AI would summarize away + if [ -f "$TRANSCRIPT_PATH" ]; then + "$MP_PYTHON" - "$TRANSCRIPT_PATH" <<'PYMINE' +import sys +try: + import hashlib + from datetime import datetime + from mempalace.normalize import normalize + from mempalace.convo_miner import chunk_exchanges, detect_convo_room + from mempalace.palace import get_collection + from mempalace.config import MempalaceConfig + palace = MempalaceConfig().palace_path + content = normalize(sys.argv[1]) + if content and len(content.strip()) >= 50: + collection = get_collection(palace) + source = sys.argv[1] + # No file_already_mined check — transcript grows during session. + # upsert is idempotent: same chunk_index → same ID → overwrite. + chunks = chunk_exchanges(content) + if chunks: + room = detect_convo_room(content) or "session" + wing = "conversations" + docs, ids, metas = [], [], [] + for chunk in chunks: + cid = hashlib.sha256( + (source + str(chunk["chunk_index"])).encode() + ).hexdigest()[:24] + docs.append(chunk["content"]) + ids.append(f"drawer_{wing}_{room}_{cid}") + metas.append({ + "wing": wing, "room": room, "source_file": source, + "chunk_index": chunk["chunk_index"], + "added_by": "hook", "filed_at": datetime.now().isoformat(), + "ingest_mode": "convos", "extract_mode": "exchange", + }) + for i in range(0, len(docs), 100): + collection.upsert( + documents=docs[i:i+100], ids=ids[i:i+100], + metadatas=metas[i:i+100], + ) +except Exception: + pass # Hook must never crash the AI +PYMINE + >> "$STATE_DIR/hook.log" 2>&1 fi # Block the AI and tell it to save @@ -145,7 +201,7 @@ if [ "$SINCE_LAST" -ge "$SAVE_INTERVAL" ] && [ "$EXCHANGE_COUNT" -gt 0 ]; then cat << 'HOOKJSON' { "decision": "block", - "reason": "AUTO-SAVE checkpoint. Save key topics, decisions, quotes, and code from this session to your memory system. Organize into appropriate categories. Use verbatim quotes where possible. Continue conversation after saving." + "reason": "AUTO-SAVE checkpoint. Save key topics, decisions, quotes, and code from this session to your memory system. IMPORTANT: Save tool output VERBATIM — Bash command results, probe findings, search results, build output, error messages. These are lost on compaction and exist nowhere else. Also save key topics, decisions, and quotes. Organize into appropriate categories. Continue conversation after saving." } HOOKJSON else From bddc08754be19efcd5fead2151adec0fe3665e10 Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Apr 2026 19:48:45 -0700 Subject: [PATCH 50/50] docs: update all docs for tool output mining, hook auto-mine, chromadb upgrade - CLAUDE.md: test count 648, fork changes 8-13, PR #562, hook descriptions - README.md: hooks section reflects two-layer capture, chromadb >=1.5.4, normalize.py captures tool blocks, MEMPAL_PYTHON env var documented - AGENTS.md: add normalize.py and hooks to key files section - HOOKS_TUTORIAL.md: new sections for two-layer capture and configuration - mempalace/README.md: normalize.py description updated, version.py added - normalize.py: docstring updated for tool_use/tool_result capture Co-Authored-By: Claude Opus 4.6 --- AGENTS.md | 2 ++ CLAUDE.md | 15 +++++++++++---- README.md | 20 +++++++++++--------- examples/HOOKS_TUTORIAL.md | 17 ++++++++++++++++- mempalace/README.md | 3 ++- mempalace/normalize.py | 2 +- 6 files changed, 43 insertions(+), 16 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3026013bb7..66946ef39e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -76,3 +76,5 @@ Knowledge Graph: - **Modifying mining**: `mempalace/miner.py` (project files) or `mempalace/convo_miner.py` (transcripts) - **Input validation**: `mempalace/config.py` — `sanitize_name()` / `sanitize_content()` - **Tests**: mirror source structure in `tests/test_.py` +- **Changing transcript normalization**: `mempalace/normalize.py` — format detection, tool_use/tool_result extraction +- **Hook auto-mining**: `hooks/mempal_save_hook.sh` and `hooks/mempal_precompact_hook.sh` — inline Python for transcript mining diff --git a/CLAUDE.md b/CLAUDE.md index 0bb9dd51ed..c4524806c2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,7 +20,7 @@ JP's fork of [milla-jovovich/mempalace](https://github.com/milla-jovovich/mempal ```bash source venv/bin/activate -python -m pytest tests/ -x -q # run tests (615 expected) +python -m pytest tests/ -x -q # run tests (648 expected) mempalace status # check palace state mempalace search "query" # test search python -m mempalace.mcp_server # run MCP server standalone @@ -37,19 +37,26 @@ Ruff for linting (`ruff check`), line length 100, target Python 3.9. 5. **fix: entity detector STOPWORDS** — 73 technical terms added (Handler, Node, Service, etc.) 6. **feat: similarity threshold** — `max_distance` parameter in search (renamed from `min_similarity`), default 1.5 L2 distance in MCP 7. **feat: hooks_cli** — stop hook saves directly via Python API with systemMessage notification, precompact blocks for AI-driven save, auto-ingest transcripts +8. **feat: --version flag** — CLI supports `mempalace --version` (from upstream PR #559 pattern) +9. **feat: tool output mining** — normalize.py captures tool_use/tool_result blocks from Claude Code JSONL with per-tool formatting strategies (Bash head+tail, Read/Edit/Write path-only, Grep/Glob capped) +10. **fix: dry-run room=None crash** — miner.py handles None room from unreadable files (#586) +11. **fix: precompact hook SESSION_ID sanitization** — applies same safe() regex as save hook (#589) +12. **feat: chromadb >=1.5.4** — upgraded from 0.6.x pin, auto-migrates existing databases (#581) +13. **feat: hooks auto-mine transcript** — both hooks now auto-mine JSONL transcript into palace (captures raw tool output), updated reason messages to request verbatim tool output, MP_PYTHON auto-detection ## Upstream PRs - milla-jovovich/mempalace#483 — mtime dedup fix - milla-jovovich/mempalace#484 — search limit + pagination + cache fix +- milla-jovovich/mempalace#562 — tool output mining, bug fixes, chromadb upgrade (18 upstream issues addressed) ## Integration - **Claude Code plugin**: installed at user scope via marketplace - **MCP server**: global user scope — available in all projects -- **Stop hook**: fires every 15 messages, saves directly via Python API + systemMessage notification + auto-ingests transcript -- **PreCompact hook**: emergency save before context compaction +- **Stop hook**: fires every 15 messages, auto-mines transcript for tool output, blocks AI to save with verbatim tool output instructions +- **PreCompact hook**: emergency save before context compaction, auto-mines transcript, finds transcript by session_id fallback ## Testing -Always run `python -m pytest tests/ -x -q` after changes. 615 tests expected to pass. Benchmark and stress tests are excluded by default (use `-m benchmark` or `-m stress` to include). +Always run `python -m pytest tests/ -x -q` after changes. 648 tests expected to pass. Benchmark and stress tests are excluded by default (use `-m benchmark` or `-m stress` to include). diff --git a/README.md b/README.md index c3540e5a22..26ec03afc0 100644 --- a/README.md +++ b/README.md @@ -512,9 +512,9 @@ The AI learns AAAK and the memory protocol automatically from the `mempalace_sta Two hooks for Claude Code that automatically save memories during work: -**Save Hook** — every 15 messages, triggers a structured save. Topics, decisions, quotes, code changes. Also regenerates the critical facts layer. +**Save Hook** — Every 15 messages, auto-mines the JSONL transcript directly into the palace (capturing raw tool output — Bash results, search findings, build errors), then triggers a structured AI save with explicit instructions to preserve tool output verbatim. -**PreCompact Hook** — fires before context compression. Emergency save before the window shrinks. +**PreCompact Hook** — Fires before context compression. Auto-mines the transcript, then forces emergency save with verbatim tool output instructions. ```json { @@ -527,6 +527,8 @@ Two hooks for Claude Code that automatically save memories during work: **Optional auto-ingest:** Set the `MEMPAL_DIR` environment variable to a directory path and the hooks will automatically run `mempalace mine` on that directory during each save trigger (background on stop, synchronous on precompact). +Set `MEMPAL_PYTHON` to your Python interpreter with mempalace + chromadb installed. Auto-detects repo venv if not set. + --- ## Benchmarks @@ -632,7 +634,7 @@ Plain text. Becomes Layer 0 — loaded every session. |------|------| | `cli.py` | CLI entry point | | `config.py` | Configuration loading and defaults | -| `normalize.py` | Converts 5 chat formats to standard transcript | +| `normalize.py` | Converts 6 chat formats to standard transcript, captures tool_use/tool_result blocks from Claude Code JSONL | | `mcp_server.py` | MCP server — 19 tools, AAAK auto-teach, memory protocol | | `miner.py` | Project file ingest | | `convo_miner.py` | Conversation ingest — chunks by exchange pair | @@ -645,8 +647,8 @@ Plain text. Becomes Layer 0 — loaded every session. | `entity_registry.py` | Entity code registry | | `entity_detector.py` | Auto-detect people and projects from content | | `split_mega_files.py` | Split concatenated transcripts into per-session files | -| `hooks/mempal_save_hook.sh` | Auto-save every N messages | -| `hooks/mempal_precompact_hook.sh` | Emergency save before compaction | +| `hooks/mempal_save_hook.sh` | auto-mine + save every N messages | +| `hooks/mempal_precompact_hook.sh` | auto-mine + emergency save | --- @@ -674,15 +676,15 @@ mempalace/ │ └── membench_bench.py ← MemBench runner ├── hooks/ ← Claude Code auto-save hooks │ ├── README.md ← hook setup guide -│ ├── mempal_save_hook.sh ← save every N messages -│ └── mempal_precompact_hook.sh ← emergency save +│ ├── mempal_save_hook.sh ← auto-mine + save every N messages +│ └── mempal_precompact_hook.sh ← auto-mine + emergency save ├── examples/ ← usage examples │ ├── basic_mining.py │ ├── convo_import.py │ └── mcp_setup.md ├── tests/ ← test suite (README) ├── assets/ ← logo + brand assets -└── pyproject.toml ← package config (v3.0.0) +└── pyproject.toml ← package config (v3.1.0) ``` --- @@ -690,7 +692,7 @@ mempalace/ ## Requirements - Python 3.9+ -- `chromadb>=0.4.0` +- `chromadb>=1.5.4` - `pyyaml>=6.0` No API key. No internet after install. Everything local. diff --git a/examples/HOOKS_TUTORIAL.md b/examples/HOOKS_TUTORIAL.md index 1b09467fd0..c065beebac 100644 --- a/examples/HOOKS_TUTORIAL.md +++ b/examples/HOOKS_TUTORIAL.md @@ -25,4 +25,19 @@ Add this to your configuration file to enable automatic background saving: } ] } -} \ No newline at end of file +} +``` + +### 3. What changed (v3.1.0+) + +Both hooks now have **two-layer capture**: + +1. **Auto-mine**: Before blocking the AI, the hook runs the normalizer on the JSONL transcript and upserts chunks directly into the palace. This captures raw tool output (Bash results, search findings, build errors) that the AI would otherwise summarize away. + +2. **Updated reason messages**: The block reason now explicitly tells the AI to save tool output verbatim — not just topics and decisions. + +### 4. Configuration + +- **`SAVE_INTERVAL=15`** — How many human messages between saves +- **`MEMPAL_PYTHON`** — Python interpreter with mempalace + chromadb. Auto-detects: env var → repo venv → system python3 +- **`MEMPAL_DIR`** — Optional directory for auto-ingest via `mempalace mine` \ No newline at end of file diff --git a/mempalace/README.md b/mempalace/README.md index fdbbb62066..5553119e71 100644 --- a/mempalace/README.md +++ b/mempalace/README.md @@ -8,7 +8,7 @@ The Python package that powers MemPalace. All modules, all logic. |--------|-------------| | `cli.py` | CLI entry point — routes to mine, search, init, compress, wake-up | | `config.py` | Configuration loading — `~/.mempalace/config.json`, env vars, defaults | -| `normalize.py` | Converts 5 chat formats (Claude Code JSONL, Claude.ai JSON, ChatGPT JSON, Slack JSON, plain text) to standard transcript format | +| `normalize.py` | Converts 6 chat formats (Claude Code JSONL, OpenAI Codex CLI JSONL, Claude.ai JSON, ChatGPT JSON, Slack JSON, plain text) to standard transcript format. Captures tool_use/tool_result blocks from Claude Code JSONL with per-tool formatting. | | `miner.py` | Project file ingest — scans directories, chunks by paragraph, stores to ChromaDB | | `convo_miner.py` | Conversation ingest — chunks by exchange pair (Q+A), detects rooms from content | | `searcher.py` | Semantic search via ChromaDB vectors — filters by wing/room, returns verbatim + scores | @@ -24,6 +24,7 @@ The Python package that powers MemPalace. All modules, all logic. | `room_detector_local.py` | Maps folders to room names using 70+ patterns — no API | | `spellcheck.py` | Name-aware spellcheck — won't "correct" proper nouns in your entity registry | | `split_mega_files.py` | Splits concatenated transcript files into per-session files | +| `version.py` | Single source of truth for package version | ## Architecture diff --git a/mempalace/normalize.py b/mempalace/normalize.py index 9810332702..40a17c6e16 100644 --- a/mempalace/normalize.py +++ b/mempalace/normalize.py @@ -6,7 +6,7 @@ - Plain text with > markers (pass through) - Claude.ai JSON export - ChatGPT conversations.json - - Claude Code JSONL + - Claude Code JSONL (with tool_use/tool_result block capture) - OpenAI Codex CLI JSONL - Slack JSON export - Plain text (pass through for paragraph chunking)