diff --git a/.claude-plugin/hooks/mempal-precompact-hook.sh b/.claude-plugin/hooks/mempal-precompact-hook.sh index 0ac46ddc4b..1d4b7bfca9 100644 --- a/.claude-plugin/hooks/mempal-precompact-hook.sh +++ b/.claude-plugin/hooks/mempal-precompact-hook.sh @@ -1,5 +1,20 @@ #!/bin/bash # MemPalace PreCompact Hook — thin wrapper calling Python CLI # All logic lives in mempalace.hooks_cli for cross-harness extensibility +# +# Python resolution order: +# 1. MEMPALACE_PYTHON env var (user override) +# 2. Plugin root's venv (development installs) +# 3. System python3 (pip install --user / pipx) +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PLUGIN_ROOT="$(dirname "$SCRIPT_DIR")" + +if [ -n "$MEMPALACE_PYTHON" ] && [ -x "$MEMPALACE_PYTHON" ]; then + PYTHON="$MEMPALACE_PYTHON" +elif [ -x "$PLUGIN_ROOT/venv/bin/python3" ]; then + PYTHON="$PLUGIN_ROOT/venv/bin/python3" +else + PYTHON="python3" +fi INPUT=$(cat) -echo "$INPUT" | python3 -m mempalace hook run --hook precompact --harness claude-code +echo "$INPUT" | "$PYTHON" -m mempalace hook run --hook precompact --harness claude-code diff --git a/.claude-plugin/hooks/mempal-stop-hook.sh b/.claude-plugin/hooks/mempal-stop-hook.sh index cba3284961..2149c2175f 100644 --- a/.claude-plugin/hooks/mempal-stop-hook.sh +++ b/.claude-plugin/hooks/mempal-stop-hook.sh @@ -1,5 +1,20 @@ #!/bin/bash # MemPalace Stop Hook — thin wrapper calling Python CLI # All logic lives in mempalace.hooks_cli for cross-harness extensibility +# +# Python resolution order: +# 1. MEMPALACE_PYTHON env var (user override) +# 2. Plugin root's venv (development installs) +# 3. System python3 (pip install --user / pipx) +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PLUGIN_ROOT="$(dirname "$SCRIPT_DIR")" + +if [ -n "$MEMPALACE_PYTHON" ] && [ -x "$MEMPALACE_PYTHON" ]; then + PYTHON="$MEMPALACE_PYTHON" +elif [ -x "$PLUGIN_ROOT/venv/bin/python3" ]; then + PYTHON="$PLUGIN_ROOT/venv/bin/python3" +else + PYTHON="python3" +fi INPUT=$(cat) -echo "$INPUT" | python3 -m mempalace hook run --hook stop --harness claude-code +echo "$INPUT" | "$PYTHON" -m mempalace hook run --hook stop --harness claude-code diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index fa05a15ab9..f153c55ecb 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "mempalace", - "version": "3.0.14", + "version": "3.1.0", "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 19 MCP tools, auto-save hooks, and guided setup.", "author": { "name": "milla-jovovich" diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 5784847186..1c8e4c9bf7 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "mempalace", - "version": "3.0.14", + "version": "3.1.0", "description": "Give your AI a memory — mine projects and conversations into a searchable palace. 19 MCP tools, auto-save hooks, and guided setup.", "author": { "name": "milla-jovovich" diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000000..b32265a7c1 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,55 @@ +# CLAUDE.md — memorypalace + +## What This Is + +JP's fork of [milla-jovovich/mempalace](https://github.com/milla-jovovich/mempalace) — a local AI memory system using ChromaDB for verbatim storage and semantic search. + +- **Fork**: `jphein/mempalace` (origin) / `milla-jovovich/mempalace` (upstream) +- **Version**: 3.1.0 + local fixes +- **Python**: venv at `./venv/`, editable install with dev deps +- **Palace data**: `~/.mempalace/palace` (ChromaDB) + `~/.mempalace/config.json` + +## Key Files + +- `~/Projects/mempalace.yaml` — **do not delete**. Mining config with wing/room definitions. Regenerate with `mempalace init ~/Projects --yes` if lost. +- `~/.mempalace/config.json` — topic wings and hall keywords, customized for JP's domains (infrastructure, development, tools, creative, projects, system). +- `~/.mempalace/palace/` — ChromaDB vector store. The actual data. +- `~/.mempalace/hook_state/` — stop hook session tracking. + +## Development + +```bash +source venv/bin/activate +python -m pytest tests/ -x -q # run tests (576 expected) +mempalace status # check palace state +mempalace search "query" # test search +python -m mempalace.mcp_server # run MCP server standalone +``` + +Ruff for linting (`ruff check`), line length 100, target Python 3.9. + +## Fork Changes (ahead of upstream) + +1. **fix: epsilon mtime comparison** — `palace.py` uses `abs() < 0.01` instead of `==` for float mtime dedup +2. **feat: bulk_check_mined()** — paginated pre-fetch of all source_file/mtime pairs +3. **fix: MCP server** — search limit capped [1,100], status/taxonomy tools paginated past 10K, duplicate cache decls removed +4. **perf: batch ChromaDB writes** — one upsert per file instead of per chunk in both miners +5. **fix: entity detector STOPWORDS** — 73 technical terms added (Handler, Node, Service, etc.) +6. **feat: similarity threshold** — `min_similarity` parameter in search, default 1.5 L2 distance in MCP +7. **feat: hooks_cli** — stop hook saves directly via Python API with systemMessage notification, precompact blocks for AI-driven save, auto-ingest transcripts + +## Upstream PRs + +- milla-jovovich/mempalace#483 — mtime dedup fix +- milla-jovovich/mempalace#484 — search limit + pagination + cache fix + +## Integration + +- **Claude Code plugin**: installed at user scope via marketplace +- **MCP server**: global user scope — available in all projects +- **Stop hook**: fires every 15 messages, saves directly via Python API + systemMessage notification + auto-ingests transcript +- **PreCompact hook**: emergency save before context compaction + +## Testing + +Always run `python -m pytest tests/ -x -q` after changes. 576 tests expected to pass. Benchmark and stress tests are excluded by default (use `-m benchmark` or `-m stress` to include). diff --git a/mempalace/cli.py b/mempalace/cli.py index d8dc6970c5..dcbd31a6bd 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -95,6 +95,7 @@ def cmd_mine(args): dry_run=args.dry_run, respect_gitignore=not args.no_gitignore, include_ignored=include_ignored, + workers=args.workers, ) @@ -150,6 +151,23 @@ def cmd_split(args): sys.argv = old_argv +def cmd_export(args): + from .exporter import export_palace + + palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path + output_dir = os.path.expanduser(args.output) + + print(f"\n{'=' * 55}") + print(" MemPalace Export") + print(f"{'=' * 55}\n") + print(f" Palace: {palace_path}") + print(f" Output: {output_dir}\n") + + export_palace(palace_path=palace_path, output_dir=output_dir) + + print(f"\n{'=' * 55}\n") + + def cmd_status(args): from .miner import status @@ -442,6 +460,12 @@ def main(): default="exchange", help="Extraction strategy for convos mode: 'exchange' (default) or 'general' (5 memory types)", ) + p_mine.add_argument( + "--workers", + type=int, + default=0, + help="Parallel workers for file processing (default: min(8, cpu_count); 1 = sequential)", + ) # search p_search = sub.add_parser("search", help="Find anything, exact words") @@ -462,6 +486,15 @@ def main(): "--config", default=None, help="Entity config JSON (e.g. entities.json)" ) + # export + p_export = sub.add_parser("export", help="Export palace as browsable markdown files") + p_export.add_argument( + "--output", + "-o", + default="./palace-export", + help="Output directory (default: ./palace-export)", + ) + # wake-up p_wakeup = sub.add_parser("wake-up", help="Show L0 + L1 wake-up context (~600-900 tokens)") p_wakeup.add_argument("--wing", default=None, help="Wake-up for a specific project/wing") @@ -561,6 +594,7 @@ def main(): "mine": cmd_mine, "split": cmd_split, "search": cmd_search, + "export": cmd_export, "mcp": cmd_mcp, "compress": cmd_compress, "wake-up": cmd_wakeup, diff --git a/mempalace/config.py b/mempalace/config.py index fcfb2c8afe..81d8f2223d 100644 --- a/mempalace/config.py +++ b/mempalace/config.py @@ -173,6 +173,42 @@ def hall_keywords(self): """Mapping of hall names to keyword lists.""" return self._file_config.get("hall_keywords", DEFAULT_HALL_KEYWORDS) + @property + def hook_silent_save(self): + """Whether the stop hook saves directly (True) or blocks for MCP calls (False).""" + return self._file_config.get("hooks", {}).get("silent_save", True) + + @property + def hook_desktop_toast(self): + """Whether the stop hook shows a desktop notification via notify-send.""" + return self._file_config.get("hooks", {}).get("desktop_toast", False) + + def set_hook_setting(self, key: str, value: bool): + """Update a hook setting and write config to disk.""" + if "hooks" not in self._file_config: + self._file_config["hooks"] = {} + self._file_config["hooks"][key] = value + try: + with open(self._config_file, "w") as f: + json.dump(self._file_config, f, indent=2) + except OSError: + pass + + @property + def chunk_size(self): + """Characters per drawer chunk.""" + return self._file_config.get("chunk_size", 800) + + @property + def chunk_overlap(self): + """Overlap between adjacent chunks.""" + return self._file_config.get("chunk_overlap", 100) + + @property + def min_chunk_size(self): + """Minimum chunk size — skip smaller chunks.""" + return self._file_config.get("min_chunk_size", 50) + def init(self): """Create config directory and write default config.json if it doesn't exist.""" self._config_dir.mkdir(parents=True, exist_ok=True) @@ -187,6 +223,9 @@ def init(self): "collection_name": DEFAULT_COLLECTION_NAME, "topic_wings": DEFAULT_TOPIC_WINGS, "hall_keywords": DEFAULT_HALL_KEYWORDS, + "chunk_size": 800, + "chunk_overlap": 100, + "min_chunk_size": 50, } with open(self._config_file, "w") as f: json.dump(default_config, f, indent=2) diff --git a/mempalace/convo_miner.py b/mempalace/convo_miner.py index 7879f96652..d54e0c3bbd 100644 --- a/mempalace/convo_miner.py +++ b/mempalace/convo_miner.py @@ -326,34 +326,44 @@ def mine_convos( if extract_mode != "general": room_counts[room] += 1 - # File each chunk - drawers_added = 0 + # Batch all chunks into a single add call per file + batch_docs = [] + batch_ids = [] + batch_metas = [] for chunk in chunks: chunk_room = chunk.get("memory_type", room) if extract_mode == "general" else room if extract_mode == "general": room_counts[chunk_room] += 1 drawer_id = f"drawer_{wing}_{chunk_room}_{hashlib.sha256((source_file + str(chunk['chunk_index'])).encode()).hexdigest()[:24]}" - try: - collection.add( - documents=[chunk["content"]], - ids=[drawer_id], - metadatas=[ - { - "wing": wing, - "room": chunk_room, - "source_file": source_file, - "chunk_index": chunk["chunk_index"], - "added_by": agent, - "filed_at": datetime.now().isoformat(), - "ingest_mode": "convos", - "extract_mode": extract_mode, - } - ], - ) - drawers_added += 1 - except Exception as e: - if "already exists" not in str(e).lower(): - raise + batch_docs.append(chunk["content"]) + batch_ids.append(drawer_id) + batch_metas.append( + { + "wing": wing, + "room": chunk_room, + "source_file": source_file, + "chunk_index": chunk["chunk_index"], + "added_by": agent, + "filed_at": datetime.now().isoformat(), + "ingest_mode": "convos", + "extract_mode": extract_mode, + } + ) + drawers_added = 0 + _ADD_BATCH_SIZE = 100 + if batch_docs: + for batch_start in range(0, len(batch_docs), _ADD_BATCH_SIZE): + batch_end = batch_start + _ADD_BATCH_SIZE + try: + collection.add( + documents=batch_docs[batch_start:batch_end], + ids=batch_ids[batch_start:batch_end], + metadatas=batch_metas[batch_start:batch_end], + ) + drawers_added += len(batch_docs[batch_start:batch_end]) + except Exception as e: + if "already exists" not in str(e).lower(): + raise total_drawers += drawers_added print(f" ✓ [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers_added}") diff --git a/mempalace/entity_detector.py b/mempalace/entity_detector.py index 061778c53c..5464d1975c 100644 --- a/mempalace/entity_detector.py +++ b/mempalace/entity_detector.py @@ -393,6 +393,78 @@ "networks", "training", "inference", + # Common technical/documentation terms that appear capitalized but aren't entities + "handler", + "node", + "service", + "manager", + "client", + "server", + "worker", + "plugin", + "module", + "interface", + "event", + "request", + "response", + "update", + "config", + "builder", + "factory", + "component", + "controller", + "provider", + "wrapper", + "helper", + "util", + "parser", + "loader", + "renderer", + "adapter", + "proxy", + "listener", + "observer", + "validator", + "formatter", + "converter", + "resolver", + "selector", + "reducer", + "dispatcher", + "compiler", + "optimizer", + "analyzer", + "generator", + "template", + "registry", + "repository", + "gateway", + "middleware", + "pipeline", + "container", + "context", + "session", + "token", + "stream", + "buffer", + "cache", + "queue", + "schema", + "entity", + "instance", + "object", + "method", + "property", + "attribute", + "parameter", + "argument", + "variable", + "constant", + "function", + "package", + "framework", + "runtime", + "platform", } # For entity detection — prose only, no code files diff --git a/mempalace/exporter.py b/mempalace/exporter.py new file mode 100644 index 0000000000..c08215f307 --- /dev/null +++ b/mempalace/exporter.py @@ -0,0 +1,153 @@ +""" +exporter.py — Export the palace as a browsable folder of markdown files. + +Produces: + output_dir/ + index.md — table of contents + wing_name/ + room_name.md — one file per room, drawers as sections + +Streams drawers in paginated batches so memory usage stays bounded +regardless of palace size. +""" + +import os +import re +from collections import defaultdict +from datetime import datetime + +from .palace import get_collection + + +def _safe_path_component(name: str) -> str: + """Sanitize a string for use as a directory/file name component.""" + name = re.sub(r'[/\\:*?"<>|]', '_', name) + name = name.strip('. ') + return name or 'unknown' + + +def export_palace(palace_path: str, output_dir: str, format: str = "markdown") -> dict: + """Export all palace drawers as markdown files organized by wing/room. + + Streams drawers in batches of 1000 and writes each wing/room file + incrementally, keeping memory usage proportional to batch size rather + than total palace size. + + Args: + palace_path: Path to the ChromaDB palace directory. + output_dir: Where to write the exported markdown tree. + format: Output format (currently only "markdown"). + + Returns: + Stats dict: {"wings": N, "rooms": N, "drawers": N} + """ + col = get_collection(palace_path) + total = col.count() + + if total == 0: + print(" Palace is empty — nothing to export.") + return {"wings": 0, "rooms": 0, "drawers": 0} + + os.makedirs(output_dir, exist_ok=True) + + # Track which room files have been opened (so we can append vs overwrite) + opened_rooms: set[tuple[str, str]] = set() + # Track stats per wing: {wing: {room: count}} + wing_stats: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int)) + total_drawers = 0 + + print(f" Streaming {total} drawers...") + offset = 0 + while offset < total: + batch = col.get(limit=1000, offset=offset, include=["documents", "metadatas"]) + if not batch["ids"]: + break + + # Group this batch by wing/room so we do one file write per room per batch + batch_grouped: dict[str, dict[str, list]] = defaultdict(lambda: defaultdict(list)) + for doc_id, doc, meta in zip(batch["ids"], batch["documents"], batch["metadatas"]): + wing = meta.get("wing", "unknown") + room = meta.get("room", "general") + batch_grouped[wing][room].append({ + "id": doc_id, + "content": doc, + "source": meta.get("source_file", ""), + "filed_at": meta.get("filed_at", ""), + "added_by": meta.get("added_by", ""), + }) + + # Write/append each room file + for wing, rooms in batch_grouped.items(): + safe_wing = _safe_path_component(wing) + wing_dir = os.path.join(output_dir, safe_wing) + os.makedirs(wing_dir, exist_ok=True) + + for room, drawers in rooms.items(): + safe_room = _safe_path_component(room) + room_path = os.path.join(wing_dir, f"{safe_room}.md") + key = (wing, room) + is_new = key not in opened_rooms + + with open(room_path, "a" if not is_new else "w", encoding="utf-8") as f: + if is_new: + f.write(f"# {wing} / {room}\n\n") + opened_rooms.add(key) + + for drawer in drawers: + source = drawer["source"] or "unknown" + filed = drawer["filed_at"] or "unknown" + added_by = drawer["added_by"] or "unknown" + + f.write( + f"## {drawer['id']}\n" + f"\n" + f"> {_quote_content(drawer['content'])}\n" + f"\n" + f"| Field | Value |\n" + f"|-------|-------|\n" + f"| Source | {source} |\n" + f"| Filed | {filed} |\n" + f"| Added by | {added_by} |\n" + f"\n" + f"---\n\n" + ) + + wing_stats[wing][room] += len(drawers) + total_drawers += len(drawers) + + offset += len(batch["ids"]) + + # Build and print stats + index_rows = [] + for wing in sorted(wing_stats): + rooms = wing_stats[wing] + wing_drawer_count = sum(rooms.values()) + index_rows.append((wing, len(rooms), wing_drawer_count)) + print(f" {wing}: {len(rooms)} rooms, {wing_drawer_count} drawers") + + # Write index.md + today = datetime.now().strftime("%Y-%m-%d") + index_lines = [ + f"# Palace Export — {today}\n", + "", + "| Wing | Rooms | Drawers |", + "|------|-------|---------|", + ] + for wing, room_count, drawer_count in index_rows: + index_lines.append(f"| [{wing}]({wing}/) | {room_count} | {drawer_count} |") + index_lines.append("") + + index_path = os.path.join(output_dir, "index.md") + with open(index_path, "w", encoding="utf-8") as f: + f.write("\n".join(index_lines)) + + stats = {"wings": len(wing_stats), "rooms": sum(r for _, r, _ in index_rows), "drawers": total_drawers} + print(f"\n Exported {stats['drawers']} drawers across {stats['wings']} wings, {stats['rooms']} rooms") + print(f" Output: {output_dir}") + return stats + + +def _quote_content(text: str) -> str: + """Format content for a markdown blockquote, handling multiline.""" + lines = text.rstrip("\n").split("\n") + return "\n> ".join(lines) diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index 3f3fc09eae..57c0bbf08d 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -17,19 +17,26 @@ SAVE_INTERVAL = 15 STATE_DIR = Path.home() / ".mempalace" / "hook_state" +_RECENT_MSG_COUNT = 30 # how many recent user messages to summarize + STOP_BLOCK_REASON = ( "AUTO-SAVE checkpoint. Save key topics, decisions, quotes, and code " - "from this session to your memory system. Organize into appropriate " - "categories. Use verbatim quotes where possible. Continue conversation " - "after saving." + "from this session to MemPalace using the MCP tools:\n" + "1. Use mempalace_diary_write to save a session summary (what was discussed, " + "key decisions, current state of work).\n" + "2. Use mempalace_add_drawer for each important decision, quote, or code " + "snippet — place in the appropriate wing and room.\n" + "Use verbatim quotes where possible. Continue conversation after saving." ) PRECOMPACT_BLOCK_REASON = ( - "COMPACTION IMMINENT. Save ALL topics, decisions, quotes, code, and " - "important context from this session to your memory system. Be thorough " - "\u2014 after compaction, detailed context will be lost. Organize into " - "appropriate categories. Use verbatim quotes where possible. Save " - "everything, then allow compaction to proceed." + "COMPACTION IMMINENT — detailed context will be lost. Save ALL topics, " + "decisions, quotes, code, and important context to MemPalace using MCP tools:\n" + "1. Use mempalace_diary_write for a thorough session summary.\n" + "2. Use mempalace_add_drawer for EVERY key decision, finding, quote, and " + "code snippet — place each in the appropriate wing and room.\n" + "Be thorough — after compaction this is all that survives. Use verbatim " + "quotes. Save everything, then allow compaction to proceed." ) @@ -87,6 +94,18 @@ def _output(data: dict): print(json.dumps(data, indent=2, ensure_ascii=False)) +def _desktop_toast(body: str, title: str = "MemPalace"): + """Send a desktop notification via notify-send. Fails silently.""" + try: + subprocess.Popen( + ["notify-send", "--app-name=MemPalace", "--icon=brain", title, body], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + except OSError: + pass + + def _maybe_auto_ingest(): """If MEMPAL_DIR is set and exists, run mempalace mine in background.""" mempal_dir = os.environ.get("MEMPAL_DIR", "") @@ -103,6 +122,145 @@ def _maybe_auto_ingest(): pass +def _extract_recent_messages(transcript_path: str, count: int = _RECENT_MSG_COUNT) -> list[str]: + """Extract the last N user messages from a JSONL transcript.""" + path = Path(transcript_path).expanduser() + if not path.is_file(): + return [] + messages = [] + try: + with open(path, encoding="utf-8", errors="replace") as f: + for line in f: + try: + entry = json.loads(line) + msg = entry.get("message", {}) + if not isinstance(msg, dict) or msg.get("role") != "user": + continue + content = msg.get("content", "") + if isinstance(content, list): + content = " ".join( + b.get("text", "") for b in content if isinstance(b, dict) + ) + if not isinstance(content, str) or not content.strip(): + continue + if "" in content or "" in content: + continue + # Truncate long messages + text = content.strip()[:200] + messages.append(text) + except (json.JSONDecodeError, AttributeError): + pass + except OSError: + return [] + return messages[-count:] + + +_THEME_STOPWORDS = frozenset( + "the a an and or but in on at to for of is it i me my you your we our " + "this that with from by was were be been are not no yes can do did don't " + "will would should could have has had let's let just also like so if then " + "ok okay sure yeah hey hi here there what when where how why which some " + "all any each every about into out up down over after before between " + "get got make made need want use used using check look see run try " + "know think right now still already really very much more most too " + "file files code one two new first last next thing things way well".split() +) + + +def _extract_themes(messages: list[str], max_themes: int = 3) -> list[str]: + """Pull 2-3 distinctive topic words from recent messages.""" + from collections import Counter + words: Counter[str] = Counter() + for msg in messages: + for word in msg.lower().split(): + # Strip punctuation, keep words 4+ chars + clean = word.strip(".,;:!?\"'`()[]{}#<>/\\-_=+@$%^&*~") + if len(clean) >= 4 and clean not in _THEME_STOPWORDS and clean.isalpha(): + words[clean] += 1 + return [w for w, _ in words.most_common(max_themes)] + + +def _save_diary_direct( + transcript_path: str, session_id: str, toast: bool = False, +) -> dict: + """Write a diary checkpoint directly via Python API (no MCP calls). + + Returns {"count": N, "themes": [...]} on success, {"count": 0} on failure. + """ + messages = _extract_recent_messages(transcript_path) + if not messages: + _log("No recent messages to save") + return {"count": 0} + + themes = _extract_themes(messages) + + # Build a compressed diary entry from recent conversation + now = datetime.now() + topics = "|".join(m[:80] for m in messages[-10:]) + entry = ( + f"CHECKPOINT:{now.strftime('%Y-%m-%d')}|session:{session_id}" + f"|msgs:{len(messages)}|recent:{topics}" + ) + + try: + from .mcp_server import tool_diary_write + result = tool_diary_write( + agent_name="session-hook", + entry=entry, + topic="checkpoint", + ) + if result.get("success"): + _log(f"Diary checkpoint saved: {result.get('entry_id', '?')}") + # Write state for ack tool to read + try: + ack_file = STATE_DIR / "last_checkpoint" + ack_file.write_text( + json.dumps({"msgs": len(messages), "ts": now.isoformat()}), + encoding="utf-8", + ) + except OSError: + pass + if toast: + _desktop_toast(f"Checkpoint saved \u2014 {len(messages)} messages archived") + return {"count": len(messages), "themes": themes} + else: + _log(f"Diary checkpoint failed: {result.get('error', 'unknown')}") + except Exception as e: + _log(f"Diary checkpoint error: {e}") + return {"count": 0} + + +def _ingest_transcript(transcript_path: str): + """Mine a Claude Code session transcript into the palace as a conversation.""" + path = Path(transcript_path).expanduser() + if not path.is_file() or path.stat().st_size < 100: + return + + from .config import MempalaceConfig + + try: + MempalaceConfig() # validate config loads + except Exception: + return + + try: + log_path = STATE_DIR / "hook.log" + STATE_DIR.mkdir(parents=True, exist_ok=True) + with open(log_path, "a") as log_f: + subprocess.Popen( + [ + sys.executable, "-m", "mempalace", "mine", + str(path.parent), "--mode", "convos", + "--wing", "sessions", + ], + stdout=log_f, + stderr=log_f, + ) + _log(f"Transcript ingest started: {path.name}") + except OSError: + pass + + SUPPORTED_HARNESSES = {"claude-code", "codex"} @@ -148,18 +306,52 @@ def hook_stop(data: dict, harness: str): _log(f"Session {session_id}: {exchange_count} exchanges, {since_last} since last save") if since_last >= SAVE_INTERVAL and exchange_count > 0: - # Update last save point - try: - last_save_file.write_text(str(exchange_count), encoding="utf-8") - except OSError: - pass - _log(f"TRIGGERING SAVE at exchange {exchange_count}") - # Optional: auto-ingest if MEMPAL_DIR is set - _maybe_auto_ingest() - - _output({"decision": "block", "reason": STOP_BLOCK_REASON}) + # Read hook settings from config + from .config import MempalaceConfig + try: + config = MempalaceConfig() + silent = config.hook_silent_save + toast = config.hook_desktop_toast + except Exception: + silent = True + toast = False + + if silent: + # Save directly via Python API — systemMessage renders in terminal + result = {"count": 0} + if transcript_path: + result = _save_diary_direct(transcript_path, session_id, toast=toast) + _ingest_transcript(transcript_path) + _maybe_auto_ingest() + # Only advance save marker after successful save + count = result.get("count", 0) + if count > 0: + try: + last_save_file.write_text(str(exchange_count), encoding="utf-8") + except OSError: + pass + themes = result.get("themes", []) + if themes: + tag = " \u2014 " + ", ".join(themes) + else: + tag = "" + _output({ + "systemMessage": f"\u2726 {count} memories woven into the palace{tag}", + }) + else: + _output({}) + else: + # Legacy: block and ask Claude to save via MCP tools + try: + last_save_file.write_text(str(exchange_count), encoding="utf-8") + except OSError: + pass + if transcript_path: + _ingest_transcript(transcript_path) + _maybe_auto_ingest() + _output({"decision": "block", "reason": STOP_BLOCK_REASON}) else: _output({}) @@ -184,8 +376,14 @@ def hook_precompact(data: dict, harness: str): session_id = parsed["session_id"] _log(f"PRE-COMPACT triggered for session {session_id}") + transcript_path = parsed["transcript_path"] + + # Best-effort background ingest — spawns async subprocess, not guaranteed + # to complete before compaction but gives the palace a head start + if transcript_path: + _ingest_transcript(transcript_path) - # Optional: auto-ingest synchronously before compaction (so memories land first) + # Optional: auto-ingest project dir synchronously mempal_dir = os.environ.get("MEMPAL_DIR", "") if mempal_dir and os.path.isdir(mempal_dir): try: diff --git a/mempalace/knowledge_graph.py b/mempalace/knowledge_graph.py index b094f06f7c..2199628823 100644 --- a/mempalace/knowledge_graph.py +++ b/mempalace/knowledge_graph.py @@ -193,7 +193,7 @@ def invalidate(self, subject: str, predicate: str, obj: str, ended: str = None): # ── Query operations ────────────────────────────────────────────────── - def query_entity(self, name: str, as_of: str = None, direction: str = "outgoing"): + def query_entity(self, name: str, as_of: str = None, direction: str = "both"): """ Get all relationships for an entity. diff --git a/mempalace/layers.py b/mempalace/layers.py index 6abb99bc9b..b14f00fe0c 100644 --- a/mempalace/layers.py +++ b/mempalace/layers.py @@ -24,6 +24,7 @@ import chromadb from .config import MempalaceConfig +from .searcher import build_where_filter # --------------------------------------------------------------------------- @@ -82,6 +83,7 @@ class Layer1: MAX_DRAWERS = 15 # at most 15 moments in wake-up MAX_CHARS = 3200 # hard cap on total L1 text (~800 tokens) + MAX_SCAN = 2000 # don't scan more than this for L1 generation def __init__(self, palace_path: str = None, wing: str = None): cfg = MempalaceConfig() @@ -115,7 +117,7 @@ def generate(self) -> str: docs.extend(batch_docs) metas.extend(batch_metas) offset += len(batch_docs) - if len(batch_docs) < _BATCH: + if len(batch_docs) < _BATCH or len(docs) >= self.MAX_SCAN: break if not docs: @@ -201,13 +203,7 @@ def retrieve(self, wing: str = None, room: str = None, n_results: int = 10) -> s except Exception: return "No palace found." - where = {} - if wing and room: - where = {"$and": [{"wing": wing}, {"room": room}]} - elif wing: - where = {"wing": wing} - elif room: - where = {"room": room} + where = build_where_filter(wing, room) kwargs = {"include": ["documents", "metadatas"], "limit": n_results} if where: @@ -265,13 +261,7 @@ def search(self, query: str, wing: str = None, room: str = None, n_results: int except Exception: return "No palace found." - where = {} - if wing and room: - where = {"$and": [{"wing": wing}, {"room": room}]} - elif wing: - where = {"wing": wing} - elif room: - where = {"room": room} + where = build_where_filter(wing, room) kwargs = { "query_texts": [query], @@ -321,13 +311,7 @@ def search_raw( except Exception: return [] - where = {} - if wing and room: - where = {"$and": [{"wing": wing}, {"room": room}]} - elif wing: - where = {"wing": wing} - elif room: - where = {"room": room} + where = build_where_filter(wing, room) kwargs = { "query_texts": [query], diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index bffd3b2f2d..61644f2862 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -23,6 +23,7 @@ import json import logging import hashlib +import time from datetime import datetime from pathlib import Path @@ -90,20 +91,18 @@ def _wal_log(operation: str, params: dict, result: dict = None): "result": result, } try: + created = not _WAL_FILE.exists() with open(_WAL_FILE, "a", encoding="utf-8") as f: f.write(json.dumps(entry, default=str) + "\n") - try: - _WAL_FILE.chmod(0o600) - except (OSError, NotImplementedError): - pass + if created: + try: + _WAL_FILE.chmod(0o600) + except (OSError, NotImplementedError): + pass except Exception as e: logger.error(f"WAL write failed: {e}") -_client_cache = None -_collection_cache = None - - def _get_client(): """Return a singleton ChromaDB PersistentClient.""" global _client_cache @@ -114,13 +113,17 @@ def _get_client(): def _get_collection(create=False): """Return the ChromaDB collection, caching the client between calls.""" - global _collection_cache + global _collection_cache, _metadata_cache, _metadata_cache_time try: client = _get_client() if create: _collection_cache = client.get_or_create_collection(_config.collection_name) + _metadata_cache = None + _metadata_cache_time = 0 elif _collection_cache is None: _collection_cache = client.get_collection(_config.collection_name) + _metadata_cache = None + _metadata_cache_time = 0 return _collection_cache except Exception: return None @@ -133,6 +136,49 @@ def _no_palace(): } +# ==================== HELPERS ==================== + + +def _fetch_all_metadata(col, where=None): + """Paginate col.get() to avoid the 10K silent truncation limit.""" + total = col.count() + all_meta = [] + offset = 0 + while offset < total: + kwargs = {"include": ["metadatas"], "limit": 1000, "offset": offset} + if where: + kwargs["where"] = where + batch = col.get(**kwargs) + all_meta.extend(batch["metadatas"]) + offset += len(batch["metadatas"]) + if not batch["metadatas"]: + break + return all_meta + + +_metadata_cache = None +_metadata_cache_time = 0 +_METADATA_CACHE_TTL = 5.0 # seconds +_MAX_RESULTS = 100 # upper bound for search/list limit params + + +def _get_cached_metadata(col, where=None): + """Return cached metadata if fresh, else fetch and cache.""" + global _metadata_cache, _metadata_cache_time + now = time.time() + if ( + where is None + and _metadata_cache is not None + and (now - _metadata_cache_time) < _METADATA_CACHE_TTL + ): + return _metadata_cache + result = _fetch_all_metadata(col, where=where) + if where is None: + _metadata_cache = result + _metadata_cache_time = now + return result + + # ==================== READ TOOLS ==================== @@ -144,7 +190,7 @@ def tool_status(): wings = {} rooms = {} try: - all_meta = col.get(include=["metadatas"], limit=10000)["metadatas"] + all_meta = _get_cached_metadata(col) for m in all_meta: w = m.get("wing", "unknown") r = m.get("room", "unknown") @@ -201,7 +247,7 @@ def tool_list_wings(): return _no_palace() wings = {} try: - all_meta = col.get(include=["metadatas"], limit=10000)["metadatas"] + all_meta = _get_cached_metadata(col) for m in all_meta: w = m.get("wing", "unknown") wings[w] = wings.get(w, 0) + 1 @@ -216,10 +262,8 @@ def tool_list_rooms(wing: str = None): return _no_palace() rooms = {} try: - kwargs = {"include": ["metadatas"], "limit": 10000} - if wing: - kwargs["where"] = {"wing": wing} - all_meta = col.get(**kwargs)["metadatas"] + where = {"wing": wing} if wing else None + all_meta = _fetch_all_metadata(col, where=where) for m in all_meta: r = m.get("room", "unknown") rooms[r] = rooms.get(r, 0) + 1 @@ -234,7 +278,7 @@ def tool_get_taxonomy(): return _no_palace() taxonomy = {} try: - all_meta = col.get(include=["metadatas"], limit=10000)["metadatas"] + all_meta = _get_cached_metadata(col) for m in all_meta: w = m.get("wing", "unknown") r = m.get("room", "unknown") @@ -246,13 +290,20 @@ def tool_get_taxonomy(): return {"taxonomy": taxonomy} -def tool_search(query: str, limit: int = 5, wing: str = None, room: str = None): +def tool_search( + query: str, limit: int = 5, wing: str = None, room: str = None, + max_distance: float = 1.5, min_similarity: float = None, +): + limit = max(1, min(limit, _MAX_RESULTS)) + # Backwards compat: accept old name + dist = min_similarity if min_similarity is not None else max_distance return search_memories( query, palace_path=_config.palace_path, wing=wing, room=room, n_results=limit, + max_distance=dist, ) @@ -410,6 +461,145 @@ def tool_delete_drawer(drawer_id: str): return {"success": False, "error": str(e)} +def tool_get_drawer(drawer_id: str): + """Fetch a single drawer by ID. Returns full content and metadata.""" + col = _get_collection() + if not col: + return _no_palace() + try: + result = col.get(ids=[drawer_id], include=["documents", "metadatas"]) + if not result["ids"]: + return {"error": f"Drawer not found: {drawer_id}"} + meta = result["metadatas"][0] + doc = result["documents"][0] + return { + "drawer_id": drawer_id, + "content": doc, + "wing": meta.get("wing", ""), + "room": meta.get("room", ""), + "metadata": meta, + } + except Exception as e: + return {"error": str(e)} + + +def tool_list_drawers(wing: str = None, room: str = None, limit: int = 20, offset: int = 0): + """List drawers with pagination. Optional wing/room filter.""" + limit = max(1, min(limit, _MAX_RESULTS)) + offset = max(0, offset) + col = _get_collection() + if not col: + return _no_palace() + try: + where = None + conditions = [] + if wing: + conditions.append({"wing": wing}) + if room: + conditions.append({"room": room}) + if len(conditions) == 1: + where = conditions[0] + elif len(conditions) > 1: + where = {"$and": conditions} + + kwargs = {"include": ["documents", "metadatas"], "limit": limit, "offset": offset} + if where: + kwargs["where"] = where + result = col.get(**kwargs) + + drawers = [] + for i, did in enumerate(result["ids"]): + meta = result["metadatas"][i] + doc = result["documents"][i] + drawers.append( + { + "drawer_id": did, + "wing": meta.get("wing", ""), + "room": meta.get("room", ""), + "content_preview": doc[:200] + "..." if len(doc) > 200 else doc, + } + ) + return { + "drawers": drawers, + "count": len(drawers), + "offset": offset, + "limit": limit, + } + except Exception as e: + return {"error": str(e)} + + +def tool_update_drawer(drawer_id: str, content: str = None, wing: str = None, room: str = None): + """Update an existing drawer's content and/or metadata.""" + global _metadata_cache + + if content is None and wing is None and room is None: + return {"success": True, "drawer_id": drawer_id, "noop": True} + + col = _get_collection() + if not col: + return _no_palace() + try: + existing = col.get(ids=[drawer_id], include=["documents", "metadatas"]) + if not existing["ids"]: + return {"success": False, "error": f"Drawer not found: {drawer_id}"} + + old_meta = existing["metadatas"][0] + old_doc = existing["documents"][0] + + # Sanitize inputs + new_doc = old_doc + if content is not None: + try: + new_doc = sanitize_content(content) + except ValueError as e: + return {"success": False, "error": str(e)} + + new_meta = dict(old_meta) + if wing is not None: + try: + new_meta["wing"] = sanitize_name(wing, "wing") + except ValueError as e: + return {"success": False, "error": str(e)} + if room is not None: + try: + new_meta["room"] = sanitize_name(room, "room") + except ValueError as e: + return {"success": False, "error": str(e)} + + _wal_log( + "update_drawer", + { + "drawer_id": drawer_id, + "old_wing": old_meta.get("wing", ""), + "old_room": old_meta.get("room", ""), + "new_wing": new_meta.get("wing", ""), + "new_room": new_meta.get("room", ""), + "content_changed": content is not None, + "content_preview": new_doc[:200] if content is not None else None, + }, + ) + + update_kwargs = {"ids": [drawer_id]} + if content is not None: + update_kwargs["documents"] = [new_doc] + update_kwargs["metadatas"] = [new_meta] + col.update(**update_kwargs) + + # Invalidate metadata cache so status/taxonomy reflect changes + _metadata_cache = None + + logger.info(f"Updated drawer: {drawer_id}") + return { + "success": True, + "drawer_id": drawer_id, + "wing": new_meta.get("wing", ""), + "room": new_meta.get("room", ""), + } + except Exception as e: + return {"success": False, "error": str(e)} + + # ==================== KNOWLEDGE GRAPH ==================== @@ -585,6 +775,71 @@ def tool_diary_read(agent_name: str, last_n: int = 10): return {"error": str(e)} +# ==================== SETTINGS TOOLS ==================== + + +def tool_hook_settings(silent_save: bool = None, desktop_toast: bool = None): + """ + Get or set hook behavior settings. + + - silent_save: True = stop hook saves directly (no MCP clutter), + False = legacy blocking MCP calls. Default: True. + - desktop_toast: True = show notify-send desktop toast on save, + False = terminal-only notification. Default: False. + + Call with no arguments to see current settings. + """ + from .config import MempalaceConfig + + try: + config = MempalaceConfig() + except Exception as e: + return {"success": False, "error": str(e)} + + changed = [] + if silent_save is not None: + config.set_hook_setting("silent_save", silent_save) + changed.append(f"silent_save → {silent_save}") + if desktop_toast is not None: + config.set_hook_setting("desktop_toast", desktop_toast) + changed.append(f"desktop_toast → {desktop_toast}") + + # Re-read to return current state + try: + config = MempalaceConfig() + except Exception: + pass + + result = { + "success": True, + "settings": { + "silent_save": config.hook_silent_save, + "desktop_toast": config.hook_desktop_toast, + }, + } + if changed: + result["updated"] = changed + return result + + +def tool_memories_filed_away(): + """Acknowledge the latest silent checkpoint. Returns a short summary.""" + state_dir = Path.home() / ".mempalace" / "hook_state" + ack_file = state_dir / "last_checkpoint" + if not ack_file.is_file(): + return {"palace": "quiet", "message": "No recent journal entry"} + try: + data = json.loads(ack_file.read_text(encoding="utf-8")) + ack_file.unlink(missing_ok=True) + msgs = data.get("msgs", "?") + return { + "message": f"\u2726 {msgs} messages tucked into drawers", + "timestamp": data.get("ts", ""), + } + except (json.JSONDecodeError, OSError): + return {"message": "\u2726 Journal entry filed in the palace"} + + # ==================== MCP PROTOCOL ==================== TOOLS = { @@ -734,14 +989,18 @@ def tool_diary_read(agent_name: str, last_n: int = 10): "handler": tool_graph_stats, }, "mempalace_search": { - "description": "Semantic search. Returns verbatim drawer content with similarity scores.", + "description": "Semantic search. Returns verbatim drawer content with similarity scores. Results with L2 distance > max_distance are filtered out (lower = more similar).", "input_schema": { "type": "object", "properties": { "query": {"type": "string", "description": "What to search for"}, - "limit": {"type": "integer", "description": "Max results (default 5)"}, + "limit": {"type": "integer", "description": "Max results (default 5)", "minimum": 1, "maximum": 100}, "wing": {"type": "string", "description": "Filter by wing (optional)"}, "room": {"type": "string", "description": "Filter by room (optional)"}, + "max_distance": { + "type": "number", + "description": "Max L2 distance threshold — results further than this are dropped. Lower = stricter. Default 1.5 filters clearly irrelevant results. Set to 0 to disable filtering.", + }, }, "required": ["query"], }, @@ -794,6 +1053,62 @@ def tool_diary_read(agent_name: str, last_n: int = 10): }, "handler": tool_delete_drawer, }, + "mempalace_get_drawer": { + "description": "Fetch a single drawer by ID — returns full content and metadata.", + "input_schema": { + "type": "object", + "properties": { + "drawer_id": {"type": "string", "description": "ID of the drawer to fetch"}, + }, + "required": ["drawer_id"], + }, + "handler": tool_get_drawer, + }, + "mempalace_list_drawers": { + "description": "List drawers with pagination. Optional wing/room filter. Returns IDs, wings, rooms, and content previews.", + "input_schema": { + "type": "object", + "properties": { + "wing": {"type": "string", "description": "Filter by wing (optional)"}, + "room": {"type": "string", "description": "Filter by room (optional)"}, + "limit": { + "type": "integer", + "description": "Max results per page (default 20, max 100)", + "minimum": 1, + "maximum": 100, + }, + "offset": { + "type": "integer", + "description": "Offset for pagination (default 0)", + "minimum": 0, + }, + }, + }, + "handler": tool_list_drawers, + }, + "mempalace_update_drawer": { + "description": "Update an existing drawer's content and/or metadata (wing, room). Fetches existing drawer first; returns error if not found.", + "input_schema": { + "type": "object", + "properties": { + "drawer_id": {"type": "string", "description": "ID of the drawer to update"}, + "content": { + "type": "string", + "description": "New content (optional — omit to keep existing)", + }, + "wing": { + "type": "string", + "description": "New wing (optional — omit to keep existing)", + }, + "room": { + "type": "string", + "description": "New room (optional — omit to keep existing)", + }, + }, + "required": ["drawer_id"], + }, + "handler": tool_update_drawer, + }, "mempalace_diary_write": { "description": "Write to your personal agent diary in AAAK format. Your observations, thoughts, what you worked on, what matters. Each agent has their own diary with full history. Write in AAAK for compression — e.g. 'SESSION:2026-04-04|built.palace.graph+diary.tools|ALC.req:agent.diaries.in.aaak|★★★'. Use entity codes from the AAAK spec.", "input_schema": { @@ -834,6 +1149,32 @@ def tool_diary_read(agent_name: str, last_n: int = 10): }, "handler": tool_diary_read, }, + "mempalace_hook_settings": { + "description": ( + "Get or set hook behavior. silent_save: True = save directly " + "(no MCP clutter), False = legacy blocking. desktop_toast: " + "True = show desktop notification. Call with no args to view." + ), + "input_schema": { + "type": "object", + "properties": { + "silent_save": { + "type": "boolean", + "description": "True = silent direct save, False = blocking MCP calls", + }, + "desktop_toast": { + "type": "boolean", + "description": "True = show desktop toast via notify-send", + }, + }, + }, + "handler": tool_hook_settings, + }, + "memories_filed_away": { + "description": "Check if a recent palace checkpoint was saved. Returns message count and timestamp.", + "input_schema": {"type": "object", "properties": {}}, + "handler": tool_memories_filed_away, + }, } @@ -866,7 +1207,8 @@ def handle_request(request): "serverInfo": {"name": "mempalace", "version": __version__}, }, } - elif method == "notifications/initialized": + elif method.startswith("notifications/"): + # Notifications (no id) never get a response per JSON-RPC spec return None elif method == "tools/list": return { @@ -914,6 +1256,9 @@ def handle_request(request): "error": {"code": -32000, "message": "Internal tool error"}, } + # Notifications (missing id) must never get a response + if req_id is None: + return None return { "jsonrpc": "2.0", "id": req_id, diff --git a/mempalace/miner.py b/mempalace/miner.py index b52e6f77b1..ac7169ab3b 100644 --- a/mempalace/miner.py +++ b/mempalace/miner.py @@ -7,7 +7,9 @@ Stores verbatim chunks as drawers. No summaries. Ever. """ +import logging import os +import re import sys import hashlib import fnmatch @@ -15,9 +17,11 @@ from datetime import datetime from collections import defaultdict +logger = logging.getLogger(__name__) + import chromadb -from .palace import SKIP_DIRS, get_collection, file_already_mined +from .palace import SKIP_DIRS, get_collection, file_already_mined, bulk_check_mined READABLE_EXTENSIONS = { ".txt", @@ -279,34 +283,37 @@ def detect_room(filepath: Path, content: str, rooms: list, project_path: Path) - """ Route a file to the right room. Priority: - 1. Folder path matches a room name - 2. Filename matches a room name or keyword - 3. Content keyword scoring + 1. Folder path exactly matches a room name or keyword + 2. Filename exactly matches a room name or keyword + 3. Content keyword scoring (word-boundary matching) 4. Fallback: "general" """ relative = str(filepath.relative_to(project_path)).lower() filename = filepath.stem.lower() - content_lower = content[:2000].lower() + # Use more content for keyword scoring: full file up to 10KB, else first 5KB + scan_limit = len(content) if len(content) <= 10000 else 5000 + content_lower = content[:scan_limit].lower() - # Priority 1: folder path matches room name or keywords + # Priority 1: folder path exactly matches room name or keywords path_parts = relative.replace("\\", "/").split("/") for part in path_parts[:-1]: # skip filename itself for room in rooms: candidates = [room["name"].lower()] + [k.lower() for k in room.get("keywords", [])] - if any(part == c or c in part or part in c for c in candidates): + if any(part == c for c in candidates): return room["name"] - # Priority 2: filename matches room name + # Priority 2: filename exactly matches room name or keyword for room in rooms: - if room["name"].lower() in filename or filename in room["name"].lower(): + candidates = [room["name"].lower()] + [k.lower() for k in room.get("keywords", [])] + if any(filename == c for c in candidates): return room["name"] - # Priority 3: keyword scoring from room keywords + name + # Priority 3: keyword scoring with word-boundary matching scores = defaultdict(int) for room in rooms: keywords = room.get("keywords", []) + [room["name"]] for kw in keywords: - count = content_lower.count(kw.lower()) + count = len(re.findall(r'\b' + re.escape(kw.lower()) + r'\b', content_lower)) scores[room["name"]] += count if scores: @@ -322,12 +329,32 @@ def detect_room(filepath: Path, content: str, rooms: list, project_path: Path) - # ============================================================================= -def chunk_text(content: str, source_file: str) -> list: +def chunk_text( + content: str, + source_file: str, + chunk_size: int = None, + chunk_overlap: int = None, + min_chunk_size: int = None, +) -> list: """ Split content into drawer-sized chunks. Tries to split on paragraph/line boundaries. Returns list of {"content": str, "chunk_index": int} + + Optional params override module-level defaults when provided. """ + if chunk_size is None: + chunk_size = CHUNK_SIZE + if chunk_overlap is None: + chunk_overlap = CHUNK_OVERLAP + if min_chunk_size is None: + min_chunk_size = MIN_CHUNK_SIZE + + if chunk_overlap < 0 or chunk_overlap >= chunk_size: + raise ValueError( + f"chunk_overlap ({chunk_overlap}) must be >= 0 and < chunk_size ({chunk_size})" + ) + # Clean up content = content.strip() if not content: @@ -338,20 +365,20 @@ def chunk_text(content: str, source_file: str) -> list: chunk_index = 0 while start < len(content): - end = min(start + CHUNK_SIZE, len(content)) + end = min(start + chunk_size, len(content)) # Try to break at paragraph boundary if end < len(content): newline_pos = content.rfind("\n\n", start, end) - if newline_pos > start + CHUNK_SIZE // 2: + if newline_pos > start + chunk_size // 2: end = newline_pos else: newline_pos = content.rfind("\n", start, end) - if newline_pos > start + CHUNK_SIZE // 2: + if newline_pos > start + chunk_size // 2: end = newline_pos chunk = content[start:end].strip() - if len(chunk) >= MIN_CHUNK_SIZE: + if len(chunk) >= min_chunk_size: chunks.append( { "content": chunk, @@ -360,7 +387,7 @@ def chunk_text(content: str, source_file: str) -> list: ) chunk_index += 1 - start = end - CHUNK_OVERLAP if end < len(content) else end + start = end - chunk_overlap if end < len(content) else end return chunks @@ -404,53 +431,133 @@ def add_drawer( # ============================================================================= -def process_file( +def _prepare_file( filepath: Path, project_path: Path, - collection, wing: str, rooms: list, agent: str, - dry_run: bool, + chunk_size: int = None, + chunk_overlap: int = None, + min_chunk_size: int = None, ) -> tuple: - """Read, chunk, route, and file one file. Returns (drawer_count, room_name).""" + """Read, chunk, and route one file without writing to ChromaDB. - # Skip if already filed + Returns (batch_docs, batch_ids, batch_metas, room) or (None, None, None, None) + when the file should be skipped (unreadable, too small, etc.). + This is the pure-computation half of process_file, safe for concurrent use. + """ + effective_min = min_chunk_size if min_chunk_size is not None else MIN_CHUNK_SIZE source_file = str(filepath) - if not dry_run and file_already_mined(collection, source_file, check_mtime=True): - return 0, None try: content = filepath.read_text(encoding="utf-8", errors="replace") except OSError: - return 0, None + return None, None, None, None content = content.strip() - if len(content) < MIN_CHUNK_SIZE: - return 0, None + if len(content) < effective_min: + return None, None, None, None room = detect_room(filepath, content, rooms, project_path) - chunks = chunk_text(content, source_file) + chunks = chunk_text( + content, + source_file, + chunk_size=chunk_size, + chunk_overlap=chunk_overlap, + min_chunk_size=min_chunk_size, + ) + + if not chunks: + return None, None, None, None + + batch_docs = [] + batch_ids = [] + batch_metas = [] + try: + file_mtime = os.path.getmtime(source_file) + except OSError: + file_mtime = None + + for chunk in chunks: + drawer_id = f"drawer_{wing}_{room}_{hashlib.sha256((source_file + str(chunk['chunk_index'])).encode()).hexdigest()[:24]}" + metadata = { + "wing": wing, + "room": room, + "source_file": source_file, + "chunk_index": chunk["chunk_index"], + "added_by": agent, + "filed_at": datetime.now().isoformat(), + } + if file_mtime is not None: + metadata["source_mtime"] = file_mtime + batch_docs.append(chunk["content"]) + batch_ids.append(drawer_id) + batch_metas.append(metadata) + + return batch_docs, batch_ids, batch_metas, room + + +def process_file( + filepath: Path, + project_path: Path, + collection, + wing: str, + rooms: list, + agent: str, + dry_run: bool, + chunk_size: int = None, + chunk_overlap: int = None, + min_chunk_size: int = None, +) -> tuple: + """Read, chunk, route, and file one file. Returns (drawer_count, room_name).""" + effective_min = min_chunk_size if min_chunk_size is not None else MIN_CHUNK_SIZE + + # Skip if already filed + source_file = str(filepath) + if not dry_run and file_already_mined(collection, source_file, check_mtime=True): + return 0, None if dry_run: + # Still need to read/chunk for the dry-run report + try: + content = filepath.read_text(encoding="utf-8", errors="replace") + except OSError: + return 0, None + content = content.strip() + if len(content) < effective_min: + return 0, None + room = detect_room(filepath, content, rooms, project_path) + chunks = chunk_text( + content, + source_file, + chunk_size=chunk_size, + chunk_overlap=chunk_overlap, + min_chunk_size=min_chunk_size, + ) print(f" [DRY RUN] {filepath.name} → room:{room} ({len(chunks)} drawers)") return len(chunks), room - drawers_added = 0 - for chunk in chunks: - added = add_drawer( - collection=collection, - wing=wing, - room=room, - content=chunk["content"], - source_file=source_file, - chunk_index=chunk["chunk_index"], - agent=agent, - ) - if added: - drawers_added += 1 + batch_docs, batch_ids, batch_metas, room = _prepare_file( + filepath, + project_path, + wing, + rooms, + agent, + chunk_size=chunk_size, + chunk_overlap=chunk_overlap, + min_chunk_size=min_chunk_size, + ) + if batch_docs is None: + return 0, None + + collection.upsert( + documents=batch_docs, + ids=batch_ids, + metadatas=batch_metas, + ) - return drawers_added, room + return len(batch_docs), room # ============================================================================= @@ -527,6 +634,26 @@ def scan_project( # ============================================================================= +def _is_already_mined(source_file: str, mined_map: dict) -> bool: + """Check if a file is already mined using the bulk-fetched mined_map. + + Compares stored mtime against current file mtime using epsilon tolerance, + matching the logic in file_already_mined() but without per-file DB queries. + """ + stored_mtime = mined_map.get(source_file) + if stored_mtime is None: + return False + try: + current_mtime = os.path.getmtime(source_file) + return abs(float(stored_mtime) - current_mtime) < 0.01 + except (OSError, TypeError, ValueError): + return False + + +# Maximum documents per ChromaDB upsert call +_UPSERT_BATCH_SIZE = 100 + + def mine( project_dir: str, palace_path: str, @@ -536,11 +663,26 @@ def mine( dry_run: bool = False, respect_gitignore: bool = True, include_ignored: list = None, + workers: int = 0, ): - """Mine a project directory into the palace.""" + """Mine a project directory into the palace. + + When workers > 1, files are read/chunked/routed in parallel threads + and then written to ChromaDB sequentially (the Python client is not + thread-safe for concurrent writes to the same collection). + """ + import concurrent.futures + import threading + + from .config import MempalaceConfig project_path = Path(project_dir).expanduser().resolve() config = load_config(project_dir) + palace_config = MempalaceConfig() + + cfg_chunk_size = palace_config.chunk_size + cfg_chunk_overlap = palace_config.chunk_overlap + cfg_min_chunk_size = palace_config.min_chunk_size wing = wing_override or config["wing"] rooms = config.get("rooms", [{"name": "general", "description": "All project files"}]) @@ -553,6 +695,9 @@ def mine( if limit > 0: files = files[:limit] + if workers <= 0: + workers = min(8, os.cpu_count() or 4) + print(f"\n{'=' * 55}") print(" MemPalace Mine") print(f"{'=' * 55}") @@ -560,6 +705,8 @@ def mine( print(f" Rooms: {', '.join(r['name'] for r in rooms)}") print(f" Files: {len(files)}") print(f" Palace: {palace_path}") + if workers > 1: + print(f" Workers: {workers}") if dry_run: print(" DRY RUN — nothing will be filed") if not respect_gitignore: @@ -577,23 +724,111 @@ def mine( files_skipped = 0 room_counts = defaultdict(int) - for i, filepath in enumerate(files, 1): - drawers, room = process_file( - filepath=filepath, - project_path=project_path, - collection=collection, - wing=wing, - rooms=rooms, - agent=agent, - dry_run=dry_run, - ) - if drawers == 0 and not dry_run: - files_skipped += 1 - else: - total_drawers += drawers - room_counts[room] += 1 - if not dry_run: - print(f" ✓ [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers}") + # --- Sequential path (workers=1 or dry_run) --- + if workers <= 1 or dry_run: + for i, filepath in enumerate(files, 1): + drawers, room = process_file( + filepath=filepath, + project_path=project_path, + collection=collection, + wing=wing, + rooms=rooms, + agent=agent, + dry_run=dry_run, + chunk_size=cfg_chunk_size, + chunk_overlap=cfg_chunk_overlap, + min_chunk_size=cfg_min_chunk_size, + ) + if drawers == 0 and not dry_run: + files_skipped += 1 + else: + total_drawers += drawers + room_counts[room] += 1 + if not dry_run: + print(f" \u2713 [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers}") + else: + # --- Concurrent path (workers > 1) --- + + # Phase 0: bulk-fetch already-mined mtimes to skip files without + # per-file DB queries. + mined_map = bulk_check_mined(collection) + + # Filter out already-mined files before spawning threads. + files_to_process = [] + for filepath in files: + if _is_already_mined(str(filepath), mined_map): + files_skipped += 1 + else: + files_to_process.append(filepath) + + # Phase 1: parallel read/chunk/route + counter_lock = threading.Lock() + processed_count = 0 + + def prepare_one(filepath): + return filepath, _prepare_file( + filepath, + project_path, + wing, + rooms, + agent, + chunk_size=cfg_chunk_size, + chunk_overlap=cfg_chunk_overlap, + min_chunk_size=cfg_min_chunk_size, + ) + + # Phase 1 read/chunk + Phase 2 write as futures complete (stream to DB) + pending_docs = [] + pending_ids = [] + pending_metas = [] + + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool: + futures = {pool.submit(prepare_one, fp): fp for fp in files_to_process} + for future in concurrent.futures.as_completed(futures): + try: + filepath, (batch_docs, batch_ids, batch_metas, room) = future.result() + except Exception as exc: + failed_path = futures[future] + logger.warning("Skipping %s: %s", failed_path, exc) + with counter_lock: + files_skipped += 1 + continue + if batch_docs is None: + with counter_lock: + files_skipped += 1 + continue + + total_drawers += len(batch_docs) + room_counts[room] += 1 + pending_docs.extend(batch_docs) + pending_ids.extend(batch_ids) + pending_metas.extend(batch_metas) + + # Flush when batch is large enough + if len(pending_docs) >= _UPSERT_BATCH_SIZE: + collection.upsert( + documents=pending_docs, + ids=pending_ids, + metadatas=pending_metas, + ) + pending_docs = [] + pending_ids = [] + pending_metas = [] + + with counter_lock: + processed_count += 1 + print( + f" \u2713 [{processed_count:4}/{len(files_to_process)}] " + f"{filepath.name[:50]:50} +{len(batch_docs)}" + ) + + # Flush remainder + if pending_docs: + collection.upsert( + documents=pending_docs, + ids=pending_ids, + metadatas=pending_metas, + ) print(f"\n{'=' * 55}") print(" Done.") diff --git a/mempalace/palace.py b/mempalace/palace.py index 6ddf19084c..537200d053 100644 --- a/mempalace/palace.py +++ b/mempalace/palace.py @@ -4,9 +4,13 @@ Consolidates ChromaDB access patterns used by both miners and the MCP server. """ +import logging import os + import chromadb +logger = logging.getLogger(__name__) + SKIP_DIRS = { ".git", "node_modules", @@ -65,7 +69,36 @@ def file_already_mined(collection, source_file: str, check_mtime: bool = False) if stored_mtime is None: return False current_mtime = os.path.getmtime(source_file) - return float(stored_mtime) == current_mtime + return abs(float(stored_mtime) - current_mtime) < 0.01 return True except Exception: return False + + +def bulk_check_mined(collection) -> dict[str, float]: + """Pre-fetch source_file/source_mtime pairs for all documents in the collection. + + Returns a dict mapping source_file -> source_mtime (as float) for every + document that has both fields. Callers can check membership and compare + mtimes locally instead of issuing one ChromaDB query per file. + + Fetches the full collection in paginated batches (like palace_graph.py) + since a WHERE-IN filter on thousands of paths is not supported by ChromaDB. + """ + mined: dict[str, float] = {} + try: + total = collection.count() + offset = 0 + while offset < total: + batch = collection.get(limit=1000, offset=offset, include=["metadatas"]) + for meta in batch["metadatas"]: + src = meta.get("source_file") + mtime = meta.get("source_mtime") + if src and mtime is not None: + mined[src] = float(mtime) + if not batch["ids"]: + break + offset += len(batch["ids"]) + except Exception: + logger.warning("bulk_check_mined: partial fetch, %d files loaded", len(mined)) + return mined diff --git a/mempalace/searcher.py b/mempalace/searcher.py index 163abd88c5..1b3bbcabbd 100644 --- a/mempalace/searcher.py +++ b/mempalace/searcher.py @@ -18,6 +18,17 @@ class SearchError(Exception): """Raised when search cannot proceed (e.g. no palace found).""" +def build_where_filter(wing: str = None, room: str = None) -> dict: + """Build ChromaDB where filter for wing/room filtering.""" + if wing and room: + return {"$and": [{"wing": wing}, {"room": room}]} + elif wing: + return {"wing": wing} + elif room: + return {"room": room} + return {} + + def search(query: str, palace_path: str, wing: str = None, room: str = None, n_results: int = 5): """ Search the palace. Returns verbatim drawer content. @@ -31,14 +42,7 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r print(" Run: mempalace init then mempalace mine ") raise SearchError(f"No palace found at {palace_path}") - # Build where filter - where = {} - if wing and room: - where = {"$and": [{"wing": wing}, {"room": room}]} - elif wing: - where = {"wing": wing} - elif room: - where = {"room": room} + where = build_where_filter(wing, room) try: kwargs = { @@ -91,11 +95,27 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r def search_memories( - query: str, palace_path: str, wing: str = None, room: str = None, n_results: int = 5 + query: str, + palace_path: str, + wing: str = None, + room: str = None, + n_results: int = 5, + max_distance: float = 0.0, ) -> dict: - """ - Programmatic search — returns a dict instead of printing. + """Programmatic search — returns a dict instead of printing. + Used by the MCP server and other callers that need data. + + Args: + query: Natural language search query. + palace_path: Path to the ChromaDB palace directory. + wing: Optional wing filter. + room: Optional room filter. + n_results: Max results to return. + max_distance: Max L2 (Euclidean) distance threshold. ChromaDB uses + L2 distance by default — 0 = identical, larger = less similar. + Results with distance > this value are filtered out. A value of + 0.0 disables filtering. Typical useful range: 0.5–1.5. """ try: client = chromadb.PersistentClient(path=palace_path) @@ -107,14 +127,7 @@ def search_memories( "hint": "Run: mempalace init && mempalace mine ", } - # Build where filter - where = {} - if wing and room: - where = {"$and": [{"wing": wing}, {"room": room}]} - elif wing: - where = {"wing": wing} - elif room: - where = {"room": room} + where = build_where_filter(wing, room) try: kwargs = { @@ -135,6 +148,9 @@ def search_memories( hits = [] for doc, meta, dist in zip(docs, metas, dists): + # Filter on raw distance before rounding to avoid precision loss + if max_distance > 0.0 and dist > max_distance: + continue hits.append( { "text": doc, @@ -142,11 +158,13 @@ def search_memories( "room": meta.get("room", "unknown"), "source_file": Path(meta.get("source_file", "?")).name, "similarity": round(1 - dist, 3), + "distance": round(dist, 4), } ) return { "query": query, "filters": {"wing": wing, "room": room}, + "total_before_filter": len(docs), "results": hits, } diff --git a/tests/test_cli.py b/tests/test_cli.py index e3c68f9b57..d3280b2a97 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -163,6 +163,7 @@ def test_cmd_mine_projects_mode(mock_config_cls): no_gitignore=False, include_ignored=[], extract="exchange", + workers=0, ) with patch("mempalace.miner.mine") as mock_mine: cmd_mine(args) @@ -175,6 +176,7 @@ def test_cmd_mine_projects_mode(mock_config_cls): dry_run=False, respect_gitignore=True, include_ignored=[], + workers=0, ) @@ -220,6 +222,7 @@ def test_cmd_mine_include_ignored_comma_split(mock_config_cls): no_gitignore=False, include_ignored=["a.txt,b.txt", "c.txt"], extract="exchange", + workers=0, ) with patch("mempalace.miner.mine") as mock_mine: cmd_mine(args) diff --git a/tests/test_exporter.py b/tests/test_exporter.py new file mode 100644 index 0000000000..0597ec1c2f --- /dev/null +++ b/tests/test_exporter.py @@ -0,0 +1,136 @@ +import os +import shutil +import tempfile +from pathlib import Path + +import yaml + +from mempalace.miner import mine +from mempalace.exporter import export_palace + + +def write_file(path: Path, content: str): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def _setup_palace(tmpdir): + """Create a small palace with drawers across two wings for testing.""" + project_a = Path(tmpdir) / "project_a" + project_b = Path(tmpdir) / "project_b" + palace_path = str(Path(tmpdir) / "palace") + + # Project A: wing=alpha, rooms=backend,frontend + os.makedirs(project_a / "backend") + os.makedirs(project_a / "frontend") + write_file(project_a / "backend" / "server.py", "def serve():\n return 'ok'\n" * 20) + write_file(project_a / "frontend" / "app.js", "function render() { return 'hi'; }\n" * 20) + with open(project_a / "mempalace.yaml", "w") as f: + yaml.dump( + { + "wing": "alpha", + "rooms": [ + {"name": "backend", "description": "Backend code"}, + {"name": "frontend", "description": "Frontend code"}, + ], + }, + f, + ) + + # Project B: wing=beta, rooms=docs + os.makedirs(project_b / "docs") + write_file(project_b / "docs" / "guide.md", "# Guide\n\nThis explains things.\n" * 20) + with open(project_b / "mempalace.yaml", "w") as f: + yaml.dump( + { + "wing": "beta", + "rooms": [{"name": "docs", "description": "Documentation"}], + }, + f, + ) + + mine(str(project_a), palace_path) + mine(str(project_b), palace_path) + + return palace_path + + +def test_export_creates_structure(): + tmpdir = tempfile.mkdtemp() + try: + palace_path = _setup_palace(tmpdir) + output_dir = os.path.join(tmpdir, "export") + + stats = export_palace(palace_path, output_dir) + + # Should have two wings + assert stats["wings"] == 2 + assert stats["rooms"] >= 2 + assert stats["drawers"] >= 3 + + # Directory structure + assert os.path.isfile(os.path.join(output_dir, "index.md")) + assert os.path.isdir(os.path.join(output_dir, "alpha")) + assert os.path.isdir(os.path.join(output_dir, "beta")) + + # Room files exist + assert os.path.isfile(os.path.join(output_dir, "alpha", "backend.md")) + assert os.path.isfile(os.path.join(output_dir, "alpha", "frontend.md")) + assert os.path.isfile(os.path.join(output_dir, "beta", "docs.md")) + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def test_export_markdown_content(): + tmpdir = tempfile.mkdtemp() + try: + palace_path = _setup_palace(tmpdir) + output_dir = os.path.join(tmpdir, "export") + + export_palace(palace_path, output_dir) + + # Check that room files contain expected markdown elements + backend_md = Path(output_dir) / "alpha" / "backend.md" + content = backend_md.read_text(encoding="utf-8") + + assert content.startswith("# alpha / backend\n") + assert "## drawer_" in content + assert "| Field | Value |" in content + assert "| Source |" in content + assert "| Filed |" in content + assert "| Added by |" in content + assert "---" in content + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def test_export_index_content(): + tmpdir = tempfile.mkdtemp() + try: + palace_path = _setup_palace(tmpdir) + output_dir = os.path.join(tmpdir, "export") + + export_palace(palace_path, output_dir) + + index_md = Path(output_dir) / "index.md" + content = index_md.read_text(encoding="utf-8") + + assert "# Palace Export" in content + assert "| Wing | Rooms | Drawers |" in content + assert "[alpha](alpha/)" in content + assert "[beta](beta/)" in content + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def test_export_empty_palace(): + tmpdir = tempfile.mkdtemp() + try: + palace_path = os.path.join(tmpdir, "empty_palace") + output_dir = os.path.join(tmpdir, "export") + + stats = export_palace(palace_path, output_dir) + + assert stats == {"wings": 0, "rooms": 0, "drawers": 0} + finally: + shutil.rmtree(tmpdir, ignore_errors=True) diff --git a/tests/test_hooks_cli.py b/tests/test_hooks_cli.py index 5a1870e02f..c6c5655755 100644 --- a/tests/test_hooks_cli.py +++ b/tests/test_hooks_cli.py @@ -8,9 +8,9 @@ from mempalace.hooks_cli import ( SAVE_INTERVAL, - STOP_BLOCK_REASON, PRECOMPACT_BLOCK_REASON, _count_human_messages, + _extract_recent_messages, _log, _maybe_auto_ingest, _parse_harness_input, @@ -105,6 +105,40 @@ def test_count_malformed_json_lines(tmp_path): assert _count_human_messages(str(transcript)) == 1 +# --- _extract_recent_messages --- + + +def test_extract_recent_messages_basic(tmp_path): + transcript = tmp_path / "t.jsonl" + _write_transcript( + transcript, + [{"message": {"role": "user", "content": f"msg {i}"}} for i in range(5)], + ) + msgs = _extract_recent_messages(str(transcript), count=3) + assert len(msgs) == 3 + assert msgs[0] == "msg 2" + assert msgs[2] == "msg 4" + + +def test_extract_recent_messages_skips_commands(tmp_path): + transcript = tmp_path / "t.jsonl" + _write_transcript( + transcript, + [ + {"message": {"role": "user", "content": "real msg"}}, + {"message": {"role": "user", "content": "status"}}, + {"message": {"role": "user", "content": "hook"}}, + ], + ) + msgs = _extract_recent_messages(str(transcript)) + assert len(msgs) == 1 + assert msgs[0] == "real msg" + + +def test_extract_recent_messages_missing_file(): + assert _extract_recent_messages("/nonexistent.jsonl") == [] + + # --- hook_stop --- @@ -157,19 +191,23 @@ def test_stop_hook_passthrough_below_interval(tmp_path): assert result == {} -def test_stop_hook_blocks_at_interval(tmp_path): +def test_stop_hook_saves_silently_at_interval(tmp_path): transcript = tmp_path / "t.jsonl" _write_transcript( transcript, [{"message": {"role": "user", "content": f"msg {i}"}} for i in range(SAVE_INTERVAL)], ) - result = _capture_hook_output( - hook_stop, - {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}, - state_dir=tmp_path, - ) - assert result["decision"] == "block" - assert result["reason"] == STOP_BLOCK_REASON + save_result = {"count": 15, "themes": ["hooks", "notifications"]} + with patch("mempalace.hooks_cli._save_diary_direct", return_value=save_result) as mock_save: + result = _capture_hook_output( + hook_stop, + {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}, + state_dir=tmp_path, + ) + # Saves silently — systemMessage notification with themes, no block + assert result["systemMessage"].startswith("\u2726 15 memories woven into the palace") + assert "hooks" in result["systemMessage"] + mock_save.assert_called_once_with(str(transcript), "test", toast=False) def test_stop_hook_tracks_save_point(tmp_path): @@ -180,13 +218,17 @@ def test_stop_hook_tracks_save_point(tmp_path): ) data = {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)} - # First call blocks - result = _capture_hook_output(hook_stop, data, state_dir=tmp_path) - assert result["decision"] == "block" + # First call saves silently with systemMessage notification + save_result = {"count": 15, "themes": ["hooks"]} + with patch("mempalace.hooks_cli._save_diary_direct", return_value=save_result): + result = _capture_hook_output(hook_stop, data, state_dir=tmp_path) + assert "systemMessage" in result # Second call with same count passes through (already saved) - result = _capture_hook_output(hook_stop, data, state_dir=tmp_path) + with patch("mempalace.hooks_cli._save_diary_direct") as mock_save: + result = _capture_hook_output(hook_stop, data, state_dir=tmp_path) assert result == {} + mock_save.assert_not_called() # --- hook_session_start --- @@ -295,12 +337,15 @@ def test_stop_hook_oserror_on_last_save_read(tmp_path): ) # Write invalid content to last save file (tmp_path / "test_last_save").write_text("not_a_number") - result = _capture_hook_output( - hook_stop, - {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}, - state_dir=tmp_path, - ) - assert result["decision"] == "block" + save_result = {"count": 15, "themes": ["testing"]} + with patch("mempalace.hooks_cli._save_diary_direct", return_value=save_result): + result = _capture_hook_output( + hook_stop, + {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}, + state_dir=tmp_path, + ) + assert "systemMessage" in result + assert "15 memories" in result["systemMessage"] def test_stop_hook_oserror_on_write(tmp_path): @@ -314,18 +359,20 @@ def test_stop_hook_oserror_on_write(tmp_path): def bad_write_text(*args, **kwargs): raise OSError("disk full") + save_result = {"count": 15, "themes": []} with patch("mempalace.hooks_cli.STATE_DIR", tmp_path): - with patch.object(Path, "write_text", bad_write_text): - result = _capture_hook_output( - hook_stop, - { - "session_id": "test", - "stop_hook_active": False, - "transcript_path": str(transcript), - }, - state_dir=tmp_path, - ) - assert result["decision"] == "block" + with patch("mempalace.hooks_cli._save_diary_direct", return_value=save_result): + with patch.object(Path, "write_text", bad_write_text): + result = _capture_hook_output( + hook_stop, + { + "session_id": "test", + "stop_hook_active": False, + "transcript_path": str(transcript), + }, + state_dir=tmp_path, + ) + assert "systemMessage" in result # --- hook_precompact with MEMPAL_DIR --- diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 96fe80cd07..d777ec39ac 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -138,6 +138,26 @@ def test_unknown_method(self): resp = handle_request({"method": "unknown/method", "id": 4, "params": {}}) assert resp["error"]["code"] == -32601 + def test_any_notification_returns_none(self): + """All notifications/* methods should return None (no response).""" + from mempalace.mcp_server import handle_request + + for method in [ + "notifications/initialized", + "notifications/cancelled", + "notifications/progress", + "notifications/roots/list_changed", + ]: + resp = handle_request({"method": method, "params": {}}) + assert resp is None, f"{method} should return None" + + def test_unknown_method_no_id_returns_none(self): + """Messages without id (notifications) must never get a response.""" + from mempalace.mcp_server import handle_request + + resp = handle_request({"method": "unknown/thing", "params": {}}) + assert resp is None + def test_tools_call_dispatches(self, monkeypatch, config, palace_path, seeded_kg): _patch_mcp_server(monkeypatch, config, seeded_kg) from mempalace.mcp_server import handle_request @@ -321,6 +341,97 @@ def test_check_duplicate(self, monkeypatch, config, palace_path, seeded_collecti ) assert result["is_duplicate"] is False + def test_get_drawer(self, monkeypatch, config, palace_path, seeded_collection, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_get_drawer + + result = tool_get_drawer("drawer_proj_backend_aaa") + assert result["drawer_id"] == "drawer_proj_backend_aaa" + assert result["wing"] == "project" + assert result["room"] == "backend" + assert "JWT tokens" in result["content"] + + def test_get_drawer_not_found(self, monkeypatch, config, palace_path, seeded_collection, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_get_drawer + + result = tool_get_drawer("nonexistent_drawer") + assert "error" in result + + def test_list_drawers(self, monkeypatch, config, palace_path, seeded_collection, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_list_drawers + + result = tool_list_drawers() + assert result["count"] == 4 + assert len(result["drawers"]) == 4 + + def test_list_drawers_with_wing_filter(self, monkeypatch, config, palace_path, seeded_collection, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_list_drawers + + result = tool_list_drawers(wing="project") + assert result["count"] == 3 + assert all(d["wing"] == "project" for d in result["drawers"]) + + def test_list_drawers_with_room_filter(self, monkeypatch, config, palace_path, seeded_collection, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_list_drawers + + result = tool_list_drawers(wing="project", room="backend") + assert result["count"] == 2 + assert all(d["room"] == "backend" for d in result["drawers"]) + + def test_list_drawers_pagination(self, monkeypatch, config, palace_path, seeded_collection, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_list_drawers + + result = tool_list_drawers(limit=2, offset=0) + assert result["count"] == 2 + assert result["limit"] == 2 + assert result["offset"] == 0 + + def test_list_drawers_negative_offset_clamped(self, monkeypatch, config, palace_path, seeded_collection, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_list_drawers + + result = tool_list_drawers(offset=-5) + assert result["offset"] == 0 + + def test_update_drawer_content(self, monkeypatch, config, palace_path, seeded_collection, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_update_drawer, tool_get_drawer + + result = tool_update_drawer("drawer_proj_backend_aaa", content="Updated content about auth.") + assert result["success"] is True + + fetched = tool_get_drawer("drawer_proj_backend_aaa") + assert fetched["content"] == "Updated content about auth." + + def test_update_drawer_wing_and_room(self, monkeypatch, config, palace_path, seeded_collection, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_update_drawer + + result = tool_update_drawer("drawer_proj_backend_aaa", wing="new_wing", room="new_room") + assert result["success"] is True + assert result["wing"] == "new_wing" + assert result["room"] == "new_room" + + def test_update_drawer_not_found(self, monkeypatch, config, palace_path, seeded_collection, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_update_drawer + + result = tool_update_drawer("nonexistent_drawer", content="hello") + assert result["success"] is False + + def test_update_drawer_noop(self, monkeypatch, config, palace_path, seeded_collection, kg): + _patch_mcp_server(monkeypatch, config, kg) + from mempalace.mcp_server import tool_update_drawer + + result = tool_update_drawer("drawer_proj_backend_aaa") + assert result["success"] is True + assert result.get("noop") is True + # ── KG Tools ──────────────────────────────────────────────────────────── diff --git a/tests/test_miner.py b/tests/test_miner.py index c013d7c25f..aa6c4df4e9 100644 --- a/tests/test_miner.py +++ b/tests/test_miner.py @@ -6,7 +6,15 @@ import chromadb import yaml -from mempalace.miner import mine, scan_project +from mempalace.miner import ( + CHUNK_OVERLAP, + CHUNK_SIZE, + MIN_CHUNK_SIZE, + chunk_text, + detect_room, + mine, + scan_project, +) from mempalace.palace import file_already_mined @@ -260,3 +268,197 @@ def test_file_already_mined_check_mtime(): # Release ChromaDB file handles before cleanup (required on Windows) del col, client shutil.rmtree(tmpdir, ignore_errors=True) + + +# ============================================================================= +# detect_room tests +# ============================================================================= + +SAMPLE_ROOMS = [ + {"name": "backend", "description": "Backend code", "keywords": ["api", "server"]}, + {"name": "frontend", "description": "Frontend code", "keywords": ["ui", "component"]}, + {"name": "tests", "description": "Test files", "keywords": ["test", "spec"]}, +] + + +def _detect(relpath: str, content: str = "", rooms: list = None): + """Helper: call detect_room with a fake project path.""" + project = Path("/fake/project") + filepath = project / relpath + return detect_room(filepath, content, rooms or SAMPLE_ROOMS, project) + + +def test_detect_room_priority1_exact_folder_match(): + """Folder named exactly 'backend' routes to backend room.""" + assert _detect("backend/app.py") == "backend" + + +def test_detect_room_priority1_keyword_folder_match(): + """Folder named exactly 'api' routes to backend room (keyword match).""" + assert _detect("api/routes.py") == "backend" + + +def test_detect_room_priority1_no_substring_match(): + """Folder 'components' must NOT match room 'component' keyword via substring.""" + assert _detect("components/button.py") != "frontend" + + +def test_detect_room_priority1_no_short_name_false_positive(): + """Folder 'src' must NOT match any room just because 'src' is a substring of something.""" + result = _detect("src/main.py", content="unrelated stuff") + assert result == "general" + + +def test_detect_room_priority2_exact_filename_match(): + """Filename 'backend.py' (stem='backend') routes to backend room.""" + assert _detect("lib/backend.py") == "backend" + + +def test_detect_room_priority2_keyword_filename_match(): + """Filename 'api.py' (stem='api') routes to backend via keyword.""" + assert _detect("lib/api.py") == "backend" + + +def test_detect_room_priority2_no_substring_match(): + """Filename 'testing.py' must NOT match 'tests' room via substring.""" + result = _detect("lib/testing.py", content="unrelated stuff") + assert result != "tests" + + +def test_detect_room_priority3_keyword_scoring(): + """Content with repeated 'api' keyword routes to backend room.""" + content = "the api handles requests. api calls are fast. api is great." + assert _detect("misc/readme.txt", content=content) == "backend" + + +def test_detect_room_priority3_word_boundary(): + """'test' inside 'testing'/'latest'/'contest' must NOT count as keyword hits.""" + # Only has 'test' embedded in other words, never standalone + content = "testing the latest contest results for attestation" + result = _detect("misc/notes.txt", content=content) + assert result != "tests" + + +def test_detect_room_priority3_word_boundary_standalone(): + """Standalone 'test' words DO count as keyword hits.""" + content = "run the test suite. each test verifies correctness. test passed." + assert _detect("misc/notes.txt", content=content) == "tests" + + +def test_detect_room_priority4_fallback_general(): + """No matches at all falls back to 'general'.""" + assert _detect("misc/random.txt", content="nothing relevant here") == "general" + + +def test_detect_room_priority4_empty_content(): + """Empty content with no path/filename match falls back to 'general'.""" + assert _detect("misc/random.txt", content="") == "general" + + +def test_detect_room_folder_beats_content(): + """Priority 1 (folder) wins even when content strongly matches another room.""" + content = "test test test test test test test" + assert _detect("backend/app.py", content=content) == "backend" + + +def test_detect_room_filename_beats_content(): + """Priority 2 (filename) wins even when content strongly matches another room.""" + content = "test test test test test test test" + assert _detect("misc/backend.py", content=content) == "backend" + + +# ============================================================================= +# chunk_text tests +# ============================================================================= + + +def test_chunk_text_short_content(): + """Content shorter than CHUNK_SIZE produces a single chunk.""" + content = "a" * (CHUNK_SIZE - 1) + chunks = chunk_text(content, "/fake/file.py") + assert len(chunks) == 1 + assert chunks[0]["content"] == content + assert chunks[0]["chunk_index"] == 0 + + +def test_chunk_text_exact_chunk_size(): + """Content exactly CHUNK_SIZE long produces a single chunk.""" + content = "a" * CHUNK_SIZE + chunks = chunk_text(content, "/fake/file.py") + assert len(chunks) == 1 + assert chunks[0]["content"] == content + + +def test_chunk_text_two_chunks(): + """Content slightly over CHUNK_SIZE produces two chunks with overlap.""" + content = "a" * (CHUNK_SIZE + 1) + chunks = chunk_text(content, "/fake/file.py") + assert len(chunks) == 2 + + +def test_chunk_text_overlap_content(): + """The second chunk starts at CHUNK_SIZE - CHUNK_OVERLAP (overlap region).""" + # Use digits so each position is unique and verifiable + content = "".join(str(i % 10) for i in range(CHUNK_SIZE + 200)) + chunks = chunk_text(content, "/fake/file.py") + assert len(chunks) >= 2 + # The overlap means the second chunk's content starts from the overlap region + expected_start = CHUNK_SIZE - CHUNK_OVERLAP + assert chunks[1]["content"].startswith(content[expected_start : expected_start + 10]) + + +def test_chunk_text_chunk_indices(): + """chunk_index values increment sequentially starting from 0.""" + content = "a" * (CHUNK_SIZE * 3) + chunks = chunk_text(content, "/fake/file.py") + assert len(chunks) >= 3 + for i, chunk in enumerate(chunks): + assert chunk["chunk_index"] == i + + +def test_chunk_text_empty_content(): + """Empty string returns an empty list.""" + chunks = chunk_text("", "/fake/file.py") + assert chunks == [] + + +def test_chunk_text_below_min_size(): + """Content below MIN_CHUNK_SIZE returns an empty list.""" + content = "a" * (MIN_CHUNK_SIZE - 1) + chunks = chunk_text(content, "/fake/file.py") + assert chunks == [] + + +def test_chunk_text_whitespace_only(): + """Whitespace-only content returns an empty list (stripped to empty).""" + chunks = chunk_text(" \n\n\t \n ", "/fake/file.py") + assert chunks == [] + +def test_chunk_text_many_chunks(): + """Very long content (10x CHUNK_SIZE) produces the correct number of chunks.""" + content = "a" * (CHUNK_SIZE * 10) + chunks = chunk_text(content, "/fake/file.py") + # With overlap, each chunk after the first starts CHUNK_SIZE - CHUNK_OVERLAP ahead. + # So we need ceil((total_len - CHUNK_OVERLAP) / (CHUNK_SIZE - CHUNK_OVERLAP)) chunks, + # but the exact count depends on boundary logic. Just verify it's reasonable. + total_len = len(content) + step = CHUNK_SIZE - CHUNK_OVERLAP + expected_min = total_len // CHUNK_SIZE # at least this many + expected_max = (total_len // step) + 1 # at most this many + assert expected_min <= len(chunks) <= expected_max + + +def test_chunk_text_preserves_content(): + """All original content is covered by the union of chunks (nothing lost).""" + # Use position-unique tokens so we can verify each segment appears in a chunk + tokens = [f"[T{i:04d}]" for i in range(300)] + content = " ".join(tokens) + chunks = chunk_text(content, "/fake/file.py") + assert len(chunks) >= 2 + # Every unique token must appear in at least one chunk + all_chunk_text = "".join(c["content"] for c in chunks) + for token in tokens: + assert token in all_chunk_text, f"Token '{token}' not found in any chunk" + # The joined chunks should contain at least as many characters as the original + # (overlap means more, confirming nothing is dropped) + assert len(all_chunk_text) >= len(content)