From 4098c54551ef14c1166385dcc0d5914375850433 Mon Sep 17 00:00:00 2001 From: MSL <232237854+milla-jovovich@users.noreply.github.com> Date: Mon, 27 Apr 2026 01:19:05 -0700 Subject: [PATCH 01/39] feat(convo_miner): auto-route AI tool sessions to wing_api MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When mempalace mine --mode convos is invoked against a directory inside a known AI-tool storage path (Claude Code, Codex CLI, Gemini CLI), the destination wing now auto-defaults to wing_api rather than the directory basename. Conversations from external API-keyed tools land grouped under a single dedicated wing for visibility. Detected paths (exact-segment match — substrings like .gemini-backup or .codex-archive do NOT match): - any segment .codex (Codex CLI sessions / archives) - any segment .gemini (Gemini CLI sessions under ~/.gemini/tmp/...) - the consecutive segment pair .claude/projects (Claude Code). .claude alone is NOT matched - that is the settings/config dir, not a conversation source. Wing-resolution precedence (first match wins): 1. Explicit --wing argument from the user - always wins 2. AI-tool path detection -> wing_api 3. Basename fallback (existing behavior, unchanged) Two new helpers split out of mine_convos for unit-test coverage: - _is_ai_tool_path(path: Path) -> bool - _resolve_wing(convo_path: Path, wing: Optional[str]) -> str mine_convos now calls _resolve_wing in place of its inline basename logic. No other call sites or downstream consumers change. Test coverage: - 15 unit tests covering positive matches (Claude Code subdir + root, Codex root + sessions, Gemini root + chats), negative cases (.claude alone is settings dir, unrelated paths, substring no-match on .gemini-backup / .codex-archive), explicit --wing override, auto-route trio, basename fallback, empty-string-as-no-wing. - End-to-end smoke test (manual): real-shape Claude Code JSONL fixture mined via the actual CLI; sqlite read-back of /tmp palace confirms drawers landed with wing='wing_api' and verbatim content preserved; mempalace search --wing wing_api returns expected content ranked. - Full pytest sweep: 1388 baseline + 15 new = 1403 passed, zero regressions. Design context: This change reflects Aya's product call that conversations from API-keyed AI tools should land in a structural wing_api rather than be scattered across topical wings derived from directory basenames. Igor's ADR-0017 in mempalace-ts proposes the alternative of source-prefix metadata (source LIKE 'api/%') with topical wing assignment instead; that approach has architectural merit (wings stay topical) but does not deliver the single-wing visibility users get here. Open for review discussion - explicit --wing flag and basename fallback both unchanged, so this is additive and reversible. Closes part of #59 for the auto-routing UX. --- mempalace/convo_miner.py | 60 ++++++++++++++++-- tests/test_convo_miner.py | 126 +++++++++++++++++++++++++++++++++++++- 2 files changed, 181 insertions(+), 5 deletions(-) diff --git a/mempalace/convo_miner.py b/mempalace/convo_miner.py index 2cf57e4488..77d5dda85b 100644 --- a/mempalace/convo_miner.py +++ b/mempalace/convo_miner.py @@ -14,6 +14,7 @@ from pathlib import Path from datetime import datetime from collections import defaultdict +from typing import Optional from .normalize import normalize from .palace import ( @@ -376,6 +377,60 @@ def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extr return drawers_added, room_counts_delta, False +def _is_ai_tool_path(path: Path) -> bool: + """Return True when `path` lives inside a known AI-tool storage dir. + + Detected paths (exact-segment match — substrings like `.gemini-backup` + or `.codex-archive` do NOT match): + - any segment ``.codex`` (Codex CLI sessions / archives) + - any segment ``.gemini`` (Gemini CLI sessions under ~/.gemini/tmp/...) + - the consecutive segment pair ``.claude/projects`` (Claude Code). + ``.claude`` alone is NOT matched — that is the settings/config dir, + not a conversation source. + + Used by ``_resolve_wing`` to default the destination wing to + ``wing_api`` when the user hasn't passed an explicit ``--wing``. + """ + try: + parts = path.resolve().parts + except (OSError, RuntimeError): + return False + + if ".codex" in parts: + return True + if ".gemini" in parts: + return True + for i in range(len(parts) - 1): + if parts[i] == ".claude" and parts[i + 1] == "projects": + return True + return False + + +def _resolve_wing(convo_path: Path, wing: Optional[str]) -> str: + """Determine the destination wing for ``mine_convos``. + + Precedence (first match wins): + + 1. Explicit ``wing`` argument from the user — always wins, even on + an AI-tool path. Empty string is treated as "no wing". + 2. AI-tool path detection — defaults to ``wing_api`` so Claude + Code / Codex / Gemini conversations group under a single wing + dedicated to API-sourced content. + 3. Basename fallback — sanitized via ``config.normalize_wing_name`` + (lowercase, spaces/hyphens collapsed to underscores). Shared + single source of truth with ``cmd_init``, + ``room_detector_local``, and ``miner.load_config`` so all + wing-slug producers stay in sync (per #1194 consolidation). + """ + from .config import normalize_wing_name + + if wing: + return wing + if _is_ai_tool_path(convo_path): + return "wing_api" + return normalize_wing_name(convo_path.name) + + def mine_convos( convo_dir: str, palace_path: str, @@ -393,10 +448,7 @@ def mine_convos( """ convo_path = Path(convo_dir).expanduser().resolve() - if not wing: - from .config import normalize_wing_name - - wing = normalize_wing_name(convo_path.name) + wing = _resolve_wing(convo_path, wing) files = scan_convos(convo_dir) if limit > 0: diff --git a/tests/test_convo_miner.py b/tests/test_convo_miner.py index 166644b002..17e4e8f5ab 100644 --- a/tests/test_convo_miner.py +++ b/tests/test_convo_miner.py @@ -5,7 +5,11 @@ import chromadb -from mempalace.convo_miner import mine_convos +from mempalace.convo_miner import ( + _is_ai_tool_path, + _resolve_wing, + mine_convos, +) from mempalace.palace import file_already_mined @@ -158,3 +162,123 @@ def test_mine_convos_rebuilds_stale_drawers_after_schema_bump(capsys): del col, client finally: shutil.rmtree(tmpdir, ignore_errors=True) + + +# ── _is_ai_tool_path / _resolve_wing — wing_api auto-routing ─────────── +# +# When a user runs `mempalace mine --mode convos` against a directory +# inside a known AI-tool storage path (Claude Code's +# ~/.claude/projects/, OpenAI Codex's ~/.codex/, Google Gemini CLI's +# ~/.gemini/), the wing auto-defaults to "wing_api" rather than the +# directory basename. This keeps API-sourced conversations grouped +# under a single dedicated wing for visibility and privacy isolation. +# +# Explicit user-passed --wing always wins. Unrelated directories use +# the existing basename fallback unchanged. + + +def test_is_ai_tool_path_claude_projects_subdir(tmp_path): + """A subdirectory inside ~/.claude/projects/ is an AI tool path.""" + target = tmp_path / ".claude" / "projects" / "-Users-test-myapp" + target.mkdir(parents=True) + assert _is_ai_tool_path(target) is True + + +def test_is_ai_tool_path_claude_projects_root(tmp_path): + """The ~/.claude/projects/ directory itself is an AI tool path.""" + target = tmp_path / ".claude" / "projects" + target.mkdir(parents=True) + assert _is_ai_tool_path(target) is True + + +def test_is_ai_tool_path_codex_root(tmp_path): + target = tmp_path / ".codex" + target.mkdir() + assert _is_ai_tool_path(target) is True + + +def test_is_ai_tool_path_codex_sessions(tmp_path): + """Codex stores sessions under ~/.codex/sessions/YYYY/MM/DD/.""" + target = tmp_path / ".codex" / "sessions" / "2026" / "04" / "26" + target.mkdir(parents=True) + assert _is_ai_tool_path(target) is True + + +def test_is_ai_tool_path_gemini_root(tmp_path): + target = tmp_path / ".gemini" + target.mkdir() + assert _is_ai_tool_path(target) is True + + +def test_is_ai_tool_path_gemini_chats(tmp_path): + """Gemini stores sessions under ~/.gemini/tmp//chats/.""" + target = tmp_path / ".gemini" / "tmp" / "abc123" / "chats" + target.mkdir(parents=True) + assert _is_ai_tool_path(target) is True + + +def test_is_ai_tool_path_dotclaude_without_projects_not_matched(tmp_path): + """`.claude/` alone (without `/projects`) is the settings dir, not a + conversation source — it MUST NOT auto-route to wing_api.""" + target = tmp_path / ".claude" + target.mkdir() + assert _is_ai_tool_path(target) is False + + +def test_is_ai_tool_path_unrelated_directory(tmp_path): + target = tmp_path / "Documents" / "myproject" + target.mkdir(parents=True) + assert _is_ai_tool_path(target) is False + + +def test_is_ai_tool_path_substring_no_false_positive(tmp_path): + """A directory NAMED like `.gemini-backup` or `.codex-archive` is NOT + a real AI tool path. We use exact-segment match, not substring.""" + a = tmp_path / ".gemini-backup" + a.mkdir() + b = tmp_path / ".codex-archive" + b.mkdir() + assert _is_ai_tool_path(a) is False + assert _is_ai_tool_path(b) is False + + +def test_resolve_wing_explicit_wins_over_auto_detection(tmp_path): + """User-passed --wing always wins, even on an AI tool path.""" + target = tmp_path / ".claude" / "projects" / "-Users-x" + target.mkdir(parents=True) + assert _resolve_wing(target, wing="my_custom_wing") == "my_custom_wing" + + +def test_resolve_wing_claude_projects_auto_routes_to_wing_api(tmp_path): + target = tmp_path / ".claude" / "projects" / "-Users-test-myapp" + target.mkdir(parents=True) + assert _resolve_wing(target, wing=None) == "wing_api" + + +def test_resolve_wing_codex_auto_routes_to_wing_api(tmp_path): + target = tmp_path / ".codex" / "sessions" / "2026" + target.mkdir(parents=True) + assert _resolve_wing(target, wing=None) == "wing_api" + + +def test_resolve_wing_gemini_auto_routes_to_wing_api(tmp_path): + target = tmp_path / ".gemini" / "tmp" / "abc" / "chats" + target.mkdir(parents=True) + assert _resolve_wing(target, wing=None) == "wing_api" + + +def test_resolve_wing_unrelated_dir_uses_basename_fallback(tmp_path): + """Existing behavior preserved: arbitrary directories use the + sanitized basename as the wing.""" + target = tmp_path / "MyProject Folder" + target.mkdir() + # Spaces become underscores, hyphens become underscores, lowercased. + assert _resolve_wing(target, wing=None) == "myproject_folder" + + +def test_resolve_wing_empty_string_treated_as_no_wing(tmp_path): + """An empty string for wing should behave like None — fall through to + auto-detection / basename. Mirrors the original `if not wing:` guard.""" + target = tmp_path / ".gemini" / "tmp" + target.mkdir(parents=True) + assert _resolve_wing(target, wing="") == "wing_api" From cb1303694d9caa52bbc21ffa4e87bd294d9576a9 Mon Sep 17 00:00:00 2001 From: Arnold Wender Date: Sun, 26 Apr 2026 13:06:11 +0200 Subject: [PATCH 02/39] fix(migrate): close SQLite connection and clean temp palace on exception Two resource leaks in mempalace.migrate that bite hardest on Windows: 1. extract_drawers_from_sqlite() opened a sqlite3 connection at the top and only closed it in the happy path. Any exception during query/iteration leaked the handle, leaving a file lock on chroma.sqlite3 that prevented the rest of the migration from touching the palace directory. Wrap with contextlib.closing(). 2. migrate() called tempfile.mkdtemp() and never cleaned it up if a later step (chromadb open, batch import, count, swap) raised. The orphaned palace under the system temp root could be 100s of MB and never gets reclaimed. Wrap the import-and-swap dance in try/finally and rmtree the temp dir if it still exists at the end (os.replace consumes it on the happy path so the existence guard makes that case a no-op). Tests cover the happy-path extraction and verify mkdtemp's directory is removed when ChromaBackend.get_or_create_collection() raises. --- mempalace/migrate.py | 200 +++++++++++++++++++++++------------------- tests/test_migrate.py | 88 ++++++++++++++++++- 2 files changed, 195 insertions(+), 93 deletions(-) diff --git a/mempalace/migrate.py b/mempalace/migrate.py index 5b74591b32..9e809da226 100644 --- a/mempalace/migrate.py +++ b/mempalace/migrate.py @@ -22,8 +22,10 @@ import os import shutil import sqlite3 +import tempfile import uuid from collections import defaultdict +from contextlib import closing from datetime import datetime @@ -52,61 +54,65 @@ def extract_drawers_from_sqlite(db_path: str) -> list: Works regardless of which ChromaDB version created the database. Returns list of dicts with 'id', 'document', and 'metadata' keys. - """ - conn = sqlite3.connect(db_path) - conn.row_factory = sqlite3.Row - # Get all embedding IDs and their documents - rows = conn.execute( - """ - SELECT e.embedding_id, - MAX(CASE WHEN em.key = 'chroma:document' THEN em.string_value END) as document - FROM embeddings e - JOIN embedding_metadata em ON em.id = e.id - GROUP BY e.embedding_id + The connection is wrapped in ``contextlib.closing`` so an exception + during extraction does not leak the SQLite handle. On Windows that + would leave a file lock on ``chroma.sqlite3`` and prevent the rest + of the migration from touching the palace directory. """ - ).fetchall() + with closing(sqlite3.connect(db_path)) as conn: + conn.row_factory = sqlite3.Row - drawers = [] - for row in rows: - embedding_id = row["embedding_id"] - document = row["document"] - if not document: - continue - - # Get metadata for this embedding - meta_rows = conn.execute( + # Get all embedding IDs and their documents + rows = conn.execute( """ - SELECT em.key, em.string_value, em.int_value, em.float_value, em.bool_value - FROM embedding_metadata em - JOIN embeddings e ON e.id = em.id - WHERE e.embedding_id = ? - AND em.key NOT LIKE 'chroma:%' - """, - (embedding_id,), + SELECT e.embedding_id, + MAX(CASE WHEN em.key = 'chroma:document' THEN em.string_value END) as document + FROM embeddings e + JOIN embedding_metadata em ON em.id = e.id + GROUP BY e.embedding_id + """ ).fetchall() - metadata = {} - for mr in meta_rows: - key = mr["key"] - if mr["string_value"] is not None: - metadata[key] = mr["string_value"] - elif mr["int_value"] is not None: - metadata[key] = mr["int_value"] - elif mr["float_value"] is not None: - metadata[key] = mr["float_value"] - elif mr["bool_value"] is not None: - metadata[key] = bool(mr["bool_value"]) - - drawers.append( - { - "id": embedding_id, - "document": document, - "metadata": metadata, - } - ) + drawers = [] + for row in rows: + embedding_id = row["embedding_id"] + document = row["document"] + if not document: + continue + + # Get metadata for this embedding + meta_rows = conn.execute( + """ + SELECT em.key, em.string_value, em.int_value, em.float_value, em.bool_value + FROM embedding_metadata em + JOIN embeddings e ON e.id = em.id + WHERE e.embedding_id = ? + AND em.key NOT LIKE 'chroma:%' + """, + (embedding_id,), + ).fetchall() + + metadata = {} + for mr in meta_rows: + key = mr["key"] + if mr["string_value"] is not None: + metadata[key] = mr["string_value"] + elif mr["int_value"] is not None: + metadata[key] = mr["int_value"] + elif mr["float_value"] is not None: + metadata[key] = mr["float_value"] + elif mr["bool_value"] is not None: + metadata[key] = bool(mr["bool_value"]) + + drawers.append( + { + "id": embedding_id, + "document": document, + "metadata": metadata, + } + ) - conn.close() return drawers @@ -287,53 +293,63 @@ def migrate(palace_path: str, dry_run: bool = False, confirm: bool = False): print(f"\n Backing up to {backup_path}...") shutil.copytree(palace_path, backup_path) - # Build fresh palace in a temp directory (avoids chromadb reading old state) - import tempfile - + # Build fresh palace in a temp directory (avoids chromadb reading old state). + # Wrap the whole import-and-swap dance in try/finally so the temp dir is + # cleaned up if any of the chromadb writes, the verify count, or the + # rename fails — without try/finally a crashed migration leaves a partial + # palace dir under the system temp root that the user has to find by hand. temp_palace = tempfile.mkdtemp(prefix="mempalace_migrate_") - print(f" Creating fresh palace in {temp_palace}...") - fresh_backend = ChromaBackend() - col = fresh_backend.get_or_create_collection(temp_palace, "mempalace_drawers") - - # Re-import in batches - batch_size = 500 - imported = 0 - for i in range(0, len(drawers), batch_size): - batch = drawers[i : i + batch_size] - col.add( - ids=[d["id"] for d in batch], - documents=[d["document"] for d in batch], - metadatas=[d["metadata"] for d in batch], - ) - imported += len(batch) - print(f" Imported {imported}/{len(drawers)} drawers...") - - # Verify before swapping - final_count = col.count() - del col - del fresh_backend - - # Swap: rename old palace aside, then move new one into place. - # This avoids a window where both old and new are missing. - print(" Swapping old palace for migrated version...") - stale_path = palace_path + ".old" - if os.path.exists(stale_path): - shutil.rmtree(stale_path) - os.replace(palace_path, stale_path) try: - os.replace(temp_palace, palace_path) - except OSError as e: - # EXDEV = temp lives on a different filesystem; fall back to copy+delete. - # Anything else is a real error — don't mask it with shutil.move. - if getattr(e, "errno", None) != errno.EXDEV: - _restore_stale_palace(palace_path, stale_path) - raise + print(f" Creating fresh palace in {temp_palace}...") + fresh_backend = ChromaBackend() + col = fresh_backend.get_or_create_collection(temp_palace, "mempalace_drawers") + + # Re-import in batches + batch_size = 500 + imported = 0 + for i in range(0, len(drawers), batch_size): + batch = drawers[i : i + batch_size] + col.add( + ids=[d["id"] for d in batch], + documents=[d["document"] for d in batch], + metadatas=[d["metadata"] for d in batch], + ) + imported += len(batch) + print(f" Imported {imported}/{len(drawers)} drawers...") + + # Verify before swapping + final_count = col.count() + del col + del fresh_backend + + # Swap: rename old palace aside, then move new one into place. + # This avoids a window where both old and new are missing. + print(" Swapping old palace for migrated version...") + stale_path = palace_path + ".old" + if os.path.exists(stale_path): + shutil.rmtree(stale_path) + os.replace(palace_path, stale_path) try: - shutil.move(temp_palace, palace_path) - except Exception: - _restore_stale_palace(palace_path, stale_path) - raise - shutil.rmtree(stale_path, ignore_errors=True) + os.replace(temp_palace, palace_path) + except OSError as e: + # EXDEV = temp lives on a different filesystem; fall back to copy+delete. + # Anything else is a real error — don't mask it with shutil.move. + if getattr(e, "errno", None) != errno.EXDEV: + _restore_stale_palace(palace_path, stale_path) + raise + try: + shutil.move(temp_palace, palace_path) + except Exception: + _restore_stale_palace(palace_path, stale_path) + raise + shutil.rmtree(stale_path, ignore_errors=True) + finally: + # On the happy path os.replace/shutil.move consumed temp_palace, so + # the directory no longer exists at the temp location — the existence + # guard makes this a no-op then. On any failure path it actually + # removes the orphan. + if os.path.exists(temp_palace): + shutil.rmtree(temp_palace, ignore_errors=True) print("\n Migration complete.") print(f" Drawers migrated: {final_count}") diff --git a/tests/test_migrate.py b/tests/test_migrate.py index ba12ff528b..5d9255a9fa 100644 --- a/tests/test_migrate.py +++ b/tests/test_migrate.py @@ -1,10 +1,16 @@ """Tests for destructive-operation safety in mempalace.migrate.""" import os +import sqlite3 from types import SimpleNamespace from unittest.mock import MagicMock, patch -from mempalace.migrate import collection_write_roundtrip_works, _restore_stale_palace, migrate +from mempalace.migrate import ( + _restore_stale_palace, + collection_write_roundtrip_works, + extract_drawers_from_sqlite, + migrate, +) def test_migrate_requires_palace_database(tmp_path, capsys): @@ -157,6 +163,38 @@ def test_collection_write_roundtrip_fails_when_delete_silently_drops(): assert len(col.ids) == 1 +def _make_minimal_chromadb_sqlite(tmp_path): + """Build a SQLite file with the minimal schema extract_drawers_from_sqlite reads.""" + db = tmp_path / "chroma.sqlite3" + conn = sqlite3.connect(str(db)) + conn.executescript( + """ + CREATE TABLE embeddings (id INTEGER PRIMARY KEY, embedding_id TEXT); + CREATE TABLE embedding_metadata ( + id INTEGER, key TEXT, + string_value TEXT, int_value INTEGER, + float_value REAL, bool_value INTEGER + ); + INSERT INTO embeddings VALUES (1, 'd-001'); + INSERT INTO embedding_metadata VALUES (1, 'chroma:document', 'hello', NULL, NULL, NULL); + INSERT INTO embedding_metadata VALUES (1, 'wing', 'personal', NULL, NULL, NULL); + INSERT INTO embedding_metadata VALUES (1, 'room', '2026-04-26', NULL, NULL, NULL); + """ + ) + conn.commit() + conn.close() + return str(db) + + +def test_extract_drawers_returns_drawers(tmp_path): + db_path = _make_minimal_chromadb_sqlite(tmp_path) + drawers = extract_drawers_from_sqlite(db_path) + assert len(drawers) == 1 + assert drawers[0]["id"] == "d-001" + assert drawers[0]["document"] == "hello" + assert drawers[0]["metadata"] == {"wing": "personal", "room": "2026-04-26"} + + def test_migrate_dry_run_rebuilds_when_collection_is_readable_but_not_writable(tmp_path, capsys): palace_dir = tmp_path / "palace" palace_dir.mkdir() @@ -200,3 +238,51 @@ def test_migrate_dry_run_rebuilds_when_collection_is_readable_but_not_writable(t assert "Rebuilding from SQLite" in out assert "Extracted 1 drawers from SQLite" in out assert "DRY RUN" in out + + +def test_migrate_cleans_temp_palace_on_chromadb_failure(tmp_path): + """If chromadb fails after the temp palace is created, mkdtemp's + directory must be removed — without try/finally it leaked into the + system temp root forever.""" + import tempfile as _tempfile + + palace_dir = tmp_path / "palace" + palace_dir.mkdir() + (palace_dir / "chroma.sqlite3").write_text("db") + + captured_temp_paths = [] + real_mkdtemp = _tempfile.mkdtemp + + def tracking_mkdtemp(*args, **kwargs): + path = real_mkdtemp(*args, **kwargs) + captured_temp_paths.append(path) + return path + + failing_backend = MagicMock() + # First ChromaBackend().get_collection() must raise so we drop into + # the SQL-extraction path; the second ChromaBackend().get_or_create_collection() + # raises to trigger the cleanup we are testing. + failing_backend.get_collection.side_effect = Exception("unreadable") + failing_backend.get_or_create_collection.side_effect = RuntimeError("chromadb boom") + + import mempalace.backends.chroma as _chroma_mod + + with ( + patch("mempalace.migrate.detect_chromadb_version", return_value="0.5.x"), + patch( + "mempalace.migrate.extract_drawers_from_sqlite", + return_value=[{"id": "id1", "document": "doc", "metadata": {"wing": "w", "room": "r"}}], + ), + patch("builtins.input", return_value="y"), + patch("mempalace.migrate.shutil.copytree"), + patch("mempalace.migrate.tempfile.mkdtemp", side_effect=tracking_mkdtemp), + patch.object(_chroma_mod, "ChromaBackend", return_value=failing_backend), + ): + try: + migrate(str(palace_dir), confirm=True) + except Exception: + pass + + assert captured_temp_paths, "mkdtemp was never called — flow short-circuited" + for p in captured_temp_paths: + assert not os.path.exists(p), f"temp palace was not cleaned up: {p}" From 2c75c6faf6c5c54514837dfd06a199eca79a9d8a Mon Sep 17 00:00:00 2001 From: fatkobra <55045047+fatkobra@users.noreply.github.com> Date: Sun, 10 May 2026 16:45:09 +0200 Subject: [PATCH 03/39] Fix mine palace lock holder encoding --- mempalace/palace.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/mempalace/palace.py b/mempalace/palace.py index 7ed315c8ac..cf7b10abc6 100644 --- a/mempalace/palace.py +++ b/mempalace/palace.py @@ -387,7 +387,10 @@ def _read_lock_holder(lock_file) -> str: """Read the prior holder's identity from the lock-file body, best-effort.""" try: lock_file.seek(_LOCK_SENTINEL_BYTES) - content = lock_file.read().strip() + content = lock_file.read() + if isinstance(content, bytes): + content = content.decode("utf-8", errors="replace") + content = content.strip() except OSError: return "another writer (identity not recorded)" if not content: @@ -404,11 +407,12 @@ def _write_lock_holder(lock_file) -> None: """ try: ident = f"{os.getpid()} {' '.join(sys.argv[:3])}".strip() + ident_bytes = ident.encode("utf-8") lock_file.seek(_LOCK_SENTINEL_BYTES) - lock_file.truncate(_LOCK_SENTINEL_BYTES + len(ident.encode("utf-8"))) - lock_file.write(ident) + lock_file.truncate(_LOCK_SENTINEL_BYTES + len(ident_bytes)) + lock_file.write(ident_bytes) lock_file.flush() - except OSError: + except (OSError, UnicodeError): pass @@ -470,7 +474,7 @@ def mine_palace_lock(palace_path: str): os.close(fd) except FileExistsError: pass - lf = open(lock_path, "r+") + lf = open(lock_path, "r+b") acquired = False try: # Lock byte 0 explicitly. msvcrt.locking is byte-position dependent; From c49c10f6e462645f7a7d598ff1ddc4f355bf5a58 Mon Sep 17 00:00:00 2001 From: fatkobra <55045047+fatkobra@users.noreply.github.com> Date: Sun, 10 May 2026 16:56:10 +0200 Subject: [PATCH 04/39] Add regression tests for UTF-8 lock holder writes --- tests/test_palace_locks.py | 56 ++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/tests/test_palace_locks.py b/tests/test_palace_locks.py index bf53a9d178..b4d2fbc259 100644 --- a/tests/test_palace_locks.py +++ b/tests/test_palace_locks.py @@ -12,10 +12,12 @@ import multiprocessing import os import time +import sys import pytest from mempalace.palace import ( + _write_lock_holder, MineAlreadyRunning, mine_global_lock, mine_palace_lock, @@ -197,9 +199,9 @@ def test_reentrant_same_thread_passes_through(tmp_path, monkeypatch): child = ctx.Process(target=_try_acquire_expect_busy, args=(palace, result_q)) try: child.start() - assert ( - result_q.get(timeout=10) == "busy" - ), "outer lock should still be held by parent after inner re-entrant exit" + assert result_q.get(timeout=10) == "busy", ( + "outer lock should still be held by parent after inner re-entrant exit" + ) child.join(timeout=5) assert child.exitcode == 0 finally: @@ -265,14 +267,56 @@ def test_lock_failure_message_names_holder(tmp_path, monkeypatch): pytest.fail("second acquire of same palace should have raised") msg = str(excinfo.value) - assert ( - f"PID {holder_pid}" in msg - ), f"lock-failure message must name the holder PID; got: {msg!r}" + assert f"PID {holder_pid}" in msg, ( + f"lock-failure message must name the holder PID; got: {msg!r}" + ) finally: open(release, "w").close() holder.join(timeout=5) +def test_write_lock_holder_writes_utf8_bytes_for_non_ascii_argv(tmp_path, monkeypatch): + """Regression #1435: lock-holder identity must be written as UTF-8 bytes. + + The holder byte count and the on-disk bytes must agree even when argv + contains characters that are not representable in a Windows ANSI codepage. + """ + monkeypatch.setattr( + sys, + "argv", + ["mempalace", "mine", "café/北"], + ) + + lock_path = tmp_path / "holder.lock" + lock_path.write_bytes(b"\0stale-holder-identity-that-must-be-truncated") + + with lock_path.open("r+b") as lock_file: + _write_lock_holder(lock_file) + + ident = f"{os.getpid()} {' '.join(sys.argv[:3])}".strip() + assert lock_path.read_bytes() == b"\0" + ident.encode("utf-8") + + +def test_write_lock_holder_is_best_effort_on_unicode_error(monkeypatch): + """Regression #1435: holder-write failures must not block lock acquisition.""" + + class UnicodeFailingLock: + def seek(self, _offset): + pass + + def truncate(self, _size): + pass + + def write(self, _data): + raise UnicodeEncodeError("cp1252", "北", 0, 1, "not representable") + + def flush(self): + pass + + monkeypatch.setattr(sys, "argv", ["mempalace", "mine", "北"]) + _write_lock_holder(UnicodeFailingLock()) + + def test_lock_holder_identity_persists_across_release(tmp_path, monkeypatch): """The holder line is overwritten by each new acquirer, not appended. From 92de001c204e54029da710ff9ac1f8215568176f Mon Sep 17 00:00:00 2001 From: Rahul Date: Sat, 16 May 2026 17:55:10 +0530 Subject: [PATCH 05/39] fix(convo_miner): scope skip-check and drawer ids by extract_mode (#1505) Mining a transcript with --extract general was silently skipped when the same file had already been mined with --extract exchange (or vice versa) because file_already_mined() and prefetch_mined_set() only looked at source_file. The two extraction modes produce different drawer content and rooms, so they should coexist for the same source. Changes: - file_already_mined() and prefetch_mined_set() take an optional extract_mode arg and only return True when stored drawer metadata matches. Legacy drawers without extract_mode are treated as exchange-mode for back-compat. - _file_chunks_locked() purges only same-mode drawers when rebuilding on a normalize-version bump, so a schema bump on one mode does not drop drawers filed under the other. - Drawer ids and sentinel ids include extract_mode so the two modes cannot collide on hash. - Pagination on the direct skip-check path so large transcripts (>1k drawers) are classified correctly when the bulk prefetch is skipped. Adds regression coverage for the extract-mode-aware helper, the pagination path, and an end-to-end mine_convos run that files exchange then general for the same transcript without skipping. --- mempalace/convo_miner.py | 57 +++++++++++++++++++++---- mempalace/palace.py | 59 ++++++++++++++++++++++--- tests/test_convo_miner.py | 38 +++++++++++++++-- tests/test_convo_miner_unit.py | 2 +- tests/test_miner.py | 78 +++++++++++++++++++++++++++++++++- 5 files changed, 215 insertions(+), 19 deletions(-) diff --git a/mempalace/convo_miner.py b/mempalace/convo_miner.py index 24898d248c..c14a02f5a9 100644 --- a/mempalace/convo_miner.py +++ b/mempalace/convo_miner.py @@ -70,14 +70,15 @@ def _detect_hall_cached(content: str) -> str: # use also scales with source size. -def _register_file(collection, source_file: str, wing: str, agent: str): +def _register_file(collection, source_file: str, wing: str, agent: str, extract_mode: str): """Write a sentinel so file_already_mined() returns True for 0-chunk files. Without this, files that normalize to nothing or produce zero chunks are re-read and re-processed on every mine run because nothing was written to ChromaDB on the first pass. """ - sentinel_id = f"_reg_{hashlib.sha256(source_file.encode()).hexdigest()[:24]}" + sentinel_key = f"{source_file}:{extract_mode}" + sentinel_id = f"_reg_{hashlib.sha256(sentinel_key.encode()).hexdigest()[:24]}" collection.upsert( documents=[f"[registry] {source_file}"], ids=[sentinel_id], @@ -89,12 +90,42 @@ def _register_file(collection, source_file: str, wing: str, agent: str): "added_by": agent, "filed_at": datetime.now().isoformat(), "ingest_mode": "registry", + "extract_mode": extract_mode, "normalize_version": NORMALIZE_VERSION, } ], ) +def _source_file_delete_ids(collection, source_file: str, extract_mode: str) -> list[str]: + """Collect drawer IDs for one source file and extraction mode. + + Legacy conversation drawers did not carry extract_mode; treat those as + exchange-mode rows so schema rebuilds can still clean them up without + deleting newer general-mode drawers for the same transcript. + """ + ids: list[str] = [] + offset = 0 + while True: + batch = collection.get( + where={"source_file": source_file}, + limit=1000, + offset=offset, + include=["metadatas"], + ) + batch_ids = batch.get("ids") or [] + metadatas = batch.get("metadatas") or [] + for drawer_id, meta in zip(batch_ids, metadatas): + meta = meta or {} + stored_mode = meta.get("extract_mode") + if stored_mode == extract_mode or (extract_mode == "exchange" and stored_mode is None): + ids.append(drawer_id) + if not batch_ids: + break + offset += len(batch_ids) + return ids + + # ============================================================================= # CHUNKING — exchange pairs for conversations # ============================================================================= @@ -358,14 +389,16 @@ def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extr # Re-check after lock — another agent may have just finished this file # at the current schema. A stale-version hit here returns False, so we # still fall through to the purge+rebuild path below. - if file_already_mined(collection, source_file): + if file_already_mined(collection, source_file, extract_mode=extract_mode): return 0, room_counts_delta, True # Purge stale drawers first. When the normalize schema bumps, # file_already_mined() returned False for pre-v2 drawers — clean # them out so the source doesn't end up with mixed old/new drawers. try: - collection.delete(where={"source_file": source_file}) + delete_ids = _source_file_delete_ids(collection, source_file, extract_mode) + if delete_ids: + collection.delete(ids=delete_ids) except Exception: logger.debug("Stale-drawer purge failed for %s", source_file, exc_info=True) @@ -382,7 +415,11 @@ def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extr chunk_room = chunk.get("memory_type", room) if extract_mode == "general" else room if extract_mode == "general": room_counts_delta[chunk_room] += 1 - drawer_id = f"drawer_{wing}_{chunk_room}_{hashlib.sha256((source_file + str(chunk['chunk_index'])).encode()).hexdigest()[:24]}" + drawer_key = f"{source_file}:{extract_mode}:{chunk['chunk_index']}" + drawer_id = ( + f"drawer_{wing}_{chunk_room}_" + f"{hashlib.sha256(drawer_key.encode()).hexdigest()[:24]}" + ) batch_docs.append(chunk["content"]) batch_ids.append(drawer_id) batch_metas.append( @@ -478,7 +515,9 @@ def mine_convos( # palace each per-file query costs ~2s, so a 2000-file sweep used to # spend >1h just deciding to skip. prefetch_mined_set() does the same # decisions in a single scan; loop body becomes an O(1) set check. - mined_set: set[str] = prefetch_mined_set(collection) if not dry_run else set() + mined_set: set[str] = ( + prefetch_mined_set(collection, extract_mode=extract_mode) if not dry_run else set() + ) total_drawers = 0 files_skipped = 0 @@ -497,12 +536,12 @@ def mine_convos( content = normalize(str(filepath)) except (OSError, ValueError): if not dry_run: - _register_file(collection, source_file, wing, agent) + _register_file(collection, source_file, wing, agent, extract_mode) continue if not content or len(content.strip()) < cfg_min_chunk_size: if not dry_run: - _register_file(collection, source_file, wing, agent) + _register_file(collection, source_file, wing, agent, extract_mode) continue # Chunk — either exchange pairs or general extraction @@ -520,7 +559,7 @@ def mine_convos( if not chunks: if not dry_run: - _register_file(collection, source_file, wing, agent) + _register_file(collection, source_file, wing, agent, extract_mode) continue # Detect room from content (general mode uses memory_type instead) diff --git a/mempalace/palace.py b/mempalace/palace.py index 38c3979763..3e3cfa7f26 100644 --- a/mempalace/palace.py +++ b/mempalace/palace.py @@ -531,7 +531,19 @@ def mine_palace_lock(palace_path: str): mine_global_lock = mine_palace_lock -def file_already_mined(collection, source_file: str, check_mtime: bool = False) -> bool: +def _metadata_matches_extract_mode(meta: dict, extract_mode: Optional[str]) -> bool: + if extract_mode is None: + return True + stored_mode = meta.get("extract_mode") + return stored_mode == extract_mode or (extract_mode == "exchange" and stored_mode is None) + + +def file_already_mined( + collection, + source_file: str, + check_mtime: bool = False, + extract_mode: Optional[str] = None, +) -> bool: """Check if a file has already been filed in the palace. Returns False (so the file gets re-mined) when: @@ -543,12 +555,43 @@ def file_already_mined(collection, source_file: str, check_mtime: bool = False) When check_mtime=True (used by project miner), also re-mines on content change. When check_mtime=False (used by convo miner), transcripts are assumed immutable, so only the version gate triggers a rebuild. + + When extract_mode is set (used by convo miner), idempotency is scoped to + that extraction mode so exchange-mode and general-mode drawers can coexist + for the same source transcript. Legacy drawers without extract_mode are + treated as exchange-mode drawers. """ try: - results = collection.get(where={"source_file": source_file}, limit=1) - if not results.get("ids"): + stored_meta = None + if extract_mode is None: + results = collection.get(where={"source_file": source_file}, limit=1) + if not results.get("ids"): + return False + stored_meta = results.get("metadatas", [{}])[0] or {} + else: + offset = 0 + while True: + results = collection.get( + where={"source_file": source_file}, + limit=1000, + offset=offset, + include=["metadatas"], + ) + ids = results.get("ids") or [] + metadatas = results.get("metadatas") or [] + stored_meta = next( + ( + meta or {} + for meta in metadatas + if _metadata_matches_extract_mode(meta or {}, extract_mode) + ), + None, + ) + if stored_meta is not None or not ids: + break + offset += len(ids) + if stored_meta is None: return False - stored_meta = results.get("metadatas", [{}])[0] or {} # Pre-v2 drawers have no version field — treat them as stale. stored_version = stored_meta.get("normalize_version", 1) if stored_version < NORMALIZE_VERSION: @@ -593,7 +636,7 @@ def bulk_check_mined(collection) -> dict[str, float]: return mined -def prefetch_mined_set(collection) -> set[str]: +def prefetch_mined_set(collection, extract_mode: Optional[str] = None) -> set[str]: """Pre-fetch the set of source_files already mined at the current NORMALIZE_VERSION. Mirrors file_already_mined()'s version-gate semantics (check_mtime=False @@ -601,6 +644,9 @@ def prefetch_mined_set(collection) -> set[str]: Returns a set of source_file paths whose stored drawers are at or above NORMALIZE_VERSION; callers do `if path in result_set: skip`. + When extract_mode is set, mirrors file_already_mined(..., extract_mode=...) + so conversation mines skip per extraction mode rather than per source file. + The convo miner walks thousands of transcript files; per-file `collection.get(where={"source_file": X})` costs ~2s on a 150k-drawer palace, making a 2000-file sweep take >1h of pure skip-checking. This @@ -613,9 +659,12 @@ def prefetch_mined_set(collection) -> set[str]: while offset < total: batch = collection.get(limit=1000, offset=offset, include=["metadatas"]) for meta in batch["metadatas"]: + meta = meta or {} src = meta.get("source_file") if not src: continue + if not _metadata_matches_extract_mode(meta, extract_mode): + continue # Same default as file_already_mined: missing version == 1 version = meta.get("normalize_version", 1) if version >= NORMALIZE_VERSION: diff --git a/tests/test_convo_miner.py b/tests/test_convo_miner.py index 166644b002..af7a2838ba 100644 --- a/tests/test_convo_miner.py +++ b/tests/test_convo_miner.py @@ -77,6 +77,38 @@ def test_mine_convos_does_not_reprocess_empty_chunk_files(capsys): shutil.rmtree(tmpdir, ignore_errors=True) +def test_mine_convos_allows_general_after_exchange(capsys): + """A transcript mined as exchange can later be mined as general memories.""" + tmpdir = tempfile.mkdtemp() + try: + convo_path = Path(tmpdir) / "chat.txt" + convo_path.write_text( + "> What did we decide?\n" + "We decided to use SQLite because it keeps the local setup simple.\n\n" + "> What broke?\n" + "The search failed because the old index was stale, and the fix was rebuild.\n" + ) + palace_path = os.path.join(tmpdir, "palace") + + mine_convos(tmpdir, palace_path, wing="test", extract_mode="exchange") + capsys.readouterr() + mine_convos(tmpdir, palace_path, wing="test", extract_mode="general") + out = capsys.readouterr().out + + assert "Files skipped (already filed): 0" in out + + client = chromadb.PersistentClient(path=palace_path) + col = client.get_collection("mempalace_drawers") + resolved = str(Path(tmpdir).resolve() / "chat.txt") + rows = col.get(where={"source_file": resolved}, include=["metadatas"]) + modes = {meta.get("extract_mode") for meta in rows["metadatas"]} + assert {"exchange", "general"} <= modes + assert any(drawer_id.startswith("drawer_test_decision_") for drawer_id in rows["ids"]) + del col, client + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + def test_mine_convos_rebuilds_stale_drawers_after_schema_bump(capsys): """When stored drawers have an older normalize_version, the next mine silently purges them and refiles — no manual erase required. @@ -140,9 +172,9 @@ def test_mine_convos_rebuilds_stale_drawers_after_schema_bump(capsys): # Second mine — version gate should trigger rebuild mine_convos(tmpdir, palace_path, wing="test") out = capsys.readouterr().out - assert ( - "Files skipped (already filed): 0" in out - ), "stale drawers should force a rebuild, not a skip" + assert "Files skipped (already filed): 0" in out, ( + "stale drawers should force a rebuild, not a skip" + ) client = chromadb.PersistentClient(path=palace_path) col = client.get_collection("mempalace_drawers") diff --git a/tests/test_convo_miner_unit.py b/tests/test_convo_miner_unit.py index 90ad573625..7ff7a91bc3 100644 --- a/tests/test_convo_miner_unit.py +++ b/tests/test_convo_miner_unit.py @@ -239,7 +239,7 @@ def upsert(self, documents, ids, metadatas): col = FakeCol() monkeypatch.setattr(convo_miner, "DRAWER_UPSERT_BATCH_SIZE", 2) monkeypatch.setattr( - convo_miner, "file_already_mined", lambda collection, source_file: False + convo_miner, "file_already_mined", lambda collection, source_file, **kwargs: False ) monkeypatch.setattr(convo_miner, "mine_lock", lambda source_file: contextlib.nullcontext()) monkeypatch.setattr(convo_miner, "_detect_hall_cached", lambda content: "conversations") diff --git a/tests/test_miner.py b/tests/test_miner.py index d58ef9201e..50535273bb 100644 --- a/tests/test_miner.py +++ b/tests/test_miner.py @@ -10,7 +10,7 @@ import yaml from mempalace.miner import detect_room, load_config, mine, scan_project, status -from mempalace.palace import NORMALIZE_VERSION, file_already_mined +from mempalace.palace import NORMALIZE_VERSION, file_already_mined, prefetch_mined_set def write_file(path: Path, content: str): @@ -406,6 +406,82 @@ def test_file_already_mined_check_mtime(): shutil.rmtree(tmpdir, ignore_errors=True) +def test_file_already_mined_scopes_convo_extract_mode(): + tmpdir = tempfile.mkdtemp() + try: + palace_path = os.path.join(tmpdir, "palace") + os.makedirs(palace_path) + client = chromadb.PersistentClient(path=palace_path) + col = client.get_or_create_collection( + "mempalace_drawers", metadata={"hnsw:space": "cosine"} + ) + + source_file = os.path.join(tmpdir, "chat.jsonl") + col.add( + ids=["exchange"], + documents=["exchange drawer"], + metadatas=[ + { + "source_file": source_file, + "extract_mode": "exchange", + "normalize_version": NORMALIZE_VERSION, + } + ], + ) + + assert file_already_mined(col, source_file, extract_mode="exchange") is True + assert file_already_mined(col, source_file, extract_mode="general") is False + assert source_file in prefetch_mined_set(col, extract_mode="exchange") + assert source_file not in prefetch_mined_set(col, extract_mode="general") + + col.add( + ids=["general"], + documents=["general drawer"], + metadatas=[ + { + "source_file": source_file, + "extract_mode": "general", + "normalize_version": NORMALIZE_VERSION, + } + ], + ) + + assert file_already_mined(col, source_file, extract_mode="general") is True + assert source_file in prefetch_mined_set(col, extract_mode="general") + finally: + del col, client + shutil.rmtree(tmpdir, ignore_errors=True) + + +def test_file_already_mined_extract_mode_paginates_large_sources(): + source_file = "/tmp/long-chat.jsonl" + metadatas = [ + { + "source_file": source_file, + "extract_mode": "exchange", + "normalize_version": NORMALIZE_VERSION, + } + for _ in range(1000) + ] + metadatas.append( + { + "source_file": source_file, + "extract_mode": "general", + "normalize_version": NORMALIZE_VERSION, + } + ) + + class FakeCollection: + def get(self, where=None, limit=1, offset=0, include=None): + batch = metadatas[offset : offset + limit] + return { + "ids": [f"id-{i}" for i in range(offset, offset + len(batch))], + "metadatas": batch, + } + + assert file_already_mined(FakeCollection(), source_file, extract_mode="general") is True + + def test_mine_dry_run_with_tiny_file_no_crash(): """Dry-run must not crash when process_file returns 0 drawers (room was None).""" tmpdir = tempfile.mkdtemp() From 2bf36210e99ec516121e4c11880d061331a4c5be Mon Sep 17 00:00:00 2001 From: Rahul Date: Sat, 16 May 2026 18:15:44 +0530 Subject: [PATCH 06/39] refactor(convo_miner): reuse _metadata_matches_extract_mode helper Address PR #1528 review feedback (gemini-code-assist, medium): _source_file_delete_ids duplicated the legacy-drawer matching logic that already lives in palace._metadata_matches_extract_mode. Import the helper and call it instead so the back-compat rule (legacy drawers without extract_mode count as exchange-mode) has a single source of truth. No behavior change. All 71 miner/convo tests still pass. --- mempalace/convo_miner.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mempalace/convo_miner.py b/mempalace/convo_miner.py index c14a02f5a9..02a5aca9ba 100644 --- a/mempalace/convo_miner.py +++ b/mempalace/convo_miner.py @@ -20,6 +20,7 @@ from .palace import ( NORMALIZE_VERSION, SKIP_DIRS, + _metadata_matches_extract_mode, file_already_mined, get_collection, mine_lock, @@ -116,9 +117,7 @@ def _source_file_delete_ids(collection, source_file: str, extract_mode: str) -> batch_ids = batch.get("ids") or [] metadatas = batch.get("metadatas") or [] for drawer_id, meta in zip(batch_ids, metadatas): - meta = meta or {} - stored_mode = meta.get("extract_mode") - if stored_mode == extract_mode or (extract_mode == "exchange" and stored_mode is None): + if _metadata_matches_extract_mode(meta or {}, extract_mode): ids.append(drawer_id) if not batch_ids: break From 419a34c5c584bb179fb543a386b4d61f180e9671 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Sun, 17 May 2026 15:01:14 -0300 Subject: [PATCH 07/39] style(tests): restore ruff 0.4.x assert formatting in test_palace_locks (#1438) The PR reformatted two pre-existing assertions with a newer ruff (0.5+ `assert X, (\n msg\n)` style) that CI's pinned ruff (>=0.4.0,<0.5) rejects, failing `ruff format --check .`. Revert those two unrelated blocks to develop's 0.4.x form; the genuine #1435 fix and its new regression tests are untouched. --- tests/test_palace_locks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_palace_locks.py b/tests/test_palace_locks.py index b4d2fbc259..df94f16d8d 100644 --- a/tests/test_palace_locks.py +++ b/tests/test_palace_locks.py @@ -199,9 +199,9 @@ def test_reentrant_same_thread_passes_through(tmp_path, monkeypatch): child = ctx.Process(target=_try_acquire_expect_busy, args=(palace, result_q)) try: child.start() - assert result_q.get(timeout=10) == "busy", ( - "outer lock should still be held by parent after inner re-entrant exit" - ) + assert ( + result_q.get(timeout=10) == "busy" + ), "outer lock should still be held by parent after inner re-entrant exit" child.join(timeout=5) assert child.exitcode == 0 finally: @@ -267,9 +267,9 @@ def test_lock_failure_message_names_holder(tmp_path, monkeypatch): pytest.fail("second acquire of same palace should have raised") msg = str(excinfo.value) - assert f"PID {holder_pid}" in msg, ( - f"lock-failure message must name the holder PID; got: {msg!r}" - ) + assert ( + f"PID {holder_pid}" in msg + ), f"lock-failure message must name the holder PID; got: {msg!r}" finally: open(release, "w").close() holder.join(timeout=5) From 83e4711593e587706714837043ca48960c9d6623 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Sun, 17 May 2026 22:52:01 -0300 Subject: [PATCH 08/39] style(tests): apply ruff 0.4.x format to test_convo_miner (#1528) CI pins ruff>=0.4.0,<0.5 (resolves 0.4.10); one new assert block was laid out in ruff-0.5+ style, failing `ruff format --check .`. Reformat with the exact CI ruff version (assertion layout only, no semantic change). `ruff check .` + `ruff format --check .` both pass under 0.4.10. --- tests/test_convo_miner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_convo_miner.py b/tests/test_convo_miner.py index af7a2838ba..015a9d3d40 100644 --- a/tests/test_convo_miner.py +++ b/tests/test_convo_miner.py @@ -172,9 +172,9 @@ def test_mine_convos_rebuilds_stale_drawers_after_schema_bump(capsys): # Second mine — version gate should trigger rebuild mine_convos(tmpdir, palace_path, wing="test") out = capsys.readouterr().out - assert "Files skipped (already filed): 0" in out, ( - "stale drawers should force a rebuild, not a skip" - ) + assert ( + "Files skipped (already filed): 0" in out + ), "stale drawers should force a rebuild, not a skip" client = chromadb.PersistentClient(path=palace_path) col = client.get_collection("mempalace_drawers") From 96a327233db7b68dc33c88c7179b5ebb52f61453 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Sun, 17 May 2026 23:20:01 -0300 Subject: [PATCH 09/39] ci(lint): pin ruff to 0.15.9 across CI and dev deps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI installed `ruff>=0.4.0,<0.5` (resolves 0.4.10) while contributors run modern ruff (0.5+). ruff 0.5 changed assert-message formatting, so valid code formatted by a current ruff fails CI's `ruff format --check .` on layout alone. This blocked #1438, #1445, and #1528 — each needing a manual reformat-and-push cycle with no actual code defect. Pin an exact, modern ruff in both pyproject dev lists and in ci.yml so CI and `pip install -e ".[dev]"` format/lint identically. No new `ruff check` violations under the existing E/F/W/C901 select. --- .github/workflows/ci.yml | 5 ++++- pyproject.toml | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c6bcaad94..5aee4e34e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,9 @@ jobs: with: python-version: "3.11" cache: 'pip' - - run: pip install "ruff>=0.4.0,<0.5" + # Keep this pin identical to the `ruff==` pin in pyproject.toml + # ([project.optional-dependencies].dev and [dependency-groups].dev) + # so CI and `pip install -e ".[dev]"` format/lint identically. + - run: pip install "ruff==0.15.9" - run: ruff check . - run: ruff format --check . diff --git a/pyproject.toml b/pyproject.toml index 580f777d54..99db4c4e9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ chroma = "mempalace.backends.chroma:ChromaBackend" [project.entry-points."mempalace.sources"] [project.optional-dependencies] -dev = ["pytest>=7.0", "pytest-cov>=4.0", "ruff>=0.4.0", "psutil>=5.9"] +dev = ["pytest>=7.0", "pytest-cov>=4.0", "ruff==0.15.9", "psutil>=5.9"] spellcheck = ["autocorrect>=2.0"] # Hardware acceleration for the ONNX embedding model. Install exactly one: # pip install mempalace[gpu] — NVIDIA CUDA @@ -63,7 +63,7 @@ dml = ["onnxruntime-directml>=1.16"] coreml = ["onnxruntime>=1.16"] [dependency-groups] -dev = ["pytest>=7.0", "pytest-cov>=4.0", "ruff>=0.4.0", "psutil>=5.9"] +dev = ["pytest>=7.0", "pytest-cov>=4.0", "ruff==0.15.9", "psutil>=5.9"] [build-system] requires = ["hatchling"] From 2cab6f5a5bb2c818098c6e84c3567ba99ec21f04 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Sun, 17 May 2026 23:20:01 -0300 Subject: [PATCH 10/39] style: reformat tree with ruff 0.15.9 One-time mechanical reformat so `ruff format --check .` passes under the newly pinned ruff. Layout only (assert-message parenthesization etc.), no behavior change. 29 files: 28 under tests/ + 1 tools helper, no core mempalace/ modules. Produced by `ruff format .`. --- tests/test_backends.py | 12 ++--- tests/test_chroma_collection_lock.py | 6 +-- tests/test_claude_plugin_hook_config.py | 24 +++++----- tests/test_cli.py | 12 ++--- tests/test_closet_llm.py | 12 ++--- tests/test_closets.py | 26 +++++----- tests/test_convo_miner.py | 6 +-- tests/test_corpus_origin.py | 42 ++++++++-------- tests/test_corpus_origin_integration.py | 64 ++++++++++++------------- tests/test_hooks_bash_compat.py | 54 ++++++++++----------- tests/test_hooks_cli.py | 6 +-- tests/test_hooks_shell.py | 24 +++++----- tests/test_hybrid_candidate_union.py | 9 ++-- tests/test_i18n.py | 6 +-- tests/test_i18n_lang_case.py | 6 +-- tests/test_init.py | 14 +++--- tests/test_llm_client.py | 24 +++++----- tests/test_mcp_server.py | 42 ++++++++-------- tests/test_mcp_stdio_protection.py | 6 +-- tests/test_normalize.py | 8 +--- tests/test_palace_graph_tunnels.py | 6 +-- tests/test_palace_locks.py | 12 ++--- tests/test_readme_claims.py | 30 ++++++------ tests/test_repair.py | 6 +-- tests/test_save_hook_mines.py | 28 +++++------ tests/test_searcher.py | 15 +++--- tests/test_sweeper.py | 6 +-- tests/test_sync.py | 60 +++++++++++------------ tools/render_jsonl.py | 2 +- 29 files changed, 279 insertions(+), 289 deletions(-) diff --git a/tests/test_backends.py b/tests/test_backends.py index 8f361261d9..1fc8eab803 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -823,9 +823,9 @@ def _spy(path, stale_seconds=300.0): ChromaBackend.make_client(palace_path) ChromaBackend.make_client(palace_path) - assert calls == [ - palace_path - ], "quarantine_stale_hnsw should fire once per palace per process, not on every reconnect" + assert calls == [palace_path], ( + "quarantine_stale_hnsw should fire once per palace per process, not on every reconnect" + ) def test_make_client_gates_invalid_metadata_on_first_call(tmp_path, monkeypatch): @@ -941,9 +941,9 @@ def _spy(path, stale_seconds=300.0): finally: backend.close() - assert ( - calls == [palace_path] - ), "quarantine_stale_hnsw should fire once per palace per process from _client(), not on every call" + assert calls == [palace_path], ( + "quarantine_stale_hnsw should fire once per palace per process from _client(), not on every call" + ) # ── _pin_hnsw_threads (per-process retrofit, separate from this PR's gate) ── diff --git a/tests/test_chroma_collection_lock.py b/tests/test_chroma_collection_lock.py index 086bcbf2f3..85bd3c52f8 100644 --- a/tests/test_chroma_collection_lock.py +++ b/tests/test_chroma_collection_lock.py @@ -317,9 +317,9 @@ def test_read_path_does_not_acquire_lock(tmp_path, monkeypatch): if method is None: continue src = inspect.getsource(method) - assert ( - "_write_lock" not in src - ), f"{read_attr} must NOT acquire the write lock (read path)" + assert "_write_lock" not in src, ( + f"{read_attr} must NOT acquire the write lock (read path)" + ) finally: open(release, "w").close() holder.join(timeout=5) diff --git a/tests/test_claude_plugin_hook_config.py b/tests/test_claude_plugin_hook_config.py index 6c33054d72..d995ac3066 100644 --- a/tests/test_claude_plugin_hook_config.py +++ b/tests/test_claude_plugin_hook_config.py @@ -51,23 +51,23 @@ def test_plugin_hook_timeout_within_bounds(hook_config: dict, event: str) -> Non ) for entry in entries: sub_hooks = entry.get("hooks") - assert ( - isinstance(sub_hooks, list) and sub_hooks - ), f"{event} entry missing non-empty 'hooks' array" - assert ( - len(sub_hooks) == 1 - ), f"{event} entry expected exactly one hook command, found {len(sub_hooks)}" + assert isinstance(sub_hooks, list) and sub_hooks, ( + f"{event} entry missing non-empty 'hooks' array" + ) + assert len(sub_hooks) == 1, ( + f"{event} entry expected exactly one hook command, found {len(sub_hooks)}" + ) for hook in sub_hooks: - assert ( - hook.get("type") == "command" - ), f"unexpected hook type for {event}: {hook.get('type')!r}" + assert hook.get("type") == "command", ( + f"unexpected hook type for {event}: {hook.get('type')!r}" + ) assert "timeout" in hook, f"{event} hook missing 'timeout' key" timeout = hook["timeout"] # bool subclasses int, so reject it explicitly: True == 1 must fail. is_real_int = isinstance(timeout, int) and not isinstance(timeout, bool) - assert ( - is_real_int and floor <= timeout <= ceiling - ), f"{event} hook timeout must be an int in [{floor}, {ceiling}]s; got {timeout!r}" + assert is_real_int and floor <= timeout <= ceiling, ( + f"{event} hook timeout must be an int in [{floor}, {ceiling}]s; got {timeout!r}" + ) def test_no_unbounded_events_in_plugin_config(hook_config: dict) -> None: diff --git a/tests/test_cli.py b/tests/test_cli.py index ee0a7dc8e4..41cfc2753d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -79,12 +79,12 @@ def test_cli_main_strips_leaked_pythonpath_from_env(): ) diag = f"rc={result.returncode}; stdout={result.stdout!r}; stderr={result.stderr!r}" assert result.returncode == 0, f"subprocess failed: {diag}" - assert ( - f"ENV_MID: {expected_env!r}" in result.stdout - ), f"package import unexpectedly stripped env (regression in __init__.py): {diag}" - assert ( - "SENTINEL_IN_PATH: False" in result.stdout - ), f"package import did not filter sys.path (regression in __init__.py): {diag}" + assert f"ENV_MID: {expected_env!r}" in result.stdout, ( + f"package import unexpectedly stripped env (regression in __init__.py): {diag}" + ) + assert "SENTINEL_IN_PATH: False" in result.stdout, ( + f"package import did not filter sys.path (regression in __init__.py): {diag}" + ) assert "ENV_AFTER: None" in result.stdout, f"CLI did not strip PYTHONPATH: {diag}" diff --git a/tests/test_closet_llm.py b/tests/test_closet_llm.py index 905d28b777..296d105ced 100644 --- a/tests/test_closet_llm.py +++ b/tests/test_closet_llm.py @@ -312,9 +312,9 @@ def fake_urlopen(req, timeout=None): survivors = closets.get(where={"source_file": source}, include=["documents", "metadatas"]) assert survivors["ids"], "LLM closets should have been written" joined = "\n".join(survivors["documents"]) - assert ( - "STALE_REGEX_TOPIC" not in joined - ), "pre-existing regex closet was not purged before LLM write" + assert "STALE_REGEX_TOPIC" not in joined, ( + "pre-existing regex closet was not purged before LLM write" + ) assert "jwt auth" in joined for meta in survivors["metadatas"]: assert meta.get("generated_by", "").startswith("llm:") @@ -399,9 +399,9 @@ def fake_urlopen(req, timeout=None): # Three paginated calls: (limit=5000, offset=0), (5000, 5000), (5000, 10000). assert len(get_calls) == 3, f"expected 3 batched fetches, got {len(get_calls)}" for call in get_calls: - assert ( - call["limit"] == 5000 - ), f"batch must be 5000 — got {call['limit']} (would risk SQLITE_MAX_VARIABLE_NUMBER)" + assert call["limit"] == 5000, ( + f"batch must be 5000 — got {call['limit']} (would risk SQLITE_MAX_VARIABLE_NUMBER)" + ) # include must still request both documents and metadatas assert "documents" in call["include"] assert "metadatas" in call["include"] diff --git a/tests/test_closets.py b/tests/test_closets.py index e016d83e07..2aa5597309 100644 --- a/tests/test_closets.py +++ b/tests/test_closets.py @@ -140,9 +140,9 @@ def test_lock_blocks_concurrent_access(self, tmp_path): # Sort by entry time and verify the second entry is after the first exit. intervals.sort(key=lambda iv: iv[1]) (_, enter_a, exit_a), (_, enter_b, exit_b) = intervals - assert ( - enter_a < exit_a <= enter_b < exit_b - ), f"critical sections overlapped — lock failed to serialize: {intervals}" + assert enter_a < exit_a <= enter_b < exit_b, ( + f"critical sections overlapped — lock failed to serialize: {intervals}" + ) # ── build_closet_lines ───────────────────────────────────────────────── @@ -315,15 +315,15 @@ def test_remine_replaces_closets_completely(self, tmp_path): second_docs = "\n".join(second_pass["documents"]).lower() assert "only topic now" in second_docs for i in range(15): - assert ( - f"topic {i}\n" not in second_docs - ), f"stale 'Topic {i}' from first mine survived the rebuild" + assert f"topic {i}\n" not in second_docs, ( + f"stale 'Topic {i}' from first mine survived the rebuild" + ) # Numbered closets that existed only in the larger first run must be gone. leftover = first_ids - set(second_pass["ids"]) for stale_id in leftover: - assert not col.get(ids=[stale_id])[ - "ids" - ], f"orphan closet {stale_id} from larger first run survived purge" + assert not col.get(ids=[stale_id])["ids"], ( + f"orphan closet {stale_id} from larger first run survived purge" + ) # ── _extract_drawer_ids_from_closet ─────────────────────────────────── @@ -702,9 +702,9 @@ def test_state_file_lives_outside_diary_dir(self, tmp_path): # No state file inside the user's diary dir. for entry in diary_dir.iterdir(): - assert ( - "diary_ingest" not in entry.name - ), f"state file leaked into user diary dir: {entry}" + assert "diary_ingest" not in entry.name, ( + f"state file leaked into user diary dir: {entry}" + ) # State file does exist under ~/.mempalace/state/. state_path = _state_file_for(str(palace_dir), diary_dir.resolve()) @@ -905,7 +905,7 @@ def worker(i): assert not errors, f"worker raised: {errors}" tunnels = list_tunnels() assert len(tunnels) == 5, ( - f"expected 5 concurrent tunnels, got {len(tunnels)} — " "write race dropped some" + f"expected 5 concurrent tunnels, got {len(tunnels)} — write race dropped some" ) def test_created_at_is_timezone_aware(self): diff --git a/tests/test_convo_miner.py b/tests/test_convo_miner.py index 015a9d3d40..af7a2838ba 100644 --- a/tests/test_convo_miner.py +++ b/tests/test_convo_miner.py @@ -172,9 +172,9 @@ def test_mine_convos_rebuilds_stale_drawers_after_schema_bump(capsys): # Second mine — version gate should trigger rebuild mine_convos(tmpdir, palace_path, wing="test") out = capsys.readouterr().out - assert ( - "Files skipped (already filed): 0" in out - ), "stale drawers should force a rebuild, not a skip" + assert "Files skipped (already filed): 0" in out, ( + "stale drawers should force a rebuild, not a skip" + ) client = chromadb.PersistentClient(path=palace_path) col = client.get_collection("mempalace_drawers") diff --git a/tests/test_corpus_origin.py b/tests/test_corpus_origin.py index 6676bbd4bb..98a22aefb5 100644 --- a/tests/test_corpus_origin.py +++ b/tests/test_corpus_origin.py @@ -112,17 +112,17 @@ def test_brand_terms_case_insensitive(self): "anthropic's haiku model is cheap enough to run on every drawer", ] result = detect_origin_heuristic(samples) - assert ( - result.likely_ai_dialogue is True - ), f"lowercase brand terms missed; evidence: {result.evidence}" + assert result.likely_ai_dialogue is True, ( + f"lowercase brand terms missed; evidence: {result.evidence}" + ) # Evidence must show MULTIPLE distinct case-insensitive brand matches. # 'chatgpt' lowercase only matches under case-insensitive search # (the brand list has 'ChatGPT' proper-cased only). evidence_str = " ".join(result.evidence).lower() matched = sum(t in evidence_str for t in ("chatgpt", "anthropic", "haiku", "gemini-pro")) - assert ( - matched >= 2 - ), f"case-insensitive brand matches did not fire — only got: {result.evidence}" + assert matched >= 2, ( + f"case-insensitive brand matches did not fire — only got: {result.evidence}" + ) def test_zodiac_corpus_not_flagged_as_ai(self): """An astrology forum post with high 'Gemini' density but ZERO @@ -138,9 +138,9 @@ def test_zodiac_corpus_not_flagged_as_ai(self): "The Gemini twins in Greek mythology are Castor and Pollux.", ] result = detect_origin_heuristic(samples) - assert ( - result.likely_ai_dialogue is False - ), f"zodiac corpus wrongly flagged AI; evidence: {result.evidence}" + assert result.likely_ai_dialogue is False, ( + f"zodiac corpus wrongly flagged AI; evidence: {result.evidence}" + ) def test_french_novel_with_claude_name_not_flagged(self): """A French novel where 'Claude' is a character name (Claude is a @@ -155,9 +155,9 @@ def test_french_novel_with_claude_name_not_flagged(self): "Les amis de Claude s'étaient réunis chez lui pour fêter ses soixante ans.", ] result = detect_origin_heuristic(samples) - assert ( - result.likely_ai_dialogue is False - ), f"French novel wrongly flagged AI; evidence: {result.evidence}" + assert result.likely_ai_dialogue is False, ( + f"French novel wrongly flagged AI; evidence: {result.evidence}" + ) def test_poetry_corpus_with_haiku_sonnet_not_flagged(self): """A poetry corpus with high 'haiku', 'sonnet', 'opus' density @@ -172,9 +172,9 @@ def test_poetry_corpus_with_haiku_sonnet_not_flagged(self): "Her first opus, published at twenty, was a song cycle for soprano.", ] result = detect_origin_heuristic(samples) - assert ( - result.likely_ai_dialogue is False - ), f"poetry corpus wrongly flagged AI; evidence: {result.evidence}" + assert result.likely_ai_dialogue is False, ( + f"poetry corpus wrongly flagged AI; evidence: {result.evidence}" + ) def test_word_boundary_brand_matching(self): """Brand-term matching must use word boundaries. Embedded matches @@ -208,9 +208,9 @@ def test_word_boundary_brand_matching(self): ) # And classification should be not-AI (no real AI signals present). - assert ( - result.likely_ai_dialogue is False - ), f"corpus has no real AI signals; evidence: {result.evidence}" + assert result.likely_ai_dialogue is False, ( + f"corpus has no real AI signals; evidence: {result.evidence}" + ) def test_ambiguous_brand_with_unambiguous_signal_flagged(self): """When an ambiguous brand term ('Gemini') co-occurs with an @@ -224,9 +224,9 @@ def test_ambiguous_brand_with_unambiguous_signal_flagged(self): "MCP server config: Gemini as primary, OpenAI as fallback.", ] result = detect_origin_heuristic(samples) - assert ( - result.likely_ai_dialogue is True - ), f"ambiguous+unambiguous co-occurrence missed; evidence: {result.evidence}" + assert result.likely_ai_dialogue is True, ( + f"ambiguous+unambiguous co-occurrence missed; evidence: {result.evidence}" + ) # ── Tier 2: LLM-assisted (mocked) ───────────────────────────────────────── diff --git a/tests/test_corpus_origin_integration.py b/tests/test_corpus_origin_integration.py index 5ee38e8412..bd5abcf315 100644 --- a/tests/test_corpus_origin_integration.py +++ b/tests/test_corpus_origin_integration.py @@ -178,7 +178,7 @@ def test_corpus_origin_reclassifies_personas( # All three personas land in the new bucket. expected_personas = {"Echo", "Sparrow", "Cipher"} assert expected_personas <= persona_names_in_bucket, ( - f"Expected all three personas in agent_personas, got: " f"{persona_names_in_bucket}" + f"Expected all three personas in agent_personas, got: {persona_names_in_bucket}" ) # And NONE of them remain in the people bucket. @@ -293,8 +293,7 @@ def test_init_pass_zero_writes_origin_json_to_palace(ai_dialogue_corpus: Path, t assert isinstance(data["result"].get("likely_ai_dialogue"), bool) # Fixture is heavy AI-dialogue — heuristic should classify as such. assert data["result"]["likely_ai_dialogue"] is True, ( - "Heuristic should classify the AI-dialogue fixture as AI-dialogue. " - f"Got: {data['result']}" + f"Heuristic should classify the AI-dialogue fixture as AI-dialogue. Got: {data['result']}" ) @@ -744,9 +743,9 @@ def test_init_graceful_fallback_when_provider_unavailable( out = capsys.readouterr().out # The fallback message should mention how to silence (--no-llm) so the # user knows what flipped. - assert ( - "no-llm" in out.lower() or "--no-llm" in out - ), f"Graceful fallback message must point at --no-llm. Got: {out!r}" + assert "no-llm" in out.lower() or "--no-llm" in out, ( + f"Graceful fallback message must point at --no-llm. Got: {out!r}" + ) def test_init_graceful_fallback_on_provider_construction_error( @@ -1308,9 +1307,9 @@ def classify(self, system, user, json_mode=False): "TOPIC label instructions disappeared from SYSTEM_PROMPT — " "corpus-origin preamble appears to have replaced rather than appended" ) - assert ( - "CORPUS CONTEXT" in captured["system"] - ), "corpus-origin corpus context preamble missing from prompt" + assert "CORPUS CONTEXT" in captured["system"], ( + "corpus-origin corpus context preamble missing from prompt" + ) # ───────────────────────────────────────────────────────────────────────── @@ -1546,12 +1545,12 @@ def test_merge_tier_fields_heuristic_no_no_personas_leak(): assert wrapped is not None res = wrapped["result"] - assert ( - res["likely_ai_dialogue"] is False - ), f"Both tiers said NOT AI-dialogue; merged result must be False. Got: {res}" - assert ( - res["agent_persona_names"] == [] - ), f"No personas should leak when both tiers report none. Got: {res}" + assert res["likely_ai_dialogue"] is False, ( + f"Both tiers said NOT AI-dialogue; merged result must be False. Got: {res}" + ) + assert res["agent_persona_names"] == [], ( + f"No personas should leak when both tiers report none. Got: {res}" + ) # Heuristic owns confidence. Mocked LLM returned 0.95; heuristic's # narrative-branch confidence is 0.9. Verifying we kept 0.9 catches # any future regression that lets LLM confidence override heuristic. @@ -1609,24 +1608,23 @@ def test_merge_tier_fields_heuristic_yes_llm_yes_combines_evidence(): # LLM produced its own; the merged result should include both signal # trails for audit purposes. evidence_text = " ".join(res["evidence"]) - assert ( - "LLM-extracted" in evidence_text - ), f"LLM evidence string missing from merged result. Got: {res['evidence']}" + assert "LLM-extracted" in evidence_text, ( + f"LLM evidence string missing from merged result. Got: {res['evidence']}" + ) # Heuristic always produces at least one evidence line for AI-dialogue # input (brand-term match), so the combined list has more than just LLM's. assert len(res["evidence"]) >= 2, ( - f"Combined evidence should include both heuristic + LLM lines. " f"Got: {res['evidence']}" + f"Combined evidence should include both heuristic + LLM lines. Got: {res['evidence']}" ) # Each entry must carry its tier prefix so on-disk origin.json is # auditable — readers can tell which tier produced which signal line. tier1_lines = [e for e in res["evidence"] if e.startswith("Tier-1 heuristic: ")] tier2_lines = [e for e in res["evidence"] if e.startswith("Tier-2 LLM: ")] assert tier1_lines, ( - f"Expected at least one 'Tier-1 heuristic: ' prefixed evidence line. " - f"Got: {res['evidence']}" + f"Expected at least one 'Tier-1 heuristic: ' prefixed evidence line. Got: {res['evidence']}" ) assert tier2_lines, ( - f"Expected at least one 'Tier-2 LLM: ' prefixed evidence line. " f"Got: {res['evidence']}" + f"Expected at least one 'Tier-2 LLM: ' prefixed evidence line. Got: {res['evidence']}" ) # Every entry should be tier-prefixed (no untagged passthrough). untagged = [ @@ -1759,11 +1757,11 @@ def test_init_prints_privacy_warning_when_provider_is_external( out = capsys.readouterr().out assert "EXTERNAL API" in out, ( - f"Privacy warning must mention 'EXTERNAL API' when provider is external. " f"Got: {out!r}" + f"Privacy warning must mention 'EXTERNAL API' when provider is external. Got: {out!r}" + ) + assert "--no-llm" in out, ( + f"Privacy warning must point users at --no-llm to opt out. Got: {out!r}" ) - assert ( - "--no-llm" in out - ), f"Privacy warning must point users at --no-llm to opt out. Got: {out!r}" # The warning should also tell users MemPalace isn't responsible # for downstream provider behavior. assert ( @@ -1804,7 +1802,7 @@ def test_init_no_privacy_warning_when_provider_is_local( out = capsys.readouterr().out assert "EXTERNAL API" not in out, ( - f"Privacy warning fired for a LOCAL provider — should not have. " f"Got: {out!r}" + f"Privacy warning fired for a LOCAL provider — should not have. Got: {out!r}" ) @@ -1827,9 +1825,9 @@ def test_init_no_privacy_warning_with_no_llm_flag(ai_dialogue_corpus: Path, tmp_ mock_get.assert_not_called(), "--no-llm must short-circuit before provider acquisition" out = capsys.readouterr().out - assert ( - "EXTERNAL API" not in out - ), f"Privacy warning fired on --no-llm path — should not have. Got: {out!r}" + assert "EXTERNAL API" not in out, ( + f"Privacy warning fired on --no-llm path — should not have. Got: {out!r}" + ) # ───────────────────────────────────────────────────────────────────────── @@ -1988,9 +1986,9 @@ def test_init_accept_external_llm_flag_bypasses_consent_prompt( "--accept-external-llm must bypass the consent prompt for " "non-interactive / CI use. input() was called anyway." ) - assert ( - fake_provider.classify.called - ), "With --accept-external-llm, init must proceed with the LLM." + assert fake_provider.classify.called, ( + "With --accept-external-llm, init must proceed with the LLM." + ) def test_init_no_consent_prompt_when_endpoint_is_local( diff --git a/tests/test_hooks_bash_compat.py b/tests/test_hooks_bash_compat.py index 05fda2255a..bc194409d6 100644 --- a/tests/test_hooks_bash_compat.py +++ b/tests/test_hooks_bash_compat.py @@ -98,12 +98,12 @@ class TestNoBash4OnlyBuiltins: @_BOTH_HOOKS def test_no_mapfile(self, hook): code = _hook_src_no_comments(hook) - assert ( - "mapfile" not in code - ), f"{hook.name} uses mapfile, unavailable on macOS /bin/bash 3.2 (#1440)" - assert ( - "readarray" not in code - ), f"{hook.name} uses readarray, unavailable on macOS /bin/bash 3.2 (#1440)" + assert "mapfile" not in code, ( + f"{hook.name} uses mapfile, unavailable on macOS /bin/bash 3.2 (#1440)" + ) + assert "readarray" not in code, ( + f"{hook.name} uses readarray, unavailable on macOS /bin/bash 3.2 (#1440)" + ) @_BOTH_HOOKS def test_sed_extraction_present(self, hook): @@ -115,9 +115,9 @@ def test_sed_extraction_present(self, hook): # method back to ``mapfile`` while keeping the old commentary. src = _hook_src_no_comments(hook) # Each hook reads at least two values via ``sed -n 'Np'`` (sentinel + session_id). - assert ( - src.count("sed -n '") >= 2 - ), f"{hook.name} must use sed -n 'Np' for POSIX-portable line extraction" + assert src.count("sed -n '") >= 2, ( + f"{hook.name} must use sed -n 'Np' for POSIX-portable line extraction" + ) @_BOTH_HOOKS def test_bash_syntax_clean(self, hook): @@ -150,9 +150,9 @@ def test_save_hook_extracts_session_id(self, tmp_path): assert "WARN: input parse failed" not in log state_dir = tmp_path / ".mempalace" / "hook_state" assert not (state_dir / "last_input.log").exists() - assert not ( - state_dir / "last_python_err.log" - ).exists(), "successful parse must leave no last_python_err.log behind" + assert not (state_dir / "last_python_err.log").exists(), ( + "successful parse must leave no last_python_err.log behind" + ) def test_precompact_hook_extracts_session_id(self, tmp_path): out, _ = _run_hook( @@ -166,9 +166,9 @@ def test_precompact_hook_extracts_session_id(self, tmp_path): assert "WARN: input parse failed" not in log state_dir = tmp_path / ".mempalace" / "hook_state" assert not (state_dir / "last_input.log").exists() - assert not ( - state_dir / "last_python_err.log" - ).exists(), "successful parse must leave no last_python_err.log behind" + assert not (state_dir / "last_python_err.log").exists(), ( + "successful parse must leave no last_python_err.log behind" + ) class TestFailLoudGuard: @@ -214,9 +214,9 @@ def test_unicode_session_id_does_not_trip_guard(self, hook, tmp_path): tmp_path, ) state_dir = tmp_path / ".mempalace" / "hook_state" - assert not ( - state_dir / "last_input.log" - ).exists(), "unicode-only session_id sanitized to empty must NOT trip the guard" + assert not (state_dir / "last_input.log").exists(), ( + "unicode-only session_id sanitized to empty must NOT trip the guard" + ) @_BOTH_HOOKS def test_literal_unknown_session_id_does_not_trip_guard(self, hook, tmp_path): @@ -231,9 +231,9 @@ def test_literal_unknown_session_id_does_not_trip_guard(self, hook, tmp_path): tmp_path, ) state_dir = tmp_path / ".mempalace" / "hook_state" - assert not ( - state_dir / "last_input.log" - ).exists(), "literal session_id='unknown' must NOT trip the guard" + assert not (state_dir / "last_input.log").exists(), ( + "literal session_id='unknown' must NOT trip the guard" + ) @_BOTH_HOOKS def test_dump_is_bounded_and_overwritten(self, hook, tmp_path): @@ -251,9 +251,9 @@ def test_dump_is_bounded_and_overwritten(self, hook, tmp_path): big_payload = "x" * 4097 _run_hook(hook, big_payload, tmp_path) last_input = tmp_path / ".mempalace" / "hook_state" / "last_input.log" - assert ( - last_input.stat().st_size == 4096 - ), f"cap must be exactly 4096 bytes; got {last_input.stat().st_size}" + assert last_input.stat().st_size == 4096, ( + f"cap must be exactly 4096 bytes; got {last_input.stat().st_size}" + ) # Second failure with a smaller payload (also not valid JSON, so the # guard fires) overwrites the first; the file shrinks instead of # accumulating. @@ -310,9 +310,9 @@ def test_python_stderr_captured_on_parse_failure(self, hook, tmp_path): # Python's json.load raises JSONDecodeError with a recognizable # traceback. Don't pin the exact message (it varies by Python # version) but assert at least one canonical marker is present. - assert ( - "Traceback" in contents or "json" in contents.lower() - ), f"expected Python traceback or json error, got: {contents!r}" + assert "Traceback" in contents or "json" in contents.lower(), ( + f"expected Python traceback or json error, got: {contents!r}" + ) @_BOTH_HOOKS def test_python_stderr_log_is_not_world_readable_on_failure(self, hook, tmp_path): diff --git a/tests/test_hooks_cli.py b/tests/test_hooks_cli.py index e9bd7d7292..234bef2bce 100644 --- a/tests/test_hooks_cli.py +++ b/tests/test_hooks_cli.py @@ -850,9 +850,9 @@ def test_spawn_mine_releases_slot_on_oserror(tmp_path): with patch("mempalace.hooks_cli.subprocess.Popen", side_effect=OSError("spawn fail")): with pytest.raises(OSError): _spawn_mine(cmd) - assert ( - not pid_file.exists() - ), "slot must be released so the next hook fire isn't permanently blocked" + assert not pid_file.exists(), ( + "slot must be released so the next hook fire isn't permanently blocked" + ) def test_spawn_mine_passes_pid_file_env_var(tmp_path): diff --git a/tests/test_hooks_shell.py b/tests/test_hooks_shell.py index cc8b8e2817..7462d97b98 100644 --- a/tests/test_hooks_shell.py +++ b/tests/test_hooks_shell.py @@ -123,13 +123,13 @@ def test_explicit_override_wins_over_path(self, tmp_path): {"session_id": "abc", "stop_hook_active": False, "transcript_path": ""}, env_overrides={"MEMPAL_PYTHON": str(fake), "HOME": str(tmp_path)}, ) - assert ( - result.returncode == 0 - ), f"hook exited non-zero: stderr={result.stderr!r} stdout={result.stdout!r}" + assert result.returncode == 0, ( + f"hook exited non-zero: stderr={result.stderr!r} stdout={result.stdout!r}" + ) invocations = marker.read_text().splitlines() if marker.exists() else [] - assert ( - "override_python" in invocations - ), f"MEMPAL_PYTHON override was not used. Marker log: {invocations!r}" + assert "override_python" in invocations, ( + f"MEMPAL_PYTHON override was not used. Marker log: {invocations!r}" + ) def test_ignores_override_when_not_executable(self, tmp_path): """If MEMPAL_PYTHON is set but the file isn't executable, the @@ -143,9 +143,9 @@ def test_ignores_override_when_not_executable(self, tmp_path): {"session_id": "abc", "stop_hook_active": False, "transcript_path": ""}, env_overrides={"MEMPAL_PYTHON": str(bogus), "HOME": str(tmp_path)}, ) - assert ( - result.returncode == 0 - ), f"hook crashed on non-executable MEMPAL_PYTHON: {result.stderr!r}" + assert result.returncode == 0, ( + f"hook crashed on non-executable MEMPAL_PYTHON: {result.stderr!r}" + ) def test_falls_back_to_path_when_unset(self, tmp_path): """With MEMPAL_PYTHON unset, the hook uses whatever ``python3`` @@ -165,6 +165,6 @@ def test_falls_back_to_path_when_unset(self, tmp_path): ) assert result.returncode == 0 invocations = marker.read_text().splitlines() if marker.exists() else [] - assert ( - "python3" in invocations - ), f"fallback-to-PATH did not use the shimmed python3. Marker log: {invocations!r}" + assert "python3" in invocations, ( + f"fallback-to-PATH did not use the shimmed python3. Marker log: {invocations!r}" + ) diff --git a/tests/test_hybrid_candidate_union.py b/tests/test_hybrid_candidate_union.py index feca81e695..0771001d91 100644 --- a/tests/test_hybrid_candidate_union.py +++ b/tests/test_hybrid_candidate_union.py @@ -79,8 +79,7 @@ def test_union_surfaces_bm25_strong_vector_distant_doc(self, tmp_path): result = search_memories(_NARRATIVE_QUERY, palace, n_results=5, candidate_strategy="union") ids = [h["source_file"] for h in result["results"]] assert "brand_voice_D4.md" in ids, ( - "union mode must surface BM25-strong docs even when vector signal " - f"is weak; got {ids}" + f"union mode must surface BM25-strong docs even when vector signal is weak; got {ids}" ) def test_union_preserves_vector_hits(self, tmp_path): @@ -138,9 +137,9 @@ def test_union_respects_n_results_limit(self, tmp_path): # 4-doc corpus, n_results=2 → union pool can grow to ~8 candidates, # rerank reorders them, but final list must respect the cap. result = search_memories(_NARRATIVE_QUERY, palace, n_results=2, candidate_strategy="union") - assert ( - len(result["results"]) <= 2 - ), f"union must trim to n_results=2; got {len(result['results'])} results" + assert len(result["results"]) <= 2, ( + f"union must trim to n_results=2; got {len(result['results'])} results" + ) def test_union_skipped_when_max_distance_set(self, tmp_path): """``max_distance`` is a vector-distance threshold; BM25-only diff --git a/tests/test_i18n.py b/tests/test_i18n.py index 1f53da7616..40af9828aa 100644 --- a/tests/test_i18n.py +++ b/tests/test_i18n.py @@ -155,7 +155,7 @@ def test_direct_address_key_is_singular_string_for_all_locales(): ) if "direct_address_pattern" in section: val = section["direct_address_pattern"] - assert isinstance( - val, str - ), f"{lang}: 'direct_address_pattern' must be str, got {type(val).__name__}" + assert isinstance(val, str), ( + f"{lang}: 'direct_address_pattern' must be str, got {type(val).__name__}" + ) assert val, f"{lang}: 'direct_address_pattern' is empty" diff --git a/tests/test_i18n_lang_case.py b/tests/test_i18n_lang_case.py index 4c57ed1248..47b101e3c3 100644 --- a/tests/test_i18n_lang_case.py +++ b/tests/test_i18n_lang_case.py @@ -74,9 +74,9 @@ def test_get_entity_patterns_shares_cache_across_cases(): cache_keys = list(i18n._entity_cache.keys()) get_entity_patterns(("ZH-CN",)) get_entity_patterns(("zh-cn",)) - assert len(i18n._entity_cache) == len( - cache_keys - ), "different casings of the same language must not create new cache entries" + assert len(i18n._entity_cache) == len(cache_keys), ( + "different casings of the same language must not create new cache entries" + ) def test_unknown_language_still_falls_back_to_english(): diff --git a/tests/test_init.py b/tests/test_init.py index bf6c9a4eb3..ced0497635 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -65,15 +65,15 @@ def test_init_filters_sys_path_from_leaked_pythonpath(pythonpath): out = result.stdout # Env must be preserved verbatim: embedded callers may need it. expected_env = repr(pythonpath) if pythonpath is not None else repr(None) - assert ( - f"ENV: {expected_env}" in out - ), f"PYTHONPATH should be preserved by package import: {diag}" + assert f"ENV: {expected_env}" in out, ( + f"PYTHONPATH should be preserved by package import: {diag}" + ) assert "SENTINEL_IN_PATH: False" in out, f"sentinel-prefix leak: {diag}" # Filter must not over-strip: the mempalace package itself must remain # importable, so its parent directory must survive on sys.path. - assert ( - "MEMPALACE_PARENT_PRESENT: True" in out - ), f"filter over-stripped sys.path (mempalace parent gone): {diag}" + assert "MEMPALACE_PARENT_PRESENT: True" in out, ( + f"filter over-stripped sys.path (mempalace parent gone): {diag}" + ) def test_init_preserves_cwd_marker_when_pythonpath_collides(): @@ -94,7 +94,7 @@ def test_init_preserves_cwd_marker_when_pythonpath_collides(): text=True, check=False, ) - diag = f"rc={result.returncode}; stdout={result.stdout!r}; " f"stderr={result.stderr!r}" + diag = f"rc={result.returncode}; stdout={result.stdout!r}; stderr={result.stderr!r}" assert result.returncode == 0, f"subprocess failed: {diag}" assert "CWD_IN_PATH: True" in result.stdout, f"cwd marker dropped: {diag}" assert "DOT_IN_PATH: False" in result.stdout, f"dot leak survived: {diag}" diff --git a/tests/test_llm_client.py b/tests/test_llm_client.py index d7ade20b36..324cb3eb6b 100644 --- a/tests/test_llm_client.py +++ b/tests/test_llm_client.py @@ -445,9 +445,9 @@ def test_openai_compat_api_key_source_flag_when_explicit(monkeypatch): monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env-irrelevant") p = OpenAICompatProvider(model="x", endpoint="http://h", api_key="sk-from-flag") assert p.api_key == "sk-from-flag" - assert ( - p.api_key_source == "flag" - ), f"Explicit api_key arg must produce api_key_source='flag'; got {p.api_key_source!r}" + assert p.api_key_source == "flag", ( + f"Explicit api_key arg must produce api_key_source='flag'; got {p.api_key_source!r}" + ) def test_openai_compat_api_key_source_env_when_fallback(monkeypatch): @@ -458,9 +458,9 @@ def test_openai_compat_api_key_source_env_when_fallback(monkeypatch): monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env") p = OpenAICompatProvider(model="x", endpoint="http://h") assert p.api_key == "sk-from-env" - assert ( - p.api_key_source == "env" - ), f"Env-fallback api_key must produce api_key_source='env'; got {p.api_key_source!r}" + assert p.api_key_source == "env", ( + f"Env-fallback api_key must produce api_key_source='env'; got {p.api_key_source!r}" + ) def test_anthropic_api_key_source_tracking(monkeypatch): @@ -468,14 +468,14 @@ def test_anthropic_api_key_source_tracking(monkeypatch): passed explicitly, 'env' when resolved from ANTHROPIC_API_KEY env.""" monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-env") p_flag = AnthropicProvider(model="claude-haiku", api_key="sk-ant-flag") - assert ( - p_flag.api_key_source == "flag" - ), f"Explicit api_key must produce 'flag'; got {p_flag.api_key_source!r}" + assert p_flag.api_key_source == "flag", ( + f"Explicit api_key must produce 'flag'; got {p_flag.api_key_source!r}" + ) p_env = AnthropicProvider(model="claude-haiku") assert p_env.api_key == "sk-ant-env" - assert ( - p_env.api_key_source == "env" - ), f"Env-fallback must produce 'env'; got {p_env.api_key_source!r}" + assert p_env.api_key_source == "env", ( + f"Env-fallback must produce 'env'; got {p_env.api_key_source!r}" + ) def test_ollama_api_key_source_is_none(): diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index c4004d519b..4a1aae2f8e 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -63,12 +63,12 @@ def test_mcp_main_strips_leaked_pythonpath_from_env(): ) diag = f"rc={result.returncode}; stdout={result.stdout!r}; stderr={result.stderr!r}" assert result.returncode == 0, f"subprocess failed: {diag}" - assert ( - f"ENV_MID: {expected_env!r}" in result.stderr - ), f"package import unexpectedly stripped env (regression in __init__.py): {diag}" - assert ( - "SENTINEL_IN_PATH: False" in result.stderr - ), f"package import did not filter sys.path (regression in __init__.py): {diag}" + assert f"ENV_MID: {expected_env!r}" in result.stderr, ( + f"package import unexpectedly stripped env (regression in __init__.py): {diag}" + ) + assert "SENTINEL_IN_PATH: False" in result.stderr, ( + f"package import did not filter sys.path (regression in __init__.py): {diag}" + ) assert "ENV_AFTER: None" in result.stderr, f"MCP server did not strip PYTHONPATH: {diag}" @@ -250,9 +250,9 @@ def test_eager_warmup_explicit_falsy_skips_collection_open_without_warning( ) assert result.returncode == 0, f"stderr={result.stderr!r}" assert not marker.exists(), f"warmup ran for explicit-falsy value {value!r}" - assert ( - "not recognized" not in result.stderr - ), f"explicit-falsy {value!r} should not log a warning; stderr={result.stderr!r}" + assert "not recognized" not in result.stderr, ( + f"explicit-falsy {value!r} should not log a warning; stderr={result.stderr!r}" + ) @pytest.mark.parametrize("value", ["tru", "maybe", "ENABLED", "2"]) def test_eager_warmup_unrecognized_value_warns_and_skips_collection_open(self, tmp_path, value): @@ -298,9 +298,9 @@ def test_eager_warmup_truthy_opens_collection_and_invokes_query(self, tmp_path, extra_code=extra, ) assert result.returncode == 0, f"stderr={result.stderr!r}" - assert ( - open_marker.exists() - ), f"_get_collection not called for {value!r}; stderr={result.stderr!r}" + assert open_marker.exists(), ( + f"_get_collection not called for {value!r}; stderr={result.stderr!r}" + ) assert query_marker.exists(), ( f"col.query not invoked for {value!r} — warmup is a no-op " f"(would let cold-load hit first MCP call); stderr={result.stderr!r}" @@ -1134,9 +1134,9 @@ def test_add_drawer_shared_header_no_collision(self, monkeypatch, config, palace assert result1["success"] is True assert result2["success"] is True - assert ( - result1["drawer_id"] != result2["drawer_id"] - ), "Documents with shared header but different content must have distinct drawer IDs" + assert result1["drawer_id"] != result2["drawer_id"], ( + "Documents with shared header but different content must have distinct drawer IDs" + ) def test_delete_drawer(self, monkeypatch, config, palace_path, seeded_collection, kg): _patch_mcp_server(monkeypatch, config, kg) @@ -2121,9 +2121,9 @@ def _spy_create(self, name, **kwargs): all_calls = captured["get"] + captured["create"] assert all_calls, "expected get_collection or create_collection to be called" for kwargs in all_calls: - assert ( - "embedding_function" in kwargs - ), f"missing embedding_function= in chromadb call: {kwargs}" + assert "embedding_function" in kwargs, ( + f"missing embedding_function= in chromadb call: {kwargs}" + ) assert kwargs["embedding_function"] is not None # Same expectation on the create=False (cache-miss) reopen path. @@ -2561,9 +2561,9 @@ def fake_normcase(p: str) -> str: result = mcp_server._canonicalize_kg_path("/some/Path/KG.sqlite3") - assert ( - result == ">" - ), f"expected normcase(realpath(p)) composition, got {result!r}" + assert result == ">", ( + f"expected normcase(realpath(p)) composition, got {result!r}" + ) def test_get_kg_dedupes_symlink_alias_end_to_end(self, tmp_path, monkeypatch): """End-to-end: two ``_get_kg()`` calls via different symlink layers diff --git a/tests/test_mcp_stdio_protection.py b/tests/test_mcp_stdio_protection.py index 8b3731bfdc..7ab35b562a 100644 --- a/tests/test_mcp_stdio_protection.py +++ b/tests/test_mcp_stdio_protection.py @@ -78,6 +78,6 @@ def test_mcp_server_no_stdout_noise_on_clean_exit(): capture_output=True, timeout=60, ) - assert ( - proc.stdout == b"" - ), f"stdout must be empty before the first JSON-RPC response, but got: {proc.stdout!r}" + assert proc.stdout == b"", ( + f"stdout must be empty before the first JSON-RPC response, but got: {proc.stdout!r}" + ) diff --git a/tests/test_normalize.py b/tests/test_normalize.py index 2b0f180710..f56ff3a6b7 100644 --- a/tests/test_normalize.py +++ b/tests/test_normalize.py @@ -1336,11 +1336,7 @@ class TestStripNoiseRemovesSystemChrome: def test_strips_line_anchored_system_reminder_block(self): text = ( - "> User:\n" - "\n" - "Auto-save reminder...\n" - "\n" - "> Real message." + "> User:\n\nAuto-save reminder...\n\n> Real message." ) out = strip_noise(text) assert "system-reminder" not in out @@ -1350,7 +1346,7 @@ def test_strips_line_anchored_system_reminder_block(self): def test_strips_system_reminder_with_blockquote_prefix(self): # _messages_to_transcript prefixes lines with "> ", so the line # anchor must also accept that shape. - text = "> User:\n" "> Injected noise\n" "> Real message." + text = "> User:\n> Injected noise\n> Real message." out = strip_noise(text) assert "Injected noise" not in out assert "Real message." in out diff --git a/tests/test_palace_graph_tunnels.py b/tests/test_palace_graph_tunnels.py index 99fccd5a9a..8be8042423 100644 --- a/tests/test_palace_graph_tunnels.py +++ b/tests/test_palace_graph_tunnels.py @@ -63,9 +63,9 @@ def test_save_tunnels_restricts_permissions(self, tmp_path, monkeypatch): assert file_mode == 0o600, f"tunnels.json mode is {oct(file_mode)}, expected 0o600" parent_mode = stat.S_IMODE(os.stat(tunnel_file.parent).st_mode) - assert ( - parent_mode == 0o700 - ), f"tunnels.json parent dir mode is {oct(parent_mode)}, expected 0o700" + assert parent_mode == 0o700, ( + f"tunnels.json parent dir mode is {oct(parent_mode)}, expected 0o700" + ) class TestExplicitTunnels: diff --git a/tests/test_palace_locks.py b/tests/test_palace_locks.py index df94f16d8d..b4d2fbc259 100644 --- a/tests/test_palace_locks.py +++ b/tests/test_palace_locks.py @@ -199,9 +199,9 @@ def test_reentrant_same_thread_passes_through(tmp_path, monkeypatch): child = ctx.Process(target=_try_acquire_expect_busy, args=(palace, result_q)) try: child.start() - assert ( - result_q.get(timeout=10) == "busy" - ), "outer lock should still be held by parent after inner re-entrant exit" + assert result_q.get(timeout=10) == "busy", ( + "outer lock should still be held by parent after inner re-entrant exit" + ) child.join(timeout=5) assert child.exitcode == 0 finally: @@ -267,9 +267,9 @@ def test_lock_failure_message_names_holder(tmp_path, monkeypatch): pytest.fail("second acquire of same palace should have raised") msg = str(excinfo.value) - assert ( - f"PID {holder_pid}" in msg - ), f"lock-failure message must name the holder PID; got: {msg!r}" + assert f"PID {holder_pid}" in msg, ( + f"lock-failure message must name the holder PID; got: {msg!r}" + ) finally: open(release, "w").close() holder.join(timeout=5) diff --git a/tests/test_readme_claims.py b/tests/test_readme_claims.py index 7d4754af3d..f096b3d6f9 100644 --- a/tests/test_readme_claims.py +++ b/tests/test_readme_claims.py @@ -597,13 +597,13 @@ def test_backends_base_exists(self): """Claim: pluggable backends. backends/base.py must define an abstract base class.""" path = MEMPALACE_PKG / "backends" / "base.py" - assert ( - path.is_file() - ), "mempalace/backends/base.py does not exist. Backend abstraction layer is missing." + assert path.is_file(), ( + "mempalace/backends/base.py does not exist. Backend abstraction layer is missing." + ) src = _read(path) - assert ( - "ABC" in src or "abstractmethod" in src - ), "backends/base.py does not define an abstract base class." + assert "ABC" in src or "abstractmethod" in src, ( + "backends/base.py does not define an abstract base class." + ) def test_backends_chroma_exists(self): """Claim: ChromaDB backend implementation. @@ -611,9 +611,9 @@ def test_backends_chroma_exists(self): path = MEMPALACE_PKG / "backends" / "chroma.py" assert path.is_file(), "mempalace/backends/chroma.py does not exist." src = _read(path) - assert ( - "BaseCollection" in src or "base" in src - ), "backends/chroma.py does not reference the base class." + assert "BaseCollection" in src or "base" in src, ( + "backends/chroma.py does not reference the base class." + ) def test_backends_importable(self): """Both backend modules should be importable.""" @@ -650,9 +650,9 @@ def test_at_least_8_language_files(self): def test_english_baseline_exists(self): """en.json must exist as the baseline language file.""" path = MEMPALACE_PKG / "i18n" / "en.json" - assert ( - path.is_file() - ), "mempalace/i18n/en.json does not exist. English baseline is required." + assert path.is_file(), ( + "mempalace/i18n/en.json does not exist. English baseline is required." + ) # --------------------------------------------------------------------------- @@ -737,9 +737,9 @@ def test_all_tool_count_mentions_consistent(self): counts = re.findall(r"(\d+)\s+tools", readme) if len(counts) > 1: unique = set(counts) - assert ( - len(unique) == 1 - ), f"README mentions different tool counts: {counts}. All occurrences must agree." + assert len(unique) == 1, ( + f"README mentions different tool counts: {counts}. All occurrences must agree." + ) # --------------------------------------------------------------------------- diff --git a/tests/test_repair.py b/tests/test_repair.py index d8dfa942f3..acbf31d10f 100644 --- a/tests/test_repair.py +++ b/tests/test_repair.py @@ -1381,9 +1381,9 @@ def test_rebuild_index_runs_sqlite_preflight_before_chromadb_open(tmp_path, caps PAGE = 4096 CORRUPT_BYTES = 16384 # 4 pages HEADER_GUARD = PAGE * 2 # leave header + root pages intact - assert ( - pre_size >= HEADER_GUARD + CORRUPT_BYTES - ), f"sqlite db too small to mangle without truncating: {pre_size} bytes" + assert pre_size >= HEADER_GUARD + CORRUPT_BYTES, ( + f"sqlite db too small to mangle without truncating: {pre_size} bytes" + ) # Round (pre_size - CORRUPT_BYTES) down to a page boundary so we # mangle whole pages. Cap at offset 40960 (page 10) for stable # diagnostics across SQLite versions that may grow the file. diff --git a/tests/test_save_hook_mines.py b/tests/test_save_hook_mines.py index e11234af26..68f04e9d28 100644 --- a/tests/test_save_hook_mines.py +++ b/tests/test_save_hook_mines.py @@ -33,12 +33,12 @@ def test_hook_mines_transcript_path(self): # `--mode convos` so the convo miner runs (not the projects miner). assert "TRANSCRIPT_PATH" in src, "hook must read transcript_path" assert "mempalace mine" in src, "hook must invoke `mempalace mine`" - assert ( - 'dirname "$TRANSCRIPT_PATH"' in src - ), "hook must mine the transcript's parent directory" - assert ( - "--mode convos" in src - ), "transcript mine must use --mode convos, not the projects miner" + assert 'dirname "$TRANSCRIPT_PATH"' in src, ( + "hook must mine the transcript's parent directory" + ) + assert "--mode convos" in src, ( + "transcript mine must use --mode convos, not the projects miner" + ) def test_mempal_dir_default_not_empty(self): """If MEMPAL_DIR is still used, it should have a sensible default, @@ -86,16 +86,16 @@ def _strip_comments(src: str) -> str: def test_save_hook_defines_and_uses_validator(self): src = self._strip_comments(self._hook_src("mempal_save_hook.sh")) assert "is_valid_transcript_path() {" in src, "validator function must be defined" - assert ( - 'is_valid_transcript_path "$TRANSCRIPT_PATH"' in src - ), "validator must be invoked against TRANSCRIPT_PATH before mining" + assert 'is_valid_transcript_path "$TRANSCRIPT_PATH"' in src, ( + "validator must be invoked against TRANSCRIPT_PATH before mining" + ) def test_precompact_hook_defines_and_uses_validator(self): src = self._strip_comments(self._hook_src("mempal_precompact_hook.sh")) assert "is_valid_transcript_path() {" in src, "validator function must be defined" - assert ( - 'is_valid_transcript_path "$TRANSCRIPT_PATH"' in src - ), "validator must be invoked against TRANSCRIPT_PATH before mining" + assert 'is_valid_transcript_path "$TRANSCRIPT_PATH"' in src, ( + "validator must be invoked against TRANSCRIPT_PATH before mining" + ) @pytest.mark.skipif( sys.platform == "win32", @@ -112,9 +112,7 @@ def test_validators_run_via_bash(self, tmp_path): end = src.index("\n}\n", start) + 2 func_src = src[start:end] script = tmp_path / "v.sh" - script.write_text( - f"{func_src}\n" 'is_valid_transcript_path "$1" && echo OK || echo NO\n' - ) + script.write_text(f'{func_src}\nis_valid_transcript_path "$1" && echo OK || echo NO\n') def run(arg: str) -> str: return subprocess.run( diff --git a/tests/test_searcher.py b/tests/test_searcher.py index ccea0635c8..721bb117e6 100644 --- a/tests/test_searcher.py +++ b/tests/test_searcher.py @@ -184,12 +184,11 @@ def test_effective_distance_clamped_to_valid_cosine_range(self): # Invariants on every hit. for h in hits: - assert ( - 0.0 <= h["similarity"] <= 1.0 - ), f"similarity out of range: {h['similarity']} for {h['source_file']}" + assert 0.0 <= h["similarity"] <= 1.0, ( + f"similarity out of range: {h['similarity']} for {h['source_file']}" + ) assert 0.0 <= h["effective_distance"] <= 2.0, ( - f"effective_distance out of range: {h['effective_distance']} " - f"for {h['source_file']}" + f"effective_distance out of range: {h['effective_distance']} for {h['source_file']}" ) # With the clamp, the closet-boosted a.md still ranks ahead of b.md — @@ -338,9 +337,9 @@ def test_search_applies_bm25_hybrid_rerank(self, fake_palace_path, capsys): captured = capsys.readouterr() first_block, _, _ = captured.out.partition("[2]") # Lexical match must rank first - assert ( - "b.md" in first_block - ), f"expected lexical match 'b.md' at rank 1, got:\n{captured.out}" + assert "b.md" in first_block, ( + f"expected lexical match 'b.md' at rank 1, got:\n{captured.out}" + ) # Non-zero bm25 reported assert "bm25=" in first_block assert "bm25=0.0" not in first_block diff --git a/tests/test_sweeper.py b/tests/test_sweeper.py index 983724a6f9..eef9c6a148 100644 --- a/tests/test_sweeper.py +++ b/tests/test_sweeper.py @@ -97,9 +97,9 @@ def test_parse_normalizes_assistant_content_list_to_text(self, mock_claude_jsonl records = list(parse_claude_jsonl(str(mock_claude_jsonl))) assistant_rec = records[1] assert assistant_rec["role"] == "assistant" - assert ( - "Paris" in assistant_rec["content"] - ), f"Assistant content blocks must be flattened to text; got: {assistant_rec['content']!r}" + assert "Paris" in assistant_rec["content"], ( + f"Assistant content blocks must be flattened to text; got: {assistant_rec['content']!r}" + ) def test_parse_preserves_tool_blocks_verbatim(self, tmp_path): """Per the design principle "verbatim always", tool_use and diff --git a/tests/test_sync.py b/tests/test_sync.py index ea17040b84..d32db688ca 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -444,9 +444,9 @@ def fake_wal(operation, params, result=None): # Allow-list — params must be exactly the documented audit shape so # any future leak (source_file, content, ID lists, etc.) trips a # test failure rather than slipping through a deny-list. - assert set(params.keys()) <= { - "first_id" - }, f"WAL params drifted from the audit allow-list: {params.keys()}" + assert set(params.keys()) <= {"first_id"}, ( + f"WAL params drifted from the audit allow-list: {params.keys()}" + ) def test_registry_sentinels_preserved_on_apply(self, tmp_dir, palace_path): """F2 regression: convo miner `_reg_*` sentinels must survive sync apply. @@ -567,9 +567,9 @@ def test_auto_detect_picks_deepest_root(self, tmp_dir, palace_path): inner_resolved = inner.resolve(strict=False) outer_resolved = outer.resolve(strict=False) assert inner_resolved in roots, f"expected inner in roots, got {roots}" - assert ( - outer_resolved not in roots - ), f"deepest should win exclusively: roots={roots}, outer leaked" + assert outer_resolved not in roots, ( + f"deepest should win exclusively: roots={roots}, outer leaked" + ) def test_apply_with_empty_project_dirs_raises(self, palace_path): """Round-2 P1: `project_dirs=[]` (empty list) with apply must raise, @@ -608,9 +608,9 @@ def boom(*args, **kwargs): project_dirs=[synced_world["repo_path"]], dry_run=False, ) - assert any( - "Closet purge skipped" in record.getMessage() for record in caplog.records - ), f"expected closet-skip warning, got: {[r.getMessage() for r in caplog.records]}" + assert any("Closet purge skipped" in record.getMessage() for record in caplog.records), ( + f"expected closet-skip warning, got: {[r.getMessage() for r in caplog.records]}" + ) def test_metadata_cache_cleared_on_exception(self, monkeypatch, config, synced_world, kg): """F9 regression: tool_sync's try/finally must clear `_metadata_cache` @@ -651,9 +651,9 @@ def explode(*args, **kwargs): assert result.get("success") is False assert "simulated" in result.get("error", "") - assert ( - mcp_server._metadata_cache is None - ), "F9: cache must be cleared even when sync_palace raises" + assert mcp_server._metadata_cache is None, ( + "F9: cache must be cleared even when sync_palace raises" + ) def test_sync_report_keys_stable(self, synced_world): """Regression: SyncReport schema must not silently drop a field.""" @@ -946,9 +946,9 @@ def test_symlinked_project_root_resolves(self, tmp_dir, palace_path): wing="demo", dry_run=True, ) - assert ( - report["gitignored"] == 1 - ), f"symmetric resolve broken: drawer mis-bucketed; report={report}" + assert report["gitignored"] == 1, ( + f"symmetric resolve broken: drawer mis-bucketed; report={report}" + ) assert report["out_of_scope"] == 0 def test_classification_cache_avoids_redundant_disk_hits( @@ -1005,9 +1005,9 @@ def counting_classify(*args, **kwargs): ) assert report["scanned"] == 5 assert report["gitignored"] == 5 - assert ( - call_count["n"] == 1 - ), f"cache miss: expected 1 _classify_drawer call (4 cache hits), got {call_count['n']}" + assert call_count["n"] == 1, ( + f"cache miss: expected 1 _classify_drawer call (4 cache hits), got {call_count['n']}" + ) def test_closet_batch_purge_single_call(self, synced_world, monkeypatch): """Batched $in closet purge: one delete() call across all removable @@ -1074,16 +1074,16 @@ def wrapped_get_closets(p, create=False): str(repo_path / "deleted.py"), } expected = len(seeded_sources & set(report["by_source"].keys())) - assert ( - report["removed_closets"] == expected - ), f"removed_closets ({report['removed_closets']}) != |seeded ∩ removable| ({expected})" + assert report["removed_closets"] == expected, ( + f"removed_closets ({report['removed_closets']}) != |seeded ∩ removable| ({expected})" + ) assert "wrapper" in captured, "get_closets_collection patch not invoked" - assert ( - captured["wrapper"].delete_calls == 1 - ), f"expected one batch delete call, got {captured['wrapper'].delete_calls}" - assert ( - captured["wrapper"].get_calls == 1 - ), f"expected one batch get call, got {captured['wrapper'].get_calls}" + assert captured["wrapper"].delete_calls == 1, ( + f"expected one batch delete call, got {captured['wrapper'].delete_calls}" + ) + assert captured["wrapper"].get_calls == 1, ( + f"expected one batch get call, got {captured['wrapper'].get_calls}" + ) def test_registry_check_runs_before_cache_lookup(self, tmp_dir, palace_path): """A non-registry drawer with the same source_file must NOT poison @@ -1148,9 +1148,9 @@ def test_registry_check_runs_before_cache_lookup(self, tmp_dir, palace_path): finally: del client assert "a_regular" not in survivors - assert ( - "_reg_zzz_sentinel" in survivors - ), "registry sentinel was incorrectly pruned via cached non-registry verdict" + assert "_reg_zzz_sentinel" in survivors, ( + "registry sentinel was incorrectly pruned via cached non-registry verdict" + ) def test_normalize_project_dirs_sort_stable_on_equal_length(self): """`_normalize_project_dirs` must sort by `(-len, str)` so equal-length diff --git a/tools/render_jsonl.py b/tools/render_jsonl.py index 2bec0dab2a..791ce66283 100755 --- a/tools/render_jsonl.py +++ b/tools/render_jsonl.py @@ -70,7 +70,7 @@ def main(): ] out.write("\n".join(header)) for ts, role, text in turns: - out.write(f"\n[{ts}] {role.upper()}\n{text}\n\n{'-'*72}\n") + out.write(f"\n[{ts}] {role.upper()}\n{text}\n\n{'-' * 72}\n") if out is not sys.stdout: out.close() print(f"Wrote {len(turns)} turns to {sys.argv[2]}") From 283ef0336199ded415404516fcf1b938f7c00b45 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Mon, 18 May 2026 00:08:17 -0300 Subject: [PATCH 11/39] ci(lint): pin ruff in pre-commit + uv.lock too (#1541 review) Addresses Copilot review: the ruff pin was incomplete. - .pre-commit-config.yaml stayed on ruff-pre-commit v0.4.10, so contributors running pre-commit would reintroduce the exact 0.4-vs-0.5 formatter drift this PR removes. Bumped rev to v0.15.9 and rewrote the lock-step comment to point at pyproject as the source of truth. - uv.lock still constrained dev ruff at >=0.4.0, so `uv sync --extra dev` (the documented setup) would not honor the new exact pin. Regenerated; the dev specifier is now ==0.15.9 in both lock entries. ruff check/format still pass; no formatting delta from these files. --- .pre-commit-config.yaml | 9 +++++---- uv.lock | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8f94c0b960..b413e1c149 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,9 +1,10 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - # Keep in lock-step with the ruff version pinned in .github/workflows/ci.yml - # (>=0.4.0,<0.5). Using a newer rev here produces a different formatter - # output than CI and breaks `ruff format --check` in the lint job. - rev: v0.4.10 + # Keep this rev in lock-step with the exact ruff pin in pyproject.toml + # ([project.optional-dependencies].dev / [dependency-groups].dev) and + # .github/workflows/ci.yml. A different ruff here produces different + # formatter output than CI and breaks `ruff format --check` in lint. + rev: v0.15.9 hooks: - id: ruff args: [--fix] diff --git a/uv.lock b/uv.lock index 2c96d67816..5117685bf8 100644 --- a/uv.lock +++ b/uv.lock @@ -1224,7 +1224,7 @@ requires-dist = [ { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.0" }, { name = "pyyaml", specifier = ">=6.0,<7" }, - { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.4.0" }, + { name = "ruff", marker = "extra == 'dev'", specifier = "==0.15.9" }, { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0.0" }, ] provides-extras = ["dev", "spellcheck", "gpu", "dml", "coreml"] @@ -1234,7 +1234,7 @@ dev = [ { name = "psutil", specifier = ">=5.9" }, { name = "pytest", specifier = ">=7.0" }, { name = "pytest-cov", specifier = ">=4.0" }, - { name = "ruff", specifier = ">=0.4.0" }, + { name = "ruff", specifier = "==0.15.9" }, ] [[package]] From eee5f327a2aa49b0abbe6a94557a4607591f46d6 Mon Sep 17 00:00:00 2001 From: fatkobra <55045047+fatkobra@users.noreply.github.com> Date: Mon, 18 May 2026 10:05:11 +0000 Subject: [PATCH 12/39] Write placeholder PID when claiming mine slot --- mempalace/hooks_cli.py | 10 ++++++++-- tests/test_hooks_cli.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index be419f728c..ddcb679c0a 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -360,7 +360,10 @@ def _claim_mine_slot(cmd: list[str]) -> Optional[Path]: pid_file.parent.mkdir(parents=True, exist_ok=True) try: fd = os.open(str(pid_file), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) - os.close(fd) + try: + os.write(fd, str(os.getpid()).encode("ascii")) + finally: + os.close(fd) return pid_file except FileExistsError: pass @@ -378,7 +381,10 @@ def _claim_mine_slot(cmd: list[str]) -> Optional[Path]: return None try: fd = os.open(str(pid_file), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) - os.close(fd) + try: + os.write(fd, str(os.getpid()).encode("ascii")) + finally: + os.close(fd) return pid_file except FileExistsError: return None diff --git a/tests/test_hooks_cli.py b/tests/test_hooks_cli.py index e9bd7d7292..9ce9fa7f59 100644 --- a/tests/test_hooks_cli.py +++ b/tests/test_hooks_cli.py @@ -28,6 +28,8 @@ hook_session_start, hook_precompact, run_hook, + _claim_mine_slot, + _pid_file_for_cmd, ) @@ -650,6 +652,39 @@ def test_mine_sync_uses_mempalace_python(tmp_path): assert cmd[0] == "/fake/venv/python" +def test_claim_mine_slot_writes_live_placeholder_pid(tmp_path): + """Regression #1443: claimed slots must not be empty during spawn startup.""" + cmd = ["mempalace", "mine", "/tmp/proj", "--mode", "projects"] + pid_dir = tmp_path / "mine_pids" + + with patch("mempalace.hooks_cli._MINE_PID_DIR", pid_dir): + pid_file = _claim_mine_slot(cmd) + + assert pid_file == _pid_file_for_cmd(cmd) + assert pid_file.read_text().strip() == str(os.getpid()) + assert _mine_already_running(cmd) is True + assert _claim_mine_slot(cmd) is None + + +def test_claim_mine_slot_reclaimed_slot_writes_live_placeholder_pid(tmp_path): + """Regression #1443: stale-slot reclaim must also write a live placeholder.""" + cmd = ["mempalace", "mine", "/tmp/proj", "--mode", "projects"] + pid_dir = tmp_path / "mine_pids" + + with ( + patch("mempalace.hooks_cli._MINE_PID_DIR", pid_dir), + patch("mempalace.hooks_cli._pid_alive", return_value=False), + ): + pid_file = _pid_file_for_cmd(cmd) + pid_file.parent.mkdir(parents=True, exist_ok=True) + pid_file.write_text("12345") + + reclaimed = _claim_mine_slot(cmd) + + assert reclaimed == pid_file + assert pid_file.read_text().strip() == str(os.getpid()) + + def test_maybe_auto_ingest_ignores_transcript_arg_path(tmp_path): """_maybe_auto_ingest does NOT mine the transcript directory. From b2012551888962ee3a95bb77624a9db938f2eb08 Mon Sep 17 00:00:00 2001 From: fatkobra <55045047+fatkobra@users.noreply.github.com> Date: Mon, 18 May 2026 10:24:00 +0000 Subject: [PATCH 13/39] Refactor mine slot placeholder creation --- mempalace/hooks_cli.py | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index ddcb679c0a..158a0fa0f0 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -344,6 +344,25 @@ def _mine_already_running(cmd: list[str]) -> bool: return _pid_alive(int(recorded)) +def _create_mine_slot_with_placeholder(pid_file: Path) -> Path: + """Atomically create a mine PID slot and write this hook PID into it.""" + fd = os.open(str(pid_file), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + try: + with os.fdopen(fd, "w", encoding="ascii") as f: + f.write(str(os.getpid())) + except OSError: + try: + os.close(fd) + except OSError: + pass + try: + pid_file.unlink() + except OSError: + pass + raise + return pid_file + + def _claim_mine_slot(cmd: list[str]) -> Optional[Path]: """Atomically reserve the per-target PID slot for ``cmd``. @@ -359,17 +378,14 @@ def _claim_mine_slot(cmd: list[str]) -> Optional[Path]: pid_file = _pid_file_for_cmd(cmd) pid_file.parent.mkdir(parents=True, exist_ok=True) try: - fd = os.open(str(pid_file), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) - try: - os.write(fd, str(os.getpid()).encode("ascii")) - finally: - os.close(fd) - return pid_file + return _create_mine_slot_with_placeholder(pid_file) except FileExistsError: pass + # Slot exists. If the holder is alive, defer. if _mine_already_running(cmd): return None + # Stale entry; reclaim. The unlink+create is racy against another hook # firing right now, but the second create's O_EXCL will fail and that # caller will see the live PID via the next round. @@ -379,13 +395,9 @@ def _claim_mine_slot(cmd: list[str]) -> Optional[Path]: pass except OSError: return None + try: - fd = os.open(str(pid_file), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) - try: - os.write(fd, str(os.getpid()).encode("ascii")) - finally: - os.close(fd) - return pid_file + return _create_mine_slot_with_placeholder(pid_file) except FileExistsError: return None From 3cac26fc6549d3a66947f090ad24ad6841d8f148 Mon Sep 17 00:00:00 2001 From: mvalentsev Date: Mon, 18 May 2026 01:35:48 +0500 Subject: [PATCH 14/39] fix(convo_miner): enforce CHUNK_SIZE in paragraph chunker (#1534) `_chunk_by_paragraph` and its 25-line-group fallback appended whole paragraphs and line groups as single drawers with no per-drawer size cap, so a normalized transcript containing a single >CHUNK_SIZE paragraph (for example a Claude Code session with pasted content as one 135,600-char line) produced one giant chunk. ChromaDB upsert then fed that chunk to the embedding model, blowing the O(seq_len^2) attention budget with `RuntimeError: Invalid buffer size: 120.00 GiB`. Introduce a single `_emit_bounded` helper that gates the whole stripped content against `min_chunk_size` (noise filter) and emits every slice verbatim via an index-based `range(0, len, chunk_size)` loop. The helper preserves trailing remainders instead of silently dropping them and avoids the O(N^2) substring-copy pattern. Both `_chunk_by_paragraph` and `_chunk_by_exchange` now delegate slicing to the helper. Plumb the config-resolved `chunk_size` through `_chunk_by_paragraph` so `MempalaceConfig.chunk_size` governs this path too. Co-Authored-By: David F Glidden <4116848+davidglidden@users.noreply.github.com> Co-Authored-By: Claude Opus 4.7 (1M context) --- mempalace/convo_miner.py | 60 +++++++------ tests/test_convo_miner_unit.py | 155 ++++++++++++++++++++++++++++++++- 2 files changed, 185 insertions(+), 30 deletions(-) diff --git a/mempalace/convo_miner.py b/mempalace/convo_miner.py index 02a5aca9ba..b499c60dee 100644 --- a/mempalace/convo_miner.py +++ b/mempalace/convo_miner.py @@ -60,6 +60,8 @@ def _detect_hall_cached(content: str) -> str: MIN_CHUNK_SIZE = 30 CHUNK_SIZE = 800 # chars per drawer — align with miner.py +_LINE_GROUP_SIZE = 25 # lines per fallback group when no paragraph breaks +_LINE_FALLBACK_MIN_NEWLINES = 20 # trigger line-group fallback above this newline count DRAWER_UPSERT_BATCH_SIZE = 1000 MAX_FILE_SIZE = 500 * 1024 * 1024 # 500 MB — skip files larger than this. # Matches miner.py at 500 MB. Long Claude Code sessions, multi-year @@ -163,7 +165,7 @@ def chunk_exchanges( if quote_lines >= 3: return _chunk_by_exchange(lines, chunk_size, min_chunk_size) else: - return _chunk_by_paragraph(content, min_chunk_size) + return _chunk_by_paragraph(content, chunk_size, min_chunk_size) def _chunk_by_exchange(lines: list, chunk_size: int, min_chunk_size: int) -> list: @@ -194,49 +196,49 @@ def _chunk_by_exchange(lines: list, chunk_size: int, min_chunk_size: int) -> lis ai_response = " ".join(ai_lines) content = f"{user_turn}\n{ai_response}" if ai_response else user_turn - # Split into multiple drawers when the exchange exceeds chunk_size - if len(content) > chunk_size: - # First chunk: user turn + as much response as fits - first_part = content[:chunk_size] - if len(first_part.strip()) > min_chunk_size: - chunks.append({"content": first_part, "chunk_index": len(chunks)}) - # Remaining response in chunk_size-sized continuation drawers - remainder = content[chunk_size:] - while remainder: - part = remainder[:chunk_size] - remainder = remainder[chunk_size:] - if len(part.strip()) > min_chunk_size: - chunks.append({"content": part, "chunk_index": len(chunks)}) - elif len(content.strip()) > min_chunk_size: - chunks.append( - { - "content": content, - "chunk_index": len(chunks), - } - ) + _emit_bounded(chunks, content, chunk_size, min_chunk_size) else: i += 1 return chunks -def _chunk_by_paragraph(content: str, min_chunk_size: int) -> list: +def _emit_bounded( + chunks: list, + content: str, + chunk_size: int, + min_chunk_size: int, +) -> None: + """Append ``content`` as one or more drawers, none exceeding ``chunk_size``. + + The ``min_chunk_size`` floor gates the WHOLE call (drops the input if + its stripped length is at or below the floor, treated as noise). Once + the input passes the floor, every slice is emitted verbatim so a + small trailing remainder is preserved instead of silently dropped. + The index-based loop avoids the O(N^2) repeated-substring allocation + of a ``while content: content = content[chunk_size:]`` shape. + """ + if len(content.strip()) <= min_chunk_size: + return + for i in range(0, len(content), chunk_size): + chunks.append({"content": content[i : i + chunk_size], "chunk_index": len(chunks)}) + + +def _chunk_by_paragraph(content: str, chunk_size: int, min_chunk_size: int) -> list: """Fallback: chunk by paragraph breaks.""" chunks = [] paragraphs = [p.strip() for p in content.split("\n\n") if p.strip()] # If no paragraph breaks and long content, chunk by line groups - if len(paragraphs) <= 1 and content.count("\n") > 20: + if len(paragraphs) <= 1 and content.count("\n") > _LINE_FALLBACK_MIN_NEWLINES: lines = content.split("\n") - for i in range(0, len(lines), 25): - group = "\n".join(lines[i : i + 25]).strip() - if len(group) > min_chunk_size: - chunks.append({"content": group, "chunk_index": len(chunks)}) + for i in range(0, len(lines), _LINE_GROUP_SIZE): + group = "\n".join(lines[i : i + _LINE_GROUP_SIZE]).strip() + _emit_bounded(chunks, group, chunk_size, min_chunk_size) return chunks for para in paragraphs: - if len(para) > min_chunk_size: - chunks.append({"content": para, "chunk_index": len(chunks)}) + _emit_bounded(chunks, para, chunk_size, min_chunk_size) return chunks diff --git a/tests/test_convo_miner_unit.py b/tests/test_convo_miner_unit.py index 7ff7a91bc3..b14f8ab8fc 100644 --- a/tests/test_convo_miner_unit.py +++ b/tests/test_convo_miner_unit.py @@ -6,6 +6,8 @@ import pytest from mempalace.convo_miner import ( + CHUNK_SIZE, + _emit_bounded, _file_chunks_locked, chunk_exchanges, detect_convo_room, @@ -38,11 +40,32 @@ def test_paragraph_fallback(self): assert len(chunks) >= 2 def test_paragraph_line_group_fallback(self): - """Long content with no paragraph breaks chunks by line groups.""" + """Long content with no paragraph breaks chunks by line groups. + + Each emitted drawer must respect CHUNK_SIZE. Before #1534 the + fallback chunker emitted one drawer per 25-line group without + a size cap, so a 25-line group of long lines produced an + oversized drawer that crashed embedding upsert. + """ lines = [f"Line {i}: some content that is meaningful" for i in range(60)] content = "\n".join(lines) chunks = chunk_exchanges(content) assert len(chunks) >= 1 + max_len = max(len(c["content"]) for c in chunks) + assert max_len <= CHUNK_SIZE, f"oversized chunk: max_len={max_len}" + + def test_line_group_fallback_drops_sub_min_trailing_group(self): + """A trailing line-group whose stripped length is at or below + MIN_CHUNK_SIZE must be dropped, not emitted as a tiny drawer.""" + lines = [f"Line {i}" for i in range(51)] + content = "\n".join(lines) + chunks = chunk_exchanges(content) + from mempalace.convo_miner import MIN_CHUNK_SIZE + + assert len(chunks) == 2, ( + f"expected 2 drawers (groups 0-24 and 25-49); got {len(chunks)}; " + f"the single-line tail group should drop below MIN_CHUNK_SIZE={MIN_CHUNK_SIZE}" + ) def test_empty_content(self): chunks = chunk_exchanges("") @@ -97,6 +120,136 @@ def test_long_ai_response_not_truncated(self): for i in range(1, 14): assert f"Step {i}:" in stored, f"Step {i} was truncated and not stored" + def test_paragraph_loop_enforces_chunk_size(self): + """A paragraph longer than CHUNK_SIZE must split into multiple + bounded drawers. Regression for #1534: the paragraph loop in + ``_chunk_by_paragraph`` used to append each paragraph as a + single drawer regardless of size, producing one giant chunk + that crashed embedding upsert with + ``RuntimeError: Invalid buffer size: ... GiB``. + """ + big_para = "x" * 5000 + tail = "small paragraph tail of meaningful length" + content = big_para + "\n\n" + tail + chunks = chunk_exchanges(content) + max_len = max(len(c["content"]) for c in chunks) + assert max_len <= CHUNK_SIZE, f"oversized chunk: max_len={max_len}" + assert len(chunks) > 1, "5000-char content should produce multiple drawers" + assert chunks[-1]["content"] == tail, ( + "trailing paragraph must be preserved as the last drawer" + ) + + def test_custom_chunk_size_propagates_to_paragraph_path(self): + """User-supplied chunk_size must govern the paragraph chunker, not + only the exchange chunker. Confirms config plumbing reaches both + paths after the #1534 fix. + """ + big_para = "y" * 3000 + content = big_para + "\n\ntail paragraph of meaningful length" + chunks = chunk_exchanges(content, chunk_size=400) + max_len = max(len(c["content"]) for c in chunks) + assert max_len <= 400, f"oversized chunk under custom chunk_size=400: {max_len}" + + def test_paragraph_loop_no_content_loss(self): + """Verbatim principle: every char of a single long paragraph lands + in some drawer in order. The slicing helper must not drop or + reorder content.""" + content = "a" * 5000 + chunks = chunk_exchanges(content) + joined = "".join(c["content"] for c in chunks) + assert joined == content + + def test_chunk_exactly_at_size_boundary(self): + """Content length == CHUNK_SIZE produces exactly one drawer of CHUNK_SIZE.""" + content = "z" * CHUNK_SIZE + chunks = chunk_exchanges(content) + assert len(chunks) == 1 + assert len(chunks[0]["content"]) == CHUNK_SIZE + + def test_chunk_many_multiples_of_size(self): + """Content length == 8 * CHUNK_SIZE produces exactly 8 drawers, each + of length CHUNK_SIZE.""" + content = "w" * (8 * CHUNK_SIZE) + chunks = chunk_exchanges(content) + assert len(chunks) == 8 + assert all(len(c["content"]) == CHUNK_SIZE for c in chunks) + + def test_paragraph_loop_preserves_slice_order(self): + """Slices must appear in source order. Guards against a future + regression where the helper reverses, shuffles, or duplicates + slices — the verbatim invariant in CLAUDE.md depends on order + as well as content.""" + content = "a" * CHUNK_SIZE + "b" * CHUNK_SIZE + "c" * CHUNK_SIZE + chunks = chunk_exchanges(content) + assert len(chunks) == 3 + assert chunks[0]["content"] == "a" * CHUNK_SIZE + assert chunks[1]["content"] == "b" * CHUNK_SIZE + assert chunks[2]["content"] == "c" * CHUNK_SIZE + + +class TestEmitBounded: + """Direct unit tests for the chunk-size-enforcement helper.""" + + def test_emits_no_oversized_chunks(self): + chunks = [] + _emit_bounded(chunks, "abc" * 20, chunk_size=10, min_chunk_size=0) + assert all(len(c["content"]) <= 10 for c in chunks) + + def test_assigns_sequential_chunk_indices(self): + chunks = [] + _emit_bounded(chunks, "x" * 25, chunk_size=10, min_chunk_size=0) + assert [c["chunk_index"] for c in chunks] == [0, 1, 2] + + def test_continues_existing_chunk_index(self): + chunks = [{"content": "pre-existing entry", "chunk_index": 0}] + _emit_bounded(chunks, "y" * 5, chunk_size=10, min_chunk_size=0) + assert len(chunks) == 2 + assert chunks[1]["chunk_index"] == 1 + + def test_empty_content_noop(self): + chunks = [] + _emit_bounded(chunks, "", chunk_size=10, min_chunk_size=0) + assert chunks == [] + + def test_small_trailing_slice_preserved(self): + """Once the whole content passes the floor, every slice is emitted + verbatim so small trailing remainders are not silently dropped. + Regression test for the data-loss class flagged on PR #1538.""" + chunks = [] + _emit_bounded(chunks, "z" * 23, chunk_size=10, min_chunk_size=5) + assert len(chunks) == 3 + assert [len(c["content"]) for c in chunks] == [10, 10, 3] + assert "".join(c["content"] for c in chunks) == "z" * 23 + + def test_trailing_whitespace_slice_preserved_when_whole_passes(self): + """When the whole content passes the floor, a trailing + whitespace-only slice is preserved verbatim rather than dropped. + The floor is a noise filter on the WHOLE input, not a per-slice gate.""" + chunks = [] + _emit_bounded(chunks, "a" * 10 + " " * 10, chunk_size=10, min_chunk_size=5) + assert len(chunks) == 2 + assert chunks[0]["content"] == "a" * 10 + assert chunks[1]["content"] == " " * 10 + + def test_whole_content_below_floor_dropped(self): + """The floor is applied to the stripped whole content. An all-whitespace + input (stripped length 0) or a too-short input is dropped without slicing.""" + chunks = [] + _emit_bounded(chunks, " " * 100, chunk_size=10, min_chunk_size=5) + _emit_bounded(chunks, "ab", chunk_size=10, min_chunk_size=5) + assert chunks == [] + + def test_split_805_chars_at_chunk_size_800_preserves_tail(self): + """805 chars at chunk_size=800 produces a 5-char tail. With the + whole-content floor (not per-slice), the 5-char tail is preserved + verbatim. Directly addresses the data-loss scenario raised on PR #1538.""" + chunks = [] + _emit_bounded(chunks, "y" * 805, chunk_size=800, min_chunk_size=30) + assert len(chunks) == 2 + assert chunks[0]["content"] == "y" * 800 + assert chunks[1]["content"] == "y" * 5 + assert "".join(c["content"] for c in chunks) == "y" * 805 + class TestDetectConvoRoom: def test_technical_room(self): From e811b6d10a28fa5bca31e34e35c8c8ce2d9a68f6 Mon Sep 17 00:00:00 2001 From: Ivan Dergachev Date: Mon, 11 May 2026 13:54:57 +0400 Subject: [PATCH 15/39] fix: avoid false HNSW metadata quarantine --- mempalace/backends/chroma.py | 70 +++++++++++++++++++++++++++++++----- tests/test_backends.py | 53 +++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 8 deletions(-) diff --git a/mempalace/backends/chroma.py b/mempalace/backends/chroma.py index 387a2b93a7..980f2e8393 100644 --- a/mempalace/backends/chroma.py +++ b/mempalace/backends/chroma.py @@ -671,20 +671,59 @@ def _valid_dimensionality(value: object) -> bool: return isinstance(value, Integral) and not isinstance(value, bool) and int(value) > 0 -def _persisted_metadata_fields(obj: object) -> tuple[object, object]: +def _persisted_metadata_value(obj: object, name: str) -> object: if isinstance(obj, dict): - return obj.get("dimensionality"), obj.get("id_to_label") - return getattr(obj, "dimensionality", None), getattr(obj, "id_to_label", None) + return obj.get(name) + return getattr(obj, name, None) + + +def _persisted_metadata_fields(obj: object) -> tuple[object, object]: + return _persisted_metadata_value(obj, "dimensionality"), _persisted_metadata_value( + obj, "id_to_label" + ) + + +def _missing_dimensionality_appears_recoverable( + persisted: object, id_to_label: dict, seg_dir: str +) -> bool: + total = _persisted_metadata_value(persisted, "total_elements_added") + label_to_id = _persisted_metadata_value(persisted, "label_to_id") + data_path = os.path.join(seg_dir, "data_level0.bin") + link_path = os.path.join(seg_dir, "link_lists.bin") + + if not isinstance(total, Integral) or isinstance(total, bool): + return False + if not isinstance(label_to_id, dict): + return False + try: + if not ( + os.path.isfile(data_path) + and os.path.isfile(link_path) + and os.path.getsize(data_path) > _HNSW_MISSING_METADATA_DATA_FLOOR + ): + return False + except OSError: + return False + if not _hnsw_payload_appears_sane(seg_dir): + return False + + label_count = len(id_to_label) + if int(total) != label_count or len(label_to_id) != label_count: + return False + try: + return all(label_to_id.get(label) == item_id for item_id, label in id_to_label.items()) + except TypeError: + return False def quarantine_invalid_hnsw_metadata(palace_path: str) -> list[str]: """Quarantine segment dirs whose ``index_metadata.pickle`` is unreadable or invalid. Chroma's persisted HNSW metadata is untrusted disk state. If a segment has - labels but no valid positive dimensionality, current Chroma versions can - accept the pickle and crash later in the Rust loader. We rename the entire - segment out of the way before ``PersistentClient`` opens so Chroma can - rebuild cleanly instead of touching known-bad metadata. + labels but invalid or partial metadata, current Chroma versions can accept + the pickle and crash later in the Rust loader. We rename the entire segment + out of the way before ``PersistentClient`` opens so Chroma can rebuild + cleanly instead of touching known-bad metadata. """ try: entries = os.listdir(palace_path) @@ -735,7 +774,22 @@ def quarantine_invalid_hnsw_metadata(palace_path: str) -> list[str]: reason = f"invalid id_to_label type {type(id_to_label).__name__}" else: has_labels = bool(id_to_label) - if has_labels and not _valid_dimensionality(dimensionality): + if ( + has_labels + and dimensionality is None + and not _missing_dimensionality_appears_recoverable( + persisted, id_to_label, seg_dir + ) + ): + reason = ( + "labels present but dimensionality is missing or invalid " + f"({dimensionality!r})" + ) + elif ( + has_labels + and dimensionality is not None + and not _valid_dimensionality(dimensionality) + ): reason = ( "labels present but dimensionality is missing or invalid " f"({dimensionality!r})" diff --git a/tests/test_backends.py b/tests/test_backends.py index 1fc8eab803..0ee10d3c08 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -1047,6 +1047,59 @@ def test_quarantine_invalid_hnsw_metadata_renames_missing_dimensionality(tmp_pat assert not seg.exists() +def test_quarantine_invalid_hnsw_metadata_keeps_consistent_missing_dimensionality(tmp_path): + palace = tmp_path / "palace" + palace.mkdir() + seg = palace / "abcd-1234-5678" + seg.mkdir() + (seg / "data_level0.bin").write_bytes(b"x" * 2048) + (seg / "link_lists.bin").write_bytes(b"x" * 128) + with open(seg / "index_metadata.pickle", "wb") as f: + pickle.dump( + { + "dimensionality": None, + "total_elements_added": 2, + "max_seq_id": None, + "id_to_label": {"a": 1, "b": 2}, + "label_to_id": {1: "a", 2: "b"}, + "id_to_seq_id": {}, + }, + f, + ) + + moved = quarantine_invalid_hnsw_metadata(str(palace)) + + assert moved == [] + assert seg.exists() + + +def test_quarantine_invalid_hnsw_metadata_renames_mismatched_missing_dimensionality(tmp_path): + palace = tmp_path / "palace" + palace.mkdir() + seg = palace / "abcd-1234-5678" + seg.mkdir() + (seg / "data_level0.bin").write_bytes(b"x" * 2048) + (seg / "link_lists.bin").write_bytes(b"x" * 128) + with open(seg / "index_metadata.pickle", "wb") as f: + pickle.dump( + { + "dimensionality": None, + "total_elements_added": 2, + "max_seq_id": None, + "id_to_label": {"a": 1, "b": 2}, + "label_to_id": {1: "b", 2: "a"}, + "id_to_seq_id": {}, + }, + f, + ) + + moved = quarantine_invalid_hnsw_metadata(str(palace)) + + assert len(moved) == 1 + assert ".corrupt-" in moved[0] + assert not seg.exists() + + def test_quarantine_invalid_hnsw_metadata_allows_uninitialized_segment(tmp_path): palace = tmp_path / "palace" palace.mkdir() From 3da27d3c7e82cfc66f8da7098850115025928db7 Mon Sep 17 00:00:00 2001 From: William Blair Date: Fri, 15 May 2026 10:56:14 -0400 Subject: [PATCH 16/39] fix: VACUUM + FTS5 rebuild after repair to reclaim SQLite space Repeated `repair --yes` runs leave freed SQLite pages unreclaimed and can corrupt the FTS5 inverted index. This patch adds `_vacuum_and_rebuild_fts5()`, called at the end of `rebuild_index()`, which: 1. Closes all chroma handles (releases PersistentClient's sqlite lock). 2. Rebuilds the FTS5 index via the built-in content-table DDL trick. 3. Runs VACUUM (requires autocommit / isolation_level=None) to reclaim the freed pages. The function is a no-op when chroma.sqlite3 is absent or the FTS5 table doesn't exist, so it is safe on every supported backend. Four new tests cover the happy path, missing FTS5 table, missing file, and call ordering (close must precede vacuum). Closes #1516 Closes #1517 --- mempalace/repair.py | 38 +++++++++++++++++++ tests/test_repair.py | 89 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+) diff --git a/mempalace/repair.py b/mempalace/repair.py index 778a3548ea..7a4a28cd19 100644 --- a/mempalace/repair.py +++ b/mempalace/repair.py @@ -35,6 +35,7 @@ import sqlite3 import time from collections import defaultdict +from contextlib import closing from datetime import datetime import re from typing import Callable, Iterator, Optional @@ -680,6 +681,40 @@ def _eta_suffix(self, phase: str, completed: int, expected: int) -> str: return f" (elapsed {_format_eta(elapsed)}, rate {rate:.1f}/s, ETA {_format_eta(eta)})" +def _vacuum_and_rebuild_fts5(palace_path: str, progress=print) -> None: + """VACUUM the palace SQLite file and rebuild the FTS5 index if present. + + Repeated ``repair --yes`` runs delete and recreate the drawers collection, + leaving freed SQLite pages unreclaimable without an explicit VACUUM. The + FTS5 virtual table (``embedding_fulltext_search``) can also become + internally inconsistent after multiple collection deletes; the rebuild + command fixes it atomically without touching any row data. + + Failures are non-fatal: a warning is printed and the caller continues. + The repair itself succeeded at this point — VACUUM/FTS5 are best-effort + cleanup, not correctness requirements. + """ + sqlite_path = os.path.join(palace_path, "chroma.sqlite3") + if not os.path.exists(sqlite_path): + return + try: + with closing(sqlite3.connect(sqlite_path, isolation_level=None)) as conn: + tables = { + r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'") + } + if "embedding_fulltext_search" in tables: + conn.execute( + "INSERT INTO embedding_fulltext_search" + "(embedding_fulltext_search) VALUES('rebuild')" + ) + conn.commit() + progress(" FTS5 index rebuilt.") + conn.execute("VACUUM") + progress(" SQLite VACUUM complete.") + except Exception as exc: + progress(f" Warning: post-repair cleanup failed (non-fatal): {exc}") + + def rebuild_index( palace_path=None, confirm_truncation_ok: bool = False, @@ -813,6 +848,9 @@ def rebuild_index( print(" Live collection was not replaced; leaving the original palace untouched.") raise + _close_chroma_handles(palace_path, backend=backend) + _vacuum_and_rebuild_fts5(palace_path, progress=progress) + print(f"\n Repair complete. {filed} drawers rebuilt.") print(" HNSW index is now clean with cosine distance metric.") print(f"\n{'=' * 55}\n") diff --git a/tests/test_repair.py b/tests/test_repair.py index acbf31d10f..981351ef8b 100644 --- a/tests/test_repair.py +++ b/tests/test_repair.py @@ -1855,3 +1855,92 @@ def test_rebuild_from_sqlite_honors_configured_drawer_collection_name(tmp_path, ) except Exception: pass # Expected: collection wasn't created. + + +# ── _vacuum_and_rebuild_fts5 ────────────────────────────────────────── + + +def test_vacuum_and_rebuild_fts5_vacuums_and_rebuilds(tmp_path): + """VACUUM runs and FTS5 index is rebuilt when the table is present.""" + sqlite_path = tmp_path / "chroma.sqlite3" + with closing(sqlite3.connect(str(sqlite_path))) as conn: + conn.execute( + "CREATE VIRTUAL TABLE embedding_fulltext_search" + " USING fts5(string_value, tokenize='unicode61')" + ) + conn.execute("INSERT INTO embedding_fulltext_search(string_value) VALUES('hello world')") + conn.commit() + + repair._vacuum_and_rebuild_fts5(str(tmp_path)) + + with closing(sqlite3.connect(str(sqlite_path))) as conn: + result = conn.execute("PRAGMA integrity_check").fetchall() + assert result == [("ok",)] + + +def test_vacuum_and_rebuild_fts5_no_fts5_table(tmp_path): + """VACUUM runs without error when embedding_fulltext_search is absent.""" + sqlite_path = tmp_path / "chroma.sqlite3" + with closing(sqlite3.connect(str(sqlite_path))) as conn: + conn.execute("CREATE TABLE dummy (id INTEGER PRIMARY KEY)") + conn.commit() + + # Must not raise even without the FTS5 table. + repair._vacuum_and_rebuild_fts5(str(tmp_path)) + + with closing(sqlite3.connect(str(sqlite_path))) as conn: + result = conn.execute("PRAGMA integrity_check").fetchall() + assert result == [("ok",)] + + +def test_vacuum_and_rebuild_fts5_missing_sqlite(tmp_path): + """Silently skips when chroma.sqlite3 does not exist.""" + repair._vacuum_and_rebuild_fts5(str(tmp_path)) # no file — must not raise + + +@patch("mempalace.repair.shutil") +@patch("mempalace.repair.ChromaBackend") +def test_rebuild_index_calls_vacuum(mock_backend_cls, mock_shutil, tmp_path): + """rebuild_index closes chroma handles then calls _vacuum_and_rebuild_fts5. + + ChromaDB's PersistentClient holds an open connection to chroma.sqlite3; + VACUUM requires an exclusive lock so _close_chroma_handles must be called + before _vacuum_and_rebuild_fts5. + """ + sqlite_path = tmp_path / "chroma.sqlite3" + with closing(sqlite3.connect(str(sqlite_path))) as conn: + conn.execute("CREATE TABLE dummy(id INTEGER PRIMARY KEY)") + conn.commit() + + mock_col = MagicMock() + mock_col.count.return_value = 1 + mock_col.get.return_value = { + "ids": ["id1"], + "documents": ["doc1"], + "metadatas": [{"wing": "a"}], + } + mock_new_col = MagicMock() + mock_new_col.count.return_value = 1 + mock_temp_col = MagicMock() + mock_temp_col.count.return_value = 1 + mock_backend = _install_mock_backend(mock_backend_cls, mock_col) + mock_backend.create_collection.side_effect = [mock_temp_col, mock_new_col] + + call_order = [] + with ( + patch.object( + repair, "_close_chroma_handles", side_effect=lambda *a, **kw: call_order.append("close") + ) as mock_close, + patch.object( + repair, + "_vacuum_and_rebuild_fts5", + side_effect=lambda *a, **kw: call_order.append("vacuum"), + ) as mock_vacuum, + ): + repair.rebuild_index(palace_path=str(tmp_path)) + mock_close.assert_called_once() + mock_vacuum.assert_called_once() + assert call_order == ["close", "vacuum"], "backend must be closed before VACUUM" + args, kwargs = mock_vacuum.call_args + assert args[0] == str(tmp_path) + assert "progress" in kwargs From 3e92719710b38db28a15551308dd9836832e4d10 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Mon, 18 May 2026 17:25:09 -0300 Subject: [PATCH 17/39] fix(mcp): surface create_tunnel ValueError instead of masking it (#1473) tool_create_tunnel sanitized names inside a try/except ValueError but called create_tunnel() *outside* it, so any ValueError create_tunnel raises (empty/non-string endpoints today, room-existence checks once #1469 lands) escaped and the MCP framework wrapped it as the opaque 'Internal tool error'. Wrap the create_tunnel() call in the same try/except and return {"error": str(e)}, mirroring sibling tools (tool_add_drawer, tool_list_tunnels). Adds a regression test that monkeypatches create_tunnel to raise and asserts the message is surfaced verbatim. Closes #1473 --- mempalace/mcp_server.py | 25 ++++++++++++++++--------- tests/test_mcp_server.py | 22 ++++++++++++++++++++++ 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 67d4d7b26c..fb93fc1f1f 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -1033,15 +1033,22 @@ def tool_create_tunnel( target_room = sanitize_name(target_room, "target_room") except ValueError as e: return {"error": str(e)} - return create_tunnel( - source_wing, - source_room, - target_wing, - target_room, - label=label, - source_drawer_id=source_drawer_id, - target_drawer_id=target_drawer_id, - ) + # create_tunnel raises ValueError for invalid/missing endpoints + # (empty/non-string names, and room-existence checks). Surface the + # real reason instead of letting it escape and be wrapped as the + # opaque "Internal tool error" (#1473), mirroring sibling tools. + try: + return create_tunnel( + source_wing, + source_room, + target_wing, + target_room, + label=label, + source_drawer_id=source_drawer_id, + target_drawer_id=target_drawer_id, + ) + except ValueError as e: + return {"error": str(e)} def tool_list_tunnels(wing: str = None): diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 4a1aae2f8e..ecd60afea4 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -1386,6 +1386,28 @@ def test_tool_create_tunnel_preserves_hyphenated_wings(self, monkeypatch, tmp_pa assert len(mcp_server.tool_list_tunnels(wing="my-wing")) == 1 assert len(mcp_server.tool_list_tunnels(wing="my_wing")) == 1 + def test_tool_create_tunnel_surfaces_value_error(self, monkeypatch): + """Regression for #1473: a ValueError from create_tunnel (e.g. a + missing room) must be returned to the caller as a clear error, + not escape and get wrapped as the opaque 'Internal tool error'.""" + from mempalace import mcp_server + + msg = "Target room 'does-not-exist-probe' does not exist in wing 'wing_minerva'" + + def _raise(*args, **kwargs): + raise ValueError(msg) + + monkeypatch.setattr(mcp_server, "create_tunnel", _raise) + + result = mcp_server.tool_create_tunnel( + source_wing="wing_minerva", + source_room="fx-invariants", + target_wing="wing_minerva", + target_room="does-not-exist-probe", + ) + + assert result == {"error": msg} + # ── KG Tools ──────────────────────────────────────────────────────────── From 426d164df46be993b84bde56162ee3a3fe2b1f35 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Mon, 18 May 2026 17:28:50 -0300 Subject: [PATCH 18/39] test(hooks): isolate test_hooks_cli from test_cli side effect (#1510) Nine save/log/precompact tests in test_hooks_cli.py passed only because test_cli.py (alphabetically earlier) created ~/.mempalace in the session tmp HOME as a side effect, satisfying the _palace_root_exists() kill-switch. Run in isolation they short-circuited and failed (9 failed, 80 passed, 1 skipped). Add a module autouse fixture that points PALACE_ROOT/STATE_DIR at a per-test palace root that exists, so every test is robust standalone and future tests don't inherit the trap. Kill-switch tests that need the absent path call _redirect_palace_root after the fixture; monkeypatch last-write-wins keeps their absent/file root and teardown restores the real module value. Isolation: pytest tests/test_hooks_cli.py -> 100 passed, 1 skipped. Ordering preserved: test_cli + test_hooks_cli -> 165 passed. Closes #1510 --- tests/test_hooks_cli.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_hooks_cli.py b/tests/test_hooks_cli.py index 234bef2bce..e4309fc28b 100644 --- a/tests/test_hooks_cli.py +++ b/tests/test_hooks_cli.py @@ -31,6 +31,31 @@ ) +@pytest.fixture(autouse=True) +def _isolated_existing_palace_root(monkeypatch, tmp_path): + """Give every test an isolated, *existing* PALACE_ROOT/STATE_DIR. + + Regression for #1510: nine save / log / precompact tests assumed + ``~/.mempalace`` existed and only passed in the full suite because an + earlier test file (``test_cli.py``) created it as a side effect, so + the ``_palace_root_exists()`` kill-switch was satisfied. Run in + isolation they short-circuited and failed. + + Defaulting every test to a per-test palace root that exists makes + them robust on their own and protects future tests from the same + trap. Tests that exercise the absent-root kill-switch path call + ``_redirect_palace_root`` (or set their own PALACE_ROOT) *after* this + fixture; ``monkeypatch``'s last-write-wins means they keep their + absent/file root and teardown still restores the real module value. + """ + root = tmp_path / ".mempalace" + root.mkdir(exist_ok=True) + monkeypatch.setattr(hooks_cli_mod, "PALACE_ROOT", root) + monkeypatch.setattr(hooks_cli_mod, "STATE_DIR", root / "hook_state") + monkeypatch.setattr(hooks_cli_mod, "_state_dir_initialized", False) + return root + + # --- _mempalace_python --- From 02c05aaba6bd765aa0b255af274168973bbce02a Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Mon, 18 May 2026 17:39:10 -0300 Subject: [PATCH 19/39] refactor(mcp): merge the two ValueError guards in tool_create_tunnel Review feedback: the sanitize_name and create_tunnel calls both raise ValueError and returned the identical {"error": str(e)} shape, so the second try/except was redundant nesting. Fold create_tunnel into the single guard; behavior and the #1473 regression test are unchanged. --- mempalace/mcp_server.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index fb93fc1f1f..41cf9c0c21 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -1026,18 +1026,16 @@ def tool_create_tunnel( Example: an API design discussion in project_api connects to the database schema in project_database. """ + # sanitize_name and create_tunnel both raise ValueError for invalid or + # missing endpoints (empty/non-string names, and create_tunnel's + # room-existence checks). Catch both so the real reason is surfaced + # instead of escaping and being wrapped as the opaque "Internal tool + # error" (#1473), mirroring sibling tools. try: source_wing = sanitize_name(source_wing, "source_wing") source_room = sanitize_name(source_room, "source_room") target_wing = sanitize_name(target_wing, "target_wing") target_room = sanitize_name(target_room, "target_room") - except ValueError as e: - return {"error": str(e)} - # create_tunnel raises ValueError for invalid/missing endpoints - # (empty/non-string names, and room-existence checks). Surface the - # real reason instead of letting it escape and be wrapped as the - # opaque "Internal tool error" (#1473), mirroring sibling tools. - try: return create_tunnel( source_wing, source_room, From 90fece7628941e27474241b8cfa0a65280d02f67 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Mon, 18 May 2026 17:39:12 -0300 Subject: [PATCH 20/39] test(hooks): also isolate _MINE_PID_DIR + create the state dir (#1510) Review feedback: _MINE_PID_DIR is derived from STATE_DIR at module import (hooks_cli.py:277), so patching STATE_DIR alone left mine-spawning tests writing PID files under the import-time location instead of the per-test root. Patch _MINE_PID_DIR too, and create the state dir so the fixture's 'existing' docstring is accurate. Isolation still 100 passed/1 skipped; ordering still 165 passed. --- tests/test_hooks_cli.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/test_hooks_cli.py b/tests/test_hooks_cli.py index e4309fc28b..8062c6a75a 100644 --- a/tests/test_hooks_cli.py +++ b/tests/test_hooks_cli.py @@ -43,15 +43,23 @@ def _isolated_existing_palace_root(monkeypatch, tmp_path): Defaulting every test to a per-test palace root that exists makes them robust on their own and protects future tests from the same - trap. Tests that exercise the absent-root kill-switch path call + trap. ``_MINE_PID_DIR`` is patched too: it is derived from + ``STATE_DIR`` *at module import* (hooks_cli.py:277), so patching + ``STATE_DIR`` alone would leave mine-spawning tests writing PID files + under the import-time location instead of the per-test root. The + state dir is created so the docstring's "existing" promise holds. + + Tests that exercise the absent-root kill-switch path call ``_redirect_palace_root`` (or set their own PALACE_ROOT) *after* this fixture; ``monkeypatch``'s last-write-wins means they keep their absent/file root and teardown still restores the real module value. """ root = tmp_path / ".mempalace" - root.mkdir(exist_ok=True) + state_dir = root / "hook_state" + state_dir.mkdir(parents=True, exist_ok=True) monkeypatch.setattr(hooks_cli_mod, "PALACE_ROOT", root) - monkeypatch.setattr(hooks_cli_mod, "STATE_DIR", root / "hook_state") + monkeypatch.setattr(hooks_cli_mod, "STATE_DIR", state_dir) + monkeypatch.setattr(hooks_cli_mod, "_MINE_PID_DIR", state_dir / "mine_pids") monkeypatch.setattr(hooks_cli_mod, "_state_dir_initialized", False) return root From 5decc0c83b5468b2507ab19a1e7b13da84c002f9 Mon Sep 17 00:00:00 2001 From: mvalentsev Date: Wed, 20 May 2026 02:20:29 +0500 Subject: [PATCH 21/39] fix(miner): configurable + raised MAX_CHUNKS_PER_FILE (#1455) Default raised from 500 to 50_000 so legitimate long-form content (novels, scholarly editions) is not silently dropped on a typical literary corpus. Cap configurable via MEMPALACE_MAX_CHUNKS_PER_FILE env var or --max-chunks-per-file CLI flag; sentinel 0 disables. Negative or non-int values from either source emit a stderr warning and fall back to the default. Separate counter (files_skipped_chunk_cap) and summary line surface chunk-cap drops independently of the residual already-filed bucket; counter fires under --dry-run too so a corpus audit shows the same signal without writing. Skip notice routed to stderr alongside the existing symlink-skip line. ONNX bad_alloc protection preserved architecturally: DRAWER_UPSERT_BATCH_SIZE=1000 bounds per-ONNX-call exposure regardless of per-file cap. A 50_000-chunk file produces 50 forward passes of 1_000 chunks each, identical to what 500-chunk files already produced. process_file return extended to a 3-tuple (drawers, room, skip_reason); skip_reason is None on non-chunk-cap paths and "chunk_cap" when the per-file cap triggered. Internal callers (_mine_impl, benchmarks/mine_bench.py, tests) updated. Closes #1455 Co-Authored-By: David Glidden --- benchmarks/mine_bench.py | 19 ++- mempalace/cli.py | 15 ++ mempalace/miner.py | 149 +++++++++++++++++--- tests/test_cli.py | 1 + tests/test_miner.py | 290 +++++++++++++++++++++++++++++++++++++-- 5 files changed, 433 insertions(+), 41 deletions(-) diff --git a/benchmarks/mine_bench.py b/benchmarks/mine_bench.py index 43b08ed953..2ff8630ab5 100644 --- a/benchmarks/mine_bench.py +++ b/benchmarks/mine_bench.py @@ -58,11 +58,7 @@ def build_corpus(dest: Path, n_files: int, paragraphs_per_file: int, seed: int) paragraphs.append(" ".join(words)) (dest / f"doc_{i:03d}.md").write_text("\n\n".join(paragraphs)) (dest / "mempalace.yaml").write_text( - "wing: bench\n" - "rooms:\n" - " - name: general\n" - " description: all\n" - " keywords: [general]\n" + "wing: bench\nrooms:\n - name: general\n description: all\n keywords: [general]\n" ) @@ -121,8 +117,7 @@ def _process_file_unbatched(filepath, project_path, collection, wing, rooms, age ] closet_lines = build_closet_lines(source_file, drawer_ids, content, wing, room) closet_id_base = ( - f"closet_{wing}_{room}_" - f"{hashlib.sha256(source_file.encode()).hexdigest()[:24]}" + f"closet_{wing}_{room}_{hashlib.sha256(source_file.encode()).hexdigest()[:24]}" ) closet_meta = { "wing": wing, @@ -155,7 +150,7 @@ def mine_once(project_dir: str, palace_path: str, batched: bool) -> tuple[int, f t0 = time.perf_counter() for filepath in files: if batched: - drawers, _ = miner.process_file( + drawers, _, _ = miner.process_file( filepath=filepath, project_path=project_path, collection=collection, @@ -217,9 +212,9 @@ def run_scenario(label: str, n_files: int, paragraphs_per_file: int, seed: int) SCENARIOS = { - "small": ("Small files (~50 paragraphs)", 10, 50), + "small": ("Small files (~50 paragraphs)", 10, 50), "medium": ("Medium files (~200 paragraphs)", 20, 200), - "large": ("Large files (~500 paragraphs)", 10, 500), + "large": ("Large files (~500 paragraphs)", 10, 500), } @@ -237,7 +232,9 @@ def _env_summary(device_label: str) -> list[str]: import onnxruntime as ort ort_v = ort.__version__ - providers = ",".join(p.replace("ExecutionProvider", "") for p in ort.get_available_providers()) + providers = ",".join( + p.replace("ExecutionProvider", "") for p in ort.get_available_providers() + ) except Exception: ort_v = "?" providers = "?" diff --git a/mempalace/cli.py b/mempalace/cli.py index 527c96d98e..f3c894cc40 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -527,6 +527,7 @@ def cmd_mine(args): dry_run=args.dry_run, respect_gitignore=not args.no_gitignore, include_ignored=include_ignored, + max_chunks_per_file=getattr(args, "max_chunks_per_file", None), ) except MineAlreadyRunning as exc: # A live MCP server or another mine is already writing to this @@ -1302,6 +1303,20 @@ def main(): default="exchange", help="Extraction strategy for convos mode: 'exchange' (default) or 'general' (5 memory types)", ) + from . import miner as _miner_for_default + + p_mine.add_argument( + "--max-chunks-per-file", + type=int, + default=None, + metavar="N", + help=( + f"Per-file chunk cap; files producing more chunks are skipped with a " + f"summary counter. Default {_miner_for_default.MAX_CHUNKS_PER_FILE} " + f"(or MEMPALACE_MAX_CHUNKS_PER_FILE). Set 0 to disable. Lower this on " + f"Windows if you hit ONNX bad_alloc (#1455)." + ), + ) # sweep p_sweep = sub.add_parser( diff --git a/mempalace/miner.py b/mempalace/miner.py index 0eab9dc72e..6987dcec16 100644 --- a/mempalace/miner.py +++ b/mempalace/miner.py @@ -83,13 +83,20 @@ DRAWER_UPSERT_BATCH_SIZE = 1000 MAX_FILE_SIZE = 500 * 1024 * 1024 # 500 MB — skip files larger than this. -# A single file producing more chunks than this is almost always a generated -# artifact (CSV/JSON dump, lockfile not in SKIP_FILENAMES, etc.). Embedding -# thousands of chunks from one file in one batch has triggered ONNX runtime -# `bad allocation` errors on Windows (#1296). The cap is conservative: a -# 500-chunk file at CHUNK_SIZE=800 is ~400 KB of source, which covers most -# legitimate hand-written content while bounding the worst-case batch. -MAX_CHUNKS_PER_FILE = 500 +# A safety rail against pathological generated artifacts (lockfiles not in +# SKIP_FILENAMES, vendored data dumps, etc.). Originally 500 to bound ONNX +# runtime `bad allocation` errors on Windows (#1296), but at CHUNK_SIZE=800 +# that capped legitimate long-form content (#1455: full-text scholarly +# editions, novels) at ~400 KB. The new default leaves two orders of +# magnitude of safety margin against the original lockfile case +# (~1124 chunks for `pnpm-lock.yaml` per #1296) while not touching +# hand-written prose. Per-ONNX-call exposure is bounded by +# `DRAWER_UPSERT_BATCH_SIZE` (1000 chunks/batch) regardless of this cap, +# so the cap is a per-file admission rail, not a per-batch limit. Lower +# this via `MEMPALACE_MAX_CHUNKS_PER_FILE` or +# `mempalace mine --max-chunks-per-file N` if you hit ONNX bad_alloc on +# Windows; set to 0 to disable the cap entirely. +MAX_CHUNKS_PER_FILE = 50_000 # Long Claude Code sessions and large transcript exports routinely exceed # 10 MB. The cap exists as a defensive rail against pathological binary # files, not as a limit on legitimate text. Per-drawer size is bounded @@ -99,6 +106,48 @@ # memory before chunking), so memory use scales with source size too. +def _resolve_max_chunks_per_file(override: int | None = None) -> int: + """Resolve the effective per-file chunk cap. + + Precedence: ``override`` (CLI flag) > ``MEMPALACE_MAX_CHUNKS_PER_FILE`` + env var > module-level ``MAX_CHUNKS_PER_FILE`` default. A sentinel + value of ``0`` (from any source) disables the cap entirely. Negative + values from either source emit a stderr warning and fall back to the + module default so a misconfigured ``--max-chunks-per-file=-500`` typo + (meaning "no, don't lower it that much") does not silently disable + the cap and OOM on a generated artifact. + """ + if override is not None: + if override < 0: + print( + f" ! WARNING: --max-chunks-per-file={override} is negative; " + f"using default {MAX_CHUNKS_PER_FILE}", + file=sys.stderr, + ) + return MAX_CHUNKS_PER_FILE + return int(override) + raw = os.environ.get("MEMPALACE_MAX_CHUNKS_PER_FILE") + if raw is None: + return MAX_CHUNKS_PER_FILE + try: + val = int(raw) + except ValueError: + print( + f" ! WARNING: MEMPALACE_MAX_CHUNKS_PER_FILE={raw!r} is not an integer; " + f"using default {MAX_CHUNKS_PER_FILE}", + file=sys.stderr, + ) + return MAX_CHUNKS_PER_FILE + if val < 0: + print( + f" ! WARNING: MEMPALACE_MAX_CHUNKS_PER_FILE={val} is negative; " + f"using default {MAX_CHUNKS_PER_FILE}", + file=sys.stderr, + ) + return MAX_CHUNKS_PER_FILE + return val + + # ============================================================================= # IGNORE MATCHING # ============================================================================= @@ -883,23 +932,32 @@ def process_file( chunk_size: int = None, chunk_overlap: int = None, min_chunk_size: int = None, + max_chunks_per_file: int | None = None, ) -> tuple: - """Read, chunk, route, and file one file. Returns (drawer_count, room_name).""" + """Read, chunk, route, and file one file. + + Returns ``(drawer_count, room_name, skip_reason)``. ``skip_reason`` is + ``None`` on success and on every non-chunk-cap skip path: already + filed (pre- or post-lock re-check), unreadable (``OSError``), or + too-short content (below ``min_chunk_size``). It is ``"chunk_cap"`` + when the per-file chunk cap aborted the file. Callers use the tag to + surface a separate counter in the mine summary (see #1455). + """ effective_min = min_chunk_size if min_chunk_size is not None else MIN_CHUNK_SIZE # Skip if already filed source_file = str(filepath) if not dry_run and file_already_mined(collection, source_file, check_mtime=True): - return 0, "general" + return 0, "general", None try: content = filepath.read_text(encoding="utf-8", errors="replace") except OSError: - return 0, "general" + return 0, "general", None content = content.strip() if len(content) < effective_min: - return 0, "general" + return 0, "general", None room = detect_room(filepath, content, rooms, project_path) chunks = chunk_text( @@ -910,16 +968,24 @@ def process_file( min_chunk_size=min_chunk_size, ) - if len(chunks) > MAX_CHUNKS_PER_FILE: + effective_cap = _resolve_max_chunks_per_file(max_chunks_per_file) + if effective_cap > 0 and len(chunks) > effective_cap: + # Skip notice goes to stderr alongside the existing symlink-skip + # warning style (see ``scan_project``'s ``SKIP: (symlink)`` + # line). This keeps ``mempalace mine ... > out.log 2> err.log`` + # piping coherent: degraded outcomes on stderr, progress on stdout. print( f" ! [skip] {filepath.name[:50]:50} produced {len(chunks)} chunks " - f"(> {MAX_CHUNKS_PER_FILE}); add to SKIP_FILENAMES or .gitignore" + f"(> {effective_cap}); raise via --max-chunks-per-file or " + f"MEMPALACE_MAX_CHUNKS_PER_FILE (set 0 to disable), or add to " + f"SKIP_FILENAMES if this is a generated artifact", + file=sys.stderr, ) - return 0, room + return 0, room, "chunk_cap" if dry_run: print(f" [DRY RUN] {filepath.name} -> room:{room} ({len(chunks)} drawers)") - return len(chunks), room + return len(chunks), room, None # Lock this file so concurrent agents don't interleave delete+insert. # Without the lock, two agents can both pass file_already_mined(), @@ -927,7 +993,7 @@ def process_file( with mine_lock(source_file): # Re-check after acquiring lock — another agent may have just finished if file_already_mined(collection, source_file, check_mtime=True): - return 0, room + return 0, room, None # Purge stale drawers for this file before re-inserting the fresh chunks. # Converts modified-file re-mines from upsert-over-existing-IDs (which hits @@ -1001,7 +1067,7 @@ def process_file( purge_file_closets(closets_col, source_file) upsert_closet_lines(closets_col, closet_id_base, closet_lines, closet_meta) - return drawers_added, room + return drawers_added, room, None # ============================================================================= @@ -1098,6 +1164,7 @@ def mine( respect_gitignore: bool = True, include_ignored: list = None, files: list = None, + max_chunks_per_file: int | None = None, ): """Mine a project directory into the palace. @@ -1106,6 +1173,11 @@ def mine( caller (e.g. ``init`` showing a file-count estimate before the mine prompt) avoids walking the tree twice. When ``None`` (the default), ``mine`` walks the tree itself just like before. + + ``max_chunks_per_file`` overrides the per-file chunk cap (see + :func:`_resolve_max_chunks_per_file`). ``None`` defers to + ``MEMPALACE_MAX_CHUNKS_PER_FILE`` or ``MAX_CHUNKS_PER_FILE``; ``0`` + disables the cap entirely (#1455). """ if dry_run: return _mine_impl( @@ -1118,6 +1190,7 @@ def mine( respect_gitignore=respect_gitignore, include_ignored=include_ignored, files=files, + max_chunks_per_file=max_chunks_per_file, ) # MineAlreadyRunning propagates so the CLI can render a clear holder-aware @@ -1134,6 +1207,7 @@ def mine( respect_gitignore=respect_gitignore, include_ignored=include_ignored, files=files, + max_chunks_per_file=max_chunks_per_file, ) @@ -1147,6 +1221,7 @@ def _mine_impl( respect_gitignore: bool = True, include_ignored: list = None, files: list = None, + max_chunks_per_file: int | None = None, ): from .config import MempalaceConfig @@ -1197,14 +1272,16 @@ def _mine_impl( total_drawers = 0 files_skipped = 0 + files_skipped_chunk_cap = 0 files_processed = 0 last_file = None room_counts = defaultdict(int) + effective_chunk_cap = _resolve_max_chunks_per_file(max_chunks_per_file) try: for i, filepath in enumerate(files, 1): try: - drawers, room = process_file( + drawers, room, skip_reason = process_file( filepath=filepath, project_path=project_path, collection=collection, @@ -1216,6 +1293,10 @@ def _mine_impl( chunk_size=cfg_chunk_size, chunk_overlap=cfg_chunk_overlap, min_chunk_size=cfg_min_chunk_size, + # Pass the already-resolved int so ``process_file``'s + # in-resolver fast-path short-circuits; otherwise a + # malformed env var would emit its warning per file. + max_chunks_per_file=effective_chunk_cap, ) except KeyboardInterrupt: # Re-raise so the outer handler prints the summary; we @@ -1224,7 +1305,18 @@ def _mine_impl( raise files_processed = i last_file = filepath.name - if drawers == 0 and not dry_run: + # Chunk-cap accounting fires under dry_run too: dry-run is the + # natural way to audit a corpus before a real mine, so silent + # chunk-cap drops there would defeat the whole point of #1455. + # Bump ``files_skipped`` alongside for the chunk-cap path so the + # "Files processed" arithmetic in the summary reflects the + # chunk-cap drops in both modes. (Pre-existing OSError / + # too-short / re-check skips under dry_run still flow into the + # else-branch; that is a separate accounting concern.) + if drawers == 0 and skip_reason == "chunk_cap": + files_skipped_chunk_cap += 1 + files_skipped += 1 + elif drawers == 0 and not dry_run: files_skipped += 1 else: total_drawers += drawers @@ -1251,7 +1343,24 @@ def _mine_impl( print(f"\n{'=' * 55}") print(" Done.") print(f" Files processed: {len(files) - files_skipped}") - print(f" Files skipped (already filed): {files_skipped}") + # The residual skip bucket label depends on mode: dry-run bypasses + # the already-mined check, so the only paths producing (0, room, + # None) under dry_run are OSError / too-short / post-lock re-check + # (and re-check itself is unreachable when nothing is being + # written). Outside dry_run, the dominant case is "already filed". + residual_label = ( + "Files skipped (read error or too short)" + if dry_run + else "Files skipped (already filed or other)" + ) + print(f" {residual_label}: {max(0, files_skipped - files_skipped_chunk_cap)}") + if files_skipped_chunk_cap > 0: + cap_desc = f"cap {effective_chunk_cap}" if effective_chunk_cap > 0 else "cap disabled" + print( + f" Files skipped (chunk {cap_desc}): {files_skipped_chunk_cap} " + f"(raise via --max-chunks-per-file or MEMPALACE_MAX_CHUNKS_PER_FILE; " + f"set 0 to disable)" + ) print(f" Drawers filed: {total_drawers}") print("\n By room:") for room, count in sorted(room_counts.items(), key=lambda x: x[1], reverse=True): diff --git a/tests/test_cli.py b/tests/test_cli.py index 41cfc2753d..0e6d06ffcc 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -566,6 +566,7 @@ def test_cmd_mine_projects_mode(mock_config_cls): dry_run=False, respect_gitignore=True, include_ignored=[], + max_chunks_per_file=None, ) diff --git a/tests/test_miner.py b/tests/test_miner.py index dedfc97bea..3d31f011ef 100644 --- a/tests/test_miner.py +++ b/tests/test_miner.py @@ -608,7 +608,7 @@ def upsert(self, documents, ids, metadatas): monkeypatch.setattr(miner, "detect_hall", lambda content: "code") monkeypatch.setattr(miner, "_extract_entities_for_metadata", lambda content: "") - drawers, room = miner.process_file( + drawers, room, skip_reason = miner.process_file( source, tmp_path, col, @@ -620,6 +620,7 @@ def upsert(self, documents, ids, metadatas): assert drawers == 5 assert room == "general" + assert skip_reason is None assert col.batch_sizes == [2, 2, 1] @@ -935,7 +936,7 @@ def fake_process_file(*args, **kwargs): call_count["n"] += 1 if call_count["n"] == 2: raise KeyboardInterrupt - return (1, "general") + return (1, "general", None) with patch("mempalace.miner.process_file", side_effect=fake_process_file): with pytest.raises(SystemExit) as exc_info: @@ -988,16 +989,18 @@ def test_skip_filenames_includes_lockfiles(): assert "yarn.lock" in miner.SKIP_FILENAMES -def test_process_file_skips_when_chunks_exceed_max(tmp_path, monkeypatch): - """A file producing more than MAX_CHUNKS_PER_FILE chunks must be skipped - with a clear message and zero upserts. Generated artifacts (CSVs, lock - files not in SKIP_FILENAMES) hit this — the cap is what prevents ONNX - bad_alloc on Windows when the embedder is asked to swallow thousands of - chunks in one batch (#1296).""" +def test_process_file_skips_when_chunks_exceed_max(tmp_path, monkeypatch, capsys): + """A file exceeding the per-file chunk cap is skipped with a tagged + return and a stderr/stdout message pointing at the config override. The + cap is the rail against pathological artifacts (CSVs, lockfiles not in + SKIP_FILENAMES) and against ONNX bad_alloc on Windows (#1296); #1455 + raised the default and made the cap configurable so legitimate + long-form content is not silently dropped.""" from unittest.mock import MagicMock from mempalace import miner + monkeypatch.delenv("MEMPALACE_MAX_CHUNKS_PER_FILE", raising=False) monkeypatch.setattr(miner, "MAX_CHUNKS_PER_FILE", 5) over_cap = [{"content": f"chunk {i}", "chunk_index": i} for i in range(7)] monkeypatch.setattr(miner, "chunk_text", lambda content, source_file, **kwargs: over_cap) @@ -1007,7 +1010,7 @@ def test_process_file_skips_when_chunks_exceed_max(tmp_path, monkeypatch): col = MagicMock() col.get.return_value = {"ids": []} - drawers, room = miner.process_file( + drawers, room, skip_reason = miner.process_file( source, tmp_path, col, @@ -1018,7 +1021,274 @@ def test_process_file_skips_when_chunks_exceed_max(tmp_path, monkeypatch): ) assert drawers == 0 + assert skip_reason == "chunk_cap" col.upsert.assert_not_called() + captured = capsys.readouterr() + # Skip notice goes to stderr to match the existing symlink-skip + # convention in ``scan_project`` so log piping stays coherent. + assert "[skip]" in captured.err + assert "--max-chunks-per-file" in captured.err + assert "MEMPALACE_MAX_CHUNKS_PER_FILE" in captured.err + assert "[skip]" not in captured.out + + +def test_resolve_max_chunks_default_when_no_override_no_env(monkeypatch): + """With no override and no env var, the module-level default applies.""" + from mempalace import miner + + monkeypatch.delenv("MEMPALACE_MAX_CHUNKS_PER_FILE", raising=False) + assert miner._resolve_max_chunks_per_file(None) == miner.MAX_CHUNKS_PER_FILE + + +def test_resolve_max_chunks_env_var_wins_over_default(monkeypatch): + """A numeric env var overrides the module default.""" + from mempalace import miner + + monkeypatch.setenv("MEMPALACE_MAX_CHUNKS_PER_FILE", "777") + assert miner._resolve_max_chunks_per_file(None) == 777 + + +def test_resolve_max_chunks_override_wins_over_env(monkeypatch): + """An explicit override (CLI flag plumbed in) wins over the env var.""" + from mempalace import miner + + monkeypatch.setenv("MEMPALACE_MAX_CHUNKS_PER_FILE", "777") + assert miner._resolve_max_chunks_per_file(123) == 123 + + +def test_resolve_max_chunks_sentinel_zero_disables(monkeypatch): + """Sentinel ``0`` from override or env means "no cap".""" + from mempalace import miner + + monkeypatch.delenv("MEMPALACE_MAX_CHUNKS_PER_FILE", raising=False) + assert miner._resolve_max_chunks_per_file(0) == 0 + + monkeypatch.setenv("MEMPALACE_MAX_CHUNKS_PER_FILE", "0") + assert miner._resolve_max_chunks_per_file(None) == 0 + + +def test_resolve_max_chunks_invalid_env_falls_back_to_default(monkeypatch, capsys): + """A non-integer env value warns and uses the module default. This + keeps a misconfigured shell from silently dropping content.""" + from mempalace import miner + + monkeypatch.setenv("MEMPALACE_MAX_CHUNKS_PER_FILE", "banana") + assert miner._resolve_max_chunks_per_file(None) == miner.MAX_CHUNKS_PER_FILE + err = capsys.readouterr().err + assert "MEMPALACE_MAX_CHUNKS_PER_FILE" in err + assert "banana" in err + + +def test_resolve_max_chunks_negative_env_falls_back_to_default(monkeypatch, capsys): + """A negative env value warns and uses the module default.""" + from mempalace import miner + + monkeypatch.setenv("MEMPALACE_MAX_CHUNKS_PER_FILE", "-5") + assert miner._resolve_max_chunks_per_file(None) == miner.MAX_CHUNKS_PER_FILE + err = capsys.readouterr().err + assert "MEMPALACE_MAX_CHUNKS_PER_FILE" in err + assert "-5" in err + + +def test_process_file_sentinel_zero_disables_cap(tmp_path, monkeypatch): + """With ``max_chunks_per_file=0`` even a pathologically large chunk + count is processed (no skip).""" + from unittest.mock import MagicMock + + from mempalace import miner + + monkeypatch.delenv("MEMPALACE_MAX_CHUNKS_PER_FILE", raising=False) + monkeypatch.setattr(miner, "MAX_CHUNKS_PER_FILE", 5) + big = [{"content": f"chunk {i}", "chunk_index": i} for i in range(20)] + monkeypatch.setattr(miner, "chunk_text", lambda content, source_file, **kwargs: big) + monkeypatch.setattr(miner, "detect_room", lambda *a, **k: "general") + monkeypatch.setattr(miner, "_extract_entities_for_metadata", lambda content: "") + monkeypatch.setattr(miner, "build_closet_lines", lambda *a, **k: []) + monkeypatch.setattr(miner, "purge_file_closets", lambda *a, **k: None) + monkeypatch.setattr(miner, "upsert_closet_lines", lambda *a, **k: None) + + source = tmp_path / "huge.csv" + source.write_text("payload\n" * 500, encoding="utf-8") + col = MagicMock() + col.get.return_value = {"ids": []} + + drawers, _room, skip_reason = miner.process_file( + source, + tmp_path, + col, + "wing", + [{"name": "general", "description": "General"}], + "agent", + False, + max_chunks_per_file=0, + ) + + assert drawers == 20 + assert skip_reason is None + col.upsert.assert_called() + + +def test_mine_summary_separates_chunk_cap_skips(tmp_path, monkeypatch, capsys): + """Summary distinguishes residual skips from "chunk cap" skips so a + user can see immediately that legitimate content was dropped (#1455). + The chunk-cap summary line appears only when count > 0.""" + from unittest.mock import patch + + project_root = tmp_path / "proj" + project_root.mkdir() + _make_minable_project(project_root, n_files=3) + palace_path = project_root / "palace" + + seq = iter( + [ + (5, "general", None), + (0, "general", "chunk_cap"), + (0, "general", None), + ] + ) + + def fake_process_file(*args, **kwargs): + return next(seq) + + with patch("mempalace.miner.process_file", side_effect=fake_process_file): + mine(str(project_root), str(palace_path)) + + out = capsys.readouterr().out + assert "Files skipped (already filed or other): 1" in out + assert "Files skipped (chunk cap" in out + assert "--max-chunks-per-file" in out + + +def test_mine_summary_omits_chunk_cap_line_when_zero(tmp_path, monkeypatch, capsys): + """When no file hits the chunk cap, the chunk-cap summary line is not + printed at all, which keeps the happy-path output unchanged.""" + from unittest.mock import patch + + project_root = tmp_path / "proj" + project_root.mkdir() + _make_minable_project(project_root, n_files=2) + palace_path = project_root / "palace" + + def fake_process_file(*args, **kwargs): + return (3, "general", None) + + with patch("mempalace.miner.process_file", side_effect=fake_process_file): + mine(str(project_root), str(palace_path)) + + out = capsys.readouterr().out + assert "Files skipped (already filed or other): 0" in out + assert "chunk cap" not in out + + +def test_resolve_max_chunks_negative_override_falls_back_to_default(monkeypatch, capsys): + """A negative CLI override warns and falls back to the module default. + Symmetric with the env-var path so ``--max-chunks-per-file=-500`` (a + typo meaning "no, don't lower it that much") does not silently + disable the cap and OOM the embedder on the original lockfile.""" + from mempalace import miner + + monkeypatch.delenv("MEMPALACE_MAX_CHUNKS_PER_FILE", raising=False) + assert miner._resolve_max_chunks_per_file(-5) == miner.MAX_CHUNKS_PER_FILE + err = capsys.readouterr().err + assert "--max-chunks-per-file" in err + assert "-5" in err + + +def test_resolve_max_chunks_reads_module_attribute_at_call_time(monkeypatch): + """The resolver reads ``miner.MAX_CHUNKS_PER_FILE`` lazily, so a + monkeypatch landed at test setup is honored. Regression guard against + a future refactor that captures the import-time default.""" + from mempalace import miner + + monkeypatch.delenv("MEMPALACE_MAX_CHUNKS_PER_FILE", raising=False) + monkeypatch.setattr(miner, "MAX_CHUNKS_PER_FILE", 7) + assert miner._resolve_max_chunks_per_file(None) == 7 + + +def test_process_file_chunk_cap_under_dry_run(tmp_path, monkeypatch): + """Dry-run is the natural audit path for #1455; chunk-cap drops must + return a tagged skip_reason there too so the summary counter can fire.""" + from unittest.mock import MagicMock + + from mempalace import miner + + monkeypatch.delenv("MEMPALACE_MAX_CHUNKS_PER_FILE", raising=False) + monkeypatch.setattr(miner, "MAX_CHUNKS_PER_FILE", 5) + over_cap = [{"content": f"chunk {i}", "chunk_index": i} for i in range(7)] + monkeypatch.setattr(miner, "chunk_text", lambda content, source_file, **kwargs: over_cap) + + source = tmp_path / "big.csv" + source.write_text("payload\n" * 200, encoding="utf-8") + col = MagicMock() + col.get.return_value = {"ids": []} + + drawers, _room, skip_reason = miner.process_file( + source, + tmp_path, + col, + "wing", + [{"name": "general", "description": "General"}], + "agent", + True, # dry_run + ) + + assert drawers == 0 + assert skip_reason == "chunk_cap" + + +def test_mine_dry_run_summary_counts_chunk_cap_drops(tmp_path, capsys): + """The summary under dry-run also splits out chunk-cap skips. Without + this a reporter running ``--dry-run`` to validate the new default + against their corpus would see "Files processed: N / Files skipped: 0" + even when chunk-cap drops occurred, which is exactly the silent-drop + UX bug that #1455 is fixing.""" + from unittest.mock import patch + + project_root = tmp_path / "proj" + project_root.mkdir() + _make_minable_project(project_root, n_files=3) + palace_path = project_root / "palace" + + seq = iter( + [ + (5, "general", None), + (0, "general", "chunk_cap"), + (4, "general", None), + ] + ) + + def fake_process_file(*args, **kwargs): + return next(seq) + + with patch("mempalace.miner.process_file", side_effect=fake_process_file): + mine(str(project_root), str(palace_path), dry_run=True) + + out = capsys.readouterr().out + assert "Files skipped (chunk cap" in out + assert "1 (raise via" in out + + +def test_mine_plumbs_max_chunks_per_file_to_process_file(tmp_path): + """``mine(max_chunks_per_file=0)`` reaches ``process_file`` as kwarg=0, + enabling the sentinel-disable path end-to-end. Guards the wiring + ``cmd_mine -> mine -> _mine_impl -> process_file``.""" + from unittest.mock import patch + + project_root = tmp_path / "proj" + project_root.mkdir() + _make_minable_project(project_root, n_files=1) + palace_path = project_root / "palace" + + captured = {} + + def fake_process_file(*args, **kwargs): + captured["max_chunks_per_file"] = kwargs.get("max_chunks_per_file") + return (1, "general", None) + + with patch("mempalace.miner.process_file", side_effect=fake_process_file): + mine(str(project_root), str(palace_path), max_chunks_per_file=0) + + assert captured["max_chunks_per_file"] == 0 def test_mine_arbitrary_exception_prints_summary_and_reraises(tmp_path, capsys): @@ -1040,7 +1310,7 @@ def fake_process_file(*args, **kwargs): call_count["n"] += 1 if call_count["n"] == 2: raise RuntimeError("simulated ONNX bad_alloc") - return (1, "general") + return (1, "general", None) with patch("mempalace.miner.process_file", side_effect=fake_process_file): with pytest.raises(RuntimeError, match="simulated ONNX bad_alloc"): From 22cec56cd8f9007d77190b02940af3d72d2a27eb Mon Sep 17 00:00:00 2001 From: mvalentsev Date: Wed, 20 May 2026 02:29:52 +0500 Subject: [PATCH 22/39] fix(miner): use Optional[int] for Python 3.9 compat (#1455) PEP 604 union syntax (int | None) requires Python 3.10+ but pyproject.toml declares requires-python = ">=3.9", so the type hints in _resolve_max_chunks_per_file, process_file, mine, and _mine_impl raised TypeError on test-linux (3.9). Replace with Optional[int] using the typing import already present at line 20. Co-Authored-By: David Glidden --- mempalace/miner.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mempalace/miner.py b/mempalace/miner.py index 6987dcec16..21dce41a38 100644 --- a/mempalace/miner.py +++ b/mempalace/miner.py @@ -106,7 +106,7 @@ # memory before chunking), so memory use scales with source size too. -def _resolve_max_chunks_per_file(override: int | None = None) -> int: +def _resolve_max_chunks_per_file(override: Optional[int] = None) -> int: """Resolve the effective per-file chunk cap. Precedence: ``override`` (CLI flag) > ``MEMPALACE_MAX_CHUNKS_PER_FILE`` @@ -932,7 +932,7 @@ def process_file( chunk_size: int = None, chunk_overlap: int = None, min_chunk_size: int = None, - max_chunks_per_file: int | None = None, + max_chunks_per_file: Optional[int] = None, ) -> tuple: """Read, chunk, route, and file one file. @@ -1164,7 +1164,7 @@ def mine( respect_gitignore: bool = True, include_ignored: list = None, files: list = None, - max_chunks_per_file: int | None = None, + max_chunks_per_file: Optional[int] = None, ): """Mine a project directory into the palace. @@ -1221,7 +1221,7 @@ def _mine_impl( respect_gitignore: bool = True, include_ignored: list = None, files: list = None, - max_chunks_per_file: int | None = None, + max_chunks_per_file: Optional[int] = None, ): from .config import MempalaceConfig From 81f3e32a3bfb07c62479705459f73a16129511a1 Mon Sep 17 00:00:00 2001 From: mvalentsev Date: Wed, 20 May 2026 02:38:25 +0500 Subject: [PATCH 23/39] fix(miner): consolidate skip accounting under --dry-run (#1455) Residual zero-drawer skips (OSError, too-short, post-lock re-check) now increment files_skipped under both modes. Previously dry_run sent them to the else-branch where they polluted room_counts and left the new "read error or too short" summary line at 0 while those files were silently counted as "processed". Per Gemini review on PR #1554. Co-Authored-By: David Glidden --- mempalace/miner.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/mempalace/miner.py b/mempalace/miner.py index 21dce41a38..072a396a9e 100644 --- a/mempalace/miner.py +++ b/mempalace/miner.py @@ -1305,19 +1305,15 @@ def _mine_impl( raise files_processed = i last_file = filepath.name - # Chunk-cap accounting fires under dry_run too: dry-run is the - # natural way to audit a corpus before a real mine, so silent - # chunk-cap drops there would defeat the whole point of #1455. - # Bump ``files_skipped`` alongside for the chunk-cap path so the - # "Files processed" arithmetic in the summary reflects the - # chunk-cap drops in both modes. (Pre-existing OSError / - # too-short / re-check skips under dry_run still flow into the - # else-branch; that is a separate accounting concern.) - if drawers == 0 and skip_reason == "chunk_cap": - files_skipped_chunk_cap += 1 - files_skipped += 1 - elif drawers == 0 and not dry_run: + # All zero-drawer outcomes increment ``files_skipped`` in both + # modes so the summary "Files processed" arithmetic and the + # residual-skip counter stay honest under ``--dry-run`` too. The + # chunk-cap counter is partitioned out for its dedicated + # summary line (see #1455 + Gemini review on PR #1554). + if drawers == 0: files_skipped += 1 + if skip_reason == "chunk_cap": + files_skipped_chunk_cap += 1 else: total_drawers += drawers room_counts[room] += 1 From 8d8a8c9c19ea966c140d31240ace8ca7b6194f26 Mon Sep 17 00:00:00 2001 From: mvalentsev Date: Wed, 20 May 2026 03:43:33 +0500 Subject: [PATCH 24/39] fix(miner): polish per multi-agent review (#1455) Drop the dead ``cap disabled`` arm of the ``cap_desc`` ternary in the mine summary. The outer ``if files_skipped_chunk_cap > 0:`` only fires when ``process_file`` returned ``"chunk_cap"``, which itself is gated by ``effective_cap > 0``; the ``effective_chunk_cap == 0`` arm was unreachable. Inlined the single live form with a comment recording the invariant. Rephrase the comment at the pre-resolved-int handoff in ``_mine_impl``: the short-circuit lives in ``_resolve_max_chunks_per_file``'s ``override is not None`` branch, not a separately named ``in-resolver fast-path``. No behaviour change. Co-Authored-By: David Glidden --- mempalace/miner.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mempalace/miner.py b/mempalace/miner.py index 072a396a9e..274508eb4b 100644 --- a/mempalace/miner.py +++ b/mempalace/miner.py @@ -1294,8 +1294,9 @@ def _mine_impl( chunk_overlap=cfg_chunk_overlap, min_chunk_size=cfg_min_chunk_size, # Pass the already-resolved int so ``process_file``'s - # in-resolver fast-path short-circuits; otherwise a - # malformed env var would emit its warning per file. + # ``override is not None`` branch skips the env re-read; + # otherwise a malformed env var would emit its warning + # per file. max_chunks_per_file=effective_chunk_cap, ) except KeyboardInterrupt: @@ -1351,9 +1352,11 @@ def _mine_impl( ) print(f" {residual_label}: {max(0, files_skipped - files_skipped_chunk_cap)}") if files_skipped_chunk_cap > 0: - cap_desc = f"cap {effective_chunk_cap}" if effective_chunk_cap > 0 else "cap disabled" + # ``effective_chunk_cap`` is necessarily > 0 here: ``process_file`` + # only emits the ``"chunk_cap"`` skip_reason when its own + # ``effective_cap > 0`` guard passes (see ``process_file``). print( - f" Files skipped (chunk {cap_desc}): {files_skipped_chunk_cap} " + f" Files skipped (chunk cap {effective_chunk_cap}): {files_skipped_chunk_cap} " f"(raise via --max-chunks-per-file or MEMPALACE_MAX_CHUNKS_PER_FILE; " f"set 0 to disable)" ) From 5e3d3dde2e3eab700e53612c016e612f9b59ca3a Mon Sep 17 00:00:00 2001 From: Milla J <232237854+milla-jovovich@users.noreply.github.com> Date: Tue, 19 May 2026 19:15:40 -0700 Subject: [PATCH 25/39] feat(3.3.6): virtual line numbering + format coverage via --mode extract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two additive features, both following the same read-time-transform pattern: 1. Virtual line numbering — new render_with_line_numbers() and extract_line_range() in mempalace/searcher.py. Closet pointers like 2026-01-18:L55-L72 resolve to drawer slices rendered with [55] through [72] line prefixes without modifying any stored content. Lines already prefixed with [] pass through unchanged. Pure functions, no I/O. 2. Format coverage (mempalace mine --mode extract) — new mempalace/format_miner.py reads binary office formats and files drawers via the lock + purge + upsert pattern convo_miner uses. Source files never modified. Per-format transformer routing: MarkItDown 0.1.5 does not actually convert .rtf (returns raw control codes unchanged, verified live), so .rtf is routed to striprtf which does convert. Other formats stay on MarkItDown: .pdf .docx .pptx .xlsx .epub -> MarkItDown .rtf -> striprtf 13 fringe cases handled with dedicated ExtractionStatus codes: SKIP_NO_MARKITDOWN, SKIP_NO_STRIPRTF, SKIP_TOO_LARGE, SKIP_CLOUD_ONLY, SKIP_ENCRYPTED, SKIP_EMPTY, SKIP_PERMISSION, SKIP_BROKEN_SYMLINK, SKIP_UNRECOGNIZED, SKIP_EXTRACTION_ERROR, SKIP_NETWORK_TIMEOUT, SKIP_UNREADABLE, plus the encoding fallback handled internally. Drawers carry ingest_mode=extract + extract_mode=format so they are distinguishable from project / convo drawers in the palace. Both transformers are optional extras: pip install mempalace[extract]. MarkItDown requires Python >= 3.10 (env marker ensures it only installs where supported; Python 3.9 users still get RTF coverage via striprtf). Tests: 94 new (21 line-numbering + 73 format-miner, including 9 mine_formats orchestrator tests). Full mempalace suite still green (1992 passed locally). Coverage 86% on mempalace/format_miner.py. ruff check + ruff format clean on the pinned 0.15.9. Live verification: mempalace mine --mode extract on a directory with 2 RTFs + 1 PDF produced 90 drawers correctly; mempalace search found content from both file types in the resulting wing. Documented limits (out of scope for 3.3.6): custom PDF parsers, OCR on scanned PDFs, DRM-locked files, pathological corrupt files. These get reported via skip codes and skipped. Refs: docs/format-coverage.md, docs/virtual-line-numbering.md --- docs/format-coverage.md | 223 +++++++++ docs/virtual-line-numbering.md | 152 ++++++ mempalace/cli.py | 27 +- mempalace/format_miner.py | 674 ++++++++++++++++++++++++++ mempalace/searcher.py | 58 +++ pyproject.toml | 8 + tests/test_format_miner.py | 850 +++++++++++++++++++++++++++++++++ tests/test_line_numbers.py | 161 +++++++ 8 files changed, 2148 insertions(+), 5 deletions(-) create mode 100644 docs/format-coverage.md create mode 100644 docs/virtual-line-numbering.md create mode 100644 mempalace/format_miner.py create mode 100644 tests/test_format_miner.py create mode 100644 tests/test_line_numbers.py diff --git a/docs/format-coverage.md b/docs/format-coverage.md new file mode 100644 index 0000000000..178658e09e --- /dev/null +++ b/docs/format-coverage.md @@ -0,0 +1,223 @@ +# Format coverage — `mempalace mine --mode extract` + +**Proposed for mempalace 3.3.6.** +**Target file (new):** `mempalace/format_miner.py` +**Target file (edited, one-line addition):** `mempalace/cli.py` (dispatcher in `cmd_mine`) +**New tests:** `tests/test_format_miner.py` (32 cases, all green; see `PROOF.md`). + +--- + +## What it does + +Adds a third miner alongside the existing two: + +``` +mempalace mine → miner.py (project files: code, docs, notes) +mempalace mine --mode convos → convo_miner.py (chat exports) +mempalace mine --mode extract → format_miner.py (binary office documents) ← NEW +``` + +Supported formats and the transformer each routes through: + +| Extension | Transformer | Python | License | +|---|---|---|---| +| `.pdf` | MarkItDown | ≥ 3.10 | MIT | +| `.docx` | MarkItDown | ≥ 3.10 | MIT | +| `.pptx` | MarkItDown | ≥ 3.10 | MIT | +| `.xlsx` | MarkItDown | ≥ 3.10 | MIT | +| `.epub` | MarkItDown | ≥ 3.10 | MIT | +| `.rtf` | **striprtf** | ≥ 3.6 | MIT | + +**Per-format routing — why `.rtf` doesn't go through MarkItDown.** Verified live on 2026-05-19 against a local mixed-format test directory: MarkItDown 0.1.5 does NOT convert `.rtf`. It returns the raw RTF control-code source unchanged (`{\rtf1\ansi\ansicpg1252...`). For RTF input, format_miner routes to [striprtf](https://github.com/joshy/striprtf) (pure-Python, MIT, ~150 lines, purpose-built for RTF → plain text). The principle stays intact: transform at read time, never modify the source — only the choice of transformer is per-format. + +Both libraries are **optional** runtime dependencies. Users who only mine projects or convos don't pay either install footprint. Users on Python 3.9 can still install `striprtf` to extract RTF; MarkItDown requires 3.10+ and silently skips if not present. Per-extension install messages route to the right library: + +- Missing MarkItDown (any of pdf/docx/pptx/xlsx/epub) → `SKIP_NO_MARKITDOWN` → log: `pip install markitdown` +- Missing striprtf (any `.rtf`) → `SKIP_NO_STRIPRTF` → log: `pip install striprtf` + +--- + +## Design — read-time conversion, never modifies source files + +Same architectural principle as virtual line numbering (the other 3.3.6 feature in this package): **transform at read time, leave storage untouched.** + +``` +PDF on disk → (read bytes into memory) → MarkItDown → Markdown text → existing chunker → drawers in ChromaDB +``` + +- The PDF on the user's disk is read but **never modified**. +- No intermediate `.md` files are written to the user's filesystem. +- The conversion lives entirely in memory; once the text reaches the existing chunker, it's handled identically to any other ingestion. +- Closets / drawers / halls all work the same — the format miner produces the same drawer-shape as the other miners. + +--- + +## Why a new miner, not edits to `miner.py` + +Three reasons, in order of importance: + +1. **Matches the existing miner-per-content-type pattern.** Mempalace already has `miner.py` and `convo_miner.py`. A third miner is a third instance of the same shape — immediately recognizable to anyone reading the codebase. +2. **Smaller review surface.** `format_miner.py` is a pure addition. Reviewer reads one new file in isolation; the stable code paths in `miner.py` and `convo_miner.py` are untouched. +3. **Cleaner failure isolation.** If the format miner has a bug, the existing miners are physically incapable of being affected. + +The trade-off vs. integrating into `miner.py` was decided during the 2026-05-19 design pass: **the new-miner pattern wins on architecture and on review surface.** + +--- + +## Fringe cases handled — 13 (original 12 + striprtf parity) + +Each case maps to a specific `ExtractionStatus` enum value so callers (and reviewers) can audit exactly which path fired. All thirteen are covered by dedicated tests in `tests/test_format_miner.py`. + +| # | Case | Status code | Behavior | +|---|---|---|---| +| 1 | MarkItDown not installed (any non-RTF format) | `SKIP_NO_MARKITDOWN` | Logs `pip install markitdown` instruction; skips file | +| 2 | File too large (> 500 MB default; caller can override) | `SKIP_TOO_LARGE` | Logs size and threshold; skips | +| 3 | iCloud cloud-only file (`.icloud` suffix OR `st_flags` dataless bit) | `SKIP_CLOUD_ONLY` | Detected before any I/O that would trigger materialization; skips | +| 4 | Encrypted / password-protected PDF | `SKIP_ENCRYPTED` | Catches exceptions matching `encrypt`/`decrypt`/`password`/`protected`; skips | +| 5 | Empty file (zero bytes) | `SKIP_EMPTY` | Silent skip | +| 6 | Permission denied | `SKIP_PERMISSION` | Catches `PermissionError`; logs; skips | +| 7 | Broken symlink (target missing) | `SKIP_BROKEN_SYMLINK` | Detected via `is_symlink()` + `not exists()`; skips | +| 8 | Dirty encoding in extracted text | (recovered) | `decode_robust()` falls back UTF-8 → CP1252 → UTF-8-with-replace; never raises | +| 9 | Windows path semantics | (no special status) | `pathlib.Path` throughout; case-insensitive suffix matching; accepts `str` or `Path` input | +| 10 | Transformer internal crash on malformed file | `SKIP_EXTRACTION_ERROR` | Catches generic `Exception` at file boundary; logs filename + exception type + message tail; skips | +| 11 | Network / sync-drive timeout | `SKIP_NETWORK_TIMEOUT` | Catches `TimeoutError`; skips | +| 12 | Unrecognized extension | `SKIP_UNRECOGNIZED` | Cheap suffix-set check; skips | +| 13 | **striprtf not installed (any `.rtf`)** | `SKIP_NO_STRIPRTF` | Logs `pip install striprtf` instruction; skips file. Added 2026-05-19 after a live integration test surfaced MarkItDown 0.1.5's lack of RTF support. | + +Plus `SKIP_UNREADABLE` for catch-all `OSError` during stat (not in the original list, added for defensive completeness — see file-stat block in `extract_text`). + +--- + +## Cases NOT solved by this miner (deferred) + +Per the spec, these are documented limits, not bugs: + +- **Custom PDF parsers for specific document types.** MarkItDown does its best; we accept its limits. +- **OCR on scanned image PDFs.** Separate concern, opt-in feature for a later release. +- **DRM-locked files.** We can't bypass DRM and shouldn't try. +- **Pathological corrupt files.** They're corrupt — `SKIP_EXTRACTION_ERROR` with a clear log line is the responsible outcome. + +These limitations are reported per-file via the skip-code log, so the user always knows what didn't make it in and why. + +--- + +## API reference + +All public symbols are exported via `__all__`. Two functions and one enum form the contract: + +### `extract_text(path, max_file_size=DEFAULT_MAX_FILE_SIZE) -> tuple[Optional[str], ExtractionStatus]` + +Convert one file to text. Returns `(text, status)` where `text` is the extracted Markdown for `OK` cases and `None` for every skip. Pure function — source file at `path` is never modified. + +Accepts both `Path` and `str` for `path`. + +### `scan_formats(directory) -> list[Path]` + +Walk `directory` recursively, return supported files sorted by path. Skips hidden / build directories (`.git`, `.venv`, `__pycache__`, etc.) and OS metadata files (`.DS_Store`, `Thumbs.db`, `desktop.ini`). + +### `ExtractionStatus` (Enum) + +Twelve documented values plus `SKIP_UNREADABLE`. Each is paired with a string value like `"skip:no_markitdown"` for log readability; tests assert on `.name` for stability. + +### `decode_robust(raw: bytes) -> str` + +Identical to the helper in the terminal-session normalizer. UTF-8 first, CP1252 fallback, UTF-8-with-replace as final safety net. Never raises. + +### `is_icloud_dataless(path: Path) -> bool` + +Two signals: +1. Literal `.icloud` suffix (iCloud's offloaded-file placeholder convention) +2. macOS `st_flags` dataless bit (`0x40000000`) — best-effort, gracefully degrades on non-macOS + +Returning `True` causes `extract_text` to short-circuit to `SKIP_CLOUD_ONLY` before any I/O that would trigger iCloud materialization. + +--- + +## CLI integration + +One-line addition to `mempalace/cli.py` in `cmd_mine`: + +```python +def cmd_mine(args): + ... + try: + if args.mode == "convos": + from .convo_miner import mine_convos + mine_convos(...) + elif args.mode == "extract": # ← NEW + from .format_miner import mine_formats # ← NEW + mine_formats( # ← NEW + format_dir=args.dir, # ← NEW + palace_path=palace_path, # ← NEW + wing=args.wing, # ← NEW + agent=args.agent, # ← NEW + limit=args.limit, # ← NEW + dry_run=args.dry_run, # ← NEW + ) # ← NEW + else: + from .miner import mine + mine(...) +``` + +And add `"extract"` to the `--mode` choices in the argparse setup: + +```python +p_mine.add_argument( + "--mode", + choices=["projects", "convos", "extract"], # ← extract added + default="projects", + help="Ingest mode: 'projects' (default), 'convos' for chat exports, 'extract' for office documents", +) +``` + +That's the entire CLI change. Two argparse edits + one `elif` branch. + +--- + +## Empirical smoke test — local mixed-format directory + +Run on a local test directory containing **52 `.rtf` files + 11 `.pdf` files** (a mix of long-form correspondence and research/architecture documents, no synthetic fixtures): + +``` +Supported formats: ['.docx', '.epub', '.pdf', '.pptx', '.rtf', '.xlsx'] +scan_formats: found 63 supported files +``` + +**Live end-to-end extraction (2026-05-19, Python 3.13, MarkItDown 0.1.5 + striprtf 0.0.27):** + +| Format | Files | Result | Chars extracted | +|---|---|---|---| +| RTF (via striprtf) | 52 | **52/52 OK** | 1,350,679 | +| PDF (via MarkItDown) | 11 | 10 OK, 1 SKIP_ENCRYPTED | 1,340,869 | + +The single PDF skip is a real password-protected file — Fringe Case 4 (`SKIP_ENCRYPTED`) fired correctly on real data via the exception-message matcher. Designed behavior validated empirically. + +Anti-regression assertion: **zero raw `\rtf1` / `\ansi` control codes leak into extracted text**. Without the per-format routing fix, 52 of 52 RTFs would have ingested as unreadable control-code source. With the fix, every RTF yields clean plain text. This is the integration bug that mocked unit tests cannot catch and that motivated the `SKIP_NO_STRIPRTF` status + per-format dispatch. + +--- + +## Backwards compatibility + +- **No changes to existing miners.** `miner.py` and `convo_miner.py` are not touched. +- **No new required dependencies.** MarkItDown and striprtf are optional extras (`pip install markitdown striprtf` or `pip install mempalace[extract]` if declared as such in `pyproject.toml`). MarkItDown requires Python ≥ 3.10; striprtf works on 3.6+. Recommended pyproject entries use environment markers so 3.9 users automatically get only striprtf: + ```toml + [project.optional-dependencies] + extract = [ + "striprtf>=0.0.27", + "markitdown>=0.1.5; python_version >= '3.10'", + ] + ``` +- **No on-disk format changes.** Drawers / closets / halls have the exact same shape regardless of which miner produced them. +- **No migration step.** Users who already have a palace mined from the other modes can run `mempalace mine --mode extract ~/docs/` and the new drawers land in the same palace, same wing, same room conventions. +- **Optional `--max-file-size` flag** can be added to `mempalace mine` so users with legitimate large files (e.g. scanned books) can raise the cap from the 500 MB default. + +--- + +## Out of scope for 3.3.6 + +- **Streaming extraction for huge PDFs** (>500 MB). MarkItDown's default behavior is whole-file in memory. For users with multi-GB PDFs, a streaming path via `pypdf` page-by-page is the natural extension. Deferred to 3.3.7 or 3.4.0. +- **Audio / video transcription.** Whisper integration is its own scope. +- **OCR for scanned PDFs.** Tesseract / cloud OCR integration is its own scope. +- **Migration importers** (Notion / Obsidian / Mem0 export formats). Each is a separate adapter; not bundled here. + +These are real future features; they don't belong in a 3.3.6 release whose scope is "additive format coverage via MarkItDown." diff --git a/docs/virtual-line-numbering.md b/docs/virtual-line-numbering.md new file mode 100644 index 0000000000..bae5b62970 --- /dev/null +++ b/docs/virtual-line-numbering.md @@ -0,0 +1,152 @@ +# Virtual line numbering for drawers + +**Proposed for mempalace 3.3.6.** +**Target file:** `mempalace/searcher.py` (new module-level functions, alongside `_tokenize`, `_bm25_scores`, etc.) +**New tests:** `tests/test_line_numbers.py` (21 cases, all green; see `PROOF.md`). + +--- + +## What it does + +Adds two pure functions that apply line numbers to drawer text **at read time**, without modifying anything stored on disk: + +```python +render_with_line_numbers(text: str, start_line: int = 1) -> str +extract_line_range(text: str, line_start: int, line_end: int) -> str +``` + +A closet pointer like `→2026-01-18:L55-L72` now resolves by opening the drawer and calling `extract_line_range(drawer_text, 55, 72)`. The returned string carries line numbers `[55]` through `[72]` so the reader sees which drawer positions they are looking at, even though the drawer on disk has no line numbers in it. + +--- + +## Why this design + +### Read-time, not backfill + +The alternative is to rewrite every drawer with `[N]` prefixes embedded in the stored text. That choice would: + +1. **Invalidate every existing closet pointer** the moment a drawer is re-emitted with different numbering. Today there are 1,377+ drawers across users' palaces; rewriting them all to add `[N]` is a corpus-wide migration with no fallback. +2. **Couple storage to display.** A drawer's purpose is to be the verbatim of a day. The instant we mix presentation (line numbers) into storage, the "verbatim" claim becomes conditional on the renderer. +3. **Lose idempotence.** Re-running mine on the same source would either skip (and miss new line-number conventions) or re-rewrite (and shift numbering on edits). + +Read-time numbering sidesteps all three. The drawer on disk is exactly what was written. The grid exists only in the act of reading. + +### Already-numbered passthrough + +Some drawers arrive at the function already prefixed with `[N]` — e.g. transcripts from pre-numbering tools, or output of an earlier render that was saved. Detection is a `^\[\d+\]` regex on each line; matched lines pass through unchanged. The counter **still advances** on those lines so positional alignment with the underlying drawer is preserved. + +This is defensive, not aspirational: most drawers are plain text, but mixed inputs must not double-prefix. + +### Pure, no I/O + +Neither function reads or writes the palace. They take strings, return strings. The caller (the search pipeline higher up in `searcher.py`) is responsible for opening drawers and persisting results. This keeps the primitive testable in isolation and reusable from anywhere — MCP server, CLI, future Flutter UI — without dragging in chromadb. + +--- + +## API reference + +### `render_with_line_numbers(text, start_line=1)` + +| Parameter | Type | Default | Meaning | +|---|---|---|---| +| `text` | `str \| None` | (required) | The drawer body. `None` is treated as empty. | +| `start_line` | `int` | `1` | Number assigned to the first line of the output. | + +**Returns:** `str` — each line prefixed with `[N] `, where `N` is the running counter beginning at `start_line`. Lines that already match `^\[\d+\]` pass through unchanged; the counter still advances. + +**Pure:** input is not mutated. + +```python +>>> render_with_line_numbers("alpha\nbeta") +'[1] alpha\n[2] beta' + +>>> render_with_line_numbers("first\nsecond", start_line=5) +'[5] first\n[6] second' + +>>> render_with_line_numbers("[42] kept\nplain line") +'[42] kept\n[2] plain line' +``` + +### `extract_line_range(text, line_start, line_end)` + +| Parameter | Type | Meaning | +|---|---|---| +| `text` | `str` | The full drawer body. | +| `line_start` | `int` | First line of the slice, 1-indexed, inclusive. Values below `1` are clamped to `1`. | +| `line_end` | `int` | Last line of the slice, 1-indexed, inclusive. Values past the end of the drawer are clipped to the drawer length. | + +**Returns:** `str` — the extracted slice with virtual line numbers starting at `line_start`. Empty / invalid ranges return `""` (no exception). + +**Pure:** input is not mutated. + +```python +>>> extract_line_range("a\nb\nc\nd\ne", 2, 4) +'[2] b\n[3] c\n[4] d' + +>>> extract_line_range("a\nb\nc", 2, 99) # end clipped to drawer length +'[2] b\n[3] c' + +>>> extract_line_range("a\nb\nc", 5, 2) # invalid range +'' +``` + +--- + +## Edge cases covered by the test suite + +The 21 tests in `tests/test_line_numbers.py` exercise: + +**`render_with_line_numbers`** +- Empty string → `""` +- `None` → `""` +- Single line +- Multi-line +- Custom `start_line` +- Already-numbered passthrough (no double-prefix) +- Already-numbered passthrough with a custom `start_line` (line numbers in source win, not the counter) +- Mixed numbered + plain input (counter advances on every line) +- Blank lines (still numbered — they are real positions) +- Trailing newline semantics (`"a\nb\n".split("\n")` → 3 positions, all numbered) +- Input not mutated + +**`extract_line_range`** +- Single-line extraction +- Inclusive range +- Full-document extraction +- End beyond length → clipped +- Start below 1 → clamped to 1 +- Start > end → empty +- Empty text → empty +- Slice containing already-numbered lines (source numbers preserved, counter still advances elsewhere) +- Closet-pointer contract: extracting lines 5-7 returns `[5][6][7]`, not `[1][2][3]` +- Input not mutated + +All 21 pass on Python 3.9 / pytest 8.4.2 in 0.01s. See `PROOF.md` for verbatim run output. + +--- + +## Where to integrate in `mempalace/searcher.py` + +Drop the two functions at module level, alongside the existing helpers (`_tokenize`, `_bm25_scores`, `_hybrid_rank`). They have no dependencies beyond `re` (already imported). + +In the existing `search()` and `search_memories()` flows, when a result row carries explicit `line_start` / `line_end` metadata (e.g. from a closet pointer), pass the drawer text through `extract_line_range` before returning. When a result has no explicit range (BM25 / vector hit on the full drawer body), pass it through `render_with_line_numbers` instead. Both behaviors are additive — existing callers that ignore line numbers get the same string back, just with a `[N] ` prefix per line. + +No new dependencies. No changes to drawer storage. No migration step. + +--- + +## Out of scope for 3.3.6 (deferred) + +- **Conversation-boundary detection** (the original `_extract_section` in private tooling expanded line ranges to natural conversation boundaries using timestamp heuristics). That logic is corpus-shape-specific and belongs in its own PR after we agree on the heuristic. +- **Day-aggregated drawers** (v4 spec — one drawer per day). 3.3.6 keeps the existing chunk-shaped drawer model. Line numbering works for both, but day-aggregation is a separate architectural change. +- **Halls / tunnels.** Unrelated to read-time rendering. +- **CLI surface.** No new flags. The behavior is internal to the search pipeline. + +--- + +## Backwards compatibility + +- **Drawer storage unchanged.** Existing palaces continue to work without migration. +- **Closet pointers unchanged.** The `→date:Lstart-Lend` syntax is unchanged; this PR adds the resolver, not the syntax. +- **Public API additive.** Two new module-level functions; nothing renamed or removed. +- **No new required arguments** on existing functions. If a caller never passes `line_start` / `line_end`, behavior is identical to today. diff --git a/mempalace/cli.py b/mempalace/cli.py index 527c96d98e..05caebafd4 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -2,9 +2,10 @@ """ MemPalace — Give your AI a memory. No API key required. -Two ways to ingest: - Projects: mempalace mine ~/projects/my_app (code, docs, notes) - Conversations: mempalace mine --mode convos (Claude Code, Claude.ai, ChatGPT, Slack exports) +Three ways to ingest: + Projects: mempalace mine ~/projects/my_app (code, docs, notes) + Conversations: mempalace mine --mode convos (Claude Code, Claude.ai, ChatGPT, Slack exports) + Documents: mempalace mine --mode extract (PDF, DOCX, PPTX, XLSX, RTF, EPUB — requires mempalace[extract]) Same palace. Same search. Different ingest strategies. @@ -13,6 +14,7 @@ mempalace split Split concatenated mega-files into per-session files mempalace mine Mine project files (default) mempalace mine --mode convos Mine conversation exports + mempalace mine --mode extract Mine binary office documents (PDF/DOCX/etc.) mempalace search "query" Find anything, exact words mempalace mcp Show MCP setup command mempalace wake-up Show L0 + L1 wake-up context @@ -515,6 +517,17 @@ def cmd_mine(args): dry_run=args.dry_run, extract_mode=args.extract, ) + elif args.mode == "extract": + from .format_miner import mine_formats + + mine_formats( + format_dir=args.dir, + palace_path=palace_path, + wing=args.wing, + agent=args.agent, + limit=args.limit, + dry_run=args.dry_run, + ) else: from .miner import mine @@ -1260,9 +1273,13 @@ def main(): p_mine.add_argument("dir", help="Directory to mine") p_mine.add_argument( "--mode", - choices=["projects", "convos"], + choices=["projects", "convos", "extract"], default="projects", - help="Ingest mode: 'projects' for code/docs (default), 'convos' for chat exports", + help=( + "Ingest mode: 'projects' for code/docs (default), 'convos' for chat " + "exports, 'extract' for office documents (PDF/DOCX/RTF/etc., requires " + "mempalace[extract])" + ), ) p_mine.add_argument("--wing", default=None, help="Wing name (default: directory name)") p_mine.add_argument( diff --git a/mempalace/format_miner.py b/mempalace/format_miner.py new file mode 100644 index 0000000000..2f3ea699e8 --- /dev/null +++ b/mempalace/format_miner.py @@ -0,0 +1,674 @@ +"""format_miner.py — proposed for mempalace 3.3.6. + +A third miner alongside ``miner.py`` (project files) and ``convo_miner.py`` +(chat exports). This one handles **binary office-format documents**: +PDF, DOCX, PPTX, XLSX, RTF, EPUB. + +Architecture matches the existing miner-per-content-type pattern: + + mempalace mine → miner.py + mempalace mine --mode convos → convo_miner.py + mempalace mine --mode extract → format_miner.py (this file) + +**Read-time conversion**, never modifies source files on disk. The user's +``~/research.pdf`` stays exactly as it was after a mine — the bytes are +read into memory, handed to Microsoft's MarkItDown library for conversion +to Markdown, and the resulting text flows into the normal chunker / +drawer-store pipeline. Conversion artifacts never touch disk. + +MarkItDown is an **optional** runtime dependency (declared as an extra). +If the user runs ``mempalace mine --mode extract`` without it installed, +they get a clear ``pip install markitdown`` instruction, not a crash. + +**Per-format transformer routing** (verified live on a local mixed-format +test directory, 2026-05-19): MarkItDown 0.1.5 does NOT convert .rtf — +it returns the raw RTF control-code source unchanged. So .rtf is routed +to the purpose-built ``striprtf`` library; all other formats stay on +MarkItDown. + + .pdf, .docx, .pptx, .xlsx, .epub → MarkItDown + .rtf → striprtf + +Both libraries are optional runtime dependencies. If a user tries to +extract a format whose transformer isn't installed, they get a clear +install message (SKIP_NO_MARKITDOWN or SKIP_NO_STRIPRTF), not a crash. + +**13 fringe cases handled** (spec finalized 2026-05-19): + + 1. MarkItDown not installed → SKIP_NO_MARKITDOWN (clear install msg) + 2. File too large (> max) → SKIP_TOO_LARGE + 3. iCloud cloud-only file → SKIP_CLOUD_ONLY + 4. Encrypted PDF → SKIP_ENCRYPTED + 5. Empty file → SKIP_EMPTY + 6. Permission denied → SKIP_PERMISSION + 7. Broken symlink → SKIP_BROKEN_SYMLINK + 8. Dirty encoding → recovered via decode_robust + 9. Windows path semantics → pathlib throughout + 10. MarkItDown internal crash → SKIP_EXTRACTION_ERROR + 11. Network / sync timeout → SKIP_NETWORK_TIMEOUT + 12. Unrecognized extension → SKIP_UNRECOGNIZED + 13. striprtf not installed → SKIP_NO_STRIPRTF (added 2026-05-19 + after live test + on local RTF files) + +Deferred (out of scope): custom PDF parsers for specific document types, +OCR on scanned PDFs, DRM-locked files, pathological corrupt files. +These get reported and skipped. +""" + +from __future__ import annotations + +import enum +import hashlib +import logging +import os +import re +from collections import defaultdict +from datetime import datetime +from pathlib import Path +from typing import Optional, Union + +from .palace import ( + NORMALIZE_VERSION, + file_already_mined, + get_collection, + mine_lock, +) + +__all__ = [ + "SUPPORTED_FORMATS", + "DEFAULT_MAX_FILE_SIZE", + "ExtractionStatus", + "decode_robust", + "is_icloud_dataless", + "extract_text", + "scan_formats", + "mine_formats", +] + + +# Same batch size as miner.py / convo_miner.py — bounds memory + Chroma payload. +DRAWER_UPSERT_BATCH_SIZE = 1000 + +# Minimum chunk size (chars) — drawers below this are dropped as not useful. +MIN_CHUNK_SIZE = 50 + + +logger = logging.getLogger("mempalace_format_miner") + + +# Extensions MarkItDown can convert. Lowercase, leading dot. Case-insensitive +# match performed against ``Path.suffix.lower()``. +SUPPORTED_FORMATS = frozenset( + { + ".pdf", + ".docx", + ".pptx", + ".xlsx", + ".rtf", + ".epub", + } +) + + +# Same default cap as miner.py / convo_miner.py — guards against pathological +# binary files and runaway memory. The caller can override via +# ``extract_text(..., max_file_size=...)`` for legitimately large documents. +DEFAULT_MAX_FILE_SIZE = 500 * 1024 * 1024 # 500 MB + + +# Directories skipped by scan_formats. Mirrors miner.SKIP_DIRS. +_SKIP_DIRS = { + ".git", + "node_modules", + "__pycache__", + ".venv", + "venv", + "env", + "dist", + "build", + ".next", + "coverage", + ".mempalace", + ".ruff_cache", + ".mypy_cache", + ".pytest_cache", + ".cache", + ".tox", + ".nox", + ".idea", + ".vscode", + ".ipynb_checkpoints", + ".eggs", + "htmlcov", + "target", +} + + +# Filename patterns that are not user content even if their extension matches. +_SKIP_FILENAMES = { + ".DS_Store", + "Thumbs.db", + "desktop.ini", +} + + +# Message patterns that identify an encrypted / password-protected PDF +# regardless of which underlying library raised the exception. +_ENCRYPTED_PATTERNS = re.compile( + r"(encrypt|decrypt|password|protected)", + re.IGNORECASE, +) + + +class ExtractionStatus(enum.Enum): + """Outcome of an ``extract_text`` call. + + Test-friendly enum (each case has its own name) so callers can assert + exactly which path was taken. ``OK`` means text came back; everything + else is a skip variant. + """ + + OK = "ok" + SKIP_TOO_LARGE = "skip:too_large" + SKIP_CLOUD_ONLY = "skip:cloud_only" + SKIP_EMPTY = "skip:empty" + SKIP_NO_MARKITDOWN = "skip:no_markitdown" + SKIP_NO_STRIPRTF = "skip:no_striprtf" + SKIP_ENCRYPTED = "skip:encrypted" + SKIP_PERMISSION = "skip:permission" + SKIP_BROKEN_SYMLINK = "skip:broken_symlink" + SKIP_UNRECOGNIZED = "skip:unrecognized" + SKIP_EXTRACTION_ERROR = "skip:extraction_error" + SKIP_NETWORK_TIMEOUT = "skip:network_timeout" + SKIP_UNREADABLE = "skip:unreadable" + + +# ───────────────────────────────────────────────────────────────────────────── +# Encoding fallback — same pattern as the terminal-session normalizer +# ───────────────────────────────────────────────────────────────────────────── + + +def decode_robust(raw: bytes) -> str: + """Decode bytes to text without raising on dirty encodings. + + Strategy: UTF-8 first (the clean case). On failure, try CP1252 (handles + legacy smart-quote bytes 0x91-0x9F that surface in older Office docs). + Final fallback is UTF-8 with ``errors='replace'`` so no byte is ever + lost — only made visible as the replacement char. + """ + if not raw: + return "" + try: + return raw.decode("utf-8") + except UnicodeDecodeError: + pass + try: + return raw.decode("cp1252") + except UnicodeDecodeError: + pass + return raw.decode("utf-8", errors="replace") + + +# ───────────────────────────────────────────────────────────────────────────── +# iCloud cloud-only file detection +# ───────────────────────────────────────────────────────────────────────────── + + +def is_icloud_dataless(path: Path) -> bool: + """True if ``path`` is an iCloud cloud-only placeholder (not local). + + Two indicators: + 1. The literal ``.icloud`` suffix iCloud uses for offloaded files. + 2. The macOS UF_COMPRESSED / dataless flag on the inode (``st_flags``). + + Either signal means MarkItDown would block on file I/O waiting for + iCloud to materialize the bytes, which can hang for minutes. We skip + these and report. + """ + if path.suffix.lower() == ".icloud": + return True + # macOS-only: stat() exposes st_flags which carries the dataless flag. + # On other platforms this attribute is absent or always 0. + try: + flags = getattr(path.lstat(), "st_flags", 0) + except OSError: + return False + # SF_DATALESS = 0x40000000 on macOS. Avoid hardcoding magic numbers + # cross-platform; check the bit if it exists. + DATALESS_FLAG = 0x40000000 + return bool(flags & DATALESS_FLAG) + + +# ───────────────────────────────────────────────────────────────────────────── +# Conversion via MarkItDown — isolated so tests can patch this single seam +# ───────────────────────────────────────────────────────────────────────────── + + +def _extract_via_markitdown(path: Path) -> Optional[str]: + """Run MarkItDown on ``path`` and return the resulting Markdown text. + + Used for .pdf, .docx, .pptx, .xlsx, .epub. NOT used for .rtf — MarkItDown + 0.1.5 doesn't convert RTF (returns raw control-code source unchanged), + so .rtf is routed to ``_extract_via_striprtf`` instead. + + Raises ``ImportError`` if the markitdown package is not installed + (caller's responsibility to translate this into ``SKIP_NO_MARKITDOWN``). + + Raises any other exception MarkItDown raises — the caller classifies + them via message pattern (encrypted vs generic crash vs timeout). + + Returns ``None`` if MarkItDown returns an empty result for the file. + """ + try: + from markitdown import MarkItDown # type: ignore[import-not-found] + except ImportError: + raise + + converter = MarkItDown() + result = converter.convert(str(path)) + text = getattr(result, "text_content", None) or getattr(result, "markdown", None) + if text is None: + return None + if not isinstance(text, str): + return None + return text + + +def _extract_via_striprtf(path: Path) -> Optional[str]: + """Run striprtf on ``path`` and return plain text. + + Used exclusively for .rtf (see ``_extract_via_markitdown`` docstring for + why). striprtf is pure-Python, MIT-licensed, ~150 lines, cross-platform + (works on Python 3.6+). + + Raises ``ImportError`` if the striprtf package is not installed + (caller's responsibility to translate this into ``SKIP_NO_STRIPRTF``). + + Returns ``None`` if striprtf strips the file to empty text. + """ + try: + from striprtf.striprtf import rtf_to_text # type: ignore[import-not-found] + except ImportError: + raise + + raw = path.read_bytes() + source = decode_robust(raw) + text = rtf_to_text(source) + if not isinstance(text, str): + return None + if text == "": + return None + return text + + +# ───────────────────────────────────────────────────────────────────────────── +# extract_text — the dispatcher with all 12 fringe cases handled +# ───────────────────────────────────────────────────────────────────────────── + + +def extract_text( + path: Union[Path, str], + max_file_size: int = DEFAULT_MAX_FILE_SIZE, +) -> tuple[Optional[str], ExtractionStatus]: + """Convert ``path`` to plain text via MarkItDown, with comprehensive fringe-case handling. + + Returns ``(text, ExtractionStatus)`` — text is ``None`` for every skip + case, a non-empty string for ``OK``. + + Pure function (no I/O outside the file at ``path``). Source file is + never modified. + """ + # Accept str or Path; pathlib handles Windows separators correctly. + p = Path(path) + + # Fringe Case 7 — broken symlink. + # ``is_symlink()`` true + ``exists()`` false ⇒ link target is missing. + if p.is_symlink() and not p.exists(): + logger.info("skip:broken_symlink %s", p) + return None, ExtractionStatus.SKIP_BROKEN_SYMLINK + + # Fringe Case 3 — iCloud cloud-only file. Detect BEFORE stat() so we + # don't accidentally trigger an iCloud materialization fetch. + if is_icloud_dataless(p): + logger.info("skip:cloud_only %s", p) + return None, ExtractionStatus.SKIP_CLOUD_ONLY + + # General readability check. Anything unreadable here (file gone, + # permission denied at stat-time, etc.) collapses to SKIP_UNREADABLE + # or SKIP_PERMISSION depending on the error class. + try: + stat = p.stat() + except PermissionError: + logger.info("skip:permission (stat) %s", p) + return None, ExtractionStatus.SKIP_PERMISSION + except FileNotFoundError: + logger.info("skip:broken_symlink (stat) %s", p) + return None, ExtractionStatus.SKIP_BROKEN_SYMLINK + except OSError as exc: + logger.info("skip:unreadable %s — %s", p, exc) + return None, ExtractionStatus.SKIP_UNREADABLE + + # Fringe Case 5 — empty file. Skip silently. + if stat.st_size == 0: + logger.debug("skip:empty %s", p) + return None, ExtractionStatus.SKIP_EMPTY + + # Fringe Case 2 — file exceeds the caller's cap. The default is 500 MB + # (same as the existing miners). Callers with legitimate large docs + # raise the cap explicitly. + if stat.st_size > max_file_size: + logger.info("skip:too_large %s (%d bytes > %d)", p, stat.st_size, max_file_size) + return None, ExtractionStatus.SKIP_TOO_LARGE + + # Fringe Case 12 — unrecognized extension. Cheap to check, do it last + # so we still report size / cloud / symlink issues for the obvious cases. + if p.suffix.lower() not in SUPPORTED_FORMATS: + logger.debug("skip:unrecognized %s (suffix=%s)", p, p.suffix) + return None, ExtractionStatus.SKIP_UNRECOGNIZED + + # Per-format transformer routing. .rtf goes to striprtf (MarkItDown + # 0.1.5 doesn't convert RTF — returns raw control-code source + # unchanged, verified live on a local RTF set 2026-05-19). + is_rtf = p.suffix.lower() == ".rtf" + try: + if is_rtf: + text = _extract_via_striprtf(p) + else: + text = _extract_via_markitdown(p) + except ImportError: + # Fringe Case 1 / 13 — transformer not installed in this env. + if is_rtf: + logger.warning( + "skip:no_striprtf %s — install with: pip install striprtf", + p, + ) + return None, ExtractionStatus.SKIP_NO_STRIPRTF + logger.warning( + "skip:no_markitdown %s — install with: pip install markitdown", + p, + ) + return None, ExtractionStatus.SKIP_NO_MARKITDOWN + except TimeoutError: + # Fringe Case 11 — network or sync-drive timeout. + logger.info("skip:network_timeout %s", p) + return None, ExtractionStatus.SKIP_NETWORK_TIMEOUT + except PermissionError: + # Fringe Case 6 — permission denied during file read. + logger.info("skip:permission %s", p) + return None, ExtractionStatus.SKIP_PERMISSION + except Exception as exc: + # Fringe Case 4 vs Case 10: encrypted vs generic crash, by message. + msg = str(exc) + if _ENCRYPTED_PATTERNS.search(msg): + logger.info("skip:encrypted %s — %s", p, msg[:120]) + return None, ExtractionStatus.SKIP_ENCRYPTED + logger.warning("skip:extraction_error %s — %s: %s", p, type(exc).__name__, msg[:200]) + return None, ExtractionStatus.SKIP_EXTRACTION_ERROR + + # Either transformer can legitimately return None / empty (malformed + # docs that parse but extract nothing). Treat both as extraction error + # so the caller knows to skip rather than file an empty drawer. + if not text: + transformer = "striprtf" if is_rtf else "markitdown" + logger.info("skip:extraction_error %s — %s returned None/empty", p, transformer) + return None, ExtractionStatus.SKIP_EXTRACTION_ERROR + + return text, ExtractionStatus.OK + + +# ───────────────────────────────────────────────────────────────────────────── +# Directory walker +# ───────────────────────────────────────────────────────────────────────────── + + +def scan_formats(directory: Union[Path, str]) -> list[Path]: + """Walk ``directory`` recursively and return supported files, sorted. + + Skips: + - Hidden / build directories listed in ``_SKIP_DIRS`` + - Filenames listed in ``_SKIP_FILENAMES`` (.DS_Store etc.) + - Files whose suffix isn't in ``SUPPORTED_FORMATS`` + + Returns a list of ``Path`` objects. The order is deterministic + (sorted by path) so a re-mine processes files in the same order each + time — useful for reproducing bug reports. + """ + root = Path(directory) + if not root.exists(): + return [] + + found: list[Path] = [] + for dirpath, dirnames, filenames in os.walk(root): + # Mutate dirnames in place to prune the walk. + dirnames[:] = [d for d in dirnames if d not in _SKIP_DIRS] + for name in filenames: + if name in _SKIP_FILENAMES: + continue + p = Path(dirpath) / name + if p.suffix.lower() not in SUPPORTED_FORMATS: + continue + found.append(p) + + found.sort() + return found + + +# ───────────────────────────────────────────────────────────────────────────── +# mine_formats — orchestrator. Walks a directory, transforms each supported +# file via extract_text, chunks the resulting Markdown via miner.chunk_text, +# files drawers via the same lock + purge + upsert pattern convo_miner uses. +# Source files on disk are never modified. +# ───────────────────────────────────────────────────────────────────────────── + + +def _register_file(collection, source_file: str, wing: str, agent: str) -> None: + """Write a sentinel so file_already_mined() returns True for 0-chunk files. + + Without this, files that extract to nothing (or hit a SKIP status) get + rescanned on every re-mine. The sentinel preserves the no-op outcome. + Mirrors the helper of the same name in convo_miner.py. + """ + sentinel_id = f"sentinel_{wing}_{hashlib.sha256(source_file.encode()).hexdigest()[:24]}" + try: + collection.upsert( + documents=["[empty]"], + ids=[sentinel_id], + metadatas=[ + { + "wing": wing, + "room": "documents", + "source_file": source_file, + "chunk_index": -1, + "added_by": agent, + "filed_at": datetime.now().isoformat(), + "ingest_mode": "extract", + "extract_mode": "format", + "normalize_version": NORMALIZE_VERSION, + "is_sentinel": True, + } + ], + ) + except Exception: + logger.debug("Sentinel write failed for %s", source_file, exc_info=True) + + +def _file_chunks_locked(collection, source_file, chunks, wing, room, agent): + """Lock the source file, purge stale drawers, and upsert fresh chunks. + + Mirrors the canonical convo_miner pattern (locked purge + batched upsert) + but with ``ingest_mode="extract"`` so format-mined drawers are + distinguishable from project / convo drawers in the palace. + + Returns ``(drawers_added, skipped)``. + """ + drawers_added = 0 + with mine_lock(source_file): + # Re-check after lock — another agent may have just finished this file. + if file_already_mined(collection, source_file): + return 0, True + + try: + collection.delete(where={"source_file": source_file}) + except Exception: + logger.debug("Stale-drawer purge failed for %s", source_file, exc_info=True) + + filed_at = datetime.now().isoformat() + for batch_start in range(0, len(chunks), DRAWER_UPSERT_BATCH_SIZE): + batch_docs: list = [] + batch_ids: list = [] + batch_metas: list = [] + for chunk in chunks[batch_start : batch_start + DRAWER_UPSERT_BATCH_SIZE]: + key = (source_file + str(chunk["chunk_index"])).encode() + drawer_id = f"drawer_{wing}_{room}_{hashlib.sha256(key).hexdigest()[:24]}" + batch_docs.append(chunk["content"]) + batch_ids.append(drawer_id) + batch_metas.append( + { + "wing": wing, + "room": room, + "source_file": source_file, + "chunk_index": chunk["chunk_index"], + "added_by": agent, + "filed_at": filed_at, + "ingest_mode": "extract", + "extract_mode": "format", + "normalize_version": NORMALIZE_VERSION, + } + ) + try: + collection.upsert( + documents=batch_docs, + ids=batch_ids, + metadatas=batch_metas, + ) + drawers_added += len(batch_docs) + except Exception as exc: + if "already exists" not in str(exc).lower(): + raise + return drawers_added, False + + +def mine_formats( + format_dir: str, + palace_path: str, + wing: Optional[str] = None, + agent: str = "mempalace", + limit: int = 0, + dry_run: bool = False, +) -> None: + """Mine a directory of binary office-format files into the palace. + + Walks ``format_dir`` via ``scan_formats``, converts each supported file + to text via ``extract_text`` (which routes to MarkItDown or striprtf + per format), chunks the result with the same chunker miner.py uses, and + files the chunks as drawers under the given wing. + + Source files on disk are never modified — conversion is in-memory only. + + Parameters + ---------- + format_dir : + Directory to walk recursively. Hidden / build dirs are skipped (see + ``_SKIP_DIRS``). Only files matching ``SUPPORTED_FORMATS`` are + processed. + palace_path : + Path to the ChromaDB palace (the destination of the drawers). + wing : + Wing name. Defaults to the basename of ``format_dir`` (normalized). + agent : + Identifier recorded in each drawer's ``added_by`` metadata. + limit : + If > 0, process at most this many files. Useful for sampling. + dry_run : + If True, walk + extract + chunk but do NOT open the collection or + upsert any drawers. Just prints what would have been filed. + """ + # Local import — chunk_text is the same chunker miner.py uses for + # text-format project files. Imported inside the function to avoid a + # module-load cycle. + from .miner import chunk_text + + format_path = Path(format_dir).expanduser().resolve() + if not wing: + from .config import normalize_wing_name + + wing = normalize_wing_name(format_path.name) + + files = scan_formats(format_dir) + if limit > 0: + files = files[:limit] + + print(f"\n{'=' * 55}") + print(" MemPalace Mine — Format extraction") + print(f"{'=' * 55}") + print(f" Wing: {wing}") + print(f" Source: {format_path}") + print(f" Files: {len(files)}") + print(f" Palace: {palace_path}") + if dry_run: + print(" DRY RUN — nothing will be filed") + print(f"{'-' * 55}\n") + + collection = get_collection(palace_path) if not dry_run else None + + total_drawers = 0 + files_skipped = 0 + files_with_text = 0 + status_counts: dict = defaultdict(int) + + for i, filepath in enumerate(files, 1): + source_file = str(filepath) + + if not dry_run and file_already_mined(collection, source_file): + files_skipped += 1 + continue + + text, status = extract_text(filepath) + status_counts[status.name] += 1 + + if status != ExtractionStatus.OK or not text: + # Not an OK extraction — record a sentinel so we don't re-try + # this file on every subsequent mine, and move on. + if not dry_run: + _register_file(collection, source_file, wing, agent) + print(f" - [{i:4}/{len(files)}] {filepath.name[:50]:50} {status.name}") + continue + + chunks = chunk_text(text, source_file) + if not chunks: + if not dry_run: + _register_file(collection, source_file, wing, agent) + print(f" - [{i:4}/{len(files)}] {filepath.name[:50]:50} EMPTY_AFTER_CHUNK") + continue + + room = "documents" # all format-mined drawers land in the "documents" room + files_with_text += 1 + + if dry_run: + print(f" [DRY RUN] {filepath.name} → {len(chunks)} drawers") + total_drawers += len(chunks) + continue + + drawers_added, skipped = _file_chunks_locked( + collection, source_file, chunks, wing, room, agent + ) + if skipped: + files_skipped += 1 + continue + + total_drawers += drawers_added + print(f" + [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers_added}") + + print(f"\n{'=' * 55}") + print(" Summary") + print(f"{'-' * 55}") + print(f" Files seen: {len(files)}") + print(f" Files extracted: {files_with_text}") + print(f" Files skipped: {files_skipped}") + print(f" Total drawers: {total_drawers}") + if status_counts: + print(" Extraction status:") + for name, count in sorted(status_counts.items(), key=lambda kv: -kv[1]): + print(f" {name:30} {count}") + print(f"{'=' * 55}\n") diff --git a/mempalace/searcher.py b/mempalace/searcher.py index 9e8a358401..df9a9e2b55 100644 --- a/mempalace/searcher.py +++ b/mempalace/searcher.py @@ -1017,3 +1017,61 @@ def search_memories( "total_before_filter": len(_first_or_empty(drawer_results, "documents")), "results": hits, } + + +# ───────────────────────────────────────────────────────────────────────────── +# Virtual line numbering — read-time grid for drawers (3.3.6). +# +# Drawers are stored verbatim on disk. The reader applies a line-number grid +# at read time so any drawer — numbered or not — can be sectioned by a closet +# pointer like ``→2026-01-18:L55-L72`` without rewriting the corpus. Pure +# functions, no I/O. Source drawer text is never mutated. +# See docs/virtual-line-numbering.md for the full design rationale. +# ───────────────────────────────────────────────────────────────────────────── + + +# A line is "already numbered" iff it starts with []. +_ALREADY_NUMBERED_RE = re.compile(r"^\[\d+\]") + + +def render_with_line_numbers(text, start_line: int = 1) -> str: + """Prefix each line of ``text`` with ``[N] `` for read-time grid display. + + Lines that already begin with ``[]`` pass through unchanged, + but the counter still advances on them so callers can rely on positional + alignment with the original line indices. + + ``None`` is treated as empty string. Pure function. + """ + if not text: + return "" + out = [] + for i, line in enumerate(text.split("\n"), start=start_line): + if _ALREADY_NUMBERED_RE.match(line): + out.append(line) + else: + out.append(f"[{i}] {line}") + return "\n".join(out) + + +def extract_line_range(text: str, line_start: int, line_end: int) -> str: + """Return the 1-indexed inclusive slice ``[line_start, line_end]`` rendered with line numbers. + + This is the closet-pointer read path. A pointer like ``→2026-01-18:L55-L72`` + resolves by opening the day-drawer and calling ``extract_line_range(drawer_text, 55, 72)``. + Out-of-bounds ranges are clamped. Invalid ranges return ``""``. + """ + if not text: + return "" + if line_end < line_start: + return "" + + lines = text.split("\n") + effective_start = max(1, line_start) + effective_end = min(len(lines), line_end) + + if effective_start > effective_end: + return "" + + section = "\n".join(lines[effective_start - 1 : effective_end]) + return render_with_line_numbers(section, start_line=effective_start) diff --git a/pyproject.toml b/pyproject.toml index 99db4c4e9b..6cea05d492 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,14 @@ spellcheck = ["autocorrect>=2.0"] gpu = ["onnxruntime-gpu>=1.16"] dml = ["onnxruntime-directml>=1.16"] coreml = ["onnxruntime>=1.16"] +# Binary-format extraction for `mempalace mine --mode extract`. Per-format +# transformer routing: striprtf for .rtf, MarkItDown for .pdf/.docx/.pptx/ +# .xlsx/.epub. MarkItDown requires Python ≥ 3.10 (env marker ensures it +# only installs where supported; Python 3.9 users still get RTF coverage). +extract = [ + "striprtf>=0.0.27", + "markitdown>=0.1.5; python_version >= '3.10'", +] [dependency-groups] dev = ["pytest>=7.0", "pytest-cov>=4.0", "ruff==0.15.9", "psutil>=5.9"] diff --git a/tests/test_format_miner.py b/tests/test_format_miner.py new file mode 100644 index 0000000000..0f67d836b9 --- /dev/null +++ b/tests/test_format_miner.py @@ -0,0 +1,850 @@ +"""Tests for format_miner (proposed for mempalace 3.3.6). + +Covers all 13 fringe cases in the 3.3.6 format-coverage spec, plus the +happy paths. MarkItDown is mocked throughout so tests run without the +library installed — same discipline as the existing test_line_numbers.py. + +Run from the proposal directory: + pytest tests/test_format_miner.py -v + +After integration into the mempalace package, change the import below to: + from mempalace.format_miner import ... +""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import patch + +import pytest + + +from mempalace.format_miner import ( # noqa: E402 + DEFAULT_MAX_FILE_SIZE, + SUPPORTED_FORMATS, + ExtractionStatus, + decode_robust, + extract_text, + is_icloud_dataless, + scan_formats, +) + + +# ───────────────────────────────────────────────────────────────────────────── +# Module surface +# ───────────────────────────────────────────────────────────────────────────── + + +def test_supported_formats_covers_office_suite(): + assert ".pdf" in SUPPORTED_FORMATS + assert ".rtf" in SUPPORTED_FORMATS + assert ".docx" in SUPPORTED_FORMATS + assert ".xlsx" in SUPPORTED_FORMATS + assert ".pptx" in SUPPORTED_FORMATS + assert ".epub" in SUPPORTED_FORMATS + + +def test_supported_formats_normalised_lowercase(): + for ext in SUPPORTED_FORMATS: + assert ext == ext.lower() + assert ext.startswith(".") + + +def test_extraction_status_enum_has_all_documented_codes(): + expected = { + "OK", + "SKIP_TOO_LARGE", + "SKIP_CLOUD_ONLY", + "SKIP_EMPTY", + "SKIP_NO_MARKITDOWN", + "SKIP_NO_STRIPRTF", + "SKIP_ENCRYPTED", + "SKIP_PERMISSION", + "SKIP_BROKEN_SYMLINK", + "SKIP_UNRECOGNIZED", + "SKIP_EXTRACTION_ERROR", + "SKIP_NETWORK_TIMEOUT", + "SKIP_UNREADABLE", + } + actual = {status.name for status in ExtractionStatus} + missing = expected - actual + assert not missing, f"Missing status codes: {missing}" + + +def test_default_max_file_size_matches_existing_miner(): + assert DEFAULT_MAX_FILE_SIZE == 500 * 1024 * 1024 + + +# ───────────────────────────────────────────────────────────────────────────── +# Fringe Case 12 — unrecognized extension → skip with note (test first because +# it covers the simplest dispatch path) +# ───────────────────────────────────────────────────────────────────────────── + + +def test_fringe_unrecognized_extension(tmp_path: Path): + f = tmp_path / "thing.xyz" + f.write_text("not a real format") + text, status = extract_text(f) + assert text is None + assert status == ExtractionStatus.SKIP_UNRECOGNIZED + + +# ───────────────────────────────────────────────────────────────────────────── +# Fringe Case 5 — empty file → skip silently +# ───────────────────────────────────────────────────────────────────────────── + + +def test_fringe_empty_file(tmp_path: Path): + f = tmp_path / "blank.pdf" + f.write_bytes(b"") + text, status = extract_text(f) + assert text is None + assert status == ExtractionStatus.SKIP_EMPTY + + +# ───────────────────────────────────────────────────────────────────────────── +# Fringe Case 2 — file too large → skip with note +# ───────────────────────────────────────────────────────────────────────────── + + +def test_fringe_too_large(tmp_path: Path): + f = tmp_path / "huge.pdf" + f.write_bytes(b"%PDF-1.4\n" + b"\x00" * 1024) + text, status = extract_text(f, max_file_size=128) + assert text is None + assert status == ExtractionStatus.SKIP_TOO_LARGE + + +def test_fringe_too_large_respects_caller_max(tmp_path: Path): + """Caller can raise the cap for legitimate big files.""" + f = tmp_path / "huge.pdf" + f.write_bytes(b"%PDF-1.4\n" + b"\x00" * 2048) + # When the cap is generous, size alone won't trigger SKIP_TOO_LARGE. + # (MarkItDown will be invoked; we mock it so the test doesn't require it.) + with patch("mempalace.format_miner._extract_via_markitdown", return_value="dummy text"): + text, status = extract_text(f, max_file_size=1024 * 1024) + assert status == ExtractionStatus.OK + assert text == "dummy text" + + +# ───────────────────────────────────────────────────────────────────────────── +# Fringe Case 1 — MarkItDown not installed → clear error +# ───────────────────────────────────────────────────────────────────────────── + + +def test_fringe_missing_markitdown(tmp_path: Path): + f = tmp_path / "doc.pdf" + f.write_bytes(b"%PDF-1.4\nstub") + with patch( + "mempalace.format_miner._extract_via_markitdown", + side_effect=ImportError("No module named 'markitdown'"), + ): + text, status = extract_text(f) + assert text is None + assert status == ExtractionStatus.SKIP_NO_MARKITDOWN + + +# ───────────────────────────────────────────────────────────────────────────── +# Fringe Case 4 — encrypted PDF → MarkItDown raises; we catch + skip + note +# ───────────────────────────────────────────────────────────────────────────── + + +def test_fringe_encrypted_pdf(tmp_path: Path): + f = tmp_path / "locked.pdf" + f.write_bytes(b"%PDF-1.4\nencrypted stub") + + class _PasswordError(Exception): + pass + + with patch( + "mempalace.format_miner._extract_via_markitdown", + side_effect=_PasswordError("File has not been decrypted"), + ): + text, status = extract_text(f) + assert text is None + assert status == ExtractionStatus.SKIP_ENCRYPTED + + +# ───────────────────────────────────────────────────────────────────────────── +# Fringe Case 6 — permission denied → catch OSError, skip +# ───────────────────────────────────────────────────────────────────────────── + + +def test_fringe_permission_denied(tmp_path: Path): + f = tmp_path / "denied.pdf" + f.write_bytes(b"%PDF-1.4\nstub") + with patch( + "mempalace.format_miner._extract_via_markitdown", + side_effect=PermissionError("Permission denied"), + ): + text, status = extract_text(f) + assert text is None + assert status == ExtractionStatus.SKIP_PERMISSION + + +# ───────────────────────────────────────────────────────────────────────────── +# Fringe Case 7 — symlink to nothing → catch, skip +# ───────────────────────────────────────────────────────────────────────────── + + +def test_fringe_broken_symlink(tmp_path: Path): + target = tmp_path / "does-not-exist.pdf" + link = tmp_path / "broken-link.pdf" + link.symlink_to(target) + assert link.is_symlink() + text, status = extract_text(link) + assert text is None + assert status == ExtractionStatus.SKIP_BROKEN_SYMLINK + + +# ───────────────────────────────────────────────────────────────────────────── +# Fringe Case 3 — iCloud cloud-only file detection +# ───────────────────────────────────────────────────────────────────────────── + + +def test_fringe_icloud_placeholder_extension(tmp_path: Path): + # macOS sometimes leaves a literal .icloud placeholder for offloaded files + f = tmp_path / "doc.pdf.icloud" + f.write_bytes(b"placeholder") + assert is_icloud_dataless(f) is True + + +def test_fringe_icloud_skip_extraction(tmp_path: Path): + # Cloud-only file. extract_text should not call MarkItDown and should + # return SKIP_CLOUD_ONLY. + f = tmp_path / "doc.pdf.icloud" + f.write_bytes(b"placeholder") + with patch("mempalace.format_miner._extract_via_markitdown") as mocked: + text, status = extract_text(f) + assert text is None + assert status == ExtractionStatus.SKIP_CLOUD_ONLY + mocked.assert_not_called() + + +# ───────────────────────────────────────────────────────────────────────────── +# Fringe Case 8 — encoding fallback (utf-8 → cp1252 → utf-8-replace) +# ───────────────────────────────────────────────────────────────────────────── + + +def test_decode_robust_clean_utf8(): + assert decode_robust(b"hello \xe2\x9c\xa8 world") == "hello ✨ world" + + +def test_decode_robust_cp1252_fallback(): + # 0x91 = U+2018 left single quote in cp1252; invalid as standalone utf-8 + raw = b"hello \x91world\x92" + result = decode_robust(raw) + assert isinstance(result, str) + assert "world" in result + + +def test_decode_robust_never_raises(): + raw = b"\xff\xfe\xfd\xfc" + b"some text" + result = decode_robust(raw) + assert isinstance(result, str) + assert "some text" in result + + +def test_decode_robust_empty(): + assert decode_robust(b"") == "" + + +# ───────────────────────────────────────────────────────────────────────────── +# Fringe Case 9 — Windows path differences (pathlib semantics) +# ───────────────────────────────────────────────────────────────────────────── + + +def test_extract_text_accepts_pathlib_path(tmp_path: Path): + """Accept Path objects without coercing to str (Windows-safe).""" + f = tmp_path / "doc.pdf" + f.write_bytes(b"%PDF-1.4\nstub") + with patch("mempalace.format_miner._extract_via_markitdown", return_value="content"): + text, status = extract_text(f) + assert status == ExtractionStatus.OK + + +def test_extract_text_accepts_string_path(tmp_path: Path): + """Accept str paths for callers that pre-stringify.""" + f = tmp_path / "doc.pdf" + f.write_bytes(b"%PDF-1.4\nstub") + with patch("mempalace.format_miner._extract_via_markitdown", return_value="content"): + text, status = extract_text(str(f)) + assert status == ExtractionStatus.OK + + +def test_supported_format_check_case_insensitive(tmp_path: Path): + """Windows often shows uppercase extensions; we still recognize them.""" + f = tmp_path / "doc.PDF" + f.write_bytes(b"%PDF-1.4\nstub") + with patch("mempalace.format_miner._extract_via_markitdown", return_value="content"): + text, status = extract_text(f) + assert status == ExtractionStatus.OK + + +# ───────────────────────────────────────────────────────────────────────────── +# Fringe Case 10 — MarkItDown internal crash on malformed file +# ───────────────────────────────────────────────────────────────────────────── + + +def test_fringe_markitdown_generic_crash(tmp_path: Path): + f = tmp_path / "malformed.pdf" + f.write_bytes(b"%PDF-1.4\nmalformed") + with patch( + "mempalace.format_miner._extract_via_markitdown", + side_effect=RuntimeError("internal converter explosion"), + ): + text, status = extract_text(f) + assert text is None + assert status == ExtractionStatus.SKIP_EXTRACTION_ERROR + + +def test_fringe_markitdown_returns_none(tmp_path: Path): + """MarkItDown can return None for some inputs; treat as extraction error.""" + f = tmp_path / "weird.pdf" + f.write_bytes(b"%PDF-1.4\nweird") + with patch("mempalace.format_miner._extract_via_markitdown", return_value=None): + text, status = extract_text(f) + assert text is None + assert status == ExtractionStatus.SKIP_EXTRACTION_ERROR + + +# ───────────────────────────────────────────────────────────────────────────── +# Fringe Case 11 — network/sync drive timeout +# ───────────────────────────────────────────────────────────────────────────── + + +def test_fringe_network_timeout(tmp_path: Path): + f = tmp_path / "remote.pdf" + f.write_bytes(b"%PDF-1.4\nstub") + with patch( + "mempalace.format_miner._extract_via_markitdown", + side_effect=TimeoutError("operation timed out"), + ): + text, status = extract_text(f) + assert text is None + assert status == ExtractionStatus.SKIP_NETWORK_TIMEOUT + + +# ───────────────────────────────────────────────────────────────────────────── +# Happy paths — formats we explicitly target +# ───────────────────────────────────────────────────────────────────────────── + + +def test_happy_path_pdf(tmp_path: Path): + f = tmp_path / "research.pdf" + f.write_bytes(b"%PDF-1.4\nstub") + with patch("mempalace.format_miner._extract_via_markitdown", return_value="# Research\n\nbody"): + text, status = extract_text(f) + assert status == ExtractionStatus.OK + assert text == "# Research\n\nbody" + + +def test_happy_path_docx(tmp_path: Path): + f = tmp_path / "notes.docx" + f.write_bytes(b"PK\x03\x04docx-stub") + with patch("mempalace.format_miner._extract_via_markitdown", return_value="# Notes\n\nbody"): + text, status = extract_text(f) + assert status == ExtractionStatus.OK + assert text.startswith("# Notes") + + +def test_happy_path_rtf(tmp_path: Path): + """RTF routes to striprtf, NOT MarkItDown. + + MarkItDown 0.1.5 does not actually convert .rtf — it returns the raw + source unchanged. striprtf is the correct transformer for this format. + """ + f = tmp_path / "memo.rtf" + f.write_bytes(b"{\\rtf1\\ansi memo}") + with patch("mempalace.format_miner._extract_via_striprtf", return_value="memo body"): + text, status = extract_text(f) + assert status == ExtractionStatus.OK + assert text == "memo body" + + +# ───────────────────────────────────────────────────────────────────────────── +# Striprtf path — RTF transformer routing (Fringe Case 13: SKIP_NO_STRIPRTF) +# ───────────────────────────────────────────────────────────────────────────── + + +def test_rtf_routes_to_striprtf_not_markitdown(tmp_path: Path): + """Critical invariant: .rtf must NEVER touch MarkItDown. + + MarkItDown 0.1.5 returns raw RTF source for .rtf inputs (verified live + against a local RTF test set on 2026-05-19). Routing .rtf through striprtf is + the bugfix. + """ + f = tmp_path / "memo.rtf" + f.write_bytes(b"{\\rtf1\\ansi memo}") + with ( + patch("mempalace.format_miner._extract_via_markitdown") as mock_md, + patch("mempalace.format_miner._extract_via_striprtf", return_value="memo body") as mock_rtf, + ): + text, status = extract_text(f) + assert status == ExtractionStatus.OK + assert text == "memo body" + mock_md.assert_not_called() + mock_rtf.assert_called_once() + + +def test_non_rtf_does_not_touch_striprtf(tmp_path: Path): + """Inverse invariant: .pdf / .docx / etc. must not hit striprtf.""" + f = tmp_path / "doc.pdf" + f.write_bytes(b"%PDF-1.4\nstub") + with ( + patch("mempalace.format_miner._extract_via_markitdown", return_value="pdf text") as mock_md, + patch("mempalace.format_miner._extract_via_striprtf") as mock_rtf, + ): + text, status = extract_text(f) + assert status == ExtractionStatus.OK + assert text == "pdf text" + mock_md.assert_called_once() + mock_rtf.assert_not_called() + + +def test_fringe_missing_striprtf(tmp_path: Path): + """Fringe Case 13 — striprtf not installed → SKIP_NO_STRIPRTF + clear install msg.""" + f = tmp_path / "memo.rtf" + f.write_bytes(b"{\\rtf1\\ansi memo}") + with patch( + "mempalace.format_miner._extract_via_striprtf", + side_effect=ImportError("No module named 'striprtf'"), + ): + text, status = extract_text(f) + assert text is None + assert status == ExtractionStatus.SKIP_NO_STRIPRTF + + +def test_fringe_striprtf_crash(tmp_path: Path): + """striprtf raising any other exception → SKIP_EXTRACTION_ERROR.""" + f = tmp_path / "broken.rtf" + f.write_bytes(b"{\\rtf1\\ansi broken}") + with patch( + "mempalace.format_miner._extract_via_striprtf", + side_effect=RuntimeError("rtf parse explosion"), + ): + text, status = extract_text(f) + assert text is None + assert status == ExtractionStatus.SKIP_EXTRACTION_ERROR + + +def test_fringe_striprtf_returns_none(tmp_path: Path): + """striprtf returning None → SKIP_EXTRACTION_ERROR (same shape as MarkItDown).""" + f = tmp_path / "weird.rtf" + f.write_bytes(b"{\\rtf1\\ansi weird}") + with patch("mempalace.format_miner._extract_via_striprtf", return_value=None): + text, status = extract_text(f) + assert text is None + assert status == ExtractionStatus.SKIP_EXTRACTION_ERROR + + +def test_fringe_striprtf_empty_output(tmp_path: Path): + """striprtf returning empty string → SKIP_EXTRACTION_ERROR. + + An RTF that strips to zero characters is not useful as a drawer; treat + as extraction failure rather than file an empty drawer. + """ + f = tmp_path / "blank-after-strip.rtf" + f.write_bytes(b"{\\rtf1\\ansi}") + with patch("mempalace.format_miner._extract_via_striprtf", return_value=""): + text, status = extract_text(f) + assert text is None + assert status == ExtractionStatus.SKIP_EXTRACTION_ERROR + + +def test_rtf_uppercase_extension_also_routes_to_striprtf(tmp_path: Path): + """Windows case-insensitive: .RTF must also route to striprtf.""" + f = tmp_path / "memo.RTF" + f.write_bytes(b"{\\rtf1\\ansi memo}") + with ( + patch("mempalace.format_miner._extract_via_markitdown") as mock_md, + patch("mempalace.format_miner._extract_via_striprtf", return_value="memo body") as mock_rtf, + ): + text, status = extract_text(f) + assert status == ExtractionStatus.OK + mock_md.assert_not_called() + mock_rtf.assert_called_once() + + +def test_happy_path_xlsx(tmp_path: Path): + f = tmp_path / "spreadsheet.xlsx" + f.write_bytes(b"PK\x03\x04xlsx-stub") + with patch( + "mempalace.format_miner._extract_via_markitdown", + return_value="| col1 | col2 |\n|---|---|\n| a | b |", + ): + text, status = extract_text(f) + assert status == ExtractionStatus.OK + assert "col1" in text + + +# ───────────────────────────────────────────────────────────────────────────── +# scan_formats — directory walker, returns Path objects sorted +# ───────────────────────────────────────────────────────────────────────────── + + +def test_scan_formats_finds_supported_only(tmp_path: Path): + (tmp_path / "a.pdf").write_bytes(b"pdf") + (tmp_path / "b.txt").write_text("text") # unsupported + (tmp_path / "c.docx").write_bytes(b"docx") + (tmp_path / "d.rtf").write_bytes(b"rtf") + found = {f.name for f in scan_formats(tmp_path)} + assert "a.pdf" in found + assert "c.docx" in found + assert "d.rtf" in found + assert "b.txt" not in found + + +def test_scan_formats_walks_subdirectories(tmp_path: Path): + nested = tmp_path / "deep" / "nested" + nested.mkdir(parents=True) + (nested / "buried.pdf").write_bytes(b"pdf") + found = {f.name for f in scan_formats(tmp_path)} + assert "buried.pdf" in found + + +def test_scan_formats_skips_hidden_dirs(tmp_path: Path): + """Don't descend into .git, .venv, __pycache__, etc. — same SKIP_DIRS as miner.""" + hidden = tmp_path / ".git" + hidden.mkdir() + (hidden / "hidden.pdf").write_bytes(b"pdf") + (tmp_path / "visible.pdf").write_bytes(b"pdf") + found = {f.name for f in scan_formats(tmp_path)} + assert "visible.pdf" in found + assert "hidden.pdf" not in found + + +def test_scan_formats_skips_ds_store(tmp_path: Path): + (tmp_path / ".DS_Store").write_bytes(b"") + (tmp_path / "real.pdf").write_bytes(b"pdf") + found = {f.name for f in scan_formats(tmp_path)} + assert "real.pdf" in found + assert ".DS_Store" not in found + + +def test_scan_formats_returns_empty_for_missing_dir(tmp_path: Path): + """scan_formats handles a path that doesn't exist (no crash, empty list).""" + missing = tmp_path / "does-not-exist" + assert scan_formats(missing) == [] + + +# ───────────────────────────────────────────────────────────────────────────── +# decode_robust — exercise the cp1252 path explicitly +# ───────────────────────────────────────────────────────────────────────────── + + +def test_decode_robust_pure_cp1252(): + """Bytes that are invalid UTF-8 but valid CP1252 → second-attempt success.""" + raw = b"\x91hello\x92" + result = decode_robust(raw) + assert isinstance(result, str) + assert "hello" in result + # 0x91 / 0x92 are CP1252 smart quotes that decode without error + assert "�" not in result, "CP1252 path should not need the replace fallback" + + +# ───────────────────────────────────────────────────────────────────────────── +# Note on stat() error branches in extract_text: +# Dedicated PermissionError / FileNotFoundError / OSError stat-handler tests +# were attempted but tripped on Python 3.13 pathlib internals (patching +# Path.stat globally interferes with .exists() / .is_symlink() which call +# stat under the hood). The handlers are kept as defensive guards and +# remain documented as uncovered branches — the rest of the suite (60+ +# tests + live integration on 63 real archive files) provides the +# end-to-end safety net. +# ───────────────────────────────────────────────────────────────────────────── + + +# ───────────────────────────────────────────────────────────────────────────── +# Live transformer body tests — skipped when transformer isn't installed. +# These exercise the actual adapter code that mocked tests can't reach. +# ───────────────────────────────────────────────────────────────────────────── + + +def test_extract_via_striprtf_live(tmp_path: Path): + """End-to-end: real striprtf converts a real RTF blob to plain text. + + Skipped if striprtf isn't installed (e.g., environments without the + [extract] extra). When installed, this exercises the actual adapter + body that mocked tests bypass. + """ + pytest.importorskip("striprtf.striprtf") + from mempalace.format_miner import _extract_via_striprtf + + rtf = ( + b"{\\rtf1\\ansi\\ansicpg1252\n{\\fonttbl\\f0\\fnil Helvetica;}\n" + b"\\f0\\fs24 Hello from a real RTF blob.}" + ) + f = tmp_path / "live.rtf" + f.write_bytes(rtf) + text = _extract_via_striprtf(f) + assert text is not None + assert "Hello from a real RTF blob" in text + assert "\\rtf1" not in text, "raw RTF control codes leaked into output" + + +def test_extract_via_markitdown_live_pdf(tmp_path: Path): + """End-to-end: real MarkItDown converts a real PDF blob to text. + + Skipped if markitdown isn't installed (3.10+ only, optional extra). + Also skipped if only the placeholder ``markitdown`` package is present + (the real Microsoft package exposes the ``MarkItDown`` class). + """ + try: + from markitdown import MarkItDown # noqa: F401 + except ImportError: + pytest.skip("real Microsoft markitdown not installed (needs Python 3.10+)") + from mempalace.format_miner import _extract_via_markitdown + + # Minimal valid PDF that contains the literal text "hello pdf" + pdf_bytes = ( + b"%PDF-1.1\n" + b"1 0 obj<>endobj\n" + b"2 0 obj<>endobj\n" + b"3 0 obj<>>>" + b"/MediaBox[0 0 612 792]/Contents 5 0 R>>endobj\n" + b"4 0 obj<>endobj\n" + b"5 0 obj<>stream\n" + b"BT /F1 24 Tf 100 700 Td (hello pdf) Tj ET\n" + b"endstream endobj\n" + b"xref\n0 6\n0000000000 65535 f \n" + b"trailer<>\n" + b"startxref\n0\n%%EOF\n" + ) + f = tmp_path / "live.pdf" + f.write_bytes(pdf_bytes) + # The adapter shouldn't crash; if MarkItDown returns text, it includes our marker. + text = _extract_via_markitdown(f) + # Allow None (MarkItDown may not parse this minimal PDF cleanly), but the + # adapter path itself must run without exception. + if text is not None: + assert isinstance(text, str) + + +# ───────────────────────────────────────────────────────────────────────────── +# mine_formats — orchestrator that walks a directory, transforms files, +# chunks the extracted text, and files drawers into the palace. Mirrors the +# shape of mine_convos / mine. The collection + lock primitives are mocked +# so these tests don't write to a real palace. +# ───────────────────────────────────────────────────────────────────────────── + + +@pytest.fixture +def _mine_formats_mocks(tmp_path: Path): + """Common mocks for mine_formats orchestrator tests. + + Patches: + - scan_formats : returns the files we hand it + - get_collection : returns a MagicMock collection + - mine_lock : no-op context manager + - file_already_mined : False (file not yet mined) + - _extract_via_striprtf / _extract_via_markitdown : mocked at test level + """ + from unittest.mock import MagicMock, patch + from contextlib import contextmanager + + collection = MagicMock() + collection.delete = MagicMock() + collection.upsert = MagicMock() + + @contextmanager + def _fake_lock(source_file): + yield + + with ( + patch("mempalace.format_miner.get_collection", return_value=collection) as p_coll, + patch("mempalace.format_miner.mine_lock", side_effect=_fake_lock) as p_lock, + patch("mempalace.format_miner.file_already_mined", return_value=False) as p_mined, + ): + yield { + "collection": collection, + "get_collection": p_coll, + "mine_lock": p_lock, + "file_already_mined": p_mined, + "tmp_path": tmp_path, + } + + +def test_mine_formats_walks_directory(_mine_formats_mocks): + """mine_formats must use scan_formats to discover supported files.""" + from unittest.mock import patch + from mempalace.format_miner import mine_formats + + tmp = _mine_formats_mocks["tmp_path"] + (tmp / "doc.pdf").write_bytes(b"pdf") + with ( + patch("mempalace.format_miner.scan_formats", return_value=[]) as p_scan, + patch("mempalace.format_miner._extract_via_markitdown"), + ): + mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace")) + p_scan.assert_called_once() + + +def test_mine_formats_skips_already_mined_files(_mine_formats_mocks): + """If file_already_mined returns True, extract_text should not be called.""" + from unittest.mock import patch + from mempalace.format_miner import mine_formats + + tmp = _mine_formats_mocks["tmp_path"] + f = tmp / "doc.pdf" + f.write_bytes(b"%PDF-1.4 stub") + _mine_formats_mocks["file_already_mined"].return_value = True + with ( + patch("mempalace.format_miner.scan_formats", return_value=[f]), + patch("mempalace.format_miner._extract_via_markitdown") as p_md, + ): + mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace")) + p_md.assert_not_called() + + +def test_mine_formats_skips_extraction_failures(_mine_formats_mocks): + """When extract_text returns a SKIP status, no real drawers are upserted. + + A sentinel upsert IS expected (so the file isn't re-extracted on every + re-mine). The sentinel carries ``is_sentinel=True`` to distinguish it + from a content drawer. + """ + from unittest.mock import patch + from mempalace.format_miner import mine_formats + + tmp = _mine_formats_mocks["tmp_path"] + f = tmp / "bad.pdf" + f.write_bytes(b"%PDF-1.4 stub") + with ( + patch("mempalace.format_miner.scan_formats", return_value=[f]), + patch( + "mempalace.format_miner._extract_via_markitdown", + side_effect=RuntimeError("converter blew up"), + ), + ): + mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace")) + # Any upsert that fired must be a sentinel, never a content drawer. + for call in _mine_formats_mocks["collection"].upsert.call_args_list: + metas = call.kwargs.get("metadatas") or call.args[2] + for m in metas: + assert m.get("is_sentinel") is True, ( + f"unexpected non-sentinel upsert on extraction failure: {m}" + ) + + +def test_mine_formats_files_drawers_for_ok_extractions(_mine_formats_mocks): + """When extract_text returns OK + text, mine_formats chunks and upserts drawers.""" + from unittest.mock import patch + from mempalace.format_miner import mine_formats + + tmp = _mine_formats_mocks["tmp_path"] + f = tmp / "good.pdf" + f.write_bytes(b"%PDF-1.4 stub") + long_text = "This is a sufficiently long extracted text. " * 30 + with ( + patch("mempalace.format_miner.scan_formats", return_value=[f]), + patch("mempalace.format_miner._extract_via_markitdown", return_value=long_text), + ): + mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace")) + # upsert should have been called at least once + assert _mine_formats_mocks["collection"].upsert.called + + +def test_mine_formats_dry_run_does_not_open_collection(_mine_formats_mocks): + """dry_run=True must not call get_collection or upsert any drawers.""" + from unittest.mock import patch + from mempalace.format_miner import mine_formats + + tmp = _mine_formats_mocks["tmp_path"] + f = tmp / "doc.pdf" + f.write_bytes(b"%PDF-1.4 stub") + _mine_formats_mocks["get_collection"].reset_mock() + with ( + patch("mempalace.format_miner.scan_formats", return_value=[f]), + patch("mempalace.format_miner._extract_via_markitdown", return_value="some text"), + ): + mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"), dry_run=True) + _mine_formats_mocks["get_collection"].assert_not_called() + _mine_formats_mocks["collection"].upsert.assert_not_called() + + +def test_mine_formats_respects_limit(_mine_formats_mocks): + """limit=N should restrict processing to the first N files.""" + from unittest.mock import patch + from mempalace.format_miner import mine_formats + + tmp = _mine_formats_mocks["tmp_path"] + files = [] + for i in range(5): + p = tmp / f"f{i}.pdf" + p.write_bytes(b"%PDF-1.4 stub") + files.append(p) + with ( + patch("mempalace.format_miner.scan_formats", return_value=files), + patch( + "mempalace.format_miner._extract_via_markitdown", return_value="long text " * 50 + ) as p_md, + ): + mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"), limit=2) + # Only 2 of the 5 files should have been extracted + assert p_md.call_count == 2 + + +def test_mine_formats_wing_defaults_from_directory_name(_mine_formats_mocks): + """When wing=None, the directory's basename becomes the wing.""" + from unittest.mock import patch + from mempalace.format_miner import mine_formats + + tmp = _mine_formats_mocks["tmp_path"] + target_dir = tmp / "my_research_corpus" + target_dir.mkdir() + f = target_dir / "doc.pdf" + f.write_bytes(b"%PDF-1.4 stub") + with ( + patch("mempalace.format_miner.scan_formats", return_value=[f]), + patch("mempalace.format_miner._extract_via_markitdown", return_value="long text " * 50), + ): + mine_formats(format_dir=str(target_dir), palace_path=str(tmp / "palace")) + # Inspect the metadata passed to upsert — the wing should derive from the dir name + call_args = _mine_formats_mocks["collection"].upsert.call_args + if call_args is not None: + metas = call_args.kwargs.get("metadatas") or call_args.args[2] + assert metas[0]["wing"] == "my_research_corpus" + + +def test_mine_formats_wing_override(_mine_formats_mocks): + """Explicit wing= param overrides the directory-name default.""" + from unittest.mock import patch + from mempalace.format_miner import mine_formats + + tmp = _mine_formats_mocks["tmp_path"] + f = tmp / "doc.pdf" + f.write_bytes(b"%PDF-1.4 stub") + with ( + patch("mempalace.format_miner.scan_formats", return_value=[f]), + patch("mempalace.format_miner._extract_via_markitdown", return_value="long text " * 50), + ): + mine_formats( + format_dir=str(tmp), + palace_path=str(tmp / "palace"), + wing="custom_wing_name", + ) + call_args = _mine_formats_mocks["collection"].upsert.call_args + if call_args is not None: + metas = call_args.kwargs.get("metadatas") or call_args.args[2] + assert metas[0]["wing"] == "custom_wing_name" + + +def test_mine_formats_ingest_mode_metadata_is_extract(_mine_formats_mocks): + """Drawers from mine_formats must carry ingest_mode='extract' so they're + distinguishable from project / convo drawers in the palace.""" + from unittest.mock import patch + from mempalace.format_miner import mine_formats + + tmp = _mine_formats_mocks["tmp_path"] + f = tmp / "doc.pdf" + f.write_bytes(b"%PDF-1.4 stub") + with ( + patch("mempalace.format_miner.scan_formats", return_value=[f]), + patch("mempalace.format_miner._extract_via_markitdown", return_value="long text " * 50), + ): + mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace")) + call_args = _mine_formats_mocks["collection"].upsert.call_args + assert call_args is not None + metas = call_args.kwargs.get("metadatas") or call_args.args[2] + assert metas[0]["ingest_mode"] == "extract" diff --git a/tests/test_line_numbers.py b/tests/test_line_numbers.py new file mode 100644 index 0000000000..9115bb7f22 --- /dev/null +++ b/tests/test_line_numbers.py @@ -0,0 +1,161 @@ +"""Tests for virtual line numbering (proposed for mempalace 3.3.6). + +Run from the proposal directory: + pytest tests/test_line_numbers.py -v + +After integration into the mempalace package, change the import below to: + from mempalace.searcher import render_with_line_numbers, extract_line_range +""" + +from mempalace.searcher import ( # noqa: E402 + extract_line_range, + render_with_line_numbers, +) + + +# ───────────────────────────────────────────────────────────────────────────── +# render_with_line_numbers +# ───────────────────────────────────────────────────────────────────────────── + + +def test_render_empty_string(): + assert render_with_line_numbers("") == "" + + +def test_render_single_line(): + assert render_with_line_numbers("hello") == "[1] hello" + + +def test_render_multi_line(): + text = "alpha\nbeta\ngamma" + expected = "[1] alpha\n[2] beta\n[3] gamma" + assert render_with_line_numbers(text) == expected + + +def test_render_custom_start_line(): + text = "first\nsecond" + expected = "[5] first\n[6] second" + assert render_with_line_numbers(text, start_line=5) == expected + + +def test_render_preserves_already_numbered_lines(): + """Lines that already start with [N] must pass through unchanged.""" + text = "[1] already numbered\n[2] also numbered" + assert render_with_line_numbers(text) == text + + +def test_render_preserves_already_numbered_with_offset(): + """Already-numbered lines pass through verbatim regardless of start_line.""" + text = "[42] keep this number\n[43] and this" + assert render_with_line_numbers(text, start_line=100) == text + + +def test_render_mixed_numbered_and_plain(): + """Counter advances on every line; numbered lines pass through, plain lines get the running counter.""" + text = "[10] kept\nplain line\n[12] also kept" + expected = "[10] kept\n[2] plain line\n[12] also kept" + assert render_with_line_numbers(text) == expected + + +def test_render_preserves_blank_lines(): + """Blank lines must still get a number — they're real positions in the drawer.""" + text = "first\n\nthird" + expected = "[1] first\n[2] \n[3] third" + assert render_with_line_numbers(text) == expected + + +def test_render_preserves_trailing_newline_semantics(): + """Splitting on \\n and rejoining preserves the original boundary count.""" + text = "a\nb\n" + result = render_with_line_numbers(text) + # "a\nb\n".split("\n") → ["a", "b", ""] — three positions + assert result == "[1] a\n[2] b\n[3] " + + +def test_render_none_input_returns_empty(): + """Defensive: None must not crash.""" + assert render_with_line_numbers(None) == "" + + +def test_render_does_not_modify_original_text(): + """Function is pure — no mutation of input.""" + original = "line one\nline two" + snapshot = original + render_with_line_numbers(original) + assert original == snapshot + + +# ───────────────────────────────────────────────────────────────────────────── +# extract_line_range +# ───────────────────────────────────────────────────────────────────────────── + + +def test_extract_single_line(): + text = "a\nb\nc\nd\ne" + assert extract_line_range(text, 3, 3) == "[3] c" + + +def test_extract_inclusive_range(): + text = "a\nb\nc\nd\ne" + expected = "[2] b\n[3] c\n[4] d" + assert extract_line_range(text, 2, 4) == expected + + +def test_extract_full_range(): + text = "first\nsecond\nthird" + expected = "[1] first\n[2] second\n[3] third" + assert extract_line_range(text, 1, 3) == expected + + +def test_extract_end_beyond_length_clips(): + """If line_end exceeds the document, return what's available — don't error.""" + text = "a\nb\nc" + expected = "[2] b\n[3] c" + assert extract_line_range(text, 2, 99) == expected + + +def test_extract_start_below_one_clamps(): + """start_line < 1 clamps to 1; numbering starts from where extraction starts.""" + text = "a\nb\nc" + # Clamping to 1 means we extract from line 1; numbering starts at 1. + expected = "[1] a\n[2] b" + assert extract_line_range(text, 0, 2) == expected + + +def test_extract_start_after_end_returns_empty(): + text = "a\nb\nc" + assert extract_line_range(text, 5, 2) == "" + + +def test_extract_empty_text(): + assert extract_line_range("", 1, 5) == "" + + +def test_extract_honors_already_numbered_lines(): + """If the slice contains already-numbered lines, they pass through.""" + text = "plain\n[42] numbered\nplain again" + expected = "[1] plain\n[42] numbered\n[3] plain again" + assert extract_line_range(text, 1, 3) == expected + + +def test_extract_uses_drawer_line_numbers_not_relative(): + """When extracting lines 5-7, the rendered numbers must be [5][6][7], not [1][2][3]. + + This is the closet-pointer contract: a pointer →2026-01-18:L55-L72 must show + [55] through [72] in the rendered output, so the user sees which drawer + positions they're reading. + """ + text = "\n".join(f"line{i}" for i in range(1, 11)) # line1..line10 + result = extract_line_range(text, 5, 7) + assert result.startswith("[5] line5") + assert "[6] line6" in result + assert result.endswith("[7] line7") + assert "[1]" not in result + assert "[8]" not in result + + +def test_extract_does_not_modify_original_text(): + original = "x\ny\nz" + snapshot = original + extract_line_range(original, 1, 2) + assert original == snapshot From e362dfa0aff853992e44b0b21931abe297f65e76 Mon Sep 17 00:00:00 2001 From: Milla J <232237854+milla-jovovich@users.noreply.github.com> Date: Tue, 19 May 2026 19:51:39 -0700 Subject: [PATCH 26/39] =?UTF-8?q?feat(format=5Fminer):=20address=20bot=20r?= =?UTF-8?q?eview=20=E2=80=94=20parity=20fixes=20with=20miner.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six follow-up fixes addressing gemini-code-assist review on PR #1555. All medium-priority parity gaps between format_miner.py and the existing miner.py / convo_miner.py patterns. 1. Reuse palace.SKIP_DIRS instead of duplicating it locally. Replaces the local _SKIP_DIRS set with the shared constant so the format miner's directory-skip set stays in sync automatically. 2. scan_formats now explicitly skips symlinks. Prevents circular links, following symlinks to /dev/urandom, and processing the same file via multiple paths. Mirrors miner.py:1068 exactly. 3. mine_formats loads MempalaceConfig at start. The current chunker (miner.chunk_text) uses module-level CHUNK_SIZE constants and doesn't accept overrides, so the loaded config values aren't yet threaded into chunking; the load is wired so future chunk_text refactors that accept per-call sizing pick this up automatically. Honest scope note: this gives format_miner the same config-load stance as miner.py without overreaching what chunk_text supports. 4. source_mtime tracking + check_mtime=True. Each drawer's metadata now carries source_mtime, and file_already_mined is called with check_mtime=True so an edited PDF/DOCX is correctly re-mined on the next pass. Mirrors miner.py semantics exactly. Previously, updated documents would have been silently skipped. 5. Per-file try/except wrap + KeyboardInterrupt handling + PID cleanup. One malformed file in a mine of thousands no longer crashes the whole orchestrator — the error is logged, a counter bumps, and the loop continues with the next file. KeyboardInterrupt still produces a clean summary on Ctrl-C. _cleanup_mine_pid_file is called in a finally block so hook-spawned mines don't leak a stale PID file on exit. 6. hall + entities metadata on every drawer. Format-mined drawers now carry the same hall (via miner.detect_hall) and entities (via miner._extract_entities_for_metadata) tags as project-mined drawers. Matches the search-result quality the other miners ship. Tests: 5 new tests covering the new behaviors: - test_scan_formats_skips_symlinks - test_mine_formats_uses_check_mtime_true - test_mine_formats_records_source_mtime_in_drawer_metadata - test_mine_formats_records_hall_in_drawer_metadata - test_mine_formats_continues_after_per_file_error Total: 78 tests passing (was 73). Full mempalace test suite green modulo one pre-existing flaky test (test_chroma_close_palace_..., a sqlite-lock-timing flake unrelated to this change — passes 4/4 in isolation). ruff check + ruff format clean on pinned 0.15.9. Coverage on mempalace/format_miner.py: 84% (clears the 80% gate). Refs: PR #1555 review comments from gemini-code-assist[bot] --- mempalace/format_miner.py | 273 ++++++++++++++++++++++++------------- tests/test_format_miner.py | 140 +++++++++++++++++++ 2 files changed, 317 insertions(+), 96 deletions(-) diff --git a/mempalace/format_miner.py b/mempalace/format_miner.py index 2f3ea699e8..be6cf6c577 100644 --- a/mempalace/format_miner.py +++ b/mempalace/format_miner.py @@ -70,6 +70,7 @@ from .palace import ( NORMALIZE_VERSION, + SKIP_DIRS, file_already_mined, get_collection, mine_lock, @@ -117,34 +118,6 @@ DEFAULT_MAX_FILE_SIZE = 500 * 1024 * 1024 # 500 MB -# Directories skipped by scan_formats. Mirrors miner.SKIP_DIRS. -_SKIP_DIRS = { - ".git", - "node_modules", - "__pycache__", - ".venv", - "venv", - "env", - "dist", - "build", - ".next", - "coverage", - ".mempalace", - ".ruff_cache", - ".mypy_cache", - ".pytest_cache", - ".cache", - ".tox", - ".nox", - ".idea", - ".vscode", - ".ipynb_checkpoints", - ".eggs", - "htmlcov", - "target", -} - - # Filename patterns that are not user content even if their extension matches. _SKIP_FILENAMES = { ".DS_Store", @@ -426,8 +399,10 @@ def scan_formats(directory: Union[Path, str]) -> list[Path]: """Walk ``directory`` recursively and return supported files, sorted. Skips: - - Hidden / build directories listed in ``_SKIP_DIRS`` + - Hidden / build directories listed in ``palace.SKIP_DIRS`` - Filenames listed in ``_SKIP_FILENAMES`` (.DS_Store etc.) + - Symlinks (prevents circular links / processing the same file via + multiple paths; mirrors ``miner.py`` and ``convo_miner.py``) - Files whose suffix isn't in ``SUPPORTED_FORMATS`` Returns a list of ``Path`` objects. The order is deterministic @@ -440,12 +415,19 @@ def scan_formats(directory: Union[Path, str]) -> list[Path]: found: list[Path] = [] for dirpath, dirnames, filenames in os.walk(root): - # Mutate dirnames in place to prune the walk. - dirnames[:] = [d for d in dirnames if d not in _SKIP_DIRS] + # Mutate dirnames in place to prune the walk. Uses the shared + # palace.SKIP_DIRS constant so this stays in sync with the other + # miners' skip set. + dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS] for name in filenames: if name in _SKIP_FILENAMES: continue p = Path(dirpath) / name + # Skip symlinks — prevents following links to /dev/urandom, + # circular links, or processing the same file twice via + # different paths. Mirrors miner.py:1068. + if p.is_symlink(): + continue if p.suffix.lower() not in SUPPORTED_FORMATS: continue found.append(p) @@ -493,19 +475,42 @@ def _register_file(collection, source_file: str, wing: str, agent: str) -> None: logger.debug("Sentinel write failed for %s", source_file, exc_info=True) -def _file_chunks_locked(collection, source_file, chunks, wing, room, agent): +def _file_chunks_locked( + collection, + source_file, + chunks, + wing, + room, + agent, + source_mtime: Optional[float] = None, +): """Lock the source file, purge stale drawers, and upsert fresh chunks. Mirrors the canonical convo_miner pattern (locked purge + batched upsert) but with ``ingest_mode="extract"`` so format-mined drawers are distinguishable from project / convo drawers in the palace. + Each drawer's metadata includes: + - ``source_mtime`` — for mtime-based idempotency (file_already_mined + with check_mtime=True will re-mine when the source file is updated) + - ``hall`` — content-keyword routing (same detect_hall as miner.py) + - ``entities`` — extracted entity tags (same _extract_entities_for_metadata) + Returns ``(drawers_added, skipped)``. """ + # Lazy imports to avoid a module-load cycle (miner.py imports from this + # module's package, so we defer these helpers until call time). + from .miner import _extract_entities_for_metadata, detect_hall + drawers_added = 0 with mine_lock(source_file): # Re-check after lock — another agent may have just finished this file. - if file_already_mined(collection, source_file): + # Use check_mtime=True so an updated source file is re-mined even + # though a prior drawer set exists (matches miner.py semantics). + # palace.file_already_mined reads current mtime from disk and compares + # against the stored source_mtime metadata, so the caller just toggles + # the flag; no need to pass mtime explicitly. + if file_already_mined(collection, source_file, check_mtime=True): return 0, True try: @@ -521,21 +526,27 @@ def _file_chunks_locked(collection, source_file, chunks, wing, room, agent): for chunk in chunks[batch_start : batch_start + DRAWER_UPSERT_BATCH_SIZE]: key = (source_file + str(chunk["chunk_index"])).encode() drawer_id = f"drawer_{wing}_{room}_{hashlib.sha256(key).hexdigest()[:24]}" - batch_docs.append(chunk["content"]) + content = chunk["content"] + meta: dict = { + "wing": wing, + "room": room, + "source_file": source_file, + "chunk_index": chunk["chunk_index"], + "added_by": agent, + "filed_at": filed_at, + "ingest_mode": "extract", + "extract_mode": "format", + "normalize_version": NORMALIZE_VERSION, + "hall": detect_hall(content), + } + if source_mtime is not None: + meta["source_mtime"] = source_mtime + entities = _extract_entities_for_metadata(content) + if entities: + meta["entities"] = entities + batch_docs.append(content) batch_ids.append(drawer_id) - batch_metas.append( - { - "wing": wing, - "room": room, - "source_file": source_file, - "chunk_index": chunk["chunk_index"], - "added_by": agent, - "filed_at": filed_at, - "ingest_mode": "extract", - "extract_mode": "format", - "normalize_version": NORMALIZE_VERSION, - } - ) + batch_metas.append(meta) try: collection.upsert( documents=batch_docs, @@ -569,9 +580,10 @@ def mine_formats( Parameters ---------- format_dir : - Directory to walk recursively. Hidden / build dirs are skipped (see - ``_SKIP_DIRS``). Only files matching ``SUPPORTED_FORMATS`` are - processed. + Directory to walk recursively. Hidden / build dirs are skipped + (see ``palace.SKIP_DIRS``). Symlinks are skipped (consistency with + ``miner.py`` and ``convo_miner.py``). Only files matching + ``SUPPORTED_FORMATS`` are processed. palace_path : Path to the ChromaDB palace (the destination of the drawers). wing : @@ -583,16 +595,36 @@ def mine_formats( dry_run : If True, walk + extract + chunk but do NOT open the collection or upsert any drawers. Just prints what would have been filed. + + Notes + ----- + - Loads ``MempalaceConfig`` once at the start. The current chunker + (``miner.chunk_text``) uses module-level CHUNK_SIZE constants and + doesn't accept overrides, so the loaded config values aren't yet + threaded into chunking; the load is wired so future ``chunk_text`` + refactors that accept per-call sizing pick this up automatically. + - Each per-file step is wrapped so one malformed file can't crash the + whole mine; the loop continues with the next file and logs the + offender. + - ``KeyboardInterrupt`` is caught at the orchestrator level so the + summary still prints on Ctrl-C; partial progress is safe to leave + in place because drawer IDs are deterministic (re-mining + upserts to the same rows). """ - # Local import — chunk_text is the same chunker miner.py uses for - # text-format project files. Imported inside the function to avoid a - # module-load cycle. + # Local imports — chunk_text is the same chunker miner.py uses; the + # config object provides chunk_size / overlap / min when needed. + from .config import MempalaceConfig, normalize_wing_name from .miner import chunk_text + # Load palace-wide config. We don't yet thread the chunk-size overrides + # through chunk_text (which would require refactoring its module-level + # constants); loading the config here satisfies the parity contract + # with miner.py and gives a single place to wire the threading later. + palace_config = MempalaceConfig() + _ = palace_config.chunk_size # touch to validate config readability + format_path = Path(format_dir).expanduser().resolve() if not wing: - from .config import normalize_wing_name - wing = normalize_wing_name(format_path.name) files = scan_formats(format_dir) @@ -615,50 +647,98 @@ def mine_formats( total_drawers = 0 files_skipped = 0 files_with_text = 0 + files_errored = 0 status_counts: dict = defaultdict(int) - for i, filepath in enumerate(files, 1): - source_file = str(filepath) - - if not dry_run and file_already_mined(collection, source_file): - files_skipped += 1 - continue - - text, status = extract_text(filepath) - status_counts[status.name] += 1 - - if status != ExtractionStatus.OK or not text: - # Not an OK extraction — record a sentinel so we don't re-try - # this file on every subsequent mine, and move on. - if not dry_run: - _register_file(collection, source_file, wing, agent) - print(f" - [{i:4}/{len(files)}] {filepath.name[:50]:50} {status.name}") - continue - - chunks = chunk_text(text, source_file) - if not chunks: - if not dry_run: - _register_file(collection, source_file, wing, agent) - print(f" - [{i:4}/{len(files)}] {filepath.name[:50]:50} EMPTY_AFTER_CHUNK") - continue - - room = "documents" # all format-mined drawers land in the "documents" room - files_with_text += 1 - - if dry_run: - print(f" [DRY RUN] {filepath.name} → {len(chunks)} drawers") - total_drawers += len(chunks) - continue - - drawers_added, skipped = _file_chunks_locked( - collection, source_file, chunks, wing, room, agent - ) - if skipped: - files_skipped += 1 - continue + try: + for i, filepath in enumerate(files, 1): + source_file = str(filepath) + + # Per-file try/except so one bad file can't crash the whole mine. + # Mirrors miner.py's robustness pattern. + try: + # Cheap mtime read up-front — used as the idempotency key on + # subsequent re-mines (check_mtime=True compares stored vs. + # current mtime). + try: + source_mtime: Optional[float] = os.path.getmtime(source_file) + except OSError: + source_mtime = None + + if not dry_run and file_already_mined(collection, source_file, check_mtime=True): + files_skipped += 1 + continue + + text, status = extract_text(filepath) + status_counts[status.name] += 1 + + if status != ExtractionStatus.OK or not text: + if not dry_run: + _register_file(collection, source_file, wing, agent) + print(f" - [{i:4}/{len(files)}] {filepath.name[:50]:50} {status.name}") + continue + + chunks = chunk_text(text, source_file) + if not chunks: + if not dry_run: + _register_file(collection, source_file, wing, agent) + print(f" - [{i:4}/{len(files)}] {filepath.name[:50]:50} EMPTY_AFTER_CHUNK") + continue + + room = "documents" # all format-mined drawers land in the "documents" room + files_with_text += 1 + + if dry_run: + print(f" [DRY RUN] {filepath.name} → {len(chunks)} drawers") + total_drawers += len(chunks) + continue + + drawers_added, skipped = _file_chunks_locked( + collection, + source_file, + chunks, + wing, + room, + agent, + source_mtime=source_mtime, + ) + if skipped: + files_skipped += 1 + continue - total_drawers += drawers_added - print(f" + [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers_added}") + total_drawers += drawers_added + print(f" + [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers_added}") + except Exception as exc: + # Log and continue — one malformed file shouldn't kill the + # whole mine. Mirrors miner.py's per-file recovery. + files_errored += 1 + logger.warning( + "mine_formats: error processing %s — %s: %s", + source_file, + type(exc).__name__, + str(exc)[:200], + ) + print( + f" ! [{i:4}/{len(files)}] {filepath.name[:50]:50} ERROR: {type(exc).__name__}" + ) + continue + except KeyboardInterrupt: + # Partial progress is safe — deterministic drawer IDs mean a re-mine + # upserts to the same rows. Print a clean summary and exit. + print("\n Mine interrupted by user (Ctrl-C).") + finally: + # Hook-spawned mines write a PID file that miner.py's + # _cleanup_mine_pid_file() clears; we mirror that so format-mode + # mines kicked off the same way don't leave a stale PID behind. + try: + from .miner import _cleanup_mine_pid_file + except ImportError: + _cleanup_mine_pid_file = None + if _cleanup_mine_pid_file is not None: + try: + _cleanup_mine_pid_file() + except Exception: + logger.debug("mine_formats: _cleanup_mine_pid_file failed", exc_info=True) print(f"\n{'=' * 55}") print(" Summary") @@ -666,6 +746,7 @@ def mine_formats( print(f" Files seen: {len(files)}") print(f" Files extracted: {files_with_text}") print(f" Files skipped: {files_skipped}") + print(f" Files errored: {files_errored}") print(f" Total drawers: {total_drawers}") if status_counts: print(" Extraction status:") diff --git a/tests/test_format_miner.py b/tests/test_format_miner.py index 0f67d836b9..0930b6e10e 100644 --- a/tests/test_format_miner.py +++ b/tests/test_format_miner.py @@ -848,3 +848,143 @@ def test_mine_formats_ingest_mode_metadata_is_extract(_mine_formats_mocks): assert call_args is not None metas = call_args.kwargs.get("metadatas") or call_args.args[2] assert metas[0]["ingest_mode"] == "extract" + + +# ───────────────────────────────────────────────────────────────────────────── +# Bot-review parity fixes (PR #1555 follow-up commit, 2026-05-19): +# - palace.SKIP_DIRS reuse (covered indirectly by scan_formats tests) +# - scan_formats skips symlinks +# - mine_formats uses check_mtime=True +# - source_mtime + hall + entities in drawer metadata +# - per-file try/except so one bad file doesn't crash the whole mine +# ───────────────────────────────────────────────────────────────────────────── + + +def test_scan_formats_skips_symlinks(tmp_path: Path): + """Symlinks must be skipped — mirrors miner.py and convo_miner.py.""" + real = tmp_path / "real.pdf" + real.write_bytes(b"%PDF-1.4 stub") + link = tmp_path / "alias.pdf" + link.symlink_to(real) + found = {f.name for f in scan_formats(tmp_path)} + assert "real.pdf" in found + assert "alias.pdf" not in found + + +def test_mine_formats_uses_check_mtime_true(_mine_formats_mocks): + """mine_formats must call file_already_mined with check_mtime=True so + updated documents get re-mined (matches miner.py semantics).""" + from unittest.mock import patch + from mempalace.format_miner import mine_formats + + tmp = _mine_formats_mocks["tmp_path"] + f = tmp / "doc.pdf" + f.write_bytes(b"%PDF-1.4 stub") + with ( + patch("mempalace.format_miner.scan_formats", return_value=[f]), + patch("mempalace.format_miner._extract_via_markitdown", return_value="long " * 50), + ): + mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace")) + # Inspect every file_already_mined call — they must all use check_mtime=True + calls = _mine_formats_mocks["file_already_mined"].call_args_list + assert calls, "file_already_mined should have been called at least once" + for call in calls: + # check_mtime can be in kwargs or positional. Inspect both. + kwargs = call.kwargs + assert kwargs.get("check_mtime") is True, f"check_mtime must be True, got {kwargs}" + + +def test_mine_formats_records_source_mtime_in_drawer_metadata(_mine_formats_mocks): + """Each drawer must carry source_mtime so file_already_mined(check_mtime=True) + can detect updates on re-mine.""" + from unittest.mock import patch + from mempalace.format_miner import mine_formats + + tmp = _mine_formats_mocks["tmp_path"] + f = tmp / "doc.pdf" + f.write_bytes(b"%PDF-1.4 stub") + with ( + patch("mempalace.format_miner.scan_formats", return_value=[f]), + patch("mempalace.format_miner._extract_via_markitdown", return_value="long " * 50), + ): + mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace")) + upsert_calls = _mine_formats_mocks["collection"].upsert.call_args_list + # Find the content upsert (not the sentinel) + found_mtime = False + for call in upsert_calls: + metas = call.kwargs.get("metadatas") or call.args[2] + for m in metas: + if m.get("is_sentinel"): + continue + assert "source_mtime" in m, f"missing source_mtime in drawer meta: {m}" + assert isinstance(m["source_mtime"], (int, float)) + found_mtime = True + assert found_mtime, "no non-sentinel drawer upsert observed" + + +def test_mine_formats_records_hall_in_drawer_metadata(_mine_formats_mocks): + """Each drawer must carry a 'hall' tag — matches miner.py drawer quality.""" + from unittest.mock import patch + from mempalace.format_miner import mine_formats + + tmp = _mine_formats_mocks["tmp_path"] + f = tmp / "doc.pdf" + f.write_bytes(b"%PDF-1.4 stub") + with ( + patch("mempalace.format_miner.scan_formats", return_value=[f]), + patch("mempalace.format_miner._extract_via_markitdown", return_value="long " * 50), + ): + mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace")) + upsert_calls = _mine_formats_mocks["collection"].upsert.call_args_list + found_hall = False + for call in upsert_calls: + metas = call.kwargs.get("metadatas") or call.args[2] + for m in metas: + if m.get("is_sentinel"): + continue + assert "hall" in m, f"missing hall in drawer meta: {m}" + assert isinstance(m["hall"], str) + found_hall = True + assert found_hall, "no non-sentinel drawer upsert observed" + + +def test_mine_formats_continues_after_per_file_error(_mine_formats_mocks): + """One bad file must not crash the whole mine — the loop continues.""" + from unittest.mock import patch + from mempalace.format_miner import mine_formats + + tmp = _mine_formats_mocks["tmp_path"] + bad = tmp / "broken.pdf" + bad.write_bytes(b"%PDF-1.4 stub") + good = tmp / "fine.pdf" + good.write_bytes(b"%PDF-1.4 stub") + + # Make chunk_text crash on the first file but succeed on the second. + call_count = {"n": 0} + + def chunk_text_first_fails(content, source_file): + call_count["n"] += 1 + if call_count["n"] == 1: + raise RuntimeError("simulated chunker explosion") + return [{"content": "good chunk content here " * 5, "chunk_index": 0}] + + with ( + patch("mempalace.format_miner.scan_formats", return_value=[bad, good]), + patch("mempalace.format_miner._extract_via_markitdown", return_value="text"), + patch("mempalace.miner.chunk_text", side_effect=chunk_text_first_fails), + ): + # Must not raise + mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace")) + + # The second file should still have produced an upsert + upsert_calls = _mine_formats_mocks["collection"].upsert.call_args_list + non_sentinel_upserts = [ + call + for call in upsert_calls + if not any( + (m or {}).get("is_sentinel") for m in (call.kwargs.get("metadatas") or call.args[2]) + ) + ] + assert non_sentinel_upserts, ( + "the good file should still have produced a content upsert after the bad file errored" + ) From 5c40cc3d70b401fdf33346d1d29bd5d32b9795bd Mon Sep 17 00:00:00 2001 From: Milla J <232237854+milla-jovovich@users.noreply.github.com> Date: Wed, 20 May 2026 03:02:39 -0700 Subject: [PATCH 27/39] feat(format_miner): detect_room + topic tunnel computation (miner.py parity) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third amendment to PR #1555. Closes the remaining parity gaps between format_miner and the existing project miner that Aya's palace audit surfaced after amendment #2. ## What was broken A smoke test against the just-mined v6 palace confirmed: WING: wing_aya ROOM: documents 2904 drawers ← ALL in one room Tunnels for wing_aya: 0 ← no cross-wing links Cause: format_miner hardcoded `room = "documents"` for every drawer and never called `_compute_topic_tunnels_for_wing()` post-mine. miner.py calls `detect_room()` per drawer and computes tunnels after the loop. format_miner did neither. ## What this commit changes 1. Module-level imports of `detect_room`, `load_config`, and `_compute_topic_tunnels_for_wing` from `.miner`. Module-level (not lazy) so test seams `patch("mempalace.format_miner.detect_room", ...)` work — lazy imports inside a function don't expose attributes on the module object. 2. `mine_formats()` loads the project's `mempalace.yaml` via `load_config()` to get the rooms list. Falls back to a single "documents" room if no config exists. 3. The hardcoded `room = "documents"` line is replaced with: room = detect_room(filepath, text, rooms, format_path) Mirrors miner.py:904 exactly — folder-match → filename-match → content-keyword scoring → fallback "general". 4. After the per-file loop completes (in an `else` branch on the outer try, so it does NOT run on KeyboardInterrupt), call `_compute_topic_tunnels_for_wing(wing)` in try/except. Exact mirror of miner.py:1241-1249. Tunnel-compute failures must never fail a mine. 5. Import `sys` at module level (used by the tunnel-compute error path for `print(..., file=sys.stderr)`). ## Tests 5 new RED-first tests in `tests/test_format_miner.py`: - test_mine_formats_calls_load_config_for_rooms - test_mine_formats_calls_detect_room_per_file - test_mine_formats_uses_detected_room_in_drawer_metadata - test_mine_formats_calls_compute_topic_tunnels_after_loop - test_mine_formats_tunnel_failure_does_not_crash_mine All 5 failed against pre-commit code with AttributeError on missing module attributes (proof the bug existed). All 5 pass after the implementation. Total format_miner test count: 62. Total proposal test count (format_miner + line_numbers): 83. Full mempalace test suite: 2002 passed, 1 skipped (no regressions). ## Verification ruff check mempalace/format_miner.py tests/test_format_miner.py → All checks passed! ruff format --check ... → already formatted (pinned ruff 0.15.9) pytest tests/test_format_miner.py tests/test_line_numbers.py → 83 passed pytest -q (full suite) → 2002 passed, 1 skipped ## Not in this commit (deferred) Within-wing hallway primitives are a separate architectural addition (separate PR — being designed). The current `_compute_topic_tunnels_for_wing` matches miner.py exactly but inherits miner.py's gap: it computes tunnels from raw topic words rather than from hallway primitives. That refactor lands in its own PR after the hallway primitive ships. --- mempalace/format_miner.py | 61 ++++++++++++++- tests/test_format_miner.py | 149 +++++++++++++++++++++++++++++++++++++ 2 files changed, 209 insertions(+), 1 deletion(-) diff --git a/mempalace/format_miner.py b/mempalace/format_miner.py index be6cf6c577..52dcbb910a 100644 --- a/mempalace/format_miner.py +++ b/mempalace/format_miner.py @@ -63,6 +63,7 @@ import logging import os import re +import sys from collections import defaultdict from datetime import datetime from pathlib import Path @@ -76,6 +77,15 @@ mine_lock, ) +# Module-level imports from .miner so tests can patch them via +# mempalace.format_miner.. Lazy imports inside functions would not +# expose these as attributes of this module, breaking the test seams. +from .miner import ( + _compute_topic_tunnels_for_wing, + detect_room, + load_config, +) + __all__ = [ "SUPPORTED_FORMATS", "DEFAULT_MAX_FILE_SIZE", @@ -627,6 +637,32 @@ def mine_formats( if not wing: wing = normalize_wing_name(format_path.name) + # Load the project's mempalace.yaml (rooms list + wing override) so + # detect_room has real categories to route into. Mirrors miner.py:1154. + # Fall back to a single "documents" room if no config exists — keeps the + # detector well-defined without forcing a yaml on every format dir. + try: + project_config = load_config(format_dir) + rooms = project_config.get( + "rooms", + [ + { + "name": "documents", + "description": "All format-mined files", + "keywords": ["documents"], + } + ], + ) + except Exception: + logger.debug("mine_formats: load_config fallback to default rooms", exc_info=True) + rooms = [ + { + "name": "documents", + "description": "All format-mined files", + "keywords": ["documents"], + } + ] + files = scan_formats(format_dir) if limit > 0: files = files[:limit] @@ -685,7 +721,10 @@ def mine_formats( print(f" - [{i:4}/{len(files)}] {filepath.name[:50]:50} EMPTY_AFTER_CHUNK") continue - room = "documents" # all format-mined drawers land in the "documents" room + # Route this drawer to a room — same detect_room miner.py + # uses (folder match → filename match → content keyword + # scoring → fallback "general"). Mirrors miner.py:904. + room = detect_room(filepath, text, rooms, format_path) files_with_text += 1 if dry_run: @@ -726,6 +765,26 @@ def mine_formats( # Partial progress is safe — deterministic drawer IDs mean a re-mine # upserts to the same rows. Print a clean summary and exit. print("\n Mine interrupted by user (Ctrl-C).") + else: + # All files processed without interruption — compute cross-wing topic + # tunnels linking this wing to others that share confirmed topics. + # Mirrors miner.py:1241-1249 exactly: tunnel-compute failures must + # never fail a mine, so any exception is logged and skipped quietly. + if not dry_run: + try: + tunnels_added = _compute_topic_tunnels_for_wing(wing) + if tunnels_added: + print(f"\n Topic tunnels: +{tunnels_added} cross-wing link(s)") + except Exception as exc: + logger.warning( + "mine_formats: topic tunnel computation skipped — %s: %s", + type(exc).__name__, + str(exc)[:200], + ) + print( + f"\n WARNING: topic tunnel computation skipped — {exc}", + file=sys.stderr, + ) finally: # Hook-spawned mines write a PID file that miner.py's # _cleanup_mine_pid_file() clears; we mirror that so format-mode diff --git a/tests/test_format_miner.py b/tests/test_format_miner.py index 0930b6e10e..1a79badf10 100644 --- a/tests/test_format_miner.py +++ b/tests/test_format_miner.py @@ -988,3 +988,152 @@ def chunk_text_first_fails(content, source_file): assert non_sentinel_upserts, ( "the good file should still have produced a content upsert after the bad file errored" ) + + +# ───────────────────────────────────────────────────────────────────────────── +# Amendment #3 — parity with miner.py: detect_room + tunnel computation. +# Tests written FIRST (TDD) to prove the gaps exist, then format_miner.py is +# amended to make them pass. Pattern mirrors miner.py exactly. +# ───────────────────────────────────────────────────────────────────────────── + + +def test_mine_formats_calls_load_config_for_rooms(_mine_formats_mocks): + """mine_formats must load mempalace.yaml (via load_config) to get the + rooms list — same as miner.py:1154. Without this, drawers fall back to + a single 'documents' room.""" + from unittest.mock import patch + from mempalace.format_miner import mine_formats + + tmp = _mine_formats_mocks["tmp_path"] + f = tmp / "doc.pdf" + f.write_bytes(b"%PDF-1.4 stub") + fake_config = { + "wing": "wing_aya", + "rooms": [{"name": "documents", "keywords": ["documents"]}], + } + with ( + patch("mempalace.format_miner.scan_formats", return_value=[f]), + patch("mempalace.format_miner._extract_via_markitdown", return_value="long " * 50), + patch("mempalace.format_miner.load_config", return_value=fake_config) as p_cfg, + ): + mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace")) + p_cfg.assert_called_once() + + +def test_mine_formats_calls_detect_room_per_file(_mine_formats_mocks): + """mine_formats must call detect_room(filepath, content, rooms, project_path) + for each file — same as miner.py:904. Hardcoding room='documents' is the + bug; this test fails until detect_room is wired in.""" + from unittest.mock import patch + from mempalace.format_miner import mine_formats + + tmp = _mine_formats_mocks["tmp_path"] + f = tmp / "doc.pdf" + f.write_bytes(b"%PDF-1.4 stub") + fake_config = { + "wing": "wing_aya", + "rooms": [{"name": "documents", "keywords": ["documents"]}], + } + with ( + patch("mempalace.format_miner.scan_formats", return_value=[f]), + patch("mempalace.format_miner._extract_via_markitdown", return_value="long " * 50), + patch("mempalace.format_miner.load_config", return_value=fake_config), + patch("mempalace.format_miner.detect_room", return_value="family") as p_room, + ): + mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace")) + p_room.assert_called_once() + # First positional arg should be the file Path; last arg should be project_path + call_args = p_room.call_args + # detect_room(filepath, content, rooms, project_path) + assert len(call_args.args) == 4 or "filepath" in call_args.kwargs + + +def test_mine_formats_uses_detected_room_in_drawer_metadata(_mine_formats_mocks): + """The room field on the drawer metadata must reflect what detect_room + returned, NOT the hardcoded 'documents'. This is the visible bug from + the 2026-05-19 mine: every drawer landed in room='documents'.""" + from unittest.mock import patch + from mempalace.format_miner import mine_formats + + tmp = _mine_formats_mocks["tmp_path"] + f = tmp / "doc.pdf" + f.write_bytes(b"%PDF-1.4 stub") + fake_config = { + "wing": "wing_aya", + "rooms": [{"name": "family", "keywords": ["family"]}], + } + with ( + patch("mempalace.format_miner.scan_formats", return_value=[f]), + patch("mempalace.format_miner._extract_via_markitdown", return_value="long " * 50), + patch("mempalace.format_miner.load_config", return_value=fake_config), + patch("mempalace.format_miner.detect_room", return_value="family"), + ): + mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace")) + upsert_calls = _mine_formats_mocks["collection"].upsert.call_args_list + found_room = None + for call in upsert_calls: + metas = call.kwargs.get("metadatas") or call.args[2] + for m in metas: + if m.get("is_sentinel"): + continue + found_room = m.get("room") + break + if found_room: + break + assert found_room == "family", ( + f"drawer room must be 'family' (detect_room's return), not {found_room!r}" + ) + + +def test_mine_formats_calls_compute_topic_tunnels_after_loop(_mine_formats_mocks): + """After the per-file loop, mine_formats must call + _compute_topic_tunnels_for_wing(wing) exactly once. Mirrors miner.py:1241. + Without this, cross-wing topic tunnels never materialize for format-mined + wings (audit confirmed 0 tunnels in the 2026-05-19 v6 mine).""" + from unittest.mock import patch + from mempalace.format_miner import mine_formats + + tmp = _mine_formats_mocks["tmp_path"] + f = tmp / "doc.pdf" + f.write_bytes(b"%PDF-1.4 stub") + fake_config = { + "wing": "wing_aya", + "rooms": [{"name": "documents", "keywords": ["documents"]}], + } + with ( + patch("mempalace.format_miner.scan_formats", return_value=[f]), + patch("mempalace.format_miner._extract_via_markitdown", return_value="long " * 50), + patch("mempalace.format_miner.load_config", return_value=fake_config), + patch("mempalace.format_miner.detect_room", return_value="documents"), + patch("mempalace.format_miner._compute_topic_tunnels_for_wing", return_value=0) as p_tun, + ): + mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"), wing="wing_aya") + p_tun.assert_called_once_with("wing_aya") + + +def test_mine_formats_tunnel_failure_does_not_crash_mine(_mine_formats_mocks): + """If _compute_topic_tunnels_for_wing raises, the mine must still + complete (summary still prints, no exception propagates). Mirrors the + try/except wrap at miner.py:1244-1249.""" + from unittest.mock import patch + from mempalace.format_miner import mine_formats + + tmp = _mine_formats_mocks["tmp_path"] + f = tmp / "doc.pdf" + f.write_bytes(b"%PDF-1.4 stub") + fake_config = { + "wing": "wing_aya", + "rooms": [{"name": "documents", "keywords": ["documents"]}], + } + with ( + patch("mempalace.format_miner.scan_formats", return_value=[f]), + patch("mempalace.format_miner._extract_via_markitdown", return_value="long " * 50), + patch("mempalace.format_miner.load_config", return_value=fake_config), + patch("mempalace.format_miner.detect_room", return_value="documents"), + patch( + "mempalace.format_miner._compute_topic_tunnels_for_wing", + side_effect=RuntimeError("simulated tunnel-compute failure"), + ), + ): + # Must NOT raise + mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"), wing="wing_aya") From 0b9c5629b756904f8239920bcca125df7cb099dc Mon Sep 17 00:00:00 2001 From: Milla J <232237854+milla-jovovich@users.noreply.github.com> Date: Wed, 20 May 2026 03:15:45 -0700 Subject: [PATCH 28/39] fix(miner): mirror init's case-insensitive entity matching in per-drawer tagger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mempalace's init-time entity scanner (`entity_detector.py:276`) already matches names case-insensitively against corpus content: name_line_indices = [i for i, line in enumerate(lines) if name_lower in line.lower()] That's how a corpus that mentions "aya" all in lowercase still surfaces "Aya" as a confirmed entity during `mempalace init`. But the per-drawer tagger in `miner.py:_extract_entities_for_metadata` was never updated to use the same flag — it walks the same `known_entities.json` seed list and matches case-sensitively: for name in known: if re.search(r"(? str: known = _load_known_entities() for name in known: - if re.search(r"(? Date: Wed, 20 May 2026 03:44:05 -0700 Subject: [PATCH 29/39] feat(hallways): within-wing entity-to-entity connector primitive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces the missing primitive in mempalace's palace architecture. ## The architectural gap Mempalace's data model today is Wing → Rooms → Tunnels. Tunnels connect rooms ACROSS wings (via shared topic words). What's missing — and what this commit adds — is the connector between ENTITIES (people, projects, concepts, interests) inside one wing. The intended full architecture is: WING → has DRAWERS (each tagged with entities) entities → connected to other entities by HALLWAYS (within-wing, built from drawer co-occurrence) hallways → are the primitive tunnels → use hallways to spawn cross-wing connections A hallway is the structural fact of "these two entities travel together inside this wing." If Aya and Lumi are mentioned together in 47 drawers across diary, letters, and ideas rooms, that's a hallway between Aya and Lumi. If Aya and "consciousness" co-occur in 19 drawers, that's another hallway. Entities aren't only people — projects, concepts, and interests share the same entity-tag substrate, so a hallway can connect a person to a concept (Aya ↔ consciousness) just as easily as person to person (Aya ↔ Lumi). `palace_graph.py:253`'s docstring currently calls cross-wing tunnels "hallways" — that conflation is fixed in a separate naming-disambiguation follow-up. This commit just introduces the primitive. ## Public API New module `mempalace/hallways.py`: compute_hallways_for_wing(wing, col=None, min_count=2) -> list[dict] Query drawers for wing. For each drawer with >=2 entities, every unordered pair of entities is one co-occurrence; increment a counter and track the room. For each entity pair with count >= min_count, materialize a hallway record. Persists to ~/.mempalace/hallways.json. Returns records created for this wing (records for other wings already on disk are preserved). list_hallways(wing=None) -> list[dict] Return all hallways, optionally filtered by wing. delete_hallway(hallway_id) -> bool Remove one record by id. Returns True on success. _load_hallways() / _save_hallways(state) JSON persistence at ~/.mempalace/hallways.json with 0600 perms on POSIX and atomic os.replace writes. Mirrors palace_graph's tunnel persistence pattern exactly. ## Data model (hallways.json) ```jsonc { "schema_version": 1, "hallways": [ { "id": "hallway____", "wing": "wing_aya", "entity_a": "Aya", "entity_b": "Lumi", "co_occurrence_count": 47, "rooms": ["diary", "letters", "ideas"], "label": "Aya ↔ Lumi (co-occur in 47 drawers across 3 rooms: diary, letters, ideas)", "created_at": "2026-05-20T03:30:00Z", "created_by": "auto" } ] } ``` Hallway IDs are deterministic — `sha256(wing + sorted(entity_a, entity_b))` truncated. Sorted-pair canonicalization means (Aya, Lumi) and (Lumi, Aya) produce the same id, so re-runs upsert idempotently and the same drawer appearing as "Aya;Lumi" or "Lumi;Aya" counts as one co-occurrence (not two). ## Tests 17 RED-first tests in `tests/test_hallways.py`: Storage: - _load_hallways returns [] for missing file - _load_hallways returns [] for corrupt JSON - _save_hallways + _load_hallways round-trips Algorithm: - returns [] for unknown wing - returns [] when no drawer has >=2 entities - creates hallway for entity pair when threshold met (Aya↔Lumi in 3 drawers) - connects person to concept (Aya↔consciousness — entities aren't only people) - respects min_count threshold (rejects 2 co-occurrences when min_count=3) - creates deterministic id per entity pair (idempotent re-runs) - entity pair is symmetric (Aya;Lumi and Lumi;Aya are the same pair) - persists to JSON file - tracks rooms across co-occurrences (rooms set vs. count distinction) - skips sentinel drawers (file_already_mined bookkeeping noise) Query API: - list_hallways returns all when no filter - list_hallways filters by wing - delete_hallway removes record and persists - delete_hallway unknown id returns False ## Verification ruff check mempalace/hallways.py tests/test_hallways.py → All checks passed! ruff format --check ... → 2 files already formatted (pinned 0.15.9) pytest tests/test_hallways.py → 17 passed pytest -q (full mempalace suite) → 1936 passed, 1 skipped, 0 regressions caused by this PR Linux CI parity replicated locally via OrbStack containers (Python 3.9, 3.11, 3.13): 17/17 hallway tests pass, ruff clean on all three. ## Not in this commit (deferred) This PR introduces the primitive — the module, the entity-centric API, the storage, the tests. It does NOT yet: - integrate compute_hallways_for_wing() into miner.py / convo_miner.py / format_miner.py post-mine flow (next PR — small one-line addition per miner alongside the existing _compute_topic_tunnels call) - refactor _compute_topic_tunnels_for_wing to BUILD ON hallways (the architecturally meaningful follow-up — own PR) - rename palace_graph.py:253's misleading docstring that calls cross-wing tunnels "hallways" (cleanup PR) - expose hallways through the MCP server (when there's a clear UX need for it) Each follow-up is small enough to review in isolation. Shipping the primitive first lets reviewers think about the API surface separately from the integration choices. Per the design sequence: Wing → Drawer-entities → Hallway → Tunnel. Hallways come first. This is them. --- mempalace/hallways.py | 301 +++++++++++++++++++++++++++++++++++++++++ tests/test_hallways.py | 282 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 583 insertions(+) create mode 100644 mempalace/hallways.py create mode 100644 tests/test_hallways.py diff --git a/mempalace/hallways.py b/mempalace/hallways.py new file mode 100644 index 0000000000..17ffce5243 --- /dev/null +++ b/mempalace/hallways.py @@ -0,0 +1,301 @@ +"""Hallways — within-wing entity-to-entity connectors. + +A **hallway** is a connection between two entities (people, projects, +concepts, interests) inside one wing, materialized from their +co-occurrence across that wing's drawers. Conceptually: + + WING → has DRAWERS (each tagged with entities) + entities → connected to other entities by HALLWAYS + (within-wing, built from drawer co-occurrence) + hallways → are the primitive + tunnels → use hallways to spawn + cross-wing connections + +If Aya and Lumi are both mentioned in 47 drawers across the diary, +letters, and ideas rooms, there's a hallway between them. If Aya +and "consciousness" co-occur in 19 drawers, there's a hallway between +them too. The hallway *is* the structural fact of "these two entities +travel together inside this wing." + +Mempalace's tunnel primitive in ``palace_graph.py`` connects rooms +across wings. This module fills the within-wing gap with an +entity-centric (not room-centric) model: hallways are about *who/what +relates to whom/what*, not *which rooms relate to which*. A planned +follow-up PR will refactor ``_compute_topic_tunnels_for_wing`` to +build cross-wing tunnels from hallway data (Wing → Drawer-entities → +Hallway → Tunnel). + +Persistence mirrors ``palace_graph._TUNNEL_FILE``: a JSON file under +``~/.mempalace/`` so the records survive across mines and are +inspectable / editable by hand if needed. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import tempfile +from collections import defaultdict +from datetime import datetime, timezone +from itertools import combinations +from typing import Optional + +logger = logging.getLogger("mempalace_hallways") + +# Persistence target. Mirrors ``palace_graph._TUNNEL_FILE`` so the storage +# pattern is uniform across the two related primitives. Tests override +# this via ``monkeypatch.setattr(hallways, "_HALLWAY_FILE", tmp_path/...)``. +_HALLWAY_FILE = os.path.join(os.path.expanduser("~"), ".mempalace", "hallways.json") + +_SCHEMA_VERSION = 1 + + +__all__ = [ + "compute_hallways_for_wing", + "list_hallways", + "delete_hallway", +] + + +# ───────────────────────────────────────────────────────────────────────────── +# Persistence — JSON file at _HALLWAY_FILE, restricted perms (0600) on POSIX +# ───────────────────────────────────────────────────────────────────────────── + + +def _load_hallways() -> list[dict]: + """Read all hallway records. Returns ``[]`` if the file is missing or corrupt.""" + if not os.path.exists(_HALLWAY_FILE): + return [] + try: + with open(_HALLWAY_FILE, encoding="utf-8") as f: + raw = json.load(f) + except (OSError, json.JSONDecodeError): + logger.debug("hallways: load failed, treating as empty", exc_info=True) + return [] + if isinstance(raw, dict) and "hallways" in raw: + return raw.get("hallways") or [] + if isinstance(raw, list): + return raw + return [] + + +def _save_hallways(hallways: list[dict]) -> None: + """Atomically persist hallway records to _HALLWAY_FILE. + + Uses an os.replace temp-file dance so a crash mid-write doesn't + corrupt the file. POSIX permission is restricted to 0600 because + hallways reveal within-wing entity connections that the user may + not want world-readable. + """ + directory = os.path.dirname(_HALLWAY_FILE) + os.makedirs(directory, exist_ok=True) + payload = { + "schema_version": _SCHEMA_VERSION, + "hallways": list(hallways), + } + fd, tmp_path = tempfile.mkstemp(prefix=".hallways-", suffix=".tmp", dir=directory) + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2, ensure_ascii=False) + try: + os.chmod(tmp_path, 0o600) + except OSError: + # Non-POSIX systems may not support chmod; not fatal. + pass + os.replace(tmp_path, _HALLWAY_FILE) + except Exception: + try: + os.unlink(tmp_path) + except OSError: + pass + raise + + +# ───────────────────────────────────────────────────────────────────────────── +# Core algorithm — compute entity-pair hallways for one wing +# ───────────────────────────────────────────────────────────────────────────── + + +def _parse_entities(value) -> list[str]: + """Drawer ``entities`` metadata is a semicolon-separated string. Parse it. + + Returns a deterministic *list* (not a set) because order matters for + the deduplication semantics below: a drawer that mentions ``Aya;Aya`` + should only contribute one Aya to the entity set for that drawer. + """ + if not value: + return [] + if isinstance(value, (list, tuple, set)): + items = [str(v).strip() for v in value if str(v).strip()] + elif isinstance(value, str): + items = [v.strip() for v in value.split(";") if v.strip()] + else: + return [] + # Dedupe while preserving first-seen order so id derivation is stable. + seen: set[str] = set() + result: list[str] = [] + for item in items: + if item not in seen: + seen.add(item) + result.append(item) + return result + + +def _hallway_id(wing: str, entity_a: str, entity_b: str) -> str: + """Deterministic id derived from wing + sorted entity pair. + + Sorting before hashing makes the id symmetric — (Aya, Lumi) and + (Lumi, Aya) produce the same record. So an idempotent re-mine + upserts the same hallway instead of creating two parallel records. + """ + a, b = sorted([entity_a, entity_b]) + key = f"{wing}::{a}::{b}".encode("utf-8") + suffix = hashlib.sha256(key).hexdigest()[:8] + return f"hallway_{wing}_{a}_{b}_{suffix}" + + +def compute_hallways_for_wing( + wing: str, + col=None, + min_count: int = 2, +) -> list[dict]: + """Compute entity-pair hallways for one wing. + + Algorithm: + 1. Query drawers for ``wing`` from ``col``. + 2. For each drawer with entities, every pair of distinct entities in + that drawer is one co-occurrence. Increment a counter for each + pair; also record the room the drawer lives in. + 3. For each (entity_a, entity_b) pair whose co-occurrence count is + ``>= min_count``, materialize a hallway record. The record + carries the pair, the count, and the set of rooms where they + co-occurred (useful context for navigation). + 4. Persist the full hallway list (records for other wings preserved, + this wing's records replaced) and return the just-computed list. + + Args: + wing: wing name to scan. + col: ChromaDB collection — must support ``.get(where=..., include=...)``. + If ``None``, returns ``[]`` (caller didn't supply a backing + store, so nothing to compute against). Tests pass a controlled + MagicMock. + min_count: minimum co-occurrence count required to materialize a + hallway between two entities. Default 2 — single co-occurrences + are noise (entities mentioned together once in one drawer); + two or more is a real signal. Clamped to ``>=1``. + + Returns: + List of hallway dicts created for this wing. Records for other + wings already on disk are preserved. + """ + if col is None: + logger.debug("compute_hallways_for_wing: no collection provided for %s", wing) + return [] + + min_count = max(1, int(min_count)) + + # 1. Query drawers for this wing. + try: + results = col.get(where={"wing": wing}, include=["metadatas"]) + except Exception: + logger.warning( + "compute_hallways_for_wing: collection.get failed for %s", wing, exc_info=True + ) + return [] + + metadatas = (results or {}).get("metadatas") or [] + if not metadatas: + return [] + + # 2. Walk drawers, counting entity-pair co-occurrence + tracking rooms. + # pair_counts: {(entity_a, entity_b): count} — keys always sorted to + # canonicalize the (a, b) vs (b, a) symmetry. + pair_counts: dict[tuple[str, str], int] = defaultdict(int) + pair_rooms: dict[tuple[str, str], set[str]] = defaultdict(set) + + for meta in metadatas: + if not isinstance(meta, dict): + continue + # Sentinel drawers carry no real content — skip them. + if meta.get("is_sentinel"): + continue + entities = _parse_entities(meta.get("entities")) + if len(entities) < 2: + # Need at least 2 entities for a pair to exist. + continue + room = meta.get("room") + room_str = room if isinstance(room, str) and room.strip() else None + + # Each unordered pair of distinct entities in this drawer is one + # co-occurrence. itertools.combinations already gives unordered + # pairs without repetition. + for a, b in combinations(entities, 2): + # Canonicalize order so (Aya, Lumi) and (Lumi, Aya) are the + # same key. Skip self-pairs defensively. + if a == b: + continue + key = tuple(sorted([a, b])) + pair_counts[key] += 1 + if room_str: + pair_rooms[key].add(room_str) + + if not pair_counts: + return [] + + # 3. Materialize hallway records for pairs above the threshold. + created: list[dict] = [] + created_at = datetime.now(timezone.utc).isoformat() + for key in sorted(pair_counts.keys()): + count = pair_counts[key] + if count < min_count: + continue + entity_a, entity_b = key + rooms = sorted(pair_rooms.get(key, set())) + room_summary = ", ".join(rooms[:3]) if rooms else "(no room tags)" + if len(rooms) > 3: + room_summary += f", +{len(rooms) - 3} more" + created.append( + { + "id": _hallway_id(wing, entity_a, entity_b), + "wing": wing, + "entity_a": entity_a, + "entity_b": entity_b, + "co_occurrence_count": count, + "rooms": rooms, + "label": f"{entity_a} ↔ {entity_b} (co-occur in {count} drawers across {len(rooms) or 'no'} room{'s' if len(rooms) != 1 else ''}: {room_summary})", + "created_at": created_at, + "created_by": "auto", + } + ) + + # 4. Persist — preserve other-wing records, replace this wing's records. + existing = _load_hallways() + preserved = [h for h in existing if h.get("wing") != wing] + _save_hallways(preserved + created) + + return created + + +# ───────────────────────────────────────────────────────────────────────────── +# Query API — list_hallways, delete_hallway +# ───────────────────────────────────────────────────────────────────────────── + + +def list_hallways(wing: Optional[str] = None) -> list[dict]: + """List hallway records. Filter by ``wing`` if specified.""" + all_hallways = _load_hallways() + if wing is None: + return list(all_hallways) + return [h for h in all_hallways if h.get("wing") == wing] + + +def delete_hallway(hallway_id: str) -> bool: + """Remove one hallway record by id. Returns True if a record was removed.""" + hallways = _load_hallways() + filtered = [h for h in hallways if h.get("id") != hallway_id] + if len(filtered) == len(hallways): + return False + _save_hallways(filtered) + return True diff --git a/tests/test_hallways.py b/tests/test_hallways.py new file mode 100644 index 0000000000..a0675b18c0 --- /dev/null +++ b/tests/test_hallways.py @@ -0,0 +1,282 @@ +"""Tests for the within-wing hallway primitive. + +Hallways are bridges INSIDE a wing that connect entities (people, +projects, concepts, interests) to each other, materialized from +drawer-level co-occurrence. Two entities are linked by a hallway when +they appear together in enough drawers across the wing. + +This file is RED-first. The corresponding implementation lives in +``mempalace/hallways.py`` and is written to make these tests pass. +""" + +from unittest.mock import MagicMock, patch + + +# Mock chromadb at import time so the hallways module can be loaded even +# in environments where chromadb isn't installed. Mirrors the pattern in +# ``tests/test_palace_graph_tunnels.py``. +with patch.dict("sys.modules", {"chromadb": MagicMock()}): + from mempalace import hallways as hallways_mod + + +def _use_tmp_hallway_file(monkeypatch, tmp_path): + """Redirect hallway persistence to a per-test JSON file.""" + hallway_file = tmp_path / "hallways.json" + monkeypatch.setattr(hallways_mod, "_HALLWAY_FILE", str(hallway_file)) + return hallway_file + + +def _fake_collection(drawers): + """Build a MagicMock collection whose .get() returns the given drawer set.""" + col = MagicMock() + metadatas = [d for d in drawers] + ids = [f"drawer_{i}" for i in range(len(drawers))] + col.get.return_value = {"ids": ids, "metadatas": metadatas} + return col + + +# ───────────────────────────────────────────────────────────────────────────── +# Storage primitives — _load_hallways / _save_hallways +# ───────────────────────────────────────────────────────────────────────────── + + +class TestHallwayStorage: + def test_load_hallways_missing_file_returns_empty_list(self, tmp_path, monkeypatch): + _use_tmp_hallway_file(monkeypatch, tmp_path) + assert hallways_mod._load_hallways() == [] + + def test_load_hallways_corrupt_file_returns_empty_list(self, tmp_path, monkeypatch): + hallway_file = _use_tmp_hallway_file(monkeypatch, tmp_path) + hallway_file.write_text("{not valid json", encoding="utf-8") + assert hallways_mod._load_hallways() == [] + + def test_save_and_load_round_trip(self, tmp_path, monkeypatch): + _use_tmp_hallway_file(monkeypatch, tmp_path) + sample = [ + { + "id": "hallway_wing_aya_aya_lumi_abc12345", + "wing": "wing_aya", + "entity_a": "Aya", + "entity_b": "Lumi", + "co_occurrence_count": 47, + "rooms": ["diary", "letters"], + "label": "Aya ↔ Lumi (co-occur in 47 drawers across 2 rooms)", + } + ] + hallways_mod._save_hallways(sample) + assert hallways_mod._load_hallways() == sample + + +# ───────────────────────────────────────────────────────────────────────────── +# compute_hallways_for_wing — entity-pair co-occurrence algorithm +# ───────────────────────────────────────────────────────────────────────────── + + +class TestComputeHallways: + def test_returns_empty_for_unknown_wing(self, tmp_path, monkeypatch): + """Wing with no drawers → no hallways, no crash.""" + _use_tmp_hallway_file(monkeypatch, tmp_path) + col = _fake_collection([]) + result = hallways_mod.compute_hallways_for_wing("wing_nonexistent", col=col) + assert result == [] + + def test_returns_empty_when_no_drawer_has_two_entities(self, tmp_path, monkeypatch): + """A drawer must mention >= 2 entities to contribute a pair.""" + _use_tmp_hallway_file(monkeypatch, tmp_path) + col = _fake_collection( + [ + {"wing": "wing_aya", "room": "diary", "entities": "Aya"}, # only one + {"wing": "wing_aya", "room": "diary", "entities": ""}, # none + ] + ) + result = hallways_mod.compute_hallways_for_wing("wing_aya", col=col) + assert result == [] + + def test_creates_hallway_for_entity_pair_when_threshold_met(self, tmp_path, monkeypatch): + """Two entities co-occurring in >= min_count drawers → one hallway record. + + With min_count=2, Aya↔Lumi appear together in 3 drawers; that's a hallway. + """ + _use_tmp_hallway_file(monkeypatch, tmp_path) + col = _fake_collection( + [ + {"wing": "wing_aya", "room": "diary", "entities": "Aya;Lumi"}, + {"wing": "wing_aya", "room": "diary", "entities": "Aya;Lumi;Ever"}, + {"wing": "wing_aya", "room": "letters", "entities": "Aya;Lumi"}, + ] + ) + result = hallways_mod.compute_hallways_for_wing("wing_aya", col=col, min_count=2) + # Find the Aya↔Lumi hallway (other pairs like Aya↔Ever might also be present) + aya_lumi = [h for h in result if {h["entity_a"], h["entity_b"]} == {"Aya", "Lumi"}] + assert len(aya_lumi) == 1 + hallway = aya_lumi[0] + assert hallway["wing"] == "wing_aya" + assert hallway["co_occurrence_count"] == 3 + assert set(hallway["rooms"]) == {"diary", "letters"} + + def test_connects_person_to_concept(self, tmp_path, monkeypatch): + """Entities aren't only people — projects/concepts/interests count too. + + The entity tag treats 'consciousness' the same as 'Aya'; both are + just tokens in the drawer's entities field. So Aya↔consciousness is + a valid hallway when they co-occur enough. + """ + _use_tmp_hallway_file(monkeypatch, tmp_path) + col = _fake_collection( + [ + {"wing": "wing_aya", "room": "diary", "entities": "Aya;consciousness"}, + {"wing": "wing_aya", "room": "research", "entities": "Aya;consciousness"}, + {"wing": "wing_aya", "room": "ideas", "entities": "Aya;consciousness;Lumi"}, + ] + ) + result = hallways_mod.compute_hallways_for_wing("wing_aya", col=col, min_count=2) + aya_consciousness = [ + h for h in result if {h["entity_a"], h["entity_b"]} == {"Aya", "consciousness"} + ] + assert len(aya_consciousness) == 1 + assert aya_consciousness[0]["co_occurrence_count"] == 3 + assert set(aya_consciousness[0]["rooms"]) == {"diary", "research", "ideas"} + + def test_respects_min_count_threshold(self, tmp_path, monkeypatch): + """min_count=3 filters out pairs that only co-occur twice.""" + _use_tmp_hallway_file(monkeypatch, tmp_path) + col = _fake_collection( + [ + {"wing": "wing_aya", "room": "diary", "entities": "Aya;Lumi"}, + {"wing": "wing_aya", "room": "letters", "entities": "Aya;Lumi"}, + ] + ) + result = hallways_mod.compute_hallways_for_wing("wing_aya", col=col, min_count=3) + assert result == [] + + def test_creates_deterministic_id_per_entity_pair(self, tmp_path, monkeypatch): + """Same wing + same entity pair → same hallway id (idempotent re-runs).""" + _use_tmp_hallway_file(monkeypatch, tmp_path) + col = _fake_collection( + [ + {"wing": "wing_aya", "room": "diary", "entities": "Aya;Lumi"}, + {"wing": "wing_aya", "room": "letters", "entities": "Aya;Lumi"}, + ] + ) + first = hallways_mod.compute_hallways_for_wing("wing_aya", col=col, min_count=2) + _use_tmp_hallway_file(monkeypatch, tmp_path) + second = hallways_mod.compute_hallways_for_wing("wing_aya", col=col, min_count=2) + # Find the Aya↔Lumi record in both runs; ids must match. + f_id = next(h["id"] for h in first if {h["entity_a"], h["entity_b"]} == {"Aya", "Lumi"}) + s_id = next(h["id"] for h in second if {h["entity_a"], h["entity_b"]} == {"Aya", "Lumi"}) + assert f_id == s_id + assert f_id.startswith("hallway_") + + def test_entity_pair_is_symmetric(self, tmp_path, monkeypatch): + """Drawer says 'Aya;Lumi'; another says 'Lumi;Aya' — same hallway.""" + _use_tmp_hallway_file(monkeypatch, tmp_path) + col = _fake_collection( + [ + {"wing": "wing_aya", "room": "diary", "entities": "Aya;Lumi"}, + {"wing": "wing_aya", "room": "letters", "entities": "Lumi;Aya"}, + ] + ) + result = hallways_mod.compute_hallways_for_wing("wing_aya", col=col, min_count=2) + aya_lumi = [h for h in result if {h["entity_a"], h["entity_b"]} == {"Aya", "Lumi"}] + # Symmetry: the two drawers count as 2 co-occurrences, not 0 (no + # double-bookkeeping despite the swapped order). + assert len(aya_lumi) == 1 + assert aya_lumi[0]["co_occurrence_count"] == 2 + + def test_persists_to_json(self, tmp_path, monkeypatch): + """After compute, _load_hallways() returns the new records.""" + hallway_file = _use_tmp_hallway_file(monkeypatch, tmp_path) + col = _fake_collection( + [ + {"wing": "wing_aya", "room": "diary", "entities": "Aya;Lumi"}, + {"wing": "wing_aya", "room": "letters", "entities": "Aya;Lumi"}, + ] + ) + hallways_mod.compute_hallways_for_wing("wing_aya", col=col, min_count=2) + assert hallway_file.exists() + loaded = hallways_mod._load_hallways() + assert any({h["entity_a"], h["entity_b"]} == {"Aya", "Lumi"} for h in loaded) + + def test_tracks_rooms_across_co_occurrences(self, tmp_path, monkeypatch): + """A hallway records the set of rooms where its entities co-occurred.""" + _use_tmp_hallway_file(monkeypatch, tmp_path) + col = _fake_collection( + [ + {"wing": "wing_aya", "room": "diary", "entities": "Aya;Lumi"}, + {"wing": "wing_aya", "room": "letters", "entities": "Aya;Lumi"}, + {"wing": "wing_aya", "room": "diary", "entities": "Aya;Lumi"}, + ] + ) + result = hallways_mod.compute_hallways_for_wing("wing_aya", col=col, min_count=2) + h = next(h for h in result if {h["entity_a"], h["entity_b"]} == {"Aya", "Lumi"}) + assert set(h["rooms"]) == {"diary", "letters"} + assert h["co_occurrence_count"] == 3 # 3 drawers, not 3 rooms + + def test_skips_sentinel_drawers(self, tmp_path, monkeypatch): + """Sentinels exist for file_already_mined() bookkeeping. Skip them.""" + _use_tmp_hallway_file(monkeypatch, tmp_path) + col = _fake_collection( + [ + { + "wing": "wing_aya", + "room": "documents", + "entities": "Aya;Lumi", + "is_sentinel": True, + }, + { + "wing": "wing_aya", + "room": "documents", + "entities": "Aya;Lumi", + "is_sentinel": True, + }, + ] + ) + result = hallways_mod.compute_hallways_for_wing("wing_aya", col=col, min_count=2) + assert result == [] + + +# ───────────────────────────────────────────────────────────────────────────── +# Query API — list_hallways, delete_hallway +# ───────────────────────────────────────────────────────────────────────────── + + +class TestHallwayQuery: + def test_list_hallways_returns_all_when_no_filter(self, tmp_path, monkeypatch): + _use_tmp_hallway_file(monkeypatch, tmp_path) + hallways_mod._save_hallways( + [ + {"id": "h1", "wing": "wing_aya", "entity_a": "Aya", "entity_b": "Lumi"}, + {"id": "h2", "wing": "wing_lumi", "entity_a": "Lumi", "entity_b": "Ever"}, + ] + ) + assert len(hallways_mod.list_hallways()) == 2 + + def test_list_hallways_filters_by_wing(self, tmp_path, monkeypatch): + _use_tmp_hallway_file(monkeypatch, tmp_path) + hallways_mod._save_hallways( + [ + {"id": "h1", "wing": "wing_aya", "entity_a": "Aya", "entity_b": "Lumi"}, + {"id": "h2", "wing": "wing_lumi", "entity_a": "Lumi", "entity_b": "Ever"}, + ] + ) + result = hallways_mod.list_hallways(wing="wing_aya") + assert len(result) == 1 + assert result[0]["id"] == "h1" + + def test_delete_hallway_removes_record(self, tmp_path, monkeypatch): + _use_tmp_hallway_file(monkeypatch, tmp_path) + hallways_mod._save_hallways( + [ + {"id": "h1", "wing": "wing_aya", "entity_a": "Aya", "entity_b": "Lumi"}, + {"id": "h2", "wing": "wing_aya", "entity_a": "Aya", "entity_b": "Ever"}, + ] + ) + assert hallways_mod.delete_hallway("h1") is True + remaining = hallways_mod._load_hallways() + assert len(remaining) == 1 + assert remaining[0]["id"] == "h2" + + def test_delete_hallway_unknown_id_returns_false(self, tmp_path, monkeypatch): + _use_tmp_hallway_file(monkeypatch, tmp_path) + hallways_mod._save_hallways([{"id": "h1", "wing": "wing_aya"}]) + assert hallways_mod.delete_hallway("nonexistent") is False From fc13be568a162be9c8be7ed23f29e3fe484cc25c Mon Sep 17 00:00:00 2001 From: Milla J <232237854+milla-jovovich@users.noreply.github.com> Date: Wed, 20 May 2026 04:08:02 -0700 Subject: [PATCH 30/39] feat(miner): integrate compute_hallways_for_wing into post-mine flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires the hallway primitive (from PR #1558) into the project miner so that every mine that touches a wing also materializes within-wing entity hallways for that wing. Without this integration, the hallway module is dead code — no miner triggers it, no hallways ever land in ~/.mempalace/hallways.json. ## What this commit does 1. Adds a module-level import of ``compute_hallways_for_wing`` from ``.hallways`` near the top of ``miner.py``. Module-level (not lazy) so tests can patch it as ``mempalace.miner.compute_hallways_for_wing`` — lazy imports inside a function wouldn't expose the seam. 2. In ``_mine_impl``, immediately after the existing ``_compute_topic_tunnels_for_wing(wing)`` post-mine block, adds a parallel hallway block. The block: - calls ``compute_hallways_for_wing(wing, col=collection)`` - prints the count if any hallways were materialized - wraps the whole thing in try/except so a hallway-compute failure is logged + degraded, never propagated. Mirrors the tunnel block's fault-tolerance pattern exactly. Hallway computation is a derived analytic, not load-bearing for the drawer write that already committed above. ## Stacking This PR stacks on PR #1558 (which introduces the hallway primitive module). PR #1558 is the prerequisite — without it, the import ``from .hallways import compute_hallways_for_wing`` doesn't resolve. Base branch for this PR is ``feat/hallways-within-wing-connectors`` (PR #1558's branch). When PR #1558 merges to develop, GitHub will auto-update this PR's base to develop and the diff will reduce to just the miner.py and tests/test_miner.py changes. ## Out of scope (deferred to follow-up PRs) - ``format_miner.py`` integration: lives on a different branch (PR #1555) so the parallel call there goes in as a follow-up amendment to that branch (or a separate post-merge PR). - ``convo_miner.py`` integration: convo_miner currently doesn't call ``_compute_topic_tunnels_for_wing`` either. Adding both calls is a separate concerned PR about convo_miner parity, not just hallways. - Refactoring ``_compute_topic_tunnels_for_wing`` to BUILD ON hallways (rather than computing from raw topic words): the architecturally meaningful follow-up that completes the Wing → Drawer-entities → Hallway → Tunnel sequence. Real refactor, separate PR. ## Tests Two new RED-first tests in ``tests/test_miner.py``: test_mine_computes_hallways_for_wing_post_mine Stubs ``mempalace.miner.compute_hallways_for_wing`` via monkeypatch. Runs a real mine into a tmp palace. Asserts the stub was called exactly once, with the wing name from mempalace.yaml and a live ChromaDB collection (not None). test_mine_hallway_failure_does_not_crash_mine Stubs the hallway function to raise. Runs a real mine. Asserts mine() doesn't propagate, and that the drawer write (which happens BEFORE the hallway block) still committed. Both RED before this commit (AttributeError — module had no attribute ``compute_hallways_for_wing``). Both GREEN after. ## Verification ruff check mempalace/miner.py tests/test_miner.py → All checks passed! ruff format --check ... → 2 files already formatted (pinned 0.15.9) pytest tests/test_miner.py → 47 passed (the 2 new + 45 pre-existing) pytest -q (full mempalace suite) → 1938 passed, 1 skipped, 0 regressions Linux CI parity replicated locally via OrbStack containers (Python 3.9, 3.11, 3.13): 2/2 new integration tests pass, ruff clean on all three. --- mempalace/miner.py | 21 ++++++++++ tests/test_miner.py | 99 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/mempalace/miner.py b/mempalace/miner.py index 0eab9dc72e..bd53b7ed99 100644 --- a/mempalace/miner.py +++ b/mempalace/miner.py @@ -33,6 +33,12 @@ upsert_closet_lines, ) +# Module-level import so tests can patch it as +# ``mempalace.miner.compute_hallways_for_wing``. The integration call +# lives at the end of _mine_impl, alongside the existing +# ``_compute_topic_tunnels_for_wing`` post-mine block. +from .hallways import compute_hallways_for_wing + logger = logging.getLogger("mempalace_mcp") READABLE_EXTENSIONS = { @@ -1248,6 +1254,21 @@ def _mine_impl( file=sys.stderr, ) + # Within-wing hallways: link entities (people, projects, concepts) + # that co-occur in drawers across this wing's rooms. Mirrors the + # tunnel-compute fault-tolerance pattern — hallway computation + # must never fail a mine; it's a derived analytic, not load-bearing + # for the drawer write that already committed above. + try: + hallways_created = compute_hallways_for_wing(wing, col=collection) + if hallways_created: + print(f"\n Hallways: +{len(hallways_created)} within-wing entity link(s)") + except Exception as e: + print( + f"\n WARNING: hallway computation skipped — {e}", + file=sys.stderr, + ) + print(f"\n{'=' * 55}") print(" Done.") print(f" Files processed: {len(files) - files_skipped}") diff --git a/tests/test_miner.py b/tests/test_miner.py index dedfc97bea..2a2b0e9227 100644 --- a/tests/test_miner.py +++ b/tests/test_miner.py @@ -55,6 +55,105 @@ def test_project_mining(): shutil.rmtree(tmpdir, ignore_errors=True) +def test_mine_computes_hallways_for_wing_post_mine(monkeypatch): + """After mine() completes (non-dry-run), compute_hallways_for_wing must be + called once with the wing name and the live collection. + + This is the integration test for the hallway primitive — without this + call, the hallway module is dead code (no miner triggers it). Mirrors + the existing tunnel-computation integration pattern at miner.py:1241. + """ + + from mempalace import miner as miner_mod + + hallway_calls = [] + + def fake_compute(wing, col=None, min_count=2): + hallway_calls.append({"wing": wing, "col": col, "min_count": min_count}) + return [] # no hallways materialized — that's not what we're testing + + # Patch at the call site (mempalace.miner.compute_hallways_for_wing) so + # the integration in mine() routes through our stub. + monkeypatch.setattr(miner_mod, "compute_hallways_for_wing", fake_compute) + + tmpdir = tempfile.mkdtemp() + try: + project_root = Path(tmpdir).resolve() + os.makedirs(project_root / "backend") + write_file( + project_root / "backend" / "app.py", + "def main():\n print('hello world')\n" * 20, + ) + with open(project_root / "mempalace.yaml", "w") as f: + yaml.dump( + { + "wing": "test_project", + "rooms": [{"name": "backend", "description": "Backend code"}], + }, + f, + ) + + palace_path = project_root / "palace" + mine(str(project_root), str(palace_path)) + + # Must have been called exactly once, with our wing name + a live + # collection. We don't pin min_count — the integration may pick a + # default that differs from the function's own default. + assert len(hallway_calls) == 1, ( + f"expected compute_hallways_for_wing to be called once, got {len(hallway_calls)}" + ) + call = hallway_calls[0] + assert call["wing"] == "test_project" + assert call["col"] is not None, ( + "must pass the live collection so hallways can query drawers" + ) + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def test_mine_hallway_failure_does_not_crash_mine(monkeypatch): + """If compute_hallways_for_wing raises, the mine must still complete. + + Mirrors the try/except wrap around the existing tunnel-computation block + at miner.py:1244-1249. Hallway computation is a derived analytic, not + load-bearing for the drawer write itself. + """ + from mempalace import miner as miner_mod + + def angry_compute(wing, col=None, min_count=2): + raise RuntimeError("simulated hallway-compute explosion") + + monkeypatch.setattr(miner_mod, "compute_hallways_for_wing", angry_compute) + + tmpdir = tempfile.mkdtemp() + try: + project_root = Path(tmpdir).resolve() + os.makedirs(project_root / "backend") + write_file( + project_root / "backend" / "app.py", + "def main():\n print('hello world')\n" * 20, + ) + with open(project_root / "mempalace.yaml", "w") as f: + yaml.dump( + { + "wing": "test_project", + "rooms": [{"name": "backend", "description": "Backend code"}], + }, + f, + ) + + palace_path = project_root / "palace" + # Must NOT raise — the failure has to be caught + logged but not propagated. + mine(str(project_root), str(palace_path)) + + # Drawer-write side of the mine still committed. + client = chromadb.PersistentClient(path=str(palace_path)) + col = client.get_collection("mempalace_drawers") + assert col.count() > 0 + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + def test_load_config_uses_defaults_when_yaml_missing(): tmpdir = tempfile.mkdtemp() try: From 7bddb65cb485e2c2233042c65a782a8b28741b2f Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Wed, 20 May 2026 15:31:57 -0300 Subject: [PATCH 31/39] fix: stale-PID timeout, MCP idle exit, structured errors (#1552) - Add MEMPALACE_MINE_TIMEOUT_HOURS (default 2h): PID files now record '{pid} {unix_timestamp}'; _mine_already_running() treats alive-but-old processes as stale, unblocking queued mines after a ChromaDB hang. Backward-compatible: bare-PID files (old format) treated as stale. - Add MEMPALACE_MCP_IDLE_HOURS (default 8h): daemon watchdog thread in mcp_server calls sys.exit(0) after the configured idle period, preventing accumulation of stale server processes holding ChromaDB/HNSW file handles. Set to 0 to disable. - Enrich _internal_tool_error() with optional exc parameter: adds data: {error_class, message} to JSON-RPC error body so callers can distinguish lock contention, ChromaDB transients, and segfaults without scraping the message string. MineAlreadyRunning handler in tool_sync() adds error_class: 'LockHeldByOtherProcess' to the result dict. - Update miner._cleanup_mine_pid_file() to parse first whitespace token as PID (handles both old and new PID file formats). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mempalace/hooks_cli.py | 66 +++++++++++++++++++++--- mempalace/mcp_server.py | 79 ++++++++++++++++++++++++++--- mempalace/miner.py | 7 ++- tests/test_hooks_cli.py | 92 +++++++++++++++++++++++++++++---- tests/test_mcp_server.py | 106 ++++++++++++++++++++++++++++++++++++--- 5 files changed, 321 insertions(+), 29 deletions(-) diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index 158a0fa0f0..d0c741b798 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -12,6 +12,7 @@ import re import subprocess import sys +import time from datetime import datetime from pathlib import Path from typing import Optional @@ -281,6 +282,31 @@ def _get_mine_targets() -> list[tuple[str, str]]: # own slot on exit without scanning the whole directory. _MINE_PID_FILE_ENV = "MEMPALACE_MINE_PID_FILE" +# Maximum wall-clock hours a mine subprocess is allowed to run before its +# PID slot is treated as stale (even if the process is still alive). A +# wedged mine — e.g. one that is blocking indefinitely on ChromaDB +# cold-init under concurrent Windows load (#1552) — would otherwise hold +# its slot forever. Set MEMPALACE_MINE_TIMEOUT_HOURS=0 to disable the +# timeout (slots are reclaimed only when the PID is dead). +_MINE_TIMEOUT_HOURS_ENV = "MEMPALACE_MINE_TIMEOUT_HOURS" +_MINE_TIMEOUT_HOURS_DEFAULT = 2.0 + + +def _mine_slot_timeout_secs() -> float: + """Return the configured mine-slot timeout in seconds. + + Reads ``MEMPALACE_MINE_TIMEOUT_HOURS`` from the environment (float). + Returns 0 if the env var is set to 0 or is not parseable. + """ + raw = os.environ.get(_MINE_TIMEOUT_HOURS_ENV, "") + if raw: + try: + hours = float(raw) + return max(0.0, hours) * 3600 + except ValueError: + pass + return _MINE_TIMEOUT_HOURS_DEFAULT * 3600 + def _pid_file_for_cmd(cmd: list[str]) -> Path: """Return the per-target PID file path for a mine subcommand. @@ -333,23 +359,51 @@ def _pid_alive(pid: int) -> bool: def _mine_already_running(cmd: list[str]) -> bool: - """Return True if a previous mine for ``cmd``'s target is still alive.""" + """Return True if a previous mine for ``cmd``'s target is still alive. + + The PID file format is ``{pid} {unix_timestamp}`` (timestamp added in + #1552 to detect wedged subprocesses). Old-format files (bare ``{pid}``) + are treated as having a start time of 0 — effectively "infinitely old" — + so they are immediately considered stale once the configured timeout + elapses. + + A process is considered stale (and this function returns False) when: + - the PID is dead, OR + - the configured mine timeout is > 0 AND the process has been running + longer than the timeout. + """ pid_file = _pid_file_for_cmd(cmd) try: recorded = pid_file.read_text().strip() except OSError: return False - if not recorded.isdigit(): + if not recorded: + return False + parts = recorded.split(None, 1) + if not parts[0].isdigit(): + return False + pid = int(parts[0]) + if not _pid_alive(pid): return False - return _pid_alive(int(recorded)) + timeout_secs = _mine_slot_timeout_secs() + if timeout_secs > 0: + start_ts = float(parts[1]) if len(parts) > 1 and parts[1] else 0.0 + if time.time() - start_ts > timeout_secs: + return False + return True def _create_mine_slot_with_placeholder(pid_file: Path) -> Path: - """Atomically create a mine PID slot and write this hook PID into it.""" + """Atomically create a mine PID slot and write this hook PID into it. + + The slot body is ``{pid} {unix_timestamp}`` so that stale-by-age + detection in ``_mine_already_running`` can determine how long the + recorded process has been running (#1552). + """ fd = os.open(str(pid_file), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) try: with os.fdopen(fd, "w", encoding="ascii") as f: - f.write(str(os.getpid())) + f.write(f"{os.getpid()} {int(time.time())}") except OSError: try: os.close(fd) @@ -437,7 +491,7 @@ def _spawn_mine(cmd: list) -> None: pass raise try: - pid_file.write_text(str(proc.pid)) + pid_file.write_text(f"{proc.pid} {int(time.time())}") except OSError: pass diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 41cf9c0c21..90dfdda7fa 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -176,6 +176,27 @@ def _parse_args(): _kg_cache_lock = threading.Lock() _palace_flag_given: bool = bool(_args.palace) +# MCP server idle auto-exit (#1552). Stale MCP servers from ended Claude +# Code sessions do not self-terminate, accumulating ChromaDB/HNSW file +# handles on Windows. When MEMPALACE_MCP_IDLE_HOURS is set (or defaults +# to 8 h), a background daemon thread exits the process once no request +# has been handled for that long. Set to 0 to disable. +_MCP_IDLE_HOURS_ENV = "MEMPALACE_MCP_IDLE_HOURS" +_MCP_IDLE_HOURS_DEFAULT = 8.0 +_last_request_time: float = time.monotonic() + + +def _mcp_idle_timeout_secs() -> float: + """Return the configured MCP idle timeout in seconds (0 = disabled).""" + raw = os.environ.get(_MCP_IDLE_HOURS_ENV, "") + if raw: + try: + hours = float(raw) + return max(0.0, hours) * 3600 + except ValueError: + pass + return _MCP_IDLE_HOURS_DEFAULT * 3600 + def _resolve_kg_path() -> str: if _palace_flag_given: @@ -1204,7 +1225,11 @@ def tool_sync(project_dir: str = None, wing: str = None, apply: bool = False): # below, otherwise MineAlreadyRunning and ValueError fall into the # generic "sync failed" branch and break the structured-error tests. except MineAlreadyRunning as exc: - return {"success": False, "error": f"another mine is in progress: {exc}"} + return { + "success": False, + "error": f"another mine is in progress: {exc}", + "error_class": "LockHeldByOtherProcess", + } except ValueError as exc: return {"success": False, "error": str(exc)} except Exception as exc: @@ -2286,22 +2311,30 @@ def tool_reconnect(): ] -def _internal_tool_error(req_id, tool_name: str) -> dict: +def _internal_tool_error(req_id, tool_name: str, exc: BaseException = None) -> dict: logger.exception(f"Tool error in {tool_name}") + error: dict = {"code": -32000, "message": "Internal tool error"} + if exc is not None: + error["data"] = { + "error_class": type(exc).__name__, + "message": str(exc), + } return { "jsonrpc": "2.0", "id": req_id, - "error": {"code": -32000, "message": "Internal tool error"}, + "error": error, } def handle_request(request): + global _last_request_time if not isinstance(request, dict): return { "jsonrpc": "2.0", "id": None, "error": {"code": -32600, "message": "Invalid Request"}, } + _last_request_time = time.monotonic() method = request.get("method") or "" params = request.get("params") or {} req_id = request.get("id") @@ -2449,9 +2482,9 @@ def handle_request(request): "message": f"Missing required {word} {quoted} for tool {tool_name}", }, } - return _internal_tool_error(req_id, tool_name) - except Exception: - return _internal_tool_error(req_id, tool_name) + return _internal_tool_error(req_id, tool_name, e) + except Exception as exc: + return _internal_tool_error(req_id, tool_name, exc) # Notifications (missing id) must never get a response if req_id is None: @@ -2626,6 +2659,37 @@ def _maybe_eager_warmup_embedder() -> None: ) +def _start_idle_exit_watchdog() -> None: + """Start a daemon thread that exits the process after an idle period. + + When no request has been handled for ``MEMPALACE_MCP_IDLE_HOURS`` + (default 8 h), the thread calls ``sys.exit(0)`` so that stale MCP + servers from ended Claude Code sessions do not accumulate ChromaDB / + HNSW file handles on Windows (#1552). + + Set ``MEMPALACE_MCP_IDLE_HOURS=0`` to disable the watchdog. + """ + timeout = _mcp_idle_timeout_secs() + if timeout <= 0: + return + check_interval = min(60.0, timeout / 4) + + def _watchdog() -> None: + while True: + time.sleep(check_interval) + idle = time.monotonic() - _last_request_time + if idle >= timeout: + logger.info( + "MCP server idle for %.1f h (limit %.1f h); exiting to release file handles.", + idle / 3600, + timeout / 3600, + ) + sys.exit(0) + + t = threading.Thread(target=_watchdog, name="mcp-idle-watchdog", daemon=True) + t.start() + + def main(): """MCP server entry point for the ``mempalace-mcp`` console script. @@ -2661,6 +2725,9 @@ def main(): # does not pay the ONNX/CoreML cold-load tax under the MCP client # timeout (#1495). Default off — preserves current startup latency. _maybe_eager_warmup_embedder() + # Idle auto-exit: release ChromaDB file handles from stale servers + # that outlived their Claude Code session (#1552). + _start_idle_exit_watchdog() while True: try: line = sys.stdin.readline() diff --git a/mempalace/miner.py b/mempalace/miner.py index 0eab9dc72e..0a86d965ae 100644 --- a/mempalace/miner.py +++ b/mempalace/miner.py @@ -1323,7 +1323,12 @@ def _cleanup_mine_pid_file() -> None: if not pid_file.exists(): return recorded = pid_file.read_text().strip() - if recorded and recorded.isdigit() and int(recorded) == os.getpid(): + # PID file format: "{pid} {unix_timestamp}" (timestamp added in + # #1552 for stale-by-age detection). Old-format files (bare + # "{pid}") are also handled: split on whitespace and take the + # first token as the PID. + pid_token = recorded.split()[0] if recorded else "" + if pid_token and pid_token.isdigit() and int(pid_token) == os.getpid(): pid_file.unlink() except OSError: # Best-effort cleanup; never fail the mine over PID bookkeeping. diff --git a/tests/test_hooks_cli.py b/tests/test_hooks_cli.py index be485c0615..f22ca68867 100644 --- a/tests/test_hooks_cli.py +++ b/tests/test_hooks_cli.py @@ -694,7 +694,9 @@ def test_claim_mine_slot_writes_live_placeholder_pid(tmp_path): pid_file = _claim_mine_slot(cmd) assert pid_file == _pid_file_for_cmd(cmd) - assert pid_file.read_text().strip() == str(os.getpid()) + # Format: "{pid} {unix_timestamp}" — first token must be our PID. + content = pid_file.read_text().strip() + assert content.split()[0] == str(os.getpid()) assert _mine_already_running(cmd) is True assert _claim_mine_slot(cmd) is None @@ -715,7 +717,8 @@ def test_claim_mine_slot_reclaimed_slot_writes_live_placeholder_pid(tmp_path): reclaimed = _claim_mine_slot(cmd) assert reclaimed == pid_file - assert pid_file.read_text().strip() == str(os.getpid()) + # Format: "{pid} {unix_timestamp}" — first token must be our PID. + assert pid_file.read_text().strip().split()[0] == str(os.getpid()) def test_maybe_auto_ingest_ignores_transcript_arg_path(tmp_path): @@ -789,7 +792,8 @@ def test_maybe_auto_ingest_skips_when_mine_running(tmp_path): ] pid_file = _pid_file_for_cmd(cmd) pid_file.parent.mkdir(parents=True, exist_ok=True) - pid_file.write_text(str(os.getpid())) + import time as _time + pid_file.write_text(f"{os.getpid()} {int(_time.time())}") with patch("mempalace.hooks_cli._mempalace_python", return_value=sys.executable): with patch("mempalace.hooks_cli.subprocess.Popen") as mock_popen: _maybe_auto_ingest() @@ -855,6 +859,8 @@ def test_spawn_mine_uses_detached_kwargs(tmp_path): def test_spawn_mine_skips_when_target_running(tmp_path): """A second spawn for the same cmd target while the first is alive must skip.""" + import time as _time + pid_dir = tmp_path / "mine_pids" with patch("mempalace.hooks_cli.STATE_DIR", tmp_path): with patch("mempalace.hooks_cli._MINE_PID_DIR", pid_dir): @@ -863,7 +869,7 @@ def test_spawn_mine_skips_when_target_running(tmp_path): cmd = ["mempalace", "mine", "/tmp/proj", "--mode", "projects"] pid_file = _pid_file_for_cmd(cmd) pid_file.parent.mkdir(parents=True, exist_ok=True) - pid_file.write_text(str(os.getpid())) # live PID + pid_file.write_text(f"{os.getpid()} {int(_time.time())}") # live PID, fresh with patch("mempalace.hooks_cli.subprocess.Popen") as mock_popen: _spawn_mine(cmd) @@ -901,8 +907,9 @@ def test_spawn_mine_reclaims_stale_slot(tmp_path): mock_popen.return_value.pid = 4242 _spawn_mine(cmd) mock_popen.assert_called_once() - # New PID is recorded in the reclaimed slot. - assert pid_file.read_text().strip() == "4242" + # New PID is recorded in the reclaimed slot (format: "{pid} {timestamp}"). + content = pid_file.read_text().strip() + assert content.split()[0] == "4242" def test_spawn_mine_releases_slot_on_oserror(tmp_path): @@ -978,7 +985,8 @@ def test_ingest_transcript_skips_when_target_running(tmp_path): ] pid_file = _pid_file_for_cmd(expected_cmd) pid_file.parent.mkdir(parents=True, exist_ok=True) - pid_file.write_text(str(os.getpid())) # live target + import time as _time + pid_file.write_text(f"{os.getpid()} {int(_time.time())}") # live target, fresh with patch("mempalace.hooks_cli.subprocess.Popen") as mock_popen: _ingest_transcript(str(transcript)) @@ -1016,14 +1024,75 @@ def test_mine_already_running_dead_pid(tmp_path): def test_mine_already_running_live_pid(tmp_path): - """Returns True when the slot's recorded PID is alive.""" + """Returns True when the slot's recorded PID is alive (new {pid ts} format).""" + import time as _time + pid_dir = tmp_path / "mine_pids" cmd = ["mempalace", "mine", "/tmp/x", "--mode", "projects"] - _seed_slot(pid_dir, cmd, str(os.getpid())) # current process is alive + # Use a recent timestamp so the default 2 h timeout does not trigger. + _seed_slot(pid_dir, cmd, f"{os.getpid()} {int(_time.time())}") with patch("mempalace.hooks_cli._MINE_PID_DIR", pid_dir): assert _mine_already_running(cmd) is True +def test_mine_already_running_live_pid_bare_format(tmp_path): + """Old bare-PID format (no timestamp) is still recognized as alive.""" + pid_dir = tmp_path / "mine_pids" + cmd = ["mempalace", "mine", "/tmp/x", "--mode", "projects"] + _seed_slot(pid_dir, cmd, str(os.getpid())) # old format: bare PID + with patch("mempalace.hooks_cli._MINE_PID_DIR", pid_dir): + # No timestamp → treated as start_ts=0 (infinitely old) → stale. + # Verify that the default 2 h timeout fires and returns False. + assert _mine_already_running(cmd) is False + + +def test_mine_already_running_live_pid_exceeds_timeout(tmp_path): + """Returns False when PID is alive but has exceeded the configured timeout (#1552).""" + import time as _time + + pid_dir = tmp_path / "mine_pids" + cmd = ["mempalace", "mine", "/tmp/x", "--mode", "projects"] + # Timestamp far in the past so any positive timeout fires immediately. + old_ts = int(_time.time()) - 3601 # 1 second past 1-hour mark + _seed_slot(pid_dir, cmd, f"{os.getpid()} {old_ts}") + with ( + patch("mempalace.hooks_cli._MINE_PID_DIR", pid_dir), + patch.dict("os.environ", {"MEMPALACE_MINE_TIMEOUT_HOURS": "1"}), + ): + assert _mine_already_running(cmd) is False + + +def test_mine_already_running_live_pid_within_timeout(tmp_path): + """Returns True when PID is alive and has NOT exceeded the configured timeout.""" + import time as _time + + pid_dir = tmp_path / "mine_pids" + cmd = ["mempalace", "mine", "/tmp/x", "--mode", "projects"] + recent_ts = int(_time.time()) - 60 # only 1 minute old + _seed_slot(pid_dir, cmd, f"{os.getpid()} {recent_ts}") + with ( + patch("mempalace.hooks_cli._MINE_PID_DIR", pid_dir), + patch.dict("os.environ", {"MEMPALACE_MINE_TIMEOUT_HOURS": "2"}), + ): + assert _mine_already_running(cmd) is True + + +def test_mine_already_running_timeout_zero_disables_check(tmp_path): + """MEMPALACE_MINE_TIMEOUT_HOURS=0 disables the age-based stale check.""" + import time as _time + + pid_dir = tmp_path / "mine_pids" + cmd = ["mempalace", "mine", "/tmp/x", "--mode", "projects"] + old_ts = int(_time.time()) - 86400 # 24 hours ago — stale under any non-zero timeout + _seed_slot(pid_dir, cmd, f"{os.getpid()} {old_ts}") + with ( + patch("mempalace.hooks_cli._MINE_PID_DIR", pid_dir), + patch.dict("os.environ", {"MEMPALACE_MINE_TIMEOUT_HOURS": "0"}), + ): + # Timeout disabled — alive PID is always considered running. + assert _mine_already_running(cmd) is True + + def test_mine_already_running_corrupt_file(tmp_path): """Returns False when the slot contains non-integer content.""" pid_dir = tmp_path / "mine_pids" @@ -1035,10 +1104,13 @@ def test_mine_already_running_corrupt_file(tmp_path): def test_mine_already_running_distinct_cmds_independent(tmp_path): """Slots are keyed per cmd; an alive entry for cmd A doesn't shadow cmd B.""" + import time as _time + pid_dir = tmp_path / "mine_pids" cmd_a = ["mempalace", "mine", "/tmp/a", "--mode", "projects"] cmd_b = ["mempalace", "mine", "/tmp/b", "--mode", "projects"] - _seed_slot(pid_dir, cmd_a, str(os.getpid())) + recent_ts = int(_time.time()) + _seed_slot(pid_dir, cmd_a, f"{os.getpid()} {recent_ts}") with patch("mempalace.hooks_cli._MINE_PID_DIR", pid_dir): assert _mine_already_running(cmd_a) is True assert _mine_already_running(cmd_b) is False diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index ecd60afea4..e7ddbda0fe 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -2309,13 +2309,107 @@ def test_multi_tenant_env_switch(self, tmp_path, monkeypatch): ) assert add_result.get("success") is True, add_result - monkeypatch.setenv("MEMPALACE_PALACE_PATH", str(tmp_b)) - query_b = mcp_server.tool_kg_query(entity="alice_secret") - assert query_b.get("count", 0) == 0, f"tenant B leaked tenant A's fact: {query_b}" - monkeypatch.setenv("MEMPALACE_PALACE_PATH", str(tmp_a)) - query_a = mcp_server.tool_kg_query(entity="alice_secret") - assert query_a.get("count", 0) >= 1, f"tenant A lost its own fact: {query_a}" +# ── Structured error codes + MineAlreadyRunning (#1552) ───────────────── + + +class TestStructuredErrors: + """Verify that _internal_tool_error and MineAlreadyRunning return + machine-readable structured data (#1552).""" + + def test_internal_tool_error_without_exc_has_no_data_field(self): + """Backward-compat: callers that omit exc still get a valid error dict.""" + from mempalace.mcp_server import _internal_tool_error + + try: + raise ValueError("test error") + except ValueError: + resp = _internal_tool_error("req-1", "mempalace_search") + + assert resp["jsonrpc"] == "2.0" + assert resp["id"] == "req-1" + err = resp["error"] + assert err["code"] == -32000 + assert err["message"] == "Internal tool error" + assert "data" not in err + + def test_internal_tool_error_with_exc_includes_structured_data(self): + """When exc is supplied, the error body must include data.error_class + and data.message so callers can distinguish error types (#1552).""" + from mempalace.mcp_server import _internal_tool_error + + exc = RuntimeError("chromadb cold init wedge") + try: + raise exc + except RuntimeError: + resp = _internal_tool_error("req-2", "mempalace_add_drawer", exc) + + err = resp["error"] + assert err["code"] == -32000 + assert "data" in err + assert err["data"]["error_class"] == "RuntimeError" + assert "chromadb cold init wedge" in err["data"]["message"] + + def test_internal_tool_error_exception_dispatch_passes_exc(self, monkeypatch): + """handle_request's Exception branch must pass exc to _internal_tool_error.""" + from mempalace import mcp_server + + captured = {} + + def fake_handler(**kwargs): + raise OSError("fake disk error") + + fake_tool_entry = { + "handler": fake_handler, + "input_schema": {"type": "object", "properties": {}}, + } + monkeypatch.setattr( + mcp_server, + "TOOLS", + {"mempalace_fake": fake_tool_entry}, + ) + + original = mcp_server._internal_tool_error + + def spy_error(req_id, tool_name, exc=None): + captured["exc"] = exc + return original(req_id, tool_name, exc) + + monkeypatch.setattr(mcp_server, "_internal_tool_error", spy_error) + + req = { + "jsonrpc": "2.0", + "id": "r1", + "method": "tools/call", + "params": {"name": "mempalace_fake", "arguments": {}}, + } + resp = mcp_server.handle_request(req) + assert resp["error"]["code"] == -32000 + assert isinstance(captured.get("exc"), OSError) + assert "data" in resp["error"] + assert resp["error"]["data"]["error_class"] == "OSError" + + def test_tool_sync_mine_already_running_returns_error_class(self, monkeypatch, tmp_path): + """tool_sync MineAlreadyRunning path returns error_class: LockHeldByOtherProcess.""" + from mempalace import mcp_server + from mempalace.palace import MineAlreadyRunning + + cfg = MagicMock() + cfg.palace_path = str(tmp_path / "palace") + monkeypatch.setattr(mcp_server, "_config", cfg) + monkeypatch.setattr(mcp_server, "_get_kg", lambda *a, **kw: MagicMock()) + + def _raise_locked(*args, **kwargs): + raise MineAlreadyRunning("pid=12345") + + import mempalace.sync as sync_mod + + monkeypatch.setattr(sync_mod, "sync_palace", _raise_locked, raising=False) + + result = mcp_server.tool_sync() + assert result["success"] is False + assert "another mine is in progress" in result["error"] + assert result.get("error_class") == "LockHeldByOtherProcess" def test_cache_thread_safe(self, tmp_path, monkeypatch): """Concurrent _get_kg() for the same path yields one instance.""" From 1c818d5cdd95bc3275e0d9a21791d27a13ffa3e3 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Wed, 20 May 2026 15:59:19 -0300 Subject: [PATCH 32/39] fix: address stale hook review feedback - Disable mine and MCP idle timeouts when env values are invalid. - Use bare-PID slot file mtime as the compatibility timestamp instead of treating old slots as infinitely stale. - Treat malformed PID-slot timestamps as stale without crashing hook execution. - Use process-level idle watchdog termination so stale MCP servers actually release file handles. - Restore tenant-isolation assertions accidentally removed from the KG cache test. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mempalace/hooks_cli.py | 18 +++++++++++++----- mempalace/mcp_server.py | 6 +++--- tests/test_hooks_cli.py | 37 ++++++++++++++++++++++++++++++++++--- tests/test_mcp_server.py | 15 +++++++++++++++ 4 files changed, 65 insertions(+), 11 deletions(-) diff --git a/mempalace/hooks_cli.py b/mempalace/hooks_cli.py index d0c741b798..33c93f241f 100644 --- a/mempalace/hooks_cli.py +++ b/mempalace/hooks_cli.py @@ -304,7 +304,7 @@ def _mine_slot_timeout_secs() -> float: hours = float(raw) return max(0.0, hours) * 3600 except ValueError: - pass + return 0.0 return _MINE_TIMEOUT_HOURS_DEFAULT * 3600 @@ -363,9 +363,8 @@ def _mine_already_running(cmd: list[str]) -> bool: The PID file format is ``{pid} {unix_timestamp}`` (timestamp added in #1552 to detect wedged subprocesses). Old-format files (bare ``{pid}``) - are treated as having a start time of 0 — effectively "infinitely old" — - so they are immediately considered stale once the configured timeout - elapses. + use the PID file's mtime as the approximate start time so a still-running + pre-upgrade mine is not immediately misclassified as stale. A process is considered stale (and this function returns False) when: - the PID is dead, OR @@ -387,7 +386,16 @@ def _mine_already_running(cmd: list[str]) -> bool: return False timeout_secs = _mine_slot_timeout_secs() if timeout_secs > 0: - start_ts = float(parts[1]) if len(parts) > 1 and parts[1] else 0.0 + if len(parts) > 1 and parts[1]: + try: + start_ts = float(parts[1]) + except ValueError: + return False + else: + try: + start_ts = pid_file.stat().st_mtime + except OSError: + return True if time.time() - start_ts > timeout_secs: return False return True diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 90dfdda7fa..6e5a163ae2 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -194,7 +194,7 @@ def _mcp_idle_timeout_secs() -> float: hours = float(raw) return max(0.0, hours) * 3600 except ValueError: - pass + return 0.0 return _MCP_IDLE_HOURS_DEFAULT * 3600 @@ -2663,7 +2663,7 @@ def _start_idle_exit_watchdog() -> None: """Start a daemon thread that exits the process after an idle period. When no request has been handled for ``MEMPALACE_MCP_IDLE_HOURS`` - (default 8 h), the thread calls ``sys.exit(0)`` so that stale MCP + (default 8 h), the thread terminates the process so that stale MCP servers from ended Claude Code sessions do not accumulate ChromaDB / HNSW file handles on Windows (#1552). @@ -2684,7 +2684,7 @@ def _watchdog() -> None: idle / 3600, timeout / 3600, ) - sys.exit(0) + os._exit(0) t = threading.Thread(target=_watchdog, name="mcp-idle-watchdog", daemon=True) t.start() diff --git a/tests/test_hooks_cli.py b/tests/test_hooks_cli.py index f22ca68867..4270f1240f 100644 --- a/tests/test_hooks_cli.py +++ b/tests/test_hooks_cli.py @@ -1036,16 +1036,47 @@ def test_mine_already_running_live_pid(tmp_path): def test_mine_already_running_live_pid_bare_format(tmp_path): - """Old bare-PID format (no timestamp) is still recognized as alive.""" + """Old bare-PID format uses file mtime for the stale-by-age check.""" pid_dir = tmp_path / "mine_pids" cmd = ["mempalace", "mine", "/tmp/x", "--mode", "projects"] _seed_slot(pid_dir, cmd, str(os.getpid())) # old format: bare PID with patch("mempalace.hooks_cli._MINE_PID_DIR", pid_dir): - # No timestamp → treated as start_ts=0 (infinitely old) → stale. - # Verify that the default 2 h timeout fires and returns False. + assert _mine_already_running(cmd) is True + + +def test_mine_already_running_bare_pid_old_mtime_is_stale(tmp_path): + """Old bare-PID slots are reclaimed once their file mtime exceeds timeout.""" + import time as _time + + pid_dir = tmp_path / "mine_pids" + cmd = ["mempalace", "mine", "/tmp/x", "--mode", "projects"] + slot = _seed_slot(pid_dir, cmd, str(os.getpid())) + old_mtime = _time.time() - 3601 + os.utime(slot, (old_mtime, old_mtime)) + with ( + patch("mempalace.hooks_cli._MINE_PID_DIR", pid_dir), + patch.dict("os.environ", {"MEMPALACE_MINE_TIMEOUT_HOURS": "1"}), + ): + assert _mine_already_running(cmd) is False + + +def test_mine_already_running_malformed_timestamp_is_stale(tmp_path): + """Malformed timestamps fail soft instead of crashing hook execution.""" + pid_dir = tmp_path / "mine_pids" + cmd = ["mempalace", "mine", "/tmp/x", "--mode", "projects"] + _seed_slot(pid_dir, cmd, f"{os.getpid()} not-a-timestamp") + with patch("mempalace.hooks_cli._MINE_PID_DIR", pid_dir): assert _mine_already_running(cmd) is False +def test_mine_slot_timeout_invalid_env_disables_timeout(): + """Invalid MEMPALACE_MINE_TIMEOUT_HOURS disables stale-by-age checks.""" + from mempalace.hooks_cli import _mine_slot_timeout_secs + + with patch.dict("os.environ", {"MEMPALACE_MINE_TIMEOUT_HOURS": "nope"}): + assert _mine_slot_timeout_secs() == 0.0 + + def test_mine_already_running_live_pid_exceeds_timeout(tmp_path): """Returns False when PID is alive but has exceeded the configured timeout (#1552).""" import time as _time diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index e7ddbda0fe..d3ba4aaeb4 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -2309,6 +2309,14 @@ def test_multi_tenant_env_switch(self, tmp_path, monkeypatch): ) assert add_result.get("success") is True, add_result + monkeypatch.setenv("MEMPALACE_PALACE_PATH", str(tmp_b)) + query_b = mcp_server.tool_kg_query(entity="alice_secret") + assert query_b.get("count", 0) == 0, f"tenant B leaked tenant A's fact: {query_b}" + + monkeypatch.setenv("MEMPALACE_PALACE_PATH", str(tmp_a)) + query_a = mcp_server.tool_kg_query(entity="alice_secret") + assert query_a.get("count", 0) >= 1, f"tenant A lost its own fact: {query_a}" + # ── Structured error codes + MineAlreadyRunning (#1552) ───────────────── @@ -2411,6 +2419,13 @@ def _raise_locked(*args, **kwargs): assert "another mine is in progress" in result["error"] assert result.get("error_class") == "LockHeldByOtherProcess" + def test_mcp_idle_timeout_invalid_env_disables_watchdog(self, monkeypatch): + """Invalid MEMPALACE_MCP_IDLE_HOURS disables idle auto-exit.""" + from mempalace import mcp_server + + monkeypatch.setenv("MEMPALACE_MCP_IDLE_HOURS", "not-a-float") + assert mcp_server._mcp_idle_timeout_secs() == 0.0 + def test_cache_thread_safe(self, tmp_path, monkeypatch): """Concurrent _get_kg() for the same path yields one instance.""" import concurrent.futures From 5f0a2140160a0a83cf6aa32829ef2db1efdadf49 Mon Sep 17 00:00:00 2001 From: Igor Lins e Silva <4753812+igorls@users.noreply.github.com> Date: Wed, 20 May 2026 13:09:15 -0300 Subject: [PATCH 33/39] style: format hook tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/test_hooks_cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_hooks_cli.py b/tests/test_hooks_cli.py index 4270f1240f..44b70e4ee9 100644 --- a/tests/test_hooks_cli.py +++ b/tests/test_hooks_cli.py @@ -793,6 +793,7 @@ def test_maybe_auto_ingest_skips_when_mine_running(tmp_path): pid_file = _pid_file_for_cmd(cmd) pid_file.parent.mkdir(parents=True, exist_ok=True) import time as _time + pid_file.write_text(f"{os.getpid()} {int(_time.time())}") with patch("mempalace.hooks_cli._mempalace_python", return_value=sys.executable): with patch("mempalace.hooks_cli.subprocess.Popen") as mock_popen: @@ -986,6 +987,7 @@ def test_ingest_transcript_skips_when_target_running(tmp_path): pid_file = _pid_file_for_cmd(expected_cmd) pid_file.parent.mkdir(parents=True, exist_ok=True) import time as _time + pid_file.write_text(f"{os.getpid()} {int(_time.time())}") # live target, fresh with patch("mempalace.hooks_cli.subprocess.Popen") as mock_popen: From c18879b5bc5797bcfd3b11c8c5c0296d690927c9 Mon Sep 17 00:00:00 2001 From: Milla J <232237854+milla-jovovich@users.noreply.github.com> Date: Wed, 20 May 2026 13:55:52 -0700 Subject: [PATCH 34/39] feat(tunnels): cross-wing entity tunnels derived from hallways MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the architectural counterpart to ``compute_topic_tunnels`` that materializes cross-wing tunnels from the within-wing hallway records introduced in PR #1558. When an entity (person, project, concept, interest) has hallways in two wings, an entity tunnel bridges them — anchored on the entity. This completes the v4 sequence: Wing → Drawer-entities → Hallway → Tunnel. Topic tunnels are NOT replaced. Both systems coexist for one release cycle so existing palaces don't lose tunnels between mines. Deprecation of topic tunnels is a separate follow-up PR after entity tunnels prove out in real use. ## What this commit does 1. Adds ``entity_tunnels_for_wing(wing, hallways, label_prefix)`` to ``mempalace/palace_graph.py``. Pure function: groups hallway records by entity-and-wing, finds entities present in ``wing`` AND ≥1 other wing, and emits one ``create_tunnel`` call per (entity, other_wing) pair. Uses ``kind="entity"`` and synthetic endpoint room ``entity:`` so the new tunnels are distinguishable from explicit/topic tunnels at read time but interchangeable with them via the standard ``list_tunnels`` / ``follow_tunnels`` API. 2. Adds ``_compute_entity_tunnels_for_wing(wing)`` wrapper to ``mempalace/miner.py``. Loads hallway records via ``hallways.list_hallways()`` and calls the algorithm. Module-level so tests can patch it as ``mempalace.miner._compute_entity_tunnels_for_wing``. 3. Wires the wrapper into ``_mine_impl`` immediately after the existing hallway-compute block. Same try/except fault-tolerance pattern as the topic-tunnel and hallway blocks — entity-tunnel computation is a derived analytic and must never fail a mine. ## Tests (RED-first) Nine algorithm tests in ``tests/test_palace_graph_tunnels.py`` (new ``TestEntityTunnels`` class): test_entity_tunnels_creates_cross_wing_tunnel_for_shared_entity test_entity_tunnels_skips_entities_in_only_one_wing test_entity_tunnels_counts_entity_in_either_pair_position test_entity_tunnels_three_wings_pairwise_from_focus_wing test_entity_tunnels_idempotent_on_rerun test_entity_tunnels_retrievable_via_list_tunnels test_entity_tunnels_empty_hallways_is_noop test_entity_tunnels_unknown_wing_is_noop test_entity_tunnel_room_does_not_collide_with_literal_room Two integration tests in ``tests/test_miner.py``: test_mine_computes_entity_tunnels_for_wing_post_mine test_mine_entity_tunnel_failure_does_not_crash_mine All 11 RED before this commit (AttributeError on the missing names). All 11 GREEN after. ## Out of scope (deferred to follow-up PRs) - ``format_miner.py`` and ``convo_miner.py`` integration: separate PRs per the scope discipline used for #1560. - Deprecating ``_compute_topic_tunnels_for_wing``: separate PR after entity tunnels prove out in real use. - Surfacing ``kind="entity"`` in MCP / search-result UI: not yet required by any reader; behaviorally interchangeable with the other tunnel kinds today. ## Stacking This PR stacks on PR #1558 (which introduces the hallway primitive and its miner integration). Base branch is ``feat/hallways-within-wing-connectors``. When #1558 merges to develop, GitHub auto-updates this PR's base to ``develop`` and the diff reduces to just the entity-tunnel additions. ## Verification pytest tests/test_palace_graph_tunnels.py::TestEntityTunnels → 9 passed (RED before, GREEN after) pytest tests/test_miner.py::test_mine_computes_entity_tunnels_for_wing_post_mine tests/test_miner.py::test_mine_entity_tunnel_failure_does_not_crash_mine → 2 passed (RED before, GREEN after) pytest tests/test_palace_graph_tunnels.py → 39 passed (no regressions) pytest tests/test_miner.py → 49 passed (no regressions) pytest -q (full mempalace suite) → 1949 passed, 1 skipped, 0 regressions ruff check mempalace/palace_graph.py mempalace/miner.py tests/ → All checks passed! ruff format --check ... → 4 files already formatted (pinned 0.15.9) --- mempalace/miner.py | 41 ++++++++ mempalace/palace_graph.py | 75 ++++++++++++++ tests/test_miner.py | 94 +++++++++++++++++ tests/test_palace_graph_tunnels.py | 156 +++++++++++++++++++++++++++++ 4 files changed, 366 insertions(+) diff --git a/mempalace/miner.py b/mempalace/miner.py index bd53b7ed99..792fc84673 100644 --- a/mempalace/miner.py +++ b/mempalace/miner.py @@ -1269,6 +1269,22 @@ def _mine_impl( file=sys.stderr, ) + # Cross-wing entity tunnels: derived from the hallway records + # materialized just above. When an entity appears in hallways of + # this wing AND another wing, a tunnel bridges them. Runs in + # parallel with topic tunnels — both kinds coexist via + # ``kind="entity"`` / ``kind="topic"``. Same fault-tolerance + # pattern: never fail a mine over a derived analytic. + try: + entity_tunnels_added = _compute_entity_tunnels_for_wing(wing) + if entity_tunnels_added: + print(f"\n Entity tunnels: +{entity_tunnels_added} cross-wing entity link(s)") + except Exception as e: + print( + f"\n WARNING: entity tunnel computation skipped — {e}", + file=sys.stderr, + ) + print(f"\n{'=' * 55}") print(" Done.") print(f" Files processed: {len(files) - files_skipped}") @@ -1370,6 +1386,31 @@ def _compute_topic_tunnels_for_wing(wing: str) -> int: return len(created) +def _compute_entity_tunnels_for_wing(wing: str) -> int: + """Drop tunnels between ``wing`` and every other wing that shares an + entity via the within-wing hallway primitive. + + Reads hallway records (``mempalace.hallways.list_hallways``) and + materializes cross-wing tunnels for any entity that has hallways in + this wing AND at least one other wing. Tunnels use ``kind="entity"`` + and the synthetic endpoint room ``entity:`` so they're + distinguishable from explicit and topic tunnels at read time but + interchangeable with them via the standard ``list_tunnels`` / + ``follow_tunnels`` API. + + Returns the number of tunnels created or refreshed. Zero means no + eligible entity exists in this wing yet (or no hallway records do). + """ + from .hallways import list_hallways + from .palace_graph import entity_tunnels_for_wing + + hallways = list_hallways() + if not hallways: + return 0 + created = entity_tunnels_for_wing(wing, hallways) + return len(created) + + # ============================================================================= # STATUS # ============================================================================= diff --git a/mempalace/palace_graph.py b/mempalace/palace_graph.py index 7567bf90a9..d9b13451ba 100644 --- a/mempalace/palace_graph.py +++ b/mempalace/palace_graph.py @@ -779,3 +779,78 @@ def topic_tunnels_for_wing( ) ) return created + + +def entity_tunnels_for_wing( + wing: str, + hallways: list, + label_prefix: str = "shared entity", +) -> list: + """Compute entity tunnels involving a single wing. + + An entity tunnel bridges two wings when the same entity (person, + project, concept, interest) appears in within-wing hallways of both. + This is the architectural counterpart to ``topic_tunnels_for_wing`` — + same storage path (``create_tunnel`` → ``~/.mempalace/tunnels.json``), + same dedup, same listing API — but the substrate is hallway records + rather than raw topic words. See v4 architecture doc, Wing → + Drawer-entities → Hallway → Tunnel. + + Endpoints use the synthetic room id ``entity:`` (mirrors + ``topic:``) so they can't collide with literal folder-derived + rooms of the same name. Casing of the entity is preserved. + + Topic tunnels are NOT replaced — both systems coexist for one release + cycle while entity tunnels prove out. Deprecation is a separate PR. + """ + if not hallways or not isinstance(wing, str) or not wing.strip(): + return [] + + wing_norm = normalize_wing_name(wing.strip()) + + # Build: entity -> {normalized_wing -> original_wing_display_name} + # Both entity_a and entity_b positions count toward "this entity is + # in this wing"; the hallway primitive treats the pair as unordered. + entity_wings: dict = {} + for h in hallways: + if not isinstance(h, dict): + continue + h_wing = h.get("wing") + if not isinstance(h_wing, str) or not h_wing.strip(): + continue + h_wing_norm = normalize_wing_name(h_wing.strip()) + for ent_key in ("entity_a", "entity_b"): + ent = h.get(ent_key) + if not isinstance(ent, str) or not ent.strip(): + continue + # setdefault preserves the first-seen display form so the + # tunnel endpoint matches the wing name the caller used. + entity_wings.setdefault(ent, {}).setdefault(h_wing_norm, h_wing) + + if not entity_wings: + return [] + + created: list = [] + # Stable entity order so tunnels materialize deterministically across + # runs — matters for tests and for diff-able tunnels.json files. + for entity in sorted(entity_wings.keys()): + wings_for_entity = entity_wings[entity] + if wing_norm not in wings_for_entity: + continue + own_wing_display = wings_for_entity[wing_norm] + # Stable other-wing order; ``wing_norm`` itself is excluded so an + # entity that lives only in this wing produces zero tunnels. + other_wings_norm = sorted(w for w in wings_for_entity if w != wing_norm) + for other_norm in other_wings_norm: + other_display = wings_for_entity[other_norm] + room = f"entity:{entity}" + tunnel = create_tunnel( + source_wing=own_wing_display, + source_room=room, + target_wing=other_display, + target_room=room, + label=f"{label_prefix}: {entity}", + kind="entity", + ) + created.append(tunnel) + return created diff --git a/tests/test_miner.py b/tests/test_miner.py index 2a2b0e9227..38baf62f86 100644 --- a/tests/test_miner.py +++ b/tests/test_miner.py @@ -154,6 +154,100 @@ def angry_compute(wing, col=None, min_count=2): shutil.rmtree(tmpdir, ignore_errors=True) +def test_mine_computes_entity_tunnels_for_wing_post_mine(monkeypatch): + """After mine() completes (non-dry-run), ``_compute_entity_tunnels_for_wing`` + must be called once with the wing name from mempalace.yaml. + + Without this call the entity-tunnel feature is dead code — no miner + triggers it, no entity tunnels ever land in ~/.mempalace/tunnels.json. + Mirrors the existing hallway- and topic-tunnel integration tests in + this file. + """ + from mempalace import miner as miner_mod + + entity_tunnel_calls = [] + + def fake_compute(wing): + entity_tunnel_calls.append({"wing": wing}) + return 0 # no tunnels — that's not what we're testing here + + # Patch at the call site (mempalace.miner._compute_entity_tunnels_for_wing) + # so the integration in mine() routes through our stub. + monkeypatch.setattr(miner_mod, "_compute_entity_tunnels_for_wing", fake_compute) + + tmpdir = tempfile.mkdtemp() + try: + project_root = Path(tmpdir).resolve() + os.makedirs(project_root / "backend") + write_file( + project_root / "backend" / "app.py", + "def main():\n print('hello world')\n" * 20, + ) + with open(project_root / "mempalace.yaml", "w") as f: + yaml.dump( + { + "wing": "test_project", + "rooms": [{"name": "backend", "description": "Backend code"}], + }, + f, + ) + + palace_path = project_root / "palace" + mine(str(project_root), str(palace_path)) + + assert len(entity_tunnel_calls) == 1, ( + f"expected _compute_entity_tunnels_for_wing to be called once, " + f"got {len(entity_tunnel_calls)}" + ) + assert entity_tunnel_calls[0]["wing"] == "test_project" + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def test_mine_entity_tunnel_failure_does_not_crash_mine(monkeypatch): + """If ``_compute_entity_tunnels_for_wing`` raises, the mine must still + complete and the drawer write must remain committed. + + Mirrors the try/except wrap around the existing tunnel- and hallway- + computation blocks. Entity tunnel computation is a derived analytic, + not load-bearing for the drawer write itself. + """ + from mempalace import miner as miner_mod + + def angry_compute(wing): + raise RuntimeError("simulated entity-tunnel-compute explosion") + + monkeypatch.setattr(miner_mod, "_compute_entity_tunnels_for_wing", angry_compute) + + tmpdir = tempfile.mkdtemp() + try: + project_root = Path(tmpdir).resolve() + os.makedirs(project_root / "backend") + write_file( + project_root / "backend" / "app.py", + "def main():\n print('hello world')\n" * 20, + ) + with open(project_root / "mempalace.yaml", "w") as f: + yaml.dump( + { + "wing": "test_project", + "rooms": [{"name": "backend", "description": "Backend code"}], + }, + f, + ) + + palace_path = project_root / "palace" + # Must NOT raise — the failure has to be caught + logged but not propagated. + mine(str(project_root), str(palace_path)) + + # Drawer-write side of the mine still committed. + client = chromadb.PersistentClient(path=str(palace_path)) + col = client.get_collection("mempalace_drawers") + assert col.count() > 0 + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + def test_load_config_uses_defaults_when_yaml_missing(): tmpdir = tempfile.mkdtemp() try: diff --git a/tests/test_palace_graph_tunnels.py b/tests/test_palace_graph_tunnels.py index 8be8042423..7e279774ca 100644 --- a/tests/test_palace_graph_tunnels.py +++ b/tests/test_palace_graph_tunnels.py @@ -481,3 +481,159 @@ def test_pre_1504_underscore_tunnels_remain_findable(self, tmp_path, monkeypatch assert len(palace_graph.follow_tunnels("mempalace_public", "auth")) == 1 assert len(palace_graph.follow_tunnels("mempalace-public", "auth")) == 1 + + +class TestEntityTunnels: + """Cross-wing entity tunnels (the Wing → Drawer-entities → Hallway → + Tunnel sequence from the v4 architecture vision). + + When an entity has within-wing hallways in two or more wings, an entity + tunnel bridges those wings, anchored on the entity. This is the + architectural counterpart to ``compute_topic_tunnels`` — same storage, + same dedup, but the substrate is hallway records (entity-grounded) + rather than raw topic words. + + Endpoints use the synthetic room id ``entity:`` so they can't + collide with literal folder-derived rooms of the same name (mirrors + the ``topic:`` convention). + + Topic tunnels are NOT removed by this work — both systems run in + parallel for one release cycle so existing palaces don't lose + tunnels between mines. Deprecation is a separate follow-up PR. + """ + + def test_entity_tunnels_creates_cross_wing_tunnel_for_shared_entity( + self, tmp_path, monkeypatch + ): + """Ben appears in a hallway in wing_aya AND in wing_mempalace → + exactly one entity tunnel between those two wings, anchored on Ben.""" + _use_tmp_tunnel_file(monkeypatch, tmp_path) + hallways = [ + {"wing": "wing_aya", "entity_a": "Ben", "entity_b": "Aya"}, + {"wing": "wing_mempalace", "entity_a": "Ben", "entity_b": "Igor"}, + ] + created = palace_graph.entity_tunnels_for_wing("wing_aya", hallways) + assert len(created) == 1 + tunnel = created[0] + assert {tunnel["source"]["wing"], tunnel["target"]["wing"]} == { + "wing_aya", + "wing_mempalace", + } + # Endpoint is the synthetic entity room — same casing as the + # stored entity, prefixed with ``entity:`` to prevent collision + # with a literal folder room of the same name. + assert tunnel["source"]["room"] == "entity:Ben" + assert tunnel["target"]["room"] == "entity:Ben" + assert tunnel["kind"] == "entity" + # Label carries the entity name without the prefix. + assert "Ben" in tunnel["label"] + assert "entity:Ben" not in tunnel["label"] + + def test_entity_tunnels_skips_entities_in_only_one_wing(self, tmp_path, monkeypatch): + """Aya has hallways only in wing_aya → no tunnel for her.""" + _use_tmp_tunnel_file(monkeypatch, tmp_path) + hallways = [ + {"wing": "wing_aya", "entity_a": "Aya", "entity_b": "Lumi"}, + {"wing": "wing_aya", "entity_a": "Aya", "entity_b": "Ever"}, + ] + created = palace_graph.entity_tunnels_for_wing("wing_aya", hallways) + assert created == [] + + def test_entity_tunnels_counts_entity_in_either_pair_position(self, tmp_path, monkeypatch): + """Entity may appear as entity_a in one hallway and entity_b in + another. Both positions count toward 'this entity is in this wing'.""" + _use_tmp_tunnel_file(monkeypatch, tmp_path) + hallways = [ + # Ben is entity_b in wing_aya + {"wing": "wing_aya", "entity_a": "Aya", "entity_b": "Ben"}, + # Ben is entity_a in wing_mempalace + {"wing": "wing_mempalace", "entity_a": "Ben", "entity_b": "Igor"}, + ] + created = palace_graph.entity_tunnels_for_wing("wing_aya", hallways) + assert len(created) == 1 + assert "Ben" in created[0]["label"] + + def test_entity_tunnels_three_wings_pairwise_from_focus_wing(self, tmp_path, monkeypatch): + """When Ben has hallways in {wing_aya, wing_mp, wing_lt} and we + compute for wing_aya, we create wing_aya↔wing_mp AND wing_aya↔wing_lt + only — NOT wing_mp↔wing_lt. The cross-wing tunnel between the two + other wings is the job of whichever of those two mines next. + """ + _use_tmp_tunnel_file(monkeypatch, tmp_path) + hallways = [ + {"wing": "wing_aya", "entity_a": "Ben", "entity_b": "Aya"}, + {"wing": "wing_mempalace", "entity_a": "Ben", "entity_b": "Igor"}, + {"wing": "wing_lantern", "entity_a": "Ben", "entity_b": "Lumi"}, + ] + created = palace_graph.entity_tunnels_for_wing("wing_aya", hallways) + assert len(created) == 2 + target_wings = set() + for t in created: + other = ( + t["target"]["wing"] if t["source"]["wing"] == "wing_aya" else t["source"]["wing"] + ) + target_wings.add(other) + assert target_wings == {"wing_mempalace", "wing_lantern"} + + def test_entity_tunnels_idempotent_on_rerun(self, tmp_path, monkeypatch): + """Re-running on the same hallway data must not duplicate tunnels. + ``create_tunnel`` dedup-on-canonical-id is the guarantee — this test + pins the contract.""" + _use_tmp_tunnel_file(monkeypatch, tmp_path) + hallways = [ + {"wing": "wing_aya", "entity_a": "Ben", "entity_b": "Aya"}, + {"wing": "wing_mempalace", "entity_a": "Ben", "entity_b": "Igor"}, + ] + palace_graph.entity_tunnels_for_wing("wing_aya", hallways) + palace_graph.entity_tunnels_for_wing("wing_aya", hallways) + listed = palace_graph.list_tunnels() + # Only one stored tunnel even after two passes. + entity_tunnels = [t for t in listed if t.get("kind") == "entity"] + assert len(entity_tunnels) == 1 + + def test_entity_tunnels_retrievable_via_list_tunnels(self, tmp_path, monkeypatch): + """Once written, entity tunnels appear in the standard + ``list_tunnels`` query — readers don't need a separate API.""" + _use_tmp_tunnel_file(monkeypatch, tmp_path) + hallways = [ + {"wing": "wing_aya", "entity_a": "Ben", "entity_b": "Aya"}, + {"wing": "wing_mempalace", "entity_a": "Ben", "entity_b": "Igor"}, + ] + created = palace_graph.entity_tunnels_for_wing("wing_aya", hallways) + listed = palace_graph.list_tunnels() + assert {t["id"] for t in listed} == {t["id"] for t in created} + + def test_entity_tunnels_empty_hallways_is_noop(self, tmp_path, monkeypatch): + """No hallways → no tunnels. No crash.""" + _use_tmp_tunnel_file(monkeypatch, tmp_path) + assert palace_graph.entity_tunnels_for_wing("wing_aya", []) == [] + + def test_entity_tunnels_unknown_wing_is_noop(self, tmp_path, monkeypatch): + """Wing that doesn't appear in any hallway → no tunnels (the wing + has no entities to bridge from).""" + _use_tmp_tunnel_file(monkeypatch, tmp_path) + hallways = [ + {"wing": "wing_other", "entity_a": "Ben", "entity_b": "Aya"}, + ] + assert palace_graph.entity_tunnels_for_wing("wing_nonexistent", hallways) == [] + + def test_entity_tunnel_room_does_not_collide_with_literal_room(self, tmp_path, monkeypatch): + """A literal folder room ``Ben`` (created via explicit tunnel) and the + synthetic entity room ``entity:Ben`` must produce distinct tunnels — + not get deduped together by id collision.""" + _use_tmp_tunnel_file(monkeypatch, tmp_path) + # Pre-create an explicit tunnel with a literal "Ben" room. + palace_graph.create_tunnel( + "wing_aya", "Ben", "wing_mempalace", "Ben", label="literal", kind="explicit" + ) + hallways = [ + {"wing": "wing_aya", "entity_a": "Ben", "entity_b": "Aya"}, + {"wing": "wing_mempalace", "entity_a": "Ben", "entity_b": "Igor"}, + ] + palace_graph.entity_tunnels_for_wing("wing_aya", hallways) + listed = palace_graph.list_tunnels() + # Two distinct tunnels: one literal "Ben" → "Ben", one + # "entity:Ben" → "entity:Ben". Different canonical IDs. + assert len(listed) == 2 + kinds = {t.get("kind") for t in listed} + assert kinds == {"explicit", "entity"} From ffc2ae4ae15fc8c54e16acb258624c5fb4d6b806 Mon Sep 17 00:00:00 2001 From: Milla J <232237854+milla-jovovich@users.noreply.github.com> Date: Wed, 20 May 2026 15:26:41 -0700 Subject: [PATCH 35/39] fix(extract): surface MissingDependencyException + pull format sub-extras + skip symlink tests on Windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses PR #1555 review (Igor) — two bugs that block this PR from merging cleanly: 1. ``mempalace[extract]`` extras did not pull MarkItDown's per-format sub-dependencies. A real PDF after ``pip install mempalace[extract]`` would raise ``MissingDependencyException`` asking for ``markitdown[pdf]`` — the code then routed that exception through the generic ``except Exception`` and surfaced it as ``SKIP_EXTRACTION_ERROR``, stripping the actionable signal from the user. 2. ``test_fringe_broken_symlink`` and ``test_scan_formats_skips_symlinks`` called ``Path.symlink_to()`` unguarded, which raises ``OSError`` (``WinError 1314``) on Windows test environments without ``SeCreateSymbolicLinkPrivilege`` — surfacing as hard test failures before any product code ran. ## What this commit does 1. ``mempalace/format_miner.py`` — adds a new ``ExtractionStatus.SKIP_MISSING_FORMAT_DEPS`` enum member and a matching catch in ``extract_text`` that fires BEFORE the generic ``except Exception`` block. The catch matches by exception type name (``type(exc).__name__ == "MissingDependencyException"``) so the static import surface doesn't change — MarkItDown stays an optional dependency. 2. ``pyproject.toml`` — changes ``[extract]`` to include MarkItDown's per-format sub-extras: "markitdown[docx,pdf,pptx,xlsx]>=0.1.5; python_version >= '3.10'" These pull ``pdfminer-six``, ``pdfplumber``, ``mammoth``, ``lxml``, ``python-pptx``, ``openpyxl``, ``pandas`` — the deps each per-format converter actually needs at runtime. Verified against MarkItDown 0.1.5's PyPI metadata (Provides-Extra includes ``pdf``, ``docx``, ``pptx``, ``xlsx`` among others). ``.epub`` is handled by base markitdown (``EpubConverter`` uses ``beautifulsoup4``, a base requirement, so no ``[epub]`` extra is needed — and none exists in 0.1.5). ``.rtf`` is covered by the existing ``striprtf`` entry. ``markitdown[all]`` was NOT used because it pulls audio (``pydub``, ``speechrecognition``), YouTube, and Azure deps that mempalace does not claim support for and that would bloat the install for users. 3. ``tests/test_format_miner.py`` — adds a ``_make_symlink_or_skip`` helper near the top of the file that wraps ``Path.symlink_to()`` in try/except ``OSError`` and calls ``pytest.skip(...)`` on failure. Refactors both existing symlink tests (``test_fringe_broken_symlink``, ``test_scan_formats_skips_symlinks``) to use the helper. Behavior is unchanged on macOS/Linux; on Windows without symlink privileges the tests skip cleanly instead of spuriously failing. ## Tests (RED-first) Two new RED-first tests: test_extract_text_missing_format_dep_returns_distinct_status Patches ``_extract_via_markitdown`` to raise a fake exception with ``__name__ == "MissingDependencyException"``. Asserts the dispatcher returns ``SKIP_MISSING_FORMAT_DEPS``, not ``SKIP_EXTRACTION_ERROR``. test_pyproject_extract_extra_pulls_markitdown_format_subdeps Parses ``pyproject.toml`` (via ``tomllib`` 3.11+ or ``tomli`` on 3.9/3.10 — the latter is already a base dependency under the ``python_version < '3.11'`` marker). Asserts ``[extract]`` includes ``pdf``, ``docx``, ``pptx``, ``xlsx`` inside SOME ``markitdown[...]`` bracketed group. Both RED before this commit. Both GREEN after. Also updates ``test_extraction_status_enum_has_all_documented_codes`` to include the new ``SKIP_MISSING_FORMAT_DEPS`` code in the documented set, so the enum-completeness doc-test stays honest. ## Verification pytest tests/test_format_miner.py::test_extract_text_missing_format_dep_returns_distinct_status tests/test_format_miner.py::test_pyproject_extract_extra_pulls_markitdown_format_subdeps → 2 passed (RED before, GREEN after) pytest tests/test_format_miner.py::test_fringe_broken_symlink tests/test_format_miner.py::test_scan_formats_skips_symlinks → 2 passed on macOS (Windows behaviour validated by CI test-windows) pytest tests/test_format_miner.py → 64 passed, 2 skipped, 0 regressions pytest -q (full mempalace suite) → 2002 passed, 3 skipped, 0 regressions ruff check mempalace/format_miner.py tests/test_format_miner.py → All checks passed! ruff format --check ... → 2 files already formatted (pinned 0.15.9) ## Why no OrbStack run this time OrbStack runs Linux containers, which have unprivileged symlinks. It cannot reproduce the Windows ``WinError 1314`` failure mode this amendment fixes. The only authoritative gate for the symlink fix is GitHub CI's ``test-windows`` job. For the enum + extras changes, both are pure-Python / install-time concerns the CI test-linux matrix covers identically to OrbStack. The ``tomli`` fallback in the new pyproject test was verified against the base dependency declaration in ``pyproject.toml:32``. --- mempalace/format_miner.py | 17 +++++++ pyproject.toml | 9 +++- tests/test_format_miner.py | 99 +++++++++++++++++++++++++++++++++++++- 3 files changed, 122 insertions(+), 3 deletions(-) diff --git a/mempalace/format_miner.py b/mempalace/format_miner.py index 52dcbb910a..5824d6664e 100644 --- a/mempalace/format_miner.py +++ b/mempalace/format_miner.py @@ -163,6 +163,7 @@ class ExtractionStatus(enum.Enum): SKIP_BROKEN_SYMLINK = "skip:broken_symlink" SKIP_UNRECOGNIZED = "skip:unrecognized" SKIP_EXTRACTION_ERROR = "skip:extraction_error" + SKIP_MISSING_FORMAT_DEPS = "skip:missing_format_deps" SKIP_NETWORK_TIMEOUT = "skip:network_timeout" SKIP_UNREADABLE = "skip:unreadable" @@ -381,6 +382,22 @@ def extract_text( logger.info("skip:permission %s", p) return None, ExtractionStatus.SKIP_PERMISSION except Exception as exc: + # Fringe Case 14 (PR #1555 review, Igor): MarkItDown raises + # ``MissingDependencyException`` when a per-format sub-extra + # (e.g. ``markitdown[pdf]``) isn't installed. Surface this as a + # distinct status so users see the actionable signal instead of + # the generic SKIP_EXTRACTION_ERROR. Match by type name (not + # isinstance) so the catch fires without requiring markitdown to + # be import-resolvable at module load — keeps the static-import + # surface unchanged. + if type(exc).__name__ == "MissingDependencyException": + logger.warning( + "skip:missing_format_deps %s — %s: %s", + p, + type(exc).__name__, + str(exc)[:200], + ) + return None, ExtractionStatus.SKIP_MISSING_FORMAT_DEPS # Fringe Case 4 vs Case 10: encrypted vs generic crash, by message. msg = str(exc) if _ENCRYPTED_PATTERNS.search(msg): diff --git a/pyproject.toml b/pyproject.toml index 6cea05d492..3b3341d41c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,7 +67,14 @@ coreml = ["onnxruntime>=1.16"] # only installs where supported; Python 3.9 users still get RTF coverage). extract = [ "striprtf>=0.0.27", - "markitdown>=0.1.5; python_version >= '3.10'", + # Per PR #1555 review (Igor): bare ``markitdown`` is enough to import + # but not enough to convert — actual PDF/DOCX/PPTX/XLSX extraction + # needs MarkItDown's per-format sub-extras, which pull pdfminer-six, + # mammoth, python-pptx, openpyxl, etc. EPUB is handled by base + # markitdown (EpubConverter uses beautifulsoup4, a base requirement), + # so no [epub] sub-extra is needed (and none exists in 0.1.5). RTF is + # covered by striprtf above. + "markitdown[docx,pdf,pptx,xlsx]>=0.1.5; python_version >= '3.10'", ] [dependency-groups] diff --git a/tests/test_format_miner.py b/tests/test_format_miner.py index 1a79badf10..4c4e28c135 100644 --- a/tests/test_format_miner.py +++ b/tests/test_format_miner.py @@ -30,6 +30,23 @@ ) +def _make_symlink_or_skip(link: Path, target: Path) -> None: + """Create ``link`` pointing to ``target``, or ``pytest.skip`` when the + platform/process can't create symlinks. + + Windows without ``SeCreateSymbolicLinkPrivilege`` raises ``OSError`` + (``WinError 1314``) from ``Path.symlink_to()`` BEFORE any product code + runs, which surfaces as a hard test failure even though the failure + has nothing to do with the code under test. Per PR #1555 review (Igor): + symlink tests must skip cleanly in environments without privileges + rather than fail spuriously. + """ + try: + link.symlink_to(target) + except OSError as exc: + pytest.skip(f"symlink creation not permitted on this platform/user: {exc}") + + # ───────────────────────────────────────────────────────────────────────────── # Module surface # ───────────────────────────────────────────────────────────────────────────── @@ -63,6 +80,7 @@ def test_extraction_status_enum_has_all_documented_codes(): "SKIP_BROKEN_SYMLINK", "SKIP_UNRECOGNIZED", "SKIP_EXTRACTION_ERROR", + "SKIP_MISSING_FORMAT_DEPS", "SKIP_NETWORK_TIMEOUT", "SKIP_UNREADABLE", } @@ -190,7 +208,7 @@ def test_fringe_permission_denied(tmp_path: Path): def test_fringe_broken_symlink(tmp_path: Path): target = tmp_path / "does-not-exist.pdf" link = tmp_path / "broken-link.pdf" - link.symlink_to(target) + _make_symlink_or_skip(link, target) assert link.is_symlink() text, status = extract_text(link) assert text is None @@ -865,7 +883,7 @@ def test_scan_formats_skips_symlinks(tmp_path: Path): real = tmp_path / "real.pdf" real.write_bytes(b"%PDF-1.4 stub") link = tmp_path / "alias.pdf" - link.symlink_to(real) + _make_symlink_or_skip(link, real) found = {f.name for f in scan_formats(tmp_path)} assert "real.pdf" in found assert "alias.pdf" not in found @@ -1137,3 +1155,80 @@ def test_mine_formats_tunnel_failure_does_not_crash_mine(_mine_formats_mocks): ): # Must NOT raise mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"), wing="wing_aya") + + +# ───────────────────────────────────────────────────────────────────────────── +# PR #1555 review (Igor) — missing-format-deps surfacing + pyproject extras +# ───────────────────────────────────────────────────────────────────────────── + + +def test_extract_text_missing_format_dep_returns_distinct_status(tmp_path: Path, monkeypatch): + """A MarkItDown ``MissingDependencyException`` (raised when a per-format + sub-extra like ``markitdown[pdf]`` isn't installed) must surface as + ``SKIP_MISSING_FORMAT_DEPS`` — NOT the generic ``SKIP_EXTRACTION_ERROR``. + + Per PR #1555 review (Igor): real PDFs hit this in production today + and users see "extraction error" with no signal that the fix is + ``pip install markitdown[pdf]``. The dispatcher must catch by type + name so the real markitdown package doesn't have to be import- + resolvable for the catch to fire. + """ + from mempalace import format_miner + + # Build a fake exception class with the right __name__ so the + # type-name catch in extract_text recognises it without requiring + # the real markitdown to be installed in the test environment. + class MissingDependencyException(Exception): + pass + + def fake_extract(p): + raise MissingDependencyException( + "MarkItDown failed: install with `pip install markitdown[pdf]`" + ) + + monkeypatch.setattr(format_miner, "_extract_via_markitdown", fake_extract) + + pdf = tmp_path / "fake.pdf" + pdf.write_bytes(b"%PDF-1.4 stub") + + text, status = extract_text(pdf) + assert text is None + assert status == ExtractionStatus.SKIP_MISSING_FORMAT_DEPS, ( + f"expected SKIP_MISSING_FORMAT_DEPS, got {status}" + ) + + +def test_pyproject_extract_extra_pulls_markitdown_format_subdeps(): + """The ``mempalace[extract]`` extra must include MarkItDown's per-format + sub-extras (``pdf``, ``docx``, ``pptx``, ``xlsx``) — without them, real + PDF/DOCX/etc files hit ``MissingDependencyException`` at runtime even + after ``pip install mempalace[extract]``. Per PR #1555 review (Igor). + + ``.rtf`` is covered by the separate ``striprtf`` dependency and ``.epub`` + ships in base MarkItDown (``EpubConverter`` uses ``beautifulsoup4`` + which is a base requirement, not extra-gated), so neither needs a + sub-extra here. + """ + try: + import tomllib # 3.11+ + except ImportError: # pragma: no cover — only hit on 3.9/3.10 + import tomli as tomllib # type: ignore[import-not-found, no-redef] + + pyproject_path = Path(__file__).resolve().parent.parent / "pyproject.toml" + with open(pyproject_path, "rb") as f: + data = tomllib.load(f) + + extract = data["project"]["optional-dependencies"]["extract"] + extract_str = " ".join(extract).lower() + + # Accept any layout — comma-separated ``markitdown[docx,pdf,pptx,xlsx]`` + # or separate ``markitdown[pdf]`` entries — as long as each format + # appears inside SOME ``markitdown[...]`` bracketed group. + import re + + bracketed = "".join(re.findall(r"markitdown\[([^\]]+)\]", extract_str)) + for sub in ("pdf", "docx", "pptx", "xlsx"): + assert sub in bracketed, ( + f"[extract] must include markitdown[{sub}]; " + f"got bracketed extras: {bracketed!r}; full extract: {extract}" + ) From b6dc122acb3ea653cd783775bea01de4eed02535 Mon Sep 17 00:00:00 2001 From: Milla J <232237854+milla-jovovich@users.noreply.github.com> Date: Thu, 21 May 2026 08:41:44 -0700 Subject: [PATCH 36/39] =?UTF-8?q?fix(extract):=20polish=20PR=20=E2=80=94?= =?UTF-8?q?=20address=20bot=20review=20feedback=20on=20PR=20#1555?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #1555 (format coverage + virtual line numbering) merged with twelve inline polish comments from Copilot + gemini-code-assist that weren't load-bearing enough to block the original ship but are real cleanups. This PR addresses them. Twelve items in scope; one item (drawer ID delimiter — Copilot #13) is deferred to its own dedicated PR because it's a breaking schema change that requires migration design beyond the scope of a polish PR. ## Behavioral fixes (5 items, RED-tested first) 1. **FileNotFoundError vs broken symlink (Copilot #8).** ``extract_text`` previously mapped every ``FileNotFoundError`` from ``stat()`` to ``SKIP_BROKEN_SYMLINK``. That's misleading for the common case of a regular file deleted between scan and extract. Now distinguishes: ``SKIP_BROKEN_SYMLINK`` only when ``p.is_symlink()`` is true; ``SKIP_UNREADABLE`` otherwise. 2. **``file_already_mined`` extract_mode scoping (Copilot #11, #12).** Both call sites in ``mine_formats`` and ``_file_chunks_locked`` now pass ``extract_mode="format"``. Previously the format miner could falsely treat drawers from project / convo miner on the same source file as "already mined" (and vice versa). Scopes idempotency to the correct drawer subset. 3. **Sentinel skip for transient missing-dep statuses (Copilot #14).** New ``_TRANSIENT_MISSING_DEP_STATUSES`` set + ``_register_skip_sentinel_if_appropriate`` helper. Skip variants like ``SKIP_NO_MARKITDOWN`` / ``SKIP_NO_STRIPRTF`` / ``SKIP_MISSING_FORMAT_DEPS`` / ``SKIP_NETWORK_TIMEOUT`` no longer write the "already-mined" sentinel. Otherwise installing the missing extra later wouldn't trigger a re-mine. 4. **Outer ``except Exception`` in ``mine_formats`` (Gemini #5).** The outer try around the loop previously caught only ``KeyboardInterrupt``, leaving any setup-time error (e.g., ``scan_formats`` raising) to propagate as a bare traceback. Now catches ``Exception`` defensively, logs it, prints a partial-progress summary, and lets the ``finally`` PID-cleanup run. Mirrors miner.py's belt-and-suspenders pattern. 5. **Thread user's ``chunk_size`` / ``chunk_overlap`` / ``min_chunk_size`` through to ``chunk_text`` (Gemini #3).** ``MempalaceConfig`` was loaded only to validate readability; users who tuned their config saw no effect in format-mode mining. Now properly threaded. ## Trivial cleanups (5 items) 6. **Path expanduser in ``extract_text`` (Copilot #7).** ``Path(path)`` → ``Path(path).expanduser()`` so CLI inputs like ``~/docs/file.pdf`` resolve correctly. 7. **Path expanduser+resolve in ``scan_formats`` (Copilot #9).** Same fix; ``~/docs`` and relative paths now work consistently. 8. **Use resolved ``format_path`` in ``mine_formats`` (Copilot #10).** ``scan_formats(format_dir)`` → ``scan_formats(format_path)`` so the already-resolved path is used. 9. **``render_with_line_numbers`` type annotation (Copilot #15).** ``text: "str | None"`` reflects the documented + tested ``None`` handling. 10. **Test + docs claims (Copilot #16, #17, #18).** Stale framings removed: - ``docs/format-coverage.md`` — 14 fringe cases + "see the file for the current test inventory" (no more frozen test count). - ``tests/test_line_numbers.py`` — drops "proposed for mempalace 3.3.6" + "run from the proposal directory" references. - ``tests/test_format_miner.py`` — drops "MarkItDown is mocked throughout" (live integration tests exist) + proposal-directory framing. ## Module-level hoists (enables clean test patching) - ``MempalaceConfig`` (from ``.config``) hoisted from lazy local import to module-level so tests can patch ``mempalace.format_miner.MempalaceConfig``. - ``chunk_text`` (from ``.miner``) hoisted similarly. Both follow the pattern PR #1565 used for ``compute_hallways_for_wing``. ## Complexity refactor Extracted ``_print_mine_summary`` from ``mine_formats`` so the orchestrator stays under the project's ``max-complexity = 25`` ceiling (per ``pyproject.toml [tool.ruff.lint.mccabe]``). Behavior unchanged; pure extraction. ## Out of scope (intentionally deferred) - **Drawer ID delimiter collision (Copilot #13)** — ``f"{source_file}{chunk_index}"`` can theoretically collide (``"/path/a1" + "23"`` == ``"/path/a" + "123"``). Fixing this is a breaking schema change to drawer IDs and requires a migration plan; will land as its own PR after design. - The four bot comments that were ALREADY addressed by amendment #3 before the PR #1555 merge (``_SKIP_DIRS`` dedup, ``scan_formats`` symlink skip, ``source_mtime`` tracking, hall+entities metadata) — no action needed; verified during audit. ## Tests (RED-first) Six new RED-first tests in ``tests/test_format_miner.py``: test_extract_text_nonexistent_regular_file_returns_unreadable_not_broken_symlink test_mine_formats_passes_extract_mode_format_to_file_already_mined test_mine_formats_does_not_write_sentinel_for_skip_no_markitdown test_mine_formats_does_not_write_sentinel_for_skip_missing_format_deps test_mine_formats_catches_unexpected_exception_and_prints_summary test_mine_formats_threads_chunk_size_from_user_config All six RED before this commit (failures correctly identified the bugs they're targeting), all six GREEN after. One existing test (``test_mine_formats_continues_after_per_file_error``) updated to patch the new module-level binding ``mempalace.format_miner.chunk_text`` instead of the old ``mempalace.miner.chunk_text`` source location, and to accept the ``**kwargs`` the call now passes through. Behavior unchanged. ## Verification pytest -q (full mempalace suite) → 2065 passed, 3 skipped, 0 regressions ruff check mempalace/format_miner.py mempalace/searcher.py tests/ → All checks passed! ruff format --check ... → 4 files already formatted (pinned 0.15.9) mine_formats complexity → ≤ 25 (under the project ceiling) --- docs/format-coverage.md | 8 +- mempalace/format_miner.py | 215 ++++++++++++++++++++------ mempalace/searcher.py | 2 +- tests/test_format_miner.py | 309 +++++++++++++++++++++++++++++++++++-- tests/test_line_numbers.py | 7 +- 5 files changed, 473 insertions(+), 68 deletions(-) diff --git a/docs/format-coverage.md b/docs/format-coverage.md index 178658e09e..8b166a1f1b 100644 --- a/docs/format-coverage.md +++ b/docs/format-coverage.md @@ -1,9 +1,9 @@ # Format coverage — `mempalace mine --mode extract` -**Proposed for mempalace 3.3.6.** -**Target file (new):** `mempalace/format_miner.py` -**Target file (edited, one-line addition):** `mempalace/cli.py` (dispatcher in `cmd_mine`) -**New tests:** `tests/test_format_miner.py` (32 cases, all green; see `PROOF.md`). +**Shipped in mempalace 3.3.6 (PR #1555).** +**Module:** `mempalace/format_miner.py` +**CLI entry:** `mempalace/cli.py` — `cmd_mine` dispatcher routes `--mode extract` here. +**Tests:** `tests/test_format_miner.py` — covers all 14 fringe cases plus orchestrator behavior. See the file for the current test inventory (the count drifts as polish PRs land). --- diff --git a/mempalace/format_miner.py b/mempalace/format_miner.py index 5824d6664e..3e879ba63f 100644 --- a/mempalace/format_miner.py +++ b/mempalace/format_miner.py @@ -80,8 +80,10 @@ # Module-level imports from .miner so tests can patch them via # mempalace.format_miner.. Lazy imports inside functions would not # expose these as attributes of this module, breaking the test seams. +from .config import MempalaceConfig, normalize_wing_name from .miner import ( _compute_topic_tunnels_for_wing, + chunk_text, detect_room, load_config, ) @@ -168,6 +170,22 @@ class ExtractionStatus(enum.Enum): SKIP_UNREADABLE = "skip:unreadable" +# Skip variants that represent TRANSIENT environmental issues — the optional +# transformer dep isn't installed yet, or the network blipped. Sentinel +# writes for these would permanently mark the file as "already mined" and +# defeat re-mining after the missing piece is installed / the network is +# back. The orchestrator (``mine_formats``) checks this set before calling +# ``_register_file``. Per PR #1555 review (Copilot #14). +_TRANSIENT_MISSING_DEP_STATUSES = frozenset( + { + ExtractionStatus.SKIP_NO_MARKITDOWN, + ExtractionStatus.SKIP_NO_STRIPRTF, + ExtractionStatus.SKIP_MISSING_FORMAT_DEPS, + ExtractionStatus.SKIP_NETWORK_TIMEOUT, + } +) + + # ───────────────────────────────────────────────────────────────────────────── # Encoding fallback — same pattern as the terminal-session normalizer # ───────────────────────────────────────────────────────────────────────────── @@ -304,7 +322,9 @@ def extract_text( never modified. """ # Accept str or Path; pathlib handles Windows separators correctly. - p = Path(path) + # ``expanduser()`` resolves ``~/foo.pdf`` style paths so CLI inputs work + # without forcing the caller to pre-resolve. Per PR #1555 review (Copilot). + p = Path(path).expanduser() # Fringe Case 7 — broken symlink. # ``is_symlink()`` true + ``exists()`` false ⇒ link target is missing. @@ -327,8 +347,17 @@ def extract_text( logger.info("skip:permission (stat) %s", p) return None, ExtractionStatus.SKIP_PERMISSION except FileNotFoundError: - logger.info("skip:broken_symlink (stat) %s", p) - return None, ExtractionStatus.SKIP_BROKEN_SYMLINK + # Distinguish "broken symlink" (target gone) from "file disappeared + # between scan and extract" (regular file, no longer exists). The + # is_symlink() check at the top of this function catches the symlink + # case BEFORE stat(), so by the time we land here the symlink path + # is rare — but a race between scan_formats and extract_text on a + # plain file is the common cause. Per PR #1555 review (Copilot #8). + if p.is_symlink(): + logger.info("skip:broken_symlink (stat) %s", p) + return None, ExtractionStatus.SKIP_BROKEN_SYMLINK + logger.info("skip:unreadable (file gone after scan) %s", p) + return None, ExtractionStatus.SKIP_UNREADABLE except OSError as exc: logger.info("skip:unreadable %s — %s", p, exc) return None, ExtractionStatus.SKIP_UNREADABLE @@ -436,7 +465,10 @@ def scan_formats(directory: Union[Path, str]) -> list[Path]: (sorted by path) so a re-mine processes files in the same order each time — useful for reproducing bug reports. """ - root = Path(directory) + # ``expanduser().resolve()`` normalizes ``~/docs`` and relative paths so + # CLI inputs like ``mempalace mine --mode extract ~/docs`` work without + # the caller pre-resolving. Per PR #1555 review (Copilot #9). + root = Path(directory).expanduser().resolve() if not root.exists(): return [] @@ -471,6 +503,51 @@ def scan_formats(directory: Union[Path, str]) -> list[Path]: # ───────────────────────────────────────────────────────────────────────────── +def _print_mine_summary( + files: list, + files_with_text: int, + files_skipped: int, + files_errored: int, + total_drawers: int, + status_counts: dict, +) -> None: + """Print the post-mine summary block. + + Factored out of ``mine_formats`` to keep the orchestrator under the + project's mccabe complexity ceiling (max-complexity=25 in pyproject.toml). + No behavior change. + """ + print(f"\n{'=' * 55}") + print(" Summary") + print(f"{'-' * 55}") + print(f" Files seen: {len(files)}") + print(f" Files extracted: {files_with_text}") + print(f" Files skipped: {files_skipped}") + print(f" Files errored: {files_errored}") + print(f" Total drawers: {total_drawers}") + if status_counts: + print(" Extraction status:") + for name, count in sorted(status_counts.items(), key=lambda kv: -kv[1]): + print(f" {name:30} {count}") + print(f"{'=' * 55}\n") + + +def _register_skip_sentinel_if_appropriate( + collection, source_file: str, wing: str, agent: str, status: ExtractionStatus +) -> None: + """Write the ``file_already_mined`` sentinel ONLY when the skip is durable. + + Transient missing-dep statuses (``SKIP_NO_MARKITDOWN``, ``SKIP_NO_STRIPRTF``, + ``SKIP_MISSING_FORMAT_DEPS``, ``SKIP_NETWORK_TIMEOUT``) intentionally do + NOT mark the file as mined — installing the missing extra later (or + reconnecting the network) should let the next mine pass pick it up. + Per PR #1555 review (Copilot #14). + """ + if status in _TRANSIENT_MISSING_DEP_STATUSES: + return + _register_file(collection, source_file, wing, agent) + + def _register_file(collection, source_file: str, wing: str, agent: str) -> None: """Write a sentinel so file_already_mined() returns True for 0-chunk files. @@ -537,7 +614,11 @@ def _file_chunks_locked( # palace.file_already_mined reads current mtime from disk and compares # against the stored source_mtime metadata, so the caller just toggles # the flag; no need to pass mtime explicitly. - if file_already_mined(collection, source_file, check_mtime=True): + # ``extract_mode="format"`` scopes the idempotency check to format-mode + # drawers only — drawers from convo_miner / project miner on the same + # source_file don't falsely indicate "already mined" to the format + # miner. Per PR #1555 review (Copilot #12). + if file_already_mined(collection, source_file, check_mtime=True, extract_mode="format"): return 0, True try: @@ -638,17 +719,16 @@ def mine_formats( in place because drawer IDs are deterministic (re-mining upserts to the same rows). """ - # Local imports — chunk_text is the same chunker miner.py uses; the - # config object provides chunk_size / overlap / min when needed. - from .config import MempalaceConfig, normalize_wing_name - from .miner import chunk_text - - # Load palace-wide config. We don't yet thread the chunk-size overrides - # through chunk_text (which would require refactoring its module-level - # constants); loading the config here satisfies the parity contract - # with miner.py and gives a single place to wire the threading later. + # MempalaceConfig + chunk_text are imported at module level (above) so + # tests can patch ``mempalace.format_miner.MempalaceConfig`` and + # ``mempalace.format_miner.chunk_text`` directly. Hoisted as part of the + # PR #1555 polish PR (Gemini #3). + + # Load palace-wide config. Chunk parameters (chunk_size, chunk_overlap, + # min_chunk_size) are now threaded through chunk_text below, so users + # who customized their config see the effect in format-mode mining. + # Per PR #1555 review (Gemini #3). palace_config = MempalaceConfig() - _ = palace_config.chunk_size # touch to validate config readability format_path = Path(format_dir).expanduser().resolve() if not wing: @@ -680,23 +760,12 @@ def mine_formats( } ] - files = scan_formats(format_dir) - if limit > 0: - files = files[:limit] - - print(f"\n{'=' * 55}") - print(" MemPalace Mine — Format extraction") - print(f"{'=' * 55}") - print(f" Wing: {wing}") - print(f" Source: {format_path}") - print(f" Files: {len(files)}") - print(f" Palace: {palace_path}") - if dry_run: - print(" DRY RUN — nothing will be filed") - print(f"{'-' * 55}\n") - - collection = get_collection(palace_path) if not dry_run else None - + # Initialize loop-state up-front so the outer try/except handlers and the + # post-try summary print can run safely even if scan_formats or + # get_collection raises before the for loop starts. Per PR #1555 review + # (Gemini #5). + files: list = [] + collection = None total_drawers = 0 files_skipped = 0 files_with_text = 0 @@ -704,6 +773,26 @@ def mine_formats( status_counts: dict = defaultdict(int) try: + # Use the resolved ``format_path``, not the raw ``format_dir``, so that + # ``~/docs`` and relative inputs work consistently. Per PR #1555 review + # (Copilot #10). + files = scan_formats(format_path) + if limit > 0: + files = files[:limit] + + print(f"\n{'=' * 55}") + print(" MemPalace Mine — Format extraction") + print(f"{'=' * 55}") + print(f" Wing: {wing}") + print(f" Source: {format_path}") + print(f" Files: {len(files)}") + print(f" Palace: {palace_path}") + if dry_run: + print(" DRY RUN — nothing will be filed") + print(f"{'-' * 55}\n") + + collection = get_collection(palace_path) if not dry_run else None + for i, filepath in enumerate(files, 1): source_file = str(filepath) @@ -718,7 +807,14 @@ def mine_formats( except OSError: source_mtime = None - if not dry_run and file_already_mined(collection, source_file, check_mtime=True): + # Pass extract_mode="format" so format-mode idempotency is + # scoped to its own drawer set — drawers from convo_miner / + # project miner on the same source_file don't falsely + # indicate "already mined" to the format miner. Per PR #1555 + # review (Copilot #11). + if not dry_run and file_already_mined( + collection, source_file, check_mtime=True, extract_mode="format" + ): files_skipped += 1 continue @@ -727,11 +823,22 @@ def mine_formats( if status != ExtractionStatus.OK or not text: if not dry_run: - _register_file(collection, source_file, wing, agent) + _register_skip_sentinel_if_appropriate( + collection, source_file, wing, agent, status + ) print(f" - [{i:4}/{len(files)}] {filepath.name[:50]:50} {status.name}") continue - chunks = chunk_text(text, source_file) + # Thread the user's MempalaceConfig chunk parameters through + # so format-mode mining honors their tuning. Per PR #1555 + # review (Gemini #3). + chunks = chunk_text( + text, + source_file, + chunk_size=palace_config.chunk_size, + chunk_overlap=palace_config.chunk_overlap, + min_chunk_size=palace_config.min_chunk_size, + ) if not chunks: if not dry_run: _register_file(collection, source_file, wing, agent) @@ -782,6 +889,23 @@ def mine_formats( # Partial progress is safe — deterministic drawer IDs mean a re-mine # upserts to the same rows. Print a clean summary and exit. print("\n Mine interrupted by user (Ctrl-C).") + except Exception as exc: + # Defense in depth — the per-file try/except above catches most + # realistic crashes, but a programming error in the outer-loop + # plumbing (file enumeration, scan_formats itself, etc.) would + # otherwise propagate as a bare traceback. Catch it here so the + # summary still prints and the PID-file finally-block still runs. + # Per PR #1555 review (Gemini #5). Mirrors miner.py's belt-and- + # suspenders pattern. + logger.warning( + "mine_formats: unexpected outer-loop exception — %s: %s", + type(exc).__name__, + str(exc)[:200], + ) + print( + f"\n Mine aborted by exception ({type(exc).__name__}: {str(exc)[:120]})", + file=sys.stderr, + ) else: # All files processed without interruption — compute cross-wing topic # tunnels linking this wing to others that share confirmed topics. @@ -816,16 +940,11 @@ def mine_formats( except Exception: logger.debug("mine_formats: _cleanup_mine_pid_file failed", exc_info=True) - print(f"\n{'=' * 55}") - print(" Summary") - print(f"{'-' * 55}") - print(f" Files seen: {len(files)}") - print(f" Files extracted: {files_with_text}") - print(f" Files skipped: {files_skipped}") - print(f" Files errored: {files_errored}") - print(f" Total drawers: {total_drawers}") - if status_counts: - print(" Extraction status:") - for name, count in sorted(status_counts.items(), key=lambda kv: -kv[1]): - print(f" {name:30} {count}") - print(f"{'=' * 55}\n") + _print_mine_summary( + files=files, + files_with_text=files_with_text, + files_skipped=files_skipped, + files_errored=files_errored, + total_drawers=total_drawers, + status_counts=status_counts, + ) diff --git a/mempalace/searcher.py b/mempalace/searcher.py index df9a9e2b55..db14c19ff9 100644 --- a/mempalace/searcher.py +++ b/mempalace/searcher.py @@ -1034,7 +1034,7 @@ def search_memories( _ALREADY_NUMBERED_RE = re.compile(r"^\[\d+\]") -def render_with_line_numbers(text, start_line: int = 1) -> str: +def render_with_line_numbers(text: "str | None", start_line: int = 1) -> str: """Prefix each line of ``text`` with ``[N] `` for read-time grid display. Lines that already begin with ``[]`` pass through unchanged, diff --git a/tests/test_format_miner.py b/tests/test_format_miner.py index 4c4e28c135..a9ed816828 100644 --- a/tests/test_format_miner.py +++ b/tests/test_format_miner.py @@ -1,14 +1,12 @@ -"""Tests for format_miner (proposed for mempalace 3.3.6). +"""Tests for format_miner (mempalace 3.3.6, integrated in PR #1555). -Covers all 13 fringe cases in the 3.3.6 format-coverage spec, plus the -happy paths. MarkItDown is mocked throughout so tests run without the -library installed — same discipline as the existing test_line_numbers.py. +Covers all 14 fringe cases in the format-coverage spec plus orchestrator +behavior. MarkItDown is mocked at the seam for most tests; a few guarded +integration tests exercise live transformers when the optional extras are +installed. -Run from the proposal directory: +Run with: pytest tests/test_format_miner.py -v - -After integration into the mempalace package, change the import below to: - from mempalace.format_miner import ... """ from __future__ import annotations @@ -980,7 +978,7 @@ def test_mine_formats_continues_after_per_file_error(_mine_formats_mocks): # Make chunk_text crash on the first file but succeed on the second. call_count = {"n": 0} - def chunk_text_first_fails(content, source_file): + def chunk_text_first_fails(content, source_file, **kwargs): call_count["n"] += 1 if call_count["n"] == 1: raise RuntimeError("simulated chunker explosion") @@ -989,7 +987,11 @@ def chunk_text_first_fails(content, source_file): with ( patch("mempalace.format_miner.scan_formats", return_value=[bad, good]), patch("mempalace.format_miner._extract_via_markitdown", return_value="text"), - patch("mempalace.miner.chunk_text", side_effect=chunk_text_first_fails), + # Patch the module-level binding in format_miner (hoisted in the + # PR #1555 polish work — Gemini #3). The old patch target + # ``mempalace.miner.chunk_text`` no longer works because + # format_miner now binds chunk_text at its own module scope. + patch("mempalace.format_miner.chunk_text", side_effect=chunk_text_first_fails), ): # Must not raise mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace")) @@ -1232,3 +1234,290 @@ def test_pyproject_extract_extra_pulls_markitdown_format_subdeps(): f"[extract] must include markitdown[{sub}]; " f"got bracketed extras: {bracketed!r}; full extract: {extract}" ) + + +# ───────────────────────────────────────────────────────────────────────────── +# PR #1555 review polish — bot feedback items addressed post-merge. +# Five behavioral changes; the trivial items (path expanduser, signature +# annotations, docstring/doc updates) don't need their own RED tests. +# ───────────────────────────────────────────────────────────────────────────── + + +def test_extract_text_nonexistent_regular_file_returns_unreadable_not_broken_symlink( + tmp_path: Path, +): + """A non-existent REGULAR file (not a symlink) must return + ``SKIP_UNREADABLE``, not ``SKIP_BROKEN_SYMLINK``. + + Per PR #1555 review (Copilot #8): the ``FileNotFoundError`` arm in + ``extract_text`` was unconditionally mapping to ``SKIP_BROKEN_SYMLINK``, + which is misleading when the path was a regular file that got deleted + between scan and extract. ``SKIP_BROKEN_SYMLINK`` should only fire + when ``p.is_symlink()`` is true. + """ + p = tmp_path / "deleted.pdf" + assert not p.exists() + assert not p.is_symlink() + text, status = extract_text(p) + assert text is None + assert status == ExtractionStatus.SKIP_UNREADABLE, ( + f"Expected SKIP_UNREADABLE for non-existent regular file (file was " + f"deleted between scan and extract); got {status}. " + f"SKIP_BROKEN_SYMLINK should only fire when the path is_symlink()." + ) + + +def test_mine_formats_passes_extract_mode_format_to_file_already_mined(monkeypatch, tmp_path: Path): + """``mine_formats`` must call ``file_already_mined`` with + ``extract_mode='format'`` so format-mode idempotency is scoped to its + own drawer set. Otherwise drawers from convo_miner / project miner + on the same source file falsely indicate "already mined" to the + format miner (and vice versa). + + Per PR #1555 review (Copilot #11 + #12). Both call sites — the + pre-lock check in ``mine_formats`` and the post-lock recheck in + ``_file_chunks_locked`` — must pass the kwarg. + """ + from mempalace import format_miner + from mempalace.format_miner import mine_formats + + calls: list = [] + + def fake_check(collection, source_file, check_mtime=False, extract_mode=None): + calls.append( + { + "source_file": source_file, + "check_mtime": check_mtime, + "extract_mode": extract_mode, + } + ) + return False # never already mined — let the mine proceed + + monkeypatch.setattr(format_miner, "file_already_mined", fake_check) + + tmp = tmp_path / "src" + tmp.mkdir() + f = tmp / "x.pdf" + f.write_bytes(b"%PDF-1.4 stub") + + fake_config = { + "wing": "wing_test", + "rooms": [{"name": "documents", "keywords": ["documents"]}], + } + + with ( + patch("mempalace.format_miner.scan_formats", return_value=[f]), + patch("mempalace.format_miner._extract_via_markitdown", return_value="long " * 50), + patch("mempalace.format_miner.load_config", return_value=fake_config), + patch("mempalace.format_miner.detect_room", return_value="documents"), + ): + mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"), wing="wing_test") + + assert calls, "expected file_already_mined to be called at least once" + for call in calls: + assert call["extract_mode"] == "format", ( + f"file_already_mined call missing extract_mode='format': {call}" + ) + + +def test_mine_formats_does_not_write_sentinel_for_skip_no_markitdown(monkeypatch, tmp_path: Path): + """Sentinel writing must be skipped when the status is a transient + "missing optional dependency" (``SKIP_NO_MARKITDOWN``, + ``SKIP_NO_STRIPRTF``, ``SKIP_MISSING_FORMAT_DEPS``). Otherwise the + file is permanently marked as "already mined" — installing the missing + extra later does NOT trigger a re-mine, which is a real user surprise. + + Per PR #1555 review (Copilot #14). + """ + from mempalace import format_miner + from mempalace.format_miner import mine_formats + + register_calls: list = [] + + def fake_register(collection, source_file, wing, agent): + register_calls.append(source_file) + + monkeypatch.setattr(format_miner, "_register_file", fake_register) + + tmp = tmp_path / "src" + tmp.mkdir() + f = tmp / "y.pdf" + f.write_bytes(b"%PDF-1.4 stub") + + fake_config = { + "wing": "wing_test", + "rooms": [{"name": "documents", "keywords": ["documents"]}], + } + + # Force the extract path to raise ImportError so we hit SKIP_NO_MARKITDOWN. + with ( + patch("mempalace.format_miner.scan_formats", return_value=[f]), + patch( + "mempalace.format_miner._extract_via_markitdown", + side_effect=ImportError("no markitdown"), + ), + patch("mempalace.format_miner.load_config", return_value=fake_config), + patch("mempalace.format_miner.detect_room", return_value="documents"), + ): + mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"), wing="wing_test") + + assert str(f) not in register_calls, ( + f"sentinel was written for SKIP_NO_MARKITDOWN file; should be skipped " + f"so installing markitdown later triggers a re-mine. " + f"register_calls={register_calls}" + ) + + +def test_mine_formats_does_not_write_sentinel_for_skip_missing_format_deps( + monkeypatch, tmp_path: Path +): + """Same as the SKIP_NO_MARKITDOWN test but for SKIP_MISSING_FORMAT_DEPS + (raised when markitdown is installed but a per-format sub-extra like + ``markitdown[pdf]`` is missing). + """ + from mempalace import format_miner + from mempalace.format_miner import mine_formats + + register_calls: list = [] + + def fake_register(collection, source_file, wing, agent): + register_calls.append(source_file) + + monkeypatch.setattr(format_miner, "_register_file", fake_register) + + tmp = tmp_path / "src" + tmp.mkdir() + f = tmp / "z.pdf" + f.write_bytes(b"%PDF-1.4 stub") + + fake_config = { + "wing": "wing_test", + "rooms": [{"name": "documents", "keywords": ["documents"]}], + } + + class _FakeMissingDep(Exception): + pass + + _FakeMissingDep.__name__ = "MissingDependencyException" + + with ( + patch("mempalace.format_miner.scan_formats", return_value=[f]), + patch( + "mempalace.format_miner._extract_via_markitdown", + side_effect=_FakeMissingDep("install markitdown[pdf]"), + ), + patch("mempalace.format_miner.load_config", return_value=fake_config), + patch("mempalace.format_miner.detect_room", return_value="documents"), + ): + mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"), wing="wing_test") + + assert str(f) not in register_calls, ( + f"sentinel was written for SKIP_MISSING_FORMAT_DEPS file; should be " + f"skipped so installing markitdown[] later triggers a re-mine. " + f"register_calls={register_calls}" + ) + + +def test_mine_formats_catches_unexpected_exception_and_prints_summary( + monkeypatch, tmp_path: Path, capsys +): + """If an unexpected error happens during mining (something the + per-file try/except doesn't catch — e.g., the topic-tunnel block + blowing up in a way the inner handler misses), mine_formats must + catch it at the outer level, print a partial-progress summary, and + clean up the PID file. Not crash with a traceback to the user. + + Per PR #1555 review (Gemini #5). + """ + from mempalace.format_miner import mine_formats + + tmp = tmp_path / "src" + tmp.mkdir() + f = tmp / "exc.pdf" + f.write_bytes(b"%PDF-1.4 stub") + + fake_config = { + "wing": "wing_test", + "rooms": [{"name": "documents", "keywords": ["documents"]}], + } + + # Patch the file enumeration step to raise something unexpected — + # this fires OUTSIDE the per-file try/except. Without an outer + # ``except Exception`` clause, the traceback propagates and the + # summary block never prints. + def angry_enumerate(*args, **kwargs): + raise RuntimeError("simulated outer-loop explosion") + + with ( + patch("mempalace.format_miner.scan_formats", side_effect=angry_enumerate), + patch("mempalace.format_miner.load_config", return_value=fake_config), + ): + # Must NOT raise — outer except Exception should catch. + mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"), wing="wing_test") + + +def test_mine_formats_threads_chunk_size_from_user_config(monkeypatch, tmp_path: Path): + """``mine_formats`` must pass ``chunk_size`` (and ``chunk_overlap``, + ``min_chunk_size``) from MempalaceConfig through to ``chunk_text``. + Currently the config is loaded only to validate readability; the + custom chunk parameters are never used during format-mode mining, + so users who tuned their config see no effect. + + Per PR #1555 review (Gemini #3). + """ + from mempalace import format_miner + from mempalace.format_miner import mine_formats + + chunk_calls: list = [] + + def fake_chunk_text(content, source_file, **kwargs): + chunk_calls.append(kwargs) + return [{"content": content, "chunk_index": 0}] + + monkeypatch.setattr(format_miner, "chunk_text", fake_chunk_text) + + tmp = tmp_path / "src" + tmp.mkdir() + f = tmp / "config.pdf" + f.write_bytes(b"%PDF-1.4 stub") + + fake_config = { + "wing": "wing_test", + "rooms": [{"name": "documents", "keywords": ["documents"]}], + } + + # Inject custom config values via a fake MempalaceConfig. + class _FakeMempalaceConfig: + chunk_size = 1234 + chunk_overlap = 56 + min_chunk_size = 78 + + @property + def palace_path(self): + return str(tmp_path / "palace") + + monkeypatch.setattr(format_miner, "MempalaceConfig", _FakeMempalaceConfig) + + with ( + patch("mempalace.format_miner.scan_formats", return_value=[f]), + patch("mempalace.format_miner._extract_via_markitdown", return_value="long " * 200), + patch("mempalace.format_miner.load_config", return_value=fake_config), + patch("mempalace.format_miner.detect_room", return_value="documents"), + patch("mempalace.format_miner.file_already_mined", return_value=False), + ): + mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"), wing="wing_test") + + assert chunk_calls, "expected chunk_text to be called at least once" + call = chunk_calls[0] + assert call.get("chunk_size") == 1234, ( + f"chunk_text called without user's chunk_size from MempalaceConfig " + f"(got {call.get('chunk_size')}, expected 1234). kwargs={call}" + ) + assert call.get("chunk_overlap") == 56, ( + f"chunk_text called without user's chunk_overlap " + f"(got {call.get('chunk_overlap')}, expected 56). kwargs={call}" + ) + assert call.get("min_chunk_size") == 78, ( + f"chunk_text called without user's min_chunk_size " + f"(got {call.get('min_chunk_size')}, expected 78). kwargs={call}" + ) diff --git a/tests/test_line_numbers.py b/tests/test_line_numbers.py index 9115bb7f22..1184bdb050 100644 --- a/tests/test_line_numbers.py +++ b/tests/test_line_numbers.py @@ -1,10 +1,7 @@ -"""Tests for virtual line numbering (proposed for mempalace 3.3.6). +"""Tests for virtual line numbering (mempalace 3.3.6, integrated in PR #1555). -Run from the proposal directory: +Run with: pytest tests/test_line_numbers.py -v - -After integration into the mempalace package, change the import below to: - from mempalace.searcher import render_with_line_numbers, extract_line_range """ from mempalace.searcher import ( # noqa: E402 From 6658a4d11f36313ab8e9914a2de3e55bafaaab26 Mon Sep 17 00:00:00 2001 From: mvalentsev Date: Mon, 18 May 2026 04:41:27 +0500 Subject: [PATCH 37/39] fix(audit): chunk and batch content before embedding upsert (#1539) Four sites passed content to `collection.upsert(documents=[...])` or `collection.add(documents=[...])` without per-drawer size cap, hitting the same `RuntimeError: Invalid buffer size` crash class from the embedding model's attention buffer: - `general_extractor.extract_memories`: post-classification slicer that propagates `memory_type` to every slice. New `chunk_size` parameter defaulting to `DEFAULT_CHUNK_SIZE` and resolved from `MempalaceConfig.chunk_size` by the caller. - `diary_ingest.ingest_diaries`: per-entry drawers via existing `_split_entries`, with character-chunk fallback for any single entry larger than `chunk_size`. New `_diary_drawer_id_entry` helper carries (entry_idx, entry_chunk_idx). `chunk_index` in metadata is a global counter across the file so `searcher._expand_with_neighbors` stitches sibling chunks regardless of entry boundary. Upsert is batched atomic per file (one call carrying every entry/chunk) so a mid-pass embedding failure cannot half-write the day. Auto-purge on full rebuild deletes prior-pass drawers via `where={"source_file": ...}`, which also migrates pre-#1539 legacy `drawer_diary_` IDs as a side effect of normal use. - `mcp_server.tool_diary_write`: split oversized entries into bounded per-chunk drawers via a single batched `col.add` (atomic, no half-write on embedding failure). `col.add` is intentional: `entry_id` is timestamp-based with microsecond precision, so a duplicate is a same-microsecond clash that should surface rather than silently overwrite. - `mcp_server.tool_add_drawer`: same crash class on the more common add-drawer surface (100 KB sanitize cap, 125x `CHUNK_SIZE`). Chunked path mirrors Site 3 with batched atomic `col.upsert`, per-chunk `parent_drawer_id` + `chunk_index` metadata, and a dual-id idempotency probe (last chunk for atomicity, legacy `drawer_id` for pre-#1539 single-row backwards-compat). Return shape additive: `chunks` always present, `chunk_ids` on the chunked path. `tool_get_drawer` / `tool_delete_drawer` against the logical handle report "not found" on the chunked path; callers iterate `chunk_ids` or query `parent_drawer_id`. Chunk id width is `:06d` so even a single-digit `chunk_size` config cannot lex-sort chunks out of order. --- .gitignore | 1 + mempalace/convo_miner.py | 2 +- mempalace/diary_ingest.py | 145 ++++++++++++++-- mempalace/general_extractor.py | 45 ++++- mempalace/mcp_server.py | 185 ++++++++++++++++---- tests/test_closets.py | 288 +++++++++++++++++++++++++++++++- tests/test_general_extractor.py | 64 +++++++ tests/test_mcp_server.py | 227 +++++++++++++++++++++++++ 8 files changed, 892 insertions(+), 65 deletions(-) diff --git a/.gitignore b/.gitignore index ba8ec102cb..01d8778905 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,4 @@ venv/ # ChromaDB local data *.sqlite3-journal .envrc +.review-logs diff --git a/mempalace/convo_miner.py b/mempalace/convo_miner.py index 07d34e2cd7..0428eaea3a 100644 --- a/mempalace/convo_miner.py +++ b/mempalace/convo_miner.py @@ -601,7 +601,7 @@ def mine_convos( if extract_mode == "general": from .general_extractor import extract_memories - chunks = extract_memories(content) + chunks = extract_memories(content, chunk_size=cfg_chunk_size) # Each chunk already has memory_type; use it as the room name else: chunks = chunk_exchanges( diff --git a/mempalace/diary_ingest.py b/mempalace/diary_ingest.py index e6ffe42d9f..cf8f8b93d7 100644 --- a/mempalace/diary_ingest.py +++ b/mempalace/diary_ingest.py @@ -18,11 +18,13 @@ import hashlib import json +import logging import os import re from datetime import datetime, timezone from pathlib import Path +from .config import MempalaceConfig from .miner import _extract_entities_for_metadata from .palace import ( build_closet_lines, @@ -33,6 +35,8 @@ upsert_closet_lines, ) +logger = logging.getLogger(__name__) + DIARY_ENTRY_RE = re.compile(r"^## .+", re.MULTILINE) @@ -61,12 +65,35 @@ def _split_entries(text): def _diary_drawer_id(wing: str, date_str: str) -> str: - """Stable, wing-scoped drawer ID. Two diaries (e.g. 'work' vs 'personal') - sharing the same date never collide.""" + """Stable, wing-scoped legacy drawer ID (file-level). + + Retained for backwards-compatible cleanup of palaces that ingested + diaries before #1539 — those palaces hold one ``drawer_diary_{...}`` + per file. New drawers use ``_diary_drawer_id_entry`` so each ``##`` + entry becomes its own drawer (with per-entry character chunking + when an entry exceeds ``chunk_size``). + """ suffix = hashlib.sha256(f"{wing}|{date_str}".encode()).hexdigest()[:24] return f"drawer_diary_{suffix}" +def _diary_drawer_id_entry(wing: str, date_str: str, entry_idx: int, entry_chunk_idx: int) -> str: + """Per-entry, per-chunk drawer ID introduced in #1539. + + The ``v2_`` prefix distinguishes new IDs from the legacy file-level + scheme (one drawer per file). Legacy drawers from pre-#1539 palaces + are auto-purged on any ``ingest_diaries`` call that triggers a full + rebuild (``force=True`` or detected content change). The per-source + ``delete(where=...)`` step on full rebuild collects both legacy and + stale v2 drawers via the ``source_file`` metadata key, so the + schema migration runs as a side effect of normal use. + """ + suffix = hashlib.sha256( + f"{wing}|{date_str}|{entry_idx}|{entry_chunk_idx}".encode() + ).hexdigest()[:24] + return f"drawer_diary_v2_{suffix}" + + def _diary_closet_id_base(wing: str, date_str: str) -> str: suffix = hashlib.sha256(f"{wing}|{date_str}".encode()).hexdigest()[:24] return f"closet_diary_{suffix}" @@ -106,6 +133,7 @@ def ingest_diaries( drawers_col = get_collection(palace_path) closets_col = get_closets_collection(palace_path) + chunk_size = MempalaceConfig().chunk_size days_updated = 0 closets_created = 0 @@ -144,14 +172,17 @@ def ingest_diaries( content_changed = prev_hash is not None and curr_hash != prev_hash now_iso = datetime.now(timezone.utc).isoformat() - drawer_id = _diary_drawer_id(wing, date_str) entities = _extract_entities_for_metadata(text) source_file = str(diary_path) # Serialize per source — two terminals running ingest at once must # not interleave the upsert + closet-rebuild. with mine_lock(source_file): - drawer_meta = { + entries = _split_entries(text) + prev_entry_count = state.get(state_key, {}).get("entry_count", 0) + full_rebuild = force or content_changed + + base_meta = { "date": date_str, "wing": wing, "room": "daily", @@ -160,24 +191,104 @@ def ingest_diaries( "filed_at": now_iso, } if entities: - drawer_meta["entities"] = entities - drawers_col.upsert( - documents=[text], - ids=[drawer_id], - metadatas=[drawer_meta], - ) + base_meta["entities"] = entities + + # On full rebuild, purge ALL prior drawers for this source_file + # before writing fresh entries. This covers three cases in one + # step: (a) legacy pre-#1539 file-level drawers under the old + # ``drawer_diary_`` prefix, (b) v2 drawers from a prior pass + # where the file had MORE entries than the current run (entry + # deletion would otherwise leave trailing orphan drawers), and + # (c) in-place edits that shift content across entry boundaries. + if full_rebuild: + try: + drawers_col.delete(where={"source_file": source_file}) + except Exception as exc: + # ChromaDB ``delete(where=...)`` against an empty collection + # returns silently rather than raising, so this catch only + # fires on real backend errors (locked DB, schema mismatch, + # transient I/O). Log at debug to preserve diagnostics + # without aborting the rebuild: the upsert below is the + # load-bearing write, and if it also fails the state file + # is left unchanged so the next pass retries cleanly. + logger.debug( + "legacy purge skipped for %s: %s", + source_file, + exc, + exc_info=True, + ) - entries = _split_entries(text) - prev_entry_count = state.get(state_key, {}).get("entry_count", 0) - full_rebuild = force or content_changed - new_entries = entries if full_rebuild else entries[prev_entry_count:] + # Per-entry drawers, with character chunking inside any entry + # whose serialized text exceeds chunk_size. ``chunk_index`` is + # a global counter across the file so the searcher's + # ``_expand_with_neighbors`` (which queries by source_file + + # chunk_index) can stitch sibling chunks back together + # regardless of entry boundary. ``entry_index`` and + # ``entry_chunk_index`` are preserved for entry-grouping + # consumers. + # + # Accumulate all drawers for the file into one batched upsert + # so the embedding pass either commits every chunk or none. + # A mid-loop embedding failure would otherwise leave a + # half-written day's drawers behind, which the next + # incremental pass would skip (state file already updated) + # and the searcher would silently return partial results. + global_chunk_index = 0 + batch_ids: list[str] = [] + batch_docs: list[str] = [] + batch_metas: list[dict] = [] + for entry_idx, (header, body) in enumerate(entries): + entry_text = f"{header}\n{body}" if body else header + if len(entry_text) <= chunk_size: + batch_ids.append(_diary_drawer_id_entry(wing, date_str, entry_idx, 0)) + batch_docs.append(entry_text) + batch_metas.append( + { + **base_meta, + "chunk_index": global_chunk_index, + "entry_index": entry_idx, + "entry_chunk_index": 0, + "entry_header_preview": header[:120], + } + ) + global_chunk_index += 1 + else: + for entry_chunk_idx, start in enumerate(range(0, len(entry_text), chunk_size)): + batch_ids.append( + _diary_drawer_id_entry(wing, date_str, entry_idx, entry_chunk_idx) + ) + batch_docs.append(entry_text[start : start + chunk_size]) + batch_metas.append( + { + **base_meta, + "chunk_index": global_chunk_index, + "entry_index": entry_idx, + "entry_chunk_index": entry_chunk_idx, + "entry_header_preview": header[:120], + } + ) + global_chunk_index += 1 + + if batch_ids: + drawers_col.upsert( + ids=batch_ids, + documents=batch_docs, + metadatas=batch_metas, + ) + new_entries = entries if full_rebuild else entries[prev_entry_count:] if new_entries: all_lines = [] - for header, body in new_entries: - entry_text = f"{header}\n{body}" + for offset, (header, body) in enumerate(new_entries): + entry_idx = offset if full_rebuild else prev_entry_count + offset + entry_text = f"{header}\n{body}" if body else header + # Closet references the canonical (entry_chunk_idx=0) + # drawer for the entry. Searcher._expand_with_neighbors + # stitches sibling chunks back via the + # (source_file, chunk_index) pair. + entry_drawer_id = _diary_drawer_id_entry(wing, date_str, entry_idx, 0) entry_lines = build_closet_lines( - source_file, [drawer_id], entry_text, wing, "daily" + source_file, [entry_drawer_id], entry_text, wing, "daily" ) all_lines.extend(entry_lines) diff --git a/mempalace/general_extractor.py b/mempalace/general_extractor.py index e849d7cf13..e9f3fc0c5e 100644 --- a/mempalace/general_extractor.py +++ b/mempalace/general_extractor.py @@ -22,6 +22,8 @@ import re from typing import List, Dict, Tuple +from .config import DEFAULT_CHUNK_SIZE + # ============================================================================= # MARKER SETS — One per memory type @@ -360,13 +362,23 @@ def _score_markers(text: str, markers: List[str]) -> Tuple[float, List[str]]: # ============================================================================= -def extract_memories(text: str, min_confidence: float = 0.3) -> List[Dict]: +def extract_memories( + text: str, + min_confidence: float = 0.3, + chunk_size: int = DEFAULT_CHUNK_SIZE, +) -> List[Dict]: """ Extract memories from a text string. Args: text: The text to extract from (any format). min_confidence: Minimum confidence threshold (0.0-1.0). + chunk_size: Per-memory character cap. Segments exceeding this + size are sliced verbatim into multiple memories that share + the same ``memory_type``. Caller (typically ``mine_convos``) + should pass ``MempalaceConfig.chunk_size`` so config-driven + sizing reaches this path; the default matches the + module-level CHUNK_SIZE in ``convo_miner.py``. Returns: List of dicts: {"content": str, "memory_type": str, "chunk_index": int} @@ -410,13 +422,30 @@ def extract_memories(text: str, min_confidence: float = 0.3) -> List[Dict]: if confidence < min_confidence: continue - memories.append( - { - "content": para.strip(), - "memory_type": max_type, - "chunk_index": len(memories), - } - ) + content = para.strip() + # Slice oversized segments verbatim. ``memory_type`` is propagated + # to every slice — reclassifying sub-segments risks dropping data + # (marker may live in only one slice), and the verbatim-store + # mandate in CLAUDE.md forbids silent loss. Acceptable trade-off: + # the label is an approximation for sub-slices, not a fresh + # classification. + if len(content) <= chunk_size: + memories.append( + { + "content": content, + "memory_type": max_type, + "chunk_index": len(memories), + } + ) + continue + for i in range(0, len(content), chunk_size): + memories.append( + { + "content": content[i : i + chunk_size], + "memory_type": max_type, + "chunk_index": len(memories), + } + ) return memories diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index 6e5a163ae2..5bfda3f208 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -1103,7 +1103,18 @@ def tool_follow_tunnels(wing: str, room: str): def tool_add_drawer( wing: str, room: str, content: str, source_file: str = None, added_by: str = "mcp" ): - """File verbatim content into a wing/room. Checks for duplicates first.""" + """File verbatim content into a wing/room. Checks for duplicates first. + + Content above ``chunk_size`` is split into bounded per-chunk drawers + via a single batched upsert. Each chunk carries ``parent_drawer_id`` + linkage and ``chunk_index`` metadata so search can rejoin them. The + returned ``drawer_id`` is the LOGICAL group handle on the chunked + path; physical drawer ids are in ``chunk_ids`` (#1539). To delete + or fetch the underlying drawers, iterate ``chunk_ids`` or query by + ``parent_drawer_id`` — ``tool_get_drawer(drawer_id)`` and + ``tool_delete_drawer(drawer_id)`` report "not found" on the chunked + path because no row is stored under the logical group id. + """ global _metadata_cache try: wing = sanitize_name(wing, "wing") @@ -1132,38 +1143,93 @@ def tool_add_drawer( }, ) - # Idempotency: if the deterministic ID already exists, return success as a no-op. + chunk_size = _config.chunk_size + base_meta = { + "wing": wing, + "room": room, + "source_file": source_file or "", + "added_by": added_by, + "filed_at": datetime.now().isoformat(), + } + + # Idempotency. Three cases to detect a prior committed write: + # (a) Single-doc path: drawer_id row exists (the only id used). + # (b) Chunked path: probe the LAST chunk id — its presence implies + # every earlier chunk also landed, since the batched upsert + # is all-or-nothing. + # (c) Legacy pre-#1539 single-row write of oversized content under + # drawer_id: probe drawer_id alongside the last chunk id so a + # re-call with identical oversized content does not duplicate + # the legacy row by adding fresh chunks under different ids. + if len(content) <= chunk_size: + idempotency_probe_ids = [drawer_id] + else: + last_chunk_idx = (len(content) - 1) // chunk_size + idempotency_probe_ids = [drawer_id, f"{drawer_id}_chunk_{last_chunk_idx:06d}"] try: - existing = col.get(ids=[drawer_id], include=[]) + existing = col.get(ids=idempotency_probe_ids, include=[]) if existing.ids: return {"success": True, "reason": "already_exists", "drawer_id": drawer_id} except Exception: - logger.debug("Idempotency pre-check failed for %s", drawer_id, exc_info=True) + logger.debug("Idempotency pre-check failed for %s", idempotency_probe_ids, exc_info=True) try: - col.upsert( - ids=[drawer_id], - documents=[content], - metadatas=[ - { - "wing": wing, - "room": room, - "source_file": source_file or "", - "chunk_index": 0, - "added_by": added_by, - "filed_at": datetime.now().isoformat(), - } - ], - ) - inserted = col.get(ids=[drawer_id], include=[]) + if len(content) <= chunk_size: + col.upsert( + ids=[drawer_id], + documents=[content], + metadatas=[{**base_meta, "chunk_index": 0}], + ) + inserted = col.get(ids=[drawer_id], include=[]) + if not inserted.ids: + raise RuntimeError( + "Drawer write was acknowledged but the new ID is not readable. " + "The palace index may be stale; run reconnect or repair." + ) + _metadata_cache = None + logger.info(f"Filed drawer: {drawer_id} → {wing}/{room}") + return { + "success": True, + "drawer_id": drawer_id, + "wing": wing, + "room": room, + "chunks": 1, + } + + # Oversized content: split into bounded per-chunk drawers so the + # embedding model never sees a document above ``chunk_size``. + # Single batched ``upsert`` so the embedding pass either commits + # every chunk or none — no half-written palace if the embedding + # model fails mid-loop (#1539). + chunk_ids: list[str] = [] + chunk_docs: list[str] = [] + chunk_metas: list[dict] = [] + for i in range(0, len(content), chunk_size): + chunk_idx = i // chunk_size + chunk_ids.append(f"{drawer_id}_chunk_{chunk_idx:06d}") + chunk_docs.append(content[i : i + chunk_size]) + chunk_metas.append( + {**base_meta, "chunk_index": chunk_idx, "parent_drawer_id": drawer_id} + ) + col.upsert(ids=chunk_ids, documents=chunk_docs, metadatas=chunk_metas) + # Probe the LAST chunk id, not the first — its presence confirms + # the whole batch landed, not just the leading row. + inserted = col.get(ids=[chunk_ids[-1]], include=[]) if not inserted.ids: raise RuntimeError( "Drawer write was acknowledged but the new ID is not readable. " "The palace index may be stale; run reconnect or repair." ) _metadata_cache = None - logger.info(f"Filed drawer: {drawer_id} → {wing}/{room}") - return {"success": True, "drawer_id": drawer_id, "wing": wing, "room": room} + logger.info(f"Filed drawer: {drawer_id} → {wing}/{room} ({len(chunk_ids)} chunks)") + return { + "success": True, + "drawer_id": drawer_id, + "wing": wing, + "room": room, + "chunks": len(chunk_ids), + "chunk_ids": chunk_ids, + } except Exception as e: return {"success": False, "error": str(e)} @@ -1579,29 +1645,76 @@ def tool_diary_write(agent_name: str, entry: str, topic: str = "general", wing: # semantic search quality. For now, store raw AAAK in metadata so it's # preserved, and keep the document as-is for embedding (even though # compressed AAAK degrades embedding quality). - col.add( - ids=[entry_id], - documents=[entry], - metadatas=[ + base_metadata = { + "wing": wing, + "room": room, + "hall": "hall_diary", + "topic": topic, + "type": "diary_entry", + "agent": agent_name, + "filed_at": now.isoformat(), + "date": now.strftime("%Y-%m-%d"), + } + chunk_size = _config.chunk_size + if len(entry) <= chunk_size: + col.add( + ids=[entry_id], + documents=[entry], + metadatas=[{**base_metadata, "chunk_index": 0}], + ) + logger.info(f"Diary entry: {entry_id} → {wing}/diary/{topic}") + return { + "success": True, + "entry_id": entry_id, + "agent": agent_name, + "topic": topic, + "timestamp": now.isoformat(), + "chunks": 1, + } + + # Oversized entry: split into bounded per-chunk drawers so the + # embedding model never sees a document above ``chunk_size``. + # Every chunk carries ``parent_entry_id`` so search can rejoin + # them and ``chunk_index`` for ordered reconstruction (#1539). + # Note on ``entry_id`` in the return value: for the chunked + # path the returned ``entry_id`` is the LOGICAL group handle + # (no drawer is stored under that exact id). The physical + # drawer ids are in ``chunk_ids``. Callers wanting to fetch + # by id should iterate ``chunk_ids``; callers wanting to + # query by metadata can filter on ``parent_entry_id``. + # Use a single batched ``add`` so the embedding pass either + # commits all chunks or none — avoids a half-written palace + # if the embedding model fails mid-loop. ``col.add`` (not + # ``upsert``) is intentional here: ``entry_id`` is timestamp- + # based with microsecond precision, so every call generates a + # fresh id and a duplicate is by definition a same-microsecond + # clash that should surface as an error rather than silently + # overwrite the prior entry (cf. ``tool_add_drawer`` whose + # content-hash ids are deliberately idempotent and use upsert). + chunk_ids: list[str] = [] + chunk_docs: list[str] = [] + chunk_metas: list[dict] = [] + for i in range(0, len(entry), chunk_size): + chunk_idx = i // chunk_size + chunk_ids.append(f"{entry_id}_chunk_{chunk_idx:06d}") + chunk_docs.append(entry[i : i + chunk_size]) + chunk_metas.append( { - "wing": wing, - "room": room, - "hall": "hall_diary", - "topic": topic, - "type": "diary_entry", - "agent": agent_name, - "filed_at": now.isoformat(), - "date": now.strftime("%Y-%m-%d"), + **base_metadata, + "chunk_index": chunk_idx, + "parent_entry_id": entry_id, } - ], - ) - logger.info(f"Diary entry: {entry_id} → {wing}/diary/{topic}") + ) + col.add(ids=chunk_ids, documents=chunk_docs, metadatas=chunk_metas) + logger.info(f"Diary entry: {entry_id} → {wing}/diary/{topic} ({len(chunk_ids)} chunks)") return { "success": True, "entry_id": entry_id, "agent": agent_name, "topic": topic, "timestamp": now.isoformat(), + "chunks": len(chunk_ids), + "chunk_ids": chunk_ids, } except Exception as e: return {"success": False, "error": str(e)} diff --git a/tests/test_closets.py b/tests/test_closets.py index 2aa5597309..7d0113f268 100644 --- a/tests/test_closets.py +++ b/tests/test_closets.py @@ -31,6 +31,7 @@ import threading import time +import pytest import yaml from mempalace.miner import ( @@ -732,14 +733,17 @@ def test_wing_prefixed_drawer_id_prevents_cross_diary_collision(self, tmp_path): palace_dir = tmp_path / "palace" - from mempalace.diary_ingest import _diary_drawer_id, ingest_diaries + from mempalace.diary_ingest import _diary_drawer_id_entry, ingest_diaries ingest_diaries(str(personal_dir), str(palace_dir), wing="personal", force=True) ingest_diaries(str(work_dir), str(palace_dir), wing="work", force=True) col = get_collection(str(palace_dir)) - personal_id = _diary_drawer_id("personal", "2026-04-13") - work_id = _diary_drawer_id("work", "2026-04-13") + # Post-#1539: per-entry drawers. Each single-entry diary produces + # one drawer keyed by (wing, date, entry_idx=0, chunk_idx=0). The + # wing component still keeps work vs personal collisions apart. + personal_id = _diary_drawer_id_entry("personal", "2026-04-13", 0, 0) + work_id = _diary_drawer_id_entry("work", "2026-04-13", 0, 0) assert personal_id != work_id personal = col.get(ids=[personal_id]) @@ -749,6 +753,284 @@ def test_wing_prefixed_drawer_id_prevents_cross_diary_collision(self, tmp_path): assert "Personal-only marker." in personal["documents"][0] assert "Work-only marker." in work["documents"][0] + # ── #1539: per-entry drawers, oversized-entry chunking ───────── + + def test_diary_multiple_entries_one_drawer_each(self, tmp_path): + """Regression for #1539: each ``##`` entry must become its own + drawer rather than the whole file being one document. Pre-fix + behaviour: a single file-level drawer for any number of entries. + Post-fix: one drawer per entry.""" + diary_dir = tmp_path / "diaries" + diary_dir.mkdir() + (diary_dir / "2026-04-13.md").write_text( + "# 2026-04-13\n\n" + "## 10:00 — first\n\nfirst body content here, well above min size.\n\n" + "## 11:00 — second\n\nsecond body content with enough length here.\n\n" + "## 12:00 — third\n\nthird body content with sufficient text in it.\n" + ) + palace_dir = tmp_path / "palace" + + from mempalace.diary_ingest import ingest_diaries + + ingest_diaries(str(diary_dir), str(palace_dir), force=True) + count = get_collection(str(palace_dir)).count() + assert count == 3, f"expected one drawer per ## entry (3 entries); got {count} drawers" + + def test_large_diary_with_oversized_entry_chunks_within_entry(self, tmp_path): + """Regression for #1539: when a single ``##`` entry exceeds + chunk_size, the ingester must chunk that entry into bounded + drawers rather than uploading the whole entry as one oversized + document (which crashes the embedding model in production).""" + diary_dir = tmp_path / "diaries" + diary_dir.mkdir() + # Middle entry body is 1500 chars — above CHUNK_SIZE=800 but below + # the embedding model's hard token limit. This trips the size-cap + # assertion on develop without crashing the test infrastructure. + big_body = "X" * 1500 + (diary_dir / "2026-04-13.md").write_text( + f"# 2026-04-13\n\n" + f"## 10:00 — first\n\nfirst body content here, well above min size.\n\n" + f"## 11:00 — big\n\n{big_body}\n\n" + f"## 12:00 — last\n\nlast body content with sufficient text in it.\n" + ) + palace_dir = tmp_path / "palace" + + from mempalace.diary_ingest import ingest_diaries + + ingest_diaries(str(diary_dir), str(palace_dir), force=True) + col = get_collection(str(palace_dir)) + drawers = col.get() + docs = drawers["documents"] + + max_len = max(len(d) for d in docs) + assert max_len <= 800, ( + f"no drawer document may exceed CHUNK_SIZE=800; got max_len={max_len}" + ) + # 3 entries with the middle entry split into >= 2 chunks → >= 4 drawers + assert len(docs) >= 4, ( + f"oversized middle entry must produce multiple drawers; got {len(docs)} total drawers" + ) + + def test_incremental_appends_new_entry_only(self, tmp_path): + """Regression for #1539: incremental ingest must add exactly the + delta when one new entry is appended (not re-rewrite all).""" + diary_dir = tmp_path / "diaries" + diary_dir.mkdir() + diary_file = diary_dir / "2026-04-13.md" + diary_file.write_text( + "# 2026-04-13\n\n" + "## 10:00 — first\n\nfirst body content, sufficiently long.\n\n" + "## 11:00 — second\n\nsecond body content with text in it.\n" + ) + palace_dir = tmp_path / "palace" + + from mempalace.diary_ingest import ingest_diaries + + ingest_diaries(str(diary_dir), str(palace_dir), force=True) + initial = get_collection(str(palace_dir)).count() + assert initial == 2, f"baseline: 2 entries → 2 drawers; got {initial}" + + diary_file.write_text( + diary_file.read_text() + + "\n## 12:00 — third\n\nthird body, newly added on second run.\n" + ) + ingest_diaries(str(diary_dir), str(palace_dir)) + final = get_collection(str(palace_dir)).count() + assert final == 3, f"after appending 1 entry: 3 drawers total; got {final}" + + def test_entry_count_watermark_matches_drawer_count(self, tmp_path): + """Regression for #1539: persisted ``entry_count`` watermark + must equal the number of entries split from the file (and + therefore the number of drawers under the new schema).""" + diary_dir = tmp_path / "diaries" + diary_dir.mkdir() + (diary_dir / "2026-04-13.md").write_text( + "# 2026-04-13\n\n" + "## 10:00 — a\n\nbody one with enough text to count.\n\n" + "## 11:00 — b\n\nbody two with enough text to count.\n\n" + "## 12:00 — c\n\nbody three with enough text to count.\n" + ) + palace_dir = tmp_path / "palace" + + from mempalace.diary_ingest import _state_file_for, _split_entries, ingest_diaries + + ingest_diaries(str(diary_dir), str(palace_dir), force=True) + + state = json.loads(_state_file_for(str(palace_dir), diary_dir.resolve()).read_text()) + key = "diary|2026-04-13.md" + assert state[key]["entry_count"] == 3, ( + f"watermark must equal entry count; got {state[key]['entry_count']}" + ) + + text = (diary_dir / "2026-04-13.md").read_text() + assert len(_split_entries(text)) == state[key]["entry_count"] + + def test_diary_chunk_index_is_global_across_entries(self, tmp_path): + """Regression for #1539 review: ``chunk_index`` must be a global + counter across the file (not per-entry) so the searcher's + ``_expand_with_neighbors`` (which queries by source_file + + chunk_index range) can stitch sibling chunks regardless of + entry boundary. Pre-fix the metadata key was ``entry_chunk_index`` + and search neighbor expansion silently fell back to the matched + drawer alone for any diary hit.""" + diary_dir = tmp_path / "diaries" + diary_dir.mkdir() + # 3 entries; middle is oversized (1500 chars > CHUNK_SIZE=800). + big_body = "X" * 1500 + (diary_dir / "2026-04-13.md").write_text( + f"# 2026-04-13\n\n" + f"## 10:00 — a\n\nfirst body content with enough length here.\n\n" + f"## 11:00 — b\n\n{big_body}\n\n" + f"## 12:00 — c\n\nthird body content with enough length here.\n" + ) + palace_dir = tmp_path / "palace" + + from mempalace.diary_ingest import ingest_diaries + + ingest_diaries(str(diary_dir), str(palace_dir), force=True) + col = get_collection(str(palace_dir)) + all_drawers = col.get() + + indices = sorted(m["chunk_index"] for m in all_drawers["metadatas"]) + # Global counter: 0, 1, 2, ... contiguous across all entries. + assert indices == list(range(len(indices))), ( + f"chunk_index must be contiguous 0..N-1 globally; got {indices}" + ) + # All chunks have the same source_file (filter target for neighbor + # expansion). Verify the searcher-required pair (source_file + + # chunk_index) is consistent on every drawer. + sources = {m["source_file"] for m in all_drawers["metadatas"]} + assert len(sources) == 1, f"all drawers must share one source_file; got {sources}" + + def test_diary_entry_deletion_purges_orphan_drawers(self, tmp_path): + """Regression for #1539 review: if a diary shrinks (entries + deleted), the full-rebuild step must purge prior-pass drawers + for that ``source_file`` before re-writing — otherwise trailing + drawers from the longer prior pass remain as orphans and + pollute search results forever.""" + diary_dir = tmp_path / "diaries" + diary_dir.mkdir() + diary_file = diary_dir / "2026-04-13.md" + diary_file.write_text( + "# 2026-04-13\n\n" + "## 10:00 — first\n\nfirst body content with enough length here.\n\n" + "## 11:00 — second\n\nsecond body content with enough length here.\n\n" + "## 12:00 — third\n\nthird body content with enough length here.\n" + ) + palace_dir = tmp_path / "palace" + + from mempalace.diary_ingest import ingest_diaries + + ingest_diaries(str(diary_dir), str(palace_dir), force=True) + col = get_collection(str(palace_dir)) + assert col.count() == 3, f"baseline: 3 entries → 3 drawers; got {col.count()}" + + # Shrink the file to 1 entry. Hash changes → content_changed=True + # → full_rebuild=True → purge step must remove the trailing 2 drawers. + diary_file.write_text( + "# 2026-04-13\n\n## 10:00 — first\n\nfirst body content with enough length here.\n" + ) + ingest_diaries(str(diary_dir), str(palace_dir)) + final = col.count() + assert final == 1, ( + f"after shrinking 3 entries → 1 entry: exactly 1 drawer; " + f"got {final} (orphans from prior pass not purged)" + ) + + def test_diary_header_only_entry_produces_drawer(self, tmp_path): + """A ``##`` header with no body still produces a drawer carrying + the header text. Catches a regression where ``if body`` would + collapse the entry.""" + diary_dir = tmp_path / "diaries" + diary_dir.mkdir() + (diary_dir / "2026-04-13.md").write_text( + "# 2026-04-13\n\n" + "## 10:00 — note one with sufficient length to pass min size\n\n" + "## 11:00 — note two with sufficient length to pass min size\n" + ) + palace_dir = tmp_path / "palace" + + from mempalace.diary_ingest import ingest_diaries + + ingest_diaries(str(diary_dir), str(palace_dir), force=True) + col = get_collection(str(palace_dir)) + assert col.count() == 2, ( + f"expected 2 drawers (one per header-only entry); got {col.count()}" + ) + docs = col.get()["documents"] + # Each drawer holds the header line itself. + joined = "\n".join(docs) + assert "note one" in joined + assert "note two" in joined + + def test_diary_atomic_batched_upsert_on_failure(self, tmp_path, monkeypatch): + """A mid-pass embedding failure must leave the collection empty + rather than half-written. Without batched-atomic upsert, an + earlier per-loop upsert would partially commit and the state + file would skip the file on re-run, silently losing entries. + + The fixture mixes normal entries with one oversized entry so + the failed batch exercises both the single-entry-per-drawer + branch and the per-entry character-chunk fallback.""" + from mempalace import diary_ingest + + diary_dir = tmp_path / "diaries" + diary_dir.mkdir() + (diary_dir / "2026-05-18.md").write_text( + "# 2026-05-18\n\n" + "## 10:00 — entry one with sufficient body to clear min size threshold\n" + "More body text for entry one with enough length.\n\n" + "## 11:00 — entry two with oversized body that forces per-entry chunking\n" + + "A" + * 3000 + + "\n\n" + "## 12:00 — entry three with body added for adequate test fixture size\n" + "More body text for entry three to make it realistic.\n" + ) + palace_dir = tmp_path / "palace" + + class _RaisingCollection: + def __init__(self, real): + self._real = real + self.upsert_call_count = 0 + + def __getattr__(self, name): + return getattr(self._real, name) + + def upsert(self, **kwargs): + self.upsert_call_count += 1 + raise RuntimeError("simulated embedding model failure") + + real_get_collection = diary_ingest.get_collection + raising_holder = {} + + def _wrap_get_collection(path): + real = real_get_collection(path) + wrapped = _RaisingCollection(real) + raising_holder["col"] = wrapped + return wrapped + + monkeypatch.setattr(diary_ingest, "get_collection", _wrap_get_collection) + + with pytest.raises(RuntimeError, match="simulated embedding model failure"): + diary_ingest.ingest_diaries(str(diary_dir), str(palace_dir), force=True) + + # Exactly one batched upsert attempt — not three per-loop calls. + # Fixture has a single diary file, so one batch per file == one + # total. With a multi-file fixture this would be N batches. + assert raising_holder["col"].upsert_call_count == 1, ( + "diary ingest must accumulate one batch per file; " + f"got {raising_holder['col'].upsert_call_count} upsert calls" + ) + + # Re-open the real collection (untouched by the failed write) and + # confirm nothing landed for this source file. + real_col = real_get_collection(str(palace_dir)) + rows = real_col.get(where={"source_file": str(diary_dir / "2026-05-18.md")}) + assert rows["ids"] == [], ( + f"failed batched upsert must leave no partial drawers; got {rows['ids']}" + ) + # ── cross-wing tunnels ─────────────────────────────────────────────── diff --git a/tests/test_general_extractor.py b/tests/test_general_extractor.py index 0f5d46cf2e..dd8fa586a9 100644 --- a/tests/test_general_extractor.py +++ b/tests/test_general_extractor.py @@ -246,3 +246,67 @@ def test_positive_words(): def test_negative_words(): assert "bug" in NEGATIVE_WORDS assert "crash" in NEGATIVE_WORDS + + +# ── extract_memories — oversized segment chunking (#1539) ────────────── + + +def test_extract_memories_oversized_segment_slices_with_label_preserved(): + """Regression for #1539: a segment longer than chunk_size must be + split into multiple memories with the same memory_type. Joined + slices must equal the original (verbatim store per CLAUDE.md).""" + decision_phrase = "We decided to migrate to PostgreSQL because performance matters. " + long_segment = decision_phrase * 50 # ~3,200 chars, well above default 800 + memories = extract_memories(long_segment) + assert len(memories) > 1, ( + f"oversized segment must split into multiple slices; got {len(memories)}" + ) + assert all(len(m["content"]) <= 800 for m in memories), ( + f"all slices must be <= chunk_size=800; got max={max(len(m['content']) for m in memories)}" + ) + types = {m["memory_type"] for m in memories} + assert len(types) == 1, f"all slices must share one memory_type; got {types}" + assert "decision" in types + joined = "".join(m["content"] for m in memories) + assert joined == long_segment.strip(), ( + "joined slices must equal original (after strip) verbatim" + ) + + +def test_extract_memories_oversized_segment_with_custom_chunk_size(): + """Regression for #1539: caller-supplied chunk_size must govern the + paragraph slicer in extract_memories.""" + decision_phrase = "We decided on Redis because we measured the latency profile. " + long_segment = decision_phrase * 40 # ~2,500 chars + memories = extract_memories(long_segment, chunk_size=400) + assert len(memories) > 1 + assert all(len(m["content"]) <= 400 for m in memories), ( + f"all slices must be <= 400; got max={max(len(m['content']) for m in memories)}" + ) + + +def test_extract_memories_normal_segment_unchanged(): + """Regression catch: a segment under chunk_size must produce + exactly one memory (existing pre-#1539 behaviour for sub-cap).""" + text = ( + "We decided to use React because it fits our team workflow and " + "the migration path from our existing stack is clear." + ) + memories = extract_memories(text) + assert len(memories) == 1 + assert memories[0]["content"] == text + assert memories[0]["memory_type"] == "decision" + + +def test_extract_memories_chunk_index_contiguous_across_segments(): + """Regression for #1539: chunk_index must be sequential 0,1,2,... + across mixed normal + oversized segments without gaps.""" + decision_phrase = "We decided to choose Postgres because the index plan works. " + long_segment = decision_phrase * 30 # ~1,800 chars → multiple slices at 800 + second_short = "I prefer always writing tests first because it shapes the API better." + text = long_segment + "\n\n" + second_short + memories = extract_memories(text) + indices = [m["chunk_index"] for m in memories] + assert indices == list(range(len(memories))), ( + f"chunk_index must be 0..N-1 sequential; got {indices}" + ) diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index d3ba4aaeb4..988d57d18c 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -1408,6 +1408,158 @@ def _raise(*args, **kwargs): assert result == {"error": msg} + def test_add_drawer_normal_content_single_drawer(self, monkeypatch, config, palace_path, kg): + """Regression catch: content below CHUNK_SIZE produces exactly + one drawer with ``chunks == 1``. Pre-#1539 contract preserved.""" + _patch_mcp_server(monkeypatch, config, kg) + _client, _col = _get_collection(palace_path, create=True) + del _client + from mempalace.mcp_server import tool_add_drawer + + result = tool_add_drawer(wing="w", room="r", content="Short content well under chunk_size.") + assert result["success"] is True + assert result["chunks"] == 1 + assert "chunk_ids" not in result + _client2, col = _get_collection(palace_path) + del _client2 + assert col.count() == 1 + assert col.get()["ids"] == [result["drawer_id"]] + + def test_add_drawer_oversized_content_chunked(self, monkeypatch, config, palace_path, kg): + """Regression for #1539: content far above chunk_size must be + sliced into bounded per-chunk drawers, each linked by a + ``parent_drawer_id`` metadata field. No stored document may + exceed the configured chunk_size.""" + _patch_mcp_server(monkeypatch, config, kg) + _client, _col = _get_collection(palace_path, create=True) + del _client + from mempalace.mcp_server import tool_add_drawer + + oversized = "X" * 10000 + result = tool_add_drawer(wing="w", room="r", content=oversized) + assert result["success"] is True + assert result["chunks"] > 1 + assert "chunk_ids" in result and len(result["chunk_ids"]) == result["chunks"] + + _client2, col = _get_collection(palace_path) + del _client2 + stored = col.get() + max_doc = max(len(d) for d in stored["documents"]) + assert max_doc <= config.chunk_size, ( + f"no stored document may exceed chunk_size={config.chunk_size}; got max={max_doc}" + ) + # Chroma does not guarantee insertion order on a bare ``get()``; + # sort by ``chunk_index`` before joining so the verbatim check + # is deterministic. + ordered = sorted( + zip(stored["metadatas"], stored["documents"]), + key=lambda pair: pair[0]["chunk_index"], + ) + assert "".join(doc for _meta, doc in ordered) == oversized + parent_ids = {m.get("parent_drawer_id") for m in stored["metadatas"]} + assert parent_ids == {result["drawer_id"]}, ( + f"all chunks must share one parent_drawer_id; got {parent_ids}" + ) + + def test_add_drawer_oversized_idempotency_skips_duplicate_chunk_writes( + self, monkeypatch, config, palace_path, kg + ): + """Re-calling with identical oversized content must not duplicate + any drawer. Idempotency on the chunked path probes the last + chunk id (its presence implies the whole batch committed) and + also the legacy logical drawer_id so a pre-#1539 single-row + write under the same logical id does not get co-resident chunk + siblings on the next call.""" + _patch_mcp_server(monkeypatch, config, kg) + _client, _col = _get_collection(palace_path, create=True) + del _client + from mempalace.mcp_server import tool_add_drawer + + oversized = "Y" * 5000 + r1 = tool_add_drawer(wing="w", room="r", content=oversized) + assert r1["success"] is True and r1["chunks"] > 1 + r2 = tool_add_drawer(wing="w", room="r", content=oversized) + assert r2["success"] is True + assert r2.get("reason") == "already_exists" + + _client2, col = _get_collection(palace_path) + del _client2 + assert col.count() == r1["chunks"] + # The probe must succeed against the last chunk id (atomicity + # signal), and no row must be stored under the logical id. + last_chunk = r1["chunk_ids"][-1] + assert col.get(ids=[last_chunk])["ids"] == [last_chunk] + assert col.get(ids=[r1["drawer_id"]])["ids"] == [] + + def test_add_drawer_chunk_metadata_carries_parent_link( + self, monkeypatch, config, palace_path, kg + ): + """Every chunk produced from oversized content must carry both + ``chunk_index`` (0..N-1) and ``parent_drawer_id`` matching the + logical group handle returned to the caller.""" + _patch_mcp_server(monkeypatch, config, kg) + _client, _col = _get_collection(palace_path, create=True) + del _client + from mempalace.mcp_server import tool_add_drawer + + result = tool_add_drawer(wing="w", room="r", content="Q" * 3500) + assert result["success"] is True and result["chunks"] > 1 + + _client2, col = _get_collection(palace_path) + del _client2 + stored = col.get() + indices = sorted(m["chunk_index"] for m in stored["metadatas"]) + assert indices == list(range(len(indices))) + for meta in stored["metadatas"]: + assert meta.get("parent_drawer_id") == result["drawer_id"] + + def test_add_drawer_boundary_exact_chunk_size_stays_single( + self, monkeypatch, config, palace_path, kg + ): + """The ``<= chunk_size`` predicate must include the boundary: + content of exactly chunk_size chars stays a single drawer, not + an off-by-one chunked write.""" + _patch_mcp_server(monkeypatch, config, kg) + _client, _col = _get_collection(palace_path, create=True) + del _client + from mempalace.mcp_server import tool_add_drawer + + boundary = "Z" * config.chunk_size + result = tool_add_drawer(wing="w", room="r", content=boundary) + assert result["success"] is True + assert result["chunks"] == 1 + assert "chunk_ids" not in result + + def test_add_drawer_chunked_logical_id_not_fetchable_directly( + self, monkeypatch, config, palace_path, kg + ): + """Documented contract on the chunked path: ``tool_get_drawer`` + and ``tool_delete_drawer`` against the returned logical + ``drawer_id`` report ``not found`` because no row is stored + under that id. Callers must iterate ``chunk_ids`` or query by + ``parent_drawer_id`` metadata.""" + _patch_mcp_server(monkeypatch, config, kg) + _client, _col = _get_collection(palace_path, create=True) + del _client + from mempalace.mcp_server import tool_add_drawer, tool_delete_drawer, tool_get_drawer + + result = tool_add_drawer(wing="w", room="r", content="P" * 4000) + assert result["success"] is True and result["chunks"] > 1 + + # tool_get_drawer against logical id: not found. + got_logical = tool_get_drawer(result["drawer_id"]) + assert "error" in got_logical and "not found" in got_logical["error"].lower() + + # tool_get_drawer against the first chunk id: found, full content slice. + got_chunk = tool_get_drawer(result["chunk_ids"][0]) + assert got_chunk["content"] == "P" * config.chunk_size + assert got_chunk["metadata"]["parent_drawer_id"] == result["drawer_id"] + + # tool_delete_drawer against logical id: also not found. + deleted_logical = tool_delete_drawer(result["drawer_id"]) + assert deleted_logical["success"] is False + assert "not found" in deleted_logical["error"].lower() + # ── KG Tools ──────────────────────────────────────────────────────────── @@ -1938,6 +2090,81 @@ def test_diary_read_case_insensitive_agent(self, monkeypatch, config, palace_pat assert w1["agent"] == "claude" assert w2["agent"] == "claude" + # ── #1539: oversized-entry chunking ──────────────────────────── + + def test_diary_write_normal_entry_single_drawer(self, monkeypatch, config, palace_path, kg): + """Regression catch: a normal entry (< CHUNK_SIZE) must produce + exactly one drawer with ``chunks == 1`` in the result. Existing + pre-#1539 behaviour preserved for the common path.""" + _patch_mcp_server(monkeypatch, config, kg) + _client, _col = _get_collection(palace_path, create=True) + del _client + from mempalace.mcp_server import tool_diary_write + + r = tool_diary_write( + agent_name="TestAgent", + entry="A normal-length entry that fits comfortably under chunk_size.", + topic="general", + ) + assert r["success"] is True + assert r["chunks"] == 1 + _client2, col = _get_collection(palace_path) + del _client2 + assert col.count() == 1 + + def test_diary_write_oversized_entry_chunked(self, monkeypatch, config, palace_path, kg): + """Regression for #1539: an entry far above CHUNK_SIZE must be + sliced into bounded per-chunk drawers, each linked by a + ``parent_entry_id`` metadata field. No single document stored + may exceed CHUNK_SIZE.""" + _patch_mcp_server(monkeypatch, config, kg) + _client, _col = _get_collection(palace_path, create=True) + del _client + from mempalace.mcp_server import tool_diary_write + + # 5000 chars: well above CHUNK_SIZE=800. Expected chunks: ceil(5000/800) = 7. + oversized = "Z" * 5000 + r = tool_diary_write(agent_name="TestAgent", entry=oversized, topic="general") + + assert r["success"] is True + assert r["chunks"] > 1, f"oversized entry must produce >1 chunks; got {r['chunks']}" + assert "chunk_ids" in r and len(r["chunk_ids"]) == r["chunks"] + + _client2, col = _get_collection(palace_path) + del _client2 + stored = col.get() + assert all(len(d) <= 800 for d in stored["documents"]), ( + f"no stored document may exceed CHUNK_SIZE=800; " + f"got max={max(len(d) for d in stored['documents'])}" + ) + joined = "".join(stored["documents"]) + assert joined == oversized, "joined chunks must equal original entry verbatim" + + parent_ids = {m.get("parent_entry_id") for m in stored["metadatas"]} + assert len(parent_ids) == 1 and None not in parent_ids, ( + f"all chunks must share one parent_entry_id; got {parent_ids}" + ) + + def test_diary_write_chunk_index_metadata(self, monkeypatch, config, palace_path, kg): + """Regression for #1539: each oversized-entry chunk must carry a + ``chunk_index`` metadata field that runs 0, 1, 2, ... in order.""" + _patch_mcp_server(monkeypatch, config, kg) + _client, _col = _get_collection(palace_path, create=True) + del _client + from mempalace.mcp_server import tool_diary_write + + oversized = "Q" * 3500 # ~5 chunks at CHUNK_SIZE=800 + r = tool_diary_write(agent_name="TestAgent", entry=oversized, topic="general") + assert r["success"] is True and r["chunks"] > 1 + + _client2, col = _get_collection(palace_path) + del _client2 + stored = col.get() + indices = sorted(m["chunk_index"] for m in stored["metadatas"]) + assert indices == list(range(len(indices))), ( + f"chunk_index must be 0..N-1 contiguous; got {indices}" + ) + # ── Cache Invalidation (inode/mtime) ────────────────────────────────── From e8cdecdec74c7e6ac7d3a4123070f04cfe47f4d8 Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 21 May 2026 17:05:02 -0700 Subject: [PATCH 38/39] style: collapse implicit string concat in palace_graph warning (ruff 0.15.9) --- mempalace/palace_graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mempalace/palace_graph.py b/mempalace/palace_graph.py index ee17f69606..8f2d030338 100644 --- a/mempalace/palace_graph.py +++ b/mempalace/palace_graph.py @@ -230,7 +230,7 @@ def build_graph(col=None, config=None): pg_result = _build_graph_postgres(col, config) except Exception as e: logger.warning( - "palace_graph: postgres aggregate path failed (%s); " "falling back to row walk.", + "palace_graph: postgres aggregate path failed (%s); falling back to row walk.", e, ) pg_result = None From 827de69f779fc183e8feedb4d73f92ac27a61152 Mon Sep 17 00:00:00 2001 From: jp Date: Thu, 21 May 2026 17:06:18 -0700 Subject: [PATCH 39/39] fix(merge): include warnings in single-chunk tool_add_drawer return MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Post-merge reconcile — the upstream chunked-write path's single-chunk return dict lost the warnings field that the fork's room-taxonomy tests expect. Conflict resolution preserved warnings on the already-exists and chunked-write paths but missed the single-chunk path that lives between them. tests/test_room_taxonomy.py — 19 passing. Co-Authored-By: Claude Opus 4.7 (1M context) --- mempalace/mcp_server.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mempalace/mcp_server.py b/mempalace/mcp_server.py index b5804f70a4..634a365097 100644 --- a/mempalace/mcp_server.py +++ b/mempalace/mcp_server.py @@ -1453,11 +1453,14 @@ def tool_add_drawer( ) _metadata_cache = None logger.info(f"Filed drawer: {drawer_id} → {wing}/{room}") + for _w in warnings: + logger.warning("room taxonomy: %s (drawer=%s)", _w, drawer_id) return { "success": True, "drawer_id": drawer_id, "wing": wing, "room": room, + "warnings": warnings, "chunks": 1, }