fix: use upsert and deterministic IDs to prevent data stagnation

mempalace/mcp_server.py

@@ -44,7 +44,8 @@ def _parse_args():
         metavar="PATH",
         help="Path to the palace directory (overrides config file and env var)",
     )
-    return parser.parse_args()
+    args, _ = parser.parse_known_args()
+    return args
 
 
 _args = _parse_args()
@@ -283,19 +284,18 @@ def tool_add_drawer(
     if not col:
         return _no_palace()
 
-    # Duplicate check
-    dup = tool_check_duplicate(content, threshold=0.9)
-    if dup.get("is_duplicate"):
-        return {
-            "success": False,
-            "reason": "duplicate",
-            "matches": dup["matches"],
-        }
+    drawer_id = f"drawer_{wing}_{room}_{hashlib.md5(content.encode()).hexdigest()[:16]}"
 
-    drawer_id = f"drawer_{wing}_{room}_{hashlib.md5((content[:100] + datetime.now().isoformat()).encode()).hexdigest()[:16]}"
+    # Idempotency: if the deterministic ID already exists, return success as a no-op.
+    try:
+        existing = col.get(ids=[drawer_id])
+        if existing and existing["ids"]:
+            return {"success": True, "reason": "already_exists", "drawer_id": drawer_id}
+    except Exception:
+        pass
 
     try:
-        col.add(
+        col.upsert(
             ids=[drawer_id],
             documents=[content],
             metadatas=[

mempalace/miner.py

@@ -403,10 +403,22 @@ def get_collection(palace_path: str):
 
 
 def file_already_mined(collection, source_file: str) -> bool:
-    """Fast check: has this file been filed before?"""
+    """Fast check: has this file been filed before and is unchanged?
+
+    Compares the stored mtime in drawer metadata against the file's current
+    mtime.  Returns False (needs re-mining) when the file has been modified
+    since it was last mined, or when no mtime was stored.
+    """
     try:
         results = collection.get(where={"source_file": source_file}, limit=1)
-        return len(results.get("ids", [])) > 0
+        if not results.get("ids"):
+            return False
+        stored_meta = results["metadatas"][0] if results.get("metadatas") else {}
+        stored_mtime = stored_meta.get("source_mtime")
+        if stored_mtime is None:
+            return False
+        current_mtime = os.path.getmtime(source_file)
+        return float(stored_mtime) == current_mtime
     except Exception:
         return False
 
@@ -417,24 +429,26 @@ def add_drawer(
     """Add one drawer to the palace."""
     drawer_id = f"drawer_{wing}_{room}_{hashlib.md5((source_file + str(chunk_index)).encode(), usedforsecurity=False).hexdigest()[:16]}"
     try:
-        collection.add(
+        metadata = {
+            "wing": wing,
+            "room": room,
+            "source_file": source_file,
+            "chunk_index": chunk_index,
+            "added_by": agent,
+            "filed_at": datetime.now().isoformat(),
+        }
+        # Store file mtime so we can detect modifications later.
+        try:
+            metadata["source_mtime"] = os.path.getmtime(source_file)
+        except OSError:
+            pass
+        collection.upsert(
             documents=[content],
             ids=[drawer_id],
-            metadatas=[
-                {
-                    "wing": wing,
-                    "room": room,
-                    "source_file": source_file,
-                    "chunk_index": chunk_index,
-                    "added_by": agent,
-                    "filed_at": datetime.now().isoformat(),
-                }
-            ],
+            metadatas=[metadata],
         )
         return True
-    except Exception as e:
-        if "already exists" in str(e).lower() or "duplicate" in str(e).lower():
-            return False
+    except Exception:
         raise
 
 

tests/conftest.py

@@ -102,7 +102,9 @@ def collection(palace_path):
     """A ChromaDB collection pre-seeded in the temp palace."""
     client = chromadb.PersistentClient(path=palace_path)
     col = client.get_or_create_collection("mempalace_drawers")
-    return col
+    yield col
+    client.delete_collection("mempalace_drawers")
+    del client
 
 
 @pytest.fixture

tests/test_hooks_cli.py

@@ -42,29 +42,43 @@ def _write_transcript(path: Path, entries: list[dict]):
 
 def test_count_human_messages_basic(tmp_path):
     transcript = tmp_path / "t.jsonl"
-    _write_transcript(transcript, [
-        {"message": {"role": "user", "content": "hello"}},
-        {"message": {"role": "assistant", "content": "hi"}},
-        {"message": {"role": "user", "content": "bye"}},
-    ])
+    _write_transcript(
+        transcript,
+        [
+            {"message": {"role": "user", "content": "hello"}},
+            {"message": {"role": "assistant", "content": "hi"}},
+            {"message": {"role": "user", "content": "bye"}},
+        ],
+    )
     assert _count_human_messages(str(transcript)) == 2
 
 
 def test_count_skips_command_messages(tmp_path):
     transcript = tmp_path / "t.jsonl"
-    _write_transcript(transcript, [
-        {"message": {"role": "user", "content": "<command-message>status</command-message>"}},
-        {"message": {"role": "user", "content": "real question"}},
-    ])
+    _write_transcript(
+        transcript,
+        [
+            {"message": {"role": "user", "content": "<command-message>status</command-message>"}},
+            {"message": {"role": "user", "content": "real question"}},
+        ],
+    )
     assert _count_human_messages(str(transcript)) == 1
 
 
 def test_count_handles_list_content(tmp_path):
     transcript = tmp_path / "t.jsonl"
-    _write_transcript(transcript, [
-        {"message": {"role": "user", "content": [{"type": "text", "text": "hello"}]}},
-        {"message": {"role": "user", "content": [{"type": "text", "text": "<command-message>x</command-message>"}]}},
-    ])
+    _write_transcript(
+        transcript,
+        [
+            {"message": {"role": "user", "content": [{"type": "text", "text": "hello"}]}},
+            {
+                "message": {
+                    "role": "user",
+                    "content": [{"type": "text", "text": "<command-message>x</command-message>"}],
+                }
+            },
+        ],
+    )
     assert _count_human_messages(str(transcript)) == 1
 
 
@@ -90,6 +104,7 @@ def test_count_malformed_json_lines(tmp_path):
 def _capture_hook_output(hook_fn, data, harness="claude-code", state_dir=None):
     """Run a hook and capture its JSON stdout output."""
     import io
+
     buf = io.StringIO()
     patches = [patch("mempalace.hooks_cli._output", side_effect=lambda d: buf.write(json.dumps(d)))]
     if state_dir:
@@ -123,10 +138,10 @@ def test_stop_hook_passthrough_when_active_string(tmp_path):
 
 def test_stop_hook_passthrough_below_interval(tmp_path):
     transcript = tmp_path / "t.jsonl"
-    _write_transcript(transcript, [
-        {"message": {"role": "user", "content": f"msg {i}"}}
-        for i in range(SAVE_INTERVAL - 1)
-    ])
+    _write_transcript(
+        transcript,
+        [{"message": {"role": "user", "content": f"msg {i}"}} for i in range(SAVE_INTERVAL - 1)],
+    )
     result = _capture_hook_output(
         hook_stop,
         {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)},
@@ -137,10 +152,10 @@ def test_stop_hook_passthrough_below_interval(tmp_path):
 
 def test_stop_hook_blocks_at_interval(tmp_path):
     transcript = tmp_path / "t.jsonl"
-    _write_transcript(transcript, [
-        {"message": {"role": "user", "content": f"msg {i}"}}
-        for i in range(SAVE_INTERVAL)
-    ])
+    _write_transcript(
+        transcript,
+        [{"message": {"role": "user", "content": f"msg {i}"}} for i in range(SAVE_INTERVAL)],
+    )
     result = _capture_hook_output(
         hook_stop,
         {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)},
@@ -152,10 +167,10 @@ def test_stop_hook_blocks_at_interval(tmp_path):
 
 def test_stop_hook_tracks_save_point(tmp_path):
     transcript = tmp_path / "t.jsonl"
-    _write_transcript(transcript, [
-        {"message": {"role": "user", "content": f"msg {i}"}}
-        for i in range(SAVE_INTERVAL)
-    ])
+    _write_transcript(
+        transcript,
+        [{"message": {"role": "user", "content": f"msg {i}"}} for i in range(SAVE_INTERVAL)],
+    )
     data = {"session_id": "test", "stop_hook_active": False, "transcript_path": str(transcript)}
 
     # First call blocks