feat: add authoritative incremental decision memory

docs/changed-set-ingest.md

@@ -0,0 +1,29 @@
+# Changed-set ingest PoC
+
+`mempalace sync` can accept a producer-supplied manifest and reindex only the
+listed project-relative files. Git is not required by core; an IDE, watcher, or
+build system may create the same JSON shape.
+
+```json
+{
+  "changed": ["src/app.py", "README.md"],
+  "deleted": ["src/old.py"]
+}
+```
+
+Preview and apply:
+
+```bash
+mempalace sync /path/to/project --manifest changed.json --wing project
+mempalace sync /path/to/project --manifest changed.json --wing project --apply --daemon
+```
+
+Paths must remain within the project root. Apply holds the palace writer lock,
+purges old drawers and closets for every affected source, and invokes the normal
+project miner only for `changed`. `deleted` sources are never opened. The daemon
+payload contains the parsed manifest, avoiding a manifest-file time-of-check /
+time-of-use race between client and writer.
+
+This is intentionally a PoC contract. A production version should add a job
+idempotency key and committed palace generation before making changed-set sync a
+default hook path.

docs/decision-memory.md

@@ -0,0 +1,28 @@
+# Authority-aware decision memory PoC
+
+Decision drawers may carry an optional structured envelope while their content
+remains verbatim:
+
+- `decision_key`: stable logical identity across versions;
+- `authority_uri`: canonical local file path or `file://` URI;
+- `authority_version`: `sha256:<hex>` or `mtime_ns:<integer>`;
+- `memory_kind`: for example `decision`, `finding`, or `preference`;
+- `authority_status`: `current`, `stale`, `unverified`, or `superseded`.
+
+`mempalace_search(verify_authority=true)` compares supported local authority
+tokens and includes an authority envelope on every result. Verification is
+opt-in because hashing large files has a real read-path cost. Unsupported and
+legacy authorities remain `unverified`; they are never assumed current.
+
+`mempalace_supersede_drawer` requires both the predecessor ID and the exact same
+non-empty `decision_key`. It files a new verbatim drawer, then marks the old
+drawer `superseded` with `superseded_by=<new id>`. Default MCP search hides that
+history; `include_superseded=true` exposes it. No semantic-similarity threshold
+can supersede a decision implicitly.
+
+Checkpoint items accept the same fields plus `supersedes_id`, so an agent can
+save a reviewed decision transition in one call.
+
+This PoC resolves only local files. Production authority adapters could support
+Git blobs, GitHub issues, planners, or document systems without changing the
+drawer lifecycle contract.

mempalace/changed_set.py

@@ -0,0 +1,139 @@
+"""Changed-set project ingest without a full filesystem walk.
+
+The producer may be Git, an IDE, or a file watcher. Core accepts only project-
+relative paths and keeps all mutation semantics inside MemPalace.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TypedDict
+
+from .miner import (
+    is_gitignored,
+    load_config,
+    load_gitignore_matcher,
+    process_file,
+)
+from .palace import (
+    get_closets_collection,
+    get_collection,
+    mine_palace_lock,
+)
+
+
+class ChangedSetReport(TypedDict):
+    changed: int
+    deleted: int
+    ignored: int
+    reindexed: int
+    drawers_added: int
+    dry_run: bool
+
+
+def _resolve_source(root: Path, value: str) -> Path:
+    if not isinstance(value, str) or not value.strip():
+        raise ValueError("changed-set paths must be non-empty strings")
+    candidate = (root / value).resolve(strict=False)
+    try:
+        candidate.relative_to(root)
+    except ValueError as exc:
+        raise ValueError(f"changed-set path escapes project root: {value}") from exc
+    return candidate
+
+
+def normalize_changed_set(
+    project_root: str | Path, changed: list[str], deleted: list[str]
+) -> tuple[list[Path], list[Path]]:
+    """Validate, resolve, sort, and deduplicate an external changed manifest."""
+    if not isinstance(changed, list) or not all(isinstance(value, str) for value in changed):
+        raise ValueError("changed must be an array of strings")
+    if not isinstance(deleted, list) or not all(isinstance(value, str) for value in deleted):
+        raise ValueError("deleted must be an array of strings")
+    root = Path(project_root).expanduser().resolve()
+    if not root.is_dir():
+        raise ValueError(f"project root does not exist: {root}")
+    changed_paths = sorted({_resolve_source(root, value) for value in changed}, key=str)
+    deleted_paths = sorted({_resolve_source(root, value) for value in deleted}, key=str)
+    overlap = set(changed_paths) & set(deleted_paths)
+    if overlap:
+        raise ValueError(f"paths cannot be both changed and deleted: {sorted(map(str, overlap))}")
+    missing_changed = [str(path) for path in changed_paths if not path.is_file()]
+    if missing_changed:
+        raise ValueError(f"changed paths must exist as files: {missing_changed}")
+    return changed_paths, deleted_paths
+
+
+def _is_gitignored_source(root: Path, path: Path) -> bool:
+    """Apply root and nested gitignore rules to one explicit changed path."""
+    matchers = []
+    cache: dict[Path, object] = {}
+    current = root
+    directories = [root]
+    for part in path.relative_to(root).parts[:-1]:
+        current /= part
+        directories.append(current)
+    for directory in directories:
+        matcher = load_gitignore_matcher(directory, cache)
+        if matcher is not None:
+            matchers.append(matcher)
+    return bool(matchers and is_gitignored(path, matchers, is_dir=False))
+
+
+def sync_changed_sources(
+    *,
+    palace_path: str,
+    project_root: str | Path,
+    changed: list[str],
+    deleted: list[str],
+    wing: str | None = None,
+    agent: str = "mempalace",
+    dry_run: bool = True,
+) -> ChangedSetReport:
+    """Serialize replacement of changed sources and removal of deleted sources."""
+    root = Path(project_root).expanduser().resolve()
+    changed_paths, deleted_paths = normalize_changed_set(root, changed, deleted)
+    ignored_paths = [path for path in changed_paths if _is_gitignored_source(root, path)]
+    ignored_set = set(ignored_paths)
+    indexable_paths = [path for path in changed_paths if path not in ignored_set]
+    report: ChangedSetReport = {
+        "changed": len(changed_paths),
+        "deleted": len(deleted_paths),
+        "ignored": len(ignored_paths),
+        "reindexed": 0,
+        "drawers_added": 0,
+        "dry_run": dry_run,
+    }
+    if dry_run:
+        return report
+
+    project_config = load_config(str(root))
+    resolved_wing = wing or project_config["wing"]
+    rooms = project_config.get("rooms", [{"name": "general", "description": "All files"}])
+    affected = [*ignored_paths, *deleted_paths]
+    with mine_palace_lock(palace_path):
+        drawers = get_collection(palace_path, create=False)
+        closets = get_closets_collection(palace_path, create=True)
+        for path in affected:
+            source = str(path)
+            drawers.delete(where={"$and": [{"source_file": source}, {"wing": resolved_wing}]})
+            closets.delete(where={"$and": [{"source_file": source}, {"wing": resolved_wing}]})
+        for path in indexable_paths:
+            added, _room, skip_reason = process_file(
+                path,
+                root,
+                drawers,
+                resolved_wing,
+                rooms,
+                agent,
+                False,
+                closets_col=closets,
+                force_reindex=True,
+            )
+            if skip_reason is None and added > 0:
+                report["reindexed"] += 1
+            report["drawers_added"] += added
+    return report
+
+
+__all__ = ["ChangedSetReport", "normalize_changed_set", "sync_changed_sources"]

mempalace/cli.py

@@ -33,6 +33,7 @@
 
 import argparse
 import contextlib
+import json
 import os
 import shlex
 import sys
@@ -1058,6 +1059,27 @@ def cmd_sync(args):
     """Prune drawers whose source files are gitignored, deleted, or moved (#1252)."""
     palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path
 
+    changed_set = None
+    manifest_path = getattr(args, "manifest", None)
+    if manifest_path:
+        if not args.dir:
+            print("mempalace: sync --manifest requires a project root argument", file=sys.stderr)
+            sys.exit(2)
+        try:
+            with open(os.path.expanduser(manifest_path), encoding="utf-8") as handle:
+                changed_set = json.load(handle)
+            if not isinstance(changed_set, dict):
+                raise ValueError("manifest must be a JSON object")
+            changed_set = {
+                "changed": changed_set.get("changed") or [],
+                "deleted": changed_set.get("deleted") or [],
+            }
+            if not all(isinstance(value, list) for value in changed_set.values()):
+                raise ValueError("manifest changed/deleted values must be arrays")
+        except (OSError, ValueError, json.JSONDecodeError) as exc:
+            print(f"mempalace: invalid changed-set manifest: {exc}", file=sys.stderr)
+            sys.exit(2)
+
     if getattr(args, "background", False) and not getattr(args, "daemon", False):
         print("mempalace: --background requires --daemon", file=sys.stderr)
         sys.exit(2)
@@ -1069,6 +1091,9 @@ def cmd_sync(args):
             "wing": args.wing,
             "dry_run": args.dry_run,
         }
+        if changed_set is not None:
+            payload["changed_set"] = changed_set
+            payload["agent"] = getattr(args, "agent", "mempalace")
         _submit_daemon_cli_job("sync", payload, args, background=getattr(args, "background", False))
         return
 
@@ -1077,6 +1102,7 @@ def cmd_sync(args):
     from .backends import detect_backend_for_path
     from .palace import _backend_artifact_label, resolve_backend_name
     from .sync import sync_palace
+    from .changed_set import sync_changed_sources
 
     if not os.path.isdir(palace_path):
         print(f"\n  No palace found at {palace_path}")
@@ -1116,6 +1142,21 @@ def cmd_sync(args):
     print(f"{'-' * 55}\n")
 
     try:
+        if changed_set is not None:
+            report = sync_changed_sources(
+                palace_path=palace_path,
+                project_root=project_dirs[0],
+                changed=changed_set["changed"],
+                deleted=changed_set["deleted"],
+                wing=args.wing,
+                agent=getattr(args, "agent", "mempalace"),
+                dry_run=args.dry_run,
+            )
+            print(
+                f"  Changed-set: changed={report['changed']} deleted={report['deleted']} "
+                f"reindexed={report['reindexed']} drawers_added={report['drawers_added']}"
+            )
+            return
         report = sync_palace(
             palace_path=palace_path,
             project_dirs=project_dirs,
@@ -2646,6 +2687,11 @@ def main():
         help="Project root to sync (optional; auto-detects from drawer metadata)",
     )
     p_sync.add_argument("--wing", default=None, help="Limit to one wing")
+    p_sync.add_argument(
+        "--manifest",
+        help="JSON changed-set with project-relative changed/deleted arrays; requires dir",
+    )
+    p_sync.add_argument("--agent", default="mempalace", help="Agent recorded on reindexed drawers")
     p_sync.add_argument(
         "--root",
         action="append",