feat: add JSONL export/import for cross-device palace sync

CHANGELOG.md

@@ -44,6 +44,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 
 - **Shared-brain rules are `host:harness:project`, declared-idle, and MCP-shape aware.** `mempalace rules` takes `--host --harness --project` (stable lowercase tokens) and optional `--mcp full|light` (default `full`, matching the 45-tool server). The packaged snippet is the only coordination text: compose the identity from the current workspace, arm `logstream watch` only on listen / claim / delegate, write topics on named lanes without filtering the default inbox on them, claim with a lowest-HLC mutex, and use `kg_supersede` for single-valued fact changes. `--mcp light` swaps tool tokens onto the 3-tool triad; prose is identical. `logstream watch --agent` now defaults a sanitized `--state-file` (`:` → `_` under `~/.mempalace/watch/`) so Windows tuple identities do not need a private path overlay.
 
+- **Cross-device sync: `mempalace export` / `mempalace import`.** `export` (default `--format jsonl`)
+  writes a deterministic, git-friendly JSONL tree organized by wing/room — sorted ids, sorted keys, no
+  timestamps, so re-exporting an unchanged palace is a zero git diff — and `import <dir>` merges an
+  export into another machine's palace by drawer id: adds new drawers, skips existing ones, idempotent
+  on re-import, and re-embeds locally since exports deliberately carry no vectors. `--format markdown`
+  exposes the existing browsable markdown exporter on the CLI for the first time. `import` follows
+  the CLI write-routing policy (`--daemon` / `--direct`, #2033) like `mine` and `sync`. (#452)
+
 ### Bug Fixes
 
 - **The transcript-path fallback no longer gives every git worktree its own wing.** `_wing_from_transcript_path`'s primary path (reading `cwd` from the JSONL) already collapsed a `<project>/.claude/worktrees/<wt>` segment before deriving the wing; the fallback path, used whenever `cwd` is absent, had no equivalent strip, so the flattened `--claude-worktrees-<wt>` segment survived into the wing name. Applied the same collapse there. (#2388)

docs/cli-write-routing.md

@@ -10,6 +10,7 @@ The policy applies to:
 - `mempalace mine`;
 - `mempalace sweep`;
 - `mempalace sync`;
+- `mempalace import`;
 - the optional post-setup mine run by `mempalace init`.
 
 ## Policies

mempalace/cli/__init__.py

@@ -18,6 +18,8 @@
     mempalace mine <dir> --mode extract   Mine binary office documents (PDF/DOCX/etc.)
     mempalace mine <source> --source NAME Mine through a registered source adapter
     mempalace search "query"              Find anything, exact words
+    mempalace export --output <dir>       Export palace to JSONL for git-based sync
+    mempalace import <dir>                Merge a JSONL export into the palace
     mempalace mcp                         Show MCP setup command
     mempalace task create ...             Create a complete agent handoff
     mempalace task launch ...             Run a stored task headlessly
@@ -86,6 +88,7 @@
     "_hub.py",
     "cmd_mine.py",
     "cmd_sync.py",
+    "cmd_export.py",
     "cmd_query.py",
     "cmd_update.py",
     "cmd_coord.py",

mempalace/cli/cmd_export.py

@@ -0,0 +1,74 @@
+# Loaded into mempalace.cli via exec (see __init__.py). Not a standalone module.
+if __name__ != "mempalace.cli":
+    raise ImportError(f"{__name__} is an implementation fragment; import mempalace.cli")
+
+
+def cmd_export(args):
+    """Export the palace to a portable directory tree (#452)."""
+    palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path
+
+    from ..backends import detect_backend_for_path
+
+    if not os.path.isdir(palace_path) or detect_backend_for_path(palace_path) is None:
+        print(f"\n  No palace found at {palace_path}", file=sys.stderr)
+        sys.exit(1)
+
+    # The default follows the config directory rather than a fixed ~/.mempalace, so
+    # an XDG install keeps its export beside its palace instead of in a second root.
+    if args.output:
+        output_dir = os.path.expanduser(args.output)
+    else:
+        output_dir = os.path.join(MempalaceConfig().config_dir, "export")
+    print(f"\n{'=' * 55}")
+    print(f"  Exporting palace ({args.format})")
+    print(f"{'=' * 55}\n")
+    from ..exporter import export_palace, export_palace_jsonl
+
+    export = export_palace_jsonl if args.format == "jsonl" else export_palace
+    try:
+        export(palace_path, output_dir)
+    except (ValueError, OSError) as exc:
+        # The exporter refuses symlinked targets with ValueError, and an unwritable
+        # output directory surfaces as OSError; report either, as `import` does.
+        print(f"  ERROR: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+
+def cmd_import(args):
+    """Merge a JSONL export into the palace (#452)."""
+    palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path
+    input_dir = os.path.expanduser(args.dir)
+
+    # Routed like `mine`, dry runs included: the dry_run flag travels in the
+    # payload, and a dry run never opens the palace wherever it executes.
+    routing = _resolve_cli_write_routing_or_exit(args, "import")
+
+    print(f"\n{'=' * 55}")
+    print("  Importing palace export" + (" (dry run)" if args.dry_run else ""))
+    print(f"{'=' * 55}\n")
+
+    if routing.use_daemon:
+        _submit_daemon_cli_job(
+            "import",
+            # Absolute: the daemon resolves paths against its own cwd (#2467).
+            {"input_dir": os.path.abspath(input_dir), "dry_run": args.dry_run},
+            args,
+            background=bool(getattr(args, "background", False)),
+            auto_start=routing.decision.auto_start_daemon,
+        )
+        return
+
+    from ..importer import import_palace
+    from ..palace import MineAlreadyRunning
+
+    try:
+        import_palace(palace_path, input_dir, dry_run=args.dry_run)
+    except MineAlreadyRunning as exc:
+        # The writer lease is non-blocking: a mine or MCP server already writing
+        # this palace refuses the import. Name the holder and exit non-zero, as
+        # `mine` and `sync` do, rather than surfacing a traceback.
+        print(f"mempalace: {exc}", file=sys.stderr)
+        sys.exit(1)
+    except ValueError as exc:
+        print(f"  ERROR: {exc}", file=sys.stderr)
+        sys.exit(1)

mempalace/cli/parser.py

@@ -293,8 +293,38 @@ def main():
         help="Actually delete drawers (overrides --dry-run; requires --wing or a project root)",
     )
 
-    # search
+    # export
     add_cli_write_routing_flags(p_sync)
+    p_export = sub.add_parser(
+        "export",
+        help="Export the palace to a portable directory tree (JSONL for sync, markdown for browsing)",
+    )
+    p_export.add_argument(
+        "--output",
+        default=None,
+        help="Directory to write the export tree into (default: export/ in the config directory, resolved via XDG — see mempalace.config)",
+    )
+    p_export.add_argument(
+        "--format",
+        choices=["jsonl", "markdown"],
+        default="jsonl",
+        help="jsonl (git-friendly, importable; default) or markdown (browsable, one-way)",
+    )
+
+    # import
+    p_import = sub.add_parser(
+        "import",
+        help="Merge a JSONL export into the palace (adds new drawers, skips existing by id)",
+    )
+    p_import.add_argument("dir", help="Directory containing a JSONL palace export")
+    p_import.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Report what would be imported without writing anything",
+    )
+
+    # search
+    add_cli_write_routing_flags(p_import)
     p_search = sub.add_parser("search", help="Find anything, exact words")
     p_search.add_argument("query", help="What to search for")
     p_search.add_argument(
@@ -971,6 +1001,8 @@ def _add_logstream_filters(p):
         "search": cmd_search,
         "sweep": cmd_sweep,
         "sync": cmd_sync,
+        "export": cmd_export,
+        "import": cmd_import,
         "mcp": cmd_mcp,
         "serve": cmd_serve,
         "compress": cmd_compress,

mempalace/exporter.py

@@ -7,11 +7,14 @@
     wing_name/
       room_name.md        — one file per room, drawers as sections
 
-Streams drawers in paginated batches so memory usage stays bounded
-regardless of palace size.
+The markdown export streams drawers in paginated batches so memory usage
+stays bounded regardless of palace size; the JSONL export buffers the
+grouped drawers so files can be written fully sorted, so its memory use is
+proportional to the total exported text.
 """
 
 import errno
+import json
 import os
 import re
 from collections import defaultdict
@@ -20,6 +23,21 @@
 from .palace import get_collection
 
 
+def _open_for_export(palace_path: str):
+    """Open the palace for export — read-only whenever a palace exists.
+
+    An existing palace is opened read-only and without create — the two travel
+    together, since sqlite_exact refuses a read-only open that may create. A
+    path holding no palace keeps its long-standing behaviour: the default open
+    files an empty palace and the export comes back empty.
+    """
+    from .backends import detect_backend_for_path
+
+    if os.path.isdir(palace_path) and detect_backend_for_path(palace_path) is not None:
+        return get_collection(palace_path, create=False, read_only=True)
+    return get_collection(palace_path)
+
+
 def _safe_path_component(name: str) -> str:
     """Sanitize a string for use as a directory/file name component."""
     name = re.sub(r'[/\\:*?"<>|]', "_", name)
@@ -80,7 +98,7 @@ def export_palace(palace_path: str, output_dir: str, format: str = "markdown") -
     Returns:
         Stats dict: {"wings": N, "rooms": N, "drawers": N}
     """
-    col = get_collection(palace_path)
+    col = _open_for_export(palace_path)
     total = col.count()
 
     if total == 0:
@@ -212,3 +230,181 @@ def _quote_content(text: str) -> str:
     """Format content for a markdown blockquote, handling multiline."""
     lines = text.rstrip("\n").split("\n")
     return "\n> ".join(lines)
+
+
+def _prune_stale_exports(output_dir: str, written: set) -> int:
+    """Remove ``*.jsonl`` files a PREVIOUS export left behind. Returns the count.
+
+    Re-exporting rewrites the rooms that still exist and used to leave the rest
+    in place. That is not only a cosmetic git-diff wrinkle: import walks every
+    ``*.jsonl`` under the tree without consulting the manifest, so a room whose
+    last drawer was deleted would be re-imported on the next device and the
+    drawer would come back.
+
+    Two safety properties, both deliberate:
+
+    * The caller passes ``had_manifest`` from BEFORE the manifest is rewritten,
+      so this only ever deletes inside a directory we can prove was already one
+      of our own exports. Pointing ``export`` at an arbitrary directory removes
+      nothing on the first run.
+    * Only regular files are unlinked, and symlinks are skipped rather than
+      followed — the same posture ``_reject_symlink`` applies on the write side.
+    """
+    removed = 0
+    for root, _dirs, files in os.walk(output_dir):
+        for name in files:
+            if not name.endswith(".jsonl"):
+                continue
+            path = os.path.join(root, name)
+            if os.path.abspath(path) in written:
+                continue
+            if os.path.islink(path) or not os.path.isfile(path):
+                continue
+            try:
+                os.unlink(path)
+                removed += 1
+            except OSError:
+                pass
+    # Drop wing directories the prune emptied; never the output root itself.
+    for root, _dirs, _files in os.walk(output_dir, topdown=False):
+        if os.path.abspath(root) == os.path.abspath(output_dir):
+            continue
+        try:
+            if not os.listdir(root):
+                os.rmdir(root)
+        except OSError:
+            pass
+    return removed
+
+
+def export_palace_jsonl(palace_path: str, output_dir: str) -> dict:
+    """Export all palace drawers as JSONL files organized by wing/room.
+
+    Produces a git-friendly tree suitable for cross-device sync (#452)::
+
+        output_dir/
+          export-manifest.json      — format version + counts
+          wing_name/
+            room_name.jsonl         — one drawer per line
+
+    Each line is a JSON object with the drawer's ``id``, ``document``, and
+    ``metadata`` — the exact triple needed to re-file it on another machine.
+    Embeddings are deliberately not included: they are large, binary, and tied
+    to the embedding model; import re-embeds instead.
+
+    Output is deterministic (keys sorted, drawers sorted by id within each
+    room, no timestamps), so re-exporting an unchanged palace produces a
+    byte-identical tree and therefore an empty git diff.
+
+    Re-exporting into an existing export also PRUNES room files that no longer
+    correspond to a room in the palace, so a deleted drawer does not survive in
+    a stale file and get re-imported on another device. Pruning is gated on a
+    previous ``export-manifest.json`` being present, so exporting into an
+    unrelated directory never deletes anything, and it is skipped when the
+    palace reports zero drawers — an empty count is also what a palace that
+    failed to open looks like, and that path warns instead.
+
+    Streams drawers in paginated batches like :func:`export_palace`, but
+    buffers the grouped drawers in memory so each file can be written fully
+    sorted; memory is proportional to the total exported text. Wing/room
+    names that sanitize to the same path component are merged into one file
+    (drawer ids stay unique, so nothing is lost).
+
+    Returns:
+        Stats dict: {"wings": N, "rooms": N, "drawers": N}
+    """
+    # A pure read: ask the backend not to run schema init, migrations or
+    # metadata writes (see _open_for_export for the create rule).
+    col = _open_for_export(palace_path)
+    total = col.count()
+
+    manifest_path = os.path.join(output_dir, "export-manifest.json")
+    had_manifest = os.path.isfile(manifest_path)
+
+    if total == 0:
+        print("  Palace is empty — nothing to export.")
+        if had_manifest:
+            # Deliberately NOT pruned. An empty count is also what a palace that
+            # failed to open looks like, and silently deleting a good export on
+            # that reading is far worse than leaving a stale one in place.
+            print(
+                f"  WARNING: {output_dir} still holds a previous export. It was left "
+                f"untouched because this palace reports zero drawers — delete it by "
+                f"hand if the palace really is empty, or it will re-import elsewhere."
+            )
+        return {"wings": 0, "rooms": 0, "drawers": 0}
+
+    _reject_symlink(output_dir, "output_dir")
+    os.makedirs(output_dir, exist_ok=True)
+    try:
+        os.chmod(output_dir, 0o700)
+    except (OSError, NotImplementedError):
+        pass
+
+    # {wing: {room: {id: line_dict}}} — buffered so each file writes sorted.
+    grouped: dict[str, dict[str, dict[str, dict]]] = defaultdict(lambda: defaultdict(dict))
+
+    print(f"  Streaming {total} drawers...")
+    offset = 0
+    while offset < total:
+        batch = col.get(limit=1000, offset=offset, include=["documents", "metadatas"])
+        if not batch["ids"]:
+            break
+        for doc_id, doc, meta in zip(batch["ids"], batch["documents"], batch["metadatas"]):
+            meta = meta or {}
+            wing = _safe_path_component(meta.get("wing", "unknown"))
+            room = _safe_path_component(meta.get("room", "general"))
+            grouped[wing][room][doc_id] = {
+                "id": doc_id,
+                "document": doc,
+                "metadata": meta,
+            }
+        offset += len(batch["ids"])
+
+    total_drawers = 0
+    room_count = 0
+    written: set = set()
+    for wing in sorted(grouped):
+        wing_dir = os.path.join(output_dir, wing)
+        _reject_symlink(wing_dir, f"wing directory {wing!r}")
+        os.makedirs(wing_dir, exist_ok=True)
+        try:
+            os.chmod(wing_dir, 0o700)
+        except (OSError, NotImplementedError):
+            pass
+
+        for room in sorted(grouped[wing]):
+            drawers = grouped[wing][room]
+            room_path = os.path.join(wing_dir, f"{room}.jsonl")
+            with _safe_open_for_write(room_path, "w") as f:
+                for doc_id in sorted(drawers):
+                    f.write(json.dumps(drawers[doc_id], ensure_ascii=False, sort_keys=True))
+                    f.write("\n")
+            written.add(os.path.abspath(room_path))
+            room_count += 1
+            total_drawers += len(drawers)
+        print(
+            f"  {wing}: {len(grouped[wing])} rooms, {sum(len(r) for r in grouped[wing].values())} drawers"
+        )
+
+    if had_manifest:
+        pruned = _prune_stale_exports(output_dir, written)
+        if pruned:
+            print(f"  Pruned {pruned} stale room file(s) from the previous export")
+
+    manifest = {
+        "format_version": 1,
+        "wings": len(grouped),
+        "rooms": room_count,
+        "drawers": total_drawers,
+    }
+    with _safe_open_for_write(manifest_path, "w") as f:
+        f.write(json.dumps(manifest, indent=2, sort_keys=True))
+        f.write("\n")
+
+    stats = {"wings": len(grouped), "rooms": room_count, "drawers": total_drawers}
+    print(
+        f"\n  Exported {stats['drawers']} drawers across {stats['wings']} wings, {stats['rooms']} rooms"
+    )
+    print(f"  Output: {output_dir}")
+    return stats