feat(graph): cross-wing tunnels by shared topics (#1180)

CHANGELOG.md

@@ -6,6 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 
 ---
 
+## [3.3.4] — unreleased
+
+### Added
+
+- **Cross-wing topic tunnels.** When two wings have confirmed `TOPIC` labels in common (the LLM-refine bucket from `mempalace init --llm`), the miner now drops a symmetric tunnel between them at mine time so the palace graph reflects shared themes (frameworks, vendors, recurring concepts). Tunnels are routed through the existing `create_tunnel` storage so they share dedup and persistence with explicit tunnels. Topic tunnels are stored under a synthetic `topic:<name>` room and tagged with `kind: "topic"` on the stored dict — this keeps them distinct from literal folder-derived rooms of the same name (a wing with both an `Angular` folder room and an `Angular` topic tunnel no longer collides at `follow_tunnels` read time) and gives LLMs scanning `list_tunnels` a visible discriminator. Threshold is configurable via `MEMPALACE_TOPIC_TUNNEL_MIN_COUNT` env var or `topic_tunnel_min_count` in `~/.mempalace/config.json` (default `1`). Manifest-dependency overlap and per-topic allow/deny lists remain out of scope. (#1180)
+
+---
+
 ## [3.3.3] — 2026-04-23
 
 ### Bug Fixes

mempalace/cli.py

@@ -117,21 +117,34 @@ def cmd_init(args):
     if languages_tuple != ("en",):
         print(f"  Languages: {', '.join(languages_tuple)}")
     detected = discover_entities(args.dir, languages=languages_tuple, llm_provider=llm_provider)
-    total = len(detected["people"]) + len(detected["projects"]) + len(detected["uncertain"])
+    total = (
+        len(detected["people"])
+        + len(detected["projects"])
+        + len(detected.get("topics", []))
+        + len(detected["uncertain"])
+    )
     if total > 0:
         confirmed = confirm_entities(detected, yes=getattr(args, "yes", False))
         # Save confirmed entities to <project>/entities.json (per-project
         # audit trail — user can inspect or hand-edit) AND merge into the
-        # global registry the miner reads at mine time.
-        if confirmed["people"] or confirmed["projects"]:
-            entities_path = Path(args.dir).expanduser().resolve() / "entities.json"
+        # global registry the miner reads at mine time. Topics are kept
+        # separately so the miner can later compute cross-wing tunnels
+        # from shared topics (see palace_graph.compute_topic_tunnels).
+        if confirmed["people"] or confirmed["projects"] or confirmed.get("topics"):
+            project_path = Path(args.dir).expanduser().resolve()
+            entities_path = project_path / "entities.json"
             with open(entities_path, "w", encoding="utf-8") as f:
                 json.dump(confirmed, f, indent=2, ensure_ascii=False)
             print(f"  Entities saved: {entities_path}")
 
             from .miner import add_to_known_entities
 
-            registry_path = add_to_known_entities(confirmed)
+            # Wing matches the default produced by ``room_detector_local``
+            # (folder basename) and the miner fallback in ``load_config``.
+            # Used by the topics_by_wing map so cross-wing tunnels can be
+            # computed at mine time.
+            wing = project_path.name
+            registry_path = add_to_known_entities(confirmed, wing=wing)
             print(f"  Registry updated: {registry_path}")
     else:
         print("  No entities detected — proceeding with directory-based rooms.")

mempalace/config.py

@@ -253,6 +253,32 @@ def embedding_device(self):
             return env_val.strip().lower()
         return str(self._file_config.get("embedding_device", "auto")).strip().lower()
 
+    @property
+    def topic_tunnel_min_count(self):
+        """Minimum number of overlapping confirmed topics required to create
+        a cross-wing tunnel between two wings.
+
+        Default is ``1`` — any single shared topic produces a tunnel. Bump
+        to ``2+`` if your projects share lots of common-tech labels (Python,
+        Docker, Git) and you want only meaningfully overlapping wings to
+        link. Reads ``MEMPALACE_TOPIC_TUNNEL_MIN_COUNT`` env first, then the
+        config-file value, then ``1``.
+        """
+        env_val = os.environ.get("MEMPALACE_TOPIC_TUNNEL_MIN_COUNT")
+        if env_val:
+            try:
+                parsed = int(env_val)
+                if parsed >= 1:
+                    return parsed
+            except ValueError:
+                pass
+        cfg_val = self._file_config.get("topic_tunnel_min_count")
+        try:
+            parsed = int(cfg_val) if cfg_val is not None else 1
+        except (TypeError, ValueError):
+            parsed = 1
+        return max(1, parsed)
+
     @property
     def hook_silent_save(self):
         """Whether the stop hook saves directly (True) or blocks for MCP calls (False)."""

mempalace/entity_detector.py

@@ -440,7 +440,7 @@ def detect_entities(file_paths: list, max_files: int = 10, languages=("en",)) ->
     candidates = extract_candidates(combined_text, languages=langs)
 
     if not candidates:
-        return {"people": [], "projects": [], "uncertain": []}
+        return {"people": [], "projects": [], "topics": [], "uncertain": []}
 
     # Score and classify each candidate
     people = []
@@ -467,6 +467,7 @@ def detect_entities(file_paths: list, max_files: int = 10, languages=("en",)) ->
     return {
         "people": people[:15],
         "projects": projects[:10],
+        "topics": [],
         "uncertain": uncertain[:8],
     }
 
@@ -489,7 +490,13 @@ def confirm_entities(detected: dict, yes: bool = False) -> dict:
     """
     Interactive confirmation step.
     User reviews detected entities, removes wrong ones, adds missing ones.
-    Returns confirmed {people: [names], projects: [names]}
+    Returns confirmed {people: [names], projects: [names], topics: [names]}.
+
+    Topics are not surfaced for interactive review — they come from the
+    LLM-refined ``TOPIC`` bucket and are passed through verbatim. They
+    feed cross-wing tunnel computation at mine time (see
+    ``palace_graph.compute_topic_tunnels``); a wrong topic at worst adds
+    a low-traffic tunnel and never alters drawer storage.
 
     Pass yes=True to auto-accept all detected entities without prompting.
     """
@@ -501,18 +508,28 @@ def confirm_entities(detected: dict, yes: bool = False) -> dict:
     _print_entity_list(detected["people"], "PEOPLE")
     _print_entity_list(detected["projects"], "PROJECTS")
 
+    if detected.get("topics"):
+        _print_entity_list(detected["topics"], "TOPICS (cross-wing tunnel signal)")
+
     if detected["uncertain"]:
         _print_entity_list(detected["uncertain"], "UNCERTAIN (need your call)")
 
     confirmed_people = [e["name"] for e in detected["people"]]
     confirmed_projects = [e["name"] for e in detected["projects"]]
+    confirmed_topics = [e["name"] for e in detected.get("topics", [])]
 
     if yes:
         # Auto-accept: include all detected (skip uncertain — ambiguous without user input)
         print(
-            f"\n  Auto-accepting {len(confirmed_people)} people, {len(confirmed_projects)} projects."
+            f"\n  Auto-accepting {len(confirmed_people)} people, "
+            f"{len(confirmed_projects)} projects, "
+            f"{len(confirmed_topics)} topics."
         )
-        return {"people": confirmed_people, "projects": confirmed_projects}
+        return {
+            "people": confirmed_people,
+            "projects": confirmed_projects,
+            "topics": confirmed_topics,
+        }
 
     print(f"\n{'─' * 58}")
     print("  Options:")
@@ -570,11 +587,14 @@ def confirm_entities(detected: dict, yes: bool = False) -> dict:
     print("  Confirmed:")
     print(f"  People:   {', '.join(confirmed_people) or '(none)'}")
     print(f"  Projects: {', '.join(confirmed_projects) or '(none)'}")
+    if confirmed_topics:
+        print(f"  Topics:   {', '.join(confirmed_topics)}")
     print(f"{'=' * 58}\n")
 
     return {
         "people": confirmed_people,
         "projects": confirmed_projects,
+        "topics": confirmed_topics,
     }
 
 

mempalace/llm_refine.py

@@ -197,13 +197,23 @@ def _apply_classifications(
     """Merge LLM decisions back into the detected dict.
 
     Returns (new_detected, reclassified_count, dropped_count).
+
+    Topics get their own bucket so the caller can persist them as
+    cross-wing tunnel signal. ``AMBIGUOUS`` still falls back to
+    ``uncertain`` for human review.
     """
     label_to_bucket = {
         "PERSON": "people",
         "PROJECT": "projects",
-        "TOPIC": "uncertain",
+        "TOPIC": "topics",
         "AMBIGUOUS": "uncertain",
     }
+    bucket_to_type = {
+        "people": "person",
+        "projects": "project",
+        "topics": "topic",
+        "uncertain": "uncertain",
+    }
 
     # Index every entity by name for in-place update
     all_entries: list[tuple[str, dict]] = []
@@ -216,14 +226,15 @@ def _apply_classifications(
     new_detected: dict[str, list[dict]] = {
         "people": [],
         "projects": [],
+        "topics": [],
         "uncertain": [],
     }
 
     for old_bucket, entry in all_entries:
         decision = decisions.get(entry["name"])
         if decision is None:
             # No LLM opinion — keep as-is
-            new_detected[old_bucket].append(entry)
+            new_detected.setdefault(old_bucket, []).append(entry)
             continue
 
         label, reason = decision
@@ -245,13 +256,7 @@ def _apply_classifications(
         updated["signals"] = signals
         if target_bucket != old_bucket:
             reclassified += 1
-            updated["type"] = (
-                "person"
-                if target_bucket == "people"
-                else "project"
-                if target_bucket == "projects"
-                else "uncertain"
-            )
+            updated["type"] = bucket_to_type.get(target_bucket, "uncertain")
         new_detected[target_bucket].append(updated)
 
     return new_detected, reclassified, dropped

mempalace/miner.py

@@ -439,7 +439,16 @@ def _refresh_known_entities_cache() -> None:
             data = json.load(f)
         if isinstance(data, dict):
             raw = data
-            for cat in data.values():
+            for cat_key, cat in data.items():
+                # Special wing-keyed map — its inner values are topic
+                # names but its outer keys are wings, which must NOT be
+                # surfaced as known entities. Pull the topic names out
+                # explicitly instead of treating it as a generic category.
+                if cat_key == "topics_by_wing" and isinstance(cat, dict):
+                    for topic_list in cat.values():
+                        if isinstance(topic_list, list):
+                            names.update(str(n) for n in topic_list if n)
+                    continue
                 if isinstance(cat, list):
                     names.update(str(n) for n in cat if n)
                 elif isinstance(cat, dict):
@@ -474,7 +483,39 @@ def _load_known_entities_raw() -> dict:
     return dict(_ENTITY_REGISTRY_CACHE["raw"])
 
 
-def add_to_known_entities(entities_by_category: dict) -> str:
+def _set_wing_topics(existing: dict, wing_key: str, topics_for_wing: list, coerce) -> None:
+    """Update ``existing['topics_by_wing'][wing_key]`` to the deduped list.
+
+    Replaces (does not union) the wing's topic list — re-running ``init``
+    should reflect the user's latest confirmation rather than accumulate
+    stale labels. Empty input drops the wing entry; an empty map drops
+    the ``topics_by_wing`` key entirely.
+    """
+    topics_map = existing.get("topics_by_wing")
+    if not isinstance(topics_map, dict):
+        topics_map = {}
+    seen_lower: set = set()
+    ordered: list = []
+    for n in topics_for_wing:
+        name = coerce(n)
+        if not name:
+            continue
+        key = name.lower()
+        if key in seen_lower:
+            continue
+        seen_lower.add(key)
+        ordered.append(name)
+    if ordered:
+        topics_map[wing_key] = ordered
+    else:
+        topics_map.pop(wing_key, None)
+    if topics_map:
+        existing["topics_by_wing"] = topics_map
+    else:
+        existing.pop("topics_by_wing", None)
+
+
+def add_to_known_entities(entities_by_category: dict, wing: str = None) -> str:
     """Union ``entities_by_category`` into ``~/.mempalace/known_entities.json``.
 
     Accepts ``{category: [names]}`` shape as produced by ``mempalace init``
@@ -488,6 +529,15 @@ def add_to_known_entities(entities_by_category: dict) -> str:
     added as keys with ``None`` values so existing code mappings aren't
     overwritten. A later compress pass can assign codes.
 
+    When ``wing`` is provided AND ``entities_by_category`` contains a
+    ``topics`` list, those topics are also recorded under
+    ``topics_by_wing[wing]`` (case-insensitive dedup, preserving the
+    casing of the first observed name). This is the signal source for
+    ``palace_graph.compute_topic_tunnels`` at mine time. Topics for a
+    wing are *replaced*, not unioned, so a re-run of ``init`` reflects
+    the user's latest confirmation rather than accumulating stale labels
+    indefinitely.
+
     The in-process cache is invalidated on write so same-process callers
     (notably ``cmd_init`` → ``cmd_mine`` in sequence) see the update
     immediately instead of waiting for a mtime re-check.
@@ -515,7 +565,16 @@ def _coerce_name(value):
         name = str(value)
         return name if name else None
 
+    # Separate the topics_by_wing key from regular categories so we don't
+    # treat it as a flat name-list elsewhere in this function.
+    topics_for_wing = None
+    if wing and isinstance(wing, str) and wing.strip():
+        topics_for_wing = entities_by_category.get("topics") or []
+
     for category, names in entities_by_category.items():
+        if category == "topics_by_wing":
+            # Reserved key — managed separately below.
+            continue
         if not isinstance(names, list) or not names:
             continue
         current = existing.get(category)
@@ -551,6 +610,9 @@ def _coerce_name(value):
                 ordered.append(name)
             existing[category] = ordered
 
+    if topics_for_wing is not None:
+        _set_wing_topics(existing, wing.strip(), topics_for_wing, _coerce_name)
+
     registry_path.write_text(_json.dumps(existing, indent=2, ensure_ascii=False), encoding="utf-8")
     try:
         registry_path.chmod(0o600)
@@ -565,6 +627,28 @@ def _coerce_name(value):
     return str(registry_path)
 
 
+def get_topics_by_wing() -> dict:
+    """Return ``topics_by_wing`` from the global registry as a dict.
+
+    Returns ``{}`` if the registry is missing, malformed, or has no
+    ``topics_by_wing`` key. Casing is preserved from disk; callers that
+    need case-insensitive comparison should normalize themselves.
+    """
+    raw = _load_known_entities_raw()
+    topics_map = raw.get("topics_by_wing")
+    if not isinstance(topics_map, dict):
+        return {}
+    out: dict = {}
+    for wing, topics in topics_map.items():
+        if not isinstance(wing, str) or not wing.strip():
+            continue
+        if isinstance(topics, list):
+            cleaned = [str(t) for t in topics if isinstance(t, str) and t.strip()]
+            if cleaned:
+                out[wing.strip()] = cleaned
+    return out
+
+
 _HALL_KEYWORDS_CACHE = None
 
 
@@ -962,6 +1046,19 @@ def mine(
             if not dry_run:
                 print(f"  + [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers}")
 
+    if not dry_run:
+        # Cross-wing topic tunnels: after every file in this wing has been
+        # processed, link this wing to any other wing that shares a
+        # confirmed TOPIC label. Out of scope for v1: manifest-dependency
+        # overlap, per-topic allow/deny lists, search-result surfacing.
+        try:
+            tunnels_added = _compute_topic_tunnels_for_wing(wing)
+            if tunnels_added:
+                print(f"\n  Topic tunnels: +{tunnels_added} cross-wing link(s)")
+        except Exception as e:
+            # Tunnel computation must never fail a mine — degrade quietly.
+            print(f"\n  WARNING: topic tunnel computation skipped — {e}", file=sys.stderr)
+
     print(f"\n{'=' * 55}")
     print("  Done.")
     print(f"  Files processed: {len(files) - files_skipped}")
@@ -974,6 +1071,25 @@ def mine(
     print(f"{'=' * 55}\n")
 
 
+def _compute_topic_tunnels_for_wing(wing: str) -> int:
+    """Drop tunnels between ``wing`` and every other wing that shares
+    confirmed topics, honoring the ``topic_tunnel_min_count`` config knob.
+
+    Returns the number of tunnels created or refreshed. Zero means no
+    overlap found (or the registry has no ``topics_by_wing`` map yet).
+    """
+    from .config import MempalaceConfig
+    from .palace_graph import topic_tunnels_for_wing
+
+    topics_map = get_topics_by_wing()
+    if not topics_map or wing not in topics_map:
+        return 0
+    cfg = MempalaceConfig()
+    min_count = cfg.topic_tunnel_min_count
+    created = topic_tunnels_for_wing(wing, topics_map, min_count=min_count)
+    return len(created)
+
+
 # =============================================================================
 # STATUS
 # =============================================================================