fix: prevent UnicodeEncodeError on Windows in search output

mempalace/cli.py

@@ -124,7 +124,7 @@ def cmd_split(args):
     import sys
 
     # Rebuild argv for split_mega_files argparse
-    argv = [args.dir]
+    argv = ["--source", args.dir]
     if args.output_dir:
         argv += ["--output-dir", args.output_dir]
     if args.dry_run:

mempalace/config.py

@@ -94,8 +94,8 @@ def palace_path(self):
         """Path to the memory palace data directory."""
         env_val = os.environ.get("MEMPALACE_PALACE_PATH") or os.environ.get("MEMPAL_PALACE_PATH")
         if env_val:
-            return env_val
-        return self._file_config.get("palace_path", DEFAULT_PALACE_PATH)
+            return os.path.expanduser(env_val)
+        return os.path.expanduser(self._file_config.get("palace_path", DEFAULT_PALACE_PATH))
 
     @property
     def collection_name(self):

mempalace/miner.py

@@ -40,19 +40,169 @@
 }
 
 SKIP_DIRS = {
+    # Version control
     ".git",
+    ".svn",
+    ".hg",
+    ".bzr",
+    # Dependencies
     "node_modules",
     "__pycache__",
     ".venv",
     "venv",
     "env",
+    ".conda",
+    ".virtualenv",
+    # Build outputs
     "dist",
     "build",
+    "target",
+    "out",
+    "bin",
+    "obj",
+    "Debug",
+    "Release",
+    "x64",
+    "x86",
+    # Frontend build outputs
     ".next",
+    ".nuxt",
+    ".svelte-kit",
+    "public",
+    "static",
+    "output",
+    "export",
+    # Java/Gradle
+    ".gradle",
+    ".metadata",
+    # Rust/Cargo
+    ".cargo",
+    # Go
+    "vendor",
+    # Python
+    ".pytest_cache",
+    ".mypy_cache",
+    ".tox",
+    # Testing
     "coverage",
+    ".nyc_output",
+    "coverage-final.json",
+    # Temporary
+    "tmp",
+    "temp",
+    ".tmp",
+    ".temp",
+    # MemPalace itself
     ".mempalace",
+    # Other common build artifacts
+    ".DS_Store",
+    "Thumbs.db",
 }
 
+# Additional file extensions and patterns to exclude
+SKIP_FILE_PATTERNS = {
+    # Minified JS/CSS
+    ".min.js",
+    ".min.css",
+    ".bundle.js",
+    ".bundle.css",
+    # Compiled/Generated files
+    ".pyc",
+    ".so",
+    ".dll",
+    ".exe",
+    ".o",
+    ".obj",
+    ".a",
+    ".lib",
+    ".dylib",
+    ".jar",
+    ".war",
+    ".ear",
+    ".zip",
+    ".tar",
+    ".gz",
+    ".rar",
+    ".7z",
+    # Build artifacts
+    ".class",
+    ".elc",
+    ".beam",
+    # Logs
+    ".log",
+    # Lock files
+    "package-lock.json",
+    "yarn.lock",
+    "Cargo.lock",
+    "Pipfile.lock",
+    "poetry.lock",
+    "Gemfile.lock",
+    "composer.lock",
+    # IDE files
+    ".swp",
+    ".swo",
+}
+
+
+def should_skip_file(filepath: Path) -> bool:
+    """Check if a file should be skipped during mining."""
+    # Check if parent directory is in skip list
+    for parent in filepath.parents:
+        if parent.name in SKIP_DIRS:
+            return True
+
+    # Check if file extension or pattern matches skip list
+    if filepath.suffix.lower() in {
+        ".min.js",
+        ".min.css",
+        ".bundle.js",
+        ".bundle.css",
+        ".pyc",
+        ".so",
+        ".dll",
+        ".exe",
+        ".o",
+        ".obj",
+        ".a",
+        ".lib",
+        ".dylib",
+        ".jar",
+        ".war",
+        ".ear",
+        ".zip",
+        ".tar",
+        ".gz",
+        ".rar",
+        ".7z",
+        ".class",
+        ".elc",
+        ".beam",
+        ".log",
+    }:
+        return True
+
+    # Check for specific filenames
+    if filepath.name in {
+        "package-lock.json",
+        "yarn.lock",
+        "Cargo.lock",
+        "Pipfile.lock",
+        "poetry.lock",
+        "Gemfile.lock",
+        "composer.lock",
+        ".swp",
+        ".swo",
+    }:
+        return True
+
+    # Check for other patterns
+    for pattern in SKIP_FILE_PATTERNS:
+        if str(filepath).lower().endswith(pattern.lower()):
+            return True
+
+    return False
+
+
 CHUNK_SIZE = 800  # chars per drawer
 CHUNK_OVERLAP = 100  # overlap between chunks
 MIN_CHUNK_SIZE = 50  # skip tiny chunks
@@ -293,16 +443,16 @@ def scan_project(project_dir: str) -> list:
         for filename in filenames:
             filepath = Path(root) / filename
             if filepath.suffix.lower() in READABLE_EXTENSIONS:
-                # Skip config files
                 if filename in (
                     "mempalace.yaml",
                     "mempalace.yml",
                     "mempal.yaml",
                     "mempal.yml",
                     ".gitignore",
-                    "package-lock.json",
                 ):
                     continue
+                if should_skip_file(filepath):
+                    continue
                 files.append(filepath)
     return files
 
@@ -399,7 +549,8 @@ def status(palace_path: str):
         return
 
     # Count by wing and room
-    r = col.get(limit=10000, include=["metadatas"])
+    total = col.count()
+    r = col.get(limit=total, include=["metadatas"])
     metas = r["metadatas"]
 
     wing_rooms = defaultdict(lambda: defaultdict(int))

mempalace/searcher.py

@@ -12,6 +12,19 @@
 import chromadb
 
 
+def _ensure_utf8_encoding():
+    if sys.platform == "win32":
+        import io
+
+        if isinstance(sys.stdout, io.TextIOWrapper):
+            try:
+                sys.stdout.reconfigure(encoding="utf-8")
+            except (AttributeError, io.UnsupportedOperation):
+                import codecs
+
+                sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())
+
+
 def search(query: str, palace_path: str, wing: str = None, room: str = None, n_results: int = 5):
     """
     Search the palace. Returns verbatim drawer content.
@@ -57,6 +70,8 @@ def search(query: str, palace_path: str, wing: str = None, room: str = None, n_r
         print(f'\n  No results found for: "{query}"')
         return
 
+    _ensure_utf8_encoding()
+
     print(f"\n{'=' * 60}")
     print(f'  Results for: "{query}"')
     if wing:

tests/test_config_expansion.py

@@ -0,0 +1,37 @@
+import os
+import sys
+from pathlib import Path
+from unittest.mock import patch
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from mempalace.config import MempalaceConfig
+
+
+def test_palace_path_expands_user_home():
+    config = MempalaceConfig()
+
+    default_path = config.palace_path
+    assert "~" not in default_path
+    assert os.path.expanduser("~/.mempalace/palace") == default_path
+
+    with patch.object(config, "_file_config", {"palace_path": "~/custom/palace"}):
+        custom_path = config.palace_path
+        assert "~" not in custom_path
+        expected = os.path.expanduser("~/custom/palace")
+        assert expected == custom_path
+
+    with patch.dict(os.environ, {"MEMPALACE_PALACE_PATH": "~/env/palace"}):
+        config_with_env = MempalaceConfig()
+        env_path = config_with_env.palace_path
+        assert "~" not in env_path
+        expected = os.path.expanduser("~/env/palace")
+        assert expected == env_path
+
+
+def test_palace_path_with_env_var():
+    with patch.dict(os.environ, {"MEMPALACE_PALACE_PATH": "~/test/palace/from/env"}):
+        config = MempalaceConfig()
+        path = config.palace_path
+        expected = os.path.expanduser("~/test/palace/from/env")
+        assert path == expected

tests/test_miner_excludes.py

@@ -0,0 +1,83 @@
+import os
+import tempfile
+import shutil
+from pathlib import Path
+import sys
+from unittest.mock import Mock
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+# Mock chromadb before importing miner
+sys.modules["chromadb"] = Mock()
+import chromadb
+
+from mempalace.miner import scan_project
+
+
+def test_scan_project_excludes_build_artifacts():
+    tmpdir = tempfile.mkdtemp()
+
+    os.makedirs(os.path.join(tmpdir, "src"), exist_ok=True)
+    with open(os.path.join(tmpdir, "src", "main.py"), "w") as f:
+        f.write("print('hello')")
+
+    os.makedirs(os.path.join(tmpdir, "target"), exist_ok=True)
+    with open(os.path.join(tmpdir, "target", "binary.exe"), "w") as f:
+        f.write("binary content")
+
+    os.makedirs(os.path.join(tmpdir, "dist"), exist_ok=True)
+    with open(os.path.join(tmpdir, "dist", "bundle.js"), "w") as f:
+        f.write("minified content")
+
+    os.makedirs(os.path.join(tmpdir, "build"), exist_ok=True)
+    with open(os.path.join(tmpdir, "build", "output.log"), "w") as f:
+        f.write("log content")
+
+    os.makedirs(os.path.join(tmpdir, "tmp"), exist_ok=True)
+    with open(os.path.join(tmpdir, "tmp", "temp.txt"), "w") as f:
+        f.write("temp content")
+
+    with open(os.path.join(tmpdir, "package-lock.json"), "w") as f:
+        f.write("{}")
+    with open(os.path.join(tmpdir, "Cargo.lock"), "w") as f:
+        f.write("lock content")
+
+    files = scan_project(tmpdir)
+    file_names = [f.name for f in files]
+
+    assert "main.py" in file_names
+
+    assert "binary.exe" not in file_names
+    assert "bundle.js" not in file_names
+    assert "output.log" not in file_names
+    assert "temp.txt" not in file_names
+
+    assert "package-lock.json" not in file_names
+    assert "Cargo.lock" not in file_names
+
+    shutil.rmtree(tmpdir)
+
+
+def test_scan_project_excludes_generated_file_patterns():
+    tmpdir = tempfile.mkdtemp()
+
+    with open(os.path.join(tmpdir, "app.min.js"), "w") as f:
+        f.write("minified js")
+    with open(os.path.join(tmpdir, "style.min.css"), "w") as f:
+        f.write("minified css")
+    with open(os.path.join(tmpdir, "bundle.bundle.js"), "w") as f:
+        f.write("bundle js")
+
+    with open(os.path.join(tmpdir, "normal.js"), "w") as f:
+        f.write("normal source")
+
+    files = scan_project(tmpdir)
+    file_names = [f.name for f in files]
+
+    assert "app.min.js" not in file_names
+    assert "style.min.css" not in file_names
+    assert "bundle.bundle.js" not in file_names
+
+    assert "normal.js" in file_names
+
+    shutil.rmtree(tmpdir)