diff --git a/mempalace/entity_detector.py b/mempalace/entity_detector.py index c70dd576bd..17b5a2d793 100644 --- a/mempalace/entity_detector.py +++ b/mempalace/entity_detector.py @@ -39,6 +39,7 @@ from collections import defaultdict from mempalace.i18n import get_entity_patterns +from mempalace.readable_extensions import ENTITY_DETECTOR_READABLE_EXTENSIONS # ==================== LANGUAGE-AWARE PATTERN LOADING ==================== @@ -88,23 +89,7 @@ def _get_stopwords(languages: tuple) -> frozenset: ".csv", } -READABLE_EXTENSIONS = { - ".txt", - ".md", - ".py", - ".js", - ".ts", - ".json", - ".yaml", - ".yml", - ".csv", - ".rst", - ".toml", - ".sh", - ".rb", - ".go", - ".rs", -} +READABLE_EXTENSIONS = ENTITY_DETECTOR_READABLE_EXTENSIONS SKIP_DIRS = { ".git", diff --git a/mempalace/miner.py b/mempalace/miner.py index ba0c630631..0d2c8754af 100644 --- a/mempalace/miner.py +++ b/mempalace/miner.py @@ -30,30 +30,9 @@ purge_file_closets, upsert_closet_lines, ) +from .readable_extensions import MINER_READABLE_EXTENSIONS -READABLE_EXTENSIONS = { - ".txt", - ".md", - ".py", - ".js", - ".ts", - ".jsx", - ".tsx", - ".json", - ".jsonl", - ".yaml", - ".yml", - ".html", - ".css", - ".java", - ".go", - ".rs", - ".rb", - ".sh", - ".csv", - ".sql", - ".toml", -} +READABLE_EXTENSIONS = MINER_READABLE_EXTENSIONS SKIP_FILENAMES = { "entities.json", diff --git a/mempalace/readable_extensions.py b/mempalace/readable_extensions.py new file mode 100644 index 0000000000..8c121917f8 --- /dev/null +++ b/mempalace/readable_extensions.py @@ -0,0 +1,65 @@ +"""Shared readable-extension sets used by mining and entity detection.""" + +BASE_READABLE_EXTENSIONS = { + ".txt", + ".md", + ".py", + ".js", + ".ts", + ".json", + ".yaml", + ".yml", + ".csv", + ".toml", + ".sh", + ".rb", + ".go", + ".rs", + ".c", + ".cc", + ".cpp", + ".cxx", + ".h", + ".hh", + ".hpp", + ".hxx", + ".ipp", + ".inl", + ".m", + ".mm", + ".cs", + ".csproj", + ".props", + ".sln", + ".vcxproj", + ".kt", + ".kts", + ".erl", + ".hrl", + ".nix", + ".cmake", + ".gn", + ".gni", + ".proto", + ".ps1", + ".mdx", + ".patch", + ".ex", + ".exs", + ".eex", + ".heex", +} + +ENTITY_DETECTOR_READABLE_EXTENSIONS = BASE_READABLE_EXTENSIONS | { + ".rst", +} + +MINER_READABLE_EXTENSIONS = BASE_READABLE_EXTENSIONS | { + ".jsx", + ".tsx", + ".jsonl", + ".html", + ".css", + ".java", + ".sql", +}