[Feat] Feature: New Python-based Model Manager

config/models.lora.yaml

@@ -0,0 +1,31 @@
+# LoRA Adapters Only
+# This file contains only LoRA adapter models for incremental CI downloads.
+#
+# Usage:
+#   python -m model_manager --config config/models.lora.yaml
+#
+# Equivalent to: make download-models-lora
+
+cache_dir: "models"
+verify: "size"
+
+models:
+  # =============================================================================
+  # LoRA Adapters - BERT-based
+  # =============================================================================
+
+  - id: lora_intent_classifier_bert-base-uncased_model
+    repo_id: LLM-Semantic-Router/lora_intent_classifier_bert-base-uncased_model
+
+  - id: lora_pii_detector_bert-base-uncased_model
+    repo_id: LLM-Semantic-Router/lora_pii_detector_bert-base-uncased_model
+
+  - id: lora_jailbreak_classifier_bert-base-uncased_model
+    repo_id: LLM-Semantic-Router/lora_jailbreak_classifier_bert-base-uncased_model
+
+  # =============================================================================
+  # Embedding Models (Advanced)
+  # =============================================================================
+
+  - id: Qwen3-Embedding-0.6B
+    repo_id: Qwen/Qwen3-Embedding-0.6B

config/models.minimal.yaml

@@ -0,0 +1,77 @@
+# Minimal Model Set for CI
+# This file contains only the essential models needed for CI tests.
+# Use this to avoid rate limits and speed up CI runs.
+#
+# Usage:
+#   CI_MINIMAL_MODELS=true python -m model_manager
+#   # or explicitly:
+#   python -m model_manager --config config/models.minimal.yaml
+#
+# Equivalent to: make download-models-minimal
+#             or CI_MINIMAL_MODELS=true make download-models
+#
+# Note: embeddinggemma-300m is gated and requires HF_TOKEN, so it's excluded.
+
+cache_dir: "models"
+verify: "size" # Use size for faster CI runs
+
+models:
+  # =============================================================================
+  # Core Classifiers (ModernBERT) - Required for classification tests
+  # =============================================================================
+
+  - id: category_classifier_modernbert-base_model
+    repo_id: LLM-Semantic-Router/category_classifier_modernbert-base_model
+
+  - id: pii_classifier_modernbert-base_presidio_token_model
+    repo_id: LLM-Semantic-Router/pii_classifier_modernbert-base_presidio_token_model
+
+  - id: jailbreak_classifier_modernbert-base_model
+    repo_id: LLM-Semantic-Router/jailbreak_classifier_modernbert-base_model
+
+  - id: pii_classifier_modernbert-base_model
+    repo_id: LLM-Semantic-Router/pii_classifier_modernbert-base_model
+
+  # =============================================================================
+  # LoRA Adapters (BERT-based) - Required for unified classifier tests
+  # =============================================================================
+
+  - id: lora_intent_classifier_bert-base-uncased_model
+    repo_id: LLM-Semantic-Router/lora_intent_classifier_bert-base-uncased_model
+
+  - id: lora_pii_detector_bert-base-uncased_model
+    repo_id: LLM-Semantic-Router/lora_pii_detector_bert-base-uncased_model
+
+  - id: lora_jailbreak_classifier_bert-base-uncased_model
+    repo_id: LLM-Semantic-Router/lora_jailbreak_classifier_bert-base-uncased_model
+
+  # =============================================================================
+  # Embedding Models - Required for cache and embedding tests
+  # =============================================================================
+
+  - id: all-MiniLM-L12-v2
+    repo_id: sentence-transformers/all-MiniLM-L12-v2
+
+  - id: Qwen3-Embedding-0.6B
+    repo_id: Qwen/Qwen3-Embedding-0.6B
+
+  # =============================================================================
+  # Hallucination Detection - Required for hallucination tests
+  # =============================================================================
+
+  - id: halugate-detector
+    repo_id: KRLabsOrg/lettucedect-base-modernbert-en-v1
+
+  - id: halugate-sentinel
+    repo_id: LLM-Semantic-Router/halugate-sentinel
+
+  - id: ModernBERT-base-nli
+    repo_id: tasksource/ModernBERT-base-nli
+
+  # =============================================================================
+  # LLM Models - For llm-katan tests
+  # =============================================================================
+
+  - id: Qwen/Qwen3-0.6B
+    repo_id: Qwen/Qwen3-0.6B
+    local_dir: Qwen/Qwen3-0.6B

config/models.yaml

@@ -0,0 +1,104 @@
+# Full Model Set for Local Development
+# This file contains ALL models for full local development, matching models.mk download-models-full
+#
+# Usage:
+#   python -m model_manager
+#   python -m model_manager --config config/models.yaml
+#
+# Includes additional LoRA variants (roberta, modernbert) and gated models not in minimal set.
+#
+# Equivalent to: make download-models-full
+
+cache_dir: "models"
+verify: "sha256" # Full verification for production use
+
+models:
+  # =============================================================================
+  # Core Classifiers (ModernBERT)
+  # =============================================================================
+
+  - id: category_classifier_modernbert-base_model
+    repo_id: LLM-Semantic-Router/category_classifier_modernbert-base_model
+
+  - id: pii_classifier_modernbert-base_presidio_token_model
+    repo_id: LLM-Semantic-Router/pii_classifier_modernbert-base_presidio_token_model
+
+  - id: jailbreak_classifier_modernbert-base_model
+    repo_id: LLM-Semantic-Router/jailbreak_classifier_modernbert-base_model
+
+  - id: pii_classifier_modernbert-base_model
+    repo_id: LLM-Semantic-Router/pii_classifier_modernbert-base_model
+
+  # =============================================================================
+  # LoRA Adapters - BERT-based
+  # =============================================================================
+
+  - id: lora_intent_classifier_bert-base-uncased_model
+    repo_id: LLM-Semantic-Router/lora_intent_classifier_bert-base-uncased_model
+
+  - id: lora_pii_detector_bert-base-uncased_model
+    repo_id: LLM-Semantic-Router/lora_pii_detector_bert-base-uncased_model
+
+  - id: lora_jailbreak_classifier_bert-base-uncased_model
+    repo_id: LLM-Semantic-Router/lora_jailbreak_classifier_bert-base-uncased_model
+
+  # =============================================================================
+  # LoRA Adapters - RoBERTa-based (Full only)
+  # =============================================================================
+
+  - id: lora_intent_classifier_roberta-base_model
+    repo_id: LLM-Semantic-Router/lora_intent_classifier_roberta-base_model
+
+  - id: lora_pii_detector_roberta-base_model
+    repo_id: LLM-Semantic-Router/lora_pii_detector_roberta-base_model
+
+  - id: lora_jailbreak_classifier_roberta-base_model
+    repo_id: LLM-Semantic-Router/lora_jailbreak_classifier_roberta-base_model
+
+  # =============================================================================
+  # LoRA Adapters - ModernBERT-based (Full only)
+  # =============================================================================
+
+  - id: lora_intent_classifier_modernbert-base_model
+    repo_id: LLM-Semantic-Router/lora_intent_classifier_modernbert-base_model
+
+  - id: lora_pii_detector_modernbert-base_model
+    repo_id: LLM-Semantic-Router/lora_pii_detector_modernbert-base_model
+
+  - id: lora_jailbreak_classifier_modernbert-base_model
+    repo_id: LLM-Semantic-Router/lora_jailbreak_classifier_modernbert-base_model
+
+  # =============================================================================
+  # Embedding Models
+  # =============================================================================
+
+  - id: all-MiniLM-L12-v2
+    repo_id: sentence-transformers/all-MiniLM-L12-v2
+
+  - id: Qwen3-Embedding-0.6B
+    repo_id: Qwen/Qwen3-Embedding-0.6B
+
+  # Gated model - requires HF_TOKEN
+  - id: embeddinggemma-300m
+    repo_id: google/embeddinggemma-300m
+
+  # =============================================================================
+  # Hallucination Detection Models
+  # =============================================================================
+
+  - id: halugate-detector
+    repo_id: KRLabsOrg/lettucedect-base-modernbert-en-v1
+
+  - id: halugate-sentinel
+    repo_id: LLM-Semantic-Router/halugate-sentinel
+
+  - id: ModernBERT-base-nli
+    repo_id: tasksource/ModernBERT-base-nli
+
+  # =============================================================================
+  # LLM Models (for llm-katan)
+  # =============================================================================
+
+  - id: Qwen/Qwen3-0.6B
+    repo_id: Qwen/Qwen3-0.6B
+    local_dir: Qwen/Qwen3-0.6B

src/model_manager/__init__.py

@@ -0,0 +1,143 @@
+"""
+Model Manager - Automated model download, verification, and caching.
+
+This module provides utilities for managing ML models from HuggingFace,
+including automatic download, integrity verification, and cache management.
+
+Usage:
+    from model_manager import ensure_models
+    ensure_models("models.yaml")
+
+    # Or programmatically:
+    from model_manager import ModelManager
+    manager = ModelManager.from_config("models.yaml")
+    manager.ensure_all()
+"""
+
+from .config import ModelSpec, ModelsConfig
+from .registry import load_models_config
+from .downloader import download_model
+from .verifier import verify_model
+from .cache import is_cached, get_model_path
+from .errors import (
+    ModelManagerError,
+    MissingModelError,
+    BadChecksumError,
+    DownloadError,
+)
+
+__version__ = "0.1.0"
+__all__ = [
+    "ensure_models",
+    "ModelManager",
+    "ModelSpec",
+    "ModelsConfig",
+    "load_models_config",
+    "download_model",
+    "verify_model",
+    "is_cached",
+    "get_model_path",
+    "ModelManagerError",
+    "MissingModelError",
+    "BadChecksumError",
+    "DownloadError",
+]
+
+
+def ensure_models(
+    config_path: str = "models.yaml", cache_dir: str | None = None
+) -> None:
+    """
+    Main entry point. Reads config, downloads missing models, verifies integrity.
+    Called during application startup.
+
+    Args:
+        config_path: Path to models.yaml configuration file
+        cache_dir: Override cache directory from config
+    """
+    manager = ModelManager.from_config(config_path)
+    if cache_dir:
+        manager.config.cache_dir = cache_dir
+    manager.ensure_all()
+
+
+class ModelManager:
+    """
+    Central manager for model download, verification, and caching.
+    """
+
+    def __init__(self, config: ModelsConfig):
+        self.config = config
+
+    @classmethod
+    def from_config(cls, config_path: str) -> "ModelManager":
+        """Create a ModelManager from a configuration file."""
+        config = load_models_config(config_path)
+        return cls(config)
+
+    def ensure_all(self) -> dict[str, str]:
+        """
+        Ensure all models are downloaded and verified.
+
+        Returns:
+            Dictionary mapping model IDs to their local paths
+        """
+        import logging
+
+        logger = logging.getLogger(__name__)
+
+        results = {}
+        for spec in self.config.models:
+            model_path = get_model_path(spec, self.config.cache_dir)
+
+            if is_cached(spec, self.config.cache_dir):
+                logger.info(f"Model '{spec.id}' already cached at {model_path}")
+                results[spec.id] = model_path
+                continue
+
+            logger.info(f"Downloading model '{spec.id}' from {spec.repo_id}...")
+            local_path = download_model(spec, self.config.cache_dir)
+
+            if self.config.verify != "none":
+                logger.info(f"Verifying model '{spec.id}'...")
+                if not verify_model(local_path, self.config.verify):
+                    raise BadChecksumError(f"Verification failed for model '{spec.id}'")
+
+            results[spec.id] = local_path
+            logger.info(f"Model '{spec.id}' ready at {local_path}")
+
+        return results
+
+    def ensure_model(self, model_id: str) -> str:
+        """
+        Ensure a specific model is downloaded and verified.
+
+        Args:
+            model_id: ID of the model to ensure
+
+        Returns:
+            Local path to the model
+        """
+        spec = self.get_model_spec(model_id)
+        if spec is None:
+            raise MissingModelError(f"Model '{model_id}' not found in configuration")
+
+        model_path = get_model_path(spec, self.config.cache_dir)
+
+        if is_cached(spec, self.config.cache_dir):
+            return model_path
+
+        local_path = download_model(spec, self.config.cache_dir)
+
+        if self.config.verify != "none":
+            if not verify_model(local_path, self.config.verify):
+                raise BadChecksumError(f"Verification failed for model '{model_id}'")
+
+        return local_path
+
+    def get_model_spec(self, model_id: str) -> ModelSpec | None:
+        """Get model specification by ID."""
+        for spec in self.config.models:
+            if spec.id == model_id:
+                return spec
+        return None

src/model_manager/__main__.py

@@ -0,0 +1,11 @@
+"""
+CLI entrypoint for running model_manager as a module.
+
+Usage:
+    python -m model_manager --config models.yaml
+"""
+
+from .cli import main
+
+if __name__ == "__main__":
+    main()