update: Improve Model Manager Configuration and CI Integration

.github/workflows/integration-test-docker.yml

@@ -77,7 +77,7 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y make curl
-          pip install huggingface_hub[cli]
+          pip install -r src/model_manager/requirements.txt hf_transfer
 
       - name: Download models
         run: |
@@ -86,6 +86,7 @@ jobs:
         env:
           CI: true
           CI_MINIMAL_MODELS: true
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
           HF_HUB_ENABLE_HF_TRANSFER: 1
           HF_HUB_DISABLE_TELEMETRY: 1
 

.github/workflows/performance-nightly.yml

@@ -62,13 +62,14 @@ jobs:
       - name: Build Rust library (CPU-only)
         run: make rust-ci
 
-      - name: Install HuggingFace CLI
+      - name: Install Model Manager dependencies
         run: |
-          pip install -U "huggingface_hub[cli]" hf_transfer
+          pip install -r src/model_manager/requirements.txt hf_transfer
 
       - name: Download models (minimal set for nightly)
         env:
-          CI_MINIMAL_MODELS: true
+          CI_MINIMAL_MODELS: false
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
           HF_HUB_ENABLE_HF_TRANSFER: 1
           HF_HUB_DISABLE_TELEMETRY: 1
         run: make download-models

.github/workflows/performance-test.yml

@@ -72,13 +72,14 @@ jobs:
       - name: Build Rust library (CPU-only)
         run: make rust-ci
 
-      - name: Install HuggingFace CLI
+      - name: Install Model Manager dependencies
         run: |
-          pip install -U "huggingface_hub[cli]" hf_transfer
+          pip install -r src/model_manager/requirements.txt hf_transfer
 
       - name: Download models (minimal)
         env:
           CI_MINIMAL_MODELS: true
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
           HF_HUB_ENABLE_HF_TRANSFER: 1
           HF_HUB_DISABLE_TELEMETRY: 1
         run: make download-models

.github/workflows/test-and-build.yml

@@ -134,13 +134,14 @@ jobs:
       - name: Build Rust library (CPU-only, no CUDA)
         run: make rust-ci
 
-      - name: Install HuggingFace CLI
+      - name: Install Model Manager dependencies
         run: |
-          pip install -U "huggingface_hub[cli]" hf_transfer
+          pip install -r src/model_manager/requirements.txt hf_transfer
 
       - name: Download models (minimal on PRs)
         env:
           CI_MINIMAL_MODELS: ${{ github.event_name == 'pull_request' }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
           HF_HUB_ENABLE_HF_TRANSFER: 1
           HF_HUB_DISABLE_TELEMETRY: 1
         run: make download-models

config/models.lora.yaml → config/model_manager/models.lora.yaml

@@ -2,7 +2,7 @@
 # This file contains only LoRA adapter models for incremental CI downloads.
 #
 # Usage:
-#   python -m model_manager --config config/models.lora.yaml
+#   python -m model_manager --config config/model_manager/models.lora.yaml
 #
 # Equivalent to: make download-models-lora
 

config/models.minimal.yaml → config/model_manager/models.minimal.yaml

@@ -5,12 +5,13 @@
 # Usage:
 #   CI_MINIMAL_MODELS=true python -m model_manager
 #   # or explicitly:
-#   python -m model_manager --config config/models.minimal.yaml
+#   python -m model_manager --config config/model_manager/models.minimal.yaml
 #
 # Equivalent to: make download-models-minimal
 #             or CI_MINIMAL_MODELS=true make download-models
 #
-# Note: embeddinggemma-300m is gated and requires HF_TOKEN, so it's excluded.
+# Note: This is the minimal set for fast CI runs. Larger models like
+# embeddinggemma-300m are in models.yaml (full set) for local development.
 
 cache_dir: "models"
 verify: "size" # Use size for faster CI runs

config/models.yaml → config/model_manager/models.yaml

@@ -3,7 +3,7 @@
 #
 # Usage:
 #   python -m model_manager
-#   python -m model_manager --config config/models.yaml
+#   python -m model_manager --config config/model_manager/models.yaml
 #
 # Includes additional LoRA variants (roberta, modernbert) and gated models not in minimal set.
 #

src/model_manager/README.md

@@ -0,0 +1,216 @@
+# Model Manager
+
+A Python module for automated ML model download, verification, and caching from HuggingFace.
+
+## Features
+
+- **Automated Download**: Download models from HuggingFace Hub with support for specific revisions and file filtering
+- **Integrity Verification**: Verify model integrity using size or SHA256 checksums
+- **Smart Caching**: Skip downloads for already-cached models
+- **CI-Friendly**: Environment variable-based config selection for minimal CI downloads
+- **CLI & Programmatic API**: Use from command line or import as a Python module
+
+## Quick Start
+
+### CLI Usage
+
+```bash
+# Download all models (uses config/model_manager/models.yaml by default)
+python -m model_manager
+
+# Use a specific config file
+python -m model_manager --config config/model_manager/models.yaml
+
+# Download a specific model only
+python -m model_manager --model category_classifier_modernbert-base_model
+
+# List all configured models and their cache status
+python -m model_manager --list
+
+# Verify existing models without downloading
+python -m model_manager --verify-only
+
+# Clean all cached models
+python -m model_manager --clean
+
+# Clean a specific model
+python -m model_manager --clean-model category_classifier_modernbert-base_model
+
+# Verbose output for debugging
+python -m model_manager -v
+```
+
+### CI Mode
+
+For CI environments, use the minimal model set to speed up builds and avoid rate limits:
+
+```bash
+# Auto-select minimal config
+CI_MINIMAL_MODELS=true python -m model_manager
+
+# Or explicitly specify
+python -m model_manager --config config/model_manager/models.minimal.yaml
+```
+
+### Programmatic Usage
+
+```python
+from model_manager import ensure_models, ModelManager
+
+# Simple: ensure all models are downloaded
+ensure_models("config/model_manager/models.yaml")
+
+# Advanced: use ModelManager for more control
+manager = ModelManager.from_config("config/model_manager/models.yaml")
+
+# Ensure all models
+paths = manager.ensure_all()
+print(paths)  # {'model_id': '/path/to/model', ...}
+
+# Ensure a specific model
+path = manager.ensure_model("category_classifier_modernbert-base_model")
+
+# Check if a model is cached
+from model_manager import is_cached, get_model_path
+spec = manager.get_model_spec("category_classifier_modernbert-base_model")
+if is_cached(spec, manager.config.cache_dir):
+    print(f"Model cached at: {get_model_path(spec, manager.config.cache_dir)}")
+```
+
+## Configuration
+
+Models are defined in YAML configuration files. See `config/model_manager/` for examples.
+
+### Configuration Schema
+
+```yaml
+# Cache directory for downloaded models
+cache_dir: "models"
+
+# Verification level: "none", "size", or "sha256"
+verify: "sha256"
+
+# List of models to manage
+models:
+  - id: my-model # Unique identifier (required)
+    repo_id: org/model-name # HuggingFace repo ID (required)
+    revision: main # Git revision (optional, default: main)
+    local_dir: custom-name # Override local directory name (optional)
+    files: # Specific files to download (optional)
+      - config.json
+      - model.safetensors
+```
+
+### Verification Levels
+
+| Level    | Description          | Use Case                  |
+| -------- | -------------------- | ------------------------- |
+| `none`   | No verification      | Fast, trust HuggingFace   |
+| `size`   | Verify file sizes    | Quick verification for CI |
+| `sha256` | Full SHA256 checksum | Production environments   |
+
+### Available Configs
+
+| Config                | Description                                  |
+| --------------------- | -------------------------------------------- |
+| `models.yaml`         | Full model set for local development         |
+| `models.minimal.yaml` | Minimal set for CI (faster, no gated models) |
+| `models.lora.yaml`    | LoRA adapters only                           |
+
+## Environment Variables
+
+| Variable            | Description                                                      |
+| ------------------- | ---------------------------------------------------------------- |
+| `CI_MINIMAL_MODELS` | Set to `true`, `1`, or `yes` to auto-select minimal config       |
+| `HF_TOKEN`          | HuggingFace token for gated models (e.g., `embeddinggemma-300m`) |
+
+## API Reference
+
+### Functions
+
+#### `ensure_models(config_path, cache_dir=None)`
+
+Main entry point. Downloads and verifies all models from config.
+
+#### `is_cached(spec, cache_dir)`
+
+Check if a model is already cached.
+
+#### `get_model_path(spec, cache_dir)`
+
+Get the local path for a model.
+
+#### `download_model(spec, cache_dir)`
+
+Download a model from HuggingFace.
+
+#### `verify_model(path, verify_level)`
+
+Verify model integrity.
+
+### Classes
+
+#### `ModelManager`
+
+Central manager for model operations.
+
+- `from_config(config_path)` - Create from config file
+- `ensure_all()` - Ensure all models are ready
+- `ensure_model(model_id)` - Ensure a specific model
+- `get_model_spec(model_id)` - Get model specification
+
+#### `ModelSpec`
+
+Specification for a single model.
+
+- `id` - Unique identifier
+- `repo_id` - HuggingFace repository ID
+- `revision` - Git revision (default: "main")
+- `local_dir` - Override local directory name
+- `files` - Specific files to download
+
+#### `ModelsConfig`
+
+Configuration container.
+
+- `models` - List of ModelSpec
+- `cache_dir` - Cache directory path
+- `verify` - Verification level
+
+### Exceptions
+
+| Exception            | Description               |
+| -------------------- | ------------------------- |
+| `ModelManagerError`  | Base exception            |
+| `MissingModelError`  | Model not found in config |
+| `BadChecksumError`   | Verification failed       |
+| `DownloadError`      | Download failed           |
+| `ConfigurationError` | Invalid config            |
+
+## Development
+
+### Running Tests
+
+```bash
+# Run all model_manager tests
+pytest src/model_manager/tests/ -v
+
+# Run with coverage
+pytest src/model_manager/tests/ --cov=model_manager
+```
+
+### Project Structure
+
+```
+src/model_manager/
+├── __init__.py      # Public API and ModelManager class
+├── __main__.py      # CLI entrypoint
+├── cli.py           # CLI implementation
+├── config.py        # Configuration dataclasses
+├── registry.py      # YAML config parsing
+├── downloader.py    # HuggingFace download logic
+├── verifier.py      # Integrity verification
+├── cache.py         # Cache management
+├── errors.py        # Custom exceptions
+└── tests/           # Unit tests
+```

src/model_manager/cli.py

@@ -24,8 +24,8 @@ def get_default_config() -> str:
     """Get default config path based on CI_MINIMAL_MODELS environment variable."""
     ci_minimal = os.environ.get("CI_MINIMAL_MODELS", "").lower()
     if ci_minimal in ("true", "1", "yes"):
-        return "config/models.minimal.yaml"
-    return "config/models.yaml"
+        return "config/model_manager/models.minimal.yaml"
+    return "config/model_manager/models.yaml"
 
 
 def setup_logging(verbose: bool = False) -> None:

src/model_manager/tests/test_cli.py

@@ -18,37 +18,37 @@ def test_default_config_without_env(self):
         with patch("os.environ.get") as mock_get:
             mock_get.return_value = ""
             result = get_default_config()
-            assert result == "config/models.yaml"
+            assert result == "config/model_manager/models.yaml"
 
     def test_minimal_config_with_true(self):
         """Test that minimal config is returned when CI_MINIMAL_MODELS=true."""
         with patch.dict(os.environ, {"CI_MINIMAL_MODELS": "true"}):
             result = get_default_config()
-            assert result == "config/models.minimal.yaml"
+            assert result == "config/model_manager/models.minimal.yaml"
 
     def test_minimal_config_with_1(self):
         """Test that minimal config is returned when CI_MINIMAL_MODELS=1."""
         with patch.dict(os.environ, {"CI_MINIMAL_MODELS": "1"}):
             result = get_default_config()
-            assert result == "config/models.minimal.yaml"
+            assert result == "config/model_manager/models.minimal.yaml"
 
     def test_minimal_config_with_yes(self):
         """Test that minimal config is returned when CI_MINIMAL_MODELS=yes."""
         with patch.dict(os.environ, {"CI_MINIMAL_MODELS": "yes"}):
             result = get_default_config()
-            assert result == "config/models.minimal.yaml"
+            assert result == "config/model_manager/models.minimal.yaml"
 
     def test_minimal_config_case_insensitive(self):
         """Test that CI_MINIMAL_MODELS check is case insensitive."""
         with patch.dict(os.environ, {"CI_MINIMAL_MODELS": "TRUE"}):
             result = get_default_config()
-            assert result == "config/models.minimal.yaml"
+            assert result == "config/model_manager/models.minimal.yaml"
 
     def test_default_config_with_false(self):
         """Test that default config is returned when CI_MINIMAL_MODELS=false."""
         with patch.dict(os.environ, {"CI_MINIMAL_MODELS": "false"}):
             result = get_default_config()
-            assert result == "config/models.yaml"
+            assert result == "config/model_manager/models.yaml"
 
 
 class TestMainCLI: