added auto model downloader

install_packages.bat

@@ -0,0 +1,8 @@
+set VENV_DIR=.venv
+
+if not exist %VENV_DIR% (
+    echo Creating virtual environment
+    python -m venv %VENV_DIR%
+)
+call %VENV_DIR%\Scripts\activate.bat
+python -m llama_api.server.app_settings --install-pkgs

install_packages.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+VENV_DIR=.venv
+
+if [ ! -d "$VENV_DIR" ]; then
+    echo "Creating virtual environment"
+    python3 -m venv $VENV_DIR
+fi
+source $VENV_DIR/bin/activate
+python3 -m llama_api.server.app_settings --install-pkgs

llama_api/schemas/models.py

@@ -92,15 +92,6 @@ class LlamaCppModel(BaseLLMModel):
 class ExllamaModel(BaseLLMModel):
     """Exllama model that can be loaded from local path."""
 
-    model_path: str = field(
-        default="YOUR_GPTQ_FOLDER_NAME",
-        metadata={
-            "description": "The GPTQ model path to the model."
-            "e.g. If you have a model folder in 'models/gptq/your_model',"
-            "then you should set this to 'your_model'."
-        },
-    )
-
     compress_pos_emb: float = field(
         default=1.0,
         metadata={

llama_api/server/app_settings.py

@@ -1,7 +1,7 @@
 import argparse
 import platform
 from contextlib import asynccontextmanager
-from os import environ
+from os import environ, getpid
 from pathlib import Path
 from typing import Dict, Optional, Union
 
@@ -21,39 +21,44 @@
 logger = ApiLogger(__name__)
 
 
-def set_priority(pid: Optional[int] = None, priority: str = "high"):
-    import platform
-    from os import getpid
-
-    import psutil
-
+def set_priority(priority: str = "high", pid: Optional[int] = None) -> bool:
     """Set The Priority of a Process.  Priority is a string which can be
     'low', 'below_normal', 'normal', 'above_normal', 'high', 'realtime'.
-    'normal' is the default."""
-
-    if platform.system() == "Windows":
-        priorities = {
-            "low": psutil.IDLE_PRIORITY_CLASS,
-            "below_normal": psutil.BELOW_NORMAL_PRIORITY_CLASS,
-            "normal": psutil.NORMAL_PRIORITY_CLASS,
-            "above_normal": psutil.ABOVE_NORMAL_PRIORITY_CLASS,
-            "high": psutil.HIGH_PRIORITY_CLASS,
-            "realtime": psutil.REALTIME_PRIORITY_CLASS,
-        }
-    else:  # Linux and other Unix systems
-        priorities = {
-            "low": 19,
-            "below_normal": 10,
-            "normal": 0,
-            "above_normal": -5,
-            "high": -11,
-            "realtime": -20,
-        }
-
+    'normal' is the default.
+    Returns True if successful, False if not."""
     if pid is None:
         pid = getpid()
-    p = psutil.Process(pid)
-    p.nice(priorities[priority])
+    try:
+        import psutil
+
+        if platform.system() == "Windows":
+            priorities = {
+                "low": psutil.IDLE_PRIORITY_CLASS,
+                "below_normal": psutil.BELOW_NORMAL_PRIORITY_CLASS,
+                "normal": psutil.NORMAL_PRIORITY_CLASS,
+                "above_normal": psutil.ABOVE_NORMAL_PRIORITY_CLASS,
+                "high": psutil.HIGH_PRIORITY_CLASS,
+                "realtime": psutil.REALTIME_PRIORITY_CLASS,
+            }
+        else:  # Linux and other Unix systems
+            priorities = {
+                "low": 19,
+                "below_normal": 10,
+                "normal": 0,
+                "above_normal": -5,
+                "high": -11,
+                "realtime": -20,
+            }
+        if priority not in priorities:
+            logger.warning(f"⚠️ Invalid priority [{priority}]")
+            return False
+
+        p = psutil.Process(pid)
+        p.nice(priorities[priority])
+        return True
+    except Exception as e:
+        logger.warning(f"⚠️ Failed to set priority of process [{pid}]: {e}")
+        return False
 
 
 def initialize_before_launch(
@@ -99,11 +104,8 @@ def initialize_before_launch(
             "If any packages are missing, "
             "use `--install-pkgs` option to install them."
         )
-    try:
-        # Set the priority of the process
-        set_priority(priority="high")
-    except Exception:
-        pass
+    # Set the priority of the process
+    set_priority(priority="high")
 
 
 @asynccontextmanager

llama_api/shared/config.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import Dict, Tuple, Union
+from typing import Dict, List, Tuple, Union
 
 
 class Config:
@@ -15,3 +15,18 @@ class Config:
         "https://github.com/abetlen/llama-cpp-python": "repositories/llama_cpp",  # noqa: E501
         "https://github.com/turboderp/exllama": "repositories/exllama",
     }
+    ggml_quanitzation_preferences_order: List[str] = [
+        "q4_K_M",
+        "q4_K_S",
+        "q4_1",
+        "q4_0",
+        "q5_K_S",
+        "q5_1",
+        "q5_0",
+        "q3_K_L",
+        "q3_K_M",
+        "q3_K_S",
+        "q2_K",
+        "q6_K",
+        "q8_0",
+    ]

llama_api/utils/concurrency.py

@@ -30,11 +30,8 @@
 def init_process_pool(env_vars: Dict[str, str]) -> None:
     """Initialize the process pool,
     and set the environment variables for the child processes"""
-    try:
-        # Set the priority of the process
-        set_priority(priority="high")
-    except Exception:
-        pass
+    # Set the priority of the process
+    set_priority(priority="high")
 
     for key, value in env_vars.items():
         environ[key] = value