Fallback to pytorch engine when the model is quantized by smooth quant

autotest/utils/config_utils.py

@@ -97,7 +97,7 @@ def get_all_model_list(tp_num: int = None,
                                          model_type=model_type):
             if case not in case_list:
                 case_list.append(case)
-    return [x for x in case_list if 'w8a8' not in x]
+    return case_list
 
 
 def get_quantization_model_list(type):

lmdeploy/archs.py

@@ -193,3 +193,21 @@ def get_model_arch(model_path: str):
             raise RuntimeError(
                 f'Could not find model architecture from config: {_cfg}')
         return arch, cfg
+
+
+def get_quantization_config(config):
+    """get quantization config from a model's config."""
+    if isinstance(config, dict):
+        for k, v in config.items():
+            if k == 'quantization_config':
+                return v
+            if isinstance(v, (dict, list)):
+                result = get_quantization_config(v)
+                if result is not None:
+                    return result
+    elif isinstance(config, list):
+        for item in config:
+            result = get_quantization_config(item)
+            if result is not None:
+                return result
+    return None

lmdeploy/turbomind/deploy/converter.py

@@ -6,7 +6,7 @@
 import fire
 import torch
 
-from lmdeploy.archs import get_model_arch
+from lmdeploy.archs import get_model_arch, get_quantization_config
 from lmdeploy.messages import TurbomindEngineConfig
 from lmdeploy.model import MODELS, best_match_model
 from lmdeploy.utils import get_logger, get_model
@@ -174,23 +174,6 @@ def pack_model_repository(workspace_path: str):
                dst=osp.join(model_repo_dir, 'postprocessing'))
 
 
-def find_quantization_config(nested, target_key):
-    if isinstance(nested, dict):
-        for key, value in nested.items():
-            if key == target_key:
-                return value
-            if isinstance(value, (dict, list)):
-                result = find_quantization_config(value, target_key)
-                if result is not None:
-                    return result
-    elif isinstance(nested, list):
-        for item in nested:
-            result = find_quantization_config(item, target_key)
-            if result is not None:
-                return result
-    return None
-
-
 def get_tm_model(model_path,
                  model_name,
                  chat_template_name,
@@ -213,8 +196,7 @@ def get_tm_model(model_path,
             If it is None, the turbomind model won't be saved
     """
     _, cfg = get_model_arch(model_path)
-    quant_config = find_quantization_config(cfg.to_dict(),
-                                            'quantization_config')
+    quant_config = get_quantization_config(cfg.to_dict())
     if quant_config:
         quant_method = quant_config.get('quant_method')
         _group_size = int(quant_config.get('group_size', 0))

lmdeploy/turbomind/supported_models.py

@@ -1,5 +1,5 @@
 # Copyright (c) OpenMMLab. All rights reserved.
-from lmdeploy.archs import get_model_arch
+from lmdeploy.archs import get_model_arch, get_quantization_config
 from lmdeploy.utils import get_logger
 
 logger = get_logger('lmdeploy')
@@ -80,7 +80,13 @@ def _is_head_dim_supported(cfg):
     if os.path.exists(triton_model_path):
         support_by_turbomind = True
     else:
+
         arch, cfg = get_model_arch(model_path)
+        quant_config = get_quantization_config(cfg.to_dict())
+        if (quant_config
+                and quant_config.get('quant_method') in ['smooth_quant']):
+            # tm hasn't support quantized models by applying smoothquant
+            return False
 
         if arch in SUPPORTED_ARCHS.keys():
             support_by_turbomind = True