Fully deprecate AutoGPTQ and AutoAWQ for GPT-QModel

docker/peft-gpu/Dockerfile

@@ -39,12 +39,9 @@ RUN apt-get update && \
 RUN chsh -s /bin/bash
 SHELL ["/bin/bash", "-c"]
 
-RUN conda run -n peft pip install --no-cache-dir bitsandbytes optimum auto-gptq
+RUN conda run -n peft pip install --no-cache-dir bitsandbytes optimum gptqmodel
 
 RUN \
-    # Add autoawq for quantization testing
-    conda run -n peft pip install --no-cache-dir https://github.com/casper-hansen/AutoAWQ/releases/download/v0.2.7.post2/autoawq-0.2.7.post2-py3-none-any.whl && \
-    conda run -n peft pip install --no-cache-dir https://github.com/casper-hansen/AutoAWQ_kernels/releases/download/v0.0.9/autoawq_kernels-0.0.9-cp311-cp311-linux_x86_64.whl && \
     # Add eetq for quantization testing; needs to run without build isolation since the setup
     # script directly imports torch from the environment which would fail with isolation.
     conda run -n peft pip install --no-build-isolation git+https://github.com/NetEase-FuXi/EETQ.git

src/peft/import_utils.py

@@ -35,24 +35,10 @@ def is_bnb_4bit_available() -> bool:
     return hasattr(bnb.nn, "Linear4bit")
 
 
-@lru_cache
-def is_auto_gptq_available():
-    if importlib.util.find_spec("auto_gptq") is not None:
-        AUTOGPTQ_MINIMUM_VERSION = packaging.version.parse("0.5.0")
-        version_autogptq = packaging.version.parse(importlib_metadata.version("auto_gptq"))
-        if AUTOGPTQ_MINIMUM_VERSION <= version_autogptq:
-            return True
-        else:
-            raise ImportError(
-                f"Found an incompatible version of auto-gptq. Found version {version_autogptq}, "
-                f"but only versions above {AUTOGPTQ_MINIMUM_VERSION} are supported"
-            )
-
-
 @lru_cache
 def is_gptqmodel_available():
     if importlib.util.find_spec("gptqmodel") is not None:
-        GPTQMODEL_MINIMUM_VERSION = packaging.version.parse("2.0.0")
+        GPTQMODEL_MINIMUM_VERSION = packaging.version.parse("5.6.0")
         OPTIMUM_MINIMUM_VERSION = packaging.version.parse("1.24.0")
         version_gptqmodel = packaging.version.parse(importlib_metadata.version("gptqmodel"))
         if GPTQMODEL_MINIMUM_VERSION <= version_gptqmodel:
@@ -103,11 +89,6 @@ def is_aqlm_available():
     return importlib.util.find_spec("aqlm") is not None
 
 
-@lru_cache
-def is_auto_awq_available():
-    return importlib.util.find_spec("awq") is not None
-
-
 @lru_cache
 def is_eetq_available():
     return importlib.util.find_spec("eetq") is not None

src/peft/tuners/adalora/model.py

@@ -17,14 +17,13 @@
 import torch
 from transformers.pytorch_utils import Conv1D
 
-from peft.import_utils import is_bnb_4bit_available, is_bnb_available, is_gptqmodel_available
+from peft.import_utils import is_bnb_4bit_available, is_bnb_available
 from peft.tuners.lora import LoraConfig, LoraModel
 from peft.tuners.tuners_utils import BaseTunerLayer
 from peft.utils import (
     TRANSFORMERS_MODELS_TO_ADALORA_TARGET_MODULES_MAPPING,
     _freeze_adapter,
     _get_submodules,
-    get_auto_gptq_quant_linear,
     get_gptqmodel_quant_linear,
     get_quantization_config,
 )
@@ -164,10 +163,7 @@ def _create_new_module(lora_config, adapter_name, target, device_map=None, **kwa
 
         gptq_quantization_config = kwargs.get("gptq_quantization_config", None)
 
-        if is_gptqmodel_available():
-            QuantLinear = get_gptqmodel_quant_linear(gptq_quantization_config, device_map=device_map)
-        else:
-            QuantLinear = get_auto_gptq_quant_linear(gptq_quantization_config)
+        QuantLinear = get_gptqmodel_quant_linear(gptq_quantization_config, device_map=device_map)
 
         loaded_in_8bit = kwargs.pop("loaded_in_8bit", False)
         loaded_in_4bit = kwargs.pop("loaded_in_4bit", False)

src/peft/tuners/lora/awq.py

@@ -11,13 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import importlib.metadata as importlib_metadata
 from typing import Any, Optional
 
-import packaging.version
 import torch
 
-from peft.import_utils import is_auto_awq_available
+from peft.import_utils import is_gptqmodel_available
 from peft.tuners.lora.layer import LoraLayer
 from peft.tuners.tuners_utils import BaseTunerLayer
 
@@ -101,20 +99,10 @@ def dispatch_awq(
     else:
         target_base_layer = target
 
-    if is_auto_awq_available():
-        from awq.modules.linear import WQLinear_GEMM
-
-        if isinstance(target_base_layer, WQLinear_GEMM):
-            # Raise the error only at the dispatch level
-            AUTOAWQ_MINIMUM_VERSION = packaging.version.parse("0.2.0")
-            version_autoawq = packaging.version.parse(importlib_metadata.version("autoawq"))
-
-            if AUTOAWQ_MINIMUM_VERSION > version_autoawq:
-                raise ImportError(
-                    f"Found an incompatible version of auto-awq. Found version {version_autoawq}, "
-                    f"but only versions above {AUTOAWQ_MINIMUM_VERSION} are supported for PEFT."
-                )
+    if is_gptqmodel_available():
+        from gptqmodel.nn_modules.qlinear.gemm_awq import AwqGEMMQuantLinear
 
+        if isinstance(target_base_layer, AwqGEMMQuantLinear):
             new_module = AwqLoraLinear(target, adapter_name, **kwargs)
             target.qweight = target_base_layer.qweight
 

src/peft/tuners/lora/gptq.py

@@ -15,10 +15,8 @@
 
 import torch
 
-from peft.import_utils import is_gptqmodel_available
 from peft.tuners.lora.layer import LoraLayer
 from peft.tuners.tuners_utils import BaseTunerLayer
-from peft.utils import get_auto_gptq_quant_linear
 
 from .layer import LoraVariant
 
@@ -138,17 +136,10 @@ def dispatch_gptq(
 
     cfg = kwargs.get("gptq_quantization_config", None)
 
-    if is_gptqmodel_available():
-        from gptqmodel.nn_modules.qlinear import BaseQuantLinear
+    from gptqmodel.nn_modules.qlinear import BaseQuantLinear
 
-        if isinstance(target_base_layer, BaseQuantLinear):
-            new_module = GPTQLoraLinear(target, adapter_name, **kwargs)
-            target.qweight = target_base_layer.qweight
-    else:
-        quant_linear = get_auto_gptq_quant_linear(cfg)
-
-        if quant_linear is not None and isinstance(target_base_layer, quant_linear):
-            new_module = GPTQLoraLinear(target, adapter_name, **kwargs)
-            target.qweight = target_base_layer.qweight
+    if isinstance(target_base_layer, BaseQuantLinear):
+        new_module = GPTQLoraLinear(target, adapter_name, **kwargs)
+        target.qweight = target_base_layer.qweight
 
     return new_module

src/peft/tuners/mixed/model.py

@@ -27,7 +27,7 @@
     ModulesToSaveWrapper,
     PeftType,
     _get_submodules,
-    get_auto_gptq_quant_linear,
+    get_gptqmodel_quant_linear,
 )
 from peft.utils.other import _set_adapter
 
@@ -175,8 +175,8 @@ def _mark_only_adapters_as_trainable(self, model: nn.Module) -> None:
     @staticmethod
     def _create_new_module(config, adapter_name, target, **kwargs):
         gptq_quantization_config = kwargs.get("gptq_quantization_config", None)
-        AutoGPTQQuantLinear = get_auto_gptq_quant_linear(gptq_quantization_config)
-        if (gptq_quantization_config is not None) or (AutoGPTQQuantLinear is not None):
+        GPTQQuantLinear = get_gptqmodel_quant_linear(gptq_quantization_config)
+        if (gptq_quantization_config is not None) or (GPTQQuantLinear is not None):
             raise ValueError(f"GPTQ quantization not supported for {config.peft_type.value} (yet).")
 
         loaded_in_8bit = kwargs.pop("loaded_in_8bit", False)

src/peft/tuners/oft/awq.py

@@ -11,13 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import importlib.metadata as importlib_metadata
 from typing import Any, Optional
 
-import packaging.version
 import torch
 
-from peft.import_utils import is_auto_awq_available
+from peft.import_utils import is_gptqmodel_available
 from peft.tuners.oft.layer import OFTLayer
 from peft.tuners.tuners_utils import BaseTunerLayer
 
@@ -99,20 +97,10 @@ def dispatch_awq(
     else:
         target_base_layer = target
 
-    if is_auto_awq_available():
-        from awq.modules.linear import WQLinear_GEMM
-
-        if isinstance(target_base_layer, WQLinear_GEMM):
-            # Raise the error only at the dispatch level
-            AUTOAWQ_MINIMUM_VERSION = packaging.version.parse("0.2.0")
-            version_autoawq = packaging.version.parse(importlib_metadata.version("autoawq"))
-
-            if AUTOAWQ_MINIMUM_VERSION > version_autoawq:
-                raise ImportError(
-                    f"Found an incompatible version of auto-awq. Found version {version_autoawq}, "
-                    f"but only versions above {AUTOAWQ_MINIMUM_VERSION} are supported for PEFT."
-                )
+    if is_gptqmodel_available():
+        from gptqmodel.nn_modules.qlinear.gemm_awq import AwqGEMMQuantLinear
 
+        if isinstance(target_base_layer, AwqGEMMQuantLinear):
             new_module = AwqOFTLinear(target, adapter_name, **kwargs)
             target.qweight = target_base_layer.qweight
 

src/peft/tuners/oft/gptq.py

@@ -15,10 +15,8 @@
 
 import torch
 
-from peft.import_utils import is_gptqmodel_available
 from peft.tuners.oft.layer import OFTLayer
 from peft.tuners.tuners_utils import BaseTunerLayer
-from peft.utils import get_auto_gptq_quant_linear
 
 
 class GPTQOFTLinear(torch.nn.Module, OFTLayer):
@@ -100,19 +98,10 @@ def dispatch_gptq(
     else:
         target_base_layer = target
 
-    cfg = kwargs.get("gptq_quantization_config", None)
+    from gptqmodel.nn_modules.qlinear import BaseQuantLinear
 
-    if is_gptqmodel_available():
-        from gptqmodel.nn_modules.qlinear import BaseQuantLinear
-
-        if isinstance(target_base_layer, BaseQuantLinear):
-            new_module = GPTQOFTLinear(target, adapter_name, **kwargs)
-            target.qweight = target_base_layer.qweight
-    else:
-        quant_linear = get_auto_gptq_quant_linear(cfg)
-
-        if quant_linear is not None and isinstance(target_base_layer, quant_linear):
-            new_module = GPTQOFTLinear(target, adapter_name, **kwargs)
-            target.qweight = target_base_layer.qweight
+    if isinstance(target_base_layer, BaseQuantLinear):
+        new_module = GPTQOFTLinear(target, adapter_name, **kwargs)
+        target.qweight = target_base_layer.qweight
 
     return new_module

src/peft/tuners/oft/layer.py

@@ -354,7 +354,7 @@ def __init__(self, base_layer: nn.Module, **kwargs) -> None:
         elif hasattr(base_layer, "codebooks") and base_layer.__class__.__name__ == "QuantizedLinear":
             # AQLM QuantLinear
             in_features, out_features = base_layer.in_features, base_layer.out_features
-        elif hasattr(base_layer, "w_bit") and base_layer.__class__.__name__ == "WQLinear_GEMM":
+        elif hasattr(base_layer, "bits") and base_layer.__class__.__name__ == "AwqGEMMQuantLinear":
             # Awq layers
             in_features, out_features = base_layer.in_features, base_layer.out_features
         elif base_layer.__class__.__name__ == "EetqLinear":

src/peft/tuners/tuners_utils.py

@@ -188,7 +188,7 @@ def _get_in_out_features(module: nn.Module) -> tuple[int, int] | tuple[None, Non
     elif hasattr(module, "codebooks") and module.__class__.__name__ == "QuantizedLinear":
         # AQLM QuantLinear
         in_features, out_features = module.in_features, module.out_features
-    elif hasattr(module, "w_bit") and module.__class__.__name__ == "WQLinear_GEMM":
+    elif hasattr(module, "bits") and module.__class__.__name__ == "AwqGEMMQuantLinear":
         # Awq layers
         in_features, out_features = module.in_features, module.out_features
     elif module.__class__.__name__ == "EetqLinear":

src/peft/utils/__init__.py

@@ -55,7 +55,6 @@
     _set_trainable,
     bloom_model_postprocess_past_key_value,
     cast_mixed_precision_params,
-    get_auto_gptq_quant_linear,
     get_gptqmodel_quant_linear,
     get_quantization_config,
     id_tensor_storage,
@@ -114,7 +113,6 @@
     "_set_trainable",
     "bloom_model_postprocess_past_key_value",
     "cast_mixed_precision_params",
-    "get_auto_gptq_quant_linear",
     "get_gptqmodel_quant_linear",
     "get_peft_model_state_dict",
     "get_quantization_config",

src/peft/utils/other.py

@@ -36,7 +36,7 @@
 from safetensors.torch import storage_ptr, storage_size
 from transformers import PreTrainedModel
 
-from ..import_utils import is_auto_gptq_available, is_gptqmodel_available, is_torch_tpu_available
+from ..import_utils import is_gptqmodel_available, is_torch_tpu_available
 from .constants import (
     CONFIG_NAME,
     EMBEDDING_LAYER_NAMES,
@@ -1250,42 +1250,6 @@ def get_quantization_config(model: torch.nn.Module, method: str):
     return None
 
 
-def get_auto_gptq_quant_linear(gptq_quantization_config):
-    """
-    Get the right AutoGPTQQuantLinear class based on the quantization config file
-    """
-    if gptq_quantization_config is None:
-        return None
-
-    if is_auto_gptq_available():
-        from auto_gptq.utils.import_utils import dynamically_import_QuantLinear
-    else:
-        return None
-
-    desc_act = gptq_quantization_config.desc_act
-    group_size = gptq_quantization_config.group_size
-    bits = gptq_quantization_config.bits
-    if hasattr(gptq_quantization_config, "use_exllama"):
-        use_exllama = gptq_quantization_config.use_exllama
-    else:
-        use_exllama = not gptq_quantization_config.disable_exllama
-    if hasattr(gptq_quantization_config, "exllama_config"):
-        exllama_version = gptq_quantization_config.exllama_config["version"]
-    else:
-        exllama_version = 1
-
-    QuantLinear = dynamically_import_QuantLinear(
-        use_triton=False,
-        desc_act=desc_act,
-        group_size=group_size,
-        bits=bits,
-        disable_exllama=not (use_exllama and exllama_version == 1),
-        disable_exllamav2=not (use_exllama and exllama_version == 2),
-    )
-
-    return QuantLinear
-
-
 def get_gptqmodel_quant_linear(gptq_quantization_config, device_map=None):
     """
     Get the right GPTQQuantLinear class based on the quantization config file
@@ -1296,7 +1260,9 @@ def get_gptqmodel_quant_linear(gptq_quantization_config, device_map=None):
     if not is_gptqmodel_available():
         return None
 
-    from gptqmodel.utils.importer import hf_select_quant_linear
+    from gptqmodel import BACKEND
+    from gptqmodel.quantization import METHOD
+    from gptqmodel.utils.importer import hf_select_quant_linear_v2
 
     desc_act = gptq_quantization_config.desc_act
     group_size = gptq_quantization_config.group_size
@@ -1309,15 +1275,17 @@ def get_gptqmodel_quant_linear(gptq_quantization_config, device_map=None):
     sym = gptq_quantization_config.sym
     meta = gptq_quantization_config.meta if hasattr(gptq_quantization_config, "meta") else None
 
-    QuantLinear = hf_select_quant_linear(
+    QuantLinear = hf_select_quant_linear_v2(
         bits=bits,
         group_size=group_size,
         desc_act=desc_act,
         sym=sym,
         device_map=device_map,
-        checkpoint_format=checkpoint_format,
+        format=checkpoint_format,
+        quant_method=METHOD.GPTQ,
         meta=meta,
-        backend="auto_trainable",
+        backend=BACKEND.AUTO_TRAINABLE,
+        pack=False,
     )
 
     return QuantLinear

tests/test_common_gpu.py

@@ -55,7 +55,7 @@
     get_peft_model,
     prepare_model_for_kbit_training,
 )
-from peft.import_utils import is_bnb_4bit_available, is_bnb_available, is_xpu_available
+from peft.import_utils import is_bnb_4bit_available, is_bnb_available, is_gptqmodel_available, is_xpu_available
 from peft.tuners.lora.config import LoraRuntimeConfig
 from peft.utils import infer_device
 
@@ -69,6 +69,9 @@
 )
 
 
+if is_gptqmodel_available():
+    from gptqmodel import BACKEND
+
 if is_bnb_available():
     import bitsandbytes as bnb
 
@@ -524,7 +527,7 @@ def test_lora_gptq_quantization_from_pretrained_safetensors(self):
         from transformers import GPTQConfig
 
         model_id = "marcsun13/opt-350m-gptq-4bit"
-        quantization_config = GPTQConfig(bits=4, use_exllama=False)
+        quantization_config = GPTQConfig(bits=4, backend=BACKEND.AUTO_TRAINABLE)
         kwargs = {
             "pretrained_model_name_or_path": model_id,
             "dtype": torch.float16,