Refactor layer initialization to use PEFT config directly

src/peft/tuners/adalora/bnb.py

@@ -18,6 +18,7 @@
 
 from peft.import_utils import is_bnb_4bit_available, is_bnb_available
 
+from .config import AdaLoraConfig
 from .layer import AdaLoraLayer
 
 
@@ -29,10 +30,9 @@ def __init__(
             self,
             base_layer: torch.nn.Module,
             adapter_name: str,
+            config: AdaLoraConfig,
             r: int = 0,
             lora_alpha: int = 1,
-            lora_dropout: float = 0.0,
-            init_lora_weights: bool = True,
             **kwargs,
         ) -> None:
             super().__init__()
@@ -41,7 +41,7 @@ def __init__(
             self.get_base_layer().weight.requires_grad = False
 
             self._active_adapter = adapter_name
-            self.update_layer(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights)
+            self.update_layer(adapter_name, r, lora_alpha, config=config)
 
         def forward(self, x: torch.Tensor) -> torch.Tensor:
             # note: no check for self.merged because merging is not supported (yet)
@@ -87,10 +87,8 @@ def __init__(
             self,
             base_layer: torch.nn.Module,
             adapter_name: str,
+            config: AdaLoraConfig,
             r: int = 0,
-            lora_alpha: int = 1,
-            lora_dropout: float = 0.0,
-            init_lora_weights: bool = True,
             **kwargs,
         ) -> None:
             super().__init__()
@@ -99,7 +97,7 @@ def __init__(
             self.get_base_layer().weight.requires_grad = False
 
             self._active_adapter = adapter_name
-            self.update_layer(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights)
+            self.update_layer(adapter_name, r, config=config)
 
         def forward(self, x: torch.Tensor, *args: Any, **kwargs: Any) -> torch.Tensor:
             # note: no check for self.merged because merging is not supported (yet)

src/peft/tuners/adalora/gptq.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 import torch
 
+from .config import AdaLoraConfig
 from .layer import AdaLoraLayer
 
 
@@ -21,10 +22,8 @@ def __init__(
         self,
         base_layer,
         adapter_name,
+        config: AdaLoraConfig,
         r: int = 0,
-        lora_alpha: int = 1,
-        lora_dropout: float = 0.0,
-        init_lora_weights: bool = True,
         **kwargs,
     ) -> None:
         super().__init__()
@@ -34,7 +33,7 @@ def __init__(
         # for backwards compatibility
         self.quant_linear_module = base_layer
         self._active_adapter = adapter_name
-        self.update_layer(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights)
+        self.update_layer(adapter_name, r, config=config)
 
     def forward(self, x: torch.Tensor) -> torch.Tensor:
         result = self.quant_linear_module(x)

src/peft/tuners/adalora/layer.py

@@ -24,6 +24,8 @@
 from peft.tuners.tuners_utils import check_adapters_to_merge
 from peft.utils import transpose
 
+from .config import AdaLoraConfig
+
 
 if packaging.version.parse(transformers.__version__) >= packaging.version.parse("4.33.0"):
     from transformers.integrations import deepspeed_config
@@ -45,9 +47,10 @@ def __init__(self, base_layer: nn.Module) -> None:
         self.lora_B = nn.ParameterDict({})
         self.ranknum = nn.ParameterDict({})
 
-    def update_layer(
-        self, adapter_name, r, lora_alpha, lora_dropout, init_lora_weights, inference_mode: bool = False, **kwargs
-    ):
+    def update_layer(self, adapter_name: str, r: int, lora_alpha: int, config: AdaLoraConfig, **kwargs) -> None:
+        lora_dropout = config.lora_dropout
+        init_lora_weights = config.init_lora_weights
+        inference_mode = config.inference_mode
         if r < 0:
             # note: r == 0 is allowed for AdaLora, see #1539
             raise ValueError(f"`r` should be a positive integer or 0, but the value passed is {r}")
@@ -91,21 +94,19 @@ def __init__(
         self,
         base_layer: nn.Module,
         adapter_name: str,
+        config: AdaLoraConfig,
         r: int = 0,
         lora_alpha: int = 1,
-        lora_dropout: float = 0.0,
-        fan_in_fan_out: bool = False,
-        init_lora_weights: bool = True,
         **kwargs,
     ) -> None:
         super().__init__()
         AdaLoraLayer.__init__(self, base_layer)
         # Freezing the pre-trained weight matrix
         self.get_base_layer().weight.requires_grad = False
 
-        self.fan_in_fan_out = fan_in_fan_out
+        self.fan_in_fan_out = config.fan_in_fan_out
         self._active_adapter = adapter_name
-        self.update_layer(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights)
+        self.update_layer(adapter_name, r, lora_alpha, config=config)
 
     def merge(self, safe_merge: bool = False, adapter_names: Optional[list[str]] = None) -> None:
         """

src/peft/tuners/adalora/model.py

@@ -119,9 +119,6 @@ def _create_and_replace(
         kwargs = {
             "r": lora_config.init_r,
             "lora_alpha": lora_config.lora_alpha,
-            "lora_dropout": lora_config.lora_dropout,
-            "fan_in_fan_out": lora_config.fan_in_fan_out,
-            "init_lora_weights": lora_config.init_lora_weights,
             "loaded_in_8bit": getattr(self.model, "is_loaded_in_8bit", False),
             "loaded_in_4bit": getattr(self.model, "is_loaded_in_4bit", False),
         }
@@ -148,8 +145,7 @@ def _create_and_replace(
                 adapter_name,
                 lora_config.init_r,
                 lora_config.lora_alpha,
-                lora_config.lora_dropout,
-                lora_config.init_lora_weights,
+                config=lora_config,
             )
 
     @staticmethod
@@ -200,25 +196,25 @@ def _create_new_module(lora_config, adapter_name, target, device_map=None, **kwa
             new_module = SVDQuantLinear(target, adapter_name, **kwargs)
         else:
             if isinstance(target_base_layer, torch.nn.Linear):
-                if kwargs["fan_in_fan_out"]:
+                if lora_config.fan_in_fan_out:
                     warnings.warn(
                         "fan_in_fan_out is set to True but the target module is `torch.nn.Linear`. "
                         "Setting fan_in_fan_out to False."
                     )
-                    kwargs["fan_in_fan_out"] = lora_config.fan_in_fan_out = False
+                    lora_config.fan_in_fan_out = False
             elif isinstance(target_base_layer, Conv1D):
-                if not kwargs["fan_in_fan_out"]:
+                if not lora_config.fan_in_fan_out:
                     warnings.warn(
                         "fan_in_fan_out is set to False but the target module is `Conv1D`. "
                         "Setting fan_in_fan_out to True."
                     )
-                    kwargs["fan_in_fan_out"] = lora_config.fan_in_fan_out = True
+                    lora_config.fan_in_fan_out = True
             else:
                 raise ValueError(
                     f"Target module {target} is not supported. "
                     f"Currently, only `torch.nn.Linear` and `Conv1D` are supported."
                 )
-            new_module = SVDLinear(target, adapter_name, **kwargs)
+            new_module = SVDLinear(target, adapter_name, config=lora_config, **kwargs)
 
         return new_module
 

src/peft/tuners/lora/aqlm.py

@@ -20,6 +20,8 @@
 from peft.tuners.lora.layer import LoraLayer
 from peft.tuners.tuners_utils import BaseTunerLayer
 
+from .config import LoraConfig
+
 
 if is_aqlm_available():
     from aqlm import QuantizedLinear
@@ -30,16 +32,12 @@ def __init__(
         self,
         base_layer,
         adapter_name: str,
+        config: LoraConfig,
         r: int = 0,
         lora_alpha: int = 1,
-        lora_dropout: float = 0.0,
-        init_lora_weights: bool = True,
-        use_rslora: bool = False,
-        use_dora: bool = False,
-        lora_bias: bool = False,
         **kwargs,
     ):
-        if use_dora:
+        if config.use_dora:
             raise ValueError(f"{self.__class__.__name__} does not support DoRA yet, please set it to False")
 
         super().__init__()
@@ -50,11 +48,7 @@ def __init__(
             adapter_name,
             r,
             lora_alpha=lora_alpha,
-            lora_dropout=lora_dropout,
-            init_lora_weights=init_lora_weights,
-            use_rslora=use_rslora,
-            use_dora=use_dora,
-            lora_bias=lora_bias,
+            config=config,
         )
 
     def forward(self, x: torch.Tensor):
@@ -98,6 +92,7 @@ def __repr__(self) -> str:
 def dispatch_aqlm(
     target: torch.nn.Module,
     adapter_name: str,
+    config: LoraConfig,
     **kwargs: Any,
 ) -> Optional[torch.nn.Module]:
     new_module = None
@@ -108,7 +103,7 @@ def dispatch_aqlm(
         target_base_layer = target
 
     if is_aqlm_available() and isinstance(target_base_layer, QuantizedLinear):
-        new_module = AqlmLoraLinear(target, adapter_name, **kwargs)
+        new_module = AqlmLoraLinear(target, adapter_name, config=config, **kwargs)
         target.qweight = target_base_layer.codes
 
     return new_module

src/peft/tuners/lora/awq.py

@@ -21,22 +21,20 @@
 from peft.tuners.lora.layer import LoraLayer
 from peft.tuners.tuners_utils import BaseTunerLayer
 
+from .config import LoraConfig
+
 
 class AwqLoraLinear(torch.nn.Module, LoraLayer):
     def __init__(
         self,
         base_layer,
         adapter_name,
+        config: LoraConfig,
         r: int = 0,
         lora_alpha: int = 1,
-        lora_dropout: float = 0.0,
-        init_lora_weights: bool = True,
-        use_rslora: bool = False,
-        use_dora: bool = False,
-        lora_bias: bool = False,
         **kwargs,
     ):
-        if use_dora:
+        if config.use_dora:
             raise ValueError(f"{self.__class__.__name__} does not support DoRA yet, please set it to False")
 
         super().__init__()
@@ -51,11 +49,7 @@ def __init__(
             adapter_name,
             r,
             lora_alpha=lora_alpha,
-            lora_dropout=lora_dropout,
-            init_lora_weights=init_lora_weights,
-            use_rslora=use_rslora,
-            use_dora=use_dora,
-            lora_bias=lora_bias,
+            config=config,
         )
 
     def forward(self, x: torch.Tensor):
@@ -92,6 +86,7 @@ def __repr__(self) -> str:
 def dispatch_awq(
     target: torch.nn.Module,
     adapter_name: str,
+    config: LoraConfig,
     **kwargs: Any,
 ) -> Optional[torch.nn.Module]:
     new_module = None
@@ -115,7 +110,7 @@ def dispatch_awq(
                     f"but only versions above {AUTOAWQ_MINIMUM_VERSION} are supported for PEFT."
                 )
 
-            new_module = AwqLoraLinear(target, adapter_name, **kwargs)
+            new_module = AwqLoraLinear(target, adapter_name, config=config, **kwargs)
             target.qweight = target_base_layer.qweight
 
     return new_module