diff --git a/src/peft/tuners/adalora/bnb.py b/src/peft/tuners/adalora/bnb.py index fef3d25e65..a89d4b40ac 100644 --- a/src/peft/tuners/adalora/bnb.py +++ b/src/peft/tuners/adalora/bnb.py @@ -18,6 +18,7 @@ from peft.import_utils import is_bnb_4bit_available, is_bnb_available +from .config import AdaLoraConfig from .layer import AdaLoraLayer @@ -29,10 +30,9 @@ def __init__( self, base_layer: torch.nn.Module, adapter_name: str, + config: AdaLoraConfig, r: int = 0, lora_alpha: int = 1, - lora_dropout: float = 0.0, - init_lora_weights: bool = True, **kwargs, ) -> None: super().__init__() @@ -41,7 +41,7 @@ def __init__( self.get_base_layer().weight.requires_grad = False self._active_adapter = adapter_name - self.update_layer(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights) + self.update_layer(adapter_name, r, lora_alpha, config=config) def forward(self, x: torch.Tensor) -> torch.Tensor: # note: no check for self.merged because merging is not supported (yet) @@ -87,10 +87,8 @@ def __init__( self, base_layer: torch.nn.Module, adapter_name: str, + config: AdaLoraConfig, r: int = 0, - lora_alpha: int = 1, - lora_dropout: float = 0.0, - init_lora_weights: bool = True, **kwargs, ) -> None: super().__init__() @@ -99,7 +97,7 @@ def __init__( self.get_base_layer().weight.requires_grad = False self._active_adapter = adapter_name - self.update_layer(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights) + self.update_layer(adapter_name, r, config=config) def forward(self, x: torch.Tensor, *args: Any, **kwargs: Any) -> torch.Tensor: # note: no check for self.merged because merging is not supported (yet) diff --git a/src/peft/tuners/adalora/gptq.py b/src/peft/tuners/adalora/gptq.py index bed1a0a7ca..32d8f5c2d8 100644 --- a/src/peft/tuners/adalora/gptq.py +++ b/src/peft/tuners/adalora/gptq.py @@ -13,6 +13,7 @@ # limitations under the License. import torch +from .config import AdaLoraConfig from .layer import AdaLoraLayer @@ -21,10 +22,8 @@ def __init__( self, base_layer, adapter_name, + config: AdaLoraConfig, r: int = 0, - lora_alpha: int = 1, - lora_dropout: float = 0.0, - init_lora_weights: bool = True, **kwargs, ) -> None: super().__init__() @@ -34,7 +33,7 @@ def __init__( # for backwards compatibility self.quant_linear_module = base_layer self._active_adapter = adapter_name - self.update_layer(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights) + self.update_layer(adapter_name, r, config=config) def forward(self, x: torch.Tensor) -> torch.Tensor: result = self.quant_linear_module(x) diff --git a/src/peft/tuners/adalora/layer.py b/src/peft/tuners/adalora/layer.py index 635e510551..0a0035682c 100644 --- a/src/peft/tuners/adalora/layer.py +++ b/src/peft/tuners/adalora/layer.py @@ -24,6 +24,8 @@ from peft.tuners.tuners_utils import check_adapters_to_merge from peft.utils import transpose +from .config import AdaLoraConfig + if packaging.version.parse(transformers.__version__) >= packaging.version.parse("4.33.0"): from transformers.integrations import deepspeed_config @@ -45,9 +47,10 @@ def __init__(self, base_layer: nn.Module) -> None: self.lora_B = nn.ParameterDict({}) self.ranknum = nn.ParameterDict({}) - def update_layer( - self, adapter_name, r, lora_alpha, lora_dropout, init_lora_weights, inference_mode: bool = False, **kwargs - ): + def update_layer(self, adapter_name: str, r: int, lora_alpha: int, config: AdaLoraConfig, **kwargs) -> None: + lora_dropout = config.lora_dropout + init_lora_weights = config.init_lora_weights + inference_mode = config.inference_mode if r < 0: # note: r == 0 is allowed for AdaLora, see #1539 raise ValueError(f"`r` should be a positive integer or 0, but the value passed is {r}") @@ -91,11 +94,9 @@ def __init__( self, base_layer: nn.Module, adapter_name: str, + config: AdaLoraConfig, r: int = 0, lora_alpha: int = 1, - lora_dropout: float = 0.0, - fan_in_fan_out: bool = False, - init_lora_weights: bool = True, **kwargs, ) -> None: super().__init__() @@ -103,9 +104,9 @@ def __init__( # Freezing the pre-trained weight matrix self.get_base_layer().weight.requires_grad = False - self.fan_in_fan_out = fan_in_fan_out + self.fan_in_fan_out = config.fan_in_fan_out self._active_adapter = adapter_name - self.update_layer(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights) + self.update_layer(adapter_name, r, lora_alpha, config=config) def merge(self, safe_merge: bool = False, adapter_names: Optional[list[str]] = None) -> None: """ diff --git a/src/peft/tuners/adalora/model.py b/src/peft/tuners/adalora/model.py index c5c345c0ef..937e4a08ad 100644 --- a/src/peft/tuners/adalora/model.py +++ b/src/peft/tuners/adalora/model.py @@ -119,9 +119,6 @@ def _create_and_replace( kwargs = { "r": lora_config.init_r, "lora_alpha": lora_config.lora_alpha, - "lora_dropout": lora_config.lora_dropout, - "fan_in_fan_out": lora_config.fan_in_fan_out, - "init_lora_weights": lora_config.init_lora_weights, "loaded_in_8bit": getattr(self.model, "is_loaded_in_8bit", False), "loaded_in_4bit": getattr(self.model, "is_loaded_in_4bit", False), } @@ -148,8 +145,7 @@ def _create_and_replace( adapter_name, lora_config.init_r, lora_config.lora_alpha, - lora_config.lora_dropout, - lora_config.init_lora_weights, + config=lora_config, ) @staticmethod @@ -200,25 +196,25 @@ def _create_new_module(lora_config, adapter_name, target, device_map=None, **kwa new_module = SVDQuantLinear(target, adapter_name, **kwargs) else: if isinstance(target_base_layer, torch.nn.Linear): - if kwargs["fan_in_fan_out"]: + if lora_config.fan_in_fan_out: warnings.warn( "fan_in_fan_out is set to True but the target module is `torch.nn.Linear`. " "Setting fan_in_fan_out to False." ) - kwargs["fan_in_fan_out"] = lora_config.fan_in_fan_out = False + lora_config.fan_in_fan_out = False elif isinstance(target_base_layer, Conv1D): - if not kwargs["fan_in_fan_out"]: + if not lora_config.fan_in_fan_out: warnings.warn( "fan_in_fan_out is set to False but the target module is `Conv1D`. " "Setting fan_in_fan_out to True." ) - kwargs["fan_in_fan_out"] = lora_config.fan_in_fan_out = True + lora_config.fan_in_fan_out = True else: raise ValueError( f"Target module {target} is not supported. " f"Currently, only `torch.nn.Linear` and `Conv1D` are supported." ) - new_module = SVDLinear(target, adapter_name, **kwargs) + new_module = SVDLinear(target, adapter_name, config=lora_config, **kwargs) return new_module diff --git a/src/peft/tuners/lora/aqlm.py b/src/peft/tuners/lora/aqlm.py index 81c7cdbb4e..fd7dec3991 100644 --- a/src/peft/tuners/lora/aqlm.py +++ b/src/peft/tuners/lora/aqlm.py @@ -20,6 +20,8 @@ from peft.tuners.lora.layer import LoraLayer from peft.tuners.tuners_utils import BaseTunerLayer +from .config import LoraConfig + if is_aqlm_available(): from aqlm import QuantizedLinear @@ -30,16 +32,12 @@ def __init__( self, base_layer, adapter_name: str, + config: LoraConfig, r: int = 0, lora_alpha: int = 1, - lora_dropout: float = 0.0, - init_lora_weights: bool = True, - use_rslora: bool = False, - use_dora: bool = False, - lora_bias: bool = False, **kwargs, ): - if use_dora: + if config.use_dora: raise ValueError(f"{self.__class__.__name__} does not support DoRA yet, please set it to False") super().__init__() @@ -50,11 +48,7 @@ def __init__( adapter_name, r, lora_alpha=lora_alpha, - lora_dropout=lora_dropout, - init_lora_weights=init_lora_weights, - use_rslora=use_rslora, - use_dora=use_dora, - lora_bias=lora_bias, + config=config, ) def forward(self, x: torch.Tensor): @@ -98,6 +92,7 @@ def __repr__(self) -> str: def dispatch_aqlm( target: torch.nn.Module, adapter_name: str, + config: LoraConfig, **kwargs: Any, ) -> Optional[torch.nn.Module]: new_module = None @@ -108,7 +103,7 @@ def dispatch_aqlm( target_base_layer = target if is_aqlm_available() and isinstance(target_base_layer, QuantizedLinear): - new_module = AqlmLoraLinear(target, adapter_name, **kwargs) + new_module = AqlmLoraLinear(target, adapter_name, config=config, **kwargs) target.qweight = target_base_layer.codes return new_module diff --git a/src/peft/tuners/lora/awq.py b/src/peft/tuners/lora/awq.py index 61eb487ad6..fe950e098e 100644 --- a/src/peft/tuners/lora/awq.py +++ b/src/peft/tuners/lora/awq.py @@ -21,22 +21,20 @@ from peft.tuners.lora.layer import LoraLayer from peft.tuners.tuners_utils import BaseTunerLayer +from .config import LoraConfig + class AwqLoraLinear(torch.nn.Module, LoraLayer): def __init__( self, base_layer, adapter_name, + config: LoraConfig, r: int = 0, lora_alpha: int = 1, - lora_dropout: float = 0.0, - init_lora_weights: bool = True, - use_rslora: bool = False, - use_dora: bool = False, - lora_bias: bool = False, **kwargs, ): - if use_dora: + if config.use_dora: raise ValueError(f"{self.__class__.__name__} does not support DoRA yet, please set it to False") super().__init__() @@ -51,11 +49,7 @@ def __init__( adapter_name, r, lora_alpha=lora_alpha, - lora_dropout=lora_dropout, - init_lora_weights=init_lora_weights, - use_rslora=use_rslora, - use_dora=use_dora, - lora_bias=lora_bias, + config=config, ) def forward(self, x: torch.Tensor): @@ -92,6 +86,7 @@ def __repr__(self) -> str: def dispatch_awq( target: torch.nn.Module, adapter_name: str, + config: LoraConfig, **kwargs: Any, ) -> Optional[torch.nn.Module]: new_module = None @@ -115,7 +110,7 @@ def dispatch_awq( f"but only versions above {AUTOAWQ_MINIMUM_VERSION} are supported for PEFT." ) - new_module = AwqLoraLinear(target, adapter_name, **kwargs) + new_module = AwqLoraLinear(target, adapter_name, config=config, **kwargs) target.qweight = target_base_layer.qweight return new_module diff --git a/src/peft/tuners/lora/bnb.py b/src/peft/tuners/lora/bnb.py index 632abf5a66..c6f4e76958 100644 --- a/src/peft/tuners/lora/bnb.py +++ b/src/peft/tuners/lora/bnb.py @@ -24,7 +24,7 @@ from peft.utils.integrations import dequantize_bnb_weight from peft.utils.other import transpose -from .config import ArrowConfig +from .config import LoraConfig from .layer import LoraLayer, LoraVariant @@ -38,15 +38,9 @@ def __init__( self, base_layer: torch.nn.Module, adapter_name: str, + config: LoraConfig, r: int = 0, lora_alpha: int = 1, - lora_dropout: float = 0.0, - init_lora_weights: bool = True, - use_rslora: bool = False, - use_alora: bool = False, - use_dora: bool = False, - arrow_config: ArrowConfig = None, - lora_bias: bool = False, **kwargs, ) -> None: super().__init__() @@ -58,24 +52,17 @@ def __init__( adapter_name, r, lora_alpha=lora_alpha, - lora_dropout=lora_dropout, - init_lora_weights=init_lora_weights, - use_rslora=use_rslora, - use_dora=use_dora, - use_alora=use_alora, - lora_bias=lora_bias, - arrow_config=arrow_config, + config=config, ) - def resolve_lora_variant( - self, *, arrow_config: ArrowConfig, use_dora: bool, use_alora: bool, **kwargs - ) -> Optional[LoraVariant]: - if arrow_config is not None: + def resolve_lora_variant(self, *, config, **kwargs) -> Optional[LoraVariant]: + if config.arrow_config is not None: from .variants import ArrowLinearVariant return ArrowLinearVariant() - if not use_dora and not use_alora: + use_alora = config.alora_invocation_tokens is not None + if not config.use_dora and not use_alora: return None from .variants import ALoraLinearVariant, DoraLinearVariant @@ -296,7 +283,7 @@ def __repr__(self) -> str: rep = super().__repr__() return "lora." + rep - def dispatch_bnb_8bit(target: torch.nn.Module, adapter_name: str, **kwargs): + def dispatch_bnb_8bit(target: torch.nn.Module, adapter_name: str, config: LoraConfig, **kwargs): new_module = None if isinstance(target, BaseTunerLayer): @@ -314,7 +301,7 @@ def dispatch_bnb_8bit(target: torch.nn.Module, adapter_name: str, **kwargs): "index": target.index, } ) - new_module = Linear8bitLt(target, adapter_name, **eightbit_kwargs) + new_module = Linear8bitLt(target, adapter_name, config=config, **eightbit_kwargs) return new_module @@ -327,14 +314,9 @@ def __init__( self, base_layer: torch.nn.Module, adapter_name: str, + config: LoraConfig, r: int = 0, lora_alpha: int = 1, - lora_dropout: float = 0.0, - init_lora_weights: bool = True, - use_rslora: bool = False, - use_dora: bool = False, - arrow_config: ArrowConfig = None, - lora_bias: bool = False, **kwargs, ) -> None: super().__init__() @@ -346,23 +328,17 @@ def __init__( adapter_name, r, lora_alpha=lora_alpha, - lora_dropout=lora_dropout, - init_lora_weights=init_lora_weights, - use_rslora=use_rslora, - use_dora=use_dora, - lora_bias=lora_bias, - arrow_config=arrow_config, + config=config, ) - def resolve_lora_variant( - self, *, arrow_config: ArrowConfig, use_dora: bool, use_alora: bool, **kwargs - ) -> Optional[LoraVariant]: - if arrow_config is not None: + def resolve_lora_variant(self, *, config: LoraConfig, **kwargs) -> Optional[LoraVariant]: + if config.arrow_config is not None: from .variants import ArrowLinearVariant return ArrowLinearVariant() - if not use_dora and not use_alora: + use_alora = config.alora_invocation_tokens is not None + if not config.use_dora and not use_alora: return None from .variants import ALoraLinearVariant, DoraLinearVariant @@ -588,7 +564,7 @@ def __repr__(self) -> str: rep = super().__repr__() return "lora." + rep - def dispatch_bnb_4bit(target: torch.nn.Module, adapter_name: str, **kwargs): + def dispatch_bnb_4bit(target: torch.nn.Module, adapter_name: str, config: LoraConfig, **kwargs): new_module = None if isinstance(target, BaseTunerLayer): @@ -606,6 +582,6 @@ def dispatch_bnb_4bit(target: torch.nn.Module, adapter_name: str, **kwargs): "quant_type": target_base_layer.weight.quant_type, } ) - new_module = Linear4bit(target, adapter_name, **fourbit_kwargs) + new_module = Linear4bit(target, adapter_name, config=config, **fourbit_kwargs) return new_module diff --git a/src/peft/tuners/lora/config.py b/src/peft/tuners/lora/config.py index b4008f090a..c80567a92b 100644 --- a/src/peft/tuners/lora/config.py +++ b/src/peft/tuners/lora/config.py @@ -14,6 +14,7 @@ from __future__ import annotations +import importlib import warnings from dataclasses import dataclass, field from typing import Literal, Optional, Union @@ -806,8 +807,6 @@ def __post_init__(self): # handle init_lora_weights and loftq_config if self.init_lora_weights == "loftq": - import importlib - if not importlib.util.find_spec("scipy"): raise ImportError("The required package 'scipy' is not installed. Please install it to continue.") if not self.loftq_config: diff --git a/src/peft/tuners/lora/eetq.py b/src/peft/tuners/lora/eetq.py index b864d9fba2..e255786174 100644 --- a/src/peft/tuners/lora/eetq.py +++ b/src/peft/tuners/lora/eetq.py @@ -19,6 +19,8 @@ from peft.tuners.lora.layer import LoraLayer from peft.tuners.tuners_utils import BaseTunerLayer +from .config import LoraConfig + if is_eetq_available(): from eetq import EetqLinear @@ -28,16 +30,12 @@ def __init__( self, base_layer, adapter_name, + config: LoraConfig, r: int = 0, lora_alpha: int = 1, - lora_dropout: float = 0.0, - init_lora_weights: bool = True, - use_rslora: bool = False, - use_dora: bool = False, - lora_bias: bool = False, **kwargs, ): - if use_dora: + if config.use_dora: raise ValueError(f"{self.__class__.__name__} does not support DoRA yet, please set it to False") super().__init__() @@ -52,11 +50,7 @@ def __init__( adapter_name, r, lora_alpha=lora_alpha, - lora_dropout=lora_dropout, - init_lora_weights=init_lora_weights, - use_rslora=use_rslora, - use_dora=use_dora, - lora_bias=lora_bias, + config=config, ) def forward(self, x: torch.Tensor): @@ -99,6 +93,7 @@ def __repr__(self) -> str: def dispatch_eetq( target: torch.nn.Module, adapter_name: str, + config: LoraConfig, **kwargs: Any, ) -> Optional[torch.nn.Module]: new_module = None @@ -109,7 +104,7 @@ def dispatch_eetq( target_base_layer = target if is_eetq_available() and isinstance(target_base_layer, EetqLinear): - new_module = EetqLoraLinear(target, adapter_name, **kwargs) + new_module = EetqLoraLinear(target, adapter_name, config=config, **kwargs) target.weight = target_base_layer.weight if hasattr(target, "bias"): diff --git a/src/peft/tuners/lora/eva.py b/src/peft/tuners/lora/eva.py index 1bc75453b1..c7be0c86ce 100644 --- a/src/peft/tuners/lora/eva.py +++ b/src/peft/tuners/lora/eva.py @@ -16,7 +16,7 @@ from collections import Counter, defaultdict from collections.abc import Iterable, Mapping from contextlib import nullcontext -from copy import deepcopy +from copy import copy, deepcopy from functools import partial from itertools import cycle from typing import Optional, Union @@ -491,14 +491,7 @@ def _load_eva_state_dict( eva_state_dict: dict, adapter_name: str, ): - peft_config = model.peft_config[adapter_name] - update_layer_kwargs = { - "adapter_name": adapter_name, - "lora_dropout": peft_config.lora_dropout, - "use_rslora": peft_config.use_rslora, - "use_dora": peft_config.use_dora, - "lora_bias": peft_config.lora_bias, - } + peft_config = copy(model.peft_config[adapter_name]) missing_eva_inits = [] new_target_modules = [] other_module_names = [] @@ -525,11 +518,13 @@ def _load_eva_state_dict( if peft_config.eva_config.adjust_scaling_factors: alpha *= new_rank / r if new_rank != r or module.lora_A[adapter_name].weight.device.type == "meta": - module.update_layer(r=new_rank, lora_alpha=alpha, init_lora_weights="eva", **update_layer_kwargs) + peft_config.init_lora_weights = "eva" + module.update_layer(adapter_name=adapter_name, r=new_rank, lora_alpha=alpha, config=peft_config) module.lora_A[adapter_name].weight.copy_(w) new_target_modules.append(name_in_base_model) else: - module.update_layer(r=r, lora_alpha=alpha, init_lora_weights=True, **update_layer_kwargs) + peft_config.init_lora_weights = True + module.update_layer(adapter_name=adapter_name, r=r, lora_alpha=alpha, config=peft_config) missing_eva_inits.append(name_in_base_model) new_rank = r # update rank pattern and alpha pattern diff --git a/src/peft/tuners/lora/gptq.py b/src/peft/tuners/lora/gptq.py index 7ff40bc9ef..4453d109bc 100644 --- a/src/peft/tuners/lora/gptq.py +++ b/src/peft/tuners/lora/gptq.py @@ -20,6 +20,7 @@ from peft.tuners.tuners_utils import BaseTunerLayer from peft.utils import get_auto_gptq_quant_linear +from .config import LoraConfig from .layer import LoraVariant @@ -28,21 +29,15 @@ def __init__( self, base_layer, adapter_name: str, + config: LoraConfig, r: int = 0, lora_alpha: int = 1, - lora_dropout: float = 0.0, - init_lora_weights: bool = True, - use_rslora: bool = False, - use_dora: bool = False, - use_qalora: bool = False, - lora_bias: bool = False, - qalora_group_size: int = 32, **kwargs, ): super().__init__() LoraLayer.__init__(self, base_layer) - if use_dora: + if config.use_dora: raise ValueError(f"{self.__class__.__name__} does not support DoRA yet, please set it to False") # self.base_layer and self.quant_linear_module are the same; we need the former for consistency and the latter @@ -53,25 +48,19 @@ def __init__( adapter_name, r, lora_alpha=lora_alpha, - lora_dropout=lora_dropout, - init_lora_weights=init_lora_weights, - use_rslora=use_rslora, - use_dora=use_dora, - use_qalora=use_qalora, - lora_bias=lora_bias, - qalora_group_size=qalora_group_size, + config=config, ) - def resolve_lora_variant(self, *, use_dora: bool, use_qalora: bool, **kwargs) -> Optional[LoraVariant]: - if use_dora and use_qalora: + def resolve_lora_variant(self, *, config: LoraConfig, **kwargs) -> Optional[LoraVariant]: + if config.use_dora and config.use_qalora: raise NotImplementedError( - f"Dora and QA_lora at the same time is not supported for {self.__class__.__name__} (yet)." + f"DoRA and QA-LoRA at the same time is not supported for {self.__class__.__name__} (yet)." ) - elif use_dora: + elif config.use_dora: from .variants import DoraLinearVariant variant = DoraLinearVariant() - elif use_qalora: + elif config.use_qalora: from .variants import QALoraLinearVariant variant = QALoraLinearVariant() @@ -127,6 +116,7 @@ def __repr__(self) -> str: def dispatch_gptq( target: torch.nn.Module, adapter_name: str, + config: LoraConfig, **kwargs: Any, ) -> Optional[torch.nn.Module]: new_module = None @@ -142,13 +132,13 @@ def dispatch_gptq( from gptqmodel.nn_modules.qlinear import BaseQuantLinear if isinstance(target_base_layer, BaseQuantLinear): - new_module = GPTQLoraLinear(target, adapter_name, **kwargs) + new_module = GPTQLoraLinear(target, adapter_name, config=config, **kwargs) target.qweight = target_base_layer.qweight else: quant_linear = get_auto_gptq_quant_linear(cfg) if quant_linear is not None and isinstance(target_base_layer, quant_linear): - new_module = GPTQLoraLinear(target, adapter_name, **kwargs) + new_module = GPTQLoraLinear(target, adapter_name, config=config, **kwargs) target.qweight = target_base_layer.qweight return new_module diff --git a/src/peft/tuners/lora/hqq.py b/src/peft/tuners/lora/hqq.py index 924acb2d4d..31f9a3f893 100644 --- a/src/peft/tuners/lora/hqq.py +++ b/src/peft/tuners/lora/hqq.py @@ -23,6 +23,7 @@ from peft.tuners.tuners_utils import BaseTunerLayer, check_adapters_to_merge from peft.utils.other import transpose +from .config import LoraConfig from .layer import LoraLayer, LoraVariant @@ -35,16 +36,12 @@ def __init__( self, base_layer: torch.nn.Module, adapter_name: str, + config: LoraConfig, r: int = 0, lora_alpha: int = 1, - lora_dropout: float = 0.0, - init_lora_weights: bool = True, - use_rslora: bool = False, - use_dora: bool = False, - lora_bias: bool = False, **kwargs, ) -> None: - if lora_bias: + if config.lora_bias: raise ValueError(f"{self.__class__.__name__} does not support lora_bias yet, set it to False") super().__init__() @@ -56,15 +53,11 @@ def __init__( adapter_name, r, lora_alpha=lora_alpha, - lora_dropout=lora_dropout, - init_lora_weights=init_lora_weights, - use_rslora=use_rslora, - use_dora=use_dora, - lora_bias=lora_bias, + config=config, ) - def resolve_lora_variant(self, *, use_dora: bool, **kwargs) -> Optional[LoraVariant]: - if not use_dora: + def resolve_lora_variant(self, *, config: LoraConfig, **kwargs) -> Optional[LoraVariant]: + if not config.use_dora: return None from .variants import DoraLinearVariant @@ -237,7 +230,7 @@ def __repr__(self) -> str: return "lora." + rep -def dispatch_hqq(target: torch.nn.Module, adapter_name: str, **kwargs): +def dispatch_hqq(target: torch.nn.Module, adapter_name: str, config: LoraConfig, **kwargs): new_module = None if isinstance(target, BaseTunerLayer): @@ -246,6 +239,6 @@ def dispatch_hqq(target: torch.nn.Module, adapter_name: str, **kwargs): target_base_layer = target if is_hqq_available() and isinstance(target_base_layer, HQQLinear): - new_module = HqqLoraLinear(target_base_layer, adapter_name, **kwargs) + new_module = HqqLoraLinear(target_base_layer, adapter_name, config=config, **kwargs) return new_module diff --git a/src/peft/tuners/lora/inc.py b/src/peft/tuners/lora/inc.py index e9fea9dc26..60d92ea851 100644 --- a/src/peft/tuners/lora/inc.py +++ b/src/peft/tuners/lora/inc.py @@ -23,6 +23,7 @@ from peft.import_utils import is_inc_available from peft.tuners.tuners_utils import BaseTunerLayer +from .config import LoraConfig from .layer import Linear @@ -33,9 +34,10 @@ def __init__( self, base_layer: torch.nn.Module, adapter_name: str, + config: LoraConfig, **kwargs, ): - super().__init__(base_layer, adapter_name, **kwargs) + super().__init__(base_layer, adapter_name, config=config, **kwargs) def merge(self, safe_merge: bool = False, adapter_names: Optional[list[str]] = None) -> None: """ @@ -59,7 +61,7 @@ def unmerge(self) -> None: raise NotImplementedError("Unmerging LoRA from INC layers is not yet implemented") -def dispatch_inc(target: torch.nn.Module, adapter_name: str, **kwargs): +def dispatch_inc(target: torch.nn.Module, adapter_name: str, config: LoraConfig, **kwargs): new_module = None if isinstance(target, BaseTunerLayer): @@ -73,6 +75,6 @@ def dispatch_inc(target: torch.nn.Module, adapter_name: str, **kwargs): ) if isinstance(target_base_layer, PatchedLinear): - new_module = IncLoraLinear(target, adapter_name, **kwargs) + new_module = IncLoraLinear(target, adapter_name, config=config, **kwargs) return new_module diff --git a/src/peft/tuners/lora/layer.py b/src/peft/tuners/lora/layer.py index 13b5bd03cd..489c8e85ef 100644 --- a/src/peft/tuners/lora/layer.py +++ b/src/peft/tuners/lora/layer.py @@ -32,10 +32,11 @@ get_bnb_param_type, skip_init_on_device, ) +from peft.utils.loftq_utils import loftq_init from peft.utils.other import transpose from peft.utils.warning import PeftWarning -from .config import ArrowConfig, LoraConfig +from .config import LoraConfig VARIANT_KWARG_KEYS = ["alora_offsets"] @@ -130,7 +131,7 @@ def __init__(self, base_layer: nn.Module, ephemeral_gpu_offload: bool = False, * def _get_in_out_features(self, module: nn.Module) -> tuple[int, int] | tuple[None, None]: return _get_in_out_features(module) - def resolve_lora_variant(self, *, use_dora: bool, use_bdlora=None, **kwargs) -> Optional[LoraVariant]: + def resolve_lora_variant(self, *, config: LoraConfig, **kwargs) -> Optional[LoraVariant]: """Return a matching LoRA variant for this layer type. Given the init arguments of this layer, return the correct LoRA variant, if any. E.g., if `use_dora=True`, this @@ -146,27 +147,20 @@ def resolve_lora_variant(self, *, use_dora: bool, use_bdlora=None, **kwargs) -> def update_layer( self, - adapter_name, - r, - lora_alpha, - lora_dropout, - init_lora_weights, - use_rslora, - use_dora: bool = False, - use_alora: bool = False, - use_qalora: bool = False, - lora_bias: bool = False, - arrow_config: ArrowConfig = None, - qalora_group_size: int = 32, - inference_mode: bool = False, - lora_ga_config=None, - use_bdlora=None, + adapter_name: str, + r: int, + lora_alpha: int, + config: LoraConfig, **kwargs, - ): + ) -> None: # collect the kwargs + lora_dropout = config.lora_dropout + init_lora_weights = config.init_lora_weights + use_rslora = config.use_rslora + lora_bias = config.lora_bias + inference_mode = config.inference_mode + target_name = kwargs.get("target_name", "") # preserve target_name before overwriting kwargs - kwargs = locals().copy() - del kwargs["self"] kwargs["target_name"] = target_name # restore target_name # This code works for linear layers, override for other layer types @@ -180,14 +174,7 @@ def update_layer( PeftWarning, ) - lora_variant = self.resolve_lora_variant( - use_dora=use_dora, - use_alora=use_alora, - use_qalora=use_qalora, - qalora_group_size=qalora_group_size, - arrow_config=arrow_config, - use_bdlora=use_bdlora, - ) + lora_variant = self.resolve_lora_variant(config=config) if lora_variant is not None: self.lora_variant[adapter_name] = lora_variant @@ -212,7 +199,7 @@ def update_layer( self.use_rslora[adapter_name] = use_rslora - self.use_dora[adapter_name] = use_dora + self.use_dora[adapter_name] = config.use_dora # for inits that require access to the base weight, use gather_param_ctx so that the weight is gathered when using DeepSpeed if isinstance(init_lora_weights, str) and init_lora_weights.startswith("pissa"): @@ -226,7 +213,7 @@ def update_layer( self.olora_init(adapter_name) elif init_lora_weights == "loftq": with gather_params_ctx(self.get_base_layer().weight): - self.loftq_init(adapter_name) + self.loftq_init(adapter_name, config) elif init_lora_weights == "eva": nn.init.zeros_(self.lora_B[adapter_name].weight) elif init_lora_weights == "orthogonal": @@ -234,14 +221,14 @@ def update_layer( self.orthogonal_init(adapter_name) elif init_lora_weights == "lora_ga": with gather_params_ctx(self.get_base_layer().weight): - self.lora_ga_init(adapter_name, lora_ga_config) + self.lora_ga_init(adapter_name, config.lora_ga_config) elif init_lora_weights: self.reset_lora_parameters(adapter_name, init_lora_weights) # call this before init of the lora variants self._move_adapter_to_device_of_base_layer(adapter_name) if adapter_name in self.lora_variant: - self.lora_variant[adapter_name].init(self, **kwargs) + self.lora_variant[adapter_name].init(self, adapter_name=adapter_name, config=config, **kwargs) self.set_adapter(self.active_adapters, inference_mode=inference_mode) @@ -445,14 +432,12 @@ def corda_init(self, adapter_name, init_lora_weights): # Remove redundant fields del linear.eigens - def loftq_init(self, adapter_name): - from peft.utils.loftq_utils import loftq_init - + def loftq_init(self, adapter_name, config: LoraConfig): weight = self.get_base_layer().weight kwargs = { - "num_bits": self.kwargs.get("loftq_bits", 4), + "num_bits": config.loftq_config["loftq_bits"], "reduced_rank": self.r[adapter_name], - "num_iter": self.kwargs.get("loftq_iter", 1), + "num_iter": config.loftq_config["loftq_iter"], } qweight, lora_A, lora_B = loftq_init(weight, **kwargs) @@ -743,57 +728,39 @@ def __init__( self, base_layer, adapter_name: str, + config: LoraConfig, r: int = 0, lora_alpha: int = 1, - lora_dropout: float = 0.0, - fan_in_fan_out: bool = False, # Set this to True if the layer to replace stores weight like (fan_in, fan_out) is_target_conv_1d_layer: bool = False, - init_lora_weights: Union[bool, str] = True, - use_rslora: bool = False, - use_dora: bool = False, - use_alora: bool = False, - arrow_config: ArrowConfig = None, - use_bdlora=None, - lora_bias: bool = False, - lora_ga_config=None, **kwargs, ) -> None: super().__init__() LoraLayer.__init__(self, base_layer, **kwargs) - self.fan_in_fan_out = fan_in_fan_out + self.fan_in_fan_out = config.fan_in_fan_out self._active_adapter = adapter_name self.update_layer( adapter_name, r, lora_alpha=lora_alpha, - lora_dropout=lora_dropout, - init_lora_weights=init_lora_weights, - use_rslora=use_rslora, - use_dora=use_dora, - use_alora=use_alora, - lora_bias=lora_bias, - arrow_config=arrow_config, - lora_ga_config=lora_ga_config, - use_bdlora=use_bdlora, + config=config, **kwargs, ) self.is_target_conv_1d_layer = is_target_conv_1d_layer - def resolve_lora_variant( - self, *, arrow_config: ArrowConfig, use_dora: bool, use_alora: bool, use_bdlora=None, **kwargs - ) -> Optional[LoraVariant]: - if arrow_config is not None: + def resolve_lora_variant(self, config: LoraConfig, **kwargs) -> Optional[LoraVariant]: + if config.arrow_config is not None: from .variants import ArrowLinearVariant return ArrowLinearVariant() - if use_bdlora is not None: + if config.use_bdlora is not None: from .variants import BdLoraLinearVariant return BdLoraLinearVariant() - if not use_dora and not use_alora: + use_alora = config.alora_invocation_tokens is not None + if not config.use_dora and not use_alora: return None from .variants import ALoraLinearVariant, DoraLinearVariant @@ -984,42 +951,30 @@ def __init__( self, base_layer: nn.Module, adapter_name: str, + config: LoraConfig, r: int = 0, lora_alpha: int = 1, - lora_dropout: float = 0.0, - fan_in_fan_out: bool = False, # Set this to True if the layer to replace stores weight like (fan_in, fan_out) init_lora_weights: Union[bool, str] = True, - use_rslora: bool = False, - use_dora: bool = False, - arrow_config: ArrowConfig = None, - lora_bias: bool = False, - lora_ga_config=None, **kwargs, ) -> None: - if lora_bias: + if config.lora_bias: # lora_bias=True is not supported (yet) for embedding layers, as they use nn.Parameter - raise ValueError(f"lora_bias={lora_bias} is not supported for {self.__class__.__name__}.") + raise ValueError(f"lora_bias={config.lora_bias} is not supported for {self.__class__.__name__}.") super().__init__() LoraLayer.__init__(self, base_layer) - self.fan_in_fan_out = fan_in_fan_out + self.fan_in_fan_out = config.fan_in_fan_out self._active_adapter = adapter_name self.update_layer( adapter_name, r, lora_alpha=lora_alpha, - lora_dropout=lora_dropout, - init_lora_weights=init_lora_weights, - use_rslora=use_rslora, - use_dora=use_dora, - lora_bias=lora_bias, - arrow_config=arrow_config, - lora_ga_config=lora_ga_config, + config=config, ) - def resolve_lora_variant(self, *, use_dora: bool, **kwargs) -> Optional[LoraVariant]: - if not use_dora: + def resolve_lora_variant(self, *, config: LoraConfig, **kwargs) -> Optional[LoraVariant]: + if not config.use_dora: return None from .variants import DoraEmbeddingVariant @@ -1028,26 +983,22 @@ def resolve_lora_variant(self, *, use_dora: bool, **kwargs) -> Optional[LoraVari def update_layer( self, - adapter_name, - r, - lora_alpha, - lora_dropout, - init_lora_weights, - use_rslora, - use_dora, - lora_bias, - arrow_config: ArrowConfig = None, - inference_mode: bool = False, + adapter_name: str, + r: int, + lora_alpha: int, + config: LoraConfig, **kwargs, - ): - # collect the kwargs - kwargs = locals().copy() - del kwargs["self"] + ) -> None: + lora_dropout = config.lora_dropout + init_lora_weights = config.init_lora_weights + use_rslora = config.use_rslora + lora_bias = config.lora_bias + inference_mode = config.inference_mode if r <= 0: raise ValueError(f"`r` should be a positive integer value but the value passed is {r}") - lora_variant = self.resolve_lora_variant(use_dora=use_dora, arrow_config=arrow_config) + lora_variant = self.resolve_lora_variant(config=config) if lora_variant is not None: self.lora_variant[adapter_name] = lora_variant @@ -1073,7 +1024,7 @@ def update_layer( self.use_rslora[adapter_name] = use_rslora - self.use_dora[adapter_name] = use_dora + self.use_dora[adapter_name] = config.use_dora if init_lora_weights == "loftq": self.loftq_init(adapter_name) @@ -1087,7 +1038,7 @@ def update_layer( self._move_adapter_to_device_of_base_layer(adapter_name) if adapter_name in self.lora_variant: - self.lora_variant[adapter_name].init(self, **kwargs) + self.lora_variant[adapter_name].init(self, adapter_name=adapter_name, config=config, **kwargs) self.set_adapter(self.active_adapters, inference_mode=inference_mode) @@ -1300,15 +1251,9 @@ def __init__( self, base_layer: nn.Module, adapter_name: str, + config: LoraConfig, r: int = 0, lora_alpha: int = 1, - lora_dropout: float = 0.0, - init_lora_weights: Union[bool, str] = True, - use_rslora: bool = False, - use_dora: bool = False, - arrow_config: ArrowConfig = None, - lora_bias: bool = False, - lora_ga_config=None, **kwargs, ) -> None: super().__init__() @@ -1332,32 +1277,22 @@ def __init__( adapter_name, r, lora_alpha=lora_alpha, - lora_dropout=lora_dropout, - init_lora_weights=init_lora_weights, - use_rslora=use_rslora, - use_dora=use_dora, - lora_bias=lora_bias, - arrow_config=arrow_config, - lora_ga_config=lora_ga_config, + config=config, ) def update_layer( self, - adapter_name, - r, - lora_alpha, - lora_dropout, - init_lora_weights, - use_rslora, - use_dora, - lora_bias, - arrow_config: ArrowConfig = None, - inference_mode: bool = False, + adapter_name: str, + r: int, + lora_alpha: int, + config: LoraConfig, **kwargs, - ): - # collect the kwargs - kwargs = locals().copy() - del kwargs["self"] + ) -> None: + lora_dropout = config.lora_dropout + init_lora_weights = config.init_lora_weights + use_rslora = config.use_rslora + lora_bias = config.lora_bias + inference_mode = config.inference_mode if r <= 0: raise ValueError(f"`r` should be a positive integer value but the value passed is {r}") @@ -1369,7 +1304,7 @@ def update_layer( PeftWarning, ) - lora_variant = self.resolve_lora_variant(use_dora=use_dora, arrow_config=arrow_config) + lora_variant = self.resolve_lora_variant(config=config) if lora_variant is not None: self.lora_variant[adapter_name] = lora_variant @@ -1401,7 +1336,7 @@ def update_layer( self.use_rslora[adapter_name] = use_rslora - self.use_dora[adapter_name] = use_dora + self.use_dora[adapter_name] = config.use_dora if init_lora_weights == "loftq": self.loftq_init(adapter_name) @@ -1415,7 +1350,7 @@ def update_layer( self._move_adapter_to_device_of_base_layer(adapter_name) if adapter_name in self.lora_variant: - self.lora_variant[adapter_name].init(self, **kwargs) + self.lora_variant[adapter_name].init(self, adapter_name=adapter_name, config=config, **kwargs) self.set_adapter(self.active_adapters, inference_mode=inference_mode) @@ -1616,8 +1551,8 @@ def __init__(self, *args, **kwargs): raise ValueError(f"Conv2d layer kernel must have 4 dimensions, not {self._kernel_dim}") self.conv_fn = F.conv2d - def resolve_lora_variant(self, *, use_dora: bool, **kwargs) -> Optional[LoraVariant]: - if not use_dora: + def resolve_lora_variant(self, *, config: LoraConfig, **kwargs) -> Optional[LoraVariant]: + if not config.use_dora: return None from .variants import DoraConv2dVariant @@ -1633,8 +1568,8 @@ def __init__(self, *args, **kwargs): raise ValueError(f"Conv1d layer kernel must have 3 dimensions, not {self._kernel_dim}") self.conv_fn = F.conv1d - def resolve_lora_variant(self, *, use_dora: bool, **kwargs) -> Optional[LoraVariant]: - if not use_dora: + def resolve_lora_variant(self, *, config: LoraConfig, **kwargs) -> Optional[LoraVariant]: + if not config.use_dora: return None from .variants import DoraConv1dVariant @@ -1650,8 +1585,8 @@ def __init__(self, *args, **kwargs): raise ValueError(f"Conv3d layer kernel must have 5 dimensions, not {self._kernel_dim}") self.conv_fn = F.conv3d - def resolve_lora_variant(self, *, use_dora: bool, **kwargs) -> Optional[LoraVariant]: - if not use_dora: + def resolve_lora_variant(self, *, config: LoraConfig, **kwargs) -> Optional[LoraVariant]: + if not config.use_dora: return None from .variants import DoraConv3dVariant @@ -1679,13 +1614,9 @@ def __init__( self, base_layer, adapter_name: str, + config: LoraConfig, r: int = 0, lora_alpha: int = 1, - lora_dropout: float = 0.0, - init_lora_weights: Union[bool, str] = True, - use_rslora: bool = False, - use_dora: bool = False, - lora_ga_config=None, **kwargs, ) -> None: # TODO work with separate weights @@ -1695,7 +1626,7 @@ def __init__( raise ValueError( f"Only same embed for query/key/value is supported as of now for {self.__class__.__name__}." ) - if use_dora: + if config.use_dora: # TODO: probably not so hard to implement raise ValueError(f"{self.__class__.__name__} does not support DoRA (yet), please set use_dora to False") if kwargs.get("use_alora", False): @@ -1710,20 +1641,14 @@ def __init__( adapter_name, r=r, lora_alpha=lora_alpha, - lora_dropout=lora_dropout, - init_lora_weights=init_lora_weights, - use_rslora=use_rslora, - use_dora=use_dora, - lora_ga_config=lora_ga_config, + config=config, **kwargs, ) else: raise ValueError(f"out_proj must be an instance of nn.Linear for {self.__class__.__name__}.") self._active_adapter = adapter_name - self.update_layer( - adapter_name, r, lora_alpha, lora_dropout, init_lora_weights, use_rslora, lora_ga_config=lora_ga_config - ) + self.update_layer(adapter_name, r, lora_alpha=lora_alpha, config=config) @property def embed_dim(self) -> int: @@ -2077,47 +2002,36 @@ def __init__( base_layer, adapter_name: str, parameter_name: str, + config: LoraConfig, r: int = 0, lora_alpha: int = 1, - lora_dropout: float = 0.0, - fan_in_fan_out: bool = False, # Set this to True if the layer to replace stores weight like (fan_in, fan_out) is_target_conv_1d_layer: bool = False, - init_lora_weights: Union[bool, str] = True, - use_rslora: bool = False, - use_dora: bool = False, - lora_bias: bool = False, - lora_ga_config=None, **kwargs, ) -> None: self.parameter_name = parameter_name super().__init__() LoraLayer.__init__(self, base_layer, **kwargs) - if lora_dropout: + if config.lora_dropout: # It's not possible to factor out x from lora_B(lora_A(dropout(x))), so dropout can't be correctly # implemented raise ValueError(f"lora.{self.__class__.__name__} does not work with lora_dropout != 0.") - if fan_in_fan_out: + if config.fan_in_fan_out: raise ValueError(f"lora.{self.__class__.__name__} does not work with fan_in_fan_out.") - if lora_bias: + if config.lora_bias: raise ValueError(f"lora.{self.__class__.__name__} does not work with lora_bias=True.") - if use_dora: + if config.use_dora: raise ValueError(f"lora.{self.__class__.__name__} does not work with use_dora=True.") if is_target_conv_1d_layer: raise ValueError(f"lora.{self.__class__.__name__} does not work with is_target_conv_1d_layer=True.") - self.fan_in_fan_out = fan_in_fan_out + self.fan_in_fan_out = config.fan_in_fan_out self._active_adapter = adapter_name self.update_layer( adapter_name, r, lora_alpha=lora_alpha, - lora_dropout=lora_dropout, - init_lora_weights=init_lora_weights, - use_rslora=use_rslora, - use_dora=use_dora, - lora_bias=lora_bias, - lora_ga_config=lora_ga_config, + config=config, ) def _get_in_out_features(self, module: nn.Module) -> tuple[int, int] | tuple[None, None]: @@ -2139,32 +2053,24 @@ def _get_in_out_features(self, module: nn.Module) -> tuple[int, int] | tuple[Non def update_layer( self, - adapter_name, - r, - lora_alpha, - lora_dropout, - init_lora_weights, - use_rslora, - use_dora: bool = False, - use_qalora: bool = False, - lora_bias: bool = False, - qalora_group_size: int = 32, - inference_mode: bool = False, - lora_ga_config=None, + adapter_name: str, + r: int, + lora_alpha: int, + config: LoraConfig, **kwargs, - ): + ) -> None: # same method as in lora.Linear but taking into account that there can be multiple experts (3d parameter) - # collect the kwargs - kwargs = locals().copy() - del kwargs["self"] + lora_dropout = config.lora_dropout + init_lora_weights = config.init_lora_weights + use_rslora = config.use_rslora + lora_bias = config.lora_bias + inference_mode = config.inference_mode # This code works for linear layers, override for other layer types if r <= 0: raise ValueError(f"`r` should be a positive integer value but the value passed is {r}") - lora_variant = self.resolve_lora_variant( - use_dora=use_dora, use_qalora=use_qalora, qalora_group_size=qalora_group_size - ) + lora_variant = self.resolve_lora_variant(config=config) if lora_variant is not None: raise ValueError(f"lora.{self.__class__.__name__} does not work with LoRA variants like DoRA.") @@ -2192,7 +2098,7 @@ def update_layer( self.use_rslora[adapter_name] = use_rslora - self.use_dora[adapter_name] = use_dora + self.use_dora[adapter_name] = config.use_dora # for inits that require access to the base weight, use gather_param_ctx so that the weight is gathered when using DeepSpeed if isinstance(init_lora_weights, str) and init_lora_weights.startswith("pissa"): @@ -2214,14 +2120,14 @@ def update_layer( self.orthogonal_init(adapter_name) elif init_lora_weights == "lora_ga": with gather_params_ctx(self.get_base_layer().weight): - self.lora_ga_init(adapter_name, lora_ga_config) + self.lora_ga_init(adapter_name, config.lora_ga_config) elif init_lora_weights: self.reset_lora_parameters(adapter_name, init_lora_weights) # call this before init of the lora variants self._move_adapter_to_device_of_base_layer(adapter_name) if adapter_name in self.lora_variant: - self.lora_variant[adapter_name].init(self, **kwargs) + self.lora_variant[adapter_name].init(self, config=config, **kwargs) self.set_adapter(self.active_adapters, inference_mode=inference_mode) @@ -2422,7 +2328,7 @@ def __repr__(self) -> str: def dispatch_default( target: torch.nn.Module, adapter_name: str, - lora_config: LoraConfig, + config: LoraConfig, parameter_name: Optional[str] = None, **kwargs, ) -> Optional[torch.nn.Module]: @@ -2434,40 +2340,31 @@ def dispatch_default( target_base_layer = target if parameter_name is not None: - new_module = ParamWrapper(target, adapter_name, parameter_name=parameter_name, **kwargs) + new_module = ParamWrapper(target, adapter_name, parameter_name=parameter_name, config=config, **kwargs) elif isinstance(target_base_layer, torch.nn.Embedding): - embedding_kwargs = kwargs.copy() - embedding_kwargs.pop("fan_in_fan_out", None) - embedding_kwargs.update(lora_config.loftq_config) - new_module = Embedding(target, adapter_name, **embedding_kwargs) + new_module = Embedding(target, adapter_name, config=config, **kwargs) elif isinstance(target_base_layer, torch.nn.Conv2d): - kwargs.update(lora_config.loftq_config) - new_module = Conv2d(target, adapter_name, **kwargs) + new_module = Conv2d(target, adapter_name, config=config, **kwargs) elif isinstance(target_base_layer, torch.nn.Conv3d): - kwargs.update(lora_config.loftq_config) - new_module = Conv3d(target, adapter_name, **kwargs) + new_module = Conv3d(target, adapter_name, config=config, **kwargs) elif isinstance(target_base_layer, nn.Conv1d): - kwargs.update(lora_config.loftq_config) - new_module = Conv1d(target, adapter_name, **kwargs) + new_module = Conv1d(target, adapter_name, config=config, **kwargs) elif isinstance(target_base_layer, torch.nn.MultiheadAttention): - kwargs.update(lora_config.loftq_config) - new_module = MultiheadAttention(target, adapter_name, **kwargs) + new_module = MultiheadAttention(target, adapter_name, config=config, **kwargs) elif isinstance(target_base_layer, torch.nn.Linear): - if kwargs["fan_in_fan_out"]: + if config.fan_in_fan_out: warnings.warn( "fan_in_fan_out is set to True but the target module is `torch.nn.Linear`. " "Setting fan_in_fan_out to False." ) - kwargs["fan_in_fan_out"] = lora_config.fan_in_fan_out = False - kwargs.update(lora_config.loftq_config) - new_module = Linear(target, adapter_name, **kwargs) + config.fan_in_fan_out = False + new_module = Linear(target, adapter_name, config=config, **kwargs) elif isinstance(target_base_layer, Conv1D): - if not kwargs["fan_in_fan_out"]: + if not config.fan_in_fan_out: warnings.warn( "fan_in_fan_out is set to False but the target module is `Conv1D`. Setting fan_in_fan_out to True." ) - kwargs["fan_in_fan_out"] = lora_config.fan_in_fan_out = True - kwargs.update(lora_config.loftq_config) - new_module = Linear(target, adapter_name, is_target_conv_1d_layer=True, **kwargs) + config.fan_in_fan_out = True + new_module = Linear(target, adapter_name, is_target_conv_1d_layer=True, config=config, **kwargs) return new_module diff --git a/src/peft/tuners/lora/model.py b/src/peft/tuners/lora/model.py index e3c032d44d..b79b3c2566 100644 --- a/src/peft/tuners/lora/model.py +++ b/src/peft/tuners/lora/model.py @@ -205,19 +205,6 @@ def _create_and_replace( kwargs = { "r": r, "lora_alpha": alpha, - "lora_dropout": lora_config.lora_dropout, - "fan_in_fan_out": lora_config.fan_in_fan_out, - "init_lora_weights": lora_config.init_lora_weights, - "use_rslora": lora_config.use_rslora, - "use_dora": lora_config.use_dora, - "use_alora": lora_config.alora_invocation_tokens is not None, - "use_qalora": lora_config.use_qalora, - "qalora_group_size": lora_config.qalora_group_size, - "ephemeral_gpu_offload": lora_config.runtime_config.ephemeral_gpu_offload, - "lora_bias": lora_config.lora_bias, - "arrow_config": lora_config.arrow_config, - "lora_ga_config": lora_config.lora_ga_config, - "use_bdlora": lora_config.use_bdlora, "target_name": current_key, "loaded_in_8bit": getattr(self.model, "is_loaded_in_8bit", False), "loaded_in_4bit": getattr(self.model, "is_loaded_in_4bit", False), @@ -248,16 +235,8 @@ def _create_and_replace( adapter_name, r, lora_alpha=alpha, - lora_dropout=lora_config.lora_dropout, - init_lora_weights=lora_config.init_lora_weights, - use_rslora=lora_config.use_rslora, - use_dora=lora_config.use_dora, - lora_bias=lora_config.lora_bias, - arrow_config=lora_config.arrow_config, - lora_ga_config=lora_config.lora_ga_config, - use_bdlora=lora_config.use_bdlora, - inference_mode=lora_config.inference_mode, target_name=current_key, + config=lora_config, ) else: if isinstance(target, ParamWrapper) and (parameter_name == target.parameter_name): @@ -309,7 +288,7 @@ def _create_new_module(lora_config, adapter_name, target, **kwargs): if lora_config._custom_modules: # Experimental custom LoRA module support. Allows users to pass a custom mapping for unsupported layer # types by impelementing their own LoRA layers. - def dynamic_dispatch_func(target, adapter_name, lora_config, **kwargs): + def dynamic_dispatch_func(target, adapter_name, config, **kwargs): new_module = None if isinstance(target, BaseTunerLayer): @@ -317,9 +296,9 @@ def dynamic_dispatch_func(target, adapter_name, lora_config, **kwargs): else: target_base_layer = target - for key, custom_cls in lora_config._custom_modules.items(): + for key, custom_cls in config._custom_modules.items(): if isinstance(target_base_layer, key): - new_module = custom_cls(target, adapter_name, **kwargs) + new_module = custom_cls(target, adapter_name, config=config, **kwargs) break return new_module @@ -353,7 +332,7 @@ def dynamic_dispatch_func(target, adapter_name, lora_config, **kwargs): new_module = None for dispatcher in dispatchers: - new_module = dispatcher(target, adapter_name, lora_config=lora_config, **kwargs) + new_module = dispatcher(target, adapter_name, config=lora_config, **kwargs) if new_module is not None: # first match wins break diff --git a/src/peft/tuners/lora/torchao.py b/src/peft/tuners/lora/torchao.py index 5e7240a053..c3329a948a 100644 --- a/src/peft/tuners/lora/torchao.py +++ b/src/peft/tuners/lora/torchao.py @@ -32,7 +32,7 @@ class TorchaoLoraLinear(Linear): def __init__(self, *args, get_apply_tensor_subclass, **kwargs): # this is not strictly necessary, as kwargs are stored either way, but we want to error early if # get_apply_tensor_subclass is missing. - if kwargs.get("lora_bias", False): + if kwargs["config"].lora_bias: raise ValueError(f"{self.__class__.__name__} does not support lora_bias yet, set it to False") super().__init__(*args, **kwargs) @@ -131,7 +131,7 @@ def __repr__(self) -> str: def dispatch_torchao( target: torch.nn.Module, adapter_name: str, - lora_config: LoraConfig, + config: LoraConfig, **kwargs: Any, ) -> Optional[torch.nn.Module]: new_module = None @@ -151,6 +151,6 @@ def dispatch_torchao( from torchao.quantization import LinearActivationQuantizedTensor if isinstance(target_base_layer.weight, (AffineQuantizedTensor, LinearActivationQuantizedTensor)): - new_module = TorchaoLoraLinear(target, adapter_name, **kwargs) + new_module = TorchaoLoraLinear(target, adapter_name, config=config, **kwargs) return new_module diff --git a/src/peft/tuners/lora/tp_layer.py b/src/peft/tuners/lora/tp_layer.py index 7edd4d3a6f..e5b7e07e85 100644 --- a/src/peft/tuners/lora/tp_layer.py +++ b/src/peft/tuners/lora/tp_layer.py @@ -16,7 +16,7 @@ import importlib import math import warnings -from typing import Any, Optional, Union +from typing import Any, Optional import torch import torch.nn as nn @@ -26,6 +26,7 @@ from peft.utils import transpose from peft.utils.integrations import gather_params_ctx +from .config import LoraConfig from .layer import LoraLayer @@ -41,30 +42,25 @@ def __init__( self, base_layer, adapter_name: str, + config: LoraConfig, backend, r: int = 0, lora_alpha: int = 1, - lora_dropout: float = 0.0, - fan_in_fan_out: bool = False, is_target_conv_1d_layer: bool = False, - init_lora_weights: Union[bool, str] = True, - use_rslora: bool = False, - use_dora: bool = False, - lora_bias: bool = False, **kwargs, ): - if lora_bias: + if config.lora_bias: raise ValueError(f"{self.__class__.__name__} does not support lora_bias yet, set it to False") super().__init__() LoraLayer.__init__(self, base_layer=base_layer, **kwargs) - if use_dora: + if config.use_dora: raise ValueError(f"{self.__class__.__name__} does not support DoRA yet, please set it to False") self.backend = backend self.is_parallel_a = isinstance(base_layer, backend.RowParallelLinear) - self.fan_in_fan_out = fan_in_fan_out + self.fan_in_fan_out = config.fan_in_fan_out self._active_adapter = adapter_name megatron_config = kwargs["megatron_config"] @@ -82,10 +78,7 @@ def __init__( adapter_name, r, lora_alpha=lora_alpha, - lora_dropout=lora_dropout, - init_lora_weights=init_lora_weights, - use_rslora=use_rslora, - use_dora=use_dora, + config=config, init_method=init_method, input_is_parallel=input_is_parallel, gather_output=gather_output, @@ -100,22 +93,20 @@ def __init__( def update_layer( self, - adapter_name, - r, - lora_alpha, - lora_dropout, - init_lora_weights, - use_rslora, - use_dora=False, + adapter_name: str, + r: int, + lora_alpha: int, + config: LoraConfig, init_method=init.xavier_normal_, - input_is_parallel=True, - gather_output=False, + input_is_parallel: bool = True, + gather_output: bool = False, inference_mode: bool = False, **parallel_linear_kwargs, - ): - # collect the kwargs - kwargs = locals().copy() - del kwargs["self"] + ) -> None: + lora_dropout = config.lora_dropout + init_lora_weights = config.init_lora_weights + use_rslora = config.use_rslora + use_dora = config.use_dora if r <= 0: raise ValueError(f"`r` should be a positive integer value but the value passed is {r}") @@ -181,7 +172,7 @@ def update_layer( self._move_adapter_to_device_of_base_layer(adapter_name) if adapter_name in self.lora_variant: - self.lora_variant[adapter_name].init(self, **kwargs) + self.lora_variant[adapter_name].init(self, adapter_name=adapter_name, config=config) self.set_adapter(self.active_adapters, inference_mode=inference_mode) @@ -311,7 +302,7 @@ def __repr__(self) -> str: def dispatch_megatron( target: torch.nn.Module, adapter_name: str, - lora_config, + config: LoraConfig, **kwargs: Any, ) -> Optional[torch.nn.Module]: new_module = None @@ -321,8 +312,8 @@ def dispatch_megatron( else: target_base_layer = target - if lora_config.megatron_config: - megatron_core = importlib.import_module(lora_config.megatron_core) + if config.megatron_config: + megatron_core = importlib.import_module(config.megatron_core) else: megatron_core = None @@ -331,10 +322,10 @@ def dispatch_megatron( (megatron_core.tensor_parallel.ColumnParallelLinear, megatron_core.tensor_parallel.RowParallelLinear), ): megatron_kwargs = kwargs.copy() - megatron_config = lora_config.megatron_config + megatron_config = config.megatron_config if isinstance(megatron_config, dict): transformer_config_class = megatron_core.transformer.transformer_config.TransformerConfig - megatron_config = transformer_config_class(**lora_config.megatron_config) + megatron_config = transformer_config_class(**config.megatron_config) megatron_kwargs["megatron_config"] = megatron_config if megatron_kwargs["fan_in_fan_out"]: warnings.warn( @@ -342,9 +333,13 @@ def dispatch_megatron( "or `RowParallelLinear`. " "Setting fan_in_fan_out to False." ) - megatron_kwargs["fan_in_fan_out"] = lora_config.fan_in_fan_out = False + megatron_kwargs["fan_in_fan_out"] = config.fan_in_fan_out = False new_module = LoraParallelLinear( - base_layer=target, adapter_name=adapter_name, backend=megatron_core.tensor_parallel, **megatron_kwargs + base_layer=target, + adapter_name=adapter_name, + config=config, + backend=megatron_core.tensor_parallel, + **megatron_kwargs, ) return new_module diff --git a/src/peft/tuners/lora/variants.py b/src/peft/tuners/lora/variants.py index bf07224ab3..59643d5236 100644 --- a/src/peft/tuners/lora/variants.py +++ b/src/peft/tuners/lora/variants.py @@ -25,14 +25,14 @@ from peft.utils.other import transpose from .arrow import ArrowLoraLinearLayer -from .config import PeftConfig +from .config import LoraConfig, PeftConfig from .dora import DoraConv1dLayer, DoraConv2dLayer, DoraConv3dLayer, DoraEmbeddingLayer, DoraLinearLayer from .layer import Conv1d, Conv2d, Conv3d, Embedding, Linear, LoraVariant, _ConvNd class ArrowLinearVariant(LoraVariant): @staticmethod - def init(module: Linear, adapter_name: str, **kwargs): + def init(module: Linear, adapter_name: str, config: LoraConfig, **kwargs): """ Initialise the ArrowLoraLinearLayer() inside lora_arrow. lora_arrow is nn.ModuleDict(), serving as a container for ArrowLoraLinearLayer(). A layer of the base model with LoRA adapter loaded on it will be like: @@ -51,7 +51,7 @@ def init(module: Linear, adapter_name: str, **kwargs): The adapter_name is "arrow_router" by default, set in create_arrow_model() in ./arrow.py """ # Checking for arrow necessary config - arrow_config = kwargs.get("arrow_config") + arrow_config = config.arrow_config if arrow_config is None: raise ValueError("ArrowLinearVariant.init() did not receive an arrow_config") @@ -432,49 +432,43 @@ def forward( class DoraConv1dVariant(_DoraConvNdVariant): @staticmethod - def init(module: Conv1d, adapter_name: str, **kwargs: Any) -> None: + def init(module: Conv1d, adapter_name: str, config: LoraConfig, **kwargs: Any) -> None: dora_layer = DoraConv1dLayer(fan_in_fan_out=False) _DoraConvNdVariant.init_convd_variant(module, adapter_name, dora_layer=dora_layer) class DoraConv2dVariant(_DoraConvNdVariant): @staticmethod - def init(module: Conv2d, adapter_name: str, **kwargs: Any) -> None: + def init(module: Conv2d, adapter_name: str, config: LoraConfig, **kwargs: Any) -> None: dora_layer = DoraConv2dLayer(fan_in_fan_out=False) _DoraConvNdVariant.init_convd_variant(module, adapter_name, dora_layer=dora_layer) class DoraConv3dVariant(_DoraConvNdVariant): @staticmethod - def init(module: Conv3d, adapter_name: str, **kwargs: Any) -> None: + def init(module: Conv3d, adapter_name: str, config: LoraConfig, **kwargs: Any) -> None: dora_layer = DoraConv3dLayer(fan_in_fan_out=False) _DoraConvNdVariant.init_convd_variant(module, adapter_name, dora_layer=dora_layer) class QALoraLinearVariant(LoraVariant): @staticmethod - def init(module: Linear, adapter_name: str, **kwargs: Any) -> None: + def init(module: Linear, adapter_name: str, config: LoraConfig, **kwargs: Any) -> None: """ Initializes QALoRA specific parameters for a given adapter. Args: module (Linear): The linear module to be adapted. adapter_name (str): The name of the adapter. + config (LoraConfig): The config of the LoRA adapter. **kwargs: Additional keyword arguments. - qalora_group_size (int): The size of groups for pooling. This is expected to be passed. """ - if "qalora_group_size" not in kwargs: - raise ValueError( - "`use_qalora=True` requires 'qalora_group_size' to be provided in kwargs." - " Please ensure it is passed from the LoraConfig." - ) - - if module.in_features is not None and module.in_features % kwargs["qalora_group_size"] != 0: + qalora_group_size = config.qalora_group_size + if module.in_features is not None and module.in_features % qalora_group_size != 0: raise ValueError( f"`use_qalora=True` requires `module.in_features` ({module.in_features}) to be" - f"divisible by 'qalora_group_size' ({kwargs['qalora_group_size']})" + f"divisible by 'qalora_group_size' ({qalora_group_size})" ) - qalora_group_size = kwargs["qalora_group_size"] if "qalora_group_size" not in module.other_param_names: module.other_param_names = module.other_param_names + ("qalora_group_size",) @@ -555,7 +549,7 @@ def forward( class ALoraLinearVariant(LoraVariant): @staticmethod - def init(module: Linear, adapter_name: str, **kwargs: Any) -> None: + def init(module: Linear, adapter_name: str, config: LoraConfig, **kwargs: Any) -> None: pass @staticmethod @@ -824,8 +818,8 @@ def weight_as_blockdiagonal_matrix(self): class BdLoraLinearVariant(LoraVariant): @staticmethod - def init(module: Linear, adapter_name: str, **kwargs) -> None: - use_bdlora = kwargs.get("use_bdlora") + def init(module: Linear, adapter_name: str, config: LoraConfig, **kwargs) -> None: + use_bdlora = config.use_bdlora target_name = kwargs.get("target_name", "") # Handle case where use_bdlora is a dict (from saved config) instead of BdLoraConfig object