From 02e545a8ec8dd8102406e10008c73b64f58efa69 Mon Sep 17 00:00:00 2001 From: nemo Date: Thu, 22 Jan 2026 15:24:44 +0100 Subject: [PATCH 1/2] Fix initialization bug introduced in #2960 The quantized adalora SVD layers were not initialized in the new config-passing scheme and therefore raised errors in the GPU tests. For reproduction run ``` make tests_examples_single_gpu ``` which will yield ``` FAILED tests/test_gpu_examples.py::PeftBnbGPUExampleTests::test_4bit_adalora_causalLM - TypeError: SVDLinear4bit.__init__() missing 1 required positional argument: 'config' FAILED tests/test_gpu_examples.py::PeftBnbGPUExampleTests::test_8bit_adalora_causalLM - TypeError: SVDLinear8bitLt.__init__() missing 1 required positional argument: 'config' ``` --- src/peft/tuners/adalora/bnb.py | 3 ++- src/peft/tuners/adalora/model.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/peft/tuners/adalora/bnb.py b/src/peft/tuners/adalora/bnb.py index a89d4b40ac..52b43f0dff 100644 --- a/src/peft/tuners/adalora/bnb.py +++ b/src/peft/tuners/adalora/bnb.py @@ -89,6 +89,7 @@ def __init__( adapter_name: str, config: AdaLoraConfig, r: int = 0, + lora_alpha: int = 1, **kwargs, ) -> None: super().__init__() @@ -97,7 +98,7 @@ def __init__( self.get_base_layer().weight.requires_grad = False self._active_adapter = adapter_name - self.update_layer(adapter_name, r, config=config) + self.update_layer(adapter_name, r, lora_alpha, config=config) def forward(self, x: torch.Tensor, *args: Any, **kwargs: Any) -> torch.Tensor: # note: no check for self.merged because merging is not supported (yet) diff --git a/src/peft/tuners/adalora/model.py b/src/peft/tuners/adalora/model.py index 937e4a08ad..6ab2e47b70 100644 --- a/src/peft/tuners/adalora/model.py +++ b/src/peft/tuners/adalora/model.py @@ -181,7 +181,7 @@ def _create_new_module(lora_config, adapter_name, target, device_map=None, **kwa "index": target_base_layer.index, } ) - new_module = SVDLinear8bitLt(target, adapter_name, **kwargs) + new_module = SVDLinear8bitLt(target, adapter_name, config=lora_config, **kwargs) elif loaded_in_4bit and is_bnb_4bit_available() and isinstance(target_base_layer, bnb.nn.Linear4bit): fourbit_kwargs = kwargs.copy() fourbit_kwargs.update( @@ -191,7 +191,7 @@ def _create_new_module(lora_config, adapter_name, target, device_map=None, **kwa "quant_type": target_base_layer.weight.quant_type, } ) - new_module = SVDLinear4bit(target, adapter_name, **fourbit_kwargs) + new_module = SVDLinear4bit(target, adapter_name, config=lora_config, **fourbit_kwargs) elif QuantLinear is not None and isinstance(target, QuantLinear): new_module = SVDQuantLinear(target, adapter_name, **kwargs) else: From c3dde16adc1a62be0df4eee0ff085f00c7b11891 Mon Sep 17 00:00:00 2001 From: nemo Date: Thu, 22 Jan 2026 15:55:30 +0100 Subject: [PATCH 2/2] Fix a small error with pytorch 2.10 Now it seems that a ValueError is raised indicating that the lora linear implementation only supports 8 bit for now. --- tests/test_gpu_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_gpu_examples.py b/tests/test_gpu_examples.py index 56b1a3189f..3147464d65 100644 --- a/tests/test_gpu_examples.py +++ b/tests/test_gpu_examples.py @@ -4133,7 +4133,7 @@ def test_causal_lm_training_single_gpu_torchao_dora_int8_dynamic_activation_int8 @pytest.mark.single_gpu_tests @pytest.mark.xfail( reason="int4_weight_only still has issues", - raises=RuntimeError, + raises=(RuntimeError, ValueError), ) def test_causal_lm_training_single_gpu_torchao_int4_raises(self): # TODO: Once proper torchao support for int4 is added, remove this test and add int4 to supported_quant_types