diff --git a/src/peft/tuners/adalora/bnb.py b/src/peft/tuners/adalora/bnb.py
index a89d4b40ac..52b43f0dff 100644
--- a/src/peft/tuners/adalora/bnb.py
+++ b/src/peft/tuners/adalora/bnb.py
@@ -89,6 +89,7 @@ def __init__(
             adapter_name: str,
             config: AdaLoraConfig,
             r: int = 0,
+            lora_alpha: int = 1,
             **kwargs,
         ) -> None:
             super().__init__()
@@ -97,7 +98,7 @@ def __init__(
             self.get_base_layer().weight.requires_grad = False
 
             self._active_adapter = adapter_name
-            self.update_layer(adapter_name, r, config=config)
+            self.update_layer(adapter_name, r, lora_alpha, config=config)
 
         def forward(self, x: torch.Tensor, *args: Any, **kwargs: Any) -> torch.Tensor:
             # note: no check for self.merged because merging is not supported (yet)
diff --git a/src/peft/tuners/adalora/model.py b/src/peft/tuners/adalora/model.py
index 937e4a08ad..6ab2e47b70 100644
--- a/src/peft/tuners/adalora/model.py
+++ b/src/peft/tuners/adalora/model.py
@@ -181,7 +181,7 @@ def _create_new_module(lora_config, adapter_name, target, device_map=None, **kwa
                     "index": target_base_layer.index,
                 }
             )
-            new_module = SVDLinear8bitLt(target, adapter_name, **kwargs)
+            new_module = SVDLinear8bitLt(target, adapter_name, config=lora_config, **kwargs)
         elif loaded_in_4bit and is_bnb_4bit_available() and isinstance(target_base_layer, bnb.nn.Linear4bit):
             fourbit_kwargs = kwargs.copy()
             fourbit_kwargs.update(
@@ -191,7 +191,7 @@ def _create_new_module(lora_config, adapter_name, target, device_map=None, **kwa
                     "quant_type": target_base_layer.weight.quant_type,
                 }
             )
-            new_module = SVDLinear4bit(target, adapter_name, **fourbit_kwargs)
+            new_module = SVDLinear4bit(target, adapter_name, config=lora_config, **fourbit_kwargs)
         elif QuantLinear is not None and isinstance(target, QuantLinear):
             new_module = SVDQuantLinear(target, adapter_name, **kwargs)
         else:
diff --git a/tests/test_gpu_examples.py b/tests/test_gpu_examples.py
index 56b1a3189f..3147464d65 100644
--- a/tests/test_gpu_examples.py
+++ b/tests/test_gpu_examples.py
@@ -4133,7 +4133,7 @@ def test_causal_lm_training_single_gpu_torchao_dora_int8_dynamic_activation_int8
     @pytest.mark.single_gpu_tests
     @pytest.mark.xfail(
         reason="int4_weight_only still has issues",
-        raises=RuntimeError,
+        raises=(RuntimeError, ValueError),
     )
     def test_causal_lm_training_single_gpu_torchao_int4_raises(self):
         # TODO: Once proper torchao support for int4 is added, remove this test and add int4 to supported_quant_types