[Fix] Fix Mixtral LoRA setting (#312)

set target_modules
InternLM · Jan 12, 2024 · 8ab2762 · 8ab2762
1 parent 28c0556
commit 8ab2762
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 0 deletions.
diff --git a/xtuner/configs/mixtral/mixtral_8x7b/mixtral_8x7b_qlora_oasst1_e3.py b/xtuner/configs/mixtral/mixtral_8x7b/mixtral_8x7b_qlora_oasst1_e3.py
@@ -78,6 +78,9 @@
         r=64,
         lora_alpha=16,
         lora_dropout=0.1,
+        target_modules=[
+            'q_proj', 'k_proj', 'v_proj', 'o_proj', 'w1', 'w2', 'w3'
+        ],
         bias='none',
         task_type='CAUSAL_LM'))
 

diff --git a/xtuner/configs/mixtral/mixtral_8x7b_instruct/mixtral_8x7b_instruct_qlora_oasst1_e3.py b/xtuner/configs/mixtral/mixtral_8x7b_instruct/mixtral_8x7b_instruct_qlora_oasst1_e3.py
@@ -78,6 +78,9 @@
         r=64,
         lora_alpha=16,
         lora_dropout=0.1,
+        target_modules=[
+            'q_proj', 'k_proj', 'v_proj', 'o_proj', 'w1', 'w2', 'w3'
+        ],
         bias='none',
         task_type='CAUSAL_LM'))