From 6f1a2026bb6f1e0b048a8b59270dbc9f553ab33d Mon Sep 17 00:00:00 2001 From: Miroslav Goncharenko Date: Thu, 14 Nov 2024 00:32:56 +0100 Subject: [PATCH] readme: replace tabs with spaces --- examples/language-modeling/README.md | 64 ++++++++++++++-------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md index 2bca4551ae..f10e46d757 100644 --- a/examples/language-modeling/README.md +++ b/examples/language-modeling/README.md @@ -475,38 +475,38 @@ python ../gaudi_spawn.py \ - Multi-card finetuning of Llama2-7B with FP8: ```bash PT_HPU_AUTOCAST_LOWER_PRECISION_OPS_LIST=ops_bf16.txt python ../gaudi_spawn.py \ - --world_size 8 --use_mpi run_lora_clm.py \ - --model_name_or_path meta-llama/Llama-2-7b-hf \ - --dataset_name tatsu-lab/alpaca \ - --bf16 True \ - --output_dir ./model_lora_llama \ - --num_train_epochs 3 \ - --per_device_train_batch_size 16 \ - --gradient_accumulation_steps 1 \ - --eval_strategy "no" \ - --save_strategy "no" \ - --learning_rate 3e-4 \ - --warmup_ratio 0.03 \ - --lr_scheduler_type "constant" \ - --max_grad_norm 0.3 \ - --logging_steps 20 \ - --do_train \ - --do_eval \ - --use_habana \ - --use_lazy_mode \ - --throughput_warmup_steps 18 \ - --lora_rank=8 \ - --lora_alpha=16 \ - --lora_dropout=0.05 \ - --lora_target_modules "q_proj" "v_proj" \ - --dataset_concatenation \ - --max_seq_length 512 \ - --ddp_bucket_cap_mb 50 \ - --adam_epsilon 1e-08 \ - --validation_split_percentage 10 \ - --low_cpu_mem_usage True \ - --pipelining_fwd_bwd \ - --fp8 True + --world_size 8 --use_mpi run_lora_clm.py \ + --model_name_or_path meta-llama/Llama-2-7b-hf \ + --dataset_name tatsu-lab/alpaca \ + --bf16 True \ + --output_dir ./model_lora_llama \ + --num_train_epochs 3 \ + --per_device_train_batch_size 16 \ + --gradient_accumulation_steps 1 \ + --eval_strategy "no" \ + --save_strategy "no" \ + --learning_rate 3e-4 \ + --warmup_ratio 0.03 \ + --lr_scheduler_type "constant" \ + --max_grad_norm 0.3 \ + --logging_steps 20 \ + --do_train \ + --do_eval \ + --use_habana \ + --use_lazy_mode \ + --throughput_warmup_steps 18 \ + --lora_rank=8 \ + --lora_alpha=16 \ + --lora_dropout=0.05 \ + --lora_target_modules "q_proj" "v_proj" \ + --dataset_concatenation \ + --max_seq_length 512 \ + --ddp_bucket_cap_mb 50 \ + --adam_epsilon 1e-08 \ + --validation_split_percentage 10 \ + --low_cpu_mem_usage True \ + --pipelining_fwd_bwd \ + --fp8 True ``` - Multi-card finetuning of codegen-16B-mono: