From 8990fb3dd354b62a20dbd972bbb0388e6a431a10 Mon Sep 17 00:00:00 2001 From: Raghav Hrishikeshan Mukundan Date: Wed, 29 Oct 2025 09:15:58 -0700 Subject: [PATCH 1/3] Bug fix for Warmup Iters Signed-off-by: Raghav Hrishikeshan Mukundan --- .../configs/qwen3/qwen3_235b_a22b_llm_pretrain.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/performance/configs/qwen3/qwen3_235b_a22b_llm_pretrain.yaml b/scripts/performance/configs/qwen3/qwen3_235b_a22b_llm_pretrain.yaml index 20657c4f5aa..9463caccb07 100644 --- a/scripts/performance/configs/qwen3/qwen3_235b_a22b_llm_pretrain.yaml +++ b/scripts/performance/configs/qwen3/qwen3_235b_a22b_llm_pretrain.yaml @@ -25,6 +25,9 @@ ConfigContainer: train_iters: 100 eval_iters: 0 + scheduler: + lr_warmup_iters: 10 + rerun_state_machine: check_for_nan_in_loss: false From 4a48b1a562810f335dba8fda4c2e25fdb9f3df5e Mon Sep 17 00:00:00 2001 From: Raghav Hrishikeshan Mukundan Date: Wed, 29 Oct 2025 09:20:13 -0700 Subject: [PATCH 2/3] Updating other workloads with the fix Signed-off-by: Raghav Hrishikeshan Mukundan --- .../performance/configs/deepseek/deepseek_v3_llm_pretrain.yaml | 3 +++ .../performance/configs/llama3/llama3_70b_llm_pretrain.yaml | 3 +++ scripts/performance/configs/llama3/llama3_8b_llm_pretrain.yaml | 3 +++ .../performance/configs/qwen3/qwen3_30b_a3b_llm_pretrain.yaml | 3 +++ 4 files changed, 12 insertions(+) diff --git a/scripts/performance/configs/deepseek/deepseek_v3_llm_pretrain.yaml b/scripts/performance/configs/deepseek/deepseek_v3_llm_pretrain.yaml index 7f56df49acf..044d99536e1 100644 --- a/scripts/performance/configs/deepseek/deepseek_v3_llm_pretrain.yaml +++ b/scripts/performance/configs/deepseek/deepseek_v3_llm_pretrain.yaml @@ -24,6 +24,9 @@ ConfigContainer: train: train_iters: 100 eval_iters: 0 + + scheduler: + lr_warmup_iters: 10 rerun_state_machine: check_for_nan_in_loss: false diff --git a/scripts/performance/configs/llama3/llama3_70b_llm_pretrain.yaml b/scripts/performance/configs/llama3/llama3_70b_llm_pretrain.yaml index d59a11c3a85..11588a42311 100644 --- a/scripts/performance/configs/llama3/llama3_70b_llm_pretrain.yaml +++ b/scripts/performance/configs/llama3/llama3_70b_llm_pretrain.yaml @@ -25,6 +25,9 @@ ConfigContainer: train_iters: 100 eval_iters: 0 + scheduler: + lr_warmup_iters: 10 + rerun_state_machine: check_for_nan_in_loss: false diff --git a/scripts/performance/configs/llama3/llama3_8b_llm_pretrain.yaml b/scripts/performance/configs/llama3/llama3_8b_llm_pretrain.yaml index c987e0b712e..56f21c5698a 100644 --- a/scripts/performance/configs/llama3/llama3_8b_llm_pretrain.yaml +++ b/scripts/performance/configs/llama3/llama3_8b_llm_pretrain.yaml @@ -20,6 +20,9 @@ ConfigContainer: train: train_iters: 100 eval_iters: 0 + + scheduler: + lr_warmup_iters: 10 rerun_state_machine: check_for_nan_in_loss: false diff --git a/scripts/performance/configs/qwen3/qwen3_30b_a3b_llm_pretrain.yaml b/scripts/performance/configs/qwen3/qwen3_30b_a3b_llm_pretrain.yaml index 92d5d529ee1..55ec607e43b 100644 --- a/scripts/performance/configs/qwen3/qwen3_30b_a3b_llm_pretrain.yaml +++ b/scripts/performance/configs/qwen3/qwen3_30b_a3b_llm_pretrain.yaml @@ -24,6 +24,9 @@ ConfigContainer: train: train_iters: 100 eval_iters: 0 + + scheduler: + lr_warmup_iters: 10 rerun_state_machine: check_for_nan_in_loss: false From 2fdf0550e2c6fb28fe533a1e36e701c3c39a9636 Mon Sep 17 00:00:00 2001 From: Raghav Hrishikeshan Mukundan Date: Wed, 29 Oct 2025 09:22:10 -0700 Subject: [PATCH 3/3] Adding the fix to 405b Signed-off-by: Raghav Hrishikeshan Mukundan --- .../performance/configs/llama31/llama31_405b_llm_pretrain.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/performance/configs/llama31/llama31_405b_llm_pretrain.yaml b/scripts/performance/configs/llama31/llama31_405b_llm_pretrain.yaml index d904c73e287..4a6211d8b4d 100644 --- a/scripts/performance/configs/llama31/llama31_405b_llm_pretrain.yaml +++ b/scripts/performance/configs/llama31/llama31_405b_llm_pretrain.yaml @@ -25,6 +25,9 @@ ConfigContainer: train: train_iters: 100 eval_iters: 0 + + scheduler: + lr_warmup_iters: 10 rerun_state_machine: check_for_nan_in_loss: false