diff --git a/tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-fsdp2.sh b/tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-fsdp2.sh index 59340fa6449..59b4c77affd 100755 --- a/tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-fsdp2.sh +++ b/tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-fsdp2.sh @@ -8,7 +8,7 @@ GPUS_PER_NODE=8 STEPS_PER_RUN=10 MAX_STEPS=10 NUM_RUNS=$(( (MAX_STEPS + STEPS_PER_RUN - 1) / STEPS_PER_RUN )) # Round up -NUM_MINUTES=45 +NUM_MINUTES=50 # ===== END CONFIG ===== exit_if_max_steps_reached diff --git a/tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-megatron-pack-cp-tq_simple.sh b/tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-megatron-pack-cp-tq_simple.sh index b40db4ad49f..56850083541 100755 --- a/tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-megatron-pack-cp-tq_simple.sh +++ b/tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-megatron-pack-cp-tq_simple.sh @@ -8,7 +8,7 @@ GPUS_PER_NODE=8 STEPS_PER_RUN=3 MAX_STEPS=3 NUM_RUNS=$(( (MAX_STEPS + STEPS_PER_RUN - 1) / STEPS_PER_RUN )) # Round up -NUM_MINUTES=30 +NUM_MINUTES=35 # ===== END CONFIG ===== source "$SCRIPT_DIR/common-tq.env" diff --git a/tests/test_suites/llm/grpo-qwen2.5-32b-32n8g-fsdp2tp8-actckpt.v3.sh b/tests/test_suites/llm/grpo-qwen2.5-32b-32n8g-fsdp2tp8-actckpt.v3.sh index a79e9002f6d..8385eb5877f 100755 --- a/tests/test_suites/llm/grpo-qwen2.5-32b-32n8g-fsdp2tp8-actckpt.v3.sh +++ b/tests/test_suites/llm/grpo-qwen2.5-32b-32n8g-fsdp2tp8-actckpt.v3.sh @@ -7,7 +7,7 @@ NUM_NODES=32 STEPS_PER_RUN=2 # step_time: [220, 310] MAX_STEPS=2 NUM_RUNS=$(( (MAX_STEPS + STEPS_PER_RUN - 1) / STEPS_PER_RUN )) # Round up -NUM_MINUTES=30 +NUM_MINUTES=35 # ===== END CONFIG ===== exit_if_max_steps_reached diff --git a/tests/test_suites/llm/sft-nanov3-30BA3B-2n8g-fsdp2-lora.sh b/tests/test_suites/llm/sft-nanov3-30BA3B-2n8g-fsdp2-lora.sh index 71056bb4e1a..8c1dee55a69 100755 --- a/tests/test_suites/llm/sft-nanov3-30BA3B-2n8g-fsdp2-lora.sh +++ b/tests/test_suites/llm/sft-nanov3-30BA3B-2n8g-fsdp2-lora.sh @@ -7,7 +7,7 @@ NUM_NODES=2 STEPS_PER_RUN=20 # step_time ~ 10sec MAX_STEPS=20 NUM_RUNS=$(( (MAX_STEPS + STEPS_PER_RUN - 1) / STEPS_PER_RUN )) # Round up -NUM_MINUTES=30 # Usually 15 minutes is enough for 20 steps, but we add a buffer of 3 minutes in metrics check (30min to buffer for initial ckpt download) +NUM_MINUTES=35 # Usually 15 minutes is enough for 20 steps, but we add a buffer of 3 minutes in metrics check (30min to buffer for initial ckpt download) # ===== END CONFIG ===== exit_if_max_steps_reached diff --git a/tests/test_suites/llm/sft-qwen2.5-32b-4n8g-fsdp2tp8sp-actckpt.v3.sh b/tests/test_suites/llm/sft-qwen2.5-32b-4n8g-fsdp2tp8sp-actckpt.v3.sh index 799a04c300b..71ad3c27b3b 100755 --- a/tests/test_suites/llm/sft-qwen2.5-32b-4n8g-fsdp2tp8sp-actckpt.v3.sh +++ b/tests/test_suites/llm/sft-qwen2.5-32b-4n8g-fsdp2tp8sp-actckpt.v3.sh @@ -10,7 +10,7 @@ NUM_NODES=4 STEPS_PER_RUN=20 # step_time ~ 29sec MAX_STEPS=20 NUM_RUNS=$(( (MAX_STEPS + STEPS_PER_RUN - 1) / STEPS_PER_RUN )) # Round up -NUM_MINUTES=30 +NUM_MINUTES=35 # ===== END CONFIG ===== exit_if_max_steps_reached