Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions tests/functional_tests/shell_test_utils/run_ci_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,10 @@ for i in $(seq 1 $N_REPEAT); do

# First run never loads from a checkpoint
export RUN_NUMBER=1
DIR=$(dirname "$_TENSORBOARD_PATH")
FILE=$(basename "$_TENSORBOARD_PATH")
export TENSORBOARD_PATH=$DIR/$i/$FILE
mkdir -p $(dirname $TENSORBOARD_PATH)
export REPEAT=$i
export CHECKPOINT_SAVE_PATH=$_CHECKPOINT_SAVE_PATH
export TRAINING_EXIT_CODE=0
Expand Down
8 changes: 4 additions & 4 deletions tests/test_utils/recipes/gpt.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -462,7 +462,7 @@ products:
- test_case: [gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer]
products:
- environment: [lts]
scope: [mr]
scope: [nightly]
- environment: [dev]
scope: [mr, mr-github, mr-github-slim]
platforms: [dgx_h100]
Expand All @@ -472,11 +472,11 @@ products:
scope: [mr]
platforms: [dgx_h100]
- environment: [lts]
scope: [mr]
scope: [nightly]
- test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather]
products:
- environment: [lts]
scope: [mr]
scope: [nightly]
- environment: [dev]
scope: [mr, mr-github, mr-github-slim]
platforms: [dgx_h100]
Expand All @@ -486,7 +486,7 @@ products:
scope: [mr]
platforms: [dgx_h100]
- environment: [lts]
scope: [mr]
scope: [nightly]
# - test_case: [gpt3_mcore_te_tp4_pp2_frozen_resume_torch_dist_reshard_8x1xNone]
# products:
# - environment: [dev]
Expand Down
Loading