diff --git a/tests/functional_tests/shell_test_utils/run_ci_test.sh b/tests/functional_tests/shell_test_utils/run_ci_test.sh index 693970d3b67..20267536a0f 100644 --- a/tests/functional_tests/shell_test_utils/run_ci_test.sh +++ b/tests/functional_tests/shell_test_utils/run_ci_test.sh @@ -69,6 +69,7 @@ mkdir -p $CHECKPOINT_SAVE_PATH mkdir -p $CHECKPOINT_LOAD_PATH || true _CHECKPOINT_LOAD_PATH=$CHECKPOINT_LOAD_PATH _CHECKPOINT_SAVE_PATH=$CHECKPOINT_SAVE_PATH +_TENSORBOARD_PATH=$TENSORBOARD_PATH SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) ROOT_DIR=$(realpath $SCRIPT_DIR/../../../) @@ -135,6 +136,10 @@ for i in $(seq 1 $N_REPEAT); do # First run never loads from a checkpoint export RUN_NUMBER=1 + DIR=$(dirname "$_TENSORBOARD_PATH") + FILE=$(basename "$_TENSORBOARD_PATH") + export TENSORBOARD_PATH=$DIR/$i/$FILE + mkdir -p $(dirname $TENSORBOARD_PATH) export REPEAT=$i export CHECKPOINT_SAVE_PATH=$_CHECKPOINT_SAVE_PATH export TRAINING_EXIT_CODE=0 diff --git a/tests/test_utils/recipes/gpt.yaml b/tests/test_utils/recipes/gpt.yaml index eab62026381..90eddc55c27 100644 --- a/tests/test_utils/recipes/gpt.yaml +++ b/tests/test_utils/recipes/gpt.yaml @@ -462,7 +462,7 @@ products: - test_case: [gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer] products: - environment: [lts] - scope: [mr] + scope: [nightly] - environment: [dev] scope: [mr, mr-github, mr-github-slim] platforms: [dgx_h100] @@ -472,11 +472,11 @@ products: scope: [mr] platforms: [dgx_h100] - environment: [lts] - scope: [mr] + scope: [nightly] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather] products: - environment: [lts] - scope: [mr] + scope: [nightly] - environment: [dev] scope: [mr, mr-github, mr-github-slim] platforms: [dgx_h100] @@ -486,7 +486,7 @@ products: scope: [mr] platforms: [dgx_h100] - environment: [lts] - scope: [mr] + scope: [nightly] # - test_case: [gpt3_mcore_te_tp4_pp2_frozen_resume_torch_dist_reshard_8x1xNone] # products: # - environment: [dev]