diff --git a/tests/functional_tests/shell_test_utils/run_ci_test.sh b/tests/functional_tests/shell_test_utils/run_ci_test.sh index 693970d3b67..00daaea69e2 100644 --- a/tests/functional_tests/shell_test_utils/run_ci_test.sh +++ b/tests/functional_tests/shell_test_utils/run_ci_test.sh @@ -69,6 +69,7 @@ mkdir -p $CHECKPOINT_SAVE_PATH mkdir -p $CHECKPOINT_LOAD_PATH || true _CHECKPOINT_LOAD_PATH=$CHECKPOINT_LOAD_PATH _CHECKPOINT_SAVE_PATH=$CHECKPOINT_SAVE_PATH +_TENSORBOARD_PATH=$TENSORBOARD_PATH SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) ROOT_DIR=$(realpath $SCRIPT_DIR/../../../) @@ -130,11 +131,11 @@ for i in $(seq 1 $N_REPEAT); do if [[ $i -gt 1 ]]; then rm -rf $CHECKPOINT_SAVE_PATH/* rm -rf /tmp/checkpoints/* - rm -rf $TENSORBOARD_PATH/* fi # First run never loads from a checkpoint export RUN_NUMBER=1 + export TENSORBOARD_PATH=$_TENSORBOARD_PATH/$i/ export REPEAT=$i export CHECKPOINT_SAVE_PATH=$_CHECKPOINT_SAVE_PATH export TRAINING_EXIT_CODE=0 diff --git a/tests/test_utils/recipes/gpt-gb200.yaml b/tests/test_utils/recipes/gpt-gb200.yaml index 70b89e31a0e..fd3a8b1605c 100644 --- a/tests/test_utils/recipes/gpt-gb200.yaml +++ b/tests/test_utils/recipes/gpt-gb200.yaml @@ -67,5 +67,5 @@ products: - test_case: [gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200]