From efb0da94751a5de58eb4a733749f1e04e6eb03dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Thu, 15 Jan 2026 09:45:52 +0000 Subject: [PATCH 1/2] ci: Fix GB200 race condition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: oliver könig --- tests/functional_tests/shell_test_utils/run_ci_test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/functional_tests/shell_test_utils/run_ci_test.sh b/tests/functional_tests/shell_test_utils/run_ci_test.sh index 693970d3b67..00daaea69e2 100644 --- a/tests/functional_tests/shell_test_utils/run_ci_test.sh +++ b/tests/functional_tests/shell_test_utils/run_ci_test.sh @@ -69,6 +69,7 @@ mkdir -p $CHECKPOINT_SAVE_PATH mkdir -p $CHECKPOINT_LOAD_PATH || true _CHECKPOINT_LOAD_PATH=$CHECKPOINT_LOAD_PATH _CHECKPOINT_SAVE_PATH=$CHECKPOINT_SAVE_PATH +_TENSORBOARD_PATH=$TENSORBOARD_PATH SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) ROOT_DIR=$(realpath $SCRIPT_DIR/../../../) @@ -130,11 +131,11 @@ for i in $(seq 1 $N_REPEAT); do if [[ $i -gt 1 ]]; then rm -rf $CHECKPOINT_SAVE_PATH/* rm -rf /tmp/checkpoints/* - rm -rf $TENSORBOARD_PATH/* fi # First run never loads from a checkpoint export RUN_NUMBER=1 + export TENSORBOARD_PATH=$_TENSORBOARD_PATH/$i/ export REPEAT=$i export CHECKPOINT_SAVE_PATH=$_CHECKPOINT_SAVE_PATH export TRAINING_EXIT_CODE=0 From c76c20c29e53a026f8fe08695cb95e217c943154 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Thu, 15 Jan 2026 09:46:17 +0000 Subject: [PATCH 2/2] renable test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: oliver könig --- tests/test_utils/recipes/gpt-gb200.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils/recipes/gpt-gb200.yaml b/tests/test_utils/recipes/gpt-gb200.yaml index 70b89e31a0e..fd3a8b1605c 100644 --- a/tests/test_utils/recipes/gpt-gb200.yaml +++ b/tests/test_utils/recipes/gpt-gb200.yaml @@ -67,5 +67,5 @@ products: - test_case: [gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200]