diff --git a/tests/functional_tests/test_cases/t5/t5_mcore_te_tp1_pp1_vp1_resume_torch/model_config.yaml b/tests/functional_tests/test_cases/t5/t5_mcore_te_tp1_pp1_vp1_resume_torch/model_config.yaml index 8fc0b04c8c8..00a11b9a439 100644 --- a/tests/functional_tests/test_cases/t5/t5_mcore_te_tp1_pp1_vp1_resume_torch/model_config.yaml +++ b/tests/functional_tests/test_cases/t5/t5_mcore_te_tp1_pp1_vp1_resume_torch/model_config.yaml @@ -53,4 +53,10 @@ MODEL_ARGS: --ckpt-format: torch --attention-backend: unfused --log-memory-to-tensorboard: true + # Synchronous data loading: the default 2 persistent dataloader workers + # (pin_memory, no DataLoader timeout) can stall on the CI data mount, leaving + # the rank blocked in get_batch -> queue.get forever -> the other ranks time + # out in the grad all-reduce (600s NCCL watchdog). In-process loading removes + # the worker-queue hang surface; test data is tiny so perf impact is nil. + --num-workers: 0 TEST_TYPE: ckpt-resume diff --git a/tests/functional_tests/test_cases/t5/t5_mcore_tp1_pp1_vp1/model_config.yaml b/tests/functional_tests/test_cases/t5/t5_mcore_tp1_pp1_vp1/model_config.yaml index f8e6db5d685..805cc3e9858 100644 --- a/tests/functional_tests/test_cases/t5/t5_mcore_tp1_pp1_vp1/model_config.yaml +++ b/tests/functional_tests/test_cases/t5/t5_mcore_tp1_pp1_vp1/model_config.yaml @@ -51,4 +51,10 @@ MODEL_ARGS: --deterministic-mode: true --ckpt-format: torch --log-memory-to-tensorboard: true + # Synchronous data loading: the default 2 persistent dataloader workers + # (pin_memory, no DataLoader timeout) can stall on the CI data mount, leaving + # the rank blocked in get_batch -> queue.get forever -> the other ranks time + # out in the grad all-reduce (600s NCCL watchdog). In-process loading removes + # the worker-queue hang surface; test data is tiny so perf impact is nil. + --num-workers: 0 TEST_TYPE: regular diff --git a/tests/functional_tests/test_cases/t5/t5_mcore_tp1_pp1_vp1_resume_torch/model_config.yaml b/tests/functional_tests/test_cases/t5/t5_mcore_tp1_pp1_vp1_resume_torch/model_config.yaml index 314a960cc75..5b1e192a3a9 100644 --- a/tests/functional_tests/test_cases/t5/t5_mcore_tp1_pp1_vp1_resume_torch/model_config.yaml +++ b/tests/functional_tests/test_cases/t5/t5_mcore_tp1_pp1_vp1_resume_torch/model_config.yaml @@ -51,4 +51,10 @@ MODEL_ARGS: --deterministic-mode: true --ckpt-format: torch --log-memory-to-tensorboard: true + # Synchronous data loading: the default 2 persistent dataloader workers + # (pin_memory, no DataLoader timeout) can stall on the CI data mount, leaving + # the rank blocked in get_batch -> queue.get forever -> the other ranks time + # out in the grad all-reduce (600s NCCL watchdog). In-process loading removes + # the worker-queue hang surface; test data is tiny so perf impact is nil. + --num-workers: 0 TEST_TYPE: ckpt-resume