diff --git a/megatron/training/training.py b/megatron/training/training.py index 378dcfb3593..8d8301f8317 100644 --- a/megatron/training/training.py +++ b/megatron/training/training.py @@ -2245,7 +2245,8 @@ def save_checkpoint_and_time( # Stop timer to get accurate train interval time and exclude checkpointing duration timers('interval-time').stop() - energy_monitor.pause() + if args.log_energy: + energy_monitor.pause() # Extra barrier is added to make sure all ranks report the max time. timer_key = 'save-checkpoint-non-persistent' if non_persistent_ckpt else 'save-checkpoint' @@ -2298,7 +2299,8 @@ def save_checkpoint_and_time( ) # Recover timing - energy_monitor.resume() + if args.log_energy: + energy_monitor.resume() timers('interval-time', log_level=0).start(barrier=True) diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml index a37dd0dc658..451ee6b2e15 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml @@ -133,7 +133,6 @@ MODEL_ARGS: --overlap-moe-expert-parallel-comm: true TEST_TYPE: regular # Usually ckpt-resume, but as a WAR to #513 set to regular METRICS: - - "iteration-time" - "lm loss" - "mem-allocated-bytes" - "mem-max-allocated-bytes" diff --git a/tests/test_utils/recipes/h100/moe.yaml b/tests/test_utils/recipes/h100/moe.yaml index 10c1140ddf5..79b97b8f75a 100644 --- a/tests/test_utils/recipes/h100/moe.yaml +++ b/tests/test_utils/recipes/h100/moe.yaml @@ -176,7 +176,7 @@ products: - test_case: [gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading] products: - environment: [dev] - scope: [mr] + scope: [mr, mr-github] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective] products: @@ -205,15 +205,10 @@ products: - environment: [dev] scope: [mr, mr-github, mr-slim] platforms: [dgx_h100] - - test_case: [gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading] - products: - - environment: [dev] - scope: [mr-broken] - platforms: [dgx_h100] - test_case: [gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading] products: - environment: [dev] - scope: [mr-broken] + scope: [mr, mr-github] platforms: [dgx_h100] ####################################################################### # Super important mr, mr-github tests that run for both DEV and LTS per mr, mr-github #