diff --git a/megatron/core/models/common/model_chunk_schedule_plan.py b/megatron/core/models/common/model_chunk_schedule_plan.py index 72b12591451..b6b4debd443 100644 --- a/megatron/core/models/common/model_chunk_schedule_plan.py +++ b/megatron/core/models/common/model_chunk_schedule_plan.py @@ -241,11 +241,14 @@ def run(f_layer, b_layer, f_input=None, b_grad=None, is_last_layer_in_bwd=False) if f_layer is not None: with f_layer.get_fp8_context(): f_input = f_layer.moe_combine.forward(f_input) - f_input = f_layer.mtp_post_process.forward(f_input) if b_layer is not None and not b_layer.config.ep_overlap_early_attn_memory_release: b_grad = b_layer.attn.backward(b_grad) + if f_layer is not None: + with f_layer.get_fp8_context(): + f_input = f_layer.mtp_post_process.forward(f_input) + # Delay the last attn_dw in backward pass (attn_dw of the first layer) # for overlapping with the p2p comm if b_layer is not None and not is_last_layer_in_bwd: