diff --git a/megatron/core/transformer/cuda_graphs.py b/megatron/core/transformer/cuda_graphs.py index 3643c42c3ce..df6f88b3b93 100644 --- a/megatron/core/transformer/cuda_graphs.py +++ b/megatron/core/transformer/cuda_graphs.py @@ -1738,7 +1738,7 @@ def __init__(self, model, config, seq_length, micro_batch_size, optimizers=[]): callables.append(layer) callables_is_mtp.append(False) for layer_number in range(num_mtp_layers): - layer = chunk_with_decoder.mtp.layers[layer_number].transformer_layer + layer = chunk_with_decoder.mtp.layers[layer_number].mtp_model_layer if _layer_is_graphable(layer, config): num_graphable_layers += 1 callables.append(layer) diff --git a/megatron/training/training.py b/megatron/training/training.py index 5206b526e18..7b5b6728dd2 100644 --- a/megatron/training/training.py +++ b/megatron/training/training.py @@ -588,6 +588,9 @@ def transformer_flops(): # Calculate the number of each type of layer. num_attn_layers, num_mamba_layers, num_mlp_layers, num_moe_layers = calculate_layer_counts() + mtp_num_layers = args.mtp_num_layers + if mtp_num_layers is None: + mtp_num_layers = 0 # Compute hybrid model FLOPs. return hybrid_flops( batch_size=batch_size, @@ -614,7 +617,7 @@ def transformer_flops(): else args.moe_shared_expert_intermediate_size), num_experts_routed_to=args.moe_router_topk, vocab_size=args.padded_vocab_size, - mtp_num_layers=args.mtp_num_layers, + mtp_num_layers=mtp_num_layers, ) else: # Compute standard Transformer model FLOPs.