diff --git a/examples/configs/grpo_math_1B.yaml b/examples/configs/grpo_math_1B.yaml index 43ac05dd0c5..fbc084abaa0 100644 --- a/examples/configs/grpo_math_1B.yaml +++ b/examples/configs/grpo_math_1B.yaml @@ -181,6 +181,14 @@ policy: moe_enable_deepep: false moe_token_dispatcher_type: "alltoall" moe_shared_expert_overlap: false + # Multi-Token Prediction (MTP). mtp_num_layers=0 disables MTP. + mtp_num_layers: 0 + # MTP loss weight added to the main next-token loss (0.0 disables the MTP loss contribution). + mtp_loss_scaling_factor: 0.0 + # When true, repeat one MTP layer mtp_num_layers times instead of using distinct layers. + mtp_use_repeated_layer: false + # When true, detach MTP heads so MTP loss does not affect main-model gradients. + mtp_detach_heads: false gradient_accumulation_fusion: false peft: diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index b963a6d4ff5..cf105b338fd 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -2391,6 +2391,10 @@ def grpo_train( metrics.update( {f"moe/{k}": v for k, v in train_results["moe_metrics"].items()} ) + if "mtp_metrics" in train_results: + metrics.update( + {f"mtp/{k}": v for k, v in train_results["mtp_metrics"].items()} + ) if master_config.grpo["use_dynamic_sampling"]: metrics["filtered_reward"] = rewards.numpy() metrics["reward"] = repeated_batch["total_reward"].numpy() @@ -3595,6 +3599,10 @@ def async_grpo_train( metrics.update( {f"moe/{k}": v for k, v in train_results["moe_metrics"].items()} ) + if "mtp_metrics" in train_results: + metrics.update( + {f"mtp/{k}": v for k, v in train_results["mtp_metrics"].items()} + ) metrics.update(train_results["all_mb_metrics"]) for k, v in metrics.items(): if k in {"probs_ratio_min", "probs_ratio_clamped_min"}: diff --git a/nemo_rl/models/megatron/common.py b/nemo_rl/models/megatron/common.py index 69912f209c5..cb00277aa40 100644 --- a/nemo_rl/models/megatron/common.py +++ b/nemo_rl/models/megatron/common.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ get_moe_layer_wise_logging_tracker, reduce_aux_losses_tracker_across_ranks, ) +from megatron.core.transformer.multi_token_prediction import MTPLossLoggingHelper def _round_up_to_multiple(value: int, multiple: int) -> int: @@ -164,3 +165,34 @@ def get_moe_metrics( clear_aux_losses_tracker() return metrics + + +def get_mtp_metrics() -> dict[str, Any]: + """Returns Multi-Token Prediction (MTP) loss and acceptance rate metrics. + + This function reduces MTP metrics across ranks and returns a dictionary of metrics. + + Returns: + dict[str, Any]: A flat dict of metrics. Each MTP layer's loss is returned + under the key "mtp_{i}_loss" and acceptance rate under "mtp_{i}_acceptance_rate" + where i is 1-indexed (matching Megatron-LM). + """ + MTPLossLoggingHelper.reduce_metrics_in_tracker() + tracker = MTPLossLoggingHelper.tracker + + metrics: dict[str, Any] = {} + if "loss_values" in tracker: + mtp_losses = tracker["loss_values"].float() + mtp_corrects = tracker.get("correct_values", torch.zeros_like(mtp_losses)) + mtp_totals = tracker.get("total_values", torch.ones_like(mtp_losses)) + mtp_num_layers = mtp_losses.shape[0] + + # Log per-layer losses and acceptance rates + for i in range(mtp_num_layers): + metrics[f"mtp_{i + 1}_loss"] = float(mtp_losses[i].item()) + # Compute acceptance rate as percentage + acceptance_rate = (mtp_corrects[i] / mtp_totals[i].clamp(min=1)) * 100.0 + metrics[f"mtp_{i + 1}_acceptance_rate"] = float(acceptance_rate.item()) + + MTPLossLoggingHelper.clean_metrics_in_tracker() + return metrics diff --git a/nemo_rl/models/megatron/data.py b/nemo_rl/models/megatron/data.py index 0c08f2a814c..bb216c8e29c 100644 --- a/nemo_rl/models/megatron/data.py +++ b/nemo_rl/models/megatron/data.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -44,6 +44,7 @@ class ProcessedInputs: position_ids: Optional[torch.Tensor] packed_seq_params: Optional[PackedSeqParams] cu_seqlens_padded: Optional[torch.Tensor] + mtp_loss_mask: Optional[torch.Tensor] = None routed_experts: Optional[torch.Tensor] = None routed_experts_cp_sharded: Optional[torch.Tensor] = None @@ -63,6 +64,8 @@ class ProcessedMicrobatch: position_ids: Position IDs tensor (None for packed sequences) packed_seq_params: PackedSeqParams for sequence packing (None if not packing) cu_seqlens_padded: Padded cumulative sequence lengths (None if not packing) + mtp_loss_mask: Pre-computed MTP loss mask (token_mask × sample_mask). + None when MTP is disabled or token/sample masks are absent. routed_experts: Optional token-aligned routed expert ids routed_experts_cp_sharded: Context-parallel sharded routed expert ids """ @@ -74,6 +77,7 @@ class ProcessedMicrobatch: position_ids: Optional[torch.Tensor] packed_seq_params: Optional[PackedSeqParams] cu_seqlens_padded: Optional[torch.Tensor] + mtp_loss_mask: Optional[torch.Tensor] = None routed_experts: Optional[torch.Tensor] = None routed_experts_cp_sharded: Optional[torch.Tensor] = None @@ -131,6 +135,7 @@ def make_processed_microbatch_iterator( position_ids=processed_inputs.position_ids, packed_seq_params=processed_inputs.packed_seq_params, cu_seqlens_padded=processed_inputs.cu_seqlens_padded, + mtp_loss_mask=processed_inputs.mtp_loss_mask, routed_experts=processed_inputs.routed_experts, routed_experts_cp_sharded=processed_inputs.routed_experts_cp_sharded, ) @@ -266,6 +271,7 @@ def process_microbatch( seq_lengths = None # Will be set if using packed sequences cu_seqlens = None cu_seqlens_padded = None + mtp_loss_mask = None if pack_sequences: # For packed sequences with padded input, we need sequence lengths @@ -280,6 +286,12 @@ def process_microbatch( seq_lengths = data_dict[seq_length_key] if delegate_pack_to_model: + # The VLM packing path does not pack or propagate mtp_loss_mask, + # so MTP training would be silently dropped here. Fail loudly + # instead of producing wrong results. + assert "mtp_loss_mask" not in data_dict, ( + "MTP training is not supported with VLM sequence packing" + ) # VLM path: model (e.g. mbridge Qwen3VL) does its own # preprocess_packed_seqs; NeMo-RL must NOT pre-pack + CP-shard, # or the double-processing produces shape mismatches downstream @@ -375,6 +387,24 @@ def process_microbatch( cp_size=get_context_parallel_world_size(), ) + # Pack pre-computed mtp_loss_mask the same way as input_ids + if "mtp_loss_mask" in data_dict: + ( + _, + mtp_loss_mask, + _, + _, + _, + ) = _pack_sequences_for_megatron( + data_dict["mtp_loss_mask"], + seq_lengths, + pad_individual_seqs_to_multiple_of, + pad_packed_seq_to_multiple_of, + pad_full_seq_to, + cp_rank=get_context_parallel_rank(), + cp_size=get_context_parallel_world_size(), + ) + # For packed sequences, position_ids and attention_mask are typically None # The PackedSeqParams handles all necessary sequence information position_ids = None @@ -422,6 +452,8 @@ def process_microbatch( eod_mask_loss=False, pad_mask_loss=False, ) + if "mtp_loss_mask" in data_dict: + mtp_loss_mask = data_dict["mtp_loss_mask"] return ProcessedInputs( input_ids=input_ids, input_ids_cp_sharded=input_ids_cp_sharded, @@ -429,6 +461,7 @@ def process_microbatch( position_ids=position_ids, packed_seq_params=packed_seq_params, cu_seqlens_padded=cu_seqlens_padded, + mtp_loss_mask=mtp_loss_mask, routed_experts=routed_experts, routed_experts_cp_sharded=routed_experts_cp_sharded, ) diff --git a/nemo_rl/models/megatron/setup.py b/nemo_rl/models/megatron/setup.py index 46b30d9860e..3fe1e53f6a7 100644 --- a/nemo_rl/models/megatron/setup.py +++ b/nemo_rl/models/megatron/setup.py @@ -667,8 +667,19 @@ def _apply_moe_config(model_cfg: Any, config: PolicyConfig) -> None: def _apply_mtp_config(model_cfg: Any, config: PolicyConfig) -> None: - if "mtp_num_layers" in config["megatron_cfg"]: - model_cfg.mtp_num_layers = config["megatron_cfg"]["mtp_num_layers"] + """Apply Multi-Token Prediction settings onto the mcore model config.""" + megatron_cfg = config["megatron_cfg"] + if "mtp_num_layers" in megatron_cfg: + # In mcore, mtp_num_layers is both the number of MTP layers (when + # mtp_use_repeated_layer is False) and the number of times the MTP layer + # is repeated (when mtp_use_repeated_layer is True). + model_cfg.mtp_num_layers = megatron_cfg["mtp_num_layers"] + if "mtp_loss_scaling_factor" in megatron_cfg: + model_cfg.mtp_loss_scaling_factor = megatron_cfg["mtp_loss_scaling_factor"] + if "mtp_use_repeated_layer" in megatron_cfg: + model_cfg.mtp_use_repeated_layer = megatron_cfg["mtp_use_repeated_layer"] + if "mtp_detach_heads" in megatron_cfg: + model_cfg.mtp_detach_heads = megatron_cfg["mtp_detach_heads"] def _apply_precision_config( diff --git a/nemo_rl/models/megatron/train.py b/nemo_rl/models/megatron/train.py index 3611b40773b..5ee01e422dc 100644 --- a/nemo_rl/models/megatron/train.py +++ b/nemo_rl/models/megatron/train.py @@ -78,6 +78,7 @@ def model_forward( attention_mask: torch.Tensor, packed_seq_params: Optional[PackedSeqParams] = None, defer_fp32_logits: Optional[bool] = False, + mtp_loss_mask: Optional[torch.Tensor] = None, straggler_timer: Optional[StragglerDetector] = None, use_linear_ce_fusion_loss: bool = False, ) -> torch.Tensor: @@ -91,6 +92,7 @@ def model_forward( attention_mask: Attention mask for the sequence packed_seq_params: Parameters for packed sequences (optional) defer_fp32_logits: Whether to skip the conversion of logits to fp32 + mtp_loss_mask: MTP loss mask to exclude prompt tokens from MTP loss (optional) straggler_timer: Straggler detector for profiling the forward pass use_linear_ce_fusion_loss: Whether to use linear CE fusion loss @@ -107,6 +109,11 @@ def model_forward( # Mamba models currently do not support packed_seq_params if packed_seq_params is not None: additional_kwargs["packed_seq_params"] = packed_seq_params + + # Pass MTP loss mask to exclude prompt tokens from MTP loss + if mtp_loss_mask is not None: + additional_kwargs["loss_mask"] = mtp_loss_mask + if defer_fp32_logits: additional_kwargs["fp32_output"] = False if use_linear_ce_fusion_loss: @@ -193,6 +200,7 @@ def forward_with_post_processing_fn( position_ids = processed_mb.position_ids packed_seq_params = processed_mb.packed_seq_params cu_seqlens_padded = processed_mb.cu_seqlens_padded + mtp_loss_mask = processed_mb.mtp_loss_mask routed_experts_cp_sharded = processed_mb.routed_experts_cp_sharded if use_router_replay: @@ -214,6 +222,7 @@ def forward_with_post_processing_fn( attention_mask=attention_mask, packed_seq_params=packed_seq_params, defer_fp32_logits=defer_fp32_logits, + mtp_loss_mask=mtp_loss_mask, straggler_timer=straggler_timer, use_linear_ce_fusion_loss=use_linear_ce_fusion_loss, ) diff --git a/nemo_rl/models/policy/__init__.py b/nemo_rl/models/policy/__init__.py index d22e6a1c49e..5d7c23a397b 100644 --- a/nemo_rl/models/policy/__init__.py +++ b/nemo_rl/models/policy/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -296,6 +296,12 @@ class MegatronConfig(TypedDict): linear_ce_fusion_chunk_size: NotRequired[int] # When mtp_num_layers=0, Multi-Token Prediction is disabled. mtp_num_layers: NotRequired[int] + # MTP loss weight added to the main next-token loss (0.0 disables the MTP loss contribution). + mtp_loss_scaling_factor: NotRequired[float] + # When True, repeat a single MTP layer mtp_num_layers times instead of using distinct layers. + mtp_use_repeated_layer: NotRequired[bool] + # When True, detach MTP heads from the main model so MTP loss does not affect main-model gradients. + mtp_detach_heads: NotRequired[bool] # When True, clear the RotaryEmbedding LRU cache and MoE token dispatcher # routing tensors in offload_before_refit (before weight transfer to the # inference engine). Useful when training and logprob runs use different diff --git a/nemo_rl/models/policy/lm_policy.py b/nemo_rl/models/policy/lm_policy.py index 7b4b47e6387..3b089e613fa 100644 --- a/nemo_rl/models/policy/lm_policy.py +++ b/nemo_rl/models/policy/lm_policy.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -729,6 +729,8 @@ def train( } if "moe_metrics" in results[0]: aggregated_results["moe_metrics"] = results[0]["moe_metrics"] + if "mtp_metrics" in results[0]: + aggregated_results["mtp_metrics"] = results[0]["mtp_metrics"] if self.flops_tracker is not None: aggregated_results["total_flops"] = self.flops_tracker.total_flops diff --git a/nemo_rl/models/policy/workers/megatron_policy_worker.py b/nemo_rl/models/policy/workers/megatron_policy_worker.py index 643d0866626..f1f589bbe9d 100644 --- a/nemo_rl/models/policy/workers/megatron_policy_worker.py +++ b/nemo_rl/models/policy/workers/megatron_policy_worker.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -534,6 +534,14 @@ def train( global_valid_seqs = gb_result["global_valid_seqs"] global_valid_toks = gb_result["global_valid_toks"] + # Pre-compute MTP loss mask from token_mask and sample_mask + # before microbatch processing, so process_microbatch can pack it + if "token_mask" in batch and "sample_mask" in batch: + mtp_loss_mask = batch["token_mask"] * batch[ + "sample_mask" + ].unsqueeze(-1) + batch["mtp_loss_mask"] = mtp_loss_mask + ( data_iterator, num_microbatches, @@ -570,6 +578,10 @@ def train( self.optimizer.zero_grad() self._copy_main_params_to_param_buffer() + # Set mtp_grad_scale_func for MTP loss scaling (scales by valid tokens) + mtp_scale = 1.0 / global_valid_toks.clamp(min=1).float() + self._set_mtp_grad_scale_func(lambda: mtp_scale) + # Forward pass. draft_enabled = "draft" in self.cfg and self.cfg["draft"]["enabled"] use_router_replay = _should_use_router_replay( @@ -601,6 +613,10 @@ def train( router_replay_train=not eval_mode, ) + # Clear mtp_grad_scale_func after the forward-backward pass so + # it doesn't get serialized in the run_config.yaml when saving + self._set_mtp_grad_scale_func(None) + # Empty unused memory. if self.cfg["megatron_cfg"]["empty_unused_memory_level"] >= 1: torch.cuda.empty_cache() @@ -721,6 +737,9 @@ def train( ) if moe_metrics: metrics["moe_metrics"] = moe_metrics + # Collect MTP metrics (kept out of train()'s body so cloudpickle does not + # pull an unpicklable torch ConfigModuleInstance into the worker actor). + self._collect_mtp_metrics(metrics) return metrics @wrap_with_nvtx_name("megatron_policy_worker/get_reference_policy_logprobs") @@ -1271,6 +1290,40 @@ def prepare_refit_info(self) -> None: return refit_param_info_hf + def _collect_mtp_metrics(self, metrics: dict[str, Any]) -> None: + """Add Multi-Token Prediction metrics to ``metrics`` when MTP is enabled. + + get_mtp_metrics is imported lazily (not a module global) so cloudpickle + does not pull an unpicklable torch ConfigModuleInstance into the worker + actor's serialization. + """ + mtp_num_layers = getattr(self.model.config, "mtp_num_layers", None) + if mtp_num_layers is not None and mtp_num_layers > 0: + from nemo_rl.models.megatron.common import get_mtp_metrics + + # MTP layers live only on the last pipeline stage, so the tracker is + # populated there alone. Broadcast to all stages so downstream metric + # aggregation (which reads rank 0's results) sees them when PP > 1. + mtp_metrics = get_mtp_metrics() + mtp_metrics = broadcast_loss_metrics_from_last_stage(mtp_metrics) + if mtp_metrics: + metrics["mtp_metrics"] = mtp_metrics + + def _set_mtp_grad_scale_func(self, func): + """Set mtp_grad_scale_func on the model config for MTP loss scaling.""" + config = self._get_model_config() + if config is not None: + config.mtp_grad_scale_func = func + + def _get_model_config(self): + """Get the underlying model config (handle Float16Module wrapper).""" + model = self.model + if hasattr(model, "module") and hasattr(model.module, "config"): + return model.module.config + elif hasattr(model, "config"): + return model.config + return None + def _calculate_refit_param_info(self) -> list[tuple[str, int]]: """Calculate parameter information for refit. diff --git a/tests/unit/models/megatron/test_megatron_data.py b/tests/unit/models/megatron/test_megatron_data.py index 7235c58caf0..86c80cb320a 100644 --- a/tests/unit/models/megatron/test_megatron_data.py +++ b/tests/unit/models/megatron/test_megatron_data.py @@ -284,6 +284,8 @@ def test_process_microbatch_with_packing( data_dict.__getitem__ = MagicMock( side_effect=lambda k: input_ids if k == "input_ids" else seq_lengths ) + # This fixture provides neither mtp_loss_mask nor routed_experts, so those + # optional packing branches must not fire. data_dict.__contains__ = MagicMock( side_effect=lambda k: k in {"input_ids", "input_lengths"} ) @@ -306,6 +308,92 @@ def test_process_microbatch_with_packing( # Verify pack was called mock_pack.assert_called_once() + @patch("nemo_rl.models.megatron.data.get_ltor_masks_and_position_ids") + def test_process_microbatch_no_packing_propagates_mtp_loss_mask( + self, mock_get_masks + ): + """Without packing, a precomputed mtp_loss_mask is passed through.""" + from nemo_rl.models.megatron.data import process_microbatch + + mock_get_masks.return_value = ( + torch.ones(1, 4), + None, + torch.arange(4).unsqueeze(0), + ) + input_ids = torch.tensor([[1, 2, 3, 4]]) + mtp_loss_mask = torch.tensor([[1, 1, 0, 0]]) + data_dict = {"input_ids": input_ids, "mtp_loss_mask": mtp_loss_mask} + + result = process_microbatch( + data_dict, pack_sequences=False, straggler_timer=MagicMock() + ) + assert result.mtp_loss_mask is not None + assert torch.equal(result.mtp_loss_mask, mtp_loss_mask) + + @patch("nemo_rl.models.megatron.data.get_ltor_masks_and_position_ids") + def test_process_microbatch_no_packing_mtp_loss_mask_absent(self, mock_get_masks): + """mtp_loss_mask defaults to None when not provided.""" + from nemo_rl.models.megatron.data import process_microbatch + + mock_get_masks.return_value = ( + torch.ones(1, 4), + None, + torch.arange(4).unsqueeze(0), + ) + data_dict = {"input_ids": torch.tensor([[1, 2, 3, 4]])} + + result = process_microbatch( + data_dict, pack_sequences=False, straggler_timer=MagicMock() + ) + assert result.mtp_loss_mask is None + + @patch("nemo_rl.models.megatron.data.get_context_parallel_rank", return_value=0) + @patch( + "nemo_rl.models.megatron.data.get_context_parallel_world_size", return_value=1 + ) + @patch("nemo_rl.models.megatron.data._pack_sequences_for_megatron") + def test_process_microbatch_with_packing_packs_mtp_loss_mask( + self, mock_pack, mock_cp_world, mock_cp_rank + ): + """With packing, mtp_loss_mask is packed like input_ids and propagated.""" + from nemo_rl.models.megatron.data import process_microbatch + + # Distinct tensors at index 0 and 1 so the assertion below can catch an + # off-by-one read (the implementation must take index 1, the CP-sharded + # packed tensor, not index 0). + packed_idx0 = torch.zeros(1, 8, dtype=torch.long) + packed_idx1 = torch.tensor([[1, 2, 3, 4, 5, 6, 7, 8]]) + mock_pack.return_value = ( + packed_idx0, + packed_idx1, + MagicMock(), + torch.tensor([0, 5, 8], dtype=torch.int32), + torch.tensor([0, 5, 8], dtype=torch.int32), + ) + + data_dict = { + "input_ids": torch.tensor( + [[1, 2, 3, 4, 5, 0, 0, 0], [6, 7, 8, 0, 0, 0, 0, 0]] + ), + "input_lengths": torch.tensor([5, 3]), + "mtp_loss_mask": torch.tensor( + [[1, 1, 1, 1, 1, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]] + ), + } + + result = process_microbatch( + data_dict, + seq_length_key="input_lengths", + pack_sequences=True, + straggler_timer=MagicMock(), + ) + + # _pack_sequences_for_megatron is called once for input_ids and once for mtp_loss_mask. + assert mock_pack.call_count == 2 + # mtp_loss_mask takes the packed tensor (index 1 of the pack return tuple). + assert result.mtp_loss_mask is not None + assert torch.equal(result.mtp_loss_mask, packed_idx1) + @patch("nemo_rl.models.megatron.data.get_context_parallel_rank", return_value=0) @patch( "nemo_rl.models.megatron.data.get_context_parallel_world_size", return_value=2 diff --git a/tests/unit/models/megatron/test_mtp_metrics.py b/tests/unit/models/megatron/test_mtp_metrics.py new file mode 100644 index 00000000000..b279eb739a7 --- /dev/null +++ b/tests/unit/models/megatron/test_mtp_metrics.py @@ -0,0 +1,78 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest +import torch + + +def _seed_tracker(monkeypatch, tracker): + """Seed MTPLossLoggingHelper with a fixed tracker and disable cross-rank reduce. + + reduce_metrics_in_tracker would otherwise all-reduce over a process group; + stubbing it lets get_mtp_metrics run single-process on CPU tensors. + """ + from megatron.core.transformer.multi_token_prediction import MTPLossLoggingHelper + + monkeypatch.setattr(MTPLossLoggingHelper, "reduce_metrics_in_tracker", lambda: None) + monkeypatch.setattr(MTPLossLoggingHelper, "tracker", tracker) + return MTPLossLoggingHelper + + +@pytest.mark.mcore +def test_get_mtp_metrics_empty_tracker(monkeypatch): + """No tracked MTP losses -> empty dict, and clean is not required.""" + from nemo_rl.models.megatron.common import get_mtp_metrics + + _seed_tracker(monkeypatch, {}) + assert get_mtp_metrics() == {} + + +@pytest.mark.mcore +def test_get_mtp_metrics_per_layer_loss_and_acceptance(monkeypatch): + """Per-layer loss/acceptance are 1-indexed and the tracker is cleaned.""" + from nemo_rl.models.megatron.common import get_mtp_metrics + + tracker = { + "loss_values": torch.tensor([2.0, 4.0]), + "correct_values": torch.tensor([1.0, 3.0]), + "total_values": torch.tensor([2.0, 6.0]), + } + helper = _seed_tracker(monkeypatch, tracker) + + cleaned = {"called": False} + monkeypatch.setattr( + helper, + "clean_metrics_in_tracker", + lambda: cleaned.__setitem__("called", True), + ) + + metrics = get_mtp_metrics() + + # 1-indexed keys matching Megatron-LM; acceptance = correct / total * 100. + assert metrics["mtp_1_loss"] == pytest.approx(2.0) + assert metrics["mtp_1_acceptance_rate"] == pytest.approx(50.0) + assert metrics["mtp_2_loss"] == pytest.approx(4.0) + assert metrics["mtp_2_acceptance_rate"] == pytest.approx(50.0) + assert cleaned["called"], "clean_metrics_in_tracker should be called" + + +@pytest.mark.mcore +def test_get_mtp_metrics_defaults_when_only_loss_tracked(monkeypatch): + """correct/total absent -> acceptance defaults (correct=0, total=1) => 0%.""" + from nemo_rl.models.megatron.common import get_mtp_metrics + + _seed_tracker(monkeypatch, {"loss_values": torch.tensor([1.5])}) + + metrics = get_mtp_metrics() + assert metrics["mtp_1_loss"] == pytest.approx(1.5) + assert metrics["mtp_1_acceptance_rate"] == pytest.approx(0.0) diff --git a/tests/unit/reference_configs/grpo_math_1B.yaml b/tests/unit/reference_configs/grpo_math_1B.yaml index b5b617619bd..527c03ef96e 100644 --- a/tests/unit/reference_configs/grpo_math_1B.yaml +++ b/tests/unit/reference_configs/grpo_math_1B.yaml @@ -181,6 +181,14 @@ policy: moe_enable_deepep: false moe_token_dispatcher_type: "alltoall" moe_shared_expert_overlap: false + # Multi-Token Prediction (MTP). mtp_num_layers=0 disables MTP. + mtp_num_layers: 0 + # MTP loss weight added to the main next-token loss (0.0 disables the MTP loss contribution). + mtp_loss_scaling_factor: 0.0 + # When true, repeat one MTP layer mtp_num_layers times instead of using distinct layers. + mtp_use_repeated_layer: false + # When true, detach MTP heads so MTP loss does not affect main-model gradients. + mtp_detach_heads: false gradient_accumulation_fusion: false peft: