diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index 12baec843cd..b6d9b0a2fec 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -578,6 +578,8 @@ def init_train_dataloader(dataset, suffix: str = ""): "Reference policy logprob calculation will be skipped since `grpo.skip_reference_policy_logprobs_calculation` is set to True and `loss_fn.reference_policy_kl_penalty` is 0." ) + _validate_use_kl_in_reward_compat(master_config) + # ========================== # Cluster # ========================== @@ -2359,6 +2361,65 @@ def _log_mixed_rewards_and_advantages_information( metrics["advantages/mean"] = advantages.float().mean().item() +def _placeholder_seq_logprob_error_metrics() -> dict[str, float]: + """Zero-valued seq-level metrics used when the prev_logprobs forward is skipped.""" + return { + "max_seq_mult_prob_error": 0.0, + "mean_seq_mult_prob_error": 0.0, + "min_seq_mult_prob_error": 0.0, + "max_seq_mult_prob_error_after_mask": 0.0, + "mean_seq_mult_prob_error_after_mask": 0.0, + "min_seq_mult_prob_error_after_mask": 0.0, + "num_masked_seqs_by_logprob_error": 0, + "masked_correct_pct": 0.0, + } + + +def _validate_use_kl_in_reward_compat(master_config: MasterConfig) -> None: + """Reject ``use_kl_in_reward`` when the KL term would read zero placeholder logprobs. + + ``force_on_policy_ratio`` (without ``seq_logprob_error_threshold``) skips + the prev_logprobs forward and passes a zero placeholder to the advantage + estimator; ``use_kl_in_reward`` then applies + ``kl_coef * calculate_kl(zeros, ref)`` which corrupts the advantage. + ``kl_coef=0`` (``reference_policy_kl_penalty=0``) zeros the term regardless, + so that case is allowed. + """ + loss_config = master_config.loss_fn + if loss_config.use_kl_in_reward and loss_config.reference_policy_kl_penalty != 0: + assert not opd_module._skip_prev_logprobs(master_config), ( + "loss_fn.use_kl_in_reward with nonzero loss_fn.reference_policy_kl_penalty " + "requires real prev_logprobs, but force_on_policy_ratio (without " + "grpo.seq_logprob_error_threshold) zeros them — KL would be computed " + "against a zero placeholder." + ) + + +def _resolve_logprob_skip_flags( + master_config: MasterConfig, +) -> tuple[bool, bool | None]: + """Return (skip_prev_logprobs, skip_reference_logprobs); warn on incompatible combos. + + Skip prev_logprobs when force_on_policy_ratio=True unless + seq_logprob_error_threshold is set (which requires prev_logprobs). + Skip reference_policy_logprobs when + ``grpo.skip_reference_policy_logprobs_calculation`` is set. + """ + # todo @jiaqi: is there a better way to skip prev_logprobs computation while still computing the seq-level error metrics? + if ( + master_config.loss_fn.force_on_policy_ratio + and master_config.grpo.get("seq_logprob_error_threshold") is not None + ): + warnings.warn( + "force_on_policy_ratio=True but seq_logprob_error_threshold is set. " + "Computing prev_logprobs anyway for seq-level error masking." + ) + return ( + opd_module._skip_prev_logprobs(master_config), + master_config.grpo.get("skip_reference_policy_logprobs_calculation"), + ) + + def compute_and_apply_seq_logprob_error_masking( train_data: BatchedDataDict, rewards: torch.Tensor, @@ -2952,24 +3013,12 @@ def grpo_train( metrics_logging_data["content"] = flat_messages["content"] memory_tracker.snapshot_start_of_stage("Computing logprobs", dir()) - # Skip prev_logprobs computation when force_on_policy_ratio=True - # unless seq_logprob_error_threshold is set (which requires prev_logprobs) + skip_prev_logprobs, skip_reference_logprobs = ( + _resolve_logprob_skip_flags(master_config) + ) seq_logprob_error_threshold = master_config.grpo.get( "seq_logprob_error_threshold", None ) - force_on_policy_ratio = master_config.loss_fn.force_on_policy_ratio - skip_prev_logprobs = opd_module._skip_prev_logprobs(master_config) - # todo @jiaqi: is there a better way to skip prev_logprobs computation while still computing the seq-level error metrics? - if force_on_policy_ratio and seq_logprob_error_threshold is not None: - warnings.warn( - "force_on_policy_ratio=True but seq_logprob_error_threshold is set. " - "Computing prev_logprobs anyway for seq-level error masking." - ) - - # Skip reference_policy_logprobs computation when skip_reference_policy_logprobs_calculation=True - skip_reference_logprobs = master_config.grpo.get( - "skip_reference_policy_logprobs_calculation" - ) if not (skip_prev_logprobs and skip_reference_logprobs): print("▶ Preparing for logprob inference...", flush=True) @@ -3034,16 +3083,7 @@ def grpo_train( # Seq-level logprob error metrics/masking require real prev_logprobs if skip_prev_logprobs: # Cannot compute seq-level metrics with placeholder prev_logprobs - seq_logprob_error_metrics = { - "max_seq_mult_prob_error": 0.0, - "mean_seq_mult_prob_error": 0.0, - "min_seq_mult_prob_error": 0.0, - "max_seq_mult_prob_error_after_mask": 0.0, - "mean_seq_mult_prob_error_after_mask": 0.0, - "min_seq_mult_prob_error_after_mask": 0.0, - "num_masked_seqs_by_logprob_error": 0, - "masked_correct_pct": 0.0, - } + seq_logprob_error_metrics = _placeholder_seq_logprob_error_metrics() else: seq_error_result = compute_and_apply_seq_logprob_error_masking( train_data=train_data, @@ -4357,25 +4397,12 @@ def async_grpo_train( train_data.to("cpu") # Training phase (same as sync version) - # Skip prev_logprobs computation when force_on_policy_ratio=True - # unless seq_logprob_error_threshold is set (which requires prev_logprobs) + skip_prev_logprobs, skip_reference_logprobs = ( + _resolve_logprob_skip_flags(master_config) + ) seq_logprob_error_threshold = master_config.grpo.get( "seq_logprob_error_threshold", None ) - force_on_policy_ratio = master_config.loss_fn.force_on_policy_ratio - skip_prev_logprobs = opd_module._skip_prev_logprobs(master_config) - - # todo @jiaqi: is there a better way to skip prev_logprobs computation while still computing the seq-level error metrics? - if force_on_policy_ratio and seq_logprob_error_threshold is not None: - warnings.warn( - "force_on_policy_ratio=True but seq_logprob_error_threshold is set. " - "Computing prev_logprobs anyway for seq-level error masking." - ) - - # Skip reference_policy_logprobs computation when skip_reference_policy_logprobs_calculation=True - skip_reference_logprobs = master_config.grpo.get( - "skip_reference_policy_logprobs_calculation" - ) if not (skip_prev_logprobs and skip_reference_logprobs): print("▶ Preparing for logprob inference...", flush=True) @@ -4412,16 +4439,7 @@ def async_grpo_train( # Seq-level logprob error metrics/masking require real prev_logprobs if skip_prev_logprobs: # Cannot compute seq-level metrics with placeholder prev_logprobs - seq_logprob_error_metrics = { - "max_seq_mult_prob_error": 0.0, - "mean_seq_mult_prob_error": 0.0, - "min_seq_mult_prob_error": 0.0, - "max_seq_mult_prob_error_after_mask": 0.0, - "mean_seq_mult_prob_error_after_mask": 0.0, - "min_seq_mult_prob_error_after_mask": 0.0, - "num_masked_seqs_by_logprob_error": 0, - "masked_correct_pct": 0.0, - } + seq_logprob_error_metrics = _placeholder_seq_logprob_error_metrics() else: seq_error_result = compute_and_apply_seq_logprob_error_masking( train_data=train_data, diff --git a/nemo_rl/algorithms/grpo_sync.py b/nemo_rl/algorithms/grpo_sync.py index a0a6485d6fb..03089443b60 100644 --- a/nemo_rl/algorithms/grpo_sync.py +++ b/nemo_rl/algorithms/grpo_sync.py @@ -51,6 +51,8 @@ _clip_grpo_advantages, _create_advantage_estimator, _log_mixed_rewards_and_advantages_information, + _placeholder_seq_logprob_error_metrics, + _resolve_logprob_skip_flags, _should_log_nemo_gym_responses, _should_use_nemo_gym, compute_and_apply_seq_logprob_error_masking, @@ -71,7 +73,7 @@ from nemo_rl.data.interfaces import DatumSpec from nemo_rl.data.llm_message_utils import batched_message_log_to_flat_message from nemo_rl.data_plane.interfaces import KVBatchMeta -from nemo_rl.data_plane.schema import DP_CALIB_INPUT_FIELDS +from nemo_rl.data_plane.schema import DP_CALIB_INPUT_FIELDS, DP_TRAIN_FIELDS from nemo_rl.distributed.batched_data_dict import BatchedDataDict from nemo_rl.environments.interfaces import EnvironmentInterface from nemo_rl.experience.sync_rollout_actor import SyncRolloutActor @@ -115,6 +117,13 @@ def _raise_if_message_level_advantage_penalties_enabled( ) +def _train_fields_for_step(skip_prev_logprobs: bool) -> tuple[str, ...]: + """Fields workers fetch this step; ``prev_logprobs`` is dropped when skipped.""" + return tuple( + f for f in DP_TRAIN_FIELDS if not (skip_prev_logprobs and f == "prev_logprobs") + ) + + # ── DAPO non-zero-std dynamic sampling, slice-only ───────────────────── # Slice-only formulation of nemo_rl.algorithms.grpo.dynamic_sampling: filter # on std != 0, accumulate survivors across iterations, slice on overflow. @@ -777,9 +786,23 @@ def grpo_train_sync( baseline_for_log = baseline.clone() memory_tracker.snapshot_start_of_stage("Computing logprobs", dir()) - print("▶ Preparing for logprob inference...", flush=True) - with timer.time("logprob_inference_prep"): - policy.prepare_for_lp_inference() + skip_prev_logprobs, skip_reference_logprobs = ( + _resolve_logprob_skip_flags(master_config) + ) + compute_prev = not skip_prev_logprobs + compute_ref = not skip_reference_logprobs + seq_logprob_error_threshold = master_config.grpo.get( + "seq_logprob_error_threshold", None + ) + # Worker-side fetch schema for this step. Same skip decision + # as ``select_fields`` below, but consumed only by + # ``train_from_meta`` (driver read uses ``select_fields``). + train_fields = _train_fields_for_step(skip_prev_logprobs) + + if compute_prev or compute_ref: + print("▶ Preparing for logprob inference...", flush=True) + with timer.time("logprob_inference_prep"): + policy.prepare_for_lp_inference() print("▶ Computing logprobs...", flush=True) with timer.time("policy_and_reference_logprobs"): @@ -791,15 +814,21 @@ def grpo_train_sync( # batched fetch to avoid double-shipping the per-token # tensor through Ray's plasma store on top of the TQ # writeback. - policy.get_logprobs_from_meta(meta, timer=timer) - compute_ref = not master_config.grpo.get( - "skip_reference_policy_logprobs_calculation" - ) + select_fields = ["generation_logprobs", "token_mask"] + if compute_prev: + policy.get_logprobs_from_meta(meta, timer=timer) + select_fields.append("prev_logprobs") + else: + print( + "▶ Skipping prev_logprobs (force_on_policy_ratio=True)...", + flush=True, + ) if compute_ref: policy.get_reference_policy_logprobs_from_meta( meta, timer=timer, ) + select_fields.append("reference_policy_logprobs") # Driver pulls only the per-token columns it needs # for masking / advantage. Bulk (input_ids, multimodal, @@ -807,21 +836,26 @@ def grpo_train_sync( # TQ — workers will fetch it via ``train_presharded``. extras_bdd = policy.read_from_dataplane( meta, - select_fields=[ - "prev_logprobs", - "generation_logprobs", - "token_mask", - *(["reference_policy_logprobs"] if compute_ref else []), - ], + select_fields=select_fields, pad_value_dict=_pad_dict, ) - prev_logprobs = extras_bdd["prev_logprobs"] generation_logprobs = extras_bdd["generation_logprobs"] token_mask = extras_bdd["token_mask"] + prev_logprobs = ( + extras_bdd["prev_logprobs"] + if compute_prev + else torch.zeros_like(generation_logprobs) + ) reference_policy_logprobs = ( extras_bdd["reference_policy_logprobs"] if compute_ref else None ) + # Seq-level logprob error metrics/masking require real prev_logprobs + if skip_prev_logprobs: + sample_mask = loss_multiplier + # Cannot compute seq-level metrics with placeholder prev_logprobs + seq_logprob_error_metrics = _placeholder_seq_logprob_error_metrics() + else: sample_mask, seq_logprob_error_metrics = ( _compute_seq_logprob_error_metrics( token_mask=token_mask, @@ -829,9 +863,7 @@ def grpo_train_sync( prev_logprobs=prev_logprobs, generation_logprobs=generation_logprobs, rewards=rewards, - seq_logprob_error_threshold=master_config.grpo[ - "seq_logprob_error_threshold" - ], + seq_logprob_error_threshold=seq_logprob_error_threshold, ) ) @@ -900,6 +932,7 @@ def grpo_train_sync( meta, loss_fn=loss_fn, timer=timer, + train_fields=train_fields, ) if sync_kv_scales: diff --git a/nemo_rl/models/policy/tq_policy.py b/nemo_rl/models/policy/tq_policy.py index 7d592dc0754..61da43511c6 100644 --- a/nemo_rl/models/policy/tq_policy.py +++ b/nemo_rl/models/policy/tq_policy.py @@ -364,6 +364,7 @@ def train_from_meta( gbs: Optional[int] = None, mbs: Optional[int] = None, timer: Optional[Timer] = None, + train_fields: tuple[str, ...] = DP_TRAIN_FIELDS, ) -> dict[str, Any]: """1-hop counterpart to :meth:`train`. @@ -379,6 +380,10 @@ def train_from_meta( gbs: Global batch size; defaults to ``cfg["train_global_batch_size"]``. mbs: Micro batch size; defaults to ``cfg["train_micro_batch_size"]``. timer: Optional timer for nested ``policy_training/*`` measurements. + train_fields: TQ columns workers fetch this step; defaults to the + full ``DP_TRAIN_FIELDS`` schema. Caller may narrow it to drop + columns it skipped writing (e.g. ``prev_logprobs`` when + ``force_on_policy_ratio=True``). Returns: Aggregated training-step output dict. @@ -388,14 +393,14 @@ def train_from_meta( self._stamp_pad_seqlen(meta) spa, dba = self._packing_args("train_mb_tokens") - # Train workers fetch the full DP_TRAIN_FIELDS schema (rollout + - # logprob deltas + advantages + sample_mask). Caller is responsible - # for ensuring those columns have been written to TQ before this - # call (workers + driver delta-writes). + # ``train_fields`` (rollout + logprob deltas + advantages + sample_mask; + # default ``DP_TRAIN_FIELDS``) must be in TQ before this call — written + # by workers + driver delta-writes. Caller may narrow to drop columns + # skipped this step (e.g. ``prev_logprobs`` under force_on_policy_ratio). train_meta = replace( meta, fields=fields_with_optional_routed_experts( - DP_TRAIN_FIELDS, enabled=self._router_replay_enabled + train_fields, enabled=self._router_replay_enabled ), task_name="train", ) diff --git a/tests/unit/algorithms/test_grpo.py b/tests/unit/algorithms/test_grpo.py index cc564c6daa5..7b8b2c703fd 100644 --- a/tests/unit/algorithms/test_grpo.py +++ b/tests/unit/algorithms/test_grpo.py @@ -35,8 +35,10 @@ _default_grpo_save_state, _initial_policy_generation_stale, _raise_if_reward_penalties_enabled_without_nemo_gym, + _resolve_logprob_skip_flags, _resolve_message_level_advantage_penalties, _should_use_async_rollouts, + _validate_use_kl_in_reward_compat, aggregate_rollout_metrics, async_grpo_train, compute_and_apply_seq_logprob_error_masking, @@ -45,6 +47,7 @@ refit_policy_generation, validate, ) +from nemo_rl.algorithms.grpo_sync import _train_fields_for_step from nemo_rl.algorithms.loss import ClippedPGLossConfig, ClippedPGLossFn from nemo_rl.algorithms.reward_functions import ( RewardShapingConfig, @@ -3629,3 +3632,58 @@ def test_mixed_metrics(self): assert result["total_turns"] == 45 assert result["accuracy"] == pytest.approx(0.8) assert result["min_accuracy_rate"] == pytest.approx(0.2) + + +def _cfg( + *, force=False, threshold=None, skip_ref=None, kl_reward=False, kl_penalty=0.01 +): + return MasterConfig.model_construct( + loss_fn=ClippedPGLossConfig( + force_on_policy_ratio=force, + use_kl_in_reward=kl_reward, + reference_policy_kl_penalty=kl_penalty, + ), + grpo={ + "seq_logprob_error_threshold": threshold, + "skip_reference_policy_logprobs_calculation": skip_ref, + }, + ) + + +@pytest.mark.parametrize( + "kw, expected", + [ + ({}, (False, None)), + ({"force": True}, (True, None)), + ({"force": True, "threshold": 1.5}, (False, None)), # threshold overrides skip + ({"skip_ref": True}, (False, True)), + ], + ids=["default", "force_on_policy", "force_plus_threshold", "skip_ref"], +) +def test_resolve_logprob_skip_flags(kw, expected): + if kw.get("force") and kw.get("threshold") is not None: + with pytest.warns(UserWarning, match="seq_logprob_error_threshold is set"): + assert _resolve_logprob_skip_flags(_cfg(**kw)) == expected + else: + assert _resolve_logprob_skip_flags(_cfg(**kw)) == expected + + +def test_validate_use_kl_in_reward_rejects_force_on_policy_ratio(): + with pytest.raises(AssertionError, match="use_kl_in_reward"): + _validate_use_kl_in_reward_compat(_cfg(force=True, kl_reward=True)) + + +def test_validate_use_kl_in_reward_allows_zero_kl_penalty(): + # kl_coef=0 zeros the KL term regardless, so a zero-placeholder + # prev_logprobs can't corrupt the advantage. + _validate_use_kl_in_reward_compat(_cfg(force=True, kl_reward=True, kl_penalty=0.0)) + + +@pytest.mark.parametrize( + "skip_prev_logprobs, expect_prev", + [(False, True), (True, False)], + ids=["keep_prev_logprobs", "skip_prev_logprobs"], +) +def test_train_fields_for_step(skip_prev_logprobs, expect_prev): + fields = _train_fields_for_step(skip_prev_logprobs) + assert ("prev_logprobs" in fields) is expect_prev