From 8831d62d76e607235a85aa55c078b9f47ca2c34a Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Sat, 4 Jul 2026 20:47:23 -0700 Subject: [PATCH 01/19] fix(grpo_sync): honor force_on_policy_ratio in TQ trainer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit grpo_sync.py always ran the prev_logprobs forward, ignoring loss_fn.force_on_policy_ratio — legacy grpo_train has skipped it since d1788b274. On grpo-qwen3-235b-16n8g this added ~16s/step (~4% total step time) of wasted forward compute plus TQ writeback. Port the skip: call _resolve_logprob_skip_flags, guard get_logprobs_from_meta on skip_prev_logprobs, write placeholder zeros to TQ so train_presharded still finds prev_logprobs, and emit placeholder seq_logprob_error_metrics via the shared factory. Extract _resolve_logprob_skip_flags and _placeholder_seq_logprob_error_metrics into private helpers in grpo.py; refactor grpo_train, async_grpo_train, and grpo_train_sync to share them (prevents future drift across the three trainers). Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Zhiyu Li --- nemo_rl/algorithms/grpo.py | 108 +++++++++++++++----------------- nemo_rl/algorithms/grpo_sync.py | 66 +++++++++++++------ 2 files changed, 96 insertions(+), 78 deletions(-) diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index 12baec843cd..6fccb42a5a3 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -2359,6 +2359,43 @@ def _log_mixed_rewards_and_advantages_information( metrics["advantages/mean"] = advantages.float().mean().item() +def _placeholder_seq_logprob_error_metrics() -> dict[str, float]: + """Zero-valued seq-level metrics used when the prev_logprobs forward is skipped.""" + return { + "max_seq_mult_prob_error": 0.0, + "mean_seq_mult_prob_error": 0.0, + "min_seq_mult_prob_error": 0.0, + "max_seq_mult_prob_error_after_mask": 0.0, + "mean_seq_mult_prob_error_after_mask": 0.0, + "min_seq_mult_prob_error_after_mask": 0.0, + "num_masked_seqs_by_logprob_error": 0, + "masked_correct_pct": 0.0, + } + + +def _resolve_logprob_skip_flags( + master_config: MasterConfig, +) -> tuple[bool, Any, Optional[float]]: + """Return (skip_prev, skip_ref, seq_logprob_error_threshold); warn on incompatible combos.""" + seq_logprob_error_threshold = master_config.grpo.get( + "seq_logprob_error_threshold", None + ) + skip_prev_logprobs = opd_module._skip_prev_logprobs(master_config) + if ( + master_config.loss_fn.force_on_policy_ratio + and seq_logprob_error_threshold is not None + ): + warnings.warn( + "force_on_policy_ratio=True but seq_logprob_error_threshold is set. " + "Computing prev_logprobs anyway for seq-level error masking." + ) + return ( + skip_prev_logprobs, + master_config.grpo.get("skip_reference_policy_logprobs_calculation"), + seq_logprob_error_threshold, + ) + + def compute_and_apply_seq_logprob_error_masking( train_data: BatchedDataDict, rewards: torch.Tensor, @@ -2952,24 +2989,11 @@ def grpo_train( metrics_logging_data["content"] = flat_messages["content"] memory_tracker.snapshot_start_of_stage("Computing logprobs", dir()) - # Skip prev_logprobs computation when force_on_policy_ratio=True - # unless seq_logprob_error_threshold is set (which requires prev_logprobs) - seq_logprob_error_threshold = master_config.grpo.get( - "seq_logprob_error_threshold", None - ) - force_on_policy_ratio = master_config.loss_fn.force_on_policy_ratio - skip_prev_logprobs = opd_module._skip_prev_logprobs(master_config) - # todo @jiaqi: is there a better way to skip prev_logprobs computation while still computing the seq-level error metrics? - if force_on_policy_ratio and seq_logprob_error_threshold is not None: - warnings.warn( - "force_on_policy_ratio=True but seq_logprob_error_threshold is set. " - "Computing prev_logprobs anyway for seq-level error masking." - ) - - # Skip reference_policy_logprobs computation when skip_reference_policy_logprobs_calculation=True - skip_reference_logprobs = master_config.grpo.get( - "skip_reference_policy_logprobs_calculation" - ) + ( + skip_prev_logprobs, + skip_reference_logprobs, + seq_logprob_error_threshold, + ) = _resolve_logprob_skip_flags(master_config) if not (skip_prev_logprobs and skip_reference_logprobs): print("▶ Preparing for logprob inference...", flush=True) @@ -3033,17 +3057,7 @@ def grpo_train( # Seq-level logprob error metrics/masking require real prev_logprobs if skip_prev_logprobs: - # Cannot compute seq-level metrics with placeholder prev_logprobs - seq_logprob_error_metrics = { - "max_seq_mult_prob_error": 0.0, - "mean_seq_mult_prob_error": 0.0, - "min_seq_mult_prob_error": 0.0, - "max_seq_mult_prob_error_after_mask": 0.0, - "mean_seq_mult_prob_error_after_mask": 0.0, - "min_seq_mult_prob_error_after_mask": 0.0, - "num_masked_seqs_by_logprob_error": 0, - "masked_correct_pct": 0.0, - } + seq_logprob_error_metrics = _placeholder_seq_logprob_error_metrics() else: seq_error_result = compute_and_apply_seq_logprob_error_masking( train_data=train_data, @@ -4357,25 +4371,11 @@ def async_grpo_train( train_data.to("cpu") # Training phase (same as sync version) - # Skip prev_logprobs computation when force_on_policy_ratio=True - # unless seq_logprob_error_threshold is set (which requires prev_logprobs) - seq_logprob_error_threshold = master_config.grpo.get( - "seq_logprob_error_threshold", None - ) - force_on_policy_ratio = master_config.loss_fn.force_on_policy_ratio - skip_prev_logprobs = opd_module._skip_prev_logprobs(master_config) - - # todo @jiaqi: is there a better way to skip prev_logprobs computation while still computing the seq-level error metrics? - if force_on_policy_ratio and seq_logprob_error_threshold is not None: - warnings.warn( - "force_on_policy_ratio=True but seq_logprob_error_threshold is set. " - "Computing prev_logprobs anyway for seq-level error masking." - ) - - # Skip reference_policy_logprobs computation when skip_reference_policy_logprobs_calculation=True - skip_reference_logprobs = master_config.grpo.get( - "skip_reference_policy_logprobs_calculation" - ) + ( + skip_prev_logprobs, + skip_reference_logprobs, + seq_logprob_error_threshold, + ) = _resolve_logprob_skip_flags(master_config) if not (skip_prev_logprobs and skip_reference_logprobs): print("▶ Preparing for logprob inference...", flush=True) @@ -4411,17 +4411,7 @@ def async_grpo_train( # Seq-level logprob error metrics/masking require real prev_logprobs if skip_prev_logprobs: - # Cannot compute seq-level metrics with placeholder prev_logprobs - seq_logprob_error_metrics = { - "max_seq_mult_prob_error": 0.0, - "mean_seq_mult_prob_error": 0.0, - "min_seq_mult_prob_error": 0.0, - "max_seq_mult_prob_error_after_mask": 0.0, - "mean_seq_mult_prob_error_after_mask": 0.0, - "min_seq_mult_prob_error_after_mask": 0.0, - "num_masked_seqs_by_logprob_error": 0, - "masked_correct_pct": 0.0, - } + seq_logprob_error_metrics = _placeholder_seq_logprob_error_metrics() else: seq_error_result = compute_and_apply_seq_logprob_error_masking( train_data=train_data, diff --git a/nemo_rl/algorithms/grpo_sync.py b/nemo_rl/algorithms/grpo_sync.py index a0a6485d6fb..2b5761b5b5d 100644 --- a/nemo_rl/algorithms/grpo_sync.py +++ b/nemo_rl/algorithms/grpo_sync.py @@ -51,6 +51,8 @@ _clip_grpo_advantages, _create_advantage_estimator, _log_mixed_rewards_and_advantages_information, + _placeholder_seq_logprob_error_metrics, + _resolve_logprob_skip_flags, _should_log_nemo_gym_responses, _should_use_nemo_gym, compute_and_apply_seq_logprob_error_masking, @@ -777,9 +779,16 @@ def grpo_train_sync( baseline_for_log = baseline.clone() memory_tracker.snapshot_start_of_stage("Computing logprobs", dir()) - print("▶ Preparing for logprob inference...", flush=True) - with timer.time("logprob_inference_prep"): - policy.prepare_for_lp_inference() + ( + skip_prev_logprobs, + skip_reference_logprobs, + seq_logprob_error_threshold, + ) = _resolve_logprob_skip_flags(master_config) + + if not (skip_prev_logprobs and skip_reference_logprobs): + print("▶ Preparing for logprob inference...", flush=True) + with timer.time("logprob_inference_prep"): + policy.prepare_for_lp_inference() print("▶ Computing logprobs...", flush=True) with timer.time("policy_and_reference_logprobs"): @@ -791,11 +800,14 @@ def grpo_train_sync( # batched fetch to avoid double-shipping the per-token # tensor through Ray's plasma store on top of the TQ # writeback. - policy.get_logprobs_from_meta(meta, timer=timer) - compute_ref = not master_config.grpo.get( - "skip_reference_policy_logprobs_calculation" - ) - if compute_ref: + if not skip_prev_logprobs: + policy.get_logprobs_from_meta(meta, timer=timer) + else: + print( + "▶ Skipping prev_logprobs (force_on_policy_ratio=True)...", + flush=True, + ) + if not skip_reference_logprobs: policy.get_reference_policy_logprobs_from_meta( meta, timer=timer, @@ -805,23 +817,41 @@ def grpo_train_sync( # for masking / advantage. Bulk (input_ids, multimodal, # output_ids, attention_mask, position_ids) stays in # TQ — workers will fetch it via ``train_presharded``. + select_fields = ["generation_logprobs", "token_mask"] + if not skip_prev_logprobs: + select_fields.append("prev_logprobs") + if not skip_reference_logprobs: + select_fields.append("reference_policy_logprobs") extras_bdd = policy.read_from_dataplane( meta, - select_fields=[ - "prev_logprobs", - "generation_logprobs", - "token_mask", - *(["reference_policy_logprobs"] if compute_ref else []), - ], + select_fields=select_fields, pad_value_dict=_pad_dict, ) - prev_logprobs = extras_bdd["prev_logprobs"] generation_logprobs = extras_bdd["generation_logprobs"] token_mask = extras_bdd["token_mask"] + if skip_prev_logprobs: + # Train workers fetch prev_logprobs via + # train_presharded (it's in DP_TRAIN_FIELDS), so the + # column must exist in TQ even when the forward was + # skipped. + prev_logprobs = torch.zeros_like(generation_logprobs) + policy.write_to_dataplane( + meta, + fields={"prev_logprobs": prev_logprobs}, + ) + else: + prev_logprobs = extras_bdd["prev_logprobs"] reference_policy_logprobs = ( - extras_bdd["reference_policy_logprobs"] if compute_ref else None + extras_bdd["reference_policy_logprobs"] + if not skip_reference_logprobs + else None ) + # Seq-level logprob error metrics/masking require real prev_logprobs + if skip_prev_logprobs: + sample_mask = loss_multiplier + seq_logprob_error_metrics = _placeholder_seq_logprob_error_metrics() + else: sample_mask, seq_logprob_error_metrics = ( _compute_seq_logprob_error_metrics( token_mask=token_mask, @@ -829,9 +859,7 @@ def grpo_train_sync( prev_logprobs=prev_logprobs, generation_logprobs=generation_logprobs, rewards=rewards, - seq_logprob_error_threshold=master_config.grpo[ - "seq_logprob_error_threshold" - ], + seq_logprob_error_threshold=seq_logprob_error_threshold, ) ) From efb30813004fc36ff154e830f6415dedd807e79b Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Mon, 6 Jul 2026 10:10:44 -0700 Subject: [PATCH 02/19] refactor(grpo): drop misleading 0-fill for skipped seq_logprob metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Emitting 8 keys pinned to 0.0 whenever prev_logprobs is skipped (force_on_policy_ratio=True) polluted wandb charts with a permanent zero line that reads as "seq-level error was 0" — a lie, since no measurement was taken. metrics.update({}) leaves the keys off for those steps, which is truthful. Also restore the pre-existing "Skip prev_logprobs computation when..." and "todo @jiaqi:" comments that were dropped when the flag-resolution block was extracted into _resolve_logprob_skip_flags. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Zhiyu Li --- nemo_rl/algorithms/grpo.py | 26 ++++++++++---------------- nemo_rl/algorithms/grpo_sync.py | 5 +++-- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index 6fccb42a5a3..14865bda1cf 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -2359,28 +2359,17 @@ def _log_mixed_rewards_and_advantages_information( metrics["advantages/mean"] = advantages.float().mean().item() -def _placeholder_seq_logprob_error_metrics() -> dict[str, float]: - """Zero-valued seq-level metrics used when the prev_logprobs forward is skipped.""" - return { - "max_seq_mult_prob_error": 0.0, - "mean_seq_mult_prob_error": 0.0, - "min_seq_mult_prob_error": 0.0, - "max_seq_mult_prob_error_after_mask": 0.0, - "mean_seq_mult_prob_error_after_mask": 0.0, - "min_seq_mult_prob_error_after_mask": 0.0, - "num_masked_seqs_by_logprob_error": 0, - "masked_correct_pct": 0.0, - } - - def _resolve_logprob_skip_flags( master_config: MasterConfig, ) -> tuple[bool, Any, Optional[float]]: """Return (skip_prev, skip_ref, seq_logprob_error_threshold); warn on incompatible combos.""" + # Skip prev_logprobs computation when force_on_policy_ratio=True + # unless seq_logprob_error_threshold is set (which requires prev_logprobs) seq_logprob_error_threshold = master_config.grpo.get( "seq_logprob_error_threshold", None ) skip_prev_logprobs = opd_module._skip_prev_logprobs(master_config) + # todo @jiaqi: is there a better way to skip prev_logprobs computation while still computing the seq-level error metrics? if ( master_config.loss_fn.force_on_policy_ratio and seq_logprob_error_threshold is not None @@ -2389,6 +2378,7 @@ def _resolve_logprob_skip_flags( "force_on_policy_ratio=True but seq_logprob_error_threshold is set. " "Computing prev_logprobs anyway for seq-level error masking." ) + # Skip reference_policy_logprobs computation when skip_reference_policy_logprobs_calculation=True return ( skip_prev_logprobs, master_config.grpo.get("skip_reference_policy_logprobs_calculation"), @@ -3057,7 +3047,9 @@ def grpo_train( # Seq-level logprob error metrics/masking require real prev_logprobs if skip_prev_logprobs: - seq_logprob_error_metrics = _placeholder_seq_logprob_error_metrics() + # No valid measurements — leave the seq-level metrics + # out of the emitted dict rather than pinning them to 0. + seq_logprob_error_metrics: dict[str, Any] = {} else: seq_error_result = compute_and_apply_seq_logprob_error_masking( train_data=train_data, @@ -4411,7 +4403,9 @@ def async_grpo_train( # Seq-level logprob error metrics/masking require real prev_logprobs if skip_prev_logprobs: - seq_logprob_error_metrics = _placeholder_seq_logprob_error_metrics() + # No valid measurements — leave the seq-level metrics + # out of the emitted dict rather than pinning them to 0. + seq_logprob_error_metrics: dict[str, Any] = {} else: seq_error_result = compute_and_apply_seq_logprob_error_masking( train_data=train_data, diff --git a/nemo_rl/algorithms/grpo_sync.py b/nemo_rl/algorithms/grpo_sync.py index 2b5761b5b5d..1eab393541d 100644 --- a/nemo_rl/algorithms/grpo_sync.py +++ b/nemo_rl/algorithms/grpo_sync.py @@ -51,7 +51,6 @@ _clip_grpo_advantages, _create_advantage_estimator, _log_mixed_rewards_and_advantages_information, - _placeholder_seq_logprob_error_metrics, _resolve_logprob_skip_flags, _should_log_nemo_gym_responses, _should_use_nemo_gym, @@ -850,7 +849,9 @@ def grpo_train_sync( # Seq-level logprob error metrics/masking require real prev_logprobs if skip_prev_logprobs: sample_mask = loss_multiplier - seq_logprob_error_metrics = _placeholder_seq_logprob_error_metrics() + # No valid measurements — leave the seq-level metrics + # out of the emitted dict rather than pinning them to 0. + seq_logprob_error_metrics: dict[str, Any] = {} else: sample_mask, seq_logprob_error_metrics = ( _compute_seq_logprob_error_metrics( From d0efc1c79f62ec2893c9e6f619739e7f4e2c4fda Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Mon, 6 Jul 2026 10:24:09 -0700 Subject: [PATCH 03/19] perf(grpo_sync): drop prev_logprobs TQ round-trip when force_on_policy_ratio MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When loss_fn.force_on_policy_ratio=True the loss uses curr_logprobs in place of prev_logprobs (loss_functions.py:322-324, 337-338) — the fetch is pure waste. The prior fix wrote a placeholder zero tensor into TQ just so train_presharded's DP_TRAIN_FIELDS fetch would succeed. Drop prev_logprobs from the train fetch instead: TQPolicy.train_from_meta now inspects loss_fn.force_on_policy_ratio and removes prev_logprobs from meta.fields, and the driver no longer writes zeros to TQ. The driver-side local `prev_logprobs = torch.zeros_like(generation_logprobs)` stays (it's still passed to compute_advantage, mirroring legacy). Also restore _placeholder_seq_logprob_error_metrics() — the all-zero seq-level metrics dict is intentional (keeps wandb charts continuous across steps that skip prev_logprobs, matching legacy behavior). Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Zhiyu Li --- nemo_rl/algorithms/grpo.py | 24 ++++++++++++++++++------ nemo_rl/algorithms/grpo_sync.py | 20 +++++++++----------- nemo_rl/models/policy/tq_policy.py | 9 ++++++++- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index 14865bda1cf..d7e28a9d144 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -2359,6 +2359,20 @@ def _log_mixed_rewards_and_advantages_information( metrics["advantages/mean"] = advantages.float().mean().item() +def _placeholder_seq_logprob_error_metrics() -> dict[str, float]: + """Zero-valued seq-level metrics used when the prev_logprobs forward is skipped.""" + return { + "max_seq_mult_prob_error": 0.0, + "mean_seq_mult_prob_error": 0.0, + "min_seq_mult_prob_error": 0.0, + "max_seq_mult_prob_error_after_mask": 0.0, + "mean_seq_mult_prob_error_after_mask": 0.0, + "min_seq_mult_prob_error_after_mask": 0.0, + "num_masked_seqs_by_logprob_error": 0, + "masked_correct_pct": 0.0, + } + + def _resolve_logprob_skip_flags( master_config: MasterConfig, ) -> tuple[bool, Any, Optional[float]]: @@ -3047,9 +3061,8 @@ def grpo_train( # Seq-level logprob error metrics/masking require real prev_logprobs if skip_prev_logprobs: - # No valid measurements — leave the seq-level metrics - # out of the emitted dict rather than pinning them to 0. - seq_logprob_error_metrics: dict[str, Any] = {} + # Cannot compute seq-level metrics with placeholder prev_logprobs + seq_logprob_error_metrics = _placeholder_seq_logprob_error_metrics() else: seq_error_result = compute_and_apply_seq_logprob_error_masking( train_data=train_data, @@ -4403,9 +4416,8 @@ def async_grpo_train( # Seq-level logprob error metrics/masking require real prev_logprobs if skip_prev_logprobs: - # No valid measurements — leave the seq-level metrics - # out of the emitted dict rather than pinning them to 0. - seq_logprob_error_metrics: dict[str, Any] = {} + # Cannot compute seq-level metrics with placeholder prev_logprobs + seq_logprob_error_metrics = _placeholder_seq_logprob_error_metrics() else: seq_error_result = compute_and_apply_seq_logprob_error_masking( train_data=train_data, diff --git a/nemo_rl/algorithms/grpo_sync.py b/nemo_rl/algorithms/grpo_sync.py index 1eab393541d..3a7a3031a6f 100644 --- a/nemo_rl/algorithms/grpo_sync.py +++ b/nemo_rl/algorithms/grpo_sync.py @@ -51,6 +51,7 @@ _clip_grpo_advantages, _create_advantage_estimator, _log_mixed_rewards_and_advantages_information, + _placeholder_seq_logprob_error_metrics, _resolve_logprob_skip_flags, _should_log_nemo_gym_responses, _should_use_nemo_gym, @@ -829,15 +830,13 @@ def grpo_train_sync( generation_logprobs = extras_bdd["generation_logprobs"] token_mask = extras_bdd["token_mask"] if skip_prev_logprobs: - # Train workers fetch prev_logprobs via - # train_presharded (it's in DP_TRAIN_FIELDS), so the - # column must exist in TQ even when the forward was - # skipped. + # Driver-local placeholder for the advantage-compute + # call below; not written to TQ. Train workers drop + # prev_logprobs from the fetch when + # loss_fn.force_on_policy_ratio is True + # (see TQPolicy.train_from_meta) and the loss then + # uses curr_logprobs instead. prev_logprobs = torch.zeros_like(generation_logprobs) - policy.write_to_dataplane( - meta, - fields={"prev_logprobs": prev_logprobs}, - ) else: prev_logprobs = extras_bdd["prev_logprobs"] reference_policy_logprobs = ( @@ -849,9 +848,8 @@ def grpo_train_sync( # Seq-level logprob error metrics/masking require real prev_logprobs if skip_prev_logprobs: sample_mask = loss_multiplier - # No valid measurements — leave the seq-level metrics - # out of the emitted dict rather than pinning them to 0. - seq_logprob_error_metrics: dict[str, Any] = {} + # Cannot compute seq-level metrics with placeholder prev_logprobs + seq_logprob_error_metrics = _placeholder_seq_logprob_error_metrics() else: sample_mask, seq_logprob_error_metrics = ( _compute_seq_logprob_error_metrics( diff --git a/nemo_rl/models/policy/tq_policy.py b/nemo_rl/models/policy/tq_policy.py index 7d592dc0754..0bfbd7eb430 100644 --- a/nemo_rl/models/policy/tq_policy.py +++ b/nemo_rl/models/policy/tq_policy.py @@ -392,10 +392,17 @@ def train_from_meta( # logprob deltas + advantages + sample_mask). Caller is responsible # for ensuring those columns have been written to TQ before this # call (workers + driver delta-writes). + # When force_on_policy_ratio=True the loss uses curr_logprobs in + # place of prev_logprobs (see ClippedPGLoss), so we drop it from + # the fetch instead of forcing the driver to write placeholder + # zeros back to TQ. + train_fields: tuple[str, ...] = DP_TRAIN_FIELDS + if getattr(loss_fn, "force_on_policy_ratio", False): + train_fields = tuple(f for f in train_fields if f != "prev_logprobs") train_meta = replace( meta, fields=fields_with_optional_routed_experts( - DP_TRAIN_FIELDS, enabled=self._router_replay_enabled + train_fields, enabled=self._router_replay_enabled ), task_name="train", ) From 141afd0e4c0a59694840ad2f4d67d86d7cc5fe66 Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Mon, 6 Jul 2026 12:21:17 -0700 Subject: [PATCH 04/19] refactor(grpo_sync): tighten helper + unify select_fields with dispatch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up cleanup on the force_on_policy_ratio TQ port: - _resolve_logprob_skip_flags no longer returns seq_logprob_error_threshold; callers grab it from master_config themselves (was only used at the call site anyway). Helper is now a 2-tuple. - grpo_sync.py fuses "should we run the forward?" with "add to select_fields" into a single ``if compute_prev`` / ``if compute_ref`` branch — no more parallel guards on the same flag. Restores the positive-named locals (compute_prev, compute_ref) for readability. - TQPolicy.train_from_meta comment tightened; annotation on train_fields dropped (redundant with the tuple-comprehension type). - Local-zeros comment for the driver-side placeholder shortened to two lines pointing at the invariant (workers skip the fetch, see train_from_meta). Empirically identical: run 13498411 already validated the semantics; this patch only shuffles code inside the same functions. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Zhiyu Li --- nemo_rl/algorithms/grpo.py | 46 +++++++++++++-------------- nemo_rl/algorithms/grpo_sync.py | 51 +++++++++++++++--------------- nemo_rl/models/policy/tq_policy.py | 9 +++--- 3 files changed, 51 insertions(+), 55 deletions(-) diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index d7e28a9d144..c868527d82e 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -2373,30 +2373,26 @@ def _placeholder_seq_logprob_error_metrics() -> dict[str, float]: } -def _resolve_logprob_skip_flags( - master_config: MasterConfig, -) -> tuple[bool, Any, Optional[float]]: - """Return (skip_prev, skip_ref, seq_logprob_error_threshold); warn on incompatible combos.""" - # Skip prev_logprobs computation when force_on_policy_ratio=True - # unless seq_logprob_error_threshold is set (which requires prev_logprobs) - seq_logprob_error_threshold = master_config.grpo.get( - "seq_logprob_error_threshold", None - ) - skip_prev_logprobs = opd_module._skip_prev_logprobs(master_config) +def _resolve_logprob_skip_flags(master_config: MasterConfig) -> tuple[bool, Any]: + """Return (skip_prev_logprobs, skip_reference_logprobs); warn on incompatible combos. + + Skip prev_logprobs when force_on_policy_ratio=True unless + seq_logprob_error_threshold is set (which requires prev_logprobs). + Skip reference_policy_logprobs when + ``grpo.skip_reference_policy_logprobs_calculation`` is set. + """ # todo @jiaqi: is there a better way to skip prev_logprobs computation while still computing the seq-level error metrics? if ( master_config.loss_fn.force_on_policy_ratio - and seq_logprob_error_threshold is not None + and master_config.grpo.get("seq_logprob_error_threshold") is not None ): warnings.warn( "force_on_policy_ratio=True but seq_logprob_error_threshold is set. " "Computing prev_logprobs anyway for seq-level error masking." ) - # Skip reference_policy_logprobs computation when skip_reference_policy_logprobs_calculation=True return ( - skip_prev_logprobs, + opd_module._skip_prev_logprobs(master_config), master_config.grpo.get("skip_reference_policy_logprobs_calculation"), - seq_logprob_error_threshold, ) @@ -2993,11 +2989,12 @@ def grpo_train( metrics_logging_data["content"] = flat_messages["content"] memory_tracker.snapshot_start_of_stage("Computing logprobs", dir()) - ( - skip_prev_logprobs, - skip_reference_logprobs, - seq_logprob_error_threshold, - ) = _resolve_logprob_skip_flags(master_config) + skip_prev_logprobs, skip_reference_logprobs = ( + _resolve_logprob_skip_flags(master_config) + ) + seq_logprob_error_threshold = master_config.grpo.get( + "seq_logprob_error_threshold", None + ) if not (skip_prev_logprobs and skip_reference_logprobs): print("▶ Preparing for logprob inference...", flush=True) @@ -4376,11 +4373,12 @@ def async_grpo_train( train_data.to("cpu") # Training phase (same as sync version) - ( - skip_prev_logprobs, - skip_reference_logprobs, - seq_logprob_error_threshold, - ) = _resolve_logprob_skip_flags(master_config) + skip_prev_logprobs, skip_reference_logprobs = ( + _resolve_logprob_skip_flags(master_config) + ) + seq_logprob_error_threshold = master_config.grpo.get( + "seq_logprob_error_threshold", None + ) if not (skip_prev_logprobs and skip_reference_logprobs): print("▶ Preparing for logprob inference...", flush=True) diff --git a/nemo_rl/algorithms/grpo_sync.py b/nemo_rl/algorithms/grpo_sync.py index 3a7a3031a6f..6ce642a8e4a 100644 --- a/nemo_rl/algorithms/grpo_sync.py +++ b/nemo_rl/algorithms/grpo_sync.py @@ -779,13 +779,16 @@ def grpo_train_sync( baseline_for_log = baseline.clone() memory_tracker.snapshot_start_of_stage("Computing logprobs", dir()) - ( - skip_prev_logprobs, - skip_reference_logprobs, - seq_logprob_error_threshold, - ) = _resolve_logprob_skip_flags(master_config) + skip_prev_logprobs, skip_reference_logprobs = ( + _resolve_logprob_skip_flags(master_config) + ) + compute_prev = not skip_prev_logprobs + compute_ref = not skip_reference_logprobs + seq_logprob_error_threshold = master_config.grpo.get( + "seq_logprob_error_threshold", None + ) - if not (skip_prev_logprobs and skip_reference_logprobs): + if compute_prev or compute_ref: print("▶ Preparing for logprob inference...", flush=True) with timer.time("logprob_inference_prep"): policy.prepare_for_lp_inference() @@ -800,28 +803,29 @@ def grpo_train_sync( # batched fetch to avoid double-shipping the per-token # tensor through Ray's plasma store on top of the TQ # writeback. - if not skip_prev_logprobs: + # + # Each ``compute_X`` guard doubles as the "add to + # select_fields" gate — one flag, one branch. + select_fields = ["generation_logprobs", "token_mask"] + if compute_prev: policy.get_logprobs_from_meta(meta, timer=timer) + select_fields.append("prev_logprobs") else: print( "▶ Skipping prev_logprobs (force_on_policy_ratio=True)...", flush=True, ) - if not skip_reference_logprobs: + if compute_ref: policy.get_reference_policy_logprobs_from_meta( meta, timer=timer, ) + select_fields.append("reference_policy_logprobs") # Driver pulls only the per-token columns it needs # for masking / advantage. Bulk (input_ids, multimodal, # output_ids, attention_mask, position_ids) stays in # TQ — workers will fetch it via ``train_presharded``. - select_fields = ["generation_logprobs", "token_mask"] - if not skip_prev_logprobs: - select_fields.append("prev_logprobs") - if not skip_reference_logprobs: - select_fields.append("reference_policy_logprobs") extras_bdd = policy.read_from_dataplane( meta, select_fields=select_fields, @@ -829,20 +833,15 @@ def grpo_train_sync( ) generation_logprobs = extras_bdd["generation_logprobs"] token_mask = extras_bdd["token_mask"] - if skip_prev_logprobs: - # Driver-local placeholder for the advantage-compute - # call below; not written to TQ. Train workers drop - # prev_logprobs from the fetch when - # loss_fn.force_on_policy_ratio is True - # (see TQPolicy.train_from_meta) and the loss then - # uses curr_logprobs instead. - prev_logprobs = torch.zeros_like(generation_logprobs) - else: - prev_logprobs = extras_bdd["prev_logprobs"] + # Local zeros for compute_advantage below; workers skip + # fetching prev_logprobs (see TQPolicy.train_from_meta). + prev_logprobs = ( + extras_bdd["prev_logprobs"] + if compute_prev + else torch.zeros_like(generation_logprobs) + ) reference_policy_logprobs = ( - extras_bdd["reference_policy_logprobs"] - if not skip_reference_logprobs - else None + extras_bdd["reference_policy_logprobs"] if compute_ref else None ) # Seq-level logprob error metrics/masking require real prev_logprobs diff --git a/nemo_rl/models/policy/tq_policy.py b/nemo_rl/models/policy/tq_policy.py index 0bfbd7eb430..85761c88708 100644 --- a/nemo_rl/models/policy/tq_policy.py +++ b/nemo_rl/models/policy/tq_policy.py @@ -392,11 +392,10 @@ def train_from_meta( # logprob deltas + advantages + sample_mask). Caller is responsible # for ensuring those columns have been written to TQ before this # call (workers + driver delta-writes). - # When force_on_policy_ratio=True the loss uses curr_logprobs in - # place of prev_logprobs (see ClippedPGLoss), so we drop it from - # the fetch instead of forcing the driver to write placeholder - # zeros back to TQ. - train_fields: tuple[str, ...] = DP_TRAIN_FIELDS + # force_on_policy_ratio=True → loss uses curr_logprobs in place + # of prev_logprobs, so drop it from the fetch entirely + # (dp_client rejects missing fields — see noop adapter). + train_fields = DP_TRAIN_FIELDS if getattr(loss_fn, "force_on_policy_ratio", False): train_fields = tuple(f for f in train_fields if f != "prev_logprobs") train_meta = replace( From 088c9bbd6a9b05e9f38b41d95eb35c40284b91d7 Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Mon, 6 Jul 2026 13:04:29 -0700 Subject: [PATCH 05/19] refactor(grpo_sync): thread skip_prev_logprobs through train_from_meta Drop the duplicated skip decision. The driver already resolved skip_prev_logprobs via _resolve_logprob_skip_flags; TQPolicy.train_from_meta now accepts it as an argument instead of independently inspecting loss_fn.force_on_policy_ratio. Single config, single source of truth, threaded from driver to policy. Also remove a stray meta-comment that described the refactor itself rather than the code, and tighten the driver-local placeholder comment. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Zhiyu Li --- nemo_rl/algorithms/grpo_sync.py | 8 +++----- nemo_rl/models/policy/tq_policy.py | 9 +++++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/nemo_rl/algorithms/grpo_sync.py b/nemo_rl/algorithms/grpo_sync.py index 6ce642a8e4a..b0adb9cf104 100644 --- a/nemo_rl/algorithms/grpo_sync.py +++ b/nemo_rl/algorithms/grpo_sync.py @@ -803,9 +803,6 @@ def grpo_train_sync( # batched fetch to avoid double-shipping the per-token # tensor through Ray's plasma store on top of the TQ # writeback. - # - # Each ``compute_X`` guard doubles as the "add to - # select_fields" gate — one flag, one branch. select_fields = ["generation_logprobs", "token_mask"] if compute_prev: policy.get_logprobs_from_meta(meta, timer=timer) @@ -833,8 +830,8 @@ def grpo_train_sync( ) generation_logprobs = extras_bdd["generation_logprobs"] token_mask = extras_bdd["token_mask"] - # Local zeros for compute_advantage below; workers skip - # fetching prev_logprobs (see TQPolicy.train_from_meta). + # Placeholder zeros satisfying compute_advantage's + # signature; not shipped to workers. prev_logprobs = ( extras_bdd["prev_logprobs"] if compute_prev @@ -926,6 +923,7 @@ def grpo_train_sync( meta, loss_fn=loss_fn, timer=timer, + skip_prev_logprobs=skip_prev_logprobs, ) if sync_kv_scales: diff --git a/nemo_rl/models/policy/tq_policy.py b/nemo_rl/models/policy/tq_policy.py index 85761c88708..ecb419503d1 100644 --- a/nemo_rl/models/policy/tq_policy.py +++ b/nemo_rl/models/policy/tq_policy.py @@ -364,6 +364,7 @@ def train_from_meta( gbs: Optional[int] = None, mbs: Optional[int] = None, timer: Optional[Timer] = None, + skip_prev_logprobs: bool = False, ) -> dict[str, Any]: """1-hop counterpart to :meth:`train`. @@ -392,11 +393,11 @@ def train_from_meta( # logprob deltas + advantages + sample_mask). Caller is responsible # for ensuring those columns have been written to TQ before this # call (workers + driver delta-writes). - # force_on_policy_ratio=True → loss uses curr_logprobs in place - # of prev_logprobs, so drop it from the fetch entirely - # (dp_client rejects missing fields — see noop adapter). + # skip_prev_logprobs=True → loss uses curr_logprobs in place of + # prev_logprobs, so drop it from the fetch entirely (dp_client + # rejects unknown fields). train_fields = DP_TRAIN_FIELDS - if getattr(loss_fn, "force_on_policy_ratio", False): + if skip_prev_logprobs: train_fields = tuple(f for f in train_fields if f != "prev_logprobs") train_meta = replace( meta, From 6cb80f08c440407dd93abb16f5b592245c83ba58 Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Mon, 6 Jul 2026 13:08:24 -0700 Subject: [PATCH 06/19] refactor(grpo_sync): driver owns the train field list Move the "which fields workers should fetch" decision fully into the driver. TQPolicy.train_from_meta now takes an optional ``train_fields`` (defaulting to DP_TRAIN_FIELDS), so the policy doesn't re-derive it from loss config. The driver computes the tuple once, right after resolving skip flags, and reuses that single source for both the driver-side read_from_dataplane and the worker-side train_presharded fetch. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Zhiyu Li --- nemo_rl/algorithms/grpo_sync.py | 11 +++++++++-- nemo_rl/models/policy/tq_policy.py | 12 +++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/nemo_rl/algorithms/grpo_sync.py b/nemo_rl/algorithms/grpo_sync.py index b0adb9cf104..5d2cba10c18 100644 --- a/nemo_rl/algorithms/grpo_sync.py +++ b/nemo_rl/algorithms/grpo_sync.py @@ -73,7 +73,7 @@ from nemo_rl.data.interfaces import DatumSpec from nemo_rl.data.llm_message_utils import batched_message_log_to_flat_message from nemo_rl.data_plane.interfaces import KVBatchMeta -from nemo_rl.data_plane.schema import DP_CALIB_INPUT_FIELDS +from nemo_rl.data_plane.schema import DP_CALIB_INPUT_FIELDS, DP_TRAIN_FIELDS from nemo_rl.distributed.batched_data_dict import BatchedDataDict from nemo_rl.environments.interfaces import EnvironmentInterface from nemo_rl.experience.sync_rollout_actor import SyncRolloutActor @@ -787,6 +787,13 @@ def grpo_train_sync( seq_logprob_error_threshold = master_config.grpo.get( "seq_logprob_error_threshold", None ) + # Effective field set for this step. Filtered once; both + # the driver-side read_from_dataplane and the workers' + # train_presharded fetch (via train_from_meta) consume it. + train_fields = tuple( + f for f in DP_TRAIN_FIELDS + if not (skip_prev_logprobs and f == "prev_logprobs") + ) if compute_prev or compute_ref: print("▶ Preparing for logprob inference...", flush=True) @@ -923,7 +930,7 @@ def grpo_train_sync( meta, loss_fn=loss_fn, timer=timer, - skip_prev_logprobs=skip_prev_logprobs, + train_fields=train_fields, ) if sync_kv_scales: diff --git a/nemo_rl/models/policy/tq_policy.py b/nemo_rl/models/policy/tq_policy.py index ecb419503d1..ee1fc700677 100644 --- a/nemo_rl/models/policy/tq_policy.py +++ b/nemo_rl/models/policy/tq_policy.py @@ -364,7 +364,7 @@ def train_from_meta( gbs: Optional[int] = None, mbs: Optional[int] = None, timer: Optional[Timer] = None, - skip_prev_logprobs: bool = False, + train_fields: Optional[tuple[str, ...]] = None, ) -> dict[str, Any]: """1-hop counterpart to :meth:`train`. @@ -393,12 +393,10 @@ def train_from_meta( # logprob deltas + advantages + sample_mask). Caller is responsible # for ensuring those columns have been written to TQ before this # call (workers + driver delta-writes). - # skip_prev_logprobs=True → loss uses curr_logprobs in place of - # prev_logprobs, so drop it from the fetch entirely (dp_client - # rejects unknown fields). - train_fields = DP_TRAIN_FIELDS - if skip_prev_logprobs: - train_fields = tuple(f for f in train_fields if f != "prev_logprobs") + # Caller may narrow the fetch schema (e.g. drop prev_logprobs + # when force_on_policy_ratio=True); default to DP_TRAIN_FIELDS. + if train_fields is None: + train_fields = DP_TRAIN_FIELDS train_meta = replace( meta, fields=fields_with_optional_routed_experts( From 50b1ccffc67e6d075dfa4490d518250e40791559 Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Mon, 6 Jul 2026 13:09:13 -0700 Subject: [PATCH 07/19] refactor(tq_policy): use DP_TRAIN_FIELDS as direct default for train_fields tuple is immutable, so we can put DP_TRAIN_FIELDS in the default arg directly instead of the None sentinel + runtime fallback. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Zhiyu Li --- nemo_rl/models/policy/tq_policy.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nemo_rl/models/policy/tq_policy.py b/nemo_rl/models/policy/tq_policy.py index ee1fc700677..a669affc418 100644 --- a/nemo_rl/models/policy/tq_policy.py +++ b/nemo_rl/models/policy/tq_policy.py @@ -364,7 +364,7 @@ def train_from_meta( gbs: Optional[int] = None, mbs: Optional[int] = None, timer: Optional[Timer] = None, - train_fields: Optional[tuple[str, ...]] = None, + train_fields: tuple[str, ...] = DP_TRAIN_FIELDS, ) -> dict[str, Any]: """1-hop counterpart to :meth:`train`. @@ -394,9 +394,7 @@ def train_from_meta( # for ensuring those columns have been written to TQ before this # call (workers + driver delta-writes). # Caller may narrow the fetch schema (e.g. drop prev_logprobs - # when force_on_policy_ratio=True); default to DP_TRAIN_FIELDS. - if train_fields is None: - train_fields = DP_TRAIN_FIELDS + # when force_on_policy_ratio=True). train_meta = replace( meta, fields=fields_with_optional_routed_experts( From 94e5164f99b3c4c643b6879d8c82fe9f8ce9f0df Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Mon, 6 Jul 2026 13:14:20 -0700 Subject: [PATCH 08/19] docs(grpo_sync): correct the prev_logprobs placeholder comment Old comment mentioned "not shipped to workers", which describes the TQ path (now handled by train_fields), not the reason we allocate a zero tensor at this line. The real reason: compute_advantage below takes a logprobs_policy arg that GRPO/Reinforce++ ignore, and OPD is forbidden in the skip_prev_logprobs=True path (asserted by opd._skip_prev_logprobs). Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Zhiyu Li --- nemo_rl/algorithms/grpo_sync.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nemo_rl/algorithms/grpo_sync.py b/nemo_rl/algorithms/grpo_sync.py index 5d2cba10c18..55b5034b0a4 100644 --- a/nemo_rl/algorithms/grpo_sync.py +++ b/nemo_rl/algorithms/grpo_sync.py @@ -837,8 +837,9 @@ def grpo_train_sync( ) generation_logprobs = extras_bdd["generation_logprobs"] token_mask = extras_bdd["token_mask"] - # Placeholder zeros satisfying compute_advantage's - # signature; not shipped to workers. + # Unused placeholder for compute_advantage's + # logprobs_policy arg — GRPO/Reinforce++ ignore it, + # and OPD is forbidden here (asserted by opd module). prev_logprobs = ( extras_bdd["prev_logprobs"] if compute_prev From b839b14847026777877f3905934afe018ff24645 Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Mon, 6 Jul 2026 13:16:29 -0700 Subject: [PATCH 09/19] refactor(grpo_sync): pass None to compute_advantage when skipping prev_logprobs torch.zeros_like(generation_logprobs) allocated a full [B, S] tensor just to satisfy compute_advantage's logprobs_policy arg. All estimators handle logprobs_policy=None: GRPO/Reinforce/GDPO/GAE either swallow it in **kwargs or gate on ``is not None``, and OPD is asserted out of the skip_prev_logprobs=True path by opd._skip_prev_logprobs. Drop the allocation. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Zhiyu Li --- nemo_rl/algorithms/grpo_sync.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/nemo_rl/algorithms/grpo_sync.py b/nemo_rl/algorithms/grpo_sync.py index 55b5034b0a4..67b3e7350fe 100644 --- a/nemo_rl/algorithms/grpo_sync.py +++ b/nemo_rl/algorithms/grpo_sync.py @@ -837,13 +837,8 @@ def grpo_train_sync( ) generation_logprobs = extras_bdd["generation_logprobs"] token_mask = extras_bdd["token_mask"] - # Unused placeholder for compute_advantage's - # logprobs_policy arg — GRPO/Reinforce++ ignore it, - # and OPD is forbidden here (asserted by opd module). prev_logprobs = ( - extras_bdd["prev_logprobs"] - if compute_prev - else torch.zeros_like(generation_logprobs) + extras_bdd["prev_logprobs"] if compute_prev else None ) reference_policy_logprobs = ( extras_bdd["reference_policy_logprobs"] if compute_ref else None From a82d3ef6a8a90aa3142c151f0a0bec2996c8b58a Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Mon, 6 Jul 2026 13:42:00 -0700 Subject: [PATCH 10/19] fix(grpo_sync): guard log_data prev_logprobs.tolist() against None MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit switched prev_logprobs to None when force_on_policy_ratio skips the forward, but missed a downstream debug-log consumer that calls .tolist() on it. AttributeError: 'NoneType' object has no attribute 'tolist' on step 1. Emit None in the log dump (JSON null) when skipped — accurate reflection of what the driver actually computed. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Zhiyu Li --- nemo_rl/algorithms/grpo_sync.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nemo_rl/algorithms/grpo_sync.py b/nemo_rl/algorithms/grpo_sync.py index 67b3e7350fe..b27d91230e0 100644 --- a/nemo_rl/algorithms/grpo_sync.py +++ b/nemo_rl/algorithms/grpo_sync.py @@ -1214,7 +1214,9 @@ def grpo_train_sync( log_data["sample_loss_mask"] = sample_mask.tolist() log_data["advantages"] = advantages.tolist() log_data["generation_logprobs"] = generation_logprobs.tolist() - log_data["prev_logprobs"] = prev_logprobs.tolist() + log_data["prev_logprobs"] = ( + prev_logprobs.tolist() if prev_logprobs is not None else None + ) # input_ids was stashed before the step-end clear_samples (the # keys are no longer in TQ at this point); ``_log_input_ids`` # is None when nemo_gym-responses logging path skipped the From 435236dcd2137a83c90cef1e88a7ac10ced1a5c1 Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Mon, 6 Jul 2026 13:44:37 -0700 Subject: [PATCH 11/19] Revert "refactor(grpo_sync): pass None to compute_advantage when skipping prev_logprobs" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Legacy grpo.py::grpo_train and async_grpo_train both set train_data["prev_logprobs"] = torch.zeros_like(...) when force_on_policy_ratio skips the forward, and call .tolist() unconditionally in the debug-log path. Diverging in grpo_sync.py — passing None + guarding .tolist() — was a wash (the 32 MB allocation is trivial vs. training step time) and broke the debug-log path with an AttributeError. Revert to zeros for line-by-line parity with legacy. This reverts commits a09d39f99 and e49cb5ae7. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Zhiyu Li --- nemo_rl/algorithms/grpo_sync.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nemo_rl/algorithms/grpo_sync.py b/nemo_rl/algorithms/grpo_sync.py index b27d91230e0..24c020c7e2a 100644 --- a/nemo_rl/algorithms/grpo_sync.py +++ b/nemo_rl/algorithms/grpo_sync.py @@ -838,7 +838,9 @@ def grpo_train_sync( generation_logprobs = extras_bdd["generation_logprobs"] token_mask = extras_bdd["token_mask"] prev_logprobs = ( - extras_bdd["prev_logprobs"] if compute_prev else None + extras_bdd["prev_logprobs"] + if compute_prev + else torch.zeros_like(generation_logprobs) ) reference_policy_logprobs = ( extras_bdd["reference_policy_logprobs"] if compute_ref else None @@ -1214,9 +1216,7 @@ def grpo_train_sync( log_data["sample_loss_mask"] = sample_mask.tolist() log_data["advantages"] = advantages.tolist() log_data["generation_logprobs"] = generation_logprobs.tolist() - log_data["prev_logprobs"] = ( - prev_logprobs.tolist() if prev_logprobs is not None else None - ) + log_data["prev_logprobs"] = prev_logprobs.tolist() # input_ids was stashed before the step-end clear_samples (the # keys are no longer in TQ at this point); ``_log_input_ids`` # is None when nemo_gym-responses logging path skipped the From 86fbfe671b727392cfc3d93155ef2f837d6777c6 Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Mon, 6 Jul 2026 17:03:26 -0700 Subject: [PATCH 12/19] style(grpo_sync): apply ruff-format to train_fields generator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI lint job flagged the train_fields tuple comprehension — ruff-format prefers each generator clause on its own line. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Zhiyu Li --- nemo_rl/algorithms/grpo_sync.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nemo_rl/algorithms/grpo_sync.py b/nemo_rl/algorithms/grpo_sync.py index 24c020c7e2a..3c510e623d5 100644 --- a/nemo_rl/algorithms/grpo_sync.py +++ b/nemo_rl/algorithms/grpo_sync.py @@ -791,7 +791,8 @@ def grpo_train_sync( # the driver-side read_from_dataplane and the workers' # train_presharded fetch (via train_from_meta) consume it. train_fields = tuple( - f for f in DP_TRAIN_FIELDS + f + for f in DP_TRAIN_FIELDS if not (skip_prev_logprobs and f == "prev_logprobs") ) From 3333116f15d9544ea0e9b35db096da348270af4b Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Mon, 6 Jul 2026 22:13:32 -0700 Subject: [PATCH 13/19] fix(grpo): assert-out use_kl_in_reward + force_on_policy_ratio MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The combination is silently broken today (in legacy grpo_train and, as of this PR's port, grpo_train_sync): when force_on_policy_ratio=True and seq_logprob_error_threshold is unset, prev_logprobs is a torch.zeros_like placeholder. Reinforce++/GAE compute_advantage then calls ``calculate_kl(zeros_tensor, reference_policy_logprobs, ...)`` — the ``is not None`` guard at advantage_estimator.py:214 doesn't catch the zero placeholder, so the KL term corrupts the advantage. Fail fast in _create_advantage_estimator (shared by all trainers). Reported in PR review; scope-limited to configuration validation. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Zhiyu Li --- nemo_rl/algorithms/grpo.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index c868527d82e..563bd2d80c5 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -2130,6 +2130,13 @@ def _create_advantage_estimator(master_config: MasterConfig): grpo_config = master_config.grpo loss_config = master_config.loss_fn + assert not ( + loss_config.use_kl_in_reward and opd_module._skip_prev_logprobs(master_config) + ), ( + "loss_fn.use_kl_in_reward requires real prev_logprobs but force_on_policy_ratio " + "with no grpo.seq_logprob_error_threshold zeros them (KL would be computed against a zero placeholder)." + ) + # Provide backward-compatible defaults when adv_estimator is not in config. # Fall back to top-level grpo.normalize_rewards / grpo.use_leave_one_out_baseline # which older configs still use. From da11484ff1b092fe9c993e114484cb8ee68c324e Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Mon, 6 Jul 2026 22:38:02 -0700 Subject: [PATCH 14/19] test(grpo): parametrized coverage for _resolve_logprob_skip_flags + kl-reward assertion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a compact matrix over the four flag regimes (default / force_on_policy / force_on_policy + threshold override / skip_ref) plus a single-case check for the AssertionError guarding use_kl_in_reward + force_on_policy_ratio in _create_advantage_estimator. TQPolicy.train_from_meta's new train_fields arg is exercised by the 235B TQ e2e run linked in the PR body — unit-testing it would need the full Ray + worker_group scaffolding. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Zhiyu Li --- tests/unit/algorithms/test_grpo.py | 35 ++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/unit/algorithms/test_grpo.py b/tests/unit/algorithms/test_grpo.py index cc564c6daa5..efcd4cb04af 100644 --- a/tests/unit/algorithms/test_grpo.py +++ b/tests/unit/algorithms/test_grpo.py @@ -32,9 +32,11 @@ _apply_configured_message_level_advantage_penalties, _apply_mask_sample_filter, _apply_message_level_advantage_penalties, + _create_advantage_estimator, _default_grpo_save_state, _initial_policy_generation_stale, _raise_if_reward_penalties_enabled_without_nemo_gym, + _resolve_logprob_skip_flags, _resolve_message_level_advantage_penalties, _should_use_async_rollouts, aggregate_rollout_metrics, @@ -3629,3 +3631,36 @@ def test_mixed_metrics(self): assert result["total_turns"] == 45 assert result["accuracy"] == pytest.approx(0.8) assert result["min_accuracy_rate"] == pytest.approx(0.2) + + +def _cfg(*, force=False, threshold=None, skip_ref=None, kl_reward=False): + from types import SimpleNamespace + + return SimpleNamespace( + grpo={ + "seq_logprob_error_threshold": threshold, + "skip_reference_policy_logprobs_calculation": skip_ref, + }, + loss_fn=SimpleNamespace( + force_on_policy_ratio=force, use_kl_in_reward=kl_reward + ), + ) + + +@pytest.mark.parametrize( + "kw, expected", + [ + ({}, (False, None)), + ({"force": True}, (True, None)), + ({"force": True, "threshold": 1.5}, (False, None)), # threshold overrides skip + ({"skip_ref": True}, (False, True)), + ], + ids=["default", "force_on_policy", "force_plus_threshold", "skip_ref"], +) +def test_resolve_logprob_skip_flags(kw, expected): + assert _resolve_logprob_skip_flags(_cfg(**kw)) == expected + + +def test_advantage_estimator_rejects_kl_reward_with_force_on_policy_ratio(): + with pytest.raises(AssertionError, match="use_kl_in_reward"): + _create_advantage_estimator(_cfg(force=True, kl_reward=True)) From def6ee0d76d5bff4796f2daed74f908d1fecab01 Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Mon, 6 Jul 2026 23:24:20 -0700 Subject: [PATCH 15/19] test(grpo): use MasterConfig+ClippedPGLossConfig instead of SimpleNamespace Both types are already imported in this file. Using them makes the fake-config production-realistic and drops the SimpleNamespace shim entirely. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Zhiyu Li --- tests/unit/algorithms/test_grpo.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/unit/algorithms/test_grpo.py b/tests/unit/algorithms/test_grpo.py index efcd4cb04af..496dfa42fbf 100644 --- a/tests/unit/algorithms/test_grpo.py +++ b/tests/unit/algorithms/test_grpo.py @@ -3634,16 +3634,14 @@ def test_mixed_metrics(self): def _cfg(*, force=False, threshold=None, skip_ref=None, kl_reward=False): - from types import SimpleNamespace - - return SimpleNamespace( + return MasterConfig.model_construct( + loss_fn=ClippedPGLossConfig( + force_on_policy_ratio=force, use_kl_in_reward=kl_reward + ), grpo={ "seq_logprob_error_threshold": threshold, "skip_reference_policy_logprobs_calculation": skip_ref, }, - loss_fn=SimpleNamespace( - force_on_policy_ratio=force, use_kl_in_reward=kl_reward - ), ) From d21dba791c2947a3c715ff277bf94cc14a899bcb Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Fri, 24 Jul 2026 00:39:26 -0700 Subject: [PATCH 16/19] fix(grpo): defensive getattr for use_kl_in_reward assertion Existing test_opd.py::test_create_advantage_estimator_opd_branch builds a SimpleNamespace loss_fn without ``use_kl_in_reward``. Short-circuit on missing attribute so the assertion doesn't fault on lightweight test doubles; production ClippedPGLossConfig always has the field. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Zhiyu Li --- nemo_rl/algorithms/grpo.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index 563bd2d80c5..874e6988016 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -2131,7 +2131,8 @@ def _create_advantage_estimator(master_config: MasterConfig): loss_config = master_config.loss_fn assert not ( - loss_config.use_kl_in_reward and opd_module._skip_prev_logprobs(master_config) + getattr(loss_config, "use_kl_in_reward", False) + and opd_module._skip_prev_logprobs(master_config) ), ( "loss_fn.use_kl_in_reward requires real prev_logprobs but force_on_policy_ratio " "with no grpo.seq_logprob_error_threshold zeros them (KL would be computed against a zero placeholder)." From 4d04c3d5f60574ef1b4826e01ad9ae75f3dac614 Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Mon, 27 Jul 2026 12:58:19 -0700 Subject: [PATCH 17/19] fix(grpo): address PR #3087 review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Applies terrykong's review feedback on PR #3087: * Move use_kl_in_reward + skip_prev_logprobs guard from _create_advantage_estimator into setup() as a new _validate_use_kl_in_reward_compat helper — fails before cluster and model weights load rather than after (was grpo_sync.py:455). Guard now drops the getattr() and reads loss_config.use_kl_in_reward directly (declared field on pydantic-validated ClippedPGLossConfig). * Allow reference_policy_kl_penalty == 0 through the guard: kl_coef=0 zeros the KL term regardless, so a zero-placeholder prev_logprobs can't corrupt the advantage in that case. * Assert pytest.warns(UserWarning) on the force_plus_threshold case in test_resolve_logprob_skip_flags — pins the helper docstring's "warn on incompatible combos" promise (deleting the warnings.warn would otherwise leave all four parametrize cases green). * Correct contradictory docstring in tq_policy.py train_from_meta; add train_fields to the Args: block. * Correct inaccurate comment in grpo_sync.py: train_fields is consumed only by train_from_meta (driver read uses select_fields). * Tighten _resolve_logprob_skip_flags return type from tuple[bool, Any] to tuple[bool, bool | None] (skip_reference_policy_logprobs_calculation is NotRequired[bool]). Also adds test_validate_use_kl_in_reward_allows_zero_kl_penalty to pin the kl_coef=0 exception. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Zhiyu Li --- nemo_rl/algorithms/grpo.py | 34 ++++++++++++++++++++++-------- nemo_rl/algorithms/grpo_sync.py | 7 +++--- nemo_rl/models/policy/tq_policy.py | 16 ++++++++------ tests/unit/algorithms/test_grpo.py | 26 +++++++++++++++++------ 4 files changed, 59 insertions(+), 24 deletions(-) diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index 874e6988016..b6d9b0a2fec 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -578,6 +578,8 @@ def init_train_dataloader(dataset, suffix: str = ""): "Reference policy logprob calculation will be skipped since `grpo.skip_reference_policy_logprobs_calculation` is set to True and `loss_fn.reference_policy_kl_penalty` is 0." ) + _validate_use_kl_in_reward_compat(master_config) + # ========================== # Cluster # ========================== @@ -2130,14 +2132,6 @@ def _create_advantage_estimator(master_config: MasterConfig): grpo_config = master_config.grpo loss_config = master_config.loss_fn - assert not ( - getattr(loss_config, "use_kl_in_reward", False) - and opd_module._skip_prev_logprobs(master_config) - ), ( - "loss_fn.use_kl_in_reward requires real prev_logprobs but force_on_policy_ratio " - "with no grpo.seq_logprob_error_threshold zeros them (KL would be computed against a zero placeholder)." - ) - # Provide backward-compatible defaults when adv_estimator is not in config. # Fall back to top-level grpo.normalize_rewards / grpo.use_leave_one_out_baseline # which older configs still use. @@ -2381,7 +2375,29 @@ def _placeholder_seq_logprob_error_metrics() -> dict[str, float]: } -def _resolve_logprob_skip_flags(master_config: MasterConfig) -> tuple[bool, Any]: +def _validate_use_kl_in_reward_compat(master_config: MasterConfig) -> None: + """Reject ``use_kl_in_reward`` when the KL term would read zero placeholder logprobs. + + ``force_on_policy_ratio`` (without ``seq_logprob_error_threshold``) skips + the prev_logprobs forward and passes a zero placeholder to the advantage + estimator; ``use_kl_in_reward`` then applies + ``kl_coef * calculate_kl(zeros, ref)`` which corrupts the advantage. + ``kl_coef=0`` (``reference_policy_kl_penalty=0``) zeros the term regardless, + so that case is allowed. + """ + loss_config = master_config.loss_fn + if loss_config.use_kl_in_reward and loss_config.reference_policy_kl_penalty != 0: + assert not opd_module._skip_prev_logprobs(master_config), ( + "loss_fn.use_kl_in_reward with nonzero loss_fn.reference_policy_kl_penalty " + "requires real prev_logprobs, but force_on_policy_ratio (without " + "grpo.seq_logprob_error_threshold) zeros them — KL would be computed " + "against a zero placeholder." + ) + + +def _resolve_logprob_skip_flags( + master_config: MasterConfig, +) -> tuple[bool, bool | None]: """Return (skip_prev_logprobs, skip_reference_logprobs); warn on incompatible combos. Skip prev_logprobs when force_on_policy_ratio=True unless diff --git a/nemo_rl/algorithms/grpo_sync.py b/nemo_rl/algorithms/grpo_sync.py index 3c510e623d5..5c18ee50770 100644 --- a/nemo_rl/algorithms/grpo_sync.py +++ b/nemo_rl/algorithms/grpo_sync.py @@ -787,9 +787,10 @@ def grpo_train_sync( seq_logprob_error_threshold = master_config.grpo.get( "seq_logprob_error_threshold", None ) - # Effective field set for this step. Filtered once; both - # the driver-side read_from_dataplane and the workers' - # train_presharded fetch (via train_from_meta) consume it. + # Worker-side fetch schema for this step. Derived from the + # same skip decision as ``select_fields`` below, but consumed + # only by ``train_from_meta`` (the driver read uses + # ``select_fields``). train_fields = tuple( f for f in DP_TRAIN_FIELDS diff --git a/nemo_rl/models/policy/tq_policy.py b/nemo_rl/models/policy/tq_policy.py index a669affc418..c6b7b228a6e 100644 --- a/nemo_rl/models/policy/tq_policy.py +++ b/nemo_rl/models/policy/tq_policy.py @@ -380,6 +380,10 @@ def train_from_meta( gbs: Global batch size; defaults to ``cfg["train_global_batch_size"]``. mbs: Micro batch size; defaults to ``cfg["train_micro_batch_size"]``. timer: Optional timer for nested ``policy_training/*`` measurements. + train_fields: TQ columns workers fetch this step; defaults to the + full ``DP_TRAIN_FIELDS`` schema. Caller may narrow it to drop + columns it skipped writing (e.g. ``prev_logprobs`` when + ``force_on_policy_ratio=True``). Returns: Aggregated training-step output dict. @@ -389,12 +393,12 @@ def train_from_meta( self._stamp_pad_seqlen(meta) spa, dba = self._packing_args("train_mb_tokens") - # Train workers fetch the full DP_TRAIN_FIELDS schema (rollout + - # logprob deltas + advantages + sample_mask). Caller is responsible - # for ensuring those columns have been written to TQ before this - # call (workers + driver delta-writes). - # Caller may narrow the fetch schema (e.g. drop prev_logprobs - # when force_on_policy_ratio=True). + # Train workers fetch ``train_fields`` (rollout + logprob deltas + + # advantages + sample_mask), defaulting to the full DP_TRAIN_FIELDS + # schema. Caller is responsible for ensuring those columns have been + # written to TQ before this call (workers + driver delta-writes), and + # may narrow it to drop columns it skipped writing this step (e.g. + # prev_logprobs when force_on_policy_ratio=True). train_meta = replace( meta, fields=fields_with_optional_routed_experts( diff --git a/tests/unit/algorithms/test_grpo.py b/tests/unit/algorithms/test_grpo.py index 496dfa42fbf..73b9d25c372 100644 --- a/tests/unit/algorithms/test_grpo.py +++ b/tests/unit/algorithms/test_grpo.py @@ -32,13 +32,13 @@ _apply_configured_message_level_advantage_penalties, _apply_mask_sample_filter, _apply_message_level_advantage_penalties, - _create_advantage_estimator, _default_grpo_save_state, _initial_policy_generation_stale, _raise_if_reward_penalties_enabled_without_nemo_gym, _resolve_logprob_skip_flags, _resolve_message_level_advantage_penalties, _should_use_async_rollouts, + _validate_use_kl_in_reward_compat, aggregate_rollout_metrics, async_grpo_train, compute_and_apply_seq_logprob_error_masking, @@ -3633,10 +3633,14 @@ def test_mixed_metrics(self): assert result["min_accuracy_rate"] == pytest.approx(0.2) -def _cfg(*, force=False, threshold=None, skip_ref=None, kl_reward=False): +def _cfg( + *, force=False, threshold=None, skip_ref=None, kl_reward=False, kl_penalty=0.01 +): return MasterConfig.model_construct( loss_fn=ClippedPGLossConfig( - force_on_policy_ratio=force, use_kl_in_reward=kl_reward + force_on_policy_ratio=force, + use_kl_in_reward=kl_reward, + reference_policy_kl_penalty=kl_penalty, ), grpo={ "seq_logprob_error_threshold": threshold, @@ -3656,9 +3660,19 @@ def _cfg(*, force=False, threshold=None, skip_ref=None, kl_reward=False): ids=["default", "force_on_policy", "force_plus_threshold", "skip_ref"], ) def test_resolve_logprob_skip_flags(kw, expected): - assert _resolve_logprob_skip_flags(_cfg(**kw)) == expected + if kw.get("force") and kw.get("threshold") is not None: + with pytest.warns(UserWarning, match="seq_logprob_error_threshold is set"): + assert _resolve_logprob_skip_flags(_cfg(**kw)) == expected + else: + assert _resolve_logprob_skip_flags(_cfg(**kw)) == expected -def test_advantage_estimator_rejects_kl_reward_with_force_on_policy_ratio(): +def test_validate_use_kl_in_reward_rejects_force_on_policy_ratio(): with pytest.raises(AssertionError, match="use_kl_in_reward"): - _create_advantage_estimator(_cfg(force=True, kl_reward=True)) + _validate_use_kl_in_reward_compat(_cfg(force=True, kl_reward=True)) + + +def test_validate_use_kl_in_reward_allows_zero_kl_penalty(): + # kl_coef=0 zeros the KL term regardless, so a zero-placeholder + # prev_logprobs can't corrupt the advantage. + _validate_use_kl_in_reward_compat(_cfg(force=True, kl_reward=True, kl_penalty=0.0)) From 12ed7294140faea2733440839c5cfda288d33240 Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Mon, 27 Jul 2026 13:54:28 -0700 Subject: [PATCH 18/19] refactor(grpo_sync): extract _train_fields_for_step + tighten tq comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-ups from PR #3087 review: * Hoist the inline ``prev_logprobs`` filter in ``grpo_train_sync`` into a pure ``_train_fields_for_step(skip_prev_logprobs)`` helper (module already imports DP_TRAIN_FIELDS). Adds a 2-case parametrize test that pins the perf win from silently regressing: TQ ignores an unwritten column and ClippedPGLossFn never reads prev_logprobs under force_on_policy_ratio, so a broken filter wouldn't crash — we would just quietly pay the round-trip again with nothing red. * Shorten the ``tq_policy.train_from_meta`` inline comment: the Args docstring already documents ``train_fields``, so the inline note only needs to state the callsite invariant (columns must already be in TQ). Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Zhiyu Li --- nemo_rl/algorithms/grpo_sync.py | 20 +++++++++++--------- nemo_rl/models/policy/tq_policy.py | 8 ++------ tests/unit/algorithms/test_grpo.py | 11 +++++++++++ 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/nemo_rl/algorithms/grpo_sync.py b/nemo_rl/algorithms/grpo_sync.py index 5c18ee50770..03089443b60 100644 --- a/nemo_rl/algorithms/grpo_sync.py +++ b/nemo_rl/algorithms/grpo_sync.py @@ -117,6 +117,13 @@ def _raise_if_message_level_advantage_penalties_enabled( ) +def _train_fields_for_step(skip_prev_logprobs: bool) -> tuple[str, ...]: + """Fields workers fetch this step; ``prev_logprobs`` is dropped when skipped.""" + return tuple( + f for f in DP_TRAIN_FIELDS if not (skip_prev_logprobs and f == "prev_logprobs") + ) + + # ── DAPO non-zero-std dynamic sampling, slice-only ───────────────────── # Slice-only formulation of nemo_rl.algorithms.grpo.dynamic_sampling: filter # on std != 0, accumulate survivors across iterations, slice on overflow. @@ -787,15 +794,10 @@ def grpo_train_sync( seq_logprob_error_threshold = master_config.grpo.get( "seq_logprob_error_threshold", None ) - # Worker-side fetch schema for this step. Derived from the - # same skip decision as ``select_fields`` below, but consumed - # only by ``train_from_meta`` (the driver read uses - # ``select_fields``). - train_fields = tuple( - f - for f in DP_TRAIN_FIELDS - if not (skip_prev_logprobs and f == "prev_logprobs") - ) + # Worker-side fetch schema for this step. Same skip decision + # as ``select_fields`` below, but consumed only by + # ``train_from_meta`` (driver read uses ``select_fields``). + train_fields = _train_fields_for_step(skip_prev_logprobs) if compute_prev or compute_ref: print("▶ Preparing for logprob inference...", flush=True) diff --git a/nemo_rl/models/policy/tq_policy.py b/nemo_rl/models/policy/tq_policy.py index c6b7b228a6e..814c6a9daff 100644 --- a/nemo_rl/models/policy/tq_policy.py +++ b/nemo_rl/models/policy/tq_policy.py @@ -393,12 +393,8 @@ def train_from_meta( self._stamp_pad_seqlen(meta) spa, dba = self._packing_args("train_mb_tokens") - # Train workers fetch ``train_fields`` (rollout + logprob deltas + - # advantages + sample_mask), defaulting to the full DP_TRAIN_FIELDS - # schema. Caller is responsible for ensuring those columns have been - # written to TQ before this call (workers + driver delta-writes), and - # may narrow it to drop columns it skipped writing this step (e.g. - # prev_logprobs when force_on_policy_ratio=True). + # ``train_fields`` columns must already be in TQ (rollout + worker + # logprob deltas + driver advantage delta) before this call. train_meta = replace( meta, fields=fields_with_optional_routed_experts( diff --git a/tests/unit/algorithms/test_grpo.py b/tests/unit/algorithms/test_grpo.py index 73b9d25c372..7b8b2c703fd 100644 --- a/tests/unit/algorithms/test_grpo.py +++ b/tests/unit/algorithms/test_grpo.py @@ -47,6 +47,7 @@ refit_policy_generation, validate, ) +from nemo_rl.algorithms.grpo_sync import _train_fields_for_step from nemo_rl.algorithms.loss import ClippedPGLossConfig, ClippedPGLossFn from nemo_rl.algorithms.reward_functions import ( RewardShapingConfig, @@ -3676,3 +3677,13 @@ def test_validate_use_kl_in_reward_allows_zero_kl_penalty(): # kl_coef=0 zeros the KL term regardless, so a zero-placeholder # prev_logprobs can't corrupt the advantage. _validate_use_kl_in_reward_compat(_cfg(force=True, kl_reward=True, kl_penalty=0.0)) + + +@pytest.mark.parametrize( + "skip_prev_logprobs, expect_prev", + [(False, True), (True, False)], + ids=["keep_prev_logprobs", "skip_prev_logprobs"], +) +def test_train_fields_for_step(skip_prev_logprobs, expect_prev): + fields = _train_fields_for_step(skip_prev_logprobs) + assert ("prev_logprobs" in fields) is expect_prev From a522962a7b13fd8c07c0ed05c00f0297a0101985 Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Mon, 27 Jul 2026 14:21:01 -0700 Subject: [PATCH 19/19] docs(tq_policy): restore composition + writers in train_from_meta comment Previous simplification dropped the field composition and delta-write authorship that terrykong's suggested wording carried. Re-add them in a tighter 4-line form: composition (rollout + logprob deltas + advantages + sample_mask; default DP_TRAIN_FIELDS), invariant + writers (must be in TQ before this call, written by workers + driver delta-writes), and the narrowing example (drop prev_logprobs under force_on_policy_ratio). Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Zhiyu Li --- nemo_rl/models/policy/tq_policy.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nemo_rl/models/policy/tq_policy.py b/nemo_rl/models/policy/tq_policy.py index 814c6a9daff..61da43511c6 100644 --- a/nemo_rl/models/policy/tq_policy.py +++ b/nemo_rl/models/policy/tq_policy.py @@ -393,8 +393,10 @@ def train_from_meta( self._stamp_pad_seqlen(meta) spa, dba = self._packing_args("train_mb_tokens") - # ``train_fields`` columns must already be in TQ (rollout + worker - # logprob deltas + driver advantage delta) before this call. + # ``train_fields`` (rollout + logprob deltas + advantages + sample_mask; + # default ``DP_TRAIN_FIELDS``) must be in TQ before this call — written + # by workers + driver delta-writes. Caller may narrow to drop columns + # skipped this step (e.g. ``prev_logprobs`` under force_on_policy_ratio). train_meta = replace( meta, fields=fields_with_optional_routed_experts(