Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
8831d62
fix(grpo_sync): honor force_on_policy_ratio in TQ trainer
ZhiyuLi-Nvidia Jul 5, 2026
efb3081
refactor(grpo): drop misleading 0-fill for skipped seq_logprob metrics
ZhiyuLi-Nvidia Jul 6, 2026
d0efc1c
perf(grpo_sync): drop prev_logprobs TQ round-trip when force_on_polic…
ZhiyuLi-Nvidia Jul 6, 2026
141afd0
refactor(grpo_sync): tighten helper + unify select_fields with dispatch
ZhiyuLi-Nvidia Jul 6, 2026
088c9bb
refactor(grpo_sync): thread skip_prev_logprobs through train_from_meta
ZhiyuLi-Nvidia Jul 6, 2026
6cb80f0
refactor(grpo_sync): driver owns the train field list
ZhiyuLi-Nvidia Jul 6, 2026
50b1ccf
refactor(tq_policy): use DP_TRAIN_FIELDS as direct default for train_…
ZhiyuLi-Nvidia Jul 6, 2026
94e5164
docs(grpo_sync): correct the prev_logprobs placeholder comment
ZhiyuLi-Nvidia Jul 6, 2026
b839b14
refactor(grpo_sync): pass None to compute_advantage when skipping pre…
ZhiyuLi-Nvidia Jul 6, 2026
a82d3ef
fix(grpo_sync): guard log_data prev_logprobs.tolist() against None
ZhiyuLi-Nvidia Jul 6, 2026
435236d
Revert "refactor(grpo_sync): pass None to compute_advantage when skip…
ZhiyuLi-Nvidia Jul 6, 2026
86fbfe6
style(grpo_sync): apply ruff-format to train_fields generator
ZhiyuLi-Nvidia Jul 7, 2026
3333116
fix(grpo): assert-out use_kl_in_reward + force_on_policy_ratio
ZhiyuLi-Nvidia Jul 7, 2026
da11484
test(grpo): parametrized coverage for _resolve_logprob_skip_flags + k…
ZhiyuLi-Nvidia Jul 7, 2026
def6ee0
test(grpo): use MasterConfig+ClippedPGLossConfig instead of SimpleNam…
ZhiyuLi-Nvidia Jul 7, 2026
d21dba7
fix(grpo): defensive getattr for use_kl_in_reward assertion
ZhiyuLi-Nvidia Jul 24, 2026
4d04c3d
fix(grpo): address PR #3087 review comments
ZhiyuLi-Nvidia Jul 27, 2026
12ed729
refactor(grpo_sync): extract _train_fields_for_step + tighten tq comment
ZhiyuLi-Nvidia Jul 27, 2026
a522962
docs(tq_policy): restore composition + writers in train_from_meta com…
ZhiyuLi-Nvidia Jul 27, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 69 additions & 51 deletions nemo_rl/algorithms/grpo.py
Comment thread
ZhiyuLi-Nvidia marked this conversation as resolved.
Resolved
Comment thread
ZhiyuLi-Nvidia marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
Expand Up @@ -578,6 +578,8 @@ def init_train_dataloader(dataset, suffix: str = ""):
"Reference policy logprob calculation will be skipped since `grpo.skip_reference_policy_logprobs_calculation` is set to True and `loss_fn.reference_policy_kl_penalty` is 0."
)

_validate_use_kl_in_reward_compat(master_config)

# ==========================
# Cluster
# ==========================
Expand Down Expand Up @@ -2359,6 +2361,65 @@ def _log_mixed_rewards_and_advantages_information(
metrics["advantages/mean"] = advantages.float().mean().item()


def _placeholder_seq_logprob_error_metrics() -> dict[str, float]:
"""Zero-valued seq-level metrics used when the prev_logprobs forward is skipped."""
return {
"max_seq_mult_prob_error": 0.0,
"mean_seq_mult_prob_error": 0.0,
"min_seq_mult_prob_error": 0.0,
"max_seq_mult_prob_error_after_mask": 0.0,
"mean_seq_mult_prob_error_after_mask": 0.0,
"min_seq_mult_prob_error_after_mask": 0.0,
"num_masked_seqs_by_logprob_error": 0,
"masked_correct_pct": 0.0,
}


def _validate_use_kl_in_reward_compat(master_config: MasterConfig) -> None:
"""Reject ``use_kl_in_reward`` when the KL term would read zero placeholder logprobs.

``force_on_policy_ratio`` (without ``seq_logprob_error_threshold``) skips
the prev_logprobs forward and passes a zero placeholder to the advantage
estimator; ``use_kl_in_reward`` then applies
``kl_coef * calculate_kl(zeros, ref)`` which corrupts the advantage.
``kl_coef=0`` (``reference_policy_kl_penalty=0``) zeros the term regardless,
so that case is allowed.
"""
loss_config = master_config.loss_fn
if loss_config.use_kl_in_reward and loss_config.reference_policy_kl_penalty != 0:
assert not opd_module._skip_prev_logprobs(master_config), (
"loss_fn.use_kl_in_reward with nonzero loss_fn.reference_policy_kl_penalty "
"requires real prev_logprobs, but force_on_policy_ratio (without "
"grpo.seq_logprob_error_threshold) zeros them — KL would be computed "
"against a zero placeholder."
)


def _resolve_logprob_skip_flags(
master_config: MasterConfig,
) -> tuple[bool, bool | None]:
"""Return (skip_prev_logprobs, skip_reference_logprobs); warn on incompatible combos.

Skip prev_logprobs when force_on_policy_ratio=True unless
seq_logprob_error_threshold is set (which requires prev_logprobs).
Skip reference_policy_logprobs when
``grpo.skip_reference_policy_logprobs_calculation`` is set.
"""
# todo @jiaqi: is there a better way to skip prev_logprobs computation while still computing the seq-level error metrics?
if (
master_config.loss_fn.force_on_policy_ratio
and master_config.grpo.get("seq_logprob_error_threshold") is not None
):
warnings.warn(
"force_on_policy_ratio=True but seq_logprob_error_threshold is set. "
"Computing prev_logprobs anyway for seq-level error masking."
)
return (
opd_module._skip_prev_logprobs(master_config),
master_config.grpo.get("skip_reference_policy_logprobs_calculation"),
)


def compute_and_apply_seq_logprob_error_masking(
train_data: BatchedDataDict,
rewards: torch.Tensor,
Expand Down Expand Up @@ -2952,24 +3013,12 @@ def grpo_train(
metrics_logging_data["content"] = flat_messages["content"]

memory_tracker.snapshot_start_of_stage("Computing logprobs", dir())
# Skip prev_logprobs computation when force_on_policy_ratio=True
# unless seq_logprob_error_threshold is set (which requires prev_logprobs)
skip_prev_logprobs, skip_reference_logprobs = (
_resolve_logprob_skip_flags(master_config)
)
seq_logprob_error_threshold = master_config.grpo.get(
"seq_logprob_error_threshold", None
)
force_on_policy_ratio = master_config.loss_fn.force_on_policy_ratio
skip_prev_logprobs = opd_module._skip_prev_logprobs(master_config)
# todo @jiaqi: is there a better way to skip prev_logprobs computation while still computing the seq-level error metrics?
if force_on_policy_ratio and seq_logprob_error_threshold is not None:
warnings.warn(
"force_on_policy_ratio=True but seq_logprob_error_threshold is set. "
"Computing prev_logprobs anyway for seq-level error masking."
)

# Skip reference_policy_logprobs computation when skip_reference_policy_logprobs_calculation=True
skip_reference_logprobs = master_config.grpo.get(
"skip_reference_policy_logprobs_calculation"
)

if not (skip_prev_logprobs and skip_reference_logprobs):
print("▶ Preparing for logprob inference...", flush=True)
Expand Down Expand Up @@ -3034,16 +3083,7 @@ def grpo_train(
# Seq-level logprob error metrics/masking require real prev_logprobs
if skip_prev_logprobs:
# Cannot compute seq-level metrics with placeholder prev_logprobs
seq_logprob_error_metrics = {
"max_seq_mult_prob_error": 0.0,
"mean_seq_mult_prob_error": 0.0,
"min_seq_mult_prob_error": 0.0,
"max_seq_mult_prob_error_after_mask": 0.0,
"mean_seq_mult_prob_error_after_mask": 0.0,
"min_seq_mult_prob_error_after_mask": 0.0,
"num_masked_seqs_by_logprob_error": 0,
"masked_correct_pct": 0.0,
}
seq_logprob_error_metrics = _placeholder_seq_logprob_error_metrics()
else:
seq_error_result = compute_and_apply_seq_logprob_error_masking(
train_data=train_data,
Expand Down Expand Up @@ -4357,25 +4397,12 @@ def async_grpo_train(
train_data.to("cpu")

# Training phase (same as sync version)
# Skip prev_logprobs computation when force_on_policy_ratio=True
# unless seq_logprob_error_threshold is set (which requires prev_logprobs)
skip_prev_logprobs, skip_reference_logprobs = (
_resolve_logprob_skip_flags(master_config)
)
seq_logprob_error_threshold = master_config.grpo.get(
"seq_logprob_error_threshold", None
)
force_on_policy_ratio = master_config.loss_fn.force_on_policy_ratio
skip_prev_logprobs = opd_module._skip_prev_logprobs(master_config)

# todo @jiaqi: is there a better way to skip prev_logprobs computation while still computing the seq-level error metrics?
if force_on_policy_ratio and seq_logprob_error_threshold is not None:
warnings.warn(
"force_on_policy_ratio=True but seq_logprob_error_threshold is set. "
"Computing prev_logprobs anyway for seq-level error masking."
)

# Skip reference_policy_logprobs computation when skip_reference_policy_logprobs_calculation=True
skip_reference_logprobs = master_config.grpo.get(
"skip_reference_policy_logprobs_calculation"
)

if not (skip_prev_logprobs and skip_reference_logprobs):
print("▶ Preparing for logprob inference...", flush=True)
Expand Down Expand Up @@ -4412,16 +4439,7 @@ def async_grpo_train(
# Seq-level logprob error metrics/masking require real prev_logprobs
if skip_prev_logprobs:
# Cannot compute seq-level metrics with placeholder prev_logprobs
seq_logprob_error_metrics = {
"max_seq_mult_prob_error": 0.0,
"mean_seq_mult_prob_error": 0.0,
"min_seq_mult_prob_error": 0.0,
"max_seq_mult_prob_error_after_mask": 0.0,
"mean_seq_mult_prob_error_after_mask": 0.0,
"min_seq_mult_prob_error_after_mask": 0.0,
"num_masked_seqs_by_logprob_error": 0,
"masked_correct_pct": 0.0,
}
seq_logprob_error_metrics = _placeholder_seq_logprob_error_metrics()
else:
seq_error_result = compute_and_apply_seq_logprob_error_masking(
train_data=train_data,
Expand Down
69 changes: 51 additions & 18 deletions nemo_rl/algorithms/grpo_sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@
_clip_grpo_advantages,
_create_advantage_estimator,
_log_mixed_rewards_and_advantages_information,
_placeholder_seq_logprob_error_metrics,
_resolve_logprob_skip_flags,
_should_log_nemo_gym_responses,
_should_use_nemo_gym,
compute_and_apply_seq_logprob_error_masking,
Expand All @@ -71,7 +73,7 @@
from nemo_rl.data.interfaces import DatumSpec
from nemo_rl.data.llm_message_utils import batched_message_log_to_flat_message
from nemo_rl.data_plane.interfaces import KVBatchMeta
from nemo_rl.data_plane.schema import DP_CALIB_INPUT_FIELDS
from nemo_rl.data_plane.schema import DP_CALIB_INPUT_FIELDS, DP_TRAIN_FIELDS
from nemo_rl.distributed.batched_data_dict import BatchedDataDict
from nemo_rl.environments.interfaces import EnvironmentInterface
from nemo_rl.experience.sync_rollout_actor import SyncRolloutActor
Expand Down Expand Up @@ -115,6 +117,13 @@ def _raise_if_message_level_advantage_penalties_enabled(
)


def _train_fields_for_step(skip_prev_logprobs: bool) -> tuple[str, ...]:
"""Fields workers fetch this step; ``prev_logprobs`` is dropped when skipped."""
return tuple(
f for f in DP_TRAIN_FIELDS if not (skip_prev_logprobs and f == "prev_logprobs")
)


# ── DAPO non-zero-std dynamic sampling, slice-only ─────────────────────
# Slice-only formulation of nemo_rl.algorithms.grpo.dynamic_sampling: filter
# on std != 0, accumulate survivors across iterations, slice on overflow.
Expand Down Expand Up @@ -777,9 +786,23 @@ def grpo_train_sync(
baseline_for_log = baseline.clone()

memory_tracker.snapshot_start_of_stage("Computing logprobs", dir())
print("▶ Preparing for logprob inference...", flush=True)
with timer.time("logprob_inference_prep"):
policy.prepare_for_lp_inference()
skip_prev_logprobs, skip_reference_logprobs = (
_resolve_logprob_skip_flags(master_config)
)
compute_prev = not skip_prev_logprobs
compute_ref = not skip_reference_logprobs
seq_logprob_error_threshold = master_config.grpo.get(
"seq_logprob_error_threshold", None
)
# Worker-side fetch schema for this step. Same skip decision
# as ``select_fields`` below, but consumed only by
# ``train_from_meta`` (driver read uses ``select_fields``).
train_fields = _train_fields_for_step(skip_prev_logprobs)

if compute_prev or compute_ref:
print("▶ Preparing for logprob inference...", flush=True)
with timer.time("logprob_inference_prep"):
policy.prepare_for_lp_inference()

print("▶ Computing logprobs...", flush=True)
with timer.time("policy_and_reference_logprobs"):
Expand All @@ -791,47 +814,56 @@ def grpo_train_sync(
# batched fetch to avoid double-shipping the per-token
# tensor through Ray's plasma store on top of the TQ
# writeback.
policy.get_logprobs_from_meta(meta, timer=timer)
compute_ref = not master_config.grpo.get(
"skip_reference_policy_logprobs_calculation"
)
select_fields = ["generation_logprobs", "token_mask"]
if compute_prev:
policy.get_logprobs_from_meta(meta, timer=timer)
select_fields.append("prev_logprobs")
else:
print(
"▶ Skipping prev_logprobs (force_on_policy_ratio=True)...",
flush=True,
)
if compute_ref:
policy.get_reference_policy_logprobs_from_meta(
meta,
timer=timer,
)
select_fields.append("reference_policy_logprobs")

# Driver pulls only the per-token columns it needs
# for masking / advantage. Bulk (input_ids, multimodal,
# output_ids, attention_mask, position_ids) stays in
# TQ — workers will fetch it via ``train_presharded``.
extras_bdd = policy.read_from_dataplane(
meta,
select_fields=[
"prev_logprobs",
"generation_logprobs",
"token_mask",
*(["reference_policy_logprobs"] if compute_ref else []),
],
select_fields=select_fields,
pad_value_dict=_pad_dict,
)
prev_logprobs = extras_bdd["prev_logprobs"]
generation_logprobs = extras_bdd["generation_logprobs"]
token_mask = extras_bdd["token_mask"]
prev_logprobs = (
extras_bdd["prev_logprobs"]
if compute_prev
else torch.zeros_like(generation_logprobs)
)
reference_policy_logprobs = (
extras_bdd["reference_policy_logprobs"] if compute_ref else None
)

# Seq-level logprob error metrics/masking require real prev_logprobs
if skip_prev_logprobs:
sample_mask = loss_multiplier
# Cannot compute seq-level metrics with placeholder prev_logprobs
seq_logprob_error_metrics = _placeholder_seq_logprob_error_metrics()
else:
sample_mask, seq_logprob_error_metrics = (
_compute_seq_logprob_error_metrics(
token_mask=token_mask,
sample_mask=loss_multiplier,
prev_logprobs=prev_logprobs,
generation_logprobs=generation_logprobs,
rewards=rewards,
seq_logprob_error_threshold=master_config.grpo[
"seq_logprob_error_threshold"
],
seq_logprob_error_threshold=seq_logprob_error_threshold,
)
)

Expand Down Expand Up @@ -900,6 +932,7 @@ def grpo_train_sync(
meta,
loss_fn=loss_fn,
timer=timer,
train_fields=train_fields,
)

if sync_kv_scales:
Expand Down
15 changes: 10 additions & 5 deletions nemo_rl/models/policy/tq_policy.py
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,7 @@ def train_from_meta(
gbs: Optional[int] = None,
mbs: Optional[int] = None,
timer: Optional[Timer] = None,
train_fields: tuple[str, ...] = DP_TRAIN_FIELDS,
) -> dict[str, Any]:
"""1-hop counterpart to :meth:`train`.

Expand All @@ -379,6 +380,10 @@ def train_from_meta(
gbs: Global batch size; defaults to ``cfg["train_global_batch_size"]``.
mbs: Micro batch size; defaults to ``cfg["train_micro_batch_size"]``.
timer: Optional timer for nested ``policy_training/*`` measurements.
train_fields: TQ columns workers fetch this step; defaults to the
full ``DP_TRAIN_FIELDS`` schema. Caller may narrow it to drop
columns it skipped writing (e.g. ``prev_logprobs`` when
``force_on_policy_ratio=True``).

Returns:
Aggregated training-step output dict.
Expand All @@ -388,14 +393,14 @@ def train_from_meta(

self._stamp_pad_seqlen(meta)
spa, dba = self._packing_args("train_mb_tokens")
# Train workers fetch the full DP_TRAIN_FIELDS schema (rollout +
# logprob deltas + advantages + sample_mask). Caller is responsible
# for ensuring those columns have been written to TQ before this
# call (workers + driver delta-writes).
# ``train_fields`` (rollout + logprob deltas + advantages + sample_mask;
# default ``DP_TRAIN_FIELDS``) must be in TQ before this call — written
# by workers + driver delta-writes. Caller may narrow to drop columns
# skipped this step (e.g. ``prev_logprobs`` under force_on_policy_ratio).
train_meta = replace(
meta,
fields=fields_with_optional_routed_experts(
DP_TRAIN_FIELDS, enabled=self._router_replay_enabled
train_fields, enabled=self._router_replay_enabled
),
task_name="train",
)
Expand Down
Loading
Loading