feat(grpo): validation-only sampling params and grouped pass@k validation

examples/configs/grpo_math_1B.yaml

@@ -15,6 +15,8 @@ grpo:
   advantage_clip_low: null
   advantage_clip_high: null
   max_val_samples: 256
+  # Validation rollouts per prompt; k > 1 also reports the pass_k metric.
+  val_num_generations_per_prompt: 1
   val_batch_size: 256
   seed: 42
   use_dynamic_sampling: false
@@ -339,6 +341,10 @@ policy:
     temperature: 1.0
     top_p: 1.0
     top_k: null
+    # Validation-only sampling; defaults follow the train values above.
+    val_temperature: ${.temperature}
+    val_top_p: ${.top_p}
+    val_top_k: ${.top_k}
     stop_token_ids: null
     stop_strings: null
     # null = topology default (IPC colocated, NCCL non-colocated).

examples/configs/recipes/llm/mopd-qwen3-1.7b-3n8g-megatron-pack.yaml

@@ -2,7 +2,7 @@ defaults: ../../grpo_math_1B.yaml
 grpo:
   num_prompts_per_step: 8
   num_generations_per_prompt: 4
-  num_val_generations_per_prompt: 1
+  val_num_generations_per_prompt: 1
   max_num_steps: 5
   val_period: 1000
   overlong_filtering: true

examples/nemo_gym/grpo_nanov3.yaml

@@ -2,7 +2,7 @@
 grpo:
   num_prompts_per_step: 128
   num_generations_per_prompt: 16
-  num_val_generations_per_prompt: 4
+  val_num_generations_per_prompt: 4
   max_rollout_turns: 1 # for multi-turn rollouts. Math Environments just have 1 turn (answering the question)
   max_num_epochs: 1
   max_num_steps: 1000000
@@ -205,6 +205,9 @@ policy:
     temperature: 1.0
     top_p: 1.0
     top_k: null
+    val_temperature: ${.temperature}
+    val_top_p: ${.top_p}
+    val_top_k: ${.top_k}
     stop_token_ids: null
     stop_strings: null
     mcore_generation_config:

examples/nemo_gym/grpo_qwen3_30ba3b_thinking_swe1.yaml

@@ -13,7 +13,7 @@ checkpointing:
 grpo:
   num_prompts_per_step: 64
   num_generations_per_prompt: 8
-  num_val_generations_per_prompt: 1
+  val_num_generations_per_prompt: 1
   max_num_epochs: 100
   advantage_clip_low: -100
   advantage_clip_high: 100

examples/nemo_gym/grpo_qwen3_30ba3b_thinking_swe2.yaml

@@ -13,7 +13,7 @@ checkpointing:
 grpo:
   num_prompts_per_step: 8
   num_generations_per_prompt: 8
-  num_val_generations_per_prompt: 1
+  val_num_generations_per_prompt: 1
   max_num_epochs: 100
   advantage_clip_low: -100
   advantage_clip_high: 100

examples/nemo_gym/grpo_workplace_assistant_nemotron_nano_v2_9b.yaml

@@ -14,6 +14,7 @@ grpo:
   advantage_clip_low: null
   advantage_clip_high: null
   max_val_samples: null   # inferred from size of val dataset. for multi evals, repeat val ds via `num_repeats` in `ng_prepare_data`.
+  val_num_generations_per_prompt: 1
   val_batch_size: null
   seed: 42
   use_dynamic_sampling: false
@@ -222,6 +223,9 @@ policy:
     temperature: 1.0
     top_p: 1.0
     top_k: null
+    val_temperature: ${.temperature}
+    val_top_p: ${.top_p}
+    val_top_k: ${.top_k}
     stop_token_ids: null
     stop_strings: null
     vllm_cfg:

examples/nemo_gym/nemotron-3-super/stage1_rlvr.yaml

@@ -13,7 +13,7 @@ checkpointing:
 grpo:
   num_prompts_per_step: 256
   num_generations_per_prompt: 16
-  num_val_generations_per_prompt: 2
+  val_num_generations_per_prompt: 2
   max_rollout_turns: 1
   max_num_epochs: 1
   max_num_steps: 1000000
@@ -222,6 +222,9 @@ policy:
     temperature: 1.0
     top_p: 1.0
     top_k: null
+    val_temperature: ${.temperature}
+    val_top_p: ${.top_p}
+    val_top_k: ${.top_k}
     stop_token_ids: null
     stop_strings: null
     vllm_cfg:

examples/nemo_gym/nemotron-3-super/stage2_swe1.yaml

@@ -13,7 +13,7 @@ checkpointing:
 grpo:
   num_prompts_per_step: 64
   num_generations_per_prompt: 16
-  num_val_generations_per_prompt: 1
+  val_num_generations_per_prompt: 1
   max_rollout_turns: 1
   max_num_epochs: 100
   max_num_steps: 1000000
@@ -222,6 +222,9 @@ policy:
     temperature: 1.0
     top_p: 1.0
     top_k: null
+    val_temperature: ${.temperature}
+    val_top_p: ${.top_p}
+    val_top_k: ${.top_k}
     stop_token_ids: null
     stop_strings: null
     vllm_cfg:

examples/nemo_gym/nemotron-3-super/stage2_swe2.yaml

@@ -13,7 +13,7 @@ checkpointing:
 grpo:
   num_prompts_per_step: 16
   num_generations_per_prompt: 32
-  num_val_generations_per_prompt: 1
+  val_num_generations_per_prompt: 1
   max_rollout_turns: 1
   max_num_epochs: 100
   max_num_steps: 1000000
@@ -215,6 +215,9 @@ policy:
     temperature: 1.0
     top_p: 1.0
     top_k: null
+    val_temperature: ${.temperature}
+    val_top_p: ${.top_p}
+    val_top_k: ${.top_k}
     stop_token_ids: null
     stop_strings: null
     vllm_cfg:

examples/nemo_gym/nemotron-3-super/stage3_rlhf.yaml

@@ -13,7 +13,7 @@ checkpointing:
 grpo:
   num_prompts_per_step: 128
   num_generations_per_prompt: 16
-  num_val_generations_per_prompt: 2
+  val_num_generations_per_prompt: 2
   max_rollout_turns: 1
   max_num_epochs: 1
   max_num_steps: 1000000
@@ -222,6 +222,9 @@ policy:
     temperature: 1.0
     top_p: 1.0
     top_k: null
+    val_temperature: ${.temperature}
+    val_top_p: ${.top_p}
+    val_top_k: ${.top_k}
     stop_token_ids: null
     stop_strings: null
     vllm_cfg:

nemo_rl/algorithms/grpo.py

@@ -94,6 +94,7 @@
 from nemo_rl.models.generation.interfaces import (
     GenerationConfig,
     GenerationInterface,
+    SamplingParams,
     resolve_routed_experts_dtype_name_for_model,
 )
 from nemo_rl.models.generation.megatron import MegatronGeneration
@@ -261,6 +262,10 @@ class GRPOConfig(TypedDict):
     # final checkpoint has validation metrics, which is required for get_best_checkpoint_path().
     val_at_end: bool
     max_val_samples: int | None  # None for NeMo-Gym compatibility
+    # Number of independent validation rollouts generated for each prompt;
+    # k > 1 additionally reports pass@k over each prompt's k rollouts as the
+    # pass_k metric.
+    val_num_generations_per_prompt: int
     skip_reference_policy_logprobs_calculation: NotRequired[bool]
     seed: int
     async_grpo: NotRequired[AsyncGRPOConfig]
@@ -391,6 +396,21 @@ def setup(
     if generation_config["backend"] == "vllm":
         normalize_vllm_refit_config(cast(VllmConfig, generation_config))
 
+    # Validation-only sampling is honored only on the NeMo-Gym vLLM rollout
+    # path; everywhere else validation must sample exactly like training.
+    val_sampling_overridden = (
+        generation_config["val_temperature"] != generation_config["temperature"]
+        or generation_config["val_top_p"] != generation_config["top_p"]
+        or generation_config["val_top_k"] != generation_config["top_k"]
+    )
+    if val_sampling_overridden:
+        assert generation_config["backend"] == "vllm" and _should_use_nemo_gym(
+            master_config
+        ), (
+            "generation.val_temperature/val_top_p/val_top_k differing from the "
+            "train sampling params is only supported for vLLM NeMo-Gym rollouts."
+        )
+
     # Set seed for all random number generators
     set_seed(grpo_config["seed"])
 
@@ -3619,6 +3639,12 @@ def validate(
     timer = Timer(context={"worker": "validator"})
     with timer.time("total_validation_time"):
         print(f"▶ Starting validation at step {step}...", flush=True)
+        val_num_generations_per_prompt = master_config.grpo[
+            "val_num_generations_per_prompt"
+        ]
+        assert val_num_generations_per_prompt >= 1, (
+            "grpo.val_num_generations_per_prompt must be >= 1"
+        )
 
         total_rewards = []
         total_lengths = []
@@ -3632,19 +3658,31 @@ def validate(
             if batch_idx >= max_batches:
                 break
 
+            if val_num_generations_per_prompt > 1:
+                val_batch = val_batch.repeat_interleave(val_num_generations_per_prompt)
+
             additional_metrics_to_report = dict()
             # Generate responses (updates the LLMMessageLogType in batch_with_msg_logs)
             # Use async rollouts when enabled by config/backend defaults.
             # We cascade NeMo-Gym first since NeMo-Gym also uses async rollouts.
             if _should_use_nemo_gym(master_config):
                 generation_config = master_config.policy["generation"]
+                # Validation-only sampling (e.g. near-greedy validation);
+                # defaults to the train profile via the exemplar YAML
+                # interpolations. Training rollouts keep policy.generation.
+                val_sampling_params = SamplingParams(
+                    temperature=generation_config["val_temperature"],
+                    top_p=generation_config["val_top_p"],
+                    top_k=generation_config["val_top_k"],
+                )
                 nemo_gym_rollout_result = run_nemo_gym_rollout_sync(
                     policy_generation=policy_generation,
                     input_batch=val_batch,
                     tokenizer=tokenizer,
                     task_to_env=val_task_to_env,
                     max_seq_len=master_config.policy["max_total_sequence_length"],
                     generation_config=generation_config,
+                    sampling_params=val_sampling_params,
                     log_full_result_tables=should_log_nemo_gym_full_result_tables(
                         wandb_enabled=master_config.logger["wandb_enabled"],
                         wandb_config=master_config.logger["wandb"],
@@ -3692,11 +3730,26 @@ def validate(
 
             all_message_logs.extend(to_env)
 
-        # Calculate validation metrics
+        # Calculate validation metrics. accuracy is the mean reward over all
+        # rollouts; grouped validation (val_num_generations_per_prompt > 1)
+        # additionally reports pass@k over each prompt's k rollouts as pass_k.
         num_samples = len(total_rewards)
+        pass_k = None
         if num_samples > 0:
             rewards_t = torch.tensor(total_rewards, dtype=torch.float32)
             accuracy = rewards_t.mean().item()
+            if val_num_generations_per_prompt > 1:
+                assert num_samples % val_num_generations_per_prompt == 0, (
+                    "Validation rewards must be divisible by "
+                    "grpo.val_num_generations_per_prompt"
+                )
+                pass_k = (
+                    (rewards_t.view(-1, val_num_generations_per_prompt) > 0)
+                    .any(dim=1)
+                    .float()
+                    .mean()
+                    .item()
+                )
         else:
             accuracy = 0.0
 
@@ -3709,6 +3762,8 @@ def validate(
             "avg_length": avg_length,
             **additional_metrics_to_report,
         }
+        if pass_k is not None:
+            val_metrics["pass_k"] = pass_k
 
         # Print sample conversations only once at the end of validation
         try:

nemo_rl/experience/rollouts.py

@@ -54,6 +54,7 @@
     GenerationDatumSpec,
     GenerationInterface,
     GenerationOutputSpec,
+    SamplingParams,
 )
 from nemo_rl.utils.timer import Timer
 
@@ -1947,7 +1948,9 @@ def apply_reward_penalties(
 
 
 def _prepare_nemo_gym_rows(
-    rows: list[dict], generation_config: GenerationConfig
+    rows: list[dict],
+    generation_config: GenerationConfig,
+    sampling_params: SamplingParams,
 ) -> None:
     """Apply NeMo-RL sampling parameters and stable row indices in place."""
     for row_index, row in enumerate(rows):
@@ -1957,8 +1960,8 @@ def _prepare_nemo_gym_rows(
                 "Each NeMo-Gym row must contain a responses_create_params dict"
             )
 
-        responses_create_params["temperature"] = generation_config["temperature"]
-        responses_create_params["top_p"] = generation_config["top_p"]
+        responses_create_params["temperature"] = sampling_params.temperature
+        responses_create_params["top_p"] = sampling_params.top_p
         configured_max_tokens = generation_config["max_new_tokens"]
         row_max_tokens = responses_create_params.get("max_output_tokens")
         responses_create_params["max_output_tokens"] = (
@@ -1998,6 +2001,7 @@ async def run_async_nemo_gym_rollout(
     reward_penalty_config: dict[str, Any] | BaseModel | None = None,
     thinking_tags: list[str] | tuple[str, ...] | None = None,
     returns_entire_batch: bool = False,
+    sampling_params: Optional[SamplingParams] = None,
 ) -> AsyncGenerator[NemoGymRolloutResult, None]:
     """Stream complete NeMo-Gym prompt groups in group-completion order.
 
@@ -2026,6 +2030,9 @@ async def run_async_nemo_gym_rollout(
         returns_entire_batch: Whether to treat the input as one potentially
             heterogeneous group. This requires ``num_generations`` to equal the
             batch size and is used by synchronous callers.
+        sampling_params: Sampling profile stamped onto every NeMo-Gym row.
+            ``None`` uses the train profile from ``generation_config``;
+            validation passes its own profile explicitly.
 
     Yields:
         ``NemoGymRolloutResult`` objects in prompt-group completion order. Rows
@@ -2076,9 +2083,11 @@ async def run_async_nemo_gym_rollout(
     assert not generation_config["stop_token_ids"], (
         "Stop strings is not supported in the generation config in NeMo-Gym path!"
     )
+    if sampling_params is None:
+        sampling_params = SamplingParams.from_generation_config(generation_config)
     # Top k is not OpenAI compatible, so NeMo-Gym does not guarantee support over it.
-    assert not generation_config["top_k"], (
-        "Top k is not supported in the generation config in NeMo-Gym path!"
+    assert not sampling_params.top_k, (
+        "Top k is not supported in the sampling params in NeMo-Gym path!"
     )
     if num_generations <= 0:
         raise ValueError("num_generations must be greater than zero")
@@ -2099,7 +2108,7 @@ async def run_async_nemo_gym_rollout(
     run_rollouts_timer_label = f"{timer_prefix}/run_rollouts"
 
     with timer.time(total_timer_label):
-        _prepare_nemo_gym_rows(nemo_gym_rows, generation_config)
+        _prepare_nemo_gym_rows(nemo_gym_rows, generation_config, sampling_params)
         accumulator = _NemoGymStreamAccumulator(
             rows=nemo_gym_rows,
             num_generations=num_generations,
@@ -2184,6 +2193,7 @@ def run_nemo_gym_rollout_sync(
     effort_config: Optional[EffortLevelsConfig] = None,
     reward_penalty_config: dict[str, Any] | BaseModel | None = None,
     thinking_tags: list[str] | tuple[str, ...] | None = None,
+    sampling_params: Optional[SamplingParams] = None,
 ) -> NemoGymRolloutResult:
     """Run and return one complete NeMo-Gym batch synchronously.
 
@@ -2206,6 +2216,9 @@ def run_nemo_gym_rollout_sync(
         effort_config: Optional configuration for effort-based reward shaping.
         reward_penalty_config: Optional reward-penalty configuration.
         thinking_tags: Optional opening and closing tags used by thinking penalties.
+        sampling_params: Sampling profile stamped onto every NeMo-Gym row.
+            ``None`` uses the train profile from ``generation_config``;
+            validation passes its own profile explicitly.
 
     Returns:
         The fully postprocessed NeMo-Gym rollout batch in input-row order.
@@ -2236,6 +2249,7 @@ async def _consume_rollout() -> NemoGymRolloutResult:
             reward_penalty_config=reward_penalty_config,
             thinking_tags=thinking_tags,
             returns_entire_batch=True,
+            sampling_params=sampling_params,
         ):
             pass
         if rollout_result is None: