diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index 75269d19107..0aa834dbc7c 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -697,6 +697,8 @@ jobs: runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} - script: L1_Functional_Tests_Megatron_3 runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} + - script: L1_Functional_Tests_Megatron_4 + runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} - script: L1_Functional_Tests_AutoModel runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} - script: L1_Functional_Tests_SGLang @@ -764,6 +766,8 @@ jobs: runner: ${{ vars.GB200_RUNNER }} - script: L1_Functional_Tests_Megatron_3 runner: ${{ vars.GB200_RUNNER }} + - script: L1_Functional_Tests_Megatron_4 + runner: ${{ vars.GB200_RUNNER }} - script: L1_Functional_Tests_AutoModel runner: ${{ vars.GB200_RUNNER }} - script: L1_Functional_Tests_SGLang @@ -832,6 +836,8 @@ jobs: runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} - script: L1_Functional_Tests_Megatron_3 runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} + - script: L1_Functional_Tests_Megatron_4 + runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} - script: L1_Functional_Tests_AutoModel runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} - script: L1_Functional_Tests_Gym diff --git a/examples/configs/grpo_math_1B.yaml b/examples/configs/grpo_math_1B.yaml index 43ac05dd0c5..425709b43e2 100644 --- a/examples/configs/grpo_math_1B.yaml +++ b/examples/configs/grpo_math_1B.yaml @@ -307,13 +307,23 @@ policy: stop_token_ids: null stop_strings: null mcore_generation_config: - buffer_size_gb: 10 # Total buffer size is 2x this value (active requests + paused requests) + async_engine: false + max_model_len: ${policy.max_total_sequence_length} # Engine-side max sequence length. + cuda_graph_impl: "local" # CUDA-graph implementation. Options: "none", "local", "transformer_engine", "full_iteration". + inference_cuda_graph_scope: "block" # Inference CUDA-graph scope. Options: "none", "layer", "block". + buffer_size_gb: 10 num_cuda_graphs: 4 # Number of CUDA graphs to pre-compile for different batch sizes block_size_tokens: 256 # Size of each KV cache block in tokens (affects memory granularity) use_cuda_graphs_for_non_decode_steps: true # Enable CUDA graphs for prefill/context processing enable_chunked_prefill: true # Split long prefills into chunks for better memory management - unified_memory_level: 0 # Unified memory usage level (0=disabled, higher values enable more aggressive paging) + enable_prefix_caching: false # Reuse KV blocks across requests sharing a prompt prefix. max_tokens: 16384 # Maximum number of tokens to use in a single step. Analogous to vllm's max_num_batched_tokens + kv_cache_management_mode: "persist" # KV cache lifecycle across suspend/resume. Options: "persist", "offload". To select "recompute", set grpo.async_grpo.recompute_kv_cache_after_weight_updates=true. + materialize_only_last_token_logits: true + num_speculative_tokens: 0 + refit_backend: "nvshmem" # Copy-service backend for non-colocated megatron weight refit. Options: "gloo" or "nvshmem". + parsers: [] # OpenAI tool-call / response parser names exposed by the HTTP server (only used if expose_http_server=true). + expose_http_server: false # Start an OpenAI-compatible HTTP server alongside the persistent engine. Required for NeMo-Gym. vllm_cfg: async_engine: false precision: ${policy.precision} diff --git a/examples/configs/grpo_math_1B_megatron.yaml b/examples/configs/grpo_math_1B_megatron.yaml index 0d4cc491c3c..8c880e18df2 100644 --- a/examples/configs/grpo_math_1B_megatron.yaml +++ b/examples/configs/grpo_math_1B_megatron.yaml @@ -114,7 +114,7 @@ policy: moe_enable_deepep: false moe_token_dispatcher_type: "alltoall" moe_shared_expert_overlap: false - #gives ~20% training perf speedup with sequence packing + #gives ~20% training perf speedup with sequence packing apply_rope_fusion: True peft: @@ -175,15 +175,6 @@ policy: temperature: 1.0 top_p: 1.0 top_k: null - mcore_generation_config: - buffer_size_gb: 10 # Total buffer size is 2x this value (active requests + paused requests) - num_cuda_graphs: 4 # Number of CUDA graphs to pre-compile for different batch sizes - block_size_tokens: 256 # Size of each KV cache block in tokens (affects memory granularity) - use_cuda_graphs_for_non_decode_steps: true # Enable CUDA graphs for prefill/context processing - enable_chunked_prefill: false # Split long prefills into chunks for better memory management - unified_memory_level: 0 # Unified memory usage level (0=disabled, higher values enable more aggressive paging) - max_tokens: 16384 # Maximum number of tokens to use in a single step. Analogous to vllm's max_num_batched_tokens - vllm_cfg: tensor_parallel_size: 1 gpu_memory_utilization: 0.6 diff --git a/examples/configs/recipes/llm/grpo-llama3.2-1b-instruct-2n8g-megatron_generation-noncolocated.yaml b/examples/configs/recipes/llm/grpo-llama3.2-1b-instruct-2n8g-megatron_generation-noncolocated.yaml new file mode 100644 index 00000000000..9e9577adb18 --- /dev/null +++ b/examples/configs/recipes/llm/grpo-llama3.2-1b-instruct-2n8g-megatron_generation-noncolocated.yaml @@ -0,0 +1,41 @@ +defaults: ../../grpo_math_1B.yaml +grpo: + max_num_steps: 30 +checkpointing: + enabled: false + checkpoint_dir: results/grpo-llama3.2-1b-instruct-2n8g-megatron_generation-noncolocated + save_period: 100 +policy: + model_name: meta-llama/Llama-3.2-1B-Instruct + tokenizer: + name: meta-llama/Llama-3.2-1B-Instruct + optimizer: null + megatron_cfg: + enabled: true + scheduler: + lr_warmup_iters: 50 + dtensor_cfg: + enabled: false + make_sequence_length_divisible_by: 1 + generation: + backend: megatron + max_new_tokens: 512 + vllm_cfg: + max_model_len: 512 + colocated: + enabled: false + resources: + gpus_per_node: 8 + num_nodes: 1 +data: + max_input_seq_length: 512 +logger: + log_dir: logs/grpo-llama3.2-1b-instruct-2n8g-megatron_generation-noncolocated + wandb_enabled: true + tensorboard_enabled: true + wandb: + project: nemo-rl + name: grpo-llama3.2-1b-instruct-2n8g-megatron_generation-noncolocated +cluster: + gpus_per_node: 8 + num_nodes: 2 diff --git a/examples/configs/recipes/llm/grpo-nanov3-30BA3B-2n8g-megatron_generation-async-gym.yaml b/examples/configs/recipes/llm/grpo-nanov3-30BA3B-2n8g-megatron_generation-async-gym.yaml new file mode 100644 index 00000000000..6ed53a48d93 --- /dev/null +++ b/examples/configs/recipes/llm/grpo-nanov3-30BA3B-2n8g-megatron_generation-async-gym.yaml @@ -0,0 +1,30 @@ +defaults: ../../../nemo_gym/grpo_nanov3.yaml +grpo: + async_grpo: + enabled: true + in_flight_weight_updates: true +checkpointing: + checkpoint_dir: results/grpo-nanov3-30BA3B-2n8g-megatron_generation-async-gym +policy: + model_name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16 + tokenizer: + name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 + train_global_batch_size: 16 + max_total_sequence_length: 2048 + sequence_packing: + enabled: false + generation: + backend: megatron + colocated: + enabled: false + resources: + gpus_per_node: 8 + num_nodes: 1 +logger: + wandb_enabled: true + tensorboard_enabled: true + wandb: + project: nemo-rl + name: grpo-nanov3-30BA3B-2n8g-megatron_generation-async-gym +cluster: + num_nodes: 2 diff --git a/examples/configs/recipes/llm/grpo-nanov3-30BA3B-2n8g-megatron_generation.yaml b/examples/configs/recipes/llm/grpo-nanov3-30BA3B-2n8g-megatron_generation.yaml new file mode 100644 index 00000000000..500097f8653 --- /dev/null +++ b/examples/configs/recipes/llm/grpo-nanov3-30BA3B-2n8g-megatron_generation.yaml @@ -0,0 +1,47 @@ +defaults: ../../grpo_math_1B.yaml +grpo: + num_prompts_per_step: 2 + num_generations_per_prompt: 8 +checkpointing: + checkpoint_dir: results/grpo-nanov3-30BA3B-2n8g-megatron_generation +policy: + model_name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16 + tokenizer: + name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 + train_global_batch_size: 16 + train_micro_batch_size: 1 + logprob_batch_size: 1 + max_total_sequence_length: 2048 + dtensor_cfg: + enabled: false + megatron_cfg: + enabled: true + bias_activation_fusion: false + tensor_model_parallel_size: 2 + expert_model_parallel_size: 8 + sequence_parallel: true + sequence_packing: + enabled: false + generation: + backend: megatron + mcore_generation_config: + transformer_impl: inference_optimized + activation_checkpointing: false + tensor_model_parallel_size: 2 + expert_model_parallel_size: 8 + sequence_parallel: true + async_engine: true + colocated: + enabled: false + resources: + gpus_per_node: 8 + num_nodes: 1 +logger: + wandb_enabled: true + tensorboard_enabled: true + wandb: + project: nemo-rl + name: grpo-nanov3-30BA3B-2n8g-megatron_generation +cluster: + gpus_per_node: 8 + num_nodes: 2 diff --git a/examples/nemo_gym/grpo_nanov3.yaml b/examples/nemo_gym/grpo_nanov3.yaml index 61bdd5f5d24..35d9a881f83 100644 --- a/examples/nemo_gym/grpo_nanov3.yaml +++ b/examples/nemo_gym/grpo_nanov3.yaml @@ -196,6 +196,42 @@ policy: top_k: null stop_token_ids: null stop_strings: null + mcore_generation_config: + # Scoped to generation workers only; training keeps its own defaults + # via the base yaml. The mcore_generation_config block is merged into + # megatron_cfg for inference workers before model setup runs. + max_model_len: ${policy.max_total_sequence_length} + transformer_impl: "inference_optimized" + cuda_graph_impl: "local" + inference_cuda_graph_scope: "block" + activation_checkpointing: false + mamba_inference_ssm_states_dtype: "float32" + inference_moe_token_dispatcher_type: "nccl" # Fall back to NCCL for now + inference_grouped_gemm_backend: "vllm" + moe_router_num_groups: null # InferenceTopKRouter requires num_groups=None + moe_router_group_topk: null # paired with moe_router_num_groups=null + pipeline_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + expert_model_parallel_size: 8 + sequence_parallel: True + context_parallel_size: 1 + tensor_model_parallel_size: 2 + buffer_size_gb: 20 # Total GPU memory (in GB) allocated for KV cache buffers + num_cuda_graphs: -1 # Number of CUDA graphs to pre-compile for different batch sizes + block_size_tokens: 256 # Size of each KV cache block in tokens (affects memory granularity) + use_cuda_graphs_for_non_decode_steps: true # Enable CUDA graphs for prefill/context processing + enable_chunked_prefill: true + max_tokens: 16384 # Maximum number of tokens to use in a single step. Analogous to vllm's max_num_batched_tokens + kv_cache_management_mode: "persist" # KV cache lifecycle across suspend/resume. Options: "persist", "offload". To select "recompute", set grpo.async_grpo.recompute_kv_cache_after_weight_updates=true. + materialize_only_last_token_logits: true + num_speculative_tokens: 0 + refit_backend: "nvshmem" # Copy-service backend for non-colocated megatron weight refit. Options: "gloo" or "nvshmem". + async_engine: true + expose_http_server: true + enable_prefix_caching: true + parsers: + - deepseek-r1-reasoning + - qwen3-coder-tool vllm_cfg: # NB: can re-enable prefix cache on vllm >= 0.11.2. # enable_prefix_caching: false diff --git a/examples/nemo_gym/grpo_qwen3_30ba3b_instruct.yaml b/examples/nemo_gym/grpo_qwen3_30ba3b_instruct.yaml index 3587432e4c0..f74d60327e9 100644 --- a/examples/nemo_gym/grpo_qwen3_30ba3b_instruct.yaml +++ b/examples/nemo_gym/grpo_qwen3_30ba3b_instruct.yaml @@ -63,6 +63,24 @@ policy: tool_parser: hermes # Enable the appropriate reasoning parser here. Since this model is an instruct model, we comment it out. # reasoning_parser: deepseek_r1 + mcore_generation_config: + async_engine: false + max_model_len: ${policy.max_total_sequence_length} + cuda_graph_impl: "local" + inference_cuda_graph_scope: "block" + buffer_size_gb: 10 + num_cuda_graphs: 4 + block_size_tokens: 256 + use_cuda_graphs_for_non_decode_steps: true + enable_chunked_prefill: true + enable_prefix_caching: false + max_tokens: 16384 + kv_cache_management_mode: "persist" + materialize_only_last_token_logits: true + num_speculative_tokens: 0 + refit_backend: "nvshmem" + parsers: [] + expose_http_server: false # Needs to be set to whatever backend TP size. make_sequence_length_divisible_by: ${policy.megatron_cfg.tensor_model_parallel_size} diff --git a/nemo_rl/algorithms/async_utils/trajectory_collector.py b/nemo_rl/algorithms/async_utils/trajectory_collector.py index decb984ea2b..12d1b8e3af1 100644 --- a/nemo_rl/algorithms/async_utils/trajectory_collector.py +++ b/nemo_rl/algorithms/async_utils/trajectory_collector.py @@ -437,9 +437,9 @@ def resume(self) -> None: def prepare_for_refit(self) -> None: """Pause new generation starts and optionally wait for pending generations. - For vLLM V1 async engine, leverages in-flight weight updates via collective_rpc, - allowing ongoing generations to continue with their current KV caches while - weights are updated. This significantly improves async performance. + For backends with an async engine in-flight weight updates allows ongoing generations + to continue with their current KV caches while weights are updated. + This significantly improves async performance. For non-async engines, waits for all pending generations to complete before refit. """ @@ -450,19 +450,29 @@ def prepare_for_refit(self) -> None: self._refit_pause_cleared.clear() print("⏸️ New generation starts paused") - # Check if we're using vLLM async engine - vllm_cfg = self.master_config.policy.get("generation", {}).get("vllm_cfg", {}) - is_async_engine = vllm_cfg.get("async_engine", False) + # Check if we're using async engine + generation_cfg = self.master_config.policy.get("generation", {}) + backend = generation_cfg.get("backend", "") + if backend == "vllm": + is_async_engine = generation_cfg.get("vllm_cfg", {}).get( + "async_engine", False + ) + elif backend == "megatron": + is_async_engine = generation_cfg.get("mcore_generation_config", {}).get( + "async_engine", False + ) + else: + is_async_engine = False in_flight_weight_updates = self.master_config.grpo.get("async_grpo", {}).get( "in_flight_weight_updates", False ) if is_async_engine and in_flight_weight_updates: - # vLLM V1 async engine supports in-flight weight updates + # async engines support in-flight weight updates # Ongoing generations will continue with their current KV caches # New generations (after weight update) will use the updated weights print( - "πŸš€ Using vLLM V1 in-flight weight update - skipping wait for pending generations" + f"πŸš€ Using {backend} in-flight weight update - skipping wait for pending generations" ) print( f" {len(self._inflight_threads)} ongoing generations will complete with current weights" diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index b963a6d4ff5..d9ef6aa6621 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -81,6 +81,7 @@ run_multi_turn_rollout, ) from nemo_rl.models.generation.interfaces import GenerationInterface +from nemo_rl.models.generation.megatron import MegatronGeneration from nemo_rl.models.generation.sglang import SGLangConfig, SGLangGeneration from nemo_rl.models.generation.vllm import VllmConfig, VllmGeneration from nemo_rl.models.megatron.router_replay import configure_vllm_for_router_replay @@ -453,6 +454,54 @@ def init_train_dataloader(dataset, suffix: str = ""): nemo_gym_num_nodes = 0 ray_cur_node_id = None + def _spinup_nemo_gym(base_urls, model_name): + """Spin up the NeMo Gym actor against the given generation server URLs.""" + t0 = time.perf_counter() + nemo_gym_py_exec = get_actor_python_env("nemo_rl.environments.nemo_gym.NemoGym") + if nemo_gym_py_exec.startswith("uv"): + nemo_gym_py_exec = create_local_venv_on_each_node( + nemo_gym_py_exec, "nemo_rl.environments.nemo_gym.NemoGym" + ) + nemo_gym_dict = env_configs["nemo_gym"] + # NeMo-RL-side detection knobs are top-level NemoGymConfig fields + # (where the detector reads them), not part of Gym's global config. + invalid_tool_call_patterns = nemo_gym_dict.pop( + "invalid_tool_call_patterns", None + ) + thinking_tags = nemo_gym_dict.pop("thinking_tags", None) + # Pass prebuilt cache + venv dirs through the global config so the gym reuses + # image-baked venvs instead of rebuilding them. + uv_cache_dir = get_nemo_gym_uv_cache_dir() + if uv_cache_dir is not None: + nemo_gym_dict.setdefault("uv_cache_dir", uv_cache_dir) + uv_venv_dir = get_nemo_gym_venv_dir() + if uv_venv_dir is not None: + nemo_gym_dict.setdefault("uv_venv_dir", uv_venv_dir) + nemo_gym_cfg = NemoGymConfig( + model_name=model_name, + base_urls=base_urls, + invalid_tool_call_patterns=invalid_tool_call_patterns, + thinking_tags=thinking_tags, + initial_global_config_dict=nemo_gym_dict, + ) + nemo_gym_opts = {} + if nemo_gym_num_nodes: + nemo_gym_opts["scheduling_strategy"] = NodeAffinitySchedulingStrategy( + node_id=ray_cur_node_id, + soft=True, + ) + nemo_gym_opts["runtime_env"] = { + "py_executable": nemo_gym_py_exec, + "env_vars": { + **os.environ, + "VIRTUAL_ENV": nemo_gym_py_exec, + "UV_PROJECT_ENVIRONMENT": nemo_gym_py_exec, + }, + } + actor = NemoGym.options(**nemo_gym_opts).remote(nemo_gym_cfg) + ray.get(actor._spinup.remote()) + return actor, time.perf_counter() - t0 + total_nodes = cluster_config["num_nodes"] if rm_env_enabled: rm_resource = env_configs["reward_model"]["resources"] @@ -501,11 +550,6 @@ def init_train_dataloader(dataset, suffix: str = ""): ) else: - assert generation_config["backend"] != "megatron", ( - "Non-colocated inference is not supported for Megatron generation backends. " - "Please use vLLM backend for generation." - ) - # train resources will be updated through overall and inference resources below train_gpus_per_node = cluster_config["gpus_per_node"] train_nodes = policy_nodes @@ -614,6 +658,20 @@ def init_train_dataloader(dataset, suffix: str = ""): ) policy_config["megatron_cfg"]["train_iters"] = total_train_iters + # When the user opts into recompute-after-refit on the megatron side, + # override mcore's kv_cache_management_mode to "recompute" directly. + if "async_grpo" in grpo_config and grpo_config["async_grpo"].get( + "recompute_kv_cache_after_weight_updates", False + ): + mcore_cfg = policy_config["generation"]["mcore_generation_config"] + prior_mode = mcore_cfg.get("kv_cache_management_mode", "persist") + if prior_mode != "recompute": + print( + f"kv_cache_management_mode overridden '{prior_mode}' -> 'recompute' by " + f"grpo.async_grpo.recompute_kv_cache_after_weight_updates=True." + ) + mcore_cfg["kv_cache_management_mode"] = "recompute" + # Define initialization functions that will be used in all paths init_reference_model = loss_config.reference_policy_kl_penalty > 0 @@ -662,6 +720,26 @@ def init_sglang(): pg.finish_generation() return pg, time.perf_counter() - t0 + def init_megatron_generation(policy=None): + """Initialize Megatron generation.""" + t0 = time.perf_counter() + if colocated_inference: + mg = MegatronGeneration( + policy=policy, + config=policy_config, + tokenizer=tokenizer, + processor=processor, + ) + else: + mg = MegatronGeneration( + cluster=inference_cluster, + config=policy_config, + tokenizer=tokenizer, + processor=processor, + weights_path=weights_path, + ) + return mg, time.perf_counter() - t0 + def initialize_generation_with_policy( init_generation_fn, generation_name: str, @@ -725,16 +803,29 @@ def initialize_generation_with_policy( # Handle generation-specific setup if backend == "megatron": - # Megatron generation: policy_generation is None, only initialize policy - policy_generation = None + # Initialize training first so checkpoint conversion completes before inference starts. + policy, policy_time = init_policy() + worker_init_timing_metrics["policy_init_time_s"] = policy_time + + # Colocated wraps the training policy; non-colocated builds a dedicated inference policy. + policy_generation, megatron_gen_time = init_megatron_generation(policy) + worker_init_timing_metrics["megatron_generation_init_time_s"] = ( + megatron_gen_time + ) + + if enable_nemo_gym: + # The Megatron inference engine must be up before its server URLs exist. + nemo_gym_actor, nemo_gym_time = _spinup_nemo_gym( + policy_generation.dp_openai_server_base_urls, + generation_config["model_name"], + ) + worker_init_timing_metrics["nemo_gym_init_time_s"] = nemo_gym_time + print( f" βœ“ Using {backend} backend for generation with {policy_config['model_name']}", flush=True, ) - policy, policy_time = init_policy() - worker_init_timing_metrics["policy_init_time_s"] = policy_time - elif backend == "vllm": # vLLM generation: setup config, then initialize with policy generation_config = cast(VllmConfig, generation_config) @@ -792,55 +883,10 @@ def init_vllm_deferred(): def init_nemo_gym(): """Spin up NeMo Gym servers with the pre-assigned vLLM URLs.""" - t0 = time.perf_counter() - nemo_gym_py_exec = get_actor_python_env( - "nemo_rl.environments.nemo_gym.NemoGym" + return _spinup_nemo_gym( + deferred_vllm.dp_openai_server_base_urls, + generation_config["model_name"], ) - if nemo_gym_py_exec.startswith("uv"): - nemo_gym_py_exec = create_local_venv_on_each_node( - nemo_gym_py_exec, "nemo_rl.environments.nemo_gym.NemoGym" - ) - nemo_gym_dict = env_configs["nemo_gym"] - # NeMo-RL-side detection knobs are top-level NemoGymConfig fields - # (where the detector reads them), not part of Gym's global config. - invalid_tool_call_patterns = nemo_gym_dict.pop( - "invalid_tool_call_patterns", None - ) - thinking_tags = nemo_gym_dict.pop("thinking_tags", None) - # Pass prebuilt cache + venv dirs through the global config so the gym reuses - # image-baked venvs instead of rebuilding them. - uv_cache_dir = get_nemo_gym_uv_cache_dir() - if uv_cache_dir is not None: - nemo_gym_dict.setdefault("uv_cache_dir", uv_cache_dir) - uv_venv_dir = get_nemo_gym_venv_dir() - if uv_venv_dir is not None: - nemo_gym_dict.setdefault("uv_venv_dir", uv_venv_dir) - nemo_gym_cfg = NemoGymConfig( - model_name=generation_config["model_name"], - base_urls=deferred_vllm.dp_openai_server_base_urls, - invalid_tool_call_patterns=invalid_tool_call_patterns, - thinking_tags=thinking_tags, - initial_global_config_dict=nemo_gym_dict, - ) - nemo_gym_opts = {} - if nemo_gym_num_nodes: - nemo_gym_opts["scheduling_strategy"] = ( - NodeAffinitySchedulingStrategy( - node_id=ray_cur_node_id, - soft=True, - ) - ) - nemo_gym_opts["runtime_env"] = { - "py_executable": nemo_gym_py_exec, - "env_vars": { - **os.environ, - "VIRTUAL_ENV": nemo_gym_py_exec, - "UV_PROJECT_ENVIRONMENT": nemo_gym_py_exec, - }, - } - actor = NemoGym.options(**nemo_gym_opts).remote(nemo_gym_cfg) - ray.get(actor._spinup.remote()) - return actor, time.perf_counter() - t0 # Colocated: vLLM + policy share GPUs -> sequential; otherwise parallel. init_tasks = {} @@ -925,18 +971,37 @@ def init_vllm_then_policy(): train_world_size = train_cluster.world_size() inference_world_size = inference_nodes * inference_gpus_per_node world_size = train_world_size + inference_world_size + # init collective - futures_train = policy.init_collective( - ip, port, world_size, train_world_size=train_world_size - ) - futures_inference = policy_generation.init_collective( - ip, port, world_size, train_world_size=train_world_size - ) # type: ignore - # wait for all futures to complete - ray.get(futures_train + futures_inference) + if backend == "megatron": + refit_backend = policy_config["generation"]["mcore_generation_config"][ + "refit_backend" + ] + futures_train = policy.init_collective_mcore_generation( + ip, + port, + world_size, + rank_offset=0, + refit_backend=refit_backend, + ) + futures_inference = policy_generation.init_collective( + ip, + port, + world_size, + train_world_size=train_world_size, + refit_backend=refit_backend, + ) + ray.get(futures_train + futures_inference) + else: + futures_train = policy.init_collective( + ip, port, world_size, train_world_size=train_world_size + ) + futures_inference = policy_generation.init_collective( + ip, port, world_size, train_world_size=train_world_size + ) # type: ignore + ray.get(futures_train + futures_inference) worker_init_timing_metrics["collective_init_time_s"] = time.perf_counter() - t0 - # prepare refit info state_dict_info = policy.prepare_refit_info() if policy_generation is not None: policy_generation.prepare_refit_info(state_dict_info) @@ -1357,18 +1422,21 @@ def _apply_configured_message_level_advantage_penalties( def _should_use_async_rollouts(master_config: MasterConfig) -> bool: """Determine if async rollouts should be used based on the configuration. - Returns True if vLLM backend is used with async_engine enabled. + Returns True if async_engine is enabled. """ generation_config = master_config.policy["generation"] if generation_config is None: return False - backend = generation_config.get("backend", "") - if backend != "vllm": - return False - vllm_cfg = generation_config.get("vllm_cfg", {}) - return vllm_cfg.get("async_engine", False) + if backend == "vllm": + vllm_cfg = generation_config.get("vllm_cfg", {}) + return vllm_cfg.get("async_engine", False) + elif backend == "megatron": + mcore_cfg = generation_config.get("mcore_generation_config", {}) + return mcore_cfg.get("async_engine", False) + else: + return False def _should_use_nemo_gym(master_config: MasterConfig) -> bool: @@ -1380,14 +1448,23 @@ def _should_use_nemo_gym(master_config: MasterConfig) -> bool: # Validate the setup for training with NeMo-Gym assert _should_use_async_rollouts(master_config), ( - "❌ Error: In order to use NeMo-Gym, you must use vllm generation backend with `async_engine: true`!" + "❌ Error: In order to use NeMo-Gym, you must use a generation backend with `async_engine: true`!" ) # We piggyback off of `_should_use_async_rollouts` to guarantee the existence of these configs. generation_config = master_config.policy["generation"] - should_expose_http_server = generation_config["vllm_cfg"].get("expose_http_server") + if generation_config["backend"] == "vllm": + should_expose_http_server = generation_config["vllm_cfg"].get( + "expose_http_server" + ) + elif generation_config["backend"] == "megatron": + should_expose_http_server = generation_config["mcore_generation_config"].get( + "expose_http_server" + ) + else: + should_expose_http_server = False assert should_expose_http_server, ( - "In order to use NeMo-Gym, you must expose the vllm server via `expose_http_server: true`!" + "In order to use NeMo-Gym, you must expose the generation server via `expose_http_server: true`!" ) return should_use_nemo_gym @@ -1493,10 +1570,27 @@ def refit_policy_generation( timer: Optional Timer used to time the prepare/transfer/update phase kv_scales: Optional dictionary of KV cache scales for FP8 quantization. """ - if colocated_inference: + # Megatron generation backend needs explicit suspend/resume around refits. + if isinstance(policy_generation, MegatronGeneration): + policy_generation.suspend_for_refit() + + if colocated_inference or isinstance(policy_generation, MegatronGeneration): policy.offload_before_refit() + # Colocated inference needs to prepare for generation. + # Megatron non-colocated inference needs to enter inference mode after refit. + if colocated_inference or isinstance(policy_generation, MegatronGeneration): policy_generation.prepare_for_generation(tags=["weights"]) + if ( + not colocated_inference + and isinstance(policy_generation, MegatronGeneration) + and policy_generation.cfg["mcore_generation_config"]["refit_backend"] + == "nvshmem" + ): + futures_train = policy.preinit_nvshmem() + futures_inference = policy_generation.preinit_nvshmem_collective() + ray.get(futures_train + futures_inference) + # Create a context manager that does nothing when timer is None timer_context = ( timer.time("prepare_for_generation/transfer_and_update_weights") @@ -1543,14 +1637,19 @@ def refit_policy_generation( results = ray.get(futures_inference) update_success = all(result for result in results if result is not None) else: - # update weights through nccl + # update weights through nccl (vLLM) or megatron reshard # SGLang haven't implemented non-colocated inference mode. if isinstance(policy_generation, SGLangGeneration): raise NotImplementedError( "SGLang haven't implemented non-colocated inference mode. " ) - policy.offload_before_refit() - futures_train = policy.broadcast_weights_for_collective(kv_scales=kv_scales) + if isinstance(policy_generation, MegatronGeneration): + futures_train = policy.swap_weights_via_reshard(is_source=True) + else: + policy.offload_before_refit() + futures_train = policy.broadcast_weights_for_collective( + kv_scales=kv_scales + ) futures_inference = policy_generation.update_weights_from_collective() # wait for all futures to complete ray.get(futures_train) @@ -1570,8 +1669,14 @@ def refit_policy_generation( if colocated_inference: policy.offload_after_refit() + # Colocated inference needs to prepare for generation. + # Megatron non-colocated inference needs to enter inference mode after refit. + if colocated_inference or isinstance(policy_generation, MegatronGeneration): policy_generation.prepare_for_generation(tags=["kv_cache"]) + if isinstance(policy_generation, MegatronGeneration): + policy_generation.resume_after_refit() + def _log_mixed_rewards_and_advantages_information( logger: Logger, @@ -1757,11 +1862,10 @@ def grpo_train( kv_scales_cache = None # Cache reused for computed kv scales - NEED_REFIT = True - # If policy_generation is None, use the policy as the generation interface (megatron framework backend) - if policy_generation is None: - policy_generation = policy # type: ignore - NEED_REFIT = False + NEED_REFIT = not ( + isinstance(policy_generation, MegatronGeneration) + and master_config.policy["generation"]["colocated"]["enabled"] + ) POLICY_GENERATION_STALE = True # tracks if generation needs a refit before running assert policy_generation is not None @@ -2920,8 +3024,9 @@ def async_grpo_train( """ # Ensure we are running with a compatible async generation backend assert _should_use_async_rollouts(master_config), ( - "Async GRPO requires vLLM backend with vllm_cfg.async_engine=True. " - "Set policy.generation.vllm_cfg.async_engine to true in your config." + "Async GRPO requires an async generation engine. " + "Set either policy.generation.vllm_cfg.async_engine=true (vLLM) or " + "policy.generation.mcore_generation_config.async_engine=true (Megatron)." ) assert master_config.loss_fn.use_importance_sampling_correction, ( "Importance sampling correction must be enabled for async GRPO for good convergence due to off-policy samples!" @@ -2943,12 +3048,10 @@ def async_grpo_train( fit_last_save_time=True, ) timeout.start_iterations() - NEED_REFIT = True - - # Setup generation interface - if policy_generation is None: - policy_generation = policy - NEED_REFIT = False + NEED_REFIT = not ( + isinstance(policy_generation, MegatronGeneration) + and master_config.policy["generation"]["colocated"]["enabled"] + ) POLICY_GENERATION_STALE = True assert policy_generation is not None @@ -3508,7 +3611,9 @@ def async_grpo_train( print("πŸ”„ Performing policy generation refit...") with timer.time("weight_sync"): refit_policy_generation( - policy, policy_generation, colocated_inference + policy, + policy_generation, + colocated_inference, ) POLICY_GENERATION_STALE = False diff --git a/nemo_rl/experience/rollouts.py b/nemo_rl/experience/rollouts.py index ed39a27e337..9d7ce87bd73 100644 --- a/nemo_rl/experience/rollouts.py +++ b/nemo_rl/experience/rollouts.py @@ -330,16 +330,24 @@ async def generate_responses_async( # Ensure the key exists even if it's None, matching GenerationDatumSpec generation_input_data["stop_strings"] = [None] * len(input_lengths) - # Check if this is vLLM with async_engine enabled + # Check if this is a supported inference engine with async_engine enabled. use_async_generation = ( hasattr(policy_generation, "cfg") - and "vllm_cfg" in policy_generation.cfg - and policy_generation.cfg["vllm_cfg"]["async_engine"] + and ( + ( + "vllm_cfg" in policy_generation.cfg + and policy_generation.cfg["vllm_cfg"]["async_engine"] + ) + or ( + "mcore_generation_config" in policy_generation.cfg + and policy_generation.cfg["mcore_generation_config"]["async_engine"] + ) + ) and hasattr(policy_generation, "generate_async") ) assert use_async_generation, ( - "Async generation is not enabled. Please enable async generation by setting async_engine=True in the vllm_cfg section of the policy config." + "Async generation is not enabled for the configured generation backend. " ) # Use async generation with per-sample streaming @@ -1327,7 +1335,14 @@ def run_async_nemo_gym_rollout( assert max_rollout_turns is None, ( "`max_rollout_turns` is not supported in NeMo-Gym path!" ) - engine_max_model_len = policy_generation.cfg["vllm_cfg"]["max_model_len"] + if "vllm_cfg" in policy_generation.cfg: + engine_max_model_len = policy_generation.cfg["vllm_cfg"]["max_model_len"] + elif "mcore_generation_config" in policy_generation.cfg: + engine_max_model_len = policy_generation.cfg["mcore_generation_config"][ + "max_model_len" + ] + else: + engine_max_model_len = policy_generation.cfg["max_total_sequence_length"] if max_seq_len is not None and max_seq_len > engine_max_model_len: warnings.warn( f"policy max_total_sequence_length ({max_seq_len}) is greater than the " @@ -1396,7 +1411,18 @@ def run_async_nemo_gym_rollout( # Prepare for the rollout metrics calculation below. Not strictly necessary here, but good to have parity with `run_async_multi_turn_rollout` with timer.time(f"{timer_prefix}/prepare_for_metrics_calculation"): batch_size = len(nemo_gym_rows) - max_total_tokens_per_sample = policy_generation.cfg["vllm_cfg"]["max_model_len"] + if "vllm_cfg" in policy_generation.cfg: + max_total_tokens_per_sample = policy_generation.cfg["vllm_cfg"][ + "max_model_len" + ] + elif "mcore_generation_config" in policy_generation.cfg: + max_total_tokens_per_sample = policy_generation.cfg[ + "mcore_generation_config" + ]["max_model_len"] + else: + max_total_tokens_per_sample = policy_generation.cfg[ + "max_total_sequence_length" + ] all_sample_metrics = [ { "total_reward": r["full_result"]["reward"], diff --git a/nemo_rl/models/generation/megatron/__init__.py b/nemo_rl/models/generation/megatron/__init__.py new file mode 100644 index 00000000000..755b0e12f21 --- /dev/null +++ b/nemo_rl/models/generation/megatron/__init__.py @@ -0,0 +1,27 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_rl.models.generation.megatron.config import ( + MCoreGenerationConfig, + MCoreGenerationSpecificArgs, +) +from nemo_rl.models.generation.megatron.megatron_generation import ( + MegatronGeneration, +) + +__all__ = [ + "MCoreGenerationConfig", + "MCoreGenerationSpecificArgs", + "MegatronGeneration", +] diff --git a/nemo_rl/models/generation/megatron/config.py b/nemo_rl/models/generation/megatron/config.py new file mode 100644 index 00000000000..7cfb88cc1f4 --- /dev/null +++ b/nemo_rl/models/generation/megatron/config.py @@ -0,0 +1,72 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Literal, NotRequired, TypedDict + +from nemo_rl.models.generation.interfaces import GenerationConfig + + +class MCoreGenerationSpecificArgs(TypedDict): + """Megatron fields related only to inference. + + Any fields not declared here but declared in the training-side config can be overwritten. + For example, Megatron inference might want `transformer_impl: "inference_optimized"`, + while Megatron training might want `transformer_impl: "transformer_engine"`. + """ + + async_engine: bool + expose_http_server: bool + parsers: list[str] + + buffer_size_gb: int + block_size_tokens: int + max_tokens: int + max_model_len: int + + num_cuda_graphs: int + use_cuda_graphs_for_non_decode_steps: bool + cuda_graph_impl: str + # Inference CUDA-graph scope. Options: + # - 'none': inference runs in eager mode (no CUDA graphs). + # - 'layer': graphs are owned at the per-layer boundary (TransformerLayer / MambaLayer). + # - 'block': graphs are owned at the enclosing block (TransformerBlock / HybridBlock). + # Only meaningful when cuda_graph_impl='local'. + inference_cuda_graph_scope: NotRequired[str] + + materialize_only_last_token_logits: bool + enable_chunked_prefill: bool + enable_prefix_caching: bool + + refit_backend: Literal["gloo", "nvshmem"] + num_speculative_tokens: int + + mamba_inference_ssm_states_dtype: NotRequired[str] + mamba_inference_conv_states_dtype: NotRequired[str] + + # KV cache lifecycle across suspend/resume: + # - "persist": cache stays allocated; CUDA graphs remain valid (default) + # - "offload": cache is moved off-GPU between iterations + # + # The third mcore value, "recompute" (drop + rebuild on resume), must be set via + # `grpo.async_grpo.recompute_kv_cache_after_weight_updates=true`. + # TODO: Unify `kv_cache_management_mode` and `recompute_kv_cache_after_weight_updates`. + kv_cache_management_mode: Literal["persist", "offload"] + + logging_step_interval: NotRequired[int] + + +class MCoreGenerationConfig(GenerationConfig): + """Generation config for Megatron Inference.""" + + mcore_generation_config: MCoreGenerationSpecificArgs diff --git a/nemo_rl/models/generation/megatron/megatron_generation.py b/nemo_rl/models/generation/megatron/megatron_generation.py new file mode 100644 index 00000000000..b8c2a4f2b28 --- /dev/null +++ b/nemo_rl/models/generation/megatron/megatron_generation.py @@ -0,0 +1,267 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING, Any, AsyncGenerator, Optional + +import ray +from transformers import AutoProcessor +from transformers.tokenization_utils_base import PreTrainedTokenizerBase + +from nemo_rl.distributed.batched_data_dict import BatchedDataDict +from nemo_rl.distributed.virtual_cluster import RayVirtualCluster +from nemo_rl.models.generation.interfaces import ( + GenerationDatumSpec, + GenerationInterface, + GenerationOutputSpec, +) +from nemo_rl.models.generation.megatron.config import MCoreGenerationConfig +from nemo_rl.models.policy import PolicyConfig + +if TYPE_CHECKING: + from nemo_rl.models.policy.lm_policy import Policy + + +class MegatronGeneration(GenerationInterface): + """Generation interface backed by Megatron (colocated or non-colocated).""" + + @staticmethod + def init_cluster_placement_groups( + cluster: RayVirtualCluster, + config: PolicyConfig, + ) -> None: + """Pre-initialize the inference cluster's placement groups. + + Args: + cluster: The inference `RayVirtualCluster`. + config: The full `PolicyConfig` (megatron parallelism + colocation). + """ + megatron_cfg = config["megatron_cfg"] + model_parallel_size = ( + megatron_cfg["tensor_model_parallel_size"] + * megatron_cfg["pipeline_model_parallel_size"] + * megatron_cfg["context_parallel_size"] + ) + colocated = config["generation"]["colocated"]["enabled"] + cluster._init_placement_groups( + strategy=None if colocated else "PACK", + use_unified_pg=model_parallel_size > cluster.num_gpus_per_node, + ) + + def __init__( + self, + config: PolicyConfig, + tokenizer: PreTrainedTokenizerBase, + cluster: Optional[RayVirtualCluster] = None, + policy: Optional["Policy"] = None, + name_prefix: str = "megatron_generation", + processor: Optional[AutoProcessor] = None, + weights_path: Optional[str] = None, + ): + """Initialize a MegatronGeneration instance. + + Exactly one of `cluster` or `policy` must be provided. + + Args: + config: PolicyConfig for the Megatron model. + tokenizer: The tokenizer for the model. + cluster: Cluster to deploy a dedicated inference Policy on. + policy: Existing training Policy to reuse for generation. + name_prefix: Prefix for naming the worker group (non-colocated only). + processor: Optional processor for VLMs (non-colocated only). + weights_path: Optional path to model weights (non-colocated only). + """ + # Import here to avoid circular imports + from nemo_rl.models.policy.lm_policy import Policy + + assert (cluster is None) != (policy is None), ( + "Provide exactly one of `cluster` or `policy`." + ) + + # `self.cfg` exposes the `generation` that matches the `GenerationInterface` contract. + # `self._policy_config` keeps a reference to the full PolicyConfig. + self._policy_config = config + self.cfg: MCoreGenerationConfig = config["generation"] + # Populated after the first prepare_for_generation (which starts the HTTP server). + self.dp_openai_server_base_urls: list[Optional[str]] = [] + + if policy is not None: + # Reuse the existing training policy. + self._policy = policy + self._owns_policy = False + return + + # Stand up a dedicated inference-only policy. + self._owns_policy = True + self._policy_config["megatron_cfg"].update(self.cfg["mcore_generation_config"]) + # Activation checkpointing is not compatible or useful in inference. + self._policy_config["megatron_cfg"]["activation_checkpointing"] = False + # Reserve GPUs before Policy workers grab them, to prevent disjoint NVLS domains. + self.init_cluster_placement_groups(cluster, self._policy_config) + self._policy = Policy( + cluster=cluster, + config=self._policy_config, + tokenizer=tokenizer, + name_prefix=name_prefix, + processor=processor, + init_optimizer=False, + init_reference_model=False, + weights_path=weights_path, + ) + + # Start the persistent inference engine + HTTP server during construction. + self.prepare_for_generation() + + url_futures = self._policy.worker_group.run_all_workers_single_data( + "report_dp_openai_server_base_url" + ) + self.dp_openai_server_base_urls = [ + url for url in ray.get(url_futures) if url is not None + ] + + def init_collective( + self, + ip: str, + port: int, + world_size: int, + *, + train_world_size: int, + refit_backend: str = "gloo", + ) -> list[ray.ObjectRef]: + """Initialize the refit collective for weight synchronization. + + Args: + ip: IP address for the process group rendezvous. + port: Port for the process group rendezvous. + world_size: Total world size (train + inference workers). + train_world_size: Number of training workers (used to offset ranks). + refit_backend: Copy service backend ("gloo" or "nvshmem"). + + Returns: + List of Ray ObjectRefs for the collective init futures. + """ + return self._policy.init_collective_mcore_generation( + ip, + port, + world_size, + rank_offset=train_world_size, + refit_backend=refit_backend, + ) + + def update_weights_from_collective(self) -> list[ray.ObjectRef]: + """Receive updated weights from the training cluster via collective communication.""" + return self._policy.swap_weights_via_reshard(is_source=False) + + def generate( + self, data: BatchedDataDict[GenerationDatumSpec], greedy: bool = False + ) -> BatchedDataDict[GenerationOutputSpec]: + """Generate a batch of data using the Megatron generation backend. + + mcore's data-parallel coordinator only accepts requests from DP rank 0 β€” + the other workers' engine loops drain the coordinator queue but never + receive a Python-side call. So we dispatch straight to worker 0. + + Args: + data: BatchedDataDict containing input_ids and input_lengths. + greedy: Whether to use greedy decoding. + + Returns: + BatchedDataDict conforming to GenerationOutputSpec. + """ + future = self._policy.worker_group.run_single_worker_single_data( + method_name="generate", + worker_idx=0, + data=data, + greedy=greedy, + ) + return ray.get(future) + + async def generate_async( + self, data: BatchedDataDict[GenerationDatumSpec], greedy: bool = False + ) -> AsyncGenerator[tuple[int, BatchedDataDict[GenerationOutputSpec]], None]: + """Generate asynchronously, yielding `(index, batch)` tuples as they complete.""" + worker = self._policy.worker_group.workers[0] + futures = worker.generate_async.options(num_returns="streaming").remote( + data=data, greedy=greedy + ) + async for result_ref in futures: + index, result_batch = await result_ref + result_batch["gen_leader_worker_idx"] = [0] + yield index, result_batch + + def prepare_for_generation(self, *args: Any, **kwargs: Any) -> bool: + """Initialize / re-enter inference mode on every worker. + + First call starts the persistent inference engine, coordinator, and the OpenAI HTTP server. + Subsequent calls re-enter inference mode after a refit. + """ + futures = self._policy.worker_group.run_all_workers_single_data( + "prepare_for_generation", **kwargs + ) + ray.get(futures) + return True + + def finish_generation(self, *args: Any, **kwargs: Any) -> bool: + """Clean up after generation.""" + futures = self._policy.worker_group.run_all_workers_single_data( + "finish_generation" + ) + ray.get(futures) + return True + + def preinit_nvshmem_collective(self) -> list[ray.ObjectRef]: + """Pre-initialize NVShmem collectively after CUDA graph capture. + + Must be called simultaneously on both training and inference workers. + """ + return self._policy.preinit_nvshmem() + + def suspend_for_refit(self) -> None: + """Suspend the inference engine for safe weight updates.""" + ray.get( + self._policy.worker_group.run_all_workers_single_data("suspend_for_refit") + ) + + def resume_after_refit(self) -> None: + """Resume the inference engine after weight updates.""" + ray.get( + self._policy.worker_group.run_all_workers_single_data("resume_after_refit") + ) + + def prepare_refit_info(self, state_dict_info: Optional[dict[str, Any]]) -> None: + """Accept the cross-backend refit-prep contract; Megatron needs none of it.""" + pass + + def start_gpu_profiling(self) -> None: + """Start GPU profiling on the dedicated inference workers. + + No-op when colocated: the shared workers are already profiled through the training policy. + """ + if self._owns_policy: + self._policy.start_gpu_profiling() + + def stop_gpu_profiling(self) -> None: + """Stop GPU profiling on the dedicated inference workers.""" + if self._owns_policy: + self._policy.stop_gpu_profiling() + + def shutdown(self) -> bool: + """Shut down all inference workers and clean up resources.""" + if not self._owns_policy: + return True + return self._policy.shutdown() + + def __del__(self) -> None: + """Safety net to ensure workers are shut down.""" + if hasattr(self, "_policy") and self._owns_policy: + self._policy.shutdown() diff --git a/nemo_rl/models/generation/megatron/megatron_worker.py b/nemo_rl/models/generation/megatron/megatron_worker.py new file mode 100644 index 00000000000..0a0721dd993 --- /dev/null +++ b/nemo_rl/models/generation/megatron/megatron_worker.py @@ -0,0 +1,803 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import gc +import os +import threading +import time +import warnings +from typing import AsyncGenerator, Optional + +import requests +import torch +from megatron.core.inference.config import ( + InferenceConfig, + KVCacheManagementMode, + PrefixCachingCoordinatorPolicy, +) +from megatron.core.inference.engines.dynamic_engine import EngineState +from megatron.core.inference.sampling_params import SamplingParams +from megatron.core.transformer.enums import InferenceCudaGraphScope +from megatron.core.transformer.utils import toggle_cuda_graphs +from megatron.core.utils import unwrap_model + +from nemo_rl.distributed.batched_data_dict import BatchedDataDict +from nemo_rl.models.generation.interfaces import ( + GenerationDatumSpec, + GenerationOutputSpec, + verify_right_padding, +) +from nemo_rl.models.generation.megatron.utils import ( + log_gpu_memory, + resolve_torch_dtype, +) +from nemo_rl.utils.nsys import wrap_with_nvtx_name + + +class MegatronGenerationMixin: + """Engine lifecycle, coordinator, HTTP server, and finish-generation machinery. + + The host class must provide: + + - model: the megatron module. + - cfg: policy config (TypedDict). + - rank: global rank (used for logging). + - tokenizer: HF tokenizer. + - megatron_tokenizer: tokenizer for inference. + - is_generation_colocated: Whether colocated or distributed. + """ + + def _init_inference_engine_state(self) -> None: + """Reset all inference-engine attributes to their uninitialized state.""" + self.dynamic_inference_engine = None + self.inference_client = None + self.inference_context = None + self.inference_wrapped_model = None + self.base_url = None + self._inference_engine_initialized = False + self._inference_engine_asleep = ( + True # Start paused since we begin with training + ) + self._inference_loop = None + self._inference_thread = None + + def _initialize_inference_engine(self, mcore_generation_config: dict) -> None: + """Initialize the persistent inference engine and client.""" + # TODO: Switch to standardized Megatron API. + if self._inference_engine_initialized: + return + + from megatron.core.inference.config import MambaInferenceStateConfig + from megatron.core.inference.contexts.dynamic_context import ( + DynamicInferenceContext, + ) + from megatron.core.inference.engines.dynamic_engine import ( + DynamicInferenceEngine, + ) + from megatron.core.inference.model_inference_wrappers.gpt.gpt_inference_wrapper import ( + GPTInferenceWrapper, + ) + from megatron.core.inference.text_generation_controllers.text_generation_controller import ( + TextGenerationController, + ) + from megatron.core.utils import get_attr_wrapped_model + + pg_collection = get_attr_wrapped_model(self.model, "pg_collection") + + buffer_size_gb = mcore_generation_config["buffer_size_gb"] + num_cuda_graphs = mcore_generation_config["num_cuda_graphs"] + block_size_tokens = mcore_generation_config["block_size_tokens"] + enable_chunked_prefill = mcore_generation_config["enable_chunked_prefill"] + use_cuda_graphs_for_non_decode_steps = mcore_generation_config[ + "use_cuda_graphs_for_non_decode_steps" + ] + max_tokens = mcore_generation_config["max_tokens"] + + # The value may be overwritten by `recompute_kv_cache_after_weight_updates`. + kv_cache_management_mode = mcore_generation_config["kv_cache_management_mode"] + needs_static_kv_pointers = kv_cache_management_mode != "persist" + + materialize_only_last_token_logits = mcore_generation_config[ + "materialize_only_last_token_logits" + ] + num_speculative_tokens = mcore_generation_config["num_speculative_tokens"] + max_requests = mcore_generation_config.get("max_requests") + + mamba_inference_state_config = MambaInferenceStateConfig.from_model(self.model) + is_hybrid_model = mamba_inference_state_config is not None + if is_hybrid_model: + if ( + mcore_generation_config.get("mamba_inference_ssm_states_dtype") + is not None + ): + mamba_inference_state_config.ssm_states_dtype = resolve_torch_dtype( + mcore_generation_config["mamba_inference_ssm_states_dtype"] + ) + if ( + mcore_generation_config.get("mamba_inference_conv_states_dtype") + is not None + ): + mamba_inference_state_config.conv_states_dtype = resolve_torch_dtype( + mcore_generation_config["mamba_inference_conv_states_dtype"] + ) + + # logging_step_interval is a power-user argument that should be NotRequired. + logging_step_interval = mcore_generation_config.get("logging_step_interval") + # This will be fixed in upstream MCore, allowing an argument of `None`. + if logging_step_interval is None: + logging_step_interval = 0 + + # flashinfer's fused-RoPE kernel only dispatches fp16/bf16 q/k. + use_flashinfer_fused_rope = self.model.config.params_dtype in ( + torch.float16, + torch.bfloat16, + ) + + inference_config = InferenceConfig( + block_size_tokens=block_size_tokens, + buffer_size_gb=buffer_size_gb, + num_cuda_graphs=num_cuda_graphs, + max_tokens=max_tokens, + max_sequence_length=mcore_generation_config["max_model_len"], + kv_cache_management_mode=KVCacheManagementMode(kv_cache_management_mode), + static_kv_memory_pointers=needs_static_kv_pointers, + use_cuda_graphs_for_non_decode_steps=use_cuda_graphs_for_non_decode_steps, + use_flashinfer_fused_rope=use_flashinfer_fused_rope, + sampling_backend="flashinfer", + use_synchronous_zmq_collectives=True, + materialize_only_last_token_logits=materialize_only_last_token_logits, + enable_chunked_prefill=enable_chunked_prefill, + enable_prefix_caching=mcore_generation_config["enable_prefix_caching"], + prefix_caching_coordinator_policy=PrefixCachingCoordinatorPolicy( + "first_prefix_block" + ), + pg_collection=pg_collection, + mamba_inference_state_config=mamba_inference_state_config, + # Reserve more KV-cache space when speculative decoding is enabled. + mamba_memory_ratio=( + 0.1 + 0.1 * num_speculative_tokens if is_hybrid_model else None + ), + logging_step_interval=logging_step_interval, + num_speculative_tokens=num_speculative_tokens, + max_requests=max_requests, + ) + + if "inference_cuda_graph_scope" in mcore_generation_config: + self.model.config.inference_cuda_graph_scope = InferenceCudaGraphScope[ + mcore_generation_config["inference_cuda_graph_scope"] + ] + + self.inference_context = DynamicInferenceContext( + self.model.config, inference_config + ) + self.inference_wrapped_model = GPTInferenceWrapper( + self.model, self.inference_context + ) + text_generation_controller = TextGenerationController( + inference_wrapped_model=self.inference_wrapped_model, + tokenizer=self.megatron_tokenizer, + ) + self.dynamic_inference_engine = DynamicInferenceEngine( + text_generation_controller, self.inference_context + ) + + self._inference_engine_initialized = True + self._inference_engine_asleep = True + print(f"[Rank {self.rank}] Initialized persistent inference engine") + + async def _start_inference_coordinator(self): + """Start the inference coordinator and engine loop.""" + self.coordinator_addr = await self.dynamic_inference_engine.start_listening_to_data_parallel_coordinator( + inference_coordinator_port=None, + launch_inference_coordinator=True, + ) + if torch.distributed.get_rank() == 0: + from megatron.core.inference.inference_client import InferenceClient + + self.inference_client = InferenceClient( + inference_coordinator_address=self.coordinator_addr, deserialize=True + ) + result = self.inference_client.start() + if result is not None: + await result + + self._inference_engine_asleep = False + + def _sleep(self) -> None: + """Pause + suspend the engine. No-op if already asleep.""" + if self._inference_engine_asleep: + return + future = asyncio.run_coroutine_threadsafe( + self._sleep_engine(), self._inference_loop + ) + future.result() + torch.distributed.barrier() + self._inference_engine_asleep = True + print(f"[Rank {self.rank}] paused inference engine") + + async def _sleep_engine(self): + if torch.distributed.get_rank() == 0: + self.inference_client.pause_engines() + await self.dynamic_inference_engine.wait_until(EngineState.PAUSED) + + if torch.distributed.get_rank() == 0: + self.inference_client.suspend_engines() + await self.dynamic_inference_engine.wait_until(EngineState.SUSPENDED) + + def _wake(self) -> None: + """Resume + unpause the engine. No-op if already awake.""" + if not self._inference_engine_asleep: + return + future = asyncio.run_coroutine_threadsafe( + self._wake_engine(), self._inference_loop + ) + future.result() + torch.distributed.barrier() + self._inference_engine_asleep = False + print(f"[Rank {self.rank}] resumed inference engine") + + async def _wake_engine(self): + if torch.distributed.get_rank() == 0: + self.inference_client.resume_engines() + await self.dynamic_inference_engine.wait_until(EngineState.RESUMED) + + if torch.distributed.get_rank() == 0: + self.inference_client.unpause_engines() + await self.dynamic_inference_engine.wait_until(EngineState.RUNNING) + + def _start_inference_loop_thread(self): + """Start a background thread with a persistent event loop for inference.""" + # CUDA current_device is per-thread. + # The worker's __init__ thread called set_device(LOCAL_RANK), and this thread must match. + local_rank = int(os.environ["LOCAL_RANK"]) + + def run_loop(): + torch.cuda.set_device(local_rank) + asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy()) + self._inference_loop = asyncio.new_event_loop() + asyncio.set_event_loop(self._inference_loop) + self._inference_loop.run_forever() + + self._inference_thread = threading.Thread(target=run_loop, daemon=True) + self._inference_thread.start() + while self._inference_loop is None: + time.sleep(0.001) + + def _setup_openai_api_server(self) -> str: + """Start the OpenAI-compatible HTTP server on this worker.""" + from megatron.core.inference.text_generation_server.dynamic_text_gen_server.text_generation_server import ( + start_text_gen_server, + ) + + from nemo_rl.distributed.virtual_cluster import ( + _get_free_port_local, + _get_node_ip_local, + ) + + ip = _get_node_ip_local() + free_port = _get_free_port_local() + + start_text_gen_server( + coordinator_addr=self.coordinator_addr, + tokenizer=self.megatron_tokenizer, + rank=torch.distributed.get_rank(), + server_port=free_port, + parsers=self.cfg["generation"]["mcore_generation_config"]["parsers"], + verbose=False, + ) + + base_url = f"http://{ip}:{free_port}/v1" + max_wait_time = 300 + start_time = time.time() + with requests.Session() as session: + while True: + if time.time() - start_time > max_wait_time: + raise TimeoutError( + f"[Megatron HTTP] Rank {self.rank} OpenAI server failed " + f"to start within {max_wait_time}s" + ) + try: + response = session.get(f"{base_url}/health", timeout=10) + if response.status_code == 200: + break + except requests.RequestException: + pass + time.sleep(2) + return base_url + + def _run_async_coordinator_start(self): + """Start the coordinator and engine loop in the background thread.""" + if self._inference_loop is None: + self._start_inference_loop_thread() + + future = asyncio.run_coroutine_threadsafe( + self._start_inference_coordinator(), self._inference_loop + ) + # _start_inference_coordinator awaits RUNNING, so future.result() only returns once + # this rank's engine is fully warmed up. Cross-rank sync is handled by Ray's actor + # group semantics (the caller waits for all workers' prepare_for_generation). + future.result() + print(f"[Rank {torch.distributed.get_rank()}] Coordinator started") + + if ( + self.cfg["generation"]["mcore_generation_config"]["expose_http_server"] + and torch.distributed.get_rank() == 0 + ): + print(f"[Rank {torch.distributed.get_rank()}] Starting HTTP Server") + self.base_url = self._setup_openai_api_server() + else: + print(f"[Rank {torch.distributed.get_rank()}] HTTP Server not started") + self.base_url = None + + def finish_generation(self) -> None: + """Wind down a generation cycle.""" + print(f"[Rank {self.rank}] finishing generation", flush=True) + log_gpu_memory("finish_generation START") + + lang_module = unwrap_model(self.model) + + if self.is_generation_colocated: + if self._inference_engine_initialized and not self._inference_engine_asleep: + self._sleep() + cuda_graph_impl = self.cfg["generation"]["mcore_generation_config"][ + "cuda_graph_impl" + ] + if cuda_graph_impl != "none": + toggle_cuda_graphs(lang_module, set_to="none") + + rotary_module = getattr(lang_module, "rotary_pos_emb", None) + if rotary_module is not None and hasattr( + rotary_module.forward, "cache_parameters" + ): + rotary_module.forward.cache_clear() + + if self.is_generation_colocated: + gc.collect() + torch.cuda.empty_cache() + + log_gpu_memory("finish_generation END") + + def prepare_for_generation(self, tags=None, **kwargs) -> None: + """Enter inference mode and start (or wake) the inference engine. + + Called in both colocated and non-colocated setups. + Even in non-colocated mode, Megatron's engine has to be intentionally paused before a refit + (and its weights are not detachable), so we have to switch modes around every refit. + """ + log_gpu_memory("prepare_for_generation START") + mcore_generation_config = self.cfg["generation"]["mcore_generation_config"] + + self.model.config.flash_decode = False + if self.is_generation_colocated and self.should_disable_forward_pre_hook: + # Bring offloaded params back to CUDA before colocated generation. + self.model = self.move_model( + self.model, "cuda", move_params=True, move_grads=False + ) + + lang_module = unwrap_model(self.model) + lang_module.eval() + + rotary_module = getattr(lang_module, "rotary_pos_emb", None) + if rotary_module is not None and hasattr( + rotary_module.forward, "cache_parameters" + ): + rotary_module.forward.cache_clear() + + cuda_graph_impl = mcore_generation_config["cuda_graph_impl"] + if cuda_graph_impl != "none": + toggle_cuda_graphs(lang_module, set_to=cuda_graph_impl) + + # tags=["weights"] means we are inside refit_policy_generation between + # suspend_for_refit and the weight transfer β€” the engine was intentionally + # paused and waking it now would race NVSHMEM init / weight transfer against + # CUDA-graph replay, corrupting TE FP8 state. The subsequent + # prepare_for_generation(tags=["kv_cache"]) is what actually wakes it. + if tags is None or "weights" not in tags: + if not self._inference_engine_initialized: + self._initialize_inference_engine(mcore_generation_config) + self._run_async_coordinator_start() + else: + self._wake() + + log_gpu_memory("prepare_for_generation END") + + def report_dp_openai_server_base_url(self) -> Optional[str]: + """Return this worker's OpenAI server base URL (None if not the leader).""" + return self.base_url + + def _build_sampling_params( + self, greedy: bool, stop_words: Optional[list[str]] + ) -> SamplingParams: + """Build mcore SamplingParams for a single request.""" + top_k_cfg = self.cfg["generation"]["top_k"] + top_k_val = 1 if greedy else (int(top_k_cfg) if top_k_cfg is not None else 0) + + top_p_cfg = self.cfg["generation"]["top_p"] + top_p_val = ( + 0.0 if greedy else (float(top_p_cfg) if top_p_cfg is not None else 0.0) + ) + + return SamplingParams( + temperature=self.cfg["generation"]["temperature"] if not greedy else 0, + top_k=top_k_val, + top_p=top_p_val, + skip_prompt_log_probs=True, + return_log_probs=True, + num_tokens_to_generate=self.cfg["generation"]["max_new_tokens"], + termination_id=self.megatron_tokenizer.eod, + stop_words=stop_words, + ) + + def _merge_stop_strings( + self, batch_stop_strings: Optional[list[Optional[list[str]]]] + ) -> Optional[list[str]]: + """Union the config's stop_strings with the given per-sample stop strings.""" + stop_set: set[str] = set() + if self.cfg["generation"]["stop_strings"]: + stop_set.update(self.cfg["generation"]["stop_strings"]) + if batch_stop_strings is not None: + for sample_ss in batch_stop_strings: + if sample_ss: + stop_set.update(sample_ss) + return list(stop_set) if stop_set else None + + def _prepare_data_for_generation( + self, data: BatchedDataDict[GenerationDatumSpec], greedy: bool = False + ) -> tuple[torch.Tensor, torch.Tensor, list[SamplingParams]]: + """Build the prompt tensors and a per-request SamplingParams for each sample.""" + if data is not None: + assert isinstance(data, BatchedDataDict), ( + f"data must be a BatchedDataDict, got type: {type(data)}" + ) + is_right_padded, error_msg = verify_right_padding( + data, pad_value=self.tokenizer.pad_token_id + ) + if not is_right_padded: + warnings.warn( + f"Input to Megatron Generation worker is not properly right-padded: {error_msg}" + ) + + prompt_tokens_tensor = data["input_ids"].cuda() + prompt_lengths_tensor = data["input_lengths"] + + batch_stop_strings = data.get("stop_strings", []) + sampling_params = [] + for i in range(prompt_tokens_tensor.size(0)): + sample_stop_strings = ( + batch_stop_strings[i] if i < len(batch_stop_strings) else None + ) + stop_words = self._merge_stop_strings( + [sample_stop_strings] if sample_stop_strings else None + ) + sampling_params.append(self._build_sampling_params(greedy, stop_words)) + + return prompt_tokens_tensor, prompt_lengths_tensor, sampling_params + + def _parse_result_to_batched_data_dict( + self, + data: BatchedDataDict[GenerationDatumSpec], + result: list, + ) -> BatchedDataDict[GenerationOutputSpec]: + """Pack DynamicInferenceRequest results into a GenerationOutputSpec batch.""" + input_lengths = data["input_lengths"] + input_ids = data["input_ids"] + batch_size = input_ids.size(0) + max_gen_seq_len = max(len(x.generated_tokens) for x in result) + padded_input_length = input_ids.size(1) + + max_seq_len = padded_input_length + max_gen_seq_len + output_ids_padded = torch.full( + (batch_size, max_seq_len), + self.tokenizer.pad_token_id, + dtype=torch.long, + device=input_ids.device, + ) + + logprobs_padded = torch.zeros( + (batch_size, max_seq_len), + dtype=torch.float, + device=input_ids.device, + ) + + generation_lengths = torch.zeros( + batch_size, dtype=torch.long, device=input_ids.device + ) + unpadded_sequence_lengths = torch.zeros( + batch_size, dtype=torch.long, device=input_ids.device + ) + for i in range(batch_size): + tokens = result[i].prompt_tokens.tolist() + result[i].generated_tokens + seq_len = len(tokens) + output_ids_padded[i, :seq_len] = torch.tensor( + tokens, dtype=torch.long, device=input_ids.device + ) + prompt_len = input_lengths[i].item() + generation_lengths[i] = seq_len - prompt_len + unpadded_sequence_lengths[i] = seq_len + gen_logprobs = result[i].generated_log_probs + logprobs_padded[i, prompt_len : prompt_len + len(gen_logprobs)] = ( + torch.tensor( + gen_logprobs, + dtype=torch.float, + device=input_ids.device, + ) + ) + + out_dict = { + "output_ids": output_ids_padded, + "logprobs": logprobs_padded, + "generation_lengths": generation_lengths, + "unpadded_sequence_lengths": unpadded_sequence_lengths, + } + + return BatchedDataDict.from_batches([out_dict]).to("cpu") + + @wrap_with_nvtx_name("megatron_policy_worker/generate") + def generate( + self, *, data: BatchedDataDict[GenerationDatumSpec], greedy: bool = False + ) -> BatchedDataDict[GenerationOutputSpec]: + """Synchronous batched generation via the mcore data-parallel coordinator. + + Args: + data: BatchedDataDict containing input_ids and input_lengths tensors + greedy: Whether to use greedy decoding instead of sampling + + Returns: + BatchedDataDict conforming to GenerationOutputSpec: + - output_ids: input + generated token IDs with proper padding + - logprobs: Log probabilities for tokens + - generation_lengths: Lengths of each response + - unpadded_sequence_lengths: Lengths of each input + generated sequence + """ + prompt_tokens_tensor, prompt_lengths_tensor, sampling_params = ( + self._prepare_data_for_generation(data, greedy) + ) + if self._inference_loop is None: + raise RuntimeError( + "Inference loop not initialized. Call prepare_for_generation() first." + ) + future = asyncio.run_coroutine_threadsafe( + self._generate_with_persistent_engine( + prompt_tokens_tensor, + prompt_lengths_tensor, + sampling_params, + ), + self._inference_loop, + ) + result = future.result() + + return self._parse_result_to_batched_data_dict(data, result) + + async def generate_async( + self, data: BatchedDataDict[GenerationDatumSpec], greedy: bool = False + ) -> AsyncGenerator[tuple[int, BatchedDataDict[GenerationOutputSpec]], None]: + """Streaming generation: yield `(index, batch)` tuples as they complete. + + Args: + data: BatchedDataDict with input_ids and input_lengths + greedy: Whether to use greedy decoding instead of sampling + + Yields: + Tuple of (original_index, BatchedDataDict conforming to GenerationOutputSpec for the single sequence) + """ + if self._inference_loop is None: + raise RuntimeError( + "Inference loop not initialized. Call prepare_for_generation() first." + ) + + async def _generate_single_item( + index: int, + ) -> tuple[int, BatchedDataDict[GenerationOutputSpec]]: + datum = data.get_batch(index, 1) + prompt_tokens_tensor, prompt_lengths_tensor, sampling_params = ( + self._prepare_data_for_generation(datum, greedy) + ) + future = asyncio.run_coroutine_threadsafe( + self._generate_with_persistent_engine( + prompt_tokens_tensor, + prompt_lengths_tensor, + sampling_params, + ), + self._inference_loop, + ) + result = await asyncio.wrap_future(future) + output = self._parse_result_to_batched_data_dict(datum, result) + return (index, output) + + tasks = [ + asyncio.create_task(_generate_single_item(i)) for i in range(data.size) + ] + for result in asyncio.as_completed(tasks): + yield await result + + async def _generate_with_persistent_engine( + self, + prompt_tokens_tensor: torch.Tensor, + prompt_lengths_tensor: torch.Tensor, + sampling_params: list[SamplingParams], + ) -> list: + """Submit requests through the persistent inference client (rank 0 only).""" + from megatron.core.inference.inference_request import DynamicInferenceRequest + + dist_rank = torch.distributed.get_rank() + assert dist_rank == 0, ( + "Only rank 0 creates a client to communicate with the coordinator" + ) + + print( + f"[Rank {dist_rank}] Submitting {prompt_tokens_tensor.size(0)} requests to coordinator" + ) + + futures = [] + for prompt_tokens, prompt_len, request_sampling_params in zip( + prompt_tokens_tensor, prompt_lengths_tensor, sampling_params, strict=True + ): + prompt = prompt_tokens[: prompt_len.item()].tolist() + futures.append( + self.inference_client.add_request(prompt, request_sampling_params) + ) + + results: list[DynamicInferenceRequest] = await asyncio.gather(*futures) + print(f"[Rank {dist_rank}] Completed {len(results)} requests") + return results + + +class MegatronGenerationRefitMixin: + """Refit collective, weight transfer, and engine suspend/resume around refits.""" + + def init_collective_mcore_generation( + self, + ip: str, + port: int, + world_size: int, + rank_offset: int, + refit_backend: str = "gloo", + ) -> None: + """Initialize the refit collective for non-colocated weight transfer. + + Args: + ip: IP address for the process group rendezvous. + port: Port for the process group rendezvous. + world_size: Total world size (train + inference workers). + rank_offset: Offset for this side's ranks (`train_world_size` for inference). + refit_backend: Copy-service backend ("gloo" or "nvshmem"). + """ + from torch.distributed.distributed_c10d import ( + PrefixStore, + ProcessGroup, + ProcessGroupGloo, + _world, + ) + + local_rank = torch.distributed.get_rank() + global_rank = local_rank + rank_offset + + # port+1 to avoid collision with the caller's rendezvous on `port`. + store = torch.distributed.TCPStore( + host_name=ip, + port=port + 1, + world_size=world_size, + is_master=(global_rank == 0), + ) + + group_name = "refit" + pg_prefix_store = PrefixStore(f"{group_name}/", store) + + # Training and inference workers run in separate torch.distributed worlds. + # The public APIs (new_group, init_process_group) assume all ranks belong to one world; + # new_group validates ranks against the default PG, and init_process_group can only + # be called once. We construct the PG manually using the same internal pattern as + # _new_process_group_helper, skipping the single-world assumptions. + pg = ProcessGroup(pg_prefix_store, global_rank, world_size) + gloo_store = PrefixStore("cpu/", pg_prefix_store) + gloo_backend = ProcessGroupGloo(gloo_store, global_rank, world_size) + gloo_backend._set_sequence_number_for_group() + pg._register_backend( + torch.device("cpu"), + ProcessGroup.BackendType.GLOO, + gloo_backend, + ) + pg._set_default_backend(ProcessGroup.BackendType.GLOO) + pg._set_group_name(group_name) + + self.refit_pg = pg + + # Register in torch.distributed's global state so that high-level ops + # (all_gather_object, broadcast_object_list) work with this PG. + _world.pg_group_ranks[pg] = {i: i for i in range(world_size)} + _world.pg_map[pg] = ("gloo", pg_prefix_store) + _world.pg_names[pg] = group_name + + if refit_backend == "nvshmem": + from megatron.core.resharding.copy_services.nvshmem_copy_service import ( + NVSHMEMCopyService, + ) + + self.refit_copy_service = NVSHMEMCopyService(group=self.refit_pg) + else: + from megatron.core.resharding.copy_services.gloo_copy_service import ( + GlooCopyService, + ) + + self.refit_copy_service = GlooCopyService(group=self.refit_pg) + + from megatron.core.resharding.refit import prepare_swap_model_weights + + is_source = rank_offset == 0 + # Cache for later refit calls (swap_weights_via_reshard). + self.refit_dst_rank_offset = ( + torch.distributed.get_world_size() if is_source else rank_offset + ) + + # Build and cache the reshard plan (and any MXFP8 transforms) collectively. + # All participating ranks (training + generation) call this simultaneously. + prepare_swap_model_weights( + src_model=self.model if is_source else None, + target_model=None if is_source else self.model, + group=self.refit_pg, + src_rank_offset=0, + dst_rank_offset=self.refit_dst_rank_offset, + ) + + def preinit_nvshmem_collective(self) -> None: + """Initialize NVShmem collectively before any weight transfer. + + Must be called on ALL participating ranks (training + inference) simultaneously, + after `prepare_for_generation()` has completed and the CG has been recorded. + The `NVSHMEMCopyService` lazy init can corrupt CUDA graph state. + """ + if not hasattr(self, "refit_copy_service"): + return + if not hasattr(self.refit_copy_service, "_ensure_initialized"): + return + self.refit_copy_service._ensure_initialized() + + def swap_weights_via_reshard(self, is_source: bool) -> bool: + """Transfer weights using Megatron's `swap_model_weights` API. + + Args: + is_source: True for training workers (senders), False for inference workers (receivers). + + Returns: + True on success. + """ + from megatron.core.resharding.refit import swap_model_weights + + src_model = self.model if is_source else None + dst_model = None if is_source else self.model + + swap_model_weights( + src_model, + dst_model, + refit_method=self.refit_copy_service, + group=self.refit_pg, + src_rank_offset=0, + dst_rank_offset=self.refit_dst_rank_offset, + ) + + return True + + def suspend_for_refit(self) -> None: + """Pause+suspend the inference engine before a weight refit.""" + if not self._inference_engine_initialized: + return + self._sleep() + torch.cuda.synchronize() + + def resume_after_refit(self) -> None: + """Resume+unpause the inference engine after a weight refit.""" + if not self._inference_engine_initialized: + return + self._wake() diff --git a/nemo_rl/models/generation/megatron/utils.py b/nemo_rl/models/generation/megatron/utils.py new file mode 100644 index 00000000000..339d57e46cc --- /dev/null +++ b/nemo_rl/models/generation/megatron/utils.py @@ -0,0 +1,37 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +from megatron.core.inference.utils import device_memory_summary + + +def resolve_torch_dtype(val): + """Convert a value to `torch.dtype`.""" + if isinstance(val, torch.dtype): + return val + if isinstance(val, str): + name = val.replace("torch.", "") + dtype = getattr(torch, name, None) + if isinstance(dtype, torch.dtype): + return dtype + raise ValueError( + f"Cannot resolve torch dtype from {val!r} (type {type(val).__name__}). " + f"Expected a torch.dtype or a string like 'torch.float32' / 'float32'." + ) + + +def log_gpu_memory(tag: str) -> None: + """Print a one-line GPU-memory summary for the calling rank.""" + rank = torch.distributed.get_rank() if torch.distributed.is_initialized() else 0 + print(f"[GPU Rank {rank}] {tag} | {device_memory_summary()}") diff --git a/nemo_rl/models/megatron/community_import.py b/nemo_rl/models/megatron/community_import.py index 2cf218561d7..8165f73a07a 100644 --- a/nemo_rl/models/megatron/community_import.py +++ b/nemo_rl/models/megatron/community_import.py @@ -149,6 +149,13 @@ def import_model_from_hf_name( megatron.core.rerun_state_machine.destroy_rerun_state_machine() + # The seeding above created the global RNG tracker with default flags. + # Such a stale tracker will ignore flags that the real model build requests later. + from megatron.core.tensor_parallel import random as mcore_random + + mcore_random._CUDA_RNG_STATE_TRACKER = None + mcore_random._CUDA_RNG_STATE_TRACKER_INITIALIZED = False + def export_model_from_megatron( hf_model_name: str, diff --git a/nemo_rl/models/megatron/config.py b/nemo_rl/models/megatron/config.py index 92b3143bcca..ccda91f3325 100644 --- a/nemo_rl/models/megatron/config.py +++ b/nemo_rl/models/megatron/config.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Callable, NamedTuple, Optional, TypedDict +from typing import Any, Callable, NamedTuple, Optional import torch from megatron.bridge.training.config import ConfigContainer @@ -24,25 +24,6 @@ from nemo_rl.algorithms.logits_sampling_utils import TrainingSamplingParams -class MegatronGenerationConfig(TypedDict): - # Total buffer size is 2x this value (active requests + paused requests) - buffer_size_gb: int - # Number of CUDA graphs to pre-compile for different batch sizes - num_cuda_graphs: int - # Size of each KV cache block in tokens (affects memory granularity) - block_size_tokens: int - # Enable CUDA graphs for prefill/context processing - use_cuda_graphs_for_non_decode_steps: bool - # Split long prefills into chunks for better memory management - enable_chunked_prefill: bool - # Unified memory usage level (0=disabled, higher values enable more aggressive paging) - unified_memory_level: int - # Maximum number of tokens to use in a single step. Analogous to vllm's max_num_batched_tokens. - # Can cause OOM if set too high so should be tuned with buffer_size_gb if OOMing. If set too - # low, then will only do 512 tokens at a time, which can be slow. - max_tokens: int - - ## returned from validate_and_set_config class RuntimeConfig(NamedTuple): """Runtime configuration for model training and inference. diff --git a/nemo_rl/models/megatron/setup.py b/nemo_rl/models/megatron/setup.py index 46b30d9860e..96fa1b97449 100644 --- a/nemo_rl/models/megatron/setup.py +++ b/nemo_rl/models/megatron/setup.py @@ -15,6 +15,7 @@ import hashlib import json import os +import threading import time import warnings from typing import Any, Callable, Optional, TypeVar @@ -57,7 +58,7 @@ from megatron.core import parallel_state from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.transformer import MegatronModule -from megatron.core.transformer.enums import AttnBackend +from megatron.core.transformer.enums import AttnBackend, InferenceCudaGraphScope from megatron.core.transformer.module import Float16Module from megatron.core.transformer.transformer_config import TransformerConfig from transformers import PreTrainedTokenizerBase @@ -156,8 +157,11 @@ def setup_distributed() -> None: configure_dynamo_cache() # Ensure clean slate before import destroy_parallel_state() - # Need to initialize the process group before calling into Megatron-Bridge, otherwise Megatron-Bridge will try to set an incorrect device - torch.distributed.init_process_group("nccl") + # Pin the communicator to the correct GPU explicitly. + local_rank = int(os.environ["LOCAL_RANK"]) + torch.distributed.init_process_group( + "nccl", device_id=torch.device(f"cuda:{local_rank}") + ) def validate_and_set_config( @@ -512,6 +516,8 @@ def setup_model_config( if fmt == "megatron_lm": model_cfg.finalize() + model_cfg.__post_init__() + # Derive fp8_param_enabled once from the config dict so that load_main_params_from_ckpt # and _create_megatron_config both use the same canonical check (fp8 enabled AND fp8_param). fp8_cfg = config["megatron_cfg"].get("fp8_cfg", None) @@ -612,6 +618,26 @@ def _apply_moe_config(model_cfg: Any, config: PolicyConfig) -> None: model_cfg.moe_token_dispatcher_type = config["megatron_cfg"][ "moe_token_dispatcher_type" ] + if "inference_moe_token_dispatcher_type" in config["megatron_cfg"]: + model_cfg.inference_moe_token_dispatcher_type = config["megatron_cfg"][ + "inference_moe_token_dispatcher_type" + ] + if "inference_grouped_gemm_backend" in config["megatron_cfg"]: + model_cfg.inference_grouped_gemm_backend = config["megatron_cfg"][ + "inference_grouped_gemm_backend" + ] + if "moe_router_num_groups" in config["megatron_cfg"]: + model_cfg.moe_router_num_groups = config["megatron_cfg"][ + "moe_router_num_groups" + ] + if "moe_router_group_topk" in config["megatron_cfg"]: + model_cfg.moe_router_group_topk = config["megatron_cfg"][ + "moe_router_group_topk" + ] + if "moe_pad_experts_for_cuda_graph_inference" in config["megatron_cfg"]: + model_cfg.moe_pad_experts_for_cuda_graph_inference = config["megatron_cfg"][ + "moe_pad_experts_for_cuda_graph_inference" + ] model_cfg.moe_shared_expert_overlap = config["megatron_cfg"][ "moe_shared_expert_overlap" ] @@ -751,6 +777,28 @@ def _apply_performance_config(model_cfg: Any, config: PolicyConfig) -> None: f"Available backends are: {list(AttnBackend.__members__.keys())}" ) + # These overrides need to be applied before the workers spawn. + if "transformer_impl" in config["megatron_cfg"]: + model_cfg.transformer_impl = config["megatron_cfg"]["transformer_impl"] + if "cuda_graph_impl" in config["megatron_cfg"]: + model_cfg.cuda_graph_impl = config["megatron_cfg"]["cuda_graph_impl"] + if model_cfg.cuda_graph_impl != "none": + model_cfg.use_te_rng_tracker = True + if "inference_cuda_graph_scope" in config["megatron_cfg"]: + model_cfg.inference_cuda_graph_scope = InferenceCudaGraphScope[ + config["megatron_cfg"]["inference_cuda_graph_scope"] + ] + + # Use the graph-safe TE RNG tracker for either training graphs or inference graphs. + if "generation" in config and config["generation"] is not None: + generation_cfg = config["generation"] + if ( + generation_cfg["backend"] == "megatron" + and generation_cfg["colocated"]["enabled"] + and generation_cfg["mcore_generation_config"]["cuda_graph_impl"] != "none" + ): + model_cfg.use_te_rng_tracker = True + # FP8 configuration fp8_cfg = config["megatron_cfg"].get("fp8_cfg", None) if fp8_cfg is not None and fp8_cfg.get("enabled", False): @@ -995,6 +1043,36 @@ def draft_pre_wrap_hook(model: list[MegatronModule]) -> list[MegatronModule]: return draft_pre_wrap_hook +_BRIDGE_SIGNAL_HANDLER_PATCHED = False + + +def _patch_bridge_signal_handler_for_worker_threads() -> None: + """Make Megatron-Bridge's signal-handler install safe off the main thread. + + See https://github.com/NVIDIA-NeMo/Megatron-Bridge/pull/4375 + + TODO: Remove this hotfix once Megatron-Bridge is bumped. + """ + global _BRIDGE_SIGNAL_HANDLER_PATCHED + if _BRIDGE_SIGNAL_HANDLER_PATCHED: + return + + from megatron.bridge.training.utils import sig_utils + + original_enter = sig_utils.DistributedSignalHandler.__enter__ + + def main_thread_only_enter(self): + if threading.current_thread() is not threading.main_thread(): + self._signal_received = False + # Nothing was installed, so release()/__exit__ become no-ops. + self.released = True + return self + return original_enter(self) + + sig_utils.DistributedSignalHandler.__enter__ = main_thread_only_enter + _BRIDGE_SIGNAL_HANDLER_PATCHED = True + + def setup_model_and_optimizer( policy_cfg: PolicyConfig, megatron_cfg: ConfigContainer, @@ -1005,6 +1083,7 @@ def setup_model_and_optimizer( additional_pre_wrap_hooks: Optional[list[Callable]] = None, ): state = GlobalState() + _patch_bridge_signal_handler_for_worker_threads() state.cfg = megatron_cfg # TODO: Freeze state.cfg @@ -1162,6 +1241,7 @@ def composed_peft_hook(model: list[MegatronModule]) -> list[MegatronModule]: pre_wrap_hook=pre_wrap_hook, mixed_precision_wrapper=mixed_precision_wrapper, pg_collection=pg_collection, + wrap_with_ddp=load_optimizer, ) if load_optimizer: diff --git a/nemo_rl/models/policy/__init__.py b/nemo_rl/models/policy/__init__.py index d22e6a1c49e..53b7f9edfd8 100644 --- a/nemo_rl/models/policy/__init__.py +++ b/nemo_rl/models/policy/__init__.py @@ -266,6 +266,25 @@ class MegatronConfig(TypedDict): # Options are 'allgather','alltoall' and 'flex' # Use 'flex' when using DeepEP moe_token_dispatcher_type: str + # Inference-only MoE dispatcher selection. + # Options are 'nvls' (requires Hopper+ NVLink) and 'nccl' (fallback for non-NVLS systems). + inference_moe_token_dispatcher_type: NotRequired[str] + # Backend for grouped-GEMM during inference-optimized MoE forward. + # Options: 'flashinfer', 'torch', 'vllm' (mcore default). + inference_grouped_gemm_backend: NotRequired[str] + # InferenceTopKRouter requires moe_router_num_groups=None + # (used when transformer_impl='inference_optimized') + moe_router_num_groups: NotRequired[int | None] + moe_router_group_topk: NotRequired[int | None] + # Transformer implementation backing the model. Only valid on generation workers. + # Options are 'transformer_engine' and 'inference_optimized'. + transformer_impl: NotRequired[str] + # CUDA-graph implementation. + # Options: 'none', 'local', 'transformer_engine', 'full_iteration'. + cuda_graph_impl: NotRequired[str] + # When True, each expert sees a fixed number of tokens for cuda-graph capture. + # Required when cuda_graph_impl= 'local' with transformer_impl != 'inference_optimized'. + moe_pad_experts_for_cuda_graph_inference: NotRequired[bool] # Can be used only with 'alltoall' token dispatcher moe_shared_expert_overlap: bool # Enable grouped GEMM for MoE experts via CUTLASS. Significant throughput diff --git a/nemo_rl/models/policy/lm_policy.py b/nemo_rl/models/policy/lm_policy.py index 7b4b47e6387..21199bbbe8b 100644 --- a/nemo_rl/models/policy/lm_policy.py +++ b/nemo_rl/models/policy/lm_policy.py @@ -113,7 +113,7 @@ def __init__( pp_size = config["megatron_cfg"]["pipeline_model_parallel_size"] cp_size = config["megatron_cfg"]["context_parallel_size"] - env_vars = config["megatron_cfg"].get("env_vars", {}) + env_vars = dict(config["megatron_cfg"].get("env_vars") or {}) if "TORCH_CUDA_ARCH_LIST" not in os.environ: raise RuntimeError( @@ -373,6 +373,38 @@ def init_collective( # this function should co-work with vllm, so we should wait for all futures to complete outside return futures + def init_collective_mcore_generation( + self, + ip: str, + port: int, + world_size: int, + *, + rank_offset: int, + refit_backend: str = "gloo", + ) -> list[ray.ObjectRef]: + """Initialize the megatron refit collective on this policy's workers.""" + return self.worker_group.run_all_workers_single_data( + "init_collective_mcore_generation", + ip=ip, + port=port, + world_size=world_size, + rank_offset=rank_offset, + refit_backend=refit_backend, + ) + + def preinit_nvshmem(self) -> list[ray.ObjectRef]: + """Pre-initialize NVSHMEM on this policy's workers (no-op when not using nvshmem).""" + return self.worker_group.run_all_workers_single_data( + "preinit_nvshmem_collective" + ) + + def swap_weights_via_reshard(self, *, is_source: bool) -> list[ray.ObjectRef]: + """Send (`is_source=True`) or receive (`is_source=False`) weights via megatron reshard.""" + return self.worker_group.run_all_workers_single_data( + "swap_weights_via_reshard", + is_source=is_source, + ) + # ── DP-shard helpers ──────────────────────────────────────────────── # DRY for Policy's logprob/train methods only. The data-plane sibling # TQPolicy shards KVBatchMeta via ``shard_meta_for_dp``; the @@ -756,13 +788,13 @@ def generate( self, data: BatchedDataDict[GenerationDatumSpec], greedy: bool = False ) -> BatchedDataDict[GenerationOutputSpec]: """Generate a batch of data using the policy.""" - # Verify input data is right-padded assert isinstance(data, BatchedDataDict), ( f"data must be a BatchedDataDict, got type: {type(data)}" ) assert "input_ids" in data and "input_lengths" in data, ( "Missing required input fields" ) + assert self.cfg["generation"] is not None, "Generation config is not set" dp_size = self.sharding_annotations.get_axis_size("data_parallel") sharded_data = data.shard_by_batch_size(dp_size, batch_size=None) @@ -774,13 +806,11 @@ def generate( output_is_replicated=["tensor_parallel", "pipeline_parallel"], common_kwargs={"greedy": greedy}, ) - assert self.cfg["generation"] is not None, "Generation config is not set" - result: BatchedDataDict[GenerationOutputSpec] = BatchedDataDict.from_batches( + result = BatchedDataDict.from_batches( self.worker_group.get_all_worker_results(futures), pad_value_dict={"output_ids": self.cfg["generation"]["_pad_token_id"]}, ) - # Verify the output has all required fields required_keys = [ "output_ids", "generation_lengths", @@ -844,6 +874,10 @@ def prepare_for_generation(self, *args: Any, **kwargs: Any) -> bool: # We don't need to do anything here return True + def finish_generation(self, *args: Any, **kwargs: Any) -> bool: + # We don't need to do anything here + return True + def prepare_for_training(self, *args: Any, **kwargs: Any) -> None: # onload everything to the GPU futures = self.worker_group.run_all_workers_single_data("prepare_for_training") @@ -855,10 +889,6 @@ def prepare_for_lp_inference(self, *args: Any, **kwargs: Any) -> None: ) ray.get(futures) - def finish_generation(self, *args: Any, **kwargs: Any) -> bool: - # We don't need to do anything here - return True - def invalidate_kv_cache(self, *args: Any, **kwargs: Any) -> bool: # We don't need to do anything here return True diff --git a/nemo_rl/models/policy/workers/base_policy_worker.py b/nemo_rl/models/policy/workers/base_policy_worker.py index 34f772a175e..da792d700f4 100644 --- a/nemo_rl/models/policy/workers/base_policy_worker.py +++ b/nemo_rl/models/policy/workers/base_policy_worker.py @@ -118,7 +118,9 @@ def stop_gpu_profiling(self) -> None: def report_node_ip_and_gpu_id(self) -> tuple[str, int]: """Report the node IP and GPU ID of the current worker.""" ip = ray._private.services.get_node_ip_address() - gpu_id = ray.get_gpu_ids()[0] + # Workers that manage their own LOCAL_RANK will have an empty `ray.get_gpu_ids()`. + gpu_ids = ray.get_gpu_ids() + gpu_id = gpu_ids[0] if gpu_ids else torch.cuda.current_device() return (ip, gpu_id) # Temporary fix, 'data' is a kwarg due to some sort of ray bug diff --git a/nemo_rl/models/policy/workers/megatron_policy_worker.py b/nemo_rl/models/policy/workers/megatron_policy_worker.py index 643d0866626..336dd58bf84 100644 --- a/nemo_rl/models/policy/workers/megatron_policy_worker.py +++ b/nemo_rl/models/policy/workers/megatron_policy_worker.py @@ -37,10 +37,6 @@ from megatron.core.distributed.fsdp.mcore_fsdp_adapter import ( FullyShardedDataParallel as custom_FSDP, ) -from megatron.core.inference.config import InferenceConfig -from megatron.core.inference.text_generation_controllers.text_generation_controller import ( - TextGenerationController, -) from megatron.core.optimizer import ChainedOptimizer from megatron.core.rerun_state_machine import get_rerun_state_machine from megatron.core.utils import get_model_config @@ -51,14 +47,13 @@ from nemo_rl.data_plane.worker_mixin import TQWorkerMixin from nemo_rl.distributed.batched_data_dict import BatchedDataDict from nemo_rl.distributed.named_sharding import NamedSharding -from nemo_rl.models.generation.interfaces import ( - GenerationDatumSpec, - GenerationOutputSpec, - verify_right_padding, +from nemo_rl.models.generation.interfaces import GenerationDatumSpec +from nemo_rl.models.generation.megatron.megatron_worker import ( + MegatronGenerationMixin, + MegatronGenerationRefitMixin, ) from nemo_rl.models.generation.vllm.config import VllmConfig from nemo_rl.models.megatron.common import get_moe_metrics -from nemo_rl.models.megatron.config import MegatronGenerationConfig from nemo_rl.models.megatron.data import ( get_microbatch_iterator, process_global_batch, @@ -142,7 +137,11 @@ def _model_self_packs_for_cp(model: Any) -> bool: # Classes with @ray.remote can't be inherited from, so we split the implementation out. # This is useful when using worker extension classes. class MegatronPolicyWorkerImpl( - TQWorkerMixin, AbstractPolicyWorker, ColocatablePolicyInterface + MegatronGenerationMixin, + MegatronGenerationRefitMixin, + TQWorkerMixin, + AbstractPolicyWorker, + ColocatablePolicyInterface, ): def __repr__(self): """Customizes the actor's prefix in the Ray logs. @@ -214,6 +213,31 @@ def _get_replica_group(self) -> Optional[Any]: self._replica_group_cache = groups[my_dp_rank] return self._replica_group_cache + @staticmethod + def configure_worker( + num_gpus: int | float, + bundle_indices: Optional[tuple[int, list[int]]] = None, + ) -> tuple[dict[str, Any], dict[str, str], dict[str, Any]]: + """Worker-controlled Ray actor configuration. + + Ensures that communication via NVLS functions correctly. + + Args: + num_gpus: Original GPU allocation for this worker based on the placement group + bundle_indices: Tuple of (node_idx, local_bundle_indices) for this server + + Returns: + tuple with complete worker configuration: + - 'resources': Resource allocation (e.g., num_gpus) + - 'env_vars': Environment variables for this worker + - 'init_kwargs': Parameters to pass to __init__ of the worker + """ + del bundle_indices # one GPU per worker; no per-bundle seeding needed + resources: dict[str, Any] = {"num_gpus": num_gpus} + env_vars: dict[str, str] = {"RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES": "1"} + init_kwargs: dict[str, Any] = {} + return resources, env_vars, init_kwargs + def __init__( self, config: PolicyConfig, @@ -227,10 +251,19 @@ def __init__( **kwargs: Any, ): """Initialize the MegatronPolicyWorker.""" + # NVML-based and guarded on torch.cuda.is_initialized(), so this does + # not initialize a CUDA context ahead of the set_device below. log_gpu_memory_diagnostics( label="init_start", worker_type="MegatronPolicyWorker" ) + # Must be the first CUDA-touching call in this process. + # With `RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES=1` (set by `configure_worker()`), + gpu_ids = ray.get_gpu_ids() + local_rank = int(gpu_ids[0]) + os.environ["LOCAL_RANK"] = str(local_rank) + torch.cuda.set_device(local_rank) + # Apply patch from https://github.com/NVIDIA/TransformerEngine/pull/2286/files apply_transformer_engine_patch() @@ -246,6 +279,12 @@ def __init__( label="after_nccl_init", worker_type="MegatronPolicyWorker" ) + # Defensive assert to ensure to ensure `local_rank` setter worked correctly. + assert torch.cuda.current_device() == local_rank, ( + f"device drift after setup_distributed: current_device=" + f"{torch.cuda.current_device()}, LOCAL_RANK={local_rank}." + ) + # Step 2: Validate and setup model paths hf_model_name, pretrained_path, pt_checkpoint_exists = validate_model_paths( config @@ -273,6 +312,14 @@ def __init__( self.tokenizer.pad_token = self.tokenizer.eos_token # Step 3: Setup model configuration + # Training workers cannot use inference_optimized transformer spec. + if init_optimizer: + assert ( + config["megatron_cfg"].get("transformer_impl") != "inference_optimized" + ), ( + "transformer_impl=inference_optimized must not be set on training workers. " + "Use policy.generation.mcore_generation_config.transformer_impl=inference_optimized instead." + ) runtime_config = validate_and_set_config( config, self.rank, @@ -289,8 +336,8 @@ def __init__( runtime_config.offload_optimizer_for_logprob ) self.is_generation_colocated = runtime_config.is_generation_colocated - self.sampling_params = runtime_config.sampling_params self.final_padded_vocab_size = runtime_config.final_padded_vocab_size + self.sampling_params = runtime_config.sampling_params self.defer_fp32_logits = self.cfg["megatron_cfg"].get( "defer_fp32_logits", None @@ -299,6 +346,14 @@ def __init__( # Store FP8 config for later use self.fp8_cfg = config["megatron_cfg"].get("fp8_cfg", None) + # Full-iteration CUDA graphs cannot be interrupted, so disable the + # NaN-in-loss check that would otherwise require breaking out of the graph. + if self.megatron_cfg.model.cuda_graph_impl == "full_iteration": + warnings.warn( + "Disabling check_for_nan_in_loss: full-iteration CUDA graph cannot be interrupted." + ) + self.megatron_cfg.rerun_state_machine.check_for_nan_in_loss = False + # Validate configuration self.megatron_cfg.validate() @@ -378,6 +433,8 @@ def __init__( ## used for streaming update inference engine weights self._held_gather_buffer = None + self._init_inference_engine_state() + log_gpu_memory_diagnostics( label="init_complete", worker_type="MegatronPolicyWorker" ) @@ -912,7 +969,9 @@ def use_reference_model(self): item = item.detach().to(device="cpu", non_blocking=True, copy=True) model_state_dict[name] = item - # Swap reference model state_dict into self.model (reference weights + optional FP8 extra_state) + # Swap reference state into self.model. Use _apply_state_dict_to_model + # (rather than load_state_dict) so FP8 _extra_state with mismatched shape + # is routed through set_extra_state() correctly. self._apply_state_dict_to_model( self.reference_state_dict, raise_if_key_missing=True, @@ -1049,214 +1108,6 @@ def get_topk_logits( [{"topk_logits": topk_logits.cpu(), "topk_indices": topk_indices.cpu()}] ) - @wrap_with_nvtx_name("megatron_policy_worker/generate") - def generate( - self, *, data: BatchedDataDict[GenerationDatumSpec], greedy: bool = False - ) -> BatchedDataDict[GenerationOutputSpec]: - """Generate a batch of data using huggingface framework generation. - - Args: - data: BatchedDataDict containing input_ids and input_lengths tensors - Returns: - BatchedDataDict conforming to GenerationOutputSpec: - - ``output_ids``: input + generated token IDs - - ``logprobs``: Log probabilities for each token - - ``generation_lengths``: Lengths of each response - """ - # 512 bATCH SIZE (200 tokens) - no_grad = torch.no_grad() - no_grad.__enter__() - self.model.config.flash_decode = False - if self.should_disable_forward_pre_hook: - self.model = self.move_model( - self.model, "cuda", move_params=True, move_grads=False - ) - # Verify input is right padded - assert isinstance(data, BatchedDataDict), ( - f"data must be a BatchedDataDict, got type: {type(data)}" - ) - assert "input_ids" in data and "input_lengths" in data, ( - f"input_ids and input_lengths must be present in the BatchedDataDict, got keys: {data.keys()}" - ) - is_right_padded, error_msg = verify_right_padding( - data, pad_value=self.tokenizer.pad_token_id - ) - if not is_right_padded: - warnings.warn( - f"Input to Megatron Generation worker is not properly right-padded: {error_msg}" - ) - - mcore_generation_config = cast( - MegatronGenerationConfig, self.cfg["generation"]["mcore_generation_config"] - ) - - from megatron.core.inference.contexts.dynamic_context import ( - DynamicInferenceContext, - ) - from megatron.core.inference.engines.dynamic_engine import ( - DynamicInferenceEngine, - ) - from megatron.core.inference.model_inference_wrappers.gpt.gpt_inference_wrapper import ( - GPTInferenceWrapper, - ) - from megatron.core.inference.sampling_params import SamplingParams - from megatron.core.inference.utils import InferenceMode - - model_config = self.model.config - model_config.cuda_graph_impl = "local" - - local_rank = torch.cuda.current_device() - num_gpus_per_node = torch.cuda.device_count() - node_idx = self.rank // num_gpus_per_node if num_gpus_per_node > 0 else 0 - model_config.inference_sampling_seed = (node_idx * 1024) + local_rank - - inference_config = InferenceConfig( - max_sequence_length=self.cfg["generation"]["max_new_tokens"], - buffer_size_gb=mcore_generation_config["buffer_size_gb"], - num_cuda_graphs=mcore_generation_config["num_cuda_graphs"], - block_size_tokens=mcore_generation_config["block_size_tokens"], - use_cuda_graphs_for_non_decode_steps=mcore_generation_config[ - "use_cuda_graphs_for_non_decode_steps" - ], - enable_chunked_prefill=mcore_generation_config["enable_chunked_prefill"], - unified_memory_level=mcore_generation_config["unified_memory_level"], - max_tokens=mcore_generation_config["max_tokens"], - materialize_only_last_token_logits=False, - use_flashinfer_fused_rope=False, - ) - - dynamic_context = DynamicInferenceContext(model_config, inference_config) - inference_wrapped_model = GPTInferenceWrapper(self.model, dynamic_context) - - inference_wrapped_model.prep_model_for_inference() - # Set pipeline parallel flag - inference_wrapped_model.model_is_pipeline_parallel = ( - self.cfg["megatron_cfg"]["pipeline_model_parallel_size"] > 1 - ) - - text_generation_controller = TextGenerationController( - inference_wrapped_model=inference_wrapped_model, - tokenizer=self.megatron_tokenizer, - ) - - with InferenceMode.active(): - dynamic_engine = DynamicInferenceEngine( - text_generation_controller, - dynamic_context, - ) - - # Handle None values for top_k - convert to integer as required by Megatron - top_k_cfg = self.cfg["generation"]["top_k"] - top_k_val = ( - 1 if greedy else (int(top_k_cfg) if top_k_cfg is not None else 0) - ) - - top_p_cfg = self.cfg["generation"]["top_p"] - top_p_val = ( - 0.0 if greedy else (float(top_p_cfg) if top_p_cfg is not None else 0.0) - ) - - # New API: SamplingParams now includes termination_id and uses num_tokens_total - sampling_params = SamplingParams( - temperature=self.cfg["generation"]["temperature"] if not greedy else 0, - top_k=top_k_val, - top_p=top_p_val, - skip_prompt_log_probs=False, - return_log_probs=True, - num_tokens_total=self.cfg["generation"]["max_new_tokens"], - num_tokens_to_generate=None, - termination_id=self.megatron_tokenizer.eod, - ) - - input_ids = data["input_ids"] - prompt_tokens_tensor = input_ids.cuda() - prompt_lengths_tensor = data["input_lengths"] - request_id = 0 - - # New API: add_request now takes sampling_params as a parameter - for p, prompt_len in zip( - prompt_tokens_tensor, prompt_lengths_tensor, strict=True - ): - dynamic_engine.add_request( - request_id, - p[:prompt_len], - sampling_params=sampling_params, - ) - request_id += 1 - - result = [] - while dynamic_engine.has_unfinished_requests(): - result_step = dynamic_engine.step_modern() - result.extend(result_step["finished_request_records"]) - - # Sort results by request_id to maintain original batch order - result.sort(key=lambda x: x.request_id) - - out = { - "tokens": [ - x.requests[0].prompt_tokens.tolist() + x.requests[0].generated_tokens - for x in result - ], - "logprobs": [ - x.requests[0].prompt_log_probs + x.requests[0].generated_log_probs - for x in result - ], - } - - input_lengths = data["input_lengths"] - # pad the out "tokens" and "logprobs" and make them into tensors from lists - batch_size = data["input_ids"].size(0) - max_gen_seq_len = max([len(x.requests[0].generated_tokens) for x in result]) - padded_input_length = input_ids.size(1) - - max_seq_len = padded_input_length + max_gen_seq_len - # Create padded tensors for tokens and logprobs - output_ids_padded = torch.full( - (batch_size, max_seq_len), - self.tokenizer.pad_token_id, - dtype=torch.long, - device=data["input_ids"].device, - ) - - logprobs_padded = torch.zeros( - (batch_size, max_seq_len), - dtype=torch.float, - device=data["input_ids"].device, - ) - - # Fill in the padded tensors with actual values - generation_lengths = torch.zeros( - batch_size, dtype=torch.long, device=data["input_ids"].device - ) - unpadded_sequence_lengths = torch.zeros( - batch_size, dtype=torch.long, device=data["input_ids"].device - ) - for i in range(batch_size): - seq_len = len(out["tokens"][i]) - output_ids_padded[i, :seq_len] = torch.tensor( - out["tokens"][i], dtype=torch.long, device=data["input_ids"].device - ) - generation_lengths[i] = seq_len - input_lengths[i].item() - unpadded_sequence_lengths[i] = seq_len - logprob_len = len(out["logprobs"][i]) - logprobs_padded[i, 1 : logprob_len + 1] = torch.tensor( - out["logprobs"][i], - dtype=torch.float, - device=data["input_ids"].device, - ) - - out_dict = { - "output_ids": output_ids_padded, - "logprobs": logprobs_padded, - "generation_lengths": generation_lengths, - "unpadded_sequence_lengths": unpadded_sequence_lengths, - } - - self.model.config.flash_decode = False - no_grad.__exit__(None, None, None) - - return BatchedDataDict.from_batches([out_dict]).to("cpu") - @torch.no_grad() @wrap_with_nvtx_name("megatron_policy_worker/prepare_refit_info") def prepare_refit_info(self) -> None: @@ -1302,6 +1153,8 @@ def calculate_size_in_bytes(param, tp_size, ep_size): torch.bfloat16: 2, torch.float16: 2, torch.float32: 4, + torch.float8_e4m3fn: 1, + torch.float8_e5m2: 1, } scale = prec_to_bytes[self.dtype] / prec_to_bytes[param.dtype] size_in_bytes = ( @@ -1702,14 +1555,15 @@ def save_checkpoint( ) original_save_path = self.mcore_state.cfg.checkpoint.save - # save_dir = os.path.dirname(weights_path) - release_name = os.path.basename(weights_path) + is_async = self.mcore_state.cfg.checkpoint.async_save try: + # Block until any previous async save is fully written to disk. + # With sync save this is a no-op. maybe_finalize_async_save( self.mcore_state, ckpt_cfg=self.mcore_state.cfg.checkpoint, - blocking=False, + blocking=True, ) self.mcore_state.cfg.checkpoint.save = weights_path @@ -1739,17 +1593,18 @@ def save_checkpoint( num_floating_point_operations_so_far=self.mcore_state.train_state.floating_point_operations_so_far, checkpointing_context=self.checkpointing_context, ) - print(f"Saved checkpoint to {weights_path}") - maybe_finalize_async_save( - self.mcore_state, - ckpt_cfg=self.mcore_state.cfg.checkpoint, - blocking=True, - terminate=True, - ) + + if not is_async: + # Sync path: finalize immediately (runs finalize_fns + barrier). + maybe_finalize_async_save( + self.mcore_state, + ckpt_cfg=self.mcore_state.cfg.checkpoint, + blocking=True, + ) if self.should_disable_forward_pre_hook: self.enable_forward_pre_hook() - if not is_training: # Restore training state if it was changed + if not is_training: self.model.train() except Exception as e: diff --git a/nemo_rl/models/value/workers/megatron_value_worker.py b/nemo_rl/models/value/workers/megatron_value_worker.py index 03d07f28b10..b4d598b8e1b 100644 --- a/nemo_rl/models/value/workers/megatron_value_worker.py +++ b/nemo_rl/models/value/workers/megatron_value_worker.py @@ -276,6 +276,31 @@ def __repr__(self): else: return f"{self.__class__.__qualname__}" + @staticmethod + def configure_worker( + num_gpus: int | float, + bundle_indices: Optional[tuple[int, list[int]]] = None, + ) -> tuple[dict[str, Any], dict[str, str], dict[str, Any]]: + """Worker-controlled Ray actor configuration. + + Mirrors `MegatronPolicyWorker` to ensure NVLS communication functions correctly. + + Args: + num_gpus: Original GPU allocation for this worker based on the placement group + bundle_indices: Tuple of (node_idx, local_bundle_indices) for this server + + Returns: + tuple with complete worker configuration: + - 'resources': Resource allocation (e.g., num_gpus) + - 'env_vars': Environment variables for this worker + - 'init_kwargs': Parameters to pass to __init__ of the worker + """ + del bundle_indices # one GPU per worker; no per-bundle seeding needed + resources: dict[str, Any] = {"num_gpus": num_gpus} + env_vars: dict[str, str] = {"RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES": "1"} + init_kwargs: dict[str, Any] = {} + return resources, env_vars, init_kwargs + def __init__( self, config: ValueConfig, @@ -297,6 +322,13 @@ def __init__( init_optimizer: Whether to initialize the optimizer. worker_sharding_annotations: Sharding topology for distributed training. """ + # Must be the first CUDA-touching call in this process. + # With `RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES=1` (set by `configure_worker()`), + gpu_ids = ray.get_gpu_ids() + local_rank = int(gpu_ids[0]) + os.environ["LOCAL_RANK"] = str(local_rank) + torch.cuda.set_device(local_rank) + apply_transformer_engine_patch() self.cfg = config diff --git a/pyrefly.toml b/pyrefly.toml index a1486f0a649..ad5796d4c2e 100644 --- a/pyrefly.toml +++ b/pyrefly.toml @@ -144,6 +144,9 @@ project-includes = [ "nemo_rl/models/generation/__init__.py", "nemo_rl/models/generation/constants.py", "nemo_rl/models/generation/interfaces.py", + "nemo_rl/models/generation/megatron/__init__.py", + "nemo_rl/models/generation/megatron/config.py", + "nemo_rl/models/generation/megatron/utils.py", "nemo_rl/models/generation/sglang/__init__.py", "nemo_rl/models/generation/sglang/config.py", "nemo_rl/models/generation/vllm/__init__.py", diff --git a/research/template_project/configs/grpo_math_1B.yaml b/research/template_project/configs/grpo_math_1B.yaml index 913e667fc9c..74508dc919b 100644 --- a/research/template_project/configs/grpo_math_1B.yaml +++ b/research/template_project/configs/grpo_math_1B.yaml @@ -288,7 +288,6 @@ policy: block_size_tokens: 256 # Size of each KV cache block in tokens (affects memory granularity) use_cuda_graphs_for_non_decode_steps: true # Enable CUDA graphs for prefill/context processing enable_chunked_prefill: true # Split long prefills into chunks for better memory management - unified_memory_level: 0 # Unified memory usage level (0=disabled, higher values enable more aggressive paging) max_tokens: 16384 # Maximum number of tokens to use in a single step. Analogous to vllm's max_num_batched_tokens vllm_cfg: async_engine: false diff --git a/tests/functional/L1_Functional_Tests_Megatron_1.sh b/tests/functional/L1_Functional_Tests_Megatron_1.sh index c7c6571aa3e..000867eda15 100644 --- a/tests/functional/L1_Functional_Tests_Megatron_1.sh +++ b/tests/functional/L1_Functional_Tests_Megatron_1.sh @@ -38,8 +38,6 @@ run_test fast uv run --no-sync bash ./tests/functional/audio_grpo_megatron.sh run_test uv run --no-sync bash ./tests/functional/grpo_megatron.sh run_test uv run --no-sync bash ./tests/functional/grpo_megatron_mbridge_restore.sh run_test fast uv run --no-sync bash ./tests/functional/grpo_megatron_eagle3_online.sh -# TODO: enable after resolving megatron inference test cases -# run_test uv run --no-sync bash ./tests/functional/grpo_megatron_generation.sh cd ${PROJECT_ROOT}/tests if compgen -G ".coverage*" > /dev/null; then diff --git a/tests/functional/L1_Functional_Tests_Megatron_4.sh b/tests/functional/L1_Functional_Tests_Megatron_4.sh new file mode 100644 index 00000000000..0610ab6d818 --- /dev/null +++ b/tests/functional/L1_Functional_Tests_Megatron_4.sh @@ -0,0 +1,65 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash +set -xeuo pipefail # Exit immediately if a command exits with a non-zero status + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +PROJECT_ROOT=$(realpath ${SCRIPT_DIR}/../..) + +cd ${PROJECT_ROOT} + +# run_test [fast] +# - "run_test fast " = always runs (both fast and full modes) +# - "run_test " = only runs in full mode; skipped when FAST=1 +run_test() { + if [[ "$1" == "fast" ]]; then + shift + time "$@" + elif [[ "${FAST:-0}" == "1" ]]; then + echo "FAST: Skipping: $*" + else + time "$@" + fi +} + +# Megatron Inference currently hits an IMA on Blackwell tests. +# TODO: remove this guard once the upstream dependency is bumped. +megatron_generation_supported() { + if command -v nvidia-smi &> /dev/null; then + local compute_cap + compute_cap=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader -i 0 2>/dev/null | tr -d '. ' || true) + if [[ "${compute_cap:-0}" -ge 100 ]]; then + echo "WARNING: Skipping Blackwell x Megatron Inference tests" + return 1 + fi + fi + return 0 +} + +if megatron_generation_supported; then + run_test uv run --no-sync bash ./tests/functional/grpo_megatron_generation.sh + run_test uv run --no-sync bash ./tests/functional/grpo_megatron_generation_non_colocated.sh + run_test uv run --no-sync bash ./tests/functional/grpo_megatron_generation_async.sh + run_test uv run --no-sync bash ./tests/functional/grpo_megatron_generation_colocated_async.sh + run_test uv run --no-sync bash ./tests/functional/grpo_megatron_generation_async_gym.sh + # DISABLED: Megatron Inference returns unmasked logprobs + # run_test uv run --no-sync bash ./tests/functional/grpo_megatron_generation_topp_topk.sh + run_test uv run --no-sync bash ./tests/functional/grpo_megatron_generation_multiturn.sh +fi + +cd ${PROJECT_ROOT}/tests +if compgen -G ".coverage*" > /dev/null; then + coverage combine .coverage* +fi diff --git a/tests/functional/grpo_megatron_generation_async.sh b/tests/functional/grpo_megatron_generation_async.sh new file mode 100644 index 00000000000..16b7e5afa1c --- /dev/null +++ b/tests/functional/grpo_megatron_generation_async.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +PROJECT_ROOT=$(realpath $SCRIPT_DIR/../..) +# Mark the current repo as safe, since wandb fetches metadata about the repo +git config --global --add safe.directory $PROJECT_ROOT + +set -eou pipefail + +EXP_NAME=$(basename $0 .sh) +EXP_DIR=$SCRIPT_DIR/$EXP_NAME +LOG_DIR=$EXP_DIR/logs +JSON_METRICS=$EXP_DIR/metrics.json +RUN_LOG=$EXP_DIR/run.log +export PYTHONPATH=${PROJECT_ROOT}:${PYTHONPATH:-} + +rm -rf $EXP_DIR $LOG_DIR +mkdir -p $EXP_DIR $LOG_DIR + +# Native async path: non-colocated Megatron generation with the async mcore engine and +# NO NeMo-Gym (the megatron analog of grpo_non_colocated.sh). +# Using Qwen2.5-0.5B instead of Qwen3-0.6B because the latter is not supported by Megatron yet +cd $PROJECT_ROOT +uv run coverage run -a --data-file=$PROJECT_ROOT/tests/.coverage --source=$PROJECT_ROOT/nemo_rl \ + $PROJECT_ROOT/examples/run_grpo.py \ + --config $PROJECT_ROOT/examples/configs/grpo_math_1B_megatron.yaml \ + policy.model_name=Qwen/Qwen2.5-0.5B \ + grpo.num_prompts_per_step=2 \ + grpo.num_generations_per_prompt=4 \ + policy.train_global_batch_size=4 \ + policy.logprob_batch_size=4 \ + policy.train_micro_batch_size=1 \ + policy.generation.backend=megatron \ + policy.generation.colocated.enabled=false \ + policy.generation.colocated.resources.gpus_per_node=1 \ + policy.generation.mcore_generation_config.async_engine=true \ + cluster.gpus_per_node=2 \ + grpo.max_num_steps=2 \ + logger.tensorboard_enabled=true \ + logger.log_dir=$LOG_DIR \ + logger.wandb_enabled=false \ + logger.monitor_gpus=true \ + checkpointing.enabled=false \ + $@ \ + 2>&1 | tee $RUN_LOG + +uv run tests/json_dump_tb_logs.py $LOG_DIR --output_path $JSON_METRICS + +uv run tests/check_metrics.py $JSON_METRICS \ + 'max(data["train/token_mult_prob_error"]) < 1.05' diff --git a/tests/functional/grpo_megatron_generation_async_gym.sh b/tests/functional/grpo_megatron_generation_async_gym.sh new file mode 100755 index 00000000000..585671b52de --- /dev/null +++ b/tests/functional/grpo_megatron_generation_async_gym.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +PROJECT_ROOT=$(realpath $SCRIPT_DIR/../..) +# Mark the current repo as safe, since wandb fetches metadata about the repo +git config --global --add safe.directory $PROJECT_ROOT + +set -eou pipefail + +EXP_NAME=$(basename $0 .sh) +EXP_DIR=$SCRIPT_DIR/$EXP_NAME +LOG_DIR=$EXP_DIR/logs +JSON_METRICS=$EXP_DIR/metrics.json +RUN_LOG=$EXP_DIR/run.log +CHECKPOINT_DIR=$EXP_DIR/checkpoints +DATA_DIR=$EXP_DIR/data +export PYTHONPATH=${PROJECT_ROOT}:${PYTHONPATH:-} + +rm -rf $EXP_DIR $LOG_DIR +mkdir -p $EXP_DIR $LOG_DIR $CHECKPOINT_DIR $DATA_DIR + +# clean up checkpoint directory on exit +trap "rm -rf $CHECKPOINT_DIR" EXIT + +cd $PROJECT_ROOT + +# Follow nemo-gym instructions here to get this data: +# https://docs.nvidia.com/nemo/gym/0.1.0/tutorials/nemo-rl-grpo/setup.html#training-nemo-rl-grpo-setup +cd 3rdparty/Gym-workspace/Gym + +# We need HF_TOKEN to download the data from huggingface +if [[ ! -f env.yaml ]]; then + if [[ -z "${HF_TOKEN:-}" ]]; then + echo "[ERROR] HF_TOKEN is not set" + exit 1 + fi + echo "hf_token: $HF_TOKEN" >> env.yaml +fi + +uv run ng_prepare_data "+config_paths=[resources_servers/workplace_assistant/configs/workplace_assistant.yaml]" \ + +output_dirpath=data/workplace_assistant \ + +mode=train_preparation \ + +should_download=true \ + +data_source=huggingface +cd - + +# This trimming of the workplace assistant dataset is necessary b/c with all the tools the first prompt is >4000 tokens +# which will cause vllm to return nothing on the first prompt and crash RL. Since we want to keep this test short to +# smoke test, we trim all but the first tool +TRAIN_PATH=$DATA_DIR/workplace_assistant_train.jsonl +VALIDATION_PATH=$DATA_DIR/workplace_assistant_validation.jsonl +jq -c '.responses_create_params.tools |= (.[0:1])' 3rdparty/Gym-workspace/Gym/data/workplace_assistant/train.jsonl > $TRAIN_PATH +jq -c '.responses_create_params.tools |= (.[0:1])' 3rdparty/Gym-workspace/Gym/data/workplace_assistant/validation.jsonl > $VALIDATION_PATH + +uv run coverage run -a --data-file=$PROJECT_ROOT/tests/.coverage --source=$PROJECT_ROOT/nemo_rl \ + $PROJECT_ROOT/examples/nemo_gym/run_grpo_nemo_gym.py \ + --config $PROJECT_ROOT/examples/nemo_gym/grpo_qwen3_30ba3b_instruct.yaml \ + policy.model_name=Qwen/Qwen3-0.6B \ + policy.dtensor_cfg.enabled=false \ + policy.megatron_cfg.enabled=true \ + policy.megatron_cfg.tensor_model_parallel_size=1 \ + policy.megatron_cfg.pipeline_model_parallel_size=1 \ + policy.megatron_cfg.expert_model_parallel_size=1 \ + policy.megatron_cfg.context_parallel_size=1 \ + policy.megatron_cfg.sequence_parallel=false \ + policy.generation.backend=megatron \ + policy.generation.mcore_generation_config.async_engine=true \ + policy.generation.mcore_generation_config.expose_http_server=true \ + policy.generation.mcore_generation_config.enable_prefix_caching=true \ + policy.max_total_sequence_length=512 \ + policy.generation.max_new_tokens=128 \ + policy.generation.colocated.enabled=false \ + policy.generation.colocated.resources.num_nodes=1 \ + policy.generation.colocated.resources.gpus_per_node=1 \ + grpo.num_prompts_per_step=4 \ + grpo.num_generations_per_prompt=2 \ + grpo.max_num_steps=10 \ + grpo.val_period=5 \ + grpo.async_grpo.enabled=true \ + grpo.async_grpo.max_trajectory_age_steps=1 \ + grpo.async_grpo.in_flight_weight_updates=true \ + policy.train_global_batch_size=4 \ + policy.train_micro_batch_size=1 \ + cluster.gpus_per_node=2 \ + loss_fn.use_importance_sampling_correction=true \ + logger.tensorboard_enabled=true \ + logger.log_dir=$LOG_DIR \ + logger.wandb_enabled=false \ + logger.monitor_gpus=true \ + checkpointing.enabled=true \ + checkpointing.save_period=5 \ + checkpointing.checkpoint_dir=$CHECKPOINT_DIR \ + data.train.data_path=$TRAIN_PATH \ + data.validation.data_path=$VALIDATION_PATH \ + $@ \ + 2>&1 | tee $RUN_LOG + +uv run tests/json_dump_tb_logs.py $LOG_DIR --output_path $JSON_METRICS + +# Smoke-level thresholds. Tighten after first successful runs on CI. +uv run tests/check_metrics.py $JSON_METRICS \ + 'median(data["train/gen_kl_error"]) < 1.3' \ + 'data["validation/accuracy"]["10"] > 0.1' diff --git a/tests/functional/grpo_megatron_generation_colocated_async.sh b/tests/functional/grpo_megatron_generation_colocated_async.sh new file mode 100644 index 00000000000..72d8a4a16c5 --- /dev/null +++ b/tests/functional/grpo_megatron_generation_colocated_async.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +PROJECT_ROOT=$(realpath $SCRIPT_DIR/../..) +# Mark the current repo as safe, since wandb fetches metadata about the repo +git config --global --add safe.directory $PROJECT_ROOT + +set -eou pipefail + +EXP_NAME=$(basename $0 .sh) +EXP_DIR=$SCRIPT_DIR/$EXP_NAME +LOG_DIR=$EXP_DIR/logs +JSON_METRICS=$EXP_DIR/metrics.json +RUN_LOG=$EXP_DIR/run.log +export PYTHONPATH=${PROJECT_ROOT}:${PYTHONPATH:-} + +rm -rf $EXP_DIR $LOG_DIR +mkdir -p $EXP_DIR $LOG_DIR + +# Colocated Megatron generation with the async mcore engine (generation shares the +# training workers). No NeMo-Gym. +# Using Qwen2.5-0.5B instead of Qwen3-0.6B because the latter is not supported by Megatron yet +cd $PROJECT_ROOT +uv run coverage run -a --data-file=$PROJECT_ROOT/tests/.coverage --source=$PROJECT_ROOT/nemo_rl \ + $PROJECT_ROOT/examples/run_grpo.py \ + --config $PROJECT_ROOT/examples/configs/grpo_math_1B_megatron.yaml \ + policy.model_name=Qwen/Qwen2.5-0.5B \ + grpo.num_prompts_per_step=2 \ + grpo.num_generations_per_prompt=4 \ + policy.train_global_batch_size=4 \ + policy.logprob_batch_size=4 \ + policy.train_micro_batch_size=1 \ + policy.generation.backend=megatron \ + policy.generation.mcore_generation_config.async_engine=true \ + cluster.gpus_per_node=2 \ + grpo.max_num_steps=2 \ + logger.tensorboard_enabled=true \ + logger.log_dir=$LOG_DIR \ + logger.wandb_enabled=false \ + logger.monitor_gpus=true \ + checkpointing.enabled=false \ + $@ \ + 2>&1 | tee $RUN_LOG + +uv run tests/json_dump_tb_logs.py $LOG_DIR --output_path $JSON_METRICS + +uv run tests/check_metrics.py $JSON_METRICS \ + 'max(data["train/token_mult_prob_error"]) < 1.05' diff --git a/tests/functional/grpo_megatron_generation_multiturn.sh b/tests/functional/grpo_megatron_generation_multiturn.sh new file mode 100644 index 00000000000..7a7867f0964 --- /dev/null +++ b/tests/functional/grpo_megatron_generation_multiturn.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +PROJECT_ROOT=$(realpath $SCRIPT_DIR/../..) +# Mark the current repo as safe, since wandb fetches metadata about the repo +git config --global --add safe.directory $PROJECT_ROOT + +set -eou pipefail + +EXP_NAME=$(basename $0 .sh) +EXP_DIR=$SCRIPT_DIR/$EXP_NAME +LOG_DIR=$EXP_DIR/logs +JSON_METRICS=$EXP_DIR/metrics.json +RUN_LOG=$EXP_DIR/run.log +export PYTHONPATH=${PROJECT_ROOT}:${PYTHONPATH:-} + +rm -rf $EXP_DIR $LOG_DIR +mkdir -p $EXP_DIR $LOG_DIR + +# Native (non-gym) multi-turn rollouts with Megatron generation, exercising the per-turn +# generate path and stop strings (the megatron analog of grpo_multiturn.sh). Switches the +# sliding-puzzle config (inherits grpo_math_1B.yaml) to the Megatron policy + backend. +# Using Qwen2.5-0.5B instead of Qwen3-0.6B because the latter is not supported by Megatron yet +cd $PROJECT_ROOT +uv run coverage run -a --data-file=$PROJECT_ROOT/tests/.coverage --source=$PROJECT_ROOT/nemo_rl \ + $PROJECT_ROOT/examples/run_grpo_sliding_puzzle.py \ + policy.model_name=Qwen/Qwen2.5-0.5B \ + policy.dtensor_cfg.enabled=false \ + policy.megatron_cfg.enabled=true \ + policy.generation.backend=megatron \ + cluster.gpus_per_node=2 \ + grpo.max_rollout_turns=5 \ + grpo.max_num_steps=3 \ + grpo.num_prompts_per_step=2 \ + grpo.num_generations_per_prompt=4 \ + policy.max_total_sequence_length=1024 \ + policy.train_global_batch_size=4 \ + policy.train_micro_batch_size=1 \ + policy.generation.top_p=0.9 \ + policy.generation.top_k=8000 \ + logger.tensorboard_enabled=true \ + logger.log_dir=$LOG_DIR \ + logger.wandb_enabled=false \ + checkpointing.enabled=false \ + $@ \ + 2>&1 | tee $RUN_LOG + +uv run tests/json_dump_tb_logs.py $LOG_DIR --output_path $JSON_METRICS + +uv run tests/check_metrics.py $JSON_METRICS \ + 'median(data["train/token_mult_prob_error"]) < 1.1' diff --git a/tests/functional/grpo_megatron_generation_non_colocated.sh b/tests/functional/grpo_megatron_generation_non_colocated.sh new file mode 100755 index 00000000000..1ddf3d73f39 --- /dev/null +++ b/tests/functional/grpo_megatron_generation_non_colocated.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +PROJECT_ROOT=$(realpath $SCRIPT_DIR/../..) +# Mark the current repo as safe, since wandb fetches metadata about the repo +git config --global --add safe.directory $PROJECT_ROOT + +set -eou pipefail + +EXP_NAME=$(basename $0 .sh) +EXP_DIR=$SCRIPT_DIR/$EXP_NAME +LOG_DIR=$EXP_DIR/logs +JSON_METRICS=$EXP_DIR/metrics.json +RUN_LOG=$EXP_DIR/run.log +export PYTHONPATH=${PROJECT_ROOT}:${PYTHONPATH:-} + +rm -rf $EXP_DIR $LOG_DIR +mkdir -p $EXP_DIR $LOG_DIR + +# Using Qwen2.5-0.5B instead of Qwen3-0.6B because the latter is not supported by Megatron yet +cd $PROJECT_ROOT +uv run coverage run -a --data-file=$PROJECT_ROOT/tests/.coverage --source=$PROJECT_ROOT/nemo_rl \ + $PROJECT_ROOT/examples/run_grpo.py \ + --config $PROJECT_ROOT/examples/configs/grpo_math_1B_megatron.yaml \ + policy.model_name=Qwen/Qwen2.5-0.5B \ + grpo.num_prompts_per_step=2 \ + grpo.num_generations_per_prompt=4 \ + policy.train_global_batch_size=4 \ + policy.logprob_batch_size=4 \ + policy.train_micro_batch_size=1 \ + policy.generation.backend=megatron \ + policy.generation.colocated.enabled=false \ + policy.generation.colocated.resources.gpus_per_node=1 \ + cluster.gpus_per_node=2 \ + grpo.max_num_steps=2 \ + logger.tensorboard_enabled=true \ + logger.log_dir=$LOG_DIR \ + logger.wandb_enabled=false \ + logger.monitor_gpus=true \ + checkpointing.enabled=false \ + $@ \ + 2>&1 | tee $RUN_LOG + +uv run tests/json_dump_tb_logs.py $LOG_DIR --output_path $JSON_METRICS + +uv run tests/check_metrics.py $JSON_METRICS \ + 'max(data["train/token_mult_prob_error"]) < 1.05' diff --git a/tests/functional/grpo_megatron_generation_topp_topk.sh b/tests/functional/grpo_megatron_generation_topp_topk.sh new file mode 100644 index 00000000000..6a4118cf3c1 --- /dev/null +++ b/tests/functional/grpo_megatron_generation_topp_topk.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +PROJECT_ROOT=$(realpath $SCRIPT_DIR/../..) +# Mark the current repo as safe, since wandb fetches metadata about the repo +git config --global --add safe.directory $PROJECT_ROOT + +set -eou pipefail + +EXP_NAME=$(basename $0 .sh) +EXP_DIR=$SCRIPT_DIR/$EXP_NAME +LOG_DIR=$EXP_DIR/logs +JSON_METRICS=$EXP_DIR/metrics.json +RUN_LOG=$EXP_DIR/run.log +export PYTHONPATH=${PROJECT_ROOT}:${PYTHONPATH:-} + +rm -rf $EXP_DIR $LOG_DIR +mkdir -p $EXP_DIR $LOG_DIR + +# Megatron generation with non-default sampling (temperature / top-p / top-k), exercising +# the SamplingParams path (the megatron analog of grpo_topp_topk.sh). +# Using Qwen2.5-0.5B instead of Qwen3-0.6B because the latter is not supported by Megatron yet +cd $PROJECT_ROOT +uv run coverage run -a --data-file=$PROJECT_ROOT/tests/.coverage --source=$PROJECT_ROOT/nemo_rl \ + $PROJECT_ROOT/examples/run_grpo.py \ + --config $PROJECT_ROOT/examples/configs/grpo_math_1B_megatron.yaml \ + policy.model_name=Qwen/Qwen2.5-0.5B \ + grpo.num_prompts_per_step=2 \ + grpo.num_generations_per_prompt=4 \ + policy.train_global_batch_size=4 \ + policy.logprob_batch_size=4 \ + policy.train_micro_batch_size=1 \ + policy.generation.backend=megatron \ + policy.generation.temperature=0.8 \ + policy.generation.top_p=0.9 \ + policy.generation.top_k=50 \ + cluster.gpus_per_node=2 \ + grpo.max_num_steps=2 \ + logger.tensorboard_enabled=true \ + logger.log_dir=$LOG_DIR \ + logger.wandb_enabled=false \ + logger.monitor_gpus=true \ + checkpointing.enabled=false \ + $@ \ + 2>&1 | tee $RUN_LOG + +uv run tests/json_dump_tb_logs.py $LOG_DIR --output_path $JSON_METRICS + +uv run tests/check_metrics.py $JSON_METRICS \ + 'max(data["train/token_mult_prob_error"]) < 1.05' diff --git a/tests/test_suites/llm/grpo-llama3.2-1b-instruct-2n8g-megatron_generation-noncolocated.sh b/tests/test_suites/llm/grpo-llama3.2-1b-instruct-2n8g-megatron_generation-noncolocated.sh new file mode 100755 index 00000000000..83b1eacfacf --- /dev/null +++ b/tests/test_suites/llm/grpo-llama3.2-1b-instruct-2n8g-megatron_generation-noncolocated.sh @@ -0,0 +1,43 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +source $SCRIPT_DIR/common.env + +# ===== BEGIN CONFIG ===== +NUM_NODES=2 +STEPS_PER_RUN=30 +MAX_STEPS=30 +NUM_RUNS=$(( (MAX_STEPS + STEPS_PER_RUN - 1) / STEPS_PER_RUN )) # Round up +NUM_MINUTES=120 +# ===== END CONFIG ===== + +exit_if_max_steps_reached + +# Run the experiment +cd $PROJECT_ROOT +uv run examples/run_grpo.py \ + --config $CONFIG_PATH \ + grpo.max_num_steps=$MAX_STEPS \ + policy.generation.backend=megatron \ + logger.log_dir=$LOG_DIR \ + logger.wandb_enabled=True \ + logger.wandb.project=nemo-rl \ + logger.wandb.name=$EXP_NAME \ + logger.monitor_gpus=True \ + logger.tensorboard_enabled=True \ + checkpointing.enabled=True \ + checkpointing.checkpoint_dir=$CKPT_DIR \ + $@ \ + 2>&1 | tee $RUN_LOG + +# Convert tensorboard logs to json +uv run tests/json_dump_tb_logs.py $LOG_DIR --output_path $JSON_METRICS + +# Only run metrics if the target step is reached +if [[ $(jq 'to_entries | .[] | select(.key == "train/loss") | .value | keys | map(tonumber) | max' $JSON_METRICS) -ge $MAX_STEPS ]]; then + uv run tests/check_metrics.py $JSON_METRICS \ + 'median(data["train/token_mult_prob_error"]) < 1.1' \ + 'data["train/token_mult_prob_error"]["30"] < 1.1' + + # Clean up checkpoint directory after successful run to save space. + rm -rf "$CKPT_DIR" +fi diff --git a/tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-megatron_generation-async-gym.sh b/tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-megatron_generation-async-gym.sh new file mode 100755 index 00000000000..343c1b2c136 --- /dev/null +++ b/tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-megatron_generation-async-gym.sh @@ -0,0 +1,68 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +source $SCRIPT_DIR/common.env + +# ===== BEGIN CONFIG ===== +NUM_NODES=2 +GPUS_PER_NODE=8 +STEPS_PER_RUN=10 +MAX_STEPS=10 +NUM_RUNS=$(( (MAX_STEPS + STEPS_PER_RUN - 1) / STEPS_PER_RUN )) # Round up +NUM_MINUTES=90 +# ===== END CONFIG ===== + +exit_if_max_steps_reached + +cd $PROJECT_ROOT + +# Prepare nemo-gym workplace_assistant dataset (mirrors tests/functional/grpo_async_gym.sh). +DATA_DIR=$EXP_DIR/data +mkdir -p $DATA_DIR +cd 3rdparty/Gym-workspace/Gym +if [[ ! -f env.yaml ]]; then + if [[ -z "${HF_TOKEN:-}" ]]; then + echo "[ERROR] HF_TOKEN is not set" + exit 1 + fi + echo "hf_token: $HF_TOKEN" >> env.yaml +fi +uv run ng_prepare_data "+config_paths=[resources_servers/workplace_assistant/configs/workplace_assistant.yaml]" \ + +output_dirpath=data/workplace_assistant \ + +mode=train_preparation \ + +should_download=true \ + +data_source=huggingface +cd - + +# Trim to a single tool so first prompt fits. +TRAIN_PATH=$DATA_DIR/workplace_assistant_train.jsonl +VALIDATION_PATH=$DATA_DIR/workplace_assistant_validation.jsonl +jq -c '.responses_create_params.tools |= (.[0:1])' 3rdparty/Gym-workspace/Gym/data/workplace_assistant/train.jsonl > $TRAIN_PATH +jq -c '.responses_create_params.tools |= (.[0:1])' 3rdparty/Gym-workspace/Gym/data/workplace_assistant/validation.jsonl > $VALIDATION_PATH + +# Run the experiment via the gym entrypoint +uv run examples/nemo_gym/run_grpo_nemo_gym.py \ + --config $CONFIG_PATH \ + grpo.max_num_steps=$MAX_STEPS \ + logger.log_dir=$LOG_DIR \ + logger.wandb_enabled=True \ + logger.wandb.project=nemo-rl \ + logger.wandb.name=$EXP_NAME \ + logger.monitor_gpus=True \ + logger.tensorboard_enabled=True \ + checkpointing.enabled=True \ + checkpointing.checkpoint_dir=$CKPT_DIR \ + data.train.data_path=$TRAIN_PATH \ + data.validation.data_path=$VALIDATION_PATH \ + $@ \ + 2>&1 | tee $RUN_LOG + +uv run tests/json_dump_tb_logs.py $LOG_DIR --output_path $JSON_METRICS + +# Smoke-level threshold; tighten after observing real runs. +if [[ $(jq 'to_entries | .[] | select(.key == "train/loss") | .value | keys | map(tonumber) | max' $JSON_METRICS) -ge $MAX_STEPS ]]; then + uv run tests/check_metrics.py $JSON_METRICS \ + 'max(data["train/reward"]) > 0.0' + + # Clean up checkpoint directory after successful run to save space. + rm -rf "$CKPT_DIR" +fi diff --git a/tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-megatron_generation.sh b/tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-megatron_generation.sh new file mode 100755 index 00000000000..9e308ca457e --- /dev/null +++ b/tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-megatron_generation.sh @@ -0,0 +1,43 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +source $SCRIPT_DIR/common.env + +# ===== BEGIN CONFIG ===== +NUM_NODES=2 +GPUS_PER_NODE=8 +STEPS_PER_RUN=10 +MAX_STEPS=10 +NUM_RUNS=$(( (MAX_STEPS + STEPS_PER_RUN - 1) / STEPS_PER_RUN )) # Round up +NUM_MINUTES=60 +# ===== END CONFIG ===== + +exit_if_max_steps_reached + +# Run the experiment +cd $PROJECT_ROOT +uv run examples/run_grpo.py \ + --config $CONFIG_PATH \ + grpo.max_num_steps=$MAX_STEPS \ + policy.generation.backend=megatron \ + logger.log_dir=$LOG_DIR \ + logger.wandb_enabled=True \ + logger.wandb.project=nemo-rl \ + logger.wandb.name=$EXP_NAME \ + logger.monitor_gpus=True \ + logger.tensorboard_enabled=True \ + checkpointing.enabled=True \ + checkpointing.checkpoint_dir=$CKPT_DIR \ + $@ \ + 2>&1 | tee $RUN_LOG + +# Convert tensorboard logs to json +uv run tests/json_dump_tb_logs.py $LOG_DIR --output_path $JSON_METRICS + +# Only run metrics if the target step is reached +if [[ $(jq 'to_entries | .[] | select(.key == "train/loss") | .value | keys | map(tonumber) | max' $JSON_METRICS) -ge $MAX_STEPS ]]; then + uv run tests/check_metrics.py $JSON_METRICS \ + 'max(data["train/reward"]) > 0.0' + + # Clean up checkpoint directory after successful run to save space. + rm -rf "$CKPT_DIR" +fi diff --git a/tests/test_suites/nightly.txt b/tests/test_suites/nightly.txt index 867fe7400fc..d47fcd6b3f8 100644 --- a/tests/test_suites/nightly.txt +++ b/tests/test_suites/nightly.txt @@ -79,6 +79,7 @@ tests/test_suites/llm/grpo-qwen3-8b-base-1n8g-fp8-kvcache-megatron.sh # Non-colocated tests/test_suites/llm/grpo-llama3.1-8b-instruct-2n8g-fsdp2tp1-noncolocated.sh +tests/test_suites/llm/grpo-llama3.2-1b-instruct-2n8g-megatron_generation-noncolocated.sh # Nemotron Super 49B #https://github.com/NVIDIA-NeMo/RL/issues/1374 @@ -93,6 +94,8 @@ tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-fsdp2.sh tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-fsdp2-lora.sh tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-megatron-lora.sh tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-megatron-pack-cp.sh +tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-megatron_generation.sh +tests/test_suites/llm/grpo-nanov3-30BA3B-2n8g-megatron_generation-async-gym.sh # Lora tests/test_suites/llm/grpo-qwen3-8B-base-1n8g-fsdp2-lora.sh diff --git a/tests/unit/algorithms/test_grpo.py b/tests/unit/algorithms/test_grpo.py index 80cc5f89781..e34675c0326 100644 --- a/tests/unit/algorithms/test_grpo.py +++ b/tests/unit/algorithms/test_grpo.py @@ -51,12 +51,21 @@ EnvironmentReturn, ) from nemo_rl.experience.rollouts import calculate_rewards +from nemo_rl.models.generation.megatron import MegatronGeneration from nemo_rl.utils.timer import Timer from tests.unit.algorithms.utils import ( create_mock_batch, ) +def _mock_policy_generation() -> MagicMock: + """Generation-interface stand-in for grpo_train / async_grpo_train tests.""" + policy_generation = MagicMock(spec=MegatronGeneration) + policy_generation.requires_kv_scale_sync = False + policy_generation.get_logger_metrics.return_value = {} + return policy_generation + + @pytest.fixture def mock_grpo_components(): # Create mock components @@ -685,6 +694,20 @@ def resume(self): mock.remote = MagicMock(return_value=MagicMock()) return mock + @property + def prepare_for_refit(self): + """Prepare for refit - returns a remote-callable mock""" + mock = MagicMock() + mock.remote = MagicMock(return_value=MagicMock()) + return mock + + @property + def resume_after_refit(self): + """Resume after refit - returns a remote-callable mock""" + mock = MagicMock() + mock.remote = MagicMock(return_value=MagicMock()) + return mock + @property def stop(self): """Stop collection - returns a remote-callable mock""" @@ -1975,7 +1998,7 @@ def test_grpo_train_skips_reference_policy_logprobs(mock_grpo_components, train_ ): train_func( policy, - None, + _mock_policy_generation(), mock_grpo_components["train_dataloader"], mock_grpo_components["val_dataloader"], mock_grpo_components["tokenizer"], @@ -2005,7 +2028,7 @@ def test_grpo_train_skips_reference_policy_logprobs(mock_grpo_components, train_ ): train_func( policy, - None, + _mock_policy_generation(), mock_grpo_components["train_dataloader"], mock_grpo_components["val_dataloader"], mock_grpo_components["tokenizer"], @@ -2045,7 +2068,7 @@ def _run_single_grpo_train_step(mock_grpo_components, train_func, monkeypatch): ): train_func( policy, - None, + _mock_policy_generation(), mock_grpo_components["train_dataloader"], mock_grpo_components["val_dataloader"], mock_grpo_components["tokenizer"], @@ -2075,7 +2098,7 @@ def _run_single_grpo_train_step(mock_grpo_components, train_func, monkeypatch): ): train_func( policy, - None, + _mock_policy_generation(), mock_grpo_components["train_dataloader"], mock_grpo_components["val_dataloader"], mock_grpo_components["tokenizer"], @@ -2199,7 +2222,7 @@ def test_grpo_train_skips_prev_logprobs_when_force_on_policy_ratio( ): train_func( policy, - None, + _mock_policy_generation(), mock_grpo_components["train_dataloader"], mock_grpo_components["val_dataloader"], mock_grpo_components["tokenizer"], @@ -2229,7 +2252,7 @@ def test_grpo_train_skips_prev_logprobs_when_force_on_policy_ratio( ): train_func( policy, - None, + _mock_policy_generation(), mock_grpo_components["train_dataloader"], mock_grpo_components["val_dataloader"], mock_grpo_components["tokenizer"], @@ -2274,7 +2297,7 @@ def test_grpo_exit_on_max_steps(mock_grpo_components, train_func): with mock_async_grpo_infrastructure(mock_batch, mock_rollout_metrics): train_func( mock_grpo_components["policy"], - None, # policy_generation + _mock_policy_generation(), mock_grpo_components["train_dataloader"], mock_grpo_components["val_dataloader"], mock_grpo_components["tokenizer"], @@ -2302,7 +2325,7 @@ def test_grpo_exit_on_max_steps(mock_grpo_components, train_func): ): train_func( mock_grpo_components["policy"], - None, # policy_generation + _mock_policy_generation(), mock_grpo_components["train_dataloader"], mock_grpo_components["val_dataloader"], mock_grpo_components["tokenizer"], @@ -2356,7 +2379,7 @@ def test_grpo_exit_on_max_epochs(mock_grpo_components, train_func): # Run training train_func( mock_grpo_components["policy"], - None, # policy_generation + _mock_policy_generation(), mock_grpo_components["train_dataloader"], mock_grpo_components["val_dataloader"], mock_grpo_components["tokenizer"], @@ -2407,7 +2430,7 @@ def test_grpo_exit_on_timeout(mock_grpo_components, train_func, capsys): with mock_async_grpo_infrastructure(mock_batch, mock_rollout_metrics): train_func( mock_grpo_components["policy"], - None, # policy_generation + _mock_policy_generation(), mock_grpo_components["train_dataloader"], mock_grpo_components["val_dataloader"], mock_grpo_components["tokenizer"], @@ -2434,7 +2457,7 @@ def test_grpo_exit_on_timeout(mock_grpo_components, train_func, capsys): ): train_func( mock_grpo_components["policy"], - None, # policy_generation + _mock_policy_generation(), mock_grpo_components["train_dataloader"], mock_grpo_components["val_dataloader"], mock_grpo_components["tokenizer"], diff --git a/tests/unit/models/generation/test_megatron_generation.py b/tests/unit/models/generation/test_megatron_generation.py new file mode 100644 index 00000000000..ebfac920d03 --- /dev/null +++ b/tests/unit/models/generation/test_megatron_generation.py @@ -0,0 +1,462 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +from copy import deepcopy + +import pytest +import ray +import torch + +from nemo_rl.algorithms.grpo import refit_policy_generation +from nemo_rl.algorithms.utils import get_tokenizer +from nemo_rl.distributed.batched_data_dict import BatchedDataDict +from nemo_rl.distributed.virtual_cluster import RayVirtualCluster +from nemo_rl.models.generation.megatron import MegatronGeneration +from nemo_rl.models.policy import PolicyConfig +from nemo_rl.models.policy.lm_policy import Policy + +model_name = "Qwen/Qwen3-0.6B" + +basic_megatron_test_config: PolicyConfig = { + "model_name": model_name, + "tokenizer": {"name": model_name}, + "generation_batch_size": 2, + "train_global_batch_size": 4, + "train_micro_batch_size": 2, + "learning_rate": 5e-6, + "logprob_batch_size": 2, + # The dynamic inference engine requires fp16/bf16 + "precision": "bfloat16", + "offload_optimizer_for_logprob": False, + "dtensor_cfg": {"enabled": False}, + "dynamic_batching": {"enabled": False}, + "sequence_packing": {"enabled": False}, + "megatron_cfg": { + "enabled": True, + "empty_unused_memory_level": 0, + "activation_checkpointing": False, + "converter_type": "Qwen2ForCausalLM", # Qwen2 converter is compatible with Qwen3 + "tensor_model_parallel_size": 1, + "expert_tensor_parallel_size": 1, + "expert_model_parallel_size": 1, + "pipeline_model_parallel_size": 1, + "num_layers_in_first_pipeline_stage": None, + "num_layers_in_last_pipeline_stage": None, + "context_parallel_size": 1, + "pipeline_dtype": "bfloat16", + "sequence_parallel": False, + "freeze_moe_router": True, + "moe_router_dtype": "fp64", + "moe_router_load_balancing_type": "none", + "moe_router_bias_update_rate": 0.0, + "moe_permute_fusion": False, + "moe_enable_deepep": False, + "moe_token_dispatcher_type": "alltoall", + "moe_shared_expert_overlap": False, + "apply_rope_fusion": True, + "bias_activation_fusion": True, + "moe_per_layer_logging": False, + "gradient_accumulation_fusion": False, + "train_iters": 100, + "optimizer": { + "optimizer": "adam", + "lr": 5.0e-6, + "min_lr": 5.0e-7, + "weight_decay": 0.01, + "bf16": True, + "fp16": False, + "params_dtype": "float32", + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "adam_eps": 1e-8, + "use_distributed_optimizer": True, + "use_precision_aware_optimizer": True, + "clip_grad": 1.0, + "optimizer_cpu_offload": False, + "optimizer_offload_fraction": 0.0, + }, + "scheduler": { + "start_weight_decay": 0.01, + "end_weight_decay": 0.01, + "weight_decay_incr_style": "constant", + "lr_decay_style": "constant", + "lr_decay_iters": None, + "lr_warmup_iters": 50, + "lr_warmup_init": 5.0e-7, + }, + "distributed_data_parallel_config": { + "grad_reduce_in_fp32": False, + "overlap_grad_reduce": True, + "overlap_param_gather": False, + "data_parallel_sharding_strategy": "optim_grads_params", + }, + }, + "draft": {"enabled": False}, + "optimizer": None, + "scheduler": None, + "max_grad_norm": 1.0, + "generation": { + "backend": "megatron", + "model_name": model_name, + "max_new_tokens": 16, # Small number of tokens for testing + "temperature": 1.0, + "top_p": 1.0, + "top_k": None, + "stop_token_ids": None, + "stop_strings": None, + "colocated": { + "enabled": False, + "resources": {"gpus_per_node": None, "num_nodes": None}, + }, + "mcore_generation_config": { + "async_engine": False, + "max_model_len": 1024, + "cuda_graph_impl": "local", + "inference_cuda_graph_scope": "block", + "buffer_size_gb": 10, + "num_cuda_graphs": 4, + "block_size_tokens": 256, + "use_cuda_graphs_for_non_decode_steps": True, + "enable_chunked_prefill": True, + "enable_prefix_caching": False, + "max_tokens": 16384, + "kv_cache_management_mode": "persist", + "materialize_only_last_token_logits": True, + "num_speculative_tokens": 0, + "refit_backend": "gloo", # not nvshmem: its NVLS multicast init is unavailable in CI + "parsers": [], + "expose_http_server": False, + }, + }, +} + + +@pytest.fixture(scope="function") +def cluster(): + """A 1-node, 2-GPU virtual cluster (enough for tp/pp up to 2).""" + virtual_cluster = RayVirtualCluster( + bundle_ct_per_node_list=[2], + use_gpus=True, + max_colocated_worker_groups=2, + num_gpus_per_node=2, + name="megatron-generation-test-cluster", + ) + yield virtual_cluster + virtual_cluster.shutdown() + + +@pytest.fixture(scope="function") +def policy_cluster_separate(): + """A dedicated 1-GPU cluster for the training policy in the non-colocated test.""" + virtual_cluster = RayVirtualCluster( + bundle_ct_per_node_list=[1], + use_gpus=True, + max_colocated_worker_groups=1, + num_gpus_per_node=1, + name="megatron-generation-test-policy-cluster", + ) + yield virtual_cluster + try: + virtual_cluster.shutdown() + except Exception as e: + print(f"Error during policy_cluster_separate shutdown: {e}") + + +@pytest.fixture(scope="function") +def tokenizer(): + """Initialize tokenizer for the test model.""" + return get_tokenizer({"name": model_name}) + + +@pytest.fixture(scope="function") +def test_input_data(tokenizer): + """Create test input data for inference.""" + test_prompts = [ + "Hello, my name is", + "The capital of France is", + ] + encodings = tokenizer( + test_prompts, + padding="max_length", + max_length=20, + truncation=True, + return_tensors="pt", + padding_side="right", + ) + input_lengths = encodings["attention_mask"].sum(dim=1).to(torch.int32) + return BatchedDataDict( + { + "input_ids": encodings["input_ids"], + "input_lengths": input_lengths, + } + ) + + +def _assert_valid_generation_output(outputs, input_data, require_generation=True): + """Assert the GenerationOutputSpec contract produced by the Megatron worker.""" + required_keys = [ + "output_ids", + "logprobs", + "generation_lengths", + "unpadded_sequence_lengths", + ] + for key in required_keys: + assert key in outputs, f"{key} not found in generation output" + + batch_size = len(input_data["input_ids"]) + assert all(outputs[key].shape[0] == batch_size for key in required_keys), ( + "Wrong batch size in generation output" + ) + # output_ids and logprobs are packed on the same padded width. + assert outputs["output_ids"].shape == outputs["logprobs"].shape + + if require_generation: + assert (outputs["generation_lengths"] > 0).all(), ( + "Some samples generated nothing" + ) + + # length identity: total (unpadded) == prompt length + generated length. + expected_unpadded = input_data["input_lengths"].cpu().to(torch.int64) + outputs[ + "generation_lengths" + ].cpu().to(torch.int64) + assert torch.equal( + outputs["unpadded_sequence_lengths"].cpu().to(torch.int64), expected_unpadded + ), "unpadded_sequence_lengths != input_lengths + generation_lengths" + + # logprob offset: position 0 is always the 0.0 placeholder. + assert torch.allclose( + outputs["logprobs"][:, 0], + torch.zeros(batch_size, dtype=outputs["logprobs"].dtype), + ), "logprobs[:, 0] should be the 0.0 placeholder" + + +async def _generate_async(mg, tokenizer, test_input_data, greedy=False): + """Drive ``generate_async`` over single-sample microbatches and reassemble in order.""" + collected = [] + for single_item_input in test_input_data.make_microbatch_iterator( + microbatch_size=1 + ): + async for original_idx, single_item_output in mg.generate_async( + single_item_input, greedy=greedy + ): + # The mcore coordinator only accepts requests on DP rank 0. + assert single_item_output["gen_leader_worker_idx"] == [0] + collected.append((original_idx, single_item_output)) + + collected.sort(key=lambda x: x[0]) + outputs = [item for _, item in collected] + pad_token_id = mg.cfg.get("_pad_token_id", tokenizer.pad_token_id) + return BatchedDataDict.from_batches( + outputs, + pad_value_dict={"output_ids": pad_token_id, "logprobs": 0.0}, + ) + + +@pytest.mark.mcore +@pytest.mark.timeout(900) +@pytest.mark.parametrize( + "tensor_parallel_size,pipeline_parallel_size", + [(1, 1), (2, 1), (1, 2)], +) +def test_megatron_policy_generation( + cluster, test_input_data, tokenizer, tensor_parallel_size, pipeline_parallel_size +): + """Standalone Megatron generation across tp/pp.""" + if cluster.num_gpus_per_node < tensor_parallel_size * pipeline_parallel_size: + pytest.skip( + f"Need {tensor_parallel_size * pipeline_parallel_size} GPUs for " + f"tp={tensor_parallel_size} pp={pipeline_parallel_size}" + ) + + config = deepcopy(basic_megatron_test_config) + config["megatron_cfg"]["tensor_model_parallel_size"] = tensor_parallel_size + config["megatron_cfg"]["pipeline_model_parallel_size"] = pipeline_parallel_size + # config-level stop string, unioned with the per-sample stop strings below. + config["generation"]["stop_strings"] = [""] + + mg = None + try: + mg = MegatronGeneration(config=config, tokenizer=tokenizer, cluster=cluster) + + # greedy decoding: full output contract + non-empty text + outputs = mg.generate(test_input_data, greedy=True) + _assert_valid_generation_output(outputs, test_input_data) + generated_texts = tokenizer.batch_decode( + outputs["output_ids"], skip_special_tokens=True + ) + assert all(len(t) > 0 for t in generated_texts), "Some greedy texts are empty" + + # sampling (non-greedy) path still produces a valid contract + sampled = mg.generate(test_input_data, greedy=False) + _assert_valid_generation_output(sampled, test_input_data) + + # per-sample stop strings are merged with the config stop string (may stop early, + # so don't require a generated token) + data_with_stops = BatchedDataDict( + { + "input_ids": test_input_data["input_ids"], + "input_lengths": test_input_data["input_lengths"], + "stop_strings": [["."], ["."]], + } + ) + stopped = mg.generate(data_with_stops, greedy=True) + _assert_valid_generation_output( + stopped, test_input_data, require_generation=False + ) + + # lifecycle: leave generation mode, re-enter, and generate again + assert mg.finish_generation() is True + assert mg.prepare_for_generation() is True + reentered = mg.generate(test_input_data, greedy=True) + _assert_valid_generation_output(reentered, test_input_data) + finally: + if mg is not None: + mg.shutdown() + gc.collect() + torch.cuda.empty_cache() + + +@pytest.mark.mcore +@pytest.mark.asyncio +@pytest.mark.timeout(900) +async def test_megatron_policy_generation_async(cluster, test_input_data, tokenizer): + """Standalone Megatron async generation.""" + config = deepcopy(basic_megatron_test_config) + config["generation"]["mcore_generation_config"]["async_engine"] = True + mg = None + try: + mg = MegatronGeneration(config=config, tokenizer=tokenizer, cluster=cluster) + outputs = await _generate_async(mg, tokenizer, test_input_data, greedy=True) + _assert_valid_generation_output(outputs, test_input_data) + generated_texts = tokenizer.batch_decode( + outputs["output_ids"], skip_special_tokens=True + ) + assert all(len(t) > 0 for t in generated_texts), "Some async texts are empty" + finally: + if mg is not None: + mg.shutdown() + gc.collect() + torch.cuda.empty_cache() + + +@pytest.mark.mcore +@pytest.mark.timeout(900) +def test_megatron_generation_colocated(cluster, test_input_data, tokenizer): + """Colocated Megatron generation: wrap an existing training policy without owning it.""" + config = deepcopy(basic_megatron_test_config) + config["generation"]["colocated"]["enabled"] = True + + # construction guard: exactly one of `cluster` / `policy` is required + with pytest.raises(AssertionError): + MegatronGeneration(config=config, tokenizer=tokenizer) + with pytest.raises(AssertionError): + MegatronGeneration( + config=config, tokenizer=tokenizer, cluster=cluster, policy=object() + ) + + policy = None + try: + policy = Policy(cluster=cluster, config=config, tokenizer=tokenizer) + megatron_cfg_before = deepcopy(config["megatron_cfg"]) + + mg = MegatronGeneration(policy=policy, config=config, tokenizer=tokenizer) + # colocated wrapper reuses the training policy and must not own it + assert mg._owns_policy is False + # the colocated path must NOT merge mcore_generation_config into megatron_cfg + assert "max_tokens" not in config["megatron_cfg"] + assert config["megatron_cfg"] == megatron_cfg_before + + mg.prepare_for_generation() + outputs = mg.generate(test_input_data, greedy=True) + _assert_valid_generation_output(outputs, test_input_data) + + # ownership guard: shutdown is a no-op, so the wrapped policy keeps generating + assert mg.shutdown() is True + after_shutdown = mg.generate(test_input_data, greedy=True) + _assert_valid_generation_output(after_shutdown, test_input_data) + finally: + if policy is not None: + policy.shutdown() + gc.collect() + torch.cuda.empty_cache() + + +@pytest.mark.mcore +@pytest.mark.timeout(900) +def test_megatron_generation_non_colocated_refit( + policy_cluster_separate, test_input_data, tokenizer +): + """Non-colocated Megatron generation.""" + generation_cluster = RayVirtualCluster( + bundle_ct_per_node_list=[1], + use_gpus=True, + max_colocated_worker_groups=1, + num_gpus_per_node=1, + name="megatron-generation-test-generation-cluster", + ) + if ( + policy_cluster_separate.num_gpus_per_node < 1 + or generation_cluster.num_gpus_per_node < 1 + ): + pytest.skip("Need at least two GPUs across separate clusters") + + config = deepcopy(basic_megatron_test_config) + + policy = None + mg = None + try: + policy = Policy( + cluster=policy_cluster_separate, config=config, tokenizer=tokenizer + ) + mg = MegatronGeneration( + config=config, tokenizer=tokenizer, cluster=generation_cluster + ) + + # init the refit collective on both sides. + ip, port = policy_cluster_separate.get_master_address_and_port() + train_world_size = policy_cluster_separate.world_size() + world_size = train_world_size + generation_cluster.world_size() + refit_backend = config["generation"]["mcore_generation_config"]["refit_backend"] + futures_train = policy.init_collective_mcore_generation( + ip, port, world_size, rank_offset=0, refit_backend=refit_backend + ) + futures_inference = mg.init_collective( + ip, + port, + world_size, + train_world_size=train_world_size, + refit_backend=refit_backend, + ) + ray.get(futures_train + futures_inference) + + # refit the inference engine from the training weights, then generate + refit_policy_generation(policy, mg, False) + outputs = mg.generate(test_input_data, greedy=True) + _assert_valid_generation_output(outputs, test_input_data) + generated_texts = tokenizer.batch_decode( + outputs["output_ids"], skip_special_tokens=True + ) + assert all(len(t) > 0 for t in generated_texts), "Some texts are empty" + finally: + if mg is not None: + mg.shutdown() + if policy is not None: + policy.shutdown() + try: + generation_cluster.shutdown() + except Exception as e: + print(f"Error during generation_cluster shutdown: {e}") + gc.collect() + torch.cuda.empty_cache() diff --git a/tests/unit/models/megatron/test_megatron_setup.py b/tests/unit/models/megatron/test_megatron_setup.py index 8662dca49a2..e034b5cb0f3 100644 --- a/tests/unit/models/megatron/test_megatron_setup.py +++ b/tests/unit/models/megatron/test_megatron_setup.py @@ -1577,13 +1577,20 @@ def _apply_patches(self, request): request.addfinalizer(p.stop) return mocks + @staticmethod + def _make_model_cfg_mock() -> MagicMock: + """Mock megatron provider that tolerates __post_init__().""" + model_cfg = MagicMock() + model_cfg.__post_init__ = MagicMock() + return model_cfg + def test_megatron_lm_passes_hf_config_overrides_to_autoconfig(self, request): """hf_config_overrides must be forwarded to AutoConfig.from_pretrained for megatron_lm.""" from nemo_rl.models.megatron.setup import setup_model_config self._apply_patches(request) - mock_model_cfg = MagicMock() + mock_model_cfg = self._make_model_cfg_mock() mock_provider = MagicMock() mock_provider.to_megatron_provider.return_value = mock_model_cfg @@ -1622,7 +1629,7 @@ def test_megatron_lm_no_overrides_calls_autoconfig_without_extra_kwargs( self._apply_patches(request) mock_provider = MagicMock() - mock_provider.to_megatron_provider.return_value = MagicMock() + mock_provider.to_megatron_provider.return_value = self._make_model_cfg_mock() config = { "pretrained_checkpoint": {"format": "megatron_lm", "path": "/ckpt"}, @@ -1665,7 +1672,7 @@ def test_megatron_bridge_with_hf_config_overrides_warns(self, tmp_path, request) } mock_cfg = MagicMock() - mock_cfg.model = MagicMock() + mock_cfg.model = self._make_model_cfg_mock() with patch("nemo_rl.models.megatron.setup.ConfigContainer") as mock_cc: mock_cc.from_yaml.return_value = mock_cfg @@ -1701,7 +1708,7 @@ def test_megatron_bridge_without_hf_config_overrides_no_warning( } mock_cfg = MagicMock() - mock_cfg.model = MagicMock() + mock_cfg.model = self._make_model_cfg_mock() with patch("nemo_rl.models.megatron.setup.ConfigContainer") as mock_cc: mock_cc.from_yaml.return_value = mock_cfg diff --git a/tests/unit/models/policy/test_megatron_worker.py b/tests/unit/models/policy/test_megatron_worker.py index df5d85e6158..a58b668928f 100644 --- a/tests/unit/models/policy/test_megatron_worker.py +++ b/tests/unit/models/policy/test_megatron_worker.py @@ -35,6 +35,7 @@ from nemo_rl.distributed.batched_data_dict import BatchedDataDict from nemo_rl.distributed.virtual_cluster import RayVirtualCluster from nemo_rl.models.generation import configure_generation_config +from nemo_rl.models.generation.megatron import MegatronGeneration from nemo_rl.models.policy import PolicyConfig from nemo_rl.models.policy.lm_policy import Policy from nemo_rl.utils.checkpoint import CheckpointManager @@ -164,13 +165,22 @@ def create_megatron_test_config( "stop_token_ids": None, "stop_strings": None, "mcore_generation_config": { + "async_engine": False, + "max_model_len": 1024, "buffer_size_gb": 2, "num_cuda_graphs": 16, "block_size_tokens": 1024, "use_cuda_graphs_for_non_decode_steps": True, "enable_chunked_prefill": True, - "unified_memory_level": 0, "max_tokens": 65536, + "cuda_graph_impl": "local", + "enable_prefix_caching": False, + "kv_cache_management_mode": "persist", + "materialize_only_last_token_logits": True, + "num_speculative_tokens": 0, + "refit_backend": "gloo", # not nvshmem: its NVLS multicast init is unavailable in CI + "parsers": [], + "expose_http_server": False, }, "colocated": { "enabled": True, @@ -598,6 +608,10 @@ def generation_setup(request, tiny_llama_model_path): init_reference_model=False, ) + generation = MegatronGeneration( + config=config, tokenizer=tokenizer, policy=policy + ) + # Create test data print("Creating test batch...") torch.manual_seed(42) @@ -627,7 +641,7 @@ def generation_setup(request, tiny_llama_model_path): } ) - yield policy, cluster, data, prompts + yield policy, generation, cluster, data, prompts except Exception as e: print(f"Error during generation setup: {e}") @@ -653,7 +667,7 @@ def generation_setup(request, tiny_llama_model_path): ) def test_megatron_policy_generation(generation_setup): """Test Megatron policy generation with different backends.""" - policy, cluster, data, prompts = generation_setup + policy, generation, cluster, data, prompts = generation_setup # Verify resources were created properly assert policy is not None, "Generation policy was not created properly" @@ -662,11 +676,11 @@ def test_megatron_policy_generation(generation_setup): # Call prepare_for_generation print("Preparing for generation...") - policy.prepare_for_generation() + generation.prepare_for_generation() # Generate text print("Generating text...") - results = policy.generate(data, greedy=True) + results = generation.generate(data, greedy=True) # Verify results assert "output_ids" in results, "Generation results should contain 'output_ids'" @@ -686,7 +700,7 @@ def test_megatron_policy_generation(generation_setup): # Call finish_generation print("Finishing generation...") - policy.finish_generation() + generation.finish_generation() @pytest.fixture diff --git a/tests/unit/reference_configs/grpo_math_1B.yaml b/tests/unit/reference_configs/grpo_math_1B.yaml index b5b617619bd..8c8e2764e2a 100644 --- a/tests/unit/reference_configs/grpo_math_1B.yaml +++ b/tests/unit/reference_configs/grpo_math_1B.yaml @@ -102,7 +102,7 @@ policy: tokenizer: name: ${policy.model_name} ## specify if you'd like to use a tokenizer different from the model's default chat_template_kwargs: null # can be used to pass kwargs to the chat template, e.g., enable_thinking=true - hf_config_overrides: {} + hf_config_overrides: {} train_global_batch_size: 512 train_micro_batch_size: 4 generation_batch_size: 32 # Only used when generating using HF backend @@ -141,7 +141,7 @@ policy: dropout_position: "post" # Where to apply dropout: "pre" (before LoRA) or "post" (after LoRA) lora_A_init: "xavier" # Initialization method for LoRA A matrix: "xavier" or "uniform" use_triton: true # Use Triton-optimized kernels for LoRA (faster but requires flash-attn). Disable when tensor_parallel_size > 1 - + megatron_cfg: enabled: false force_reconvert_from_hf: False # Set to True to force reconvert of the model from Hugging Face @@ -239,7 +239,7 @@ policy: use_custom_fsdp: false data_parallel_sharding_strategy: "optim_grads_params" - fp8_cfg: + fp8_cfg: enabled: false fp8: "e4m3" fp8_recipe: "blockwise" @@ -254,7 +254,7 @@ policy: num_layers: null aux_layer_indices: null - # See docs/design-docs/sequence-packing-and-dynamic-batching.md + # See docs/design-docs/sequence-packing-and-dynamic-batching.md # for more details on dynamic batching and sequence packing. dynamic_batching: enabled: False @@ -305,13 +305,23 @@ policy: stop_token_ids: null stop_strings: null mcore_generation_config: - buffer_size_gb: 10 # Total buffer size is 2x this value (active requests + paused requests) + async_engine: false + max_model_len: ${policy.max_total_sequence_length} + cuda_graph_impl: "local" # CUDA-graph implementation. Options: "none", "local", "transformer_engine", "full_iteration". + inference_cuda_graph_scope: "block" # Inference CUDA-graph scope. Options: "none", "layer", "block". + buffer_size_gb: 10 num_cuda_graphs: 4 # Number of CUDA graphs to pre-compile for different batch sizes block_size_tokens: 256 # Size of each KV cache block in tokens (affects memory granularity) use_cuda_graphs_for_non_decode_steps: true # Enable CUDA graphs for prefill/context processing enable_chunked_prefill: true # Split long prefills into chunks for better memory management - unified_memory_level: 0 # Unified memory usage level (0=disabled, higher values enable more aggressive paging) + enable_prefix_caching: false # Reuse KV blocks across requests sharing a prompt prefix. max_tokens: 16384 # Maximum number of tokens to use in a single step. Analogous to vllm's max_num_batched_tokens + kv_cache_management_mode: "persist" # KV cache lifecycle across suspend/resume. Options: "persist", "offload". To select "recompute", set grpo.async_grpo.recompute_kv_cache_after_weight_updates=true. + materialize_only_last_token_logits: true # Materialize logits only for the last token of each sequence during decode. + num_speculative_tokens: 0 + refit_backend: "nvshmem" # Copy-service backend for non-colocated megatron weight refit. Options: "gloo" or "nvshmem". + parsers: [] # OpenAI tool-call / response parser names exposed by the HTTP server (only used if expose_http_server=true). + expose_http_server: false # Start an OpenAI-compatible HTTP server alongside the persistent engine. Required for NeMo-Gym. vllm_cfg: async_engine: false precision: ${policy.precision} diff --git a/tests/unit/test_recipes_and_test_suites.py b/tests/unit/test_recipes_and_test_suites.py index 63225fbbd5c..4e95c20db2d 100644 --- a/tests/unit/test_recipes_and_test_suites.py +++ b/tests/unit/test_recipes_and_test_suites.py @@ -234,7 +234,7 @@ def test_all_recipe_yamls_accounted_for_in_test_suites( ) -def test_nightly_compute_stays_below_2000_hours(nightly_test_suite, tracker): +def test_nightly_compute_stays_below_2080_hours(nightly_test_suite, tracker): command = f"DRYRUN=1 HF_HOME=... HF_DATASETS_CACHE=... CONTAINER= ACCOUNT= PARTITION= ./tools/launch {' '.join(nightly_test_suite)}" print(f"Running command: {command}") @@ -266,8 +266,8 @@ def test_nightly_compute_stays_below_2000_hours(nightly_test_suite, tracker): f"Last line of output was not as expected: '{last_line}'" ) total_gpu_hours = float(last_line.split(":")[-1].strip()) - assert total_gpu_hours <= 2000, ( - f"Total GPU hours exceeded 2000: {last_line}. We should revisit the test suites to reduce the total GPU hours." + assert total_gpu_hours <= 2080, ( + f"Total GPU hours exceeded 2080: {last_line}. We should revisit the test suites to reduce the total GPU hours." ) tracker.track("total_nightly_gpu_hours", total_gpu_hours)