diff --git a/megatron/rl/rl_utils.py b/megatron/rl/rl_utils.py index 8a564315dc3..4d018217cec 100644 --- a/megatron/rl/rl_utils.py +++ b/megatron/rl/rl_utils.py @@ -1973,8 +1973,8 @@ def megatron_rl_inference_mode( logger.debug(f"[{dist.get_rank()}] Entering inference mode") - # Change cudagraph scope for inference (empty list = full-layer capture) - model[0].config.cuda_graph_scope = [] + # Set cudagraph scope for inference. + model[0].config.cuda_graph_scope = args.cuda_graph_scope model[0].config.cuda_graph_impl = "local" # If we get a lower precision wrapper, we go one object deeper. @@ -2031,7 +2031,8 @@ def megatron_rl_inference_mode( # Reset drop_and_pad leaked from inference decode set_decode_expert_padding(unwrap_model(model[0]), set_to=False) - # Restore partial capture cudagraph scope for training if this is MoE + # Restore cudagraph scope for training. + # MoE partial capture requires specific scopes that aren't user-facing. if args.num_experts is not None: model[0].config.cuda_graph_scope = [ CudaGraphScope.mamba, @@ -2039,6 +2040,10 @@ def megatron_rl_inference_mode( CudaGraphScope.moe_router, CudaGraphScope.moe_preprocess, ] + else: + model[0].config.cuda_graph_scope = [ + s for s in args.cuda_graph_scope if s != CudaGraphScope.full_iteration_inference + ] # Switch MoE layers to partial CUDA graph capture for training if args.rl_training_cuda_graphs and args.num_experts is not None: