diff --git a/megatron/inference/utils.py b/megatron/inference/utils.py
index 60b6d9bb0c0..91a9d954617 100644
--- a/megatron/inference/utils.py
+++ b/megatron/inference/utils.py
@@ -367,7 +367,7 @@ def get_inference_config_from_model_and_args(model: MegatronModule, args):
         mamba_inference_state_config=mamba_inference_state_config,
         pg_collection=pg_collection,
         use_flashinfer_fused_rope=args.use_flashinfer_fused_rope,
-        materialize_only_last_token_logits=(not args.return_log_probs),
+        materialize_only_last_token_logits=not (args.return_log_probs and not args.skip_prompt_log_probs),
         track_generated_token_events=args.inference_dynamic_batching_track_generated_token_events,
         track_paused_request_events=args.inference_dynamic_batching_track_paused_request_events,
         enable_chunked_prefill=args.enable_chunked_prefill,
diff --git a/megatron/rl/inference/megatron.py b/megatron/rl/inference/megatron.py
index d9cad5c7467..c7f8b47a26c 100644
--- a/megatron/rl/inference/megatron.py
+++ b/megatron/rl/inference/megatron.py
@@ -97,6 +97,10 @@ async def launch(cls, model: GPTModel, **kwargs):
                 "WARNING: Tokenizer has no BOS token so prompt will not have BOS token",
             )
 
+        # RL needs log probs, but not prompt log probs.
+        args.return_log_probs = True
+        args.skip_prompt_log_probs = True
+
         inference_engine: DynamicInferenceEngine = get_dynamic_inference_engine(model=model)
         dp_addr = await inference_engine.start_listening_to_data_parallel_coordinator(
             inference_coordinator_port=41521, launch_inference_coordinator=True,