diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py
index e550429ce2c..abeef9bf0b3 100644
--- a/nemo_rl/algorithms/grpo.py
+++ b/nemo_rl/algorithms/grpo.py
@@ -554,6 +554,8 @@ def init_train_dataloader(dataset, suffix: str = ""):
         policy_config["megatron_cfg"]["train_iters"] = total_train_iters
 
     # Define initialization functions that will be used in all paths
+    init_reference_model = master_config["loss_fn"]["reference_policy_kl_penalty"] > 0
+
     def init_policy():
         """Initialize policy training workers."""
         t0 = time.perf_counter()
@@ -565,6 +567,7 @@ def init_policy():
             weights_path=weights_path,
             optimizer_path=optimizer_path,
             init_optimizer=True,
+            init_reference_model=init_reference_model,
         )
         return p, time.perf_counter() - t0