[rollout] fix: fix async llm config passing

.github/workflows/sandbox.yml

@@ -47,5 +47,5 @@ jobs:
           pip3 install vllm==0.5.4
       - name: Running sandbox tests on 8 L20 GPUs
         run: |
-          cd tests/sandbox
+          cd tests/utils/reward_score
           pytest -s -x .

.github/workflows/ray_gpu_test.yml → .github/workflows/single_controller_unit_tests.yml

@@ -1,4 +1,4 @@
-name: ray_gpu
+name: single_controller_unit_tests
 
 on:
   # Trigger the workflow on push or pull request,
@@ -26,7 +26,7 @@ permissions:
   contents: read
 
 jobs:
-  ray_gpu:
+  single_controller_unit_tests:
     runs-on: [L20x8]
     timeout-minutes: 10 # Increase this timeout value as needed
     env:

verl/trainer/ppo/ray_trainer.py

@@ -772,7 +772,7 @@ def init_workers(self):
 
             self.async_rollout_mode = True
             self.async_rollout_manager = AsyncLLMServerManager(
-                config=self.config.actor_rollout_ref,
+                config=self.config,
                 worker_group=self.actor_rollout_wg,
             )
 

verl/utils/checkpoint/checkpoint_manager.py

@@ -150,7 +150,7 @@ def find_latest_ckpt_path(path, directory_format="global_step_{}"):
 
     tracker_file = get_checkpoint_tracker_filename(path)
     if not os.path.exists(tracker_file):
-        print("Checkpoint tracker file does not exist: %s", tracker_file)
+        print(f"Checkpoint tracker file does not exist: {tracker_file}")
         return None
 
     with open(tracker_file, "rb") as f: