[rollout] feat: follow OpenAI tool calling schema in chat scheduler

.github/workflows/vllm.yml

@@ -66,6 +66,7 @@ jobs:
       - name: Download Model to Use
         run: |
           huggingface-cli download Qwen/Qwen2.5-0.5B-Instruct
+          huggingface-cli download Qwen/Qwen2.5-1.5B-Instruct
           huggingface-cli download 'Qwen/Qwen2-7B-Instruct'
           huggingface-cli download 'deepseek-ai/deepseek-llm-7b-chat'
           export HF_HUB_OFFLINE=1
@@ -94,4 +95,4 @@ jobs:
       - name: Running multi-turn rollout tests on 8 L20 GPUs
         run: |
           pip3 install --upgrade vllm==0.8.3 tensordict==0.7.2
-          python3 tests/workers/rollout/test_vllm_multi_turn.py
+          pytest -svvv tests/workers/rollout/test_vllm_chat_scheduler.py

examples/grpo_trainer/run_qwen2-7b_seq_balance.sh

@@ -9,7 +9,6 @@ rollout_name="sglang" # sglang or vllm
 if [ "$rollout_mode" = "async" ]; then
     export VLLM_USE_V1=1
     return_raw_chat="True"
-    chat_scheduler=examples.ppo_trainer.naive_chat_scheduler.NaiveChatCompletionScheduler
 fi
 
 python3 -m verl.trainer.main_ppo \
@@ -38,7 +37,7 @@ python3 -m verl.trainer.main_ppo \
     actor_rollout_ref.rollout.tensor_model_parallel_size=2 \
     actor_rollout_ref.rollout.name=$rollout_name \
     actor_rollout_ref.rollout.mode=$rollout_mode \
-    actor_rollout_ref.rollout.chat_scheduler=$chat_scheduler \
+    actor_rollout_ref.rollout.multi_turn.format=hermes \
     actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \
     actor_rollout_ref.rollout.n=5 \
     actor_rollout_ref.ref.fsdp_config.param_offload=True \

examples/ppo_trainer/run_qwen2-7b_seq_balance.sh

@@ -12,7 +12,6 @@ test_files="['$gsm8k_test_path', '$math_test_path']"
 rollout_mode="sync"
 if [ "$rollout_mode" = "async" ]; then
     return_raw_chat="True"
-    chat_scheduler=examples.ppo_trainer.naive_chat_scheduler.NaiveChatCompletionScheduler
 fi
 
 python3 -m verl.trainer.main_ppo \
@@ -38,7 +37,7 @@ python3 -m verl.trainer.main_ppo \
     actor_rollout_ref.rollout.tensor_model_parallel_size=2 \
     actor_rollout_ref.rollout.name=vllm \
     actor_rollout_ref.rollout.mode=$rollout_mode \
-    actor_rollout_ref.rollout.chat_scheduler=$chat_scheduler \
+    actor_rollout_ref.rollout.multi_turn.format=hermes \
     actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \
     actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=24000 \
     critic.optim.lr=1e-5 \

tests/workers/rollout/async_rollout_utils.py

@@ -11,8 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import os
-from typing import Any, Dict
 
 import ray
 from omegaconf import DictConfig
@@ -24,12 +22,7 @@
 from verl.workers.rollout.async_server import AsyncLLMServerManager
 
 
-def init_async_rollout_manager(config: DictConfig, scheduler_kwargs: Dict[str, Any] = None) -> AsyncLLMServerManager:
-    # make openai client happy
-    os.environ["no_proxy"] = ""
-    os.environ["http_proxy"] = ""
-    os.environ["https_proxy"] = ""
-
+def init_async_rollout_manager(config: DictConfig) -> AsyncLLMServerManager:
     # =========================== 1. Create hybrid ActorRollout workers ===========================
     role_worker_mapping = {
         Role.ActorRollout: ray.remote(AsyncActorRolloutRefWorker),
@@ -61,9 +54,8 @@ def init_async_rollout_manager(config: DictConfig, scheduler_kwargs: Dict[str, A
 
     # =========================== 2. Create AsyncLLMServerManager  ===========================
     async_rollout_manager = AsyncLLMServerManager(
-        config=config.actor_rollout_ref,
+        config=config,
         worker_group=actor_rollout_wg,
-        scheduler_kwargs=scheduler_kwargs,
     )
 
     return async_rollout_manager