From 953eae626dc5b91514d8ee6a5423ca3aaf67306d Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Mon, 4 Aug 2025 16:24:53 -0700 Subject: [PATCH 01/54] change the default bash of qqwen 2.5 --- examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh index 662723df445..505f5cc41cf 100644 --- a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh +++ b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh @@ -31,9 +31,10 @@ python3 -m verl.trainer.main_ppo \ actor_rollout_ref.actor.fsdp_config.param_offload=False \ actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=2 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ actor_rollout_ref.rollout.name=sglang \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ + actor_rollout_ref.rollout.multi_stage_wake_up=True \ actor_rollout_ref.rollout.n=16 \ actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ actor_rollout_ref.ref.fsdp_config.param_offload=True \ From 0384aef9dc37e87c27a8f370309844c5d837c461 Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Mon, 4 Aug 2025 17:20:35 -0700 Subject: [PATCH 02/54] add engine support --- verl/workers/rollout/sglang_rollout/sglang_rollout.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index 6fae513857f..fd50592f943 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -1059,16 +1059,6 @@ async def _handle_pending_state(self, _req: AsyncRolloutRequest) -> AsyncRollout interaction = self.interaction_map[interaction_name] await interaction.start_interaction(_req.request_id, **interaction_kwargs) - @GPUMemoryLogger(role="sglang rollout", logger=logger) - @torch.no_grad() - def generate_sequences_with_tools(self, prompts: DataProto, **kwargs) -> DataProto: - logger.warning( - "`generate_sequences_with_tools` is deprecated, please use `generate_sequences(...)`", - DeprecationWarning, - stacklevel=2, - ) - return self._req_level_generate_sequences(prompts, **kwargs) - @GPUMemoryLogger(role="sglang rollout", logger=logger) @torch.no_grad() def _req_level_generate_sequences(self, prompts: DataProto, **kwargs) -> DataProto: From a0ab8b023e3c10ba4fffb9b2ff6e27aa38537243 Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Mon, 4 Aug 2025 17:33:59 -0700 Subject: [PATCH 03/54] OVER_SAMPLE_RATE --- .../rollout/sglang_rollout/sglang_rollout.py | 154 +++++++++++++++++- 1 file changed, 153 insertions(+), 1 deletion(-) diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index fd50592f943..9496eeeb90c 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -83,6 +83,8 @@ logger = logging.getLogger(__file__) logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) +OVER_SAMPLE_RATE = 0.8 + # patch to avoid issue https://github.com/sgl-project/sglang/issues/6723 def _set_envs_and_config(server_args: ServerArgs): @@ -170,6 +172,25 @@ async def update_weights_from_tensor(self, update_weights_request: UpdateWeights async def flush_cache(self): return await self.tokenizer_manager.flush_cache() + async def abort_request(self, rid: str = "", abort_all: bool = False): + """Abort a specific request or all requests. + + Args: + rid: The request ID to abort. If empty and abort_all is False, no action is taken. + abort_all: If True, abort all running requests regardless of rid. + """ + try: + self.tokenizer_manager.abort_request(rid=rid, abort_all=abort_all) + if abort_all: + logger.info("Aborted all pending requests") + elif rid: + logger.info(f"Aborted specific request: {rid}") + else: + logger.warning("No request ID provided and abort_all is False, no action taken") + except Exception as e: + logger.error(f"Failed to abort requests: {e}") + raise + # NOTE(sgm): add for verl. We can optimize it by making # the dataloader yield List[int] without padding. @@ -1076,12 +1097,56 @@ def _req_level_generate_sequences(self, prompts: DataProto, **kwargs) -> DataPro req_list = self._preprocess_prompt_to_async_rollout_requests( prompts, ) + + # 添加进度监控和abort功能 + total_requests = len(req_list) + target_completion = int(total_requests * OVER_SAMPLE_RATE) # 80%完成时abort + completed_count = 0 + aborted_requests = [] + + # 创建进度监控和abort任务 + async def monitor_and_abort(): + nonlocal completed_count + while completed_count < target_completion: + await asyncio.sleep(0.1) + + logger.info(f"🎯 Target reached: {completed_count}/{total_requests} completed!") + logger.info("🚫 Aborting remaining requests...") + + try: + await self._engine.abort_request(abort_all=True) + logger.info("✅ Abort command sent successfully!") + except Exception as e: + logger.error(f"❌ Abort failed: {e}") + + # 修改请求处理函数，添加完成计数 + async def process_request_with_monitoring(req): + nonlocal completed_count + try: + result = await self._async_rollout_a_request(req, do_sample, is_validate, **kwargs) + completed_count += 1 + return result + except Exception as e: + # 如果请求被abort，创建padding请求 + logger.warning(f"Request {req.request_id} was aborted or failed: {e}") + aborted_requests.append(req.request_id) + completed_count += 1 + # 返回一个padding的请求，确保在后续处理中被忽略 + return self._create_padding_request(req) + + # 启动监控任务 + monitor_task = asyncio.create_task(monitor_and_abort()) + loop = asyncio.get_event_loop() output_req_list = loop.run_until_complete( asyncio.gather( - *[self._async_rollout_a_request(req, do_sample, is_validate, **kwargs) for req in req_list], + *[process_request_with_monitoring(req) for req in req_list], ) ) + + # 取消监控任务 + monitor_task.cancel() + sorted_output_req_list = sorted(output_req_list, key=lambda x: (x.batch_data_id, x.rollout_offset)) else: sorted_output_req_list = None @@ -1269,6 +1334,93 @@ def _req_level_generate_sequences(self, prompts: DataProto, **kwargs) -> DataPro non_tensor_batch=non_tensor_batch, ) + def _create_padding_request(self, original_req: AsyncRolloutRequest) -> AsyncRolloutRequest: + """创建一个padding请求，用于替代被abort的请求。 + + 这个padding请求的特点是： + 1. 状态为COMPLETED，但包含空的response + 2. response_loss_mask全为0，确保在loss计算中被忽略 + 3. 保持原始请求的结构，但内容为空 + """ + # 创建padding的response_ids (全为pad_token_id) + padding_response_length = self.config.response_length + padding_response_ids = torch.full( + (1, padding_response_length), + self.pad_token_id, + dtype=torch.long, + device=original_req.input_ids.device if original_req.input_ids is not None else "cpu", + ) + + # 创建padding的attention_mask (全为0) + padding_response_attention_mask = torch.zeros( + (1, padding_response_length), + dtype=torch.long, + device=original_req.attention_mask.device if original_req.attention_mask is not None else "cpu", + ) + + # 创建padding的position_ids + if original_req.position_ids is not None: + prompt_length = original_req.prompt_ids.shape[-1] if original_req.prompt_ids is not None else 0 + padding_response_position_ids = torch.arange( + prompt_length, prompt_length + padding_response_length, dtype=torch.long + ).unsqueeze(0) + if original_req.position_ids.dim() == 2: + # 如果是2D tensor (如qwen2vl) + padding_response_position_ids = padding_response_position_ids.repeat( + original_req.position_ids.shape[0], 1 + ) + else: + padding_response_position_ids = None + + # 创建padding的loss_mask (全为0，确保被忽略) + padding_response_loss_mask = torch.zeros( + (1, padding_response_length), + dtype=torch.long, + device=original_req.loss_mask.device if original_req.loss_mask is not None else "cpu", + ) + + # 创建新的请求，保持原始结构但使用padding数据 + padding_req = AsyncRolloutRequest( + batch_data_id=original_req.batch_data_id, + rollout_offset=original_req.rollout_offset, + request_id=original_req.request_id + "_padding", + state=AsyncRolloutRequestStateEnum.COMPLETED, + messages=original_req.messages, # 保持原始messages + multi_modal_keys=original_req.multi_modal_keys, + multi_modal_data=original_req.multi_modal_data, + multi_modal_inputs=original_req.multi_modal_inputs, + tool_schemas=original_req.tool_schemas, + tools_kwargs=original_req.tools_kwargs, + interaction_kwargs=original_req.interaction_kwargs, + input_ids=original_req.input_ids, # 保持原始input_ids + prompt_ids=original_req.prompt_ids, # 保持原始prompt_ids + response_ids=padding_response_ids, # 使用padding的response_ids + attention_mask=original_req.attention_mask, # 保持原始attention_mask + prompt_attention_mask=original_req.prompt_attention_mask, # 保持原始prompt_attention_mask + response_attention_mask=padding_response_attention_mask, # 使用padding的response_attention_mask + position_ids=original_req.position_ids, # 保持原始position_ids + prompt_position_ids=original_req.prompt_position_ids, # 保持原始prompt_position_ids + response_position_ids=padding_response_position_ids, # 使用padding的response_position_ids + loss_mask=original_req.loss_mask, # 保持原始loss_mask + prompt_loss_mask=original_req.prompt_loss_mask, # 保持原始prompt_loss_mask + response_loss_mask=padding_response_loss_mask, # 使用padding的response_loss_mask (全为0) + reward_scores={}, # 空的reward_scores + max_prompt_len=original_req.max_prompt_len, + max_response_len=original_req.max_response_len, + max_model_len=original_req.max_model_len, + metrics={}, # 空的metrics + output_token_ids=None, # 空的output_token_ids + rollout_log_probs=None, # 空的rollout_log_probs + use_inference_chat_template=original_req.use_inference_chat_template, + tokenization_sanity_check_mode=original_req.tokenization_sanity_check_mode, + generation_prompt_ids=original_req.generation_prompt_ids, + base_conv_wo_gen_prompt_end_pos=original_req.base_conv_wo_gen_prompt_end_pos, + base_conv_with_gen_prompt_end_pos=original_req.base_conv_with_gen_prompt_end_pos, + ) + + logger.info(f"Created padding request for aborted request {original_req.request_id}") + return padding_req + def _preprocess_prompt_to_async_rollout_requests(self, prompts: DataProto, n: int = 1) -> list[AsyncRolloutRequest]: assert "raw_prompt" in prompts.non_tensor_batch, ( "need data.return_raw_chat=True, due to no official way do parse_messages" From 652f0f7f95381b63ca70997bb5636d47ab2caaea Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Mon, 4 Aug 2025 18:02:03 -0700 Subject: [PATCH 04/54] update sgl rollout --- .../rollout/sglang_rollout/sglang_rollout.py | 46 +++++++++++++------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index 9496eeeb90c..3f355ae5add 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -893,7 +893,9 @@ async def _async_rollout_a_request( # Only continue the conversation if the prompt length is not greater than max_model_len - 1, # since SGLang raises an error when max_new_tokens + 1 is greater to max_model_len (the extra # token accounts for the EOS token). - if len(_req.get_generation_prompt_ids(self.processing_class)) + 1 >= self.config.max_model_len: + prompt_length = len(_req.get_generation_prompt_ids(self.processing_class)) + + if prompt_length + 1 >= self.config.max_model_len: finish_reason_type = FinishReasonTypeEnum.LENGTH break @@ -1045,9 +1047,11 @@ async def _handle_engine_generate( self, generation_prompt_ids: list[int], sampling_params: dict, image_data: Optional[list[Any]] = None ) -> dict: max_new_tokens = min(self.config.response_length, self.config.max_model_len - len(generation_prompt_ids) - 1) + kwargs = sampling_params.copy() kwargs["max_new_tokens"] = max_new_tokens kwargs["n"] = 1 # group size is supported in preprocess + output = await self._engine.async_generate( input_ids=generation_prompt_ids, sampling_params=kwargs, @@ -1093,6 +1097,7 @@ def _req_level_generate_sequences(self, prompts: DataProto, **kwargs) -> DataPro do_sample = prompts.meta_info.get("do_sample", True) is_validate = prompts.meta_info.get("validate", False) tgt_device = prompts.batch["input_ids"].device + if self._tp_rank == 0: req_list = self._preprocess_prompt_to_async_rollout_requests( prompts, @@ -1103,6 +1108,7 @@ def _req_level_generate_sequences(self, prompts: DataProto, **kwargs) -> DataPro target_completion = int(total_requests * OVER_SAMPLE_RATE) # 80%完成时abort completed_count = 0 aborted_requests = [] + print(f"🎯 Over sampling target: {target_completion}/{total_requests}") # 创建进度监控和abort任务 async def monitor_and_abort(): @@ -1110,14 +1116,14 @@ async def monitor_and_abort(): while completed_count < target_completion: await asyncio.sleep(0.1) - logger.info(f"🎯 Target reached: {completed_count}/{total_requests} completed!") - logger.info("🚫 Aborting remaining requests...") + print(f"🎯 Target reached: {completed_count}/{total_requests} completed!") + print("🚫 Aborting remaining requests...") try: await self._engine.abort_request(abort_all=True) - logger.info("✅ Abort command sent successfully!") + print("✅ Abort command sent successfully!") except Exception as e: - logger.error(f"❌ Abort failed: {e}") + print(f"❌ Abort failed: {e}") # 修改请求处理函数，添加完成计数 async def process_request_with_monitoring(req): @@ -1125,6 +1131,7 @@ async def process_request_with_monitoring(req): try: result = await self._async_rollout_a_request(req, do_sample, is_validate, **kwargs) completed_count += 1 + print(f"✅ Request {req.request_id} completed ({completed_count}/{total_requests})") return result except Exception as e: # 如果请求被abort，创建padding请求 @@ -1134,18 +1141,27 @@ async def process_request_with_monitoring(req): # 返回一个padding的请求，确保在后续处理中被忽略 return self._create_padding_request(req) - # 启动监控任务 - monitor_task = asyncio.create_task(monitor_and_abort()) + # 修复：在事件循环中创建监控任务 + async def run_with_monitoring(): + # 启动监控任务 + monitor_task = asyncio.create_task(monitor_and_abort()) + try: + output_req_list = await asyncio.gather( + *[process_request_with_monitoring(req) for req in req_list], + ) + return output_req_list + finally: + # 取消监控任务 + monitor_task.cancel() + try: + await monitor_task + except asyncio.CancelledError: + pass + + # 运行异步任务 loop = asyncio.get_event_loop() - output_req_list = loop.run_until_complete( - asyncio.gather( - *[process_request_with_monitoring(req) for req in req_list], - ) - ) - - # 取消监控任务 - monitor_task.cancel() + output_req_list = loop.run_until_complete(run_with_monitoring()) sorted_output_req_list = sorted(output_req_list, key=lambda x: (x.batch_data_id, x.rollout_offset)) else: From 3fdfd00b2fb673e63fb3218d49a3d9c67dc7d117 Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Mon, 4 Aug 2025 18:46:35 -0700 Subject: [PATCH 05/54] is val --- .../rollout/sglang_rollout/sglang_rollout.py | 116 ++++++++++-------- 1 file changed, 65 insertions(+), 51 deletions(-) diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index 3f355ae5add..fc47afe1ba0 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -1108,60 +1108,74 @@ def _req_level_generate_sequences(self, prompts: DataProto, **kwargs) -> DataPro target_completion = int(total_requests * OVER_SAMPLE_RATE) # 80%完成时abort completed_count = 0 aborted_requests = [] - print(f"🎯 Over sampling target: {target_completion}/{total_requests}") - - # 创建进度监控和abort任务 - async def monitor_and_abort(): - nonlocal completed_count - while completed_count < target_completion: - await asyncio.sleep(0.1) - - print(f"🎯 Target reached: {completed_count}/{total_requests} completed!") - print("🚫 Aborting remaining requests...") - - try: - await self._engine.abort_request(abort_all=True) - print("✅ Abort command sent successfully!") - except Exception as e: - print(f"❌ Abort failed: {e}") - - # 修改请求处理函数，添加完成计数 - async def process_request_with_monitoring(req): - nonlocal completed_count - try: - result = await self._async_rollout_a_request(req, do_sample, is_validate, **kwargs) - completed_count += 1 - print(f"✅ Request {req.request_id} completed ({completed_count}/{total_requests})") - return result - except Exception as e: - # 如果请求被abort，创建padding请求 - logger.warning(f"Request {req.request_id} was aborted or failed: {e}") - aborted_requests.append(req.request_id) - completed_count += 1 - # 返回一个padding的请求，确保在后续处理中被忽略 - return self._create_padding_request(req) - - # 修复：在事件循环中创建监控任务 - async def run_with_monitoring(): - # 启动监控任务 - monitor_task = asyncio.create_task(monitor_and_abort()) - - try: - output_req_list = await asyncio.gather( - *[process_request_with_monitoring(req) for req in req_list], + + # 区分训练和验证阶段 + if is_validate: + print(f"🔍 Validation mode: processing all {total_requests} requests without abort") + + # 验证阶段：处理所有请求，不使用abort + async def process_all_requests(): + return await asyncio.gather( + *[self._async_rollout_a_request(req, do_sample, is_validate, **kwargs) for req in req_list], ) - return output_req_list - finally: - # 取消监控任务 - monitor_task.cancel() + + loop = asyncio.get_event_loop() + output_req_list = loop.run_until_complete(process_all_requests()) + else: + print(f"🎯 Training mode: over sampling target {target_completion}/{total_requests}") + + # 创建进度监控和abort任务 + async def monitor_and_abort(): + nonlocal completed_count + while completed_count < target_completion: + await asyncio.sleep(0.1) + + print(f"🎯 Target reached: {completed_count}/{total_requests} completed!") + print("🚫 Aborting remaining requests...") + try: - await monitor_task - except asyncio.CancelledError: - pass + await self._engine.abort_request(abort_all=True) + print("✅ Abort command sent successfully!") + except Exception as e: + print(f"❌ Abort failed: {e}") + + # 修改请求处理函数，添加完成计数 + async def process_request_with_monitoring(req): + nonlocal completed_count + try: + result = await self._async_rollout_a_request(req, do_sample, is_validate, **kwargs) + completed_count += 1 + print(f"✅ Request {req.request_id} completed ({completed_count}/{total_requests})") + return result + except Exception as e: + # 如果请求被abort，创建padding请求 + logger.warning(f"Request {req.request_id} was aborted or failed: {e}") + aborted_requests.append(req.request_id) + completed_count += 1 + # 返回一个padding的请求，确保在后续处理中被忽略 + return self._create_padding_request(req) + + # 修复：在事件循环中创建监控任务 + async def run_with_monitoring(): + # 启动监控任务 + monitor_task = asyncio.create_task(monitor_and_abort()) - # 运行异步任务 - loop = asyncio.get_event_loop() - output_req_list = loop.run_until_complete(run_with_monitoring()) + try: + output_req_list = await asyncio.gather( + *[process_request_with_monitoring(req) for req in req_list], + ) + return output_req_list + finally: + # 取消监控任务 + monitor_task.cancel() + try: + await monitor_task + except asyncio.CancelledError: + pass + + # 运行异步任务 + loop = asyncio.get_event_loop() + output_req_list = loop.run_until_complete(run_with_monitoring()) sorted_output_req_list = sorted(output_req_list, key=lambda x: (x.batch_data_id, x.rollout_offset)) else: From d37f51d8791342ea9e6b912f1b6d25a5c074f378 Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Mon, 4 Aug 2025 19:01:26 -0700 Subject: [PATCH 06/54] self.abort --- verl/workers/rollout/sglang_rollout/sglang_rollout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index fc47afe1ba0..f6bfcd2bc0f 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -180,7 +180,7 @@ async def abort_request(self, rid: str = "", abort_all: bool = False): abort_all: If True, abort all running requests regardless of rid. """ try: - self.tokenizer_manager.abort_request(rid=rid, abort_all=abort_all) + self.abort_request(rid=rid, abort_all=abort_all) if abort_all: logger.info("Aborted all pending requests") elif rid: From 51410e704d3013702ff30eee795ba06a38837546 Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Mon, 4 Aug 2025 20:47:48 -0700 Subject: [PATCH 07/54] cancel task --- .../rollout/sglang_rollout/sglang_rollout.py | 75 +++++++------------ 1 file changed, 29 insertions(+), 46 deletions(-) diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index f6bfcd2bc0f..b025c6d2aac 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -180,13 +180,9 @@ async def abort_request(self, rid: str = "", abort_all: bool = False): abort_all: If True, abort all running requests regardless of rid. """ try: - self.abort_request(rid=rid, abort_all=abort_all) - if abort_all: - logger.info("Aborted all pending requests") - elif rid: - logger.info(f"Aborted specific request: {rid}") - else: - logger.warning("No request ID provided and abort_all is False, no action taken") + result = await self.tokenizer_manager.abort_request(rid=rid, abort_all=abort_all) + print(f"🔍 Abort result: {result}") + return result if result is not None else {"status": "aborted"} except Exception as e: logger.error(f"Failed to abort requests: {e}") raise @@ -1124,58 +1120,45 @@ async def process_all_requests(): else: print(f"🎯 Training mode: over sampling target {target_completion}/{total_requests}") - # 创建进度监控和abort任务 - async def monitor_and_abort(): - nonlocal completed_count - while completed_count < target_completion: - await asyncio.sleep(0.1) - - print(f"🎯 Target reached: {completed_count}/{total_requests} completed!") - print("🚫 Aborting remaining requests...") + completion_lock = asyncio.Lock() - try: - await self._engine.abort_request(abort_all=True) - print("✅ Abort command sent successfully!") - except Exception as e: - print(f"❌ Abort failed: {e}") - - # 修改请求处理函数，添加完成计数 async def process_request_with_monitoring(req): nonlocal completed_count try: + # Allow the request to run to completion first result = await self._async_rollout_a_request(req, do_sample, is_validate, **kwargs) - completed_count += 1 - print(f"✅ Request {req.request_id} completed ({completed_count}/{total_requests})") - return result + + # After it finishes, acquire the lock to check if it's still needed + async with completion_lock: + if completed_count < target_completion: + # This request made it in time, count it + completed_count += 1 + print(f"✅ Request {req.request_id} completed ({completed_count}/{total_requests})") + return result + else: + # This request finished after the target was met, discard the result + logger.warning( + f"Request {req.request_id} finished after target met. Discarding result and creating padding." + ) + return self._create_padding_request(req) except Exception as e: - # 如果请求被abort，创建padding请求 + # If the request fails, it's also a form of completion logger.warning(f"Request {req.request_id} was aborted or failed: {e}") aborted_requests.append(req.request_id) - completed_count += 1 - # 返回一个padding的请求，确保在后续处理中被忽略 - return self._create_padding_request(req) - # 修复：在事件循环中创建监控任务 - async def run_with_monitoring(): - # 启动监控任务 - monitor_task = asyncio.create_task(monitor_and_abort()) + # We still need to check if this failure should be counted + async with completion_lock: + if completed_count < target_completion: + completed_count += 1 - try: - output_req_list = await asyncio.gather( - *[process_request_with_monitoring(req) for req in req_list], - ) - return output_req_list - finally: - # 取消监控任务 - monitor_task.cancel() - try: - await monitor_task - except asyncio.CancelledError: - pass + return self._create_padding_request(req) + + async def run_all_monitored(): + return await asyncio.gather(*[process_request_with_monitoring(req) for req in req_list]) # 运行异步任务 loop = asyncio.get_event_loop() - output_req_list = loop.run_until_complete(run_with_monitoring()) + output_req_list = loop.run_until_complete(run_all_monitored()) sorted_output_req_list = sorted(output_req_list, key=lambda x: (x.batch_data_id, x.rollout_offset)) else: From de8feb3a77d8a0b6038d052f9d9225c4a29f9f64 Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Mon, 4 Aug 2025 20:55:36 -0700 Subject: [PATCH 08/54] cancel then abort --- .../rollout/sglang_rollout/sglang_rollout.py | 81 +++++++++++++++---- 1 file changed, 66 insertions(+), 15 deletions(-) diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index b025c6d2aac..113053d4338 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -1121,44 +1121,95 @@ async def process_all_requests(): print(f"🎯 Training mode: over sampling target {target_completion}/{total_requests}") completion_lock = asyncio.Lock() + all_tasks = [] async def process_request_with_monitoring(req): nonlocal completed_count try: - # Allow the request to run to completion first result = await self._async_rollout_a_request(req, do_sample, is_validate, **kwargs) - # After it finishes, acquire the lock to check if it's still needed async with completion_lock: if completed_count < target_completion: - # This request made it in time, count it completed_count += 1 print(f"✅ Request {req.request_id} completed ({completed_count}/{total_requests})") return result else: - # This request finished after the target was met, discard the result - logger.warning( - f"Request {req.request_id} finished after target met. Discarding result and creating padding." - ) + # 这个请求虽然完成了，但已经超过目标，返回padding + logger.info(f"Request {req.request_id} finished after target met, creating padding") return self._create_padding_request(req) + except asyncio.CancelledError: + # 请求被取消，返回padding + logger.info(f"Request {req.request_id} was cancelled, creating padding") + aborted_requests.append(req.request_id) + return self._create_padding_request(req) except Exception as e: - # If the request fails, it's also a form of completion - logger.warning(f"Request {req.request_id} was aborted or failed: {e}") + # 请求失败，也算作完成 + logger.warning(f"Request {req.request_id} failed: {e}") aborted_requests.append(req.request_id) - - # We still need to check if this failure should be counted async with completion_lock: if completed_count < target_completion: completed_count += 1 - return self._create_padding_request(req) - async def run_all_monitored(): - return await asyncio.gather(*[process_request_with_monitoring(req) for req in req_list]) + async def monitor_and_cancel(): + nonlocal completed_count + while completed_count < target_completion: + await asyncio.sleep(0.1) + + print(f"🎯 Target reached: {completed_count}/{total_requests} completed!") + print("🚫 Cancelling remaining requests and sending abort to engine...") + + # 取消剩余的任务 + cancelled_count = 0 + for task in all_tasks: + if not task.done(): + task.cancel() + cancelled_count += 1 + + print(f"📋 Cancelled {cancelled_count} remaining tasks") + + # 向engine发送abort信号，中断所有正在进行的请求 + try: + abort_result = await self.abort_request(abort_all=True) + print(f"✅ Abort signal sent to engine: {abort_result}") + except Exception as e: + print(f"❌ Failed to send abort signal to engine: {e}") + + async def run_with_cancellation(): + nonlocal all_tasks + + # 创建所有任务 + all_tasks = [asyncio.create_task(process_request_with_monitoring(req)) for req in req_list] + + # 启动监控任务 + monitor_task = asyncio.create_task(monitor_and_cancel()) + + try: + # 等待所有任务完成（包括被取消的） + results = await asyncio.gather(*all_tasks, return_exceptions=True) + + # 处理结果，将异常转换为padding + output_req_list = [] + for i, result in enumerate(results): + if isinstance(result, Exception): + # 如果是异常（包括CancelledError），创建padding + logger.warning(f"Task {i} resulted in exception: {result}") + output_req_list.append(self._create_padding_request(req_list[i])) + else: + output_req_list.append(result) + + return output_req_list + finally: + # 取消监控任务 + monitor_task.cancel() + try: + await monitor_task + except asyncio.CancelledError: + pass # 运行异步任务 loop = asyncio.get_event_loop() - output_req_list = loop.run_until_complete(run_all_monitored()) + output_req_list = loop.run_until_complete(run_with_cancellation()) sorted_output_req_list = sorted(output_req_list, key=lambda x: (x.batch_data_id, x.rollout_offset)) else: From da37e6affa9f8ec3c38bab7711105b9b7cadf468 Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Mon, 4 Aug 2025 21:11:30 -0700 Subject: [PATCH 09/54] fix await --- examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh | 1 + verl/workers/rollout/sglang_rollout/sglang_rollout.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh index 505f5cc41cf..90c0dac0255 100644 --- a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh +++ b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh @@ -5,6 +5,7 @@ set -x ulimit -n 65535 +VAL_BEFORE_TRAIN=False PROJECT_DIR="$(pwd)" CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index 113053d4338..603ec0a3293 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -180,7 +180,7 @@ async def abort_request(self, rid: str = "", abort_all: bool = False): abort_all: If True, abort all running requests regardless of rid. """ try: - result = await self.tokenizer_manager.abort_request(rid=rid, abort_all=abort_all) + result = self.tokenizer_manager.abort_request(rid=rid, abort_all=abort_all) print(f"🔍 Abort result: {result}") return result if result is not None else {"status": "aborted"} except Exception as e: @@ -1170,7 +1170,7 @@ async def monitor_and_cancel(): # 向engine发送abort信号，中断所有正在进行的请求 try: - abort_result = await self.abort_request(abort_all=True) + abort_result = await self._engine.abort_request(abort_all=True) print(f"✅ Abort signal sent to engine: {abort_result}") except Exception as e: print(f"❌ Failed to send abort signal to engine: {e}") From 530850747a3356078f7092fbe4e4313538eb3729 Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Mon, 4 Aug 2025 21:32:44 -0700 Subject: [PATCH 10/54] finish over sample --- .../run_qwen2.5-3b_gsm8k_multiturn.sh | 12 +++++++++--- .../workers/rollout/sglang_rollout/sglang_rollout.py | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh index 90c0dac0255..0e2ff0207d4 100644 --- a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh +++ b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh @@ -5,10 +5,15 @@ set -x ulimit -n 65535 -VAL_BEFORE_TRAIN=False PROJECT_DIR="$(pwd)" CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" +function now() { + date '+%Y-%m-%d-%H-%M' +} + +EXPERIMENT_NAME="qwen2.5-3b_baseline_$(now)" + python3 -m verl.trainer.main_ppo \ --config-path="$CONFIG_PATH" \ --config-name='gsm8k_multiturn_grpo' \ @@ -42,12 +47,13 @@ python3 -m verl.trainer.main_ppo \ algorithm.use_kl_in_reward=False \ trainer.critic_warmup=0 \ trainer.logger='["console","wandb"]' \ - trainer.project_name='gsm8k_async_rl' \ - trainer.experiment_name='qwen2.5-3b_function_rm-gsm8k-sgl-multi-w-tool-verify-n16' \ + trainer.project_name='gsm8k_baseline' \ + trainer.experiment_name=$EXPERIMENT_NAME \ trainer.n_gpus_per_node=8 \ trainer.nnodes=1 \ trainer.save_freq=-1 \ trainer.test_freq=20 \ + trainer.val_before_train=False \ data.train_files=$HOME/data/gsm8k/train.parquet \ data.val_files=$HOME/data/gsm8k/test.parquet \ actor_rollout_ref.rollout.multi_turn.tool_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/tool_config/gsm8k_tool_config.yaml" \ diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index 603ec0a3293..f1543658ef2 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -1480,6 +1480,7 @@ def _create_padding_request(self, original_req: AsyncRolloutRequest) -> AsyncRol generation_prompt_ids=original_req.generation_prompt_ids, base_conv_wo_gen_prompt_end_pos=original_req.base_conv_wo_gen_prompt_end_pos, base_conv_with_gen_prompt_end_pos=original_req.base_conv_with_gen_prompt_end_pos, + processing_class=self.processing_class, # 添加缺少的 processing_class 参数 ) logger.info(f"Created padding request for aborted request {original_req.request_id}") From 1b8bfa9f31e5c6f22603b5e72007073729579997 Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Mon, 4 Aug 2025 21:37:40 -0700 Subject: [PATCH 11/54] add log to over sample --- verl/workers/rollout/sglang_rollout/sglang_rollout.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index f1543658ef2..19749560234 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -83,7 +83,7 @@ logger = logging.getLogger(__file__) logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) -OVER_SAMPLE_RATE = 0.8 +OVER_SAMPLE_RATE = 1 # patch to avoid issue https://github.com/sgl-project/sglang/issues/6723 @@ -1102,6 +1102,7 @@ def _req_level_generate_sequences(self, prompts: DataProto, **kwargs) -> DataPro # 添加进度监控和abort功能 total_requests = len(req_list) target_completion = int(total_requests * OVER_SAMPLE_RATE) # 80%完成时abort + print(f"🎯 Training mode: over sampling target {target_completion}/{total_requests}") completed_count = 0 aborted_requests = [] From 519a1b019d700ac16d50c68bee9b02888efdea2e Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Mon, 4 Aug 2025 23:41:25 -0700 Subject: [PATCH 12/54] Add benchmakr script --- .../run_qwen2.5-3b_gsm8k_multiturn.sh | 6 +-- over_sample.sh | 50 +++++++++++++++++++ run_experiment_loop.sh | 50 +++++++++++++++++++ .../rollout/sglang_rollout/sglang_rollout.py | 2 +- 4 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 over_sample.sh create mode 100644 run_experiment_loop.sh diff --git a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh index 0e2ff0207d4..75ede080c6d 100644 --- a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh +++ b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh @@ -9,10 +9,10 @@ PROJECT_DIR="$(pwd)" CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" function now() { - date '+%Y-%m-%d-%H-%M' + date '+%d-%H-%M' } -EXPERIMENT_NAME="qwen2.5-3b_baseline_$(now)" +EXPERIMENT_NAME="qwen2.5-3b_baseline_$(now)_$OVER_SAMPLE_RATE" python3 -m verl.trainer.main_ppo \ --config-path="$CONFIG_PATH" \ @@ -47,7 +47,7 @@ python3 -m verl.trainer.main_ppo \ algorithm.use_kl_in_reward=False \ trainer.critic_warmup=0 \ trainer.logger='["console","wandb"]' \ - trainer.project_name='gsm8k_baseline' \ + trainer.project_name='benchmark_over_sample' \ trainer.experiment_name=$EXPERIMENT_NAME \ trainer.n_gpus_per_node=8 \ trainer.nnodes=1 \ diff --git a/over_sample.sh b/over_sample.sh new file mode 100644 index 00000000000..10206c8a309 --- /dev/null +++ b/over_sample.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# 实验循环脚本 +# OVER_SAMPLE_RATE 从 0.1 到 1.0，每个运行45分钟 + +cd ~/verl +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + +# 定义 OVER_SAMPLE_RATE 数组 +rates=(0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1.0) + +# 无限循环 +while true; do + for rate in "${rates[@]}"; do + echo "==========================================" + echo "开始实验: OVER_SAMPLE_RATE = $rate" + echo "时间: $(date)" + echo "==========================================" + + # 设置当前实验的 OVER_SAMPLE_RATE + export OVER_SAMPLE_RATE=$rate + + # 启动训练（后台运行） + echo "启动训练..." + bash examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh & + TRAIN_PID=$! + + # 等待45分钟 + echo "训练将在10分钟后自动终止..." + sleep 600 # 10分钟 = 600秒 + + # 终止训练进程 + echo "终止训练进程..." + kill $TRAIN_PID 2>/dev/null + + # 清理所有 sglang 相关进程 + echo "清理 sglang 进程..." + pkill -f sglang + + # 等待3分钟 + echo "等待3分钟进行下一组实验..." + sleep 180 # 3分钟 = 180秒 + + echo "实验 OVER_SAMPLE_RATE = $rate 完成" + echo "==========================================" + done + + echo "一轮实验完成，开始下一轮..." + echo "==========================================" +done \ No newline at end of file diff --git a/run_experiment_loop.sh b/run_experiment_loop.sh new file mode 100644 index 00000000000..2ccb948d126 --- /dev/null +++ b/run_experiment_loop.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# 实验循环脚本 +# OVER_SAMPLE_RATE 从 0.1 到 1.0，每个运行45分钟 + +cd ~/verl +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + +# 定义 OVER_SAMPLE_RATE 数组 +rates=(0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1.0) + +for rate in "${rates[@]}"; do + echo "==========================================" + echo "开始实验: OVER_SAMPLE_RATE = $rate" + echo "时间: $(date)" + echo "==========================================" + + # 设置当前实验的 OVER_SAMPLE_RATE + export OVER_SAMPLE_RATE=$rate + + # 拉取并预处理 gsm8k 数据集 + echo "预处理数据集..." + python examples/data_preprocess/gsm8k_multiturn_w_tool.py + + # 启动训练（后台运行） + echo "启动训练..." + bash examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh & + TRAIN_PID=$! + + # 等待45分钟 + echo "训练将在45分钟后自动终止..." + sleep 2700 # 45分钟 = 2700秒 + + # 终止训练进程 + echo "终止训练进程..." + kill $TRAIN_PID 2>/dev/null + + # 清理所有 sglang 相关进程 + echo "清理 sglang 进程..." + pkill -f sglang + + # 等待3分钟 + echo "等待3分钟进行下一组实验..." + sleep 180 # 3分钟 = 180秒 + + echo "实验 OVER_SAMPLE_RATE = $rate 完成" + echo "==========================================" +done + +echo "所有实验完成！" \ No newline at end of file diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index 19749560234..39cc6860711 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -83,7 +83,7 @@ logger = logging.getLogger(__file__) logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) -OVER_SAMPLE_RATE = 1 +OVER_SAMPLE_RATE = os.getenv("OVER_SAMPLE_RATE", 1) # patch to avoid issue https://github.com/sgl-project/sglang/issues/6723 From ae43c5a17d3c27973d450c4cc644b1d2bdcac8d9 Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Tue, 5 Aug 2025 00:40:31 -0700 Subject: [PATCH 13/54] increase to 45 --- over_sample.sh | 21 ++++++++++++++++--- .../rollout/sglang_rollout/sglang_rollout.py | 2 +- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/over_sample.sh b/over_sample.sh index 10206c8a309..0b8d9a8fb2b 100644 --- a/over_sample.sh +++ b/over_sample.sh @@ -18,16 +18,31 @@ while true; do echo "==========================================" # 设置当前实验的 OVER_SAMPLE_RATE + unset OVER_SAMPLE_RATE export OVER_SAMPLE_RATE=$rate + echo "设置 OVER_SAMPLE_RATE = $OVER_SAMPLE_RATE" + + # 验证环境变量设置正确 + if ! [[ "$OVER_SAMPLE_RATE" =~ ^[0-9]+\.?[0-9]*$ ]]; then + echo "错误：OVER_SAMPLE_RATE 格式不正确: $OVER_SAMPLE_RATE" + exit 1 + fi # 启动训练（后台运行） echo "启动训练..." bash examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh & TRAIN_PID=$! - # 等待45分钟 - echo "训练将在10分钟后自动终止..." - sleep 600 # 10分钟 = 600秒 + # 等待训练完成或超时 + echo "训练将在45分钟后自动终止，或失败时立即终止..." + for i in {1..2700}; do + # 检查进程是否还在运行 + if ! kill -0 $TRAIN_PID 2>/dev/null; then + echo "训练进程已结束（可能失败）" + break + fi + sleep 1 + done # 终止训练进程 echo "终止训练进程..." diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index 39cc6860711..f51399c612c 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -83,7 +83,7 @@ logger = logging.getLogger(__file__) logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) -OVER_SAMPLE_RATE = os.getenv("OVER_SAMPLE_RATE", 1) +OVER_SAMPLE_RATE = float(os.getenv("OVER_SAMPLE_RATE", 1)) # patch to avoid issue https://github.com/sgl-project/sglang/issues/6723 From 45265ec8cd55a6bde69bd48d584164151a38dd98 Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Tue, 5 Aug 2025 00:41:32 -0700 Subject: [PATCH 14/54] incrse --- over_sample.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/over_sample.sh b/over_sample.sh index 0b8d9a8fb2b..74492702199 100644 --- a/over_sample.sh +++ b/over_sample.sh @@ -7,7 +7,7 @@ cd ~/verl export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 # 定义 OVER_SAMPLE_RATE 数组 -rates=(0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1.0) +rates=(0.6 0.65 0.7 0.75 0.8 0.85 0.88 0.9 0.92 0.95 0.98 1.0) # 无限循环 while true; do From ef6ab934cecfc2093f693b1bf24e5e68b0705307 Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Tue, 5 Aug 2025 00:42:13 -0700 Subject: [PATCH 15/54] add over sample --- over_sample.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/over_sample.sh b/over_sample.sh index 74492702199..52a927236d4 100644 --- a/over_sample.sh +++ b/over_sample.sh @@ -7,7 +7,7 @@ cd ~/verl export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 # 定义 OVER_SAMPLE_RATE 数组 -rates=(0.6 0.65 0.7 0.75 0.8 0.85 0.88 0.9 0.92 0.95 0.98 1.0) +rates=(0.65 0.7 0.75 0.8 0.85 0.88 0.9 0.92 0.95 0.98 1.0) # 无限循环 while true; do From 5d3b9705ecf45f6980224897e617a4b2f71093e1 Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Tue, 5 Aug 2025 09:46:20 -0700 Subject: [PATCH 16/54] fix reward and loss cal --- verl/trainer/ppo/core_algos.py | 17 +++++++++++++---- verl/trainer/ppo/metric_utils.py | 32 ++++++++++++++++++++++++-------- verl/workers/fsdp_workers.py | 14 ++++++++++++++ 3 files changed, 51 insertions(+), 12 deletions(-) diff --git a/verl/trainer/ppo/core_algos.py b/verl/trainer/ppo/core_algos.py index 7ec622036d9..9ae784be867 100644 --- a/verl/trainer/ppo/core_algos.py +++ b/verl/trainer/ppo/core_algos.py @@ -715,17 +715,26 @@ def agg_loss(loss_mat: torch.Tensor, loss_mask: torch.Tensor, loss_agg_mode: str loss: `a scalar torch.Tensor` aggregated loss """ + # 检测padding请求（loss_mask全为0的样本） + # 如果某个样本的loss_mask全为0，说明这是一个被abort的padding请求 + # 我们需要将其排除在统计之外 + sample_mask = loss_mask.any(dim=1).float() # (bs,) - 只有非padding样本为1 + if loss_agg_mode == "token-mean": - loss = verl_F.masked_mean(loss_mat, loss_mask) + # 使用样本级别的mask来过滤padding请求 + loss = verl_F.masked_mean(loss_mat, loss_mask * sample_mask.unsqueeze(-1)) elif loss_agg_mode == "seq-mean-token-sum": seq_losses = torch.sum(loss_mat * loss_mask, dim=-1) # token-sum - loss = torch.mean(seq_losses) # seq-mean + # 使用样本级别的mask来过滤padding请求 + loss = verl_F.masked_mean(seq_losses, sample_mask) elif loss_agg_mode == "seq-mean-token-mean": seq_losses = torch.sum(loss_mat * loss_mask, dim=-1) / torch.sum(loss_mask, dim=-1) # token-mean - loss = torch.mean(seq_losses) # seq-mean + # 使用样本级别的mask来过滤padding请求 + loss = verl_F.masked_mean(seq_losses, sample_mask) elif loss_agg_mode == "seq-mean-token-sum-norm": seq_losses = torch.sum(loss_mat * loss_mask, dim=-1) - loss = torch.sum(seq_losses) / loss_mask.shape[-1] # The divisor + # 使用样本级别的mask来过滤padding请求 + loss = verl_F.masked_mean(seq_losses, sample_mask) / loss_mask.shape[-1] # The divisor # (loss_mask.shape[-1]) should ideally be constant # throughout training to well-replicate the DrGRPO paper. # TODO: Perhaps add user-defined normalizer argument to diff --git a/verl/trainer/ppo/metric_utils.py b/verl/trainer/ppo/metric_utils.py index 3b6b47bf04c..eb1c2616959 100644 --- a/verl/trainer/ppo/metric_utils.py +++ b/verl/trainer/ppo/metric_utils.py @@ -118,6 +118,22 @@ def compute_data_metrics(batch: DataProto, use_critic: bool = True) -> dict[str, prompt_length = response_info["prompt_length"] response_length = response_info["response_length"] + # 检测被abort的请求：response_mask全为0的请求 + # 被abort的请求的response部分完全没有有效token + aborted_mask = (response_length == 0).bool() # response_length为0表示被abort + non_aborted_mask = ~aborted_mask + + non_aborted_sequence_score = sequence_score[non_aborted_mask] + non_aborted_sequence_reward = sequence_reward[non_aborted_mask] + + score_mean = torch.mean(non_aborted_sequence_score).detach().item() + score_max = torch.max(non_aborted_sequence_score).detach().item() + score_min = torch.min(non_aborted_sequence_score).detach().item() + + reward_mean = torch.mean(non_aborted_sequence_reward).detach().item() + reward_max = torch.max(non_aborted_sequence_reward).detach().item() + reward_min = torch.min(non_aborted_sequence_reward).detach().item() + valid_adv = torch.masked_select(advantages, response_mask) valid_returns = torch.masked_select(returns, response_mask) @@ -128,14 +144,14 @@ def compute_data_metrics(batch: DataProto, use_critic: bool = True) -> dict[str, return_var = torch.var(valid_returns) metrics = { - # score - "critic/score/mean": torch.mean(sequence_score).detach().item(), - "critic/score/max": torch.max(sequence_score).detach().item(), - "critic/score/min": torch.min(sequence_score).detach().item(), - # reward - "critic/rewards/mean": torch.mean(sequence_reward).detach().item(), - "critic/rewards/max": torch.max(sequence_reward).detach().item(), - "critic/rewards/min": torch.min(sequence_reward).detach().item(), + # score (只对非abort请求计算平均值) + "critic/score/mean": score_mean, + "critic/score/max": score_max, + "critic/score/min": score_min, + # reward (只对非abort请求计算平均值) + "critic/rewards/mean": reward_mean, + "critic/rewards/max": reward_max, + "critic/rewards/min": reward_min, # adv "critic/advantages/mean": torch.mean(valid_adv).detach().item(), "critic/advantages/max": torch.max(valid_adv).detach().item(), diff --git a/verl/workers/fsdp_workers.py b/verl/workers/fsdp_workers.py index 25824fc3c00..5b32019a3c7 100644 --- a/verl/workers/fsdp_workers.py +++ b/verl/workers/fsdp_workers.py @@ -1536,6 +1536,20 @@ def _expand_to_token_level(self, data: DataProto, scores: torch.Tensor): # select the response part token_level_scores = token_level_scores[:, -response_length:] + # 检测padding请求（response_loss_mask全为0的请求） + # 如果某个请求的response_loss_mask全为0，说明这是一个被abort的padding请求 + # 我们将其reward设为0，这样在后续的reward统计中就不会被计入 + if "response_mask" in data.batch: + response_mask = data.batch["response_mask"] # 这是response_loss_mask + # 检查每个样本的response_mask是否全为0 + padding_mask = response_mask.sum(dim=-1) == 0 # (batch_size,) + + # 将padding请求的reward设为0 + if padding_mask.any(): + # 将padding请求对应的token_level_scores设为0 + token_level_scores[padding_mask] = 0 + logger.info(f"Detected {padding_mask.sum().item()} padding requests, their rewards are set to 0") + return token_level_scores def _switch_chat_template(self, data: DataProto): From 89da6d79fd84a72951927fd1c9493072eeb14924 Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Tue, 5 Aug 2025 11:37:48 -0700 Subject: [PATCH 17/54] revert loss agg --- over_sample.sh | 20 +++++++++++-- run_experiment_loop.sh | 50 -------------------------------- verl/trainer/ppo/core_algos.py | 17 +++-------- verl/trainer/ppo/metric_utils.py | 12 +++++++- 4 files changed, 33 insertions(+), 66 deletions(-) delete mode 100644 run_experiment_loop.sh diff --git a/over_sample.sh b/over_sample.sh index 52a927236d4..17172cad9cd 100644 --- a/over_sample.sh +++ b/over_sample.sh @@ -6,8 +6,24 @@ cd ~/verl export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 -# 定义 OVER_SAMPLE_RATE 数组 -rates=(0.65 0.7 0.75 0.8 0.85 0.88 0.9 0.92 0.95 0.98 1.0) +# 检查是否传入了参数 +if [ $# -eq 0 ]; then + # 如果没有传入参数，使用默认值 + rates=(0.8) + echo "使用默认的 OVER_SAMPLE_RATE 数组: ${rates[*]}" +else + # 使用传入的参数作为数组 + rates=("$@") + echo "使用传入的 OVER_SAMPLE_RATE 数组: ${rates[*]}" +fi + +# 验证传入的参数是否为有效的数字 +for rate in "${rates[@]}"; do + if ! [[ "$rate" =~ ^[0-9]+\.?[0-9]*$ ]]; then + echo "错误：传入的参数 '$rate' 不是有效的数字" + exit 1 + fi +done # 无限循环 while true; do diff --git a/run_experiment_loop.sh b/run_experiment_loop.sh deleted file mode 100644 index 2ccb948d126..00000000000 --- a/run_experiment_loop.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -# 实验循环脚本 -# OVER_SAMPLE_RATE 从 0.1 到 1.0，每个运行45分钟 - -cd ~/verl -export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 - -# 定义 OVER_SAMPLE_RATE 数组 -rates=(0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1.0) - -for rate in "${rates[@]}"; do - echo "==========================================" - echo "开始实验: OVER_SAMPLE_RATE = $rate" - echo "时间: $(date)" - echo "==========================================" - - # 设置当前实验的 OVER_SAMPLE_RATE - export OVER_SAMPLE_RATE=$rate - - # 拉取并预处理 gsm8k 数据集 - echo "预处理数据集..." - python examples/data_preprocess/gsm8k_multiturn_w_tool.py - - # 启动训练（后台运行） - echo "启动训练..." - bash examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh & - TRAIN_PID=$! - - # 等待45分钟 - echo "训练将在45分钟后自动终止..." - sleep 2700 # 45分钟 = 2700秒 - - # 终止训练进程 - echo "终止训练进程..." - kill $TRAIN_PID 2>/dev/null - - # 清理所有 sglang 相关进程 - echo "清理 sglang 进程..." - pkill -f sglang - - # 等待3分钟 - echo "等待3分钟进行下一组实验..." - sleep 180 # 3分钟 = 180秒 - - echo "实验 OVER_SAMPLE_RATE = $rate 完成" - echo "==========================================" -done - -echo "所有实验完成！" \ No newline at end of file diff --git a/verl/trainer/ppo/core_algos.py b/verl/trainer/ppo/core_algos.py index 9ae784be867..7ec622036d9 100644 --- a/verl/trainer/ppo/core_algos.py +++ b/verl/trainer/ppo/core_algos.py @@ -715,26 +715,17 @@ def agg_loss(loss_mat: torch.Tensor, loss_mask: torch.Tensor, loss_agg_mode: str loss: `a scalar torch.Tensor` aggregated loss """ - # 检测padding请求（loss_mask全为0的样本） - # 如果某个样本的loss_mask全为0，说明这是一个被abort的padding请求 - # 我们需要将其排除在统计之外 - sample_mask = loss_mask.any(dim=1).float() # (bs,) - 只有非padding样本为1 - if loss_agg_mode == "token-mean": - # 使用样本级别的mask来过滤padding请求 - loss = verl_F.masked_mean(loss_mat, loss_mask * sample_mask.unsqueeze(-1)) + loss = verl_F.masked_mean(loss_mat, loss_mask) elif loss_agg_mode == "seq-mean-token-sum": seq_losses = torch.sum(loss_mat * loss_mask, dim=-1) # token-sum - # 使用样本级别的mask来过滤padding请求 - loss = verl_F.masked_mean(seq_losses, sample_mask) + loss = torch.mean(seq_losses) # seq-mean elif loss_agg_mode == "seq-mean-token-mean": seq_losses = torch.sum(loss_mat * loss_mask, dim=-1) / torch.sum(loss_mask, dim=-1) # token-mean - # 使用样本级别的mask来过滤padding请求 - loss = verl_F.masked_mean(seq_losses, sample_mask) + loss = torch.mean(seq_losses) # seq-mean elif loss_agg_mode == "seq-mean-token-sum-norm": seq_losses = torch.sum(loss_mat * loss_mask, dim=-1) - # 使用样本级别的mask来过滤padding请求 - loss = verl_F.masked_mean(seq_losses, sample_mask) / loss_mask.shape[-1] # The divisor + loss = torch.sum(seq_losses) / loss_mask.shape[-1] # The divisor # (loss_mask.shape[-1]) should ideally be constant # throughout training to well-replicate the DrGRPO paper. # TODO: Perhaps add user-defined normalizer argument to diff --git a/verl/trainer/ppo/metric_utils.py b/verl/trainer/ppo/metric_utils.py index eb1c2616959..0bafa0c4e14 100644 --- a/verl/trainer/ppo/metric_utils.py +++ b/verl/trainer/ppo/metric_utils.py @@ -120,9 +120,19 @@ def compute_data_metrics(batch: DataProto, use_critic: bool = True) -> dict[str, # 检测被abort的请求：response_mask全为0的请求 # 被abort的请求的response部分完全没有有效token - aborted_mask = (response_length == 0).bool() # response_length为0表示被abort + # 使用batch中的response_mask来保持与agg_loss的一致性 + if "response_mask" in batch.batch: + # 使用batch中的response_mask，与agg_loss保持一致 + response_mask_from_batch = batch.batch["response_mask"] + aborted_mask = (response_mask_from_batch.sum(dim=1) == 0).bool() + else: + # 回退到原来的计算方式 + aborted_mask = (response_length == 0).bool() # response_length为0表示被abort + non_aborted_mask = ~aborted_mask + print("over sample rate in metric_utils: ", non_aborted_mask.sum() / len(non_aborted_mask)) + non_aborted_sequence_score = sequence_score[non_aborted_mask] non_aborted_sequence_reward = sequence_reward[non_aborted_mask] From 614f0ad4217080c9ffbe263aad3014019d1b73bf Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Tue, 5 Aug 2025 11:41:10 -0700 Subject: [PATCH 18/54] revert reward padding --- verl/workers/fsdp_workers.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/verl/workers/fsdp_workers.py b/verl/workers/fsdp_workers.py index 5b32019a3c7..431e73e9a08 100644 --- a/verl/workers/fsdp_workers.py +++ b/verl/workers/fsdp_workers.py @@ -1535,21 +1535,6 @@ def _expand_to_token_level(self, data: DataProto, scores: torch.Tensor): # select the response part token_level_scores = token_level_scores[:, -response_length:] - - # 检测padding请求（response_loss_mask全为0的请求） - # 如果某个请求的response_loss_mask全为0，说明这是一个被abort的padding请求 - # 我们将其reward设为0，这样在后续的reward统计中就不会被计入 - if "response_mask" in data.batch: - response_mask = data.batch["response_mask"] # 这是response_loss_mask - # 检查每个样本的response_mask是否全为0 - padding_mask = response_mask.sum(dim=-1) == 0 # (batch_size,) - - # 将padding请求的reward设为0 - if padding_mask.any(): - # 将padding请求对应的token_level_scores设为0 - token_level_scores[padding_mask] = 0 - logger.info(f"Detected {padding_mask.sum().item()} padding requests, their rewards are set to 0") - return token_level_scores def _switch_chat_template(self, data: DataProto): From 08b7e028fbb62fc963dbf7b8564370f6bb485b47 Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Tue, 5 Aug 2025 11:41:26 -0700 Subject: [PATCH 19/54] revert reward padding --- verl/workers/fsdp_workers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/verl/workers/fsdp_workers.py b/verl/workers/fsdp_workers.py index 431e73e9a08..7f54a9baa0a 100644 --- a/verl/workers/fsdp_workers.py +++ b/verl/workers/fsdp_workers.py @@ -1535,6 +1535,7 @@ def _expand_to_token_level(self, data: DataProto, scores: torch.Tensor): # select the response part token_level_scores = token_level_scores[:, -response_length:] + return token_level_scores def _switch_chat_template(self, data: DataProto): From 36abd78db7753eca766a5d5925df087c9ea44def Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Tue, 5 Aug 2025 11:43:17 -0700 Subject: [PATCH 20/54] finish --- verl/workers/fsdp_workers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verl/workers/fsdp_workers.py b/verl/workers/fsdp_workers.py index 7f54a9baa0a..25824fc3c00 100644 --- a/verl/workers/fsdp_workers.py +++ b/verl/workers/fsdp_workers.py @@ -1535,7 +1535,7 @@ def _expand_to_token_level(self, data: DataProto, scores: torch.Tensor): # select the response part token_level_scores = token_level_scores[:, -response_length:] - + return token_level_scores def _switch_chat_template(self, data: DataProto): From b979a73e358313afafab5db512cd5ae0009ccac0 Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Tue, 5 Aug 2025 11:55:19 -0700 Subject: [PATCH 21/54] revert non_aborted_mask --- verl/trainer/ppo/metric_utils.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/verl/trainer/ppo/metric_utils.py b/verl/trainer/ppo/metric_utils.py index 0bafa0c4e14..9628de51875 100644 --- a/verl/trainer/ppo/metric_utils.py +++ b/verl/trainer/ppo/metric_utils.py @@ -121,13 +121,7 @@ def compute_data_metrics(batch: DataProto, use_critic: bool = True) -> dict[str, # 检测被abort的请求：response_mask全为0的请求 # 被abort的请求的response部分完全没有有效token # 使用batch中的response_mask来保持与agg_loss的一致性 - if "response_mask" in batch.batch: - # 使用batch中的response_mask，与agg_loss保持一致 - response_mask_from_batch = batch.batch["response_mask"] - aborted_mask = (response_mask_from_batch.sum(dim=1) == 0).bool() - else: - # 回退到原来的计算方式 - aborted_mask = (response_length == 0).bool() # response_length为0表示被abort + aborted_mask = (response_length == 0).bool() # response_length为0表示被abort non_aborted_mask = ~aborted_mask From b4fdfcf6a625fc232ecb1d84a7ced6bc913f87ba Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Tue, 5 Aug 2025 14:24:16 -0700 Subject: [PATCH 22/54] clean up codes --- .../run_qwen2.5-3b_gsm8k_multiturn.sh | 4 +- over_sample.sh | 81 ------------------- 2 files changed, 2 insertions(+), 83 deletions(-) delete mode 100644 over_sample.sh diff --git a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh index 75ede080c6d..b4323964409 100644 --- a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh +++ b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh @@ -47,13 +47,13 @@ python3 -m verl.trainer.main_ppo \ algorithm.use_kl_in_reward=False \ trainer.critic_warmup=0 \ trainer.logger='["console","wandb"]' \ - trainer.project_name='benchmark_over_sample' \ + trainer.project_name='benchmark_over_sample_2' \ trainer.experiment_name=$EXPERIMENT_NAME \ trainer.n_gpus_per_node=8 \ trainer.nnodes=1 \ trainer.save_freq=-1 \ trainer.test_freq=20 \ - trainer.val_before_train=False \ + trainer.val_before_train=True \ data.train_files=$HOME/data/gsm8k/train.parquet \ data.val_files=$HOME/data/gsm8k/test.parquet \ actor_rollout_ref.rollout.multi_turn.tool_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/tool_config/gsm8k_tool_config.yaml" \ diff --git a/over_sample.sh b/over_sample.sh deleted file mode 100644 index 17172cad9cd..00000000000 --- a/over_sample.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash - -# 实验循环脚本 -# OVER_SAMPLE_RATE 从 0.1 到 1.0，每个运行45分钟 - -cd ~/verl -export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 - -# 检查是否传入了参数 -if [ $# -eq 0 ]; then - # 如果没有传入参数，使用默认值 - rates=(0.8) - echo "使用默认的 OVER_SAMPLE_RATE 数组: ${rates[*]}" -else - # 使用传入的参数作为数组 - rates=("$@") - echo "使用传入的 OVER_SAMPLE_RATE 数组: ${rates[*]}" -fi - -# 验证传入的参数是否为有效的数字 -for rate in "${rates[@]}"; do - if ! [[ "$rate" =~ ^[0-9]+\.?[0-9]*$ ]]; then - echo "错误：传入的参数 '$rate' 不是有效的数字" - exit 1 - fi -done - -# 无限循环 -while true; do - for rate in "${rates[@]}"; do - echo "==========================================" - echo "开始实验: OVER_SAMPLE_RATE = $rate" - echo "时间: $(date)" - echo "==========================================" - - # 设置当前实验的 OVER_SAMPLE_RATE - unset OVER_SAMPLE_RATE - export OVER_SAMPLE_RATE=$rate - echo "设置 OVER_SAMPLE_RATE = $OVER_SAMPLE_RATE" - - # 验证环境变量设置正确 - if ! [[ "$OVER_SAMPLE_RATE" =~ ^[0-9]+\.?[0-9]*$ ]]; then - echo "错误：OVER_SAMPLE_RATE 格式不正确: $OVER_SAMPLE_RATE" - exit 1 - fi - - # 启动训练（后台运行） - echo "启动训练..." - bash examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh & - TRAIN_PID=$! - - # 等待训练完成或超时 - echo "训练将在45分钟后自动终止，或失败时立即终止..." - for i in {1..2700}; do - # 检查进程是否还在运行 - if ! kill -0 $TRAIN_PID 2>/dev/null; then - echo "训练进程已结束（可能失败）" - break - fi - sleep 1 - done - - # 终止训练进程 - echo "终止训练进程..." - kill $TRAIN_PID 2>/dev/null - - # 清理所有 sglang 相关进程 - echo "清理 sglang 进程..." - pkill -f sglang - - # 等待3分钟 - echo "等待3分钟进行下一组实验..." - sleep 180 # 3分钟 = 180秒 - - echo "实验 OVER_SAMPLE_RATE = $rate 完成" - echo "==========================================" - done - - echo "一轮实验完成，开始下一轮..." - echo "==========================================" -done \ No newline at end of file From 4e2316b8bd74f4bfad5cf4d1615ad611c5be7753 Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Tue, 5 Aug 2025 14:56:36 -0700 Subject: [PATCH 23/54] clean up codes --- .../_generated_ppo_megatron_trainer.yaml | 1 + .../config/_generated_ppo_trainer.yaml | 1 + verl/trainer/config/rollout/rollout.yaml | 8 +- verl/trainer/ppo/metric_utils.py | 10 +- .../rollout/sglang_rollout/sglang_rollout.py | 124 +++++++----------- 5 files changed, 61 insertions(+), 83 deletions(-) diff --git a/verl/trainer/config/_generated_ppo_megatron_trainer.yaml b/verl/trainer/config/_generated_ppo_megatron_trainer.yaml index 4f50153270c..e81d18f4c37 100644 --- a/verl/trainer/config/_generated_ppo_megatron_trainer.yaml +++ b/verl/trainer/config/_generated_ppo_megatron_trainer.yaml @@ -144,6 +144,7 @@ actor_rollout_ref: disable_log_stats: true do_sample: true 'n': 1 + over_sample_rate: 0 multi_stage_wake_up: false engine_kwargs: vllm: diff --git a/verl/trainer/config/_generated_ppo_trainer.yaml b/verl/trainer/config/_generated_ppo_trainer.yaml index f69a8cfa043..7d204b9be90 100644 --- a/verl/trainer/config/_generated_ppo_trainer.yaml +++ b/verl/trainer/config/_generated_ppo_trainer.yaml @@ -108,6 +108,7 @@ actor_rollout_ref: disable_log_stats: true do_sample: true 'n': 1 + over_sample_rate: 0 multi_stage_wake_up: false engine_kwargs: vllm: diff --git a/verl/trainer/config/rollout/rollout.yaml b/verl/trainer/config/rollout/rollout.yaml index b16907b1db8..9f0f9a25c0a 100644 --- a/verl/trainer/config/rollout/rollout.yaml +++ b/verl/trainer/config/rollout/rollout.yaml @@ -73,7 +73,13 @@ do_sample: True # number of responses (i.e. num sample times). > 1 for grpo n: 1 -# Whether to wake up inference engine in multi-stage to reduce peak memory during training-rollout transition. +# The over_sample_rate parameter controls the early termination threshold for training rollouts, +# where the system will abort remaining requests when (1 - over_sample_rate) * total_requests completions are reached. +over_sample_rate: 0 + +# Whether to wake up inference engine in multi-stage for SGLang +# to reduce peak memory during training-rollout transition. +# This is only effective for SGLang rollout. multi_stage_wake_up: false # Extra inference engine arguments (vllm, sglang). diff --git a/verl/trainer/ppo/metric_utils.py b/verl/trainer/ppo/metric_utils.py index 9628de51875..f0bf71912d8 100644 --- a/verl/trainer/ppo/metric_utils.py +++ b/verl/trainer/ppo/metric_utils.py @@ -118,11 +118,7 @@ def compute_data_metrics(batch: DataProto, use_critic: bool = True) -> dict[str, prompt_length = response_info["prompt_length"] response_length = response_info["response_length"] - # 检测被abort的请求：response_mask全为0的请求 - # 被abort的请求的response部分完全没有有效token - # 使用batch中的response_mask来保持与agg_loss的一致性 - aborted_mask = (response_length == 0).bool() # response_length为0表示被abort - + aborted_mask = (response_length == 0).bool() non_aborted_mask = ~aborted_mask print("over sample rate in metric_utils: ", non_aborted_mask.sum() / len(non_aborted_mask)) @@ -148,11 +144,11 @@ def compute_data_metrics(batch: DataProto, use_critic: bool = True) -> dict[str, return_var = torch.var(valid_returns) metrics = { - # score (只对非abort请求计算平均值) + # score "critic/score/mean": score_mean, "critic/score/max": score_max, "critic/score/min": score_min, - # reward (只对非abort请求计算平均值) + # reward "critic/rewards/mean": reward_mean, "critic/rewards/max": reward_max, "critic/rewards/min": reward_min, diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index f51399c612c..e1e74a6cbd8 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -181,7 +181,6 @@ async def abort_request(self, rid: str = "", abort_all: bool = False): """ try: result = self.tokenizer_manager.abort_request(rid=rid, abort_all=abort_all) - print(f"🔍 Abort result: {result}") return result if result is not None else {"status": "aborted"} except Exception as e: logger.error(f"Failed to abort requests: {e}") @@ -1099,18 +1098,16 @@ def _req_level_generate_sequences(self, prompts: DataProto, **kwargs) -> DataPro prompts, ) - # 添加进度监控和abort功能 + # add progress monitoring and abort function total_requests = len(req_list) - target_completion = int(total_requests * OVER_SAMPLE_RATE) # 80%完成时abort - print(f"🎯 Training mode: over sampling target {target_completion}/{total_requests}") + target_completion = int(total_requests * (1 - OVER_SAMPLE_RATE)) + # abort when target_completion of requests are completed completed_count = 0 aborted_requests = [] - # 区分训练和验证阶段 + # distinguish training and validation if is_validate: - print(f"🔍 Validation mode: processing all {total_requests} requests without abort") - - # 验证阶段：处理所有请求，不使用abort + # validation mode: process all requests without abort async def process_all_requests(): return await asyncio.gather( *[self._async_rollout_a_request(req, do_sample, is_validate, **kwargs) for req in req_list], @@ -1119,8 +1116,6 @@ async def process_all_requests(): loop = asyncio.get_event_loop() output_req_list = loop.run_until_complete(process_all_requests()) else: - print(f"🎯 Training mode: over sampling target {target_completion}/{total_requests}") - completion_lock = asyncio.Lock() all_tasks = [] @@ -1132,68 +1127,53 @@ async def process_request_with_monitoring(req): async with completion_lock: if completed_count < target_completion: completed_count += 1 - print(f"✅ Request {req.request_id} completed ({completed_count}/{total_requests})") - return result - else: - # 这个请求虽然完成了，但已经超过目标，返回padding - logger.info(f"Request {req.request_id} finished after target met, creating padding") - return self._create_padding_request(req) + return result except asyncio.CancelledError: - # 请求被取消，返回padding + # request is cancelled, return padding logger.info(f"Request {req.request_id} was cancelled, creating padding") aborted_requests.append(req.request_id) return self._create_padding_request(req) except Exception as e: - # 请求失败，也算作完成 - logger.warning(f"Request {req.request_id} failed: {e}") - aborted_requests.append(req.request_id) - async with completion_lock: - if completed_count < target_completion: - completed_count += 1 - return self._create_padding_request(req) + logger.error(f"Uncaught exception in process_request_with_monitoring: {e}") + logger.error("This shall not happen, please check the code") + raise e async def monitor_and_cancel(): nonlocal completed_count while completed_count < target_completion: await asyncio.sleep(0.1) - print(f"🎯 Target reached: {completed_count}/{total_requests} completed!") - print("🚫 Cancelling remaining requests and sending abort to engine...") - - # 取消剩余的任务 + # cancel remaining tasks cancelled_count = 0 for task in all_tasks: if not task.done(): task.cancel() cancelled_count += 1 - print(f"📋 Cancelled {cancelled_count} remaining tasks") - - # 向engine发送abort信号，中断所有正在进行的请求 + # send abort signal to engine, interrupt all ongoing requests try: - abort_result = await self._engine.abort_request(abort_all=True) - print(f"✅ Abort signal sent to engine: {abort_result}") + await self._engine.abort_request(abort_all=True) except Exception as e: - print(f"❌ Failed to send abort signal to engine: {e}") + logger.error(f"Failed to send abort signal to engine: {e}") async def run_with_cancellation(): nonlocal all_tasks - # 创建所有任务 + # create all tasks all_tasks = [asyncio.create_task(process_request_with_monitoring(req)) for req in req_list] - # 启动监控任务 + # start monitoring task monitor_task = asyncio.create_task(monitor_and_cancel()) try: - # 等待所有任务完成（包括被取消的） + # wait for all tasks to complete (including cancelled ones) results = await asyncio.gather(*all_tasks, return_exceptions=True) - # 处理结果，将异常转换为padding + # process results, convert exceptions to padding output_req_list = [] for i, result in enumerate(results): if isinstance(result, Exception): - # 如果是异常（包括CancelledError），创建padding + # if it is an exception (including CancelledError), create padding logger.warning(f"Task {i} resulted in exception: {result}") output_req_list.append(self._create_padding_request(req_list[i])) else: @@ -1201,14 +1181,14 @@ async def run_with_cancellation(): return output_req_list finally: - # 取消监控任务 + # cancel monitoring task monitor_task.cancel() try: await monitor_task except asyncio.CancelledError: pass - # 运行异步任务 + # run async tasks loop = asyncio.get_event_loop() output_req_list = loop.run_until_complete(run_with_cancellation()) @@ -1400,14 +1380,12 @@ async def run_with_cancellation(): ) def _create_padding_request(self, original_req: AsyncRolloutRequest) -> AsyncRolloutRequest: - """创建一个padding请求，用于替代被abort的请求。 - - 这个padding请求的特点是： - 1. 状态为COMPLETED，但包含空的response - 2. response_loss_mask全为0，确保在loss计算中被忽略 - 3. 保持原始请求的结构，但内容为空 - """ - # 创建padding的response_ids (全为pad_token_id) + # create a padding request to replace the aborted request + # the padding request has the following characteristics: + # 1. state is COMPLETED, but contains empty response + # 2. response_loss_mask is all 0, ensuring it is ignored in loss calculation + # 3. keep the original request structure, but the content is empty + # create padding response_ids (all pad_token_id) padding_response_length = self.config.response_length padding_response_ids = torch.full( (1, padding_response_length), @@ -1416,75 +1394,71 @@ def _create_padding_request(self, original_req: AsyncRolloutRequest) -> AsyncRol device=original_req.input_ids.device if original_req.input_ids is not None else "cpu", ) - # 创建padding的attention_mask (全为0) + # create padding attention_mask (all 0) padding_response_attention_mask = torch.zeros( (1, padding_response_length), dtype=torch.long, device=original_req.attention_mask.device if original_req.attention_mask is not None else "cpu", ) - # 创建padding的position_ids + # create padding position_ids if original_req.position_ids is not None: prompt_length = original_req.prompt_ids.shape[-1] if original_req.prompt_ids is not None else 0 padding_response_position_ids = torch.arange( prompt_length, prompt_length + padding_response_length, dtype=torch.long ).unsqueeze(0) if original_req.position_ids.dim() == 2: - # 如果是2D tensor (如qwen2vl) + # if it is a 2D tensor (e.g. qwen2vl) padding_response_position_ids = padding_response_position_ids.repeat( original_req.position_ids.shape[0], 1 ) else: padding_response_position_ids = None - # 创建padding的loss_mask (全为0，确保被忽略) + # create padding loss_mask (all 0, ensuring it is ignored) padding_response_loss_mask = torch.zeros( (1, padding_response_length), dtype=torch.long, device=original_req.loss_mask.device if original_req.loss_mask is not None else "cpu", ) - # 创建新的请求，保持原始结构但使用padding数据 padding_req = AsyncRolloutRequest( batch_data_id=original_req.batch_data_id, rollout_offset=original_req.rollout_offset, request_id=original_req.request_id + "_padding", state=AsyncRolloutRequestStateEnum.COMPLETED, - messages=original_req.messages, # 保持原始messages + messages=original_req.messages, multi_modal_keys=original_req.multi_modal_keys, multi_modal_data=original_req.multi_modal_data, multi_modal_inputs=original_req.multi_modal_inputs, tool_schemas=original_req.tool_schemas, tools_kwargs=original_req.tools_kwargs, interaction_kwargs=original_req.interaction_kwargs, - input_ids=original_req.input_ids, # 保持原始input_ids - prompt_ids=original_req.prompt_ids, # 保持原始prompt_ids - response_ids=padding_response_ids, # 使用padding的response_ids - attention_mask=original_req.attention_mask, # 保持原始attention_mask - prompt_attention_mask=original_req.prompt_attention_mask, # 保持原始prompt_attention_mask - response_attention_mask=padding_response_attention_mask, # 使用padding的response_attention_mask - position_ids=original_req.position_ids, # 保持原始position_ids - prompt_position_ids=original_req.prompt_position_ids, # 保持原始prompt_position_ids - response_position_ids=padding_response_position_ids, # 使用padding的response_position_ids - loss_mask=original_req.loss_mask, # 保持原始loss_mask - prompt_loss_mask=original_req.prompt_loss_mask, # 保持原始prompt_loss_mask - response_loss_mask=padding_response_loss_mask, # 使用padding的response_loss_mask (全为0) - reward_scores={}, # 空的reward_scores + input_ids=original_req.input_ids, + prompt_ids=original_req.prompt_ids, + response_ids=padding_response_ids, + attention_mask=original_req.attention_mask, + prompt_attention_mask=original_req.prompt_attention_mask, + response_attention_mask=padding_response_attention_mask, + position_ids=original_req.position_ids, + prompt_position_ids=original_req.prompt_position_ids, + response_position_ids=padding_response_position_ids, + loss_mask=original_req.loss_mask, + prompt_loss_mask=original_req.prompt_loss_mask, + response_loss_mask=padding_response_loss_mask, + reward_scores={}, max_prompt_len=original_req.max_prompt_len, max_response_len=original_req.max_response_len, - max_model_len=original_req.max_model_len, - metrics={}, # 空的metrics - output_token_ids=None, # 空的output_token_ids - rollout_log_probs=None, # 空的rollout_log_probs + metrics={}, + output_token_ids=None, + rollout_log_probs=None, use_inference_chat_template=original_req.use_inference_chat_template, tokenization_sanity_check_mode=original_req.tokenization_sanity_check_mode, generation_prompt_ids=original_req.generation_prompt_ids, base_conv_wo_gen_prompt_end_pos=original_req.base_conv_wo_gen_prompt_end_pos, base_conv_with_gen_prompt_end_pos=original_req.base_conv_with_gen_prompt_end_pos, - processing_class=self.processing_class, # 添加缺少的 processing_class 参数 + processing_class=self.processing_class, ) - - logger.info(f"Created padding request for aborted request {original_req.request_id}") return padding_req def _preprocess_prompt_to_async_rollout_requests(self, prompts: DataProto, n: int = 1) -> list[AsyncRolloutRequest]: From 3a37c6eeebe913f991757f019b443195c7622bbf Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Tue, 5 Aug 2025 14:58:45 -0700 Subject: [PATCH 24/54] modify over sample rate --- verl/workers/rollout/sglang_rollout/sglang_rollout.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index e1e74a6cbd8..f09b6c7e582 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -83,8 +83,6 @@ logger = logging.getLogger(__file__) logger.setLevel(os.getenv("VERL_LOGGING_LEVEL", "WARN")) -OVER_SAMPLE_RATE = float(os.getenv("OVER_SAMPLE_RATE", 1)) - # patch to avoid issue https://github.com/sgl-project/sglang/issues/6723 def _set_envs_and_config(server_args: ServerArgs): @@ -1100,7 +1098,7 @@ def _req_level_generate_sequences(self, prompts: DataProto, **kwargs) -> DataPro # add progress monitoring and abort function total_requests = len(req_list) - target_completion = int(total_requests * (1 - OVER_SAMPLE_RATE)) + target_completion = int(total_requests * (1 - self.config.over_sample_rate)) # abort when target_completion of requests are completed completed_count = 0 aborted_requests = [] From 94d7c682a074ce575bce6604ca7de48502287423 Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Tue, 5 Aug 2025 15:00:15 -0700 Subject: [PATCH 25/54] update examples --- examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh index b4323964409..e5ab236a4ad 100644 --- a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh +++ b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh @@ -12,7 +12,7 @@ function now() { date '+%d-%H-%M' } -EXPERIMENT_NAME="qwen2.5-3b_baseline_$(now)_$OVER_SAMPLE_RATE" +EXPERIMENT_NAME="qwen2.5-3b_baseline_$(now)" python3 -m verl.trainer.main_ppo \ --config-path="$CONFIG_PATH" \ @@ -44,10 +44,11 @@ python3 -m verl.trainer.main_ppo \ actor_rollout_ref.rollout.n=16 \ actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.rollout.over_sample_rate=0 \ algorithm.use_kl_in_reward=False \ trainer.critic_warmup=0 \ trainer.logger='["console","wandb"]' \ - trainer.project_name='benchmark_over_sample_2' \ + trainer.project_name='multi-turn-grpo-qwen2.5-3b-sglang' \ trainer.experiment_name=$EXPERIMENT_NAME \ trainer.n_gpus_per_node=8 \ trainer.nnodes=1 \ From 271b493297c7f21e1d67497edcc370bad7fb2d3c Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Tue, 5 Aug 2025 23:13:22 -0700 Subject: [PATCH 26/54] [tmp file] benchmark --- ...un_qwen2.5-3b_gsm8k_multiturn_benchmark.sh | 65 ++++++++ over_sample_enhanced.sh | 147 ++++++++++++++++++ 2 files changed, 212 insertions(+) create mode 100644 examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh create mode 100644 over_sample_enhanced.sh diff --git a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh new file mode 100644 index 00000000000..a5741cb439e --- /dev/null +++ b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh @@ -0,0 +1,65 @@ +# run on 8xH100 +# make sure your current working directory is the root of the project + +set -x + +ulimit -n 65535 + +PROJECT_DIR="$(pwd)" +CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" + +function now() { + date '+%d-%H-%M' +} + +OVER_SAMPLE_RATE=$1 + +EXPERIMENT_NAME="qwen2.5-3b_baseline_$(now)_$OVER_SAMPLE_RATE" + +python3 -m verl.trainer.main_ppo \ + --config-path="$CONFIG_PATH" \ + --config-name='gsm8k_multiturn_grpo' \ + algorithm.adv_estimator=grpo \ + data.train_batch_size=256 \ + data.max_prompt_length=1024 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.return_raw_chat=True \ + actor_rollout_ref.model.path=Qwen/Qwen2.5-3B-Instruct \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=256 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.001 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ + actor_rollout_ref.rollout.multi_stage_wake_up=True \ + actor_rollout_ref.rollout.n=16 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.rollout.over_sample_rate=$OVER_SAMPLE_RATE \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='over_sample_benchmark' \ + trainer.experiment_name=$EXPERIMENT_NAME \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=20 \ + trainer.val_before_train=True \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + actor_rollout_ref.rollout.multi_turn.tool_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/tool_config/gsm8k_tool_config.yaml" \ + trainer.total_epochs=15 \ + actor_rollout_ref.rollout.update_weights_bucket_megabytes=512 $@ + diff --git a/over_sample_enhanced.sh b/over_sample_enhanced.sh new file mode 100644 index 00000000000..79f98f6f202 --- /dev/null +++ b/over_sample_enhanced.sh @@ -0,0 +1,147 @@ +#!/bin/bash + +# 增强版实验循环脚本 +# 支持信号处理、实验恢复、资源监控等功能 + +set -e # 遇到错误时退出 + +# 信号处理函数 +cleanup() { + echo "收到中断信号，正在清理..." + if [ ! -z "$TRAIN_PID" ] && kill -0 $TRAIN_PID 2>/dev/null; then + echo "终止训练进程..." + kill -9 $TRAIN_PID 2>/dev/null + fi + pkill -f sglang + echo "清理完成，退出" + exit 0 +} + +# 设置信号处理 +trap cleanup SIGINT SIGTERM + +cd ~/verl +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + +# 创建日志目录 +LOG_DIR="~/verl/experiment_logs/$(date '+%Y%m%d_%H%M%S')" +mkdir -p "$LOG_DIR" + +# 实验状态文件 +STATE_FILE="$LOG_DIR/experiment_state.txt" + +# 检查是否传入了参数 +if [ $# -eq 0 ]; then + # 如果没有传入参数，使用默认值 + rates=(0 0.05 0.1 0.15 0.2 0.25 0.3 0.35 0.4) + echo "使用默认的 OVER_SAMPLE_RATE 数组: ${rates[*]}" +else + # 使用传入的参数作为数组 + rates=("$@") + echo "使用传入的 OVER_SAMPLE_RATE 数组: ${rates[*]}" +fi + +# 验证传入的参数是否为有效的数字 +for rate in "${rates[@]}"; do + if ! [[ "$rate" =~ ^[0-9]+\.?[0-9]*$ ]]; then + echo "错误：传入的参数 '$rate' 不是有效的数字" + exit 1 + fi +done + +# 检查系统资源 +check_system_resources() { + echo "检查系统资源..." + + # 检查GPU + if ! command -v nvidia-smi &> /dev/null; then + echo "警告: nvidia-smi 不可用" + else + echo "GPU 状态:" + nvidia-smi --query-gpu=index,name,memory.total,memory.free --format=csv,noheader,nounits | head -8 + fi + + # 检查内存 + echo "内存使用情况:" + free -h + + # 检查磁盘空间 + echo "磁盘使用情况:" + df -h ~/verl +} + +# 记录实验开始 +echo "实验开始时间: $(date)" | tee -a "$LOG_DIR/experiment_summary.log" +echo "实验参数: ${rates[*]}" | tee -a "$LOG_DIR/experiment_summary.log" + +# 检查系统资源 +check_system_resources | tee -a "$LOG_DIR/experiment_summary.log" + +# 无限循环 +while true; do + for rate in "${rates[@]}"; do + echo "==========================================" + echo "开始实验: OVER_SAMPLE_RATE = $rate" + echo "时间: $(date)" + echo "==========================================" + + # 记录实验开始 + echo "$(date): 开始 OVER_SAMPLE_RATE=$rate 的实验" | tee -a "$LOG_DIR/experiment_summary.log" + echo "$rate" > "$STATE_FILE" + + # 启动训练（后台运行） + echo "启动训练..." + bash examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh $rate > "$LOG_DIR/train_${rate}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 & + TRAIN_PID=$! + + # 等待训练完成或超时 + echo "训练将在45分钟后自动终止，或失败时立即终止..." + for i in {1..60}; do + # 检查进程是否还在运行 + if ! kill -0 $TRAIN_PID 2>/dev/null; then + echo "训练进程已结束" + # 检查退出状态 + wait $TRAIN_PID + EXIT_CODE=$? + if [ $EXIT_CODE -eq 0 ]; then + echo "训练成功完成" | tee -a "$LOG_DIR/experiment_summary.log" + else + echo "训练失败，退出码: $EXIT_CODE" | tee -a "$LOG_DIR/experiment_summary.log" + fi + break + fi + + # 每5分钟检查一次GPU使用情况 + if [ $((i % 300)) -eq 0 ]; then + echo "GPU使用情况检查 (第$((i/60))分钟):" + if command -v nvidia-smi &> /dev/null; then + nvidia-smi --query-gpu=index,name,utilization.gpu,memory.used,memory.total --format=csv,noheader,nounits | head -8 + fi + fi + + sleep 1 + done + + # 如果进程还在运行，强制终止 + if kill -0 $TRAIN_PID 2>/dev/null; then + echo "训练超时，强制终止进程..." | tee -a "$LOG_DIR/experiment_summary.log" + kill -9 $TRAIN_PID 2>/dev/null + fi + + # 清理所有 sglang 相关进程 + echo "清理 sglang 进程..." + pkill -f sglang + + # 等待3分钟 + echo "等待3分钟进行下一组实验..." + sleep 30 # 3分钟 = 180秒 + + echo "实验 OVER_SAMPLE_RATE = $rate 完成" + echo "$(date): OVER_SAMPLE_RATE=$rate 实验完成" | tee -a "$LOG_DIR/experiment_summary.log" + echo "==========================================" + done + + echo "一轮实验完成，开始下一轮..." + echo "$(date): 完成一轮实验，开始下一轮" | tee -a "$LOG_DIR/experiment_summary.log" + echo "==========================================" +done \ No newline at end of file From 6e986962d7583b05e628c1aee659bbec8692eb88 Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Tue, 5 Aug 2025 23:18:09 -0700 Subject: [PATCH 27/54] upd --- over_sample_enhanced.sh | 97 +++++------------------------------------ 1 file changed, 10 insertions(+), 87 deletions(-) diff --git a/over_sample_enhanced.sh b/over_sample_enhanced.sh index 79f98f6f202..bf15811a698 100644 --- a/over_sample_enhanced.sh +++ b/over_sample_enhanced.sh @@ -1,35 +1,11 @@ #!/bin/bash -# 增强版实验循环脚本 -# 支持信号处理、实验恢复、资源监控等功能 - -set -e # 遇到错误时退出 - -# 信号处理函数 -cleanup() { - echo "收到中断信号，正在清理..." - if [ ! -z "$TRAIN_PID" ] && kill -0 $TRAIN_PID 2>/dev/null; then - echo "终止训练进程..." - kill -9 $TRAIN_PID 2>/dev/null - fi - pkill -f sglang - echo "清理完成，退出" - exit 0 -} - -# 设置信号处理 -trap cleanup SIGINT SIGTERM +# 简化版实验循环脚本 +# 每个实验运行60秒，间隔30秒 cd ~/verl export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 -# 创建日志目录 -LOG_DIR="~/verl/experiment_logs/$(date '+%Y%m%d_%H%M%S')" -mkdir -p "$LOG_DIR" - -# 实验状态文件 -STATE_FILE="$LOG_DIR/experiment_state.txt" - # 检查是否传入了参数 if [ $# -eq 0 ]; then # 如果没有传入参数，使用默认值 @@ -49,34 +25,6 @@ for rate in "${rates[@]}"; do fi done -# 检查系统资源 -check_system_resources() { - echo "检查系统资源..." - - # 检查GPU - if ! command -v nvidia-smi &> /dev/null; then - echo "警告: nvidia-smi 不可用" - else - echo "GPU 状态:" - nvidia-smi --query-gpu=index,name,memory.total,memory.free --format=csv,noheader,nounits | head -8 - fi - - # 检查内存 - echo "内存使用情况:" - free -h - - # 检查磁盘空间 - echo "磁盘使用情况:" - df -h ~/verl -} - -# 记录实验开始 -echo "实验开始时间: $(date)" | tee -a "$LOG_DIR/experiment_summary.log" -echo "实验参数: ${rates[*]}" | tee -a "$LOG_DIR/experiment_summary.log" - -# 检查系统资源 -check_system_resources | tee -a "$LOG_DIR/experiment_summary.log" - # 无限循环 while true; do for rate in "${rates[@]}"; do @@ -85,63 +33,38 @@ while true; do echo "时间: $(date)" echo "==========================================" - # 记录实验开始 - echo "$(date): 开始 OVER_SAMPLE_RATE=$rate 的实验" | tee -a "$LOG_DIR/experiment_summary.log" - echo "$rate" > "$STATE_FILE" - # 启动训练（后台运行） echo "启动训练..." - bash examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh $rate > "$LOG_DIR/train_${rate}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 & + bash examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh $rate & TRAIN_PID=$! # 等待训练完成或超时 - echo "训练将在45分钟后自动终止，或失败时立即终止..." + echo "训练将在60秒后自动终止..." for i in {1..60}; do # 检查进程是否还在运行 if ! kill -0 $TRAIN_PID 2>/dev/null; then echo "训练进程已结束" - # 检查退出状态 - wait $TRAIN_PID - EXIT_CODE=$? - if [ $EXIT_CODE -eq 0 ]; then - echo "训练成功完成" | tee -a "$LOG_DIR/experiment_summary.log" - else - echo "训练失败，退出码: $EXIT_CODE" | tee -a "$LOG_DIR/experiment_summary.log" - fi break fi - - # 每5分钟检查一次GPU使用情况 - if [ $((i % 300)) -eq 0 ]; then - echo "GPU使用情况检查 (第$((i/60))分钟):" - if command -v nvidia-smi &> /dev/null; then - nvidia-smi --query-gpu=index,name,utilization.gpu,memory.used,memory.total --format=csv,noheader,nounits | head -8 - fi - fi - sleep 1 done - # 如果进程还在运行，强制终止 - if kill -0 $TRAIN_PID 2>/dev/null; then - echo "训练超时，强制终止进程..." | tee -a "$LOG_DIR/experiment_summary.log" - kill -9 $TRAIN_PID 2>/dev/null - fi + # 终止训练进程 + echo "终止训练进程..." + kill $TRAIN_PID 2>/dev/null # 清理所有 sglang 相关进程 echo "清理 sglang 进程..." pkill -f sglang - # 等待3分钟 - echo "等待3分钟进行下一组实验..." - sleep 30 # 3分钟 = 180秒 + # 等待30秒 + echo "等待30秒进行下一组实验..." + sleep 30 echo "实验 OVER_SAMPLE_RATE = $rate 完成" - echo "$(date): OVER_SAMPLE_RATE=$rate 实验完成" | tee -a "$LOG_DIR/experiment_summary.log" echo "==========================================" done echo "一轮实验完成，开始下一轮..." - echo "$(date): 完成一轮实验，开始下一轮" | tee -a "$LOG_DIR/experiment_summary.log" echo "==========================================" done \ No newline at end of file From a32cfb4b7b32b4a59ae927fac5f6dff4fabd7071 Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Tue, 5 Aug 2025 23:21:16 -0700 Subject: [PATCH 28/54] upd --- over_sample_enhanced.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/over_sample_enhanced.sh b/over_sample_enhanced.sh index bf15811a698..0ffa68323b3 100644 --- a/over_sample_enhanced.sh +++ b/over_sample_enhanced.sh @@ -17,12 +17,19 @@ else echo "使用传入的 OVER_SAMPLE_RATE 数组: ${rates[*]}" fi -# 验证传入的参数是否为有效的数字 +# 验证传入的参数是否为有效的数字且在合理范围内 for rate in "${rates[@]}"; do + # 检查是否为有效数字 if ! [[ "$rate" =~ ^[0-9]+\.?[0-9]*$ ]]; then echo "错误：传入的参数 '$rate' 不是有效的数字" exit 1 fi + + # 检查是否在合理范围内 (0-1) + if (( $(echo "$rate < 0" | bc -l) )) || (( $(echo "$rate > 1" | bc -l) )); then + echo "错误：传入的参数 '$rate' 超出合理范围 (0-1)" + exit 1 + fi done # 无限循环 @@ -35,7 +42,8 @@ while true; do # 启动训练（后台运行） echo "启动训练..." - bash examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh $rate & + # 使用引号保护参数，确保正确传递 + bash examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh "$rate" & TRAIN_PID=$! # 等待训练完成或超时 From ea29d312e0b037a7e7f2250fcfe86cee84805c60 Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Tue, 5 Aug 2025 23:24:10 -0700 Subject: [PATCH 29/54] fix --- .../run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh index a5741cb439e..3381171e152 100644 --- a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh +++ b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh @@ -61,5 +61,5 @@ python3 -m verl.trainer.main_ppo \ data.val_files=$HOME/data/gsm8k/test.parquet \ actor_rollout_ref.rollout.multi_turn.tool_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/tool_config/gsm8k_tool_config.yaml" \ trainer.total_epochs=15 \ - actor_rollout_ref.rollout.update_weights_bucket_megabytes=512 $@ + actor_rollout_ref.rollout.update_weights_bucket_megabytes=512 From cc8fb16dcd21f37c04cd8cfe8cde2988060fdcaf Mon Sep 17 00:00:00 2001 From: zhaochenyang Date: Tue, 5 Aug 2025 23:28:49 -0700 Subject: [PATCH 30/54] launch --- ...un_qwen2.5-3b_gsm8k_multiturn_benchmark.sh | 2 +- over_sample_enhanced.sh | 21 +++---------------- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh index 3381171e152..4ce424352a2 100644 --- a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh +++ b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh @@ -50,7 +50,7 @@ python3 -m verl.trainer.main_ppo \ algorithm.use_kl_in_reward=False \ trainer.critic_warmup=0 \ trainer.logger='["console","wandb"]' \ - trainer.project_name='over_sample_benchmark' \ + trainer.project_name='over_sample_ratio_benchmark' \ trainer.experiment_name=$EXPERIMENT_NAME \ trainer.n_gpus_per_node=8 \ trainer.nnodes=1 \ diff --git a/over_sample_enhanced.sh b/over_sample_enhanced.sh index 0ffa68323b3..16ba58e962d 100644 --- a/over_sample_enhanced.sh +++ b/over_sample_enhanced.sh @@ -17,21 +17,6 @@ else echo "使用传入的 OVER_SAMPLE_RATE 数组: ${rates[*]}" fi -# 验证传入的参数是否为有效的数字且在合理范围内 -for rate in "${rates[@]}"; do - # 检查是否为有效数字 - if ! [[ "$rate" =~ ^[0-9]+\.?[0-9]*$ ]]; then - echo "错误：传入的参数 '$rate' 不是有效的数字" - exit 1 - fi - - # 检查是否在合理范围内 (0-1) - if (( $(echo "$rate < 0" | bc -l) )) || (( $(echo "$rate > 1" | bc -l) )); then - echo "错误：传入的参数 '$rate' 超出合理范围 (0-1)" - exit 1 - fi -done - # 无限循环 while true; do for rate in "${rates[@]}"; do @@ -48,7 +33,7 @@ while true; do # 等待训练完成或超时 echo "训练将在60秒后自动终止..." - for i in {1..60}; do + for i in {1..2700}; do # 检查进程是否还在运行 if ! kill -0 $TRAIN_PID 2>/dev/null; then echo "训练进程已结束" @@ -66,8 +51,8 @@ while true; do pkill -f sglang # 等待30秒 - echo "等待30秒进行下一组实验..." - sleep 30 + echo "等待180秒进行下一组实验..." + sleep 180 echo "实验 OVER_SAMPLE_RATE = $rate 完成" echo "==========================================" From 53099d1443a6e733792249875c318223037bac52 Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Wed, 6 Aug 2025 19:30:16 -0700 Subject: [PATCH 31/54] discard dynamic batch --- examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh b/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh index 04fb6a9cd1c..32578403b10 100644 --- a/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh +++ b/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh @@ -42,9 +42,7 @@ python3 -m verl.trainer.main_ppo \ actor_rollout_ref.actor.clip_ratio_high=0.28 \ actor_rollout_ref.actor.clip_ratio_c=10.0 \ actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.actor.use_dynamic_bsz=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=8 \ - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=1024 \ + actor_rollout_ref.actor.ppo_mini_batch_size=32 \ actor_rollout_ref.rollout.name=sglang \ actor_rollout_ref.rollout.mode=async \ actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ From 3d7fa4a7180ffa4edc51a061f7e0a7325c1a897c Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Wed, 6 Aug 2025 19:37:02 -0700 Subject: [PATCH 32/54] add micro_32 --- examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh b/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh index 32578403b10..1345265d310 100644 --- a/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh +++ b/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh @@ -43,6 +43,7 @@ python3 -m verl.trainer.main_ppo \ actor_rollout_ref.actor.clip_ratio_c=10.0 \ actor_rollout_ref.actor.optim.lr=1e-6 \ actor_rollout_ref.actor.ppo_mini_batch_size=32 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ actor_rollout_ref.rollout.name=sglang \ actor_rollout_ref.rollout.mode=async \ actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ From 5a733f853e1b0536e820a97257a7feca1783ec49 Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Wed, 6 Aug 2025 19:40:27 -0700 Subject: [PATCH 33/54] log bs --- examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh b/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh index 1345265d310..bd0f2c9559c 100644 --- a/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh +++ b/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh @@ -44,6 +44,7 @@ python3 -m verl.trainer.main_ppo \ actor_rollout_ref.actor.optim.lr=1e-6 \ actor_rollout_ref.actor.ppo_mini_batch_size=32 \ actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ actor_rollout_ref.rollout.name=sglang \ actor_rollout_ref.rollout.mode=async \ actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ From 4cc54b8fe8582a1722d758f668b2c573e3b73623 Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Wed, 6 Aug 2025 19:43:37 -0700 Subject: [PATCH 34/54] Add log_prob_micro_batch_size_per_gpu --- examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh b/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh index bd0f2c9559c..99228998e47 100644 --- a/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh +++ b/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh @@ -45,9 +45,10 @@ python3 -m verl.trainer.main_ppo \ actor_rollout_ref.actor.ppo_mini_batch_size=32 \ actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ actor_rollout_ref.rollout.name=sglang \ actor_rollout_ref.rollout.mode=async \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.tensor_model_parallel_size= 1 \ actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ actor_rollout_ref.rollout.multi_stage_wake_up=True \ actor_rollout_ref.rollout.multi_turn.enable=True \ From c67ebfffbab9a615cf08af01bc7b7784e9094d18 Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Wed, 6 Aug 2025 19:47:57 -0700 Subject: [PATCH 35/54] Add ppo_max_token_len_per_gpu --- examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh b/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh index 99228998e47..c34355f5806 100644 --- a/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh +++ b/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh @@ -44,11 +44,12 @@ python3 -m verl.trainer.main_ppo \ actor_rollout_ref.actor.optim.lr=1e-6 \ actor_rollout_ref.actor.ppo_mini_batch_size=32 \ actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=32768 \ actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ actor_rollout_ref.rollout.name=sglang \ actor_rollout_ref.rollout.mode=async \ - actor_rollout_ref.rollout.tensor_model_parallel_size= 1 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ actor_rollout_ref.rollout.multi_stage_wake_up=True \ actor_rollout_ref.rollout.multi_turn.enable=True \ From 43a9c31c07c2858672612a23217a3c66d72b3481 Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Wed, 6 Aug 2025 20:29:51 -0700 Subject: [PATCH 36/54] add qwen3 4b --- examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh b/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh index c34355f5806..69fadd730fb 100644 --- a/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh +++ b/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh @@ -33,7 +33,7 @@ python3 -m verl.trainer.main_ppo \ data.custom_cls.name=CustomRLHFDataset \ custom_reward_function.path=$PROJECT_DIR/recipe/retool/retool.py \ custom_reward_function.name=compute_score \ - actor_rollout_ref.model.path=Qwen/Qwen2.5-3B-Instruct \ + actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507 \ actor_rollout_ref.model.use_remove_padding=True \ actor_rollout_ref.model.enable_gradient_checkpointing=True \ actor_rollout_ref.actor.use_kl_loss=False \ @@ -50,7 +50,7 @@ python3 -m verl.trainer.main_ppo \ actor_rollout_ref.rollout.name=sglang \ actor_rollout_ref.rollout.mode=async \ actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \ actor_rollout_ref.rollout.multi_stage_wake_up=True \ actor_rollout_ref.rollout.multi_turn.enable=True \ actor_rollout_ref.rollout.multi_turn.max_user_turns=16 \ @@ -63,7 +63,7 @@ python3 -m verl.trainer.main_ppo \ actor_rollout_ref.rollout.val_kwargs.n=30 \ trainer.logger=['console','wandb'] \ trainer.project_name=sglang-dapo-multiturn \ - trainer.experiment_name=qwen2_5-3b_dapo_multiturn \ + trainer.experiment_name=qwen3-4b_dapo_multiturn \ trainer.n_gpus_per_node=4 \ trainer.log_val_generations=20 \ trainer.val_before_train=True \ From 8e17a936158fcb1252292b4140e4dd71a9845bec Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Wed, 6 Aug 2025 20:32:42 -0700 Subject: [PATCH 37/54] use qwen3 4b --- ..._qwen2_3b_dapo_multiturn.sh => run_qwen3_4b_dapo_multiturn.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/sglang_multiturn/{run_qwen2_3b_dapo_multiturn.sh => run_qwen3_4b_dapo_multiturn.sh} (100%) diff --git a/examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh b/examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh similarity index 100% rename from examples/sglang_multiturn/run_qwen2_3b_dapo_multiturn.sh rename to examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh From 71687d765ff8a4e216a3c4aca206790b7e4c8c84 Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Wed, 6 Aug 2025 20:54:58 -0700 Subject: [PATCH 38/54] descrease micro --- examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh b/examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh index 69fadd730fb..a1f765f3bef 100644 --- a/examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh +++ b/examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh @@ -43,10 +43,10 @@ python3 -m verl.trainer.main_ppo \ actor_rollout_ref.actor.clip_ratio_c=10.0 \ actor_rollout_ref.actor.optim.lr=1e-6 \ actor_rollout_ref.actor.ppo_mini_batch_size=32 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \ actor_rollout_ref.actor.ppo_max_token_len_per_gpu=32768 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ actor_rollout_ref.rollout.name=sglang \ actor_rollout_ref.rollout.mode=async \ actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ From b084dfd1e5db11aeca917cee73e0bb61d63fb112 Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Wed, 6 Aug 2025 21:28:33 -0700 Subject: [PATCH 39/54] update dapo for benchmarking --- ...un_qwen2.5-3b_gsm8k_multiturn_benchmark.sh | 65 ------------- .../run_qwen3-4b_dpo_multiturn_benchmark.sh | 92 +++++++++++++++++++ .../run_qwen3_4b_dapo_multiturn.sh | 1 + over_sample_enhanced.sh | 15 ++- 4 files changed, 103 insertions(+), 70 deletions(-) delete mode 100644 examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh create mode 100644 examples/sglang_multiturn/run_qwen3-4b_dpo_multiturn_benchmark.sh diff --git a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh deleted file mode 100644 index 4ce424352a2..00000000000 --- a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh +++ /dev/null @@ -1,65 +0,0 @@ -# run on 8xH100 -# make sure your current working directory is the root of the project - -set -x - -ulimit -n 65535 - -PROJECT_DIR="$(pwd)" -CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" - -function now() { - date '+%d-%H-%M' -} - -OVER_SAMPLE_RATE=$1 - -EXPERIMENT_NAME="qwen2.5-3b_baseline_$(now)_$OVER_SAMPLE_RATE" - -python3 -m verl.trainer.main_ppo \ - --config-path="$CONFIG_PATH" \ - --config-name='gsm8k_multiturn_grpo' \ - algorithm.adv_estimator=grpo \ - data.train_batch_size=256 \ - data.max_prompt_length=1024 \ - data.max_response_length=1024 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - data.return_raw_chat=True \ - actor_rollout_ref.model.path=Qwen/Qwen2.5-3B-Instruct \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.actor.ppo_mini_batch_size=256 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=32 \ - actor_rollout_ref.actor.use_kl_loss=True \ - actor_rollout_ref.actor.kl_loss_coef=0.001 \ - actor_rollout_ref.actor.kl_loss_type=low_var_kl \ - actor_rollout_ref.actor.entropy_coeff=0 \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.fsdp_config.param_offload=False \ - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=32 \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.name=sglang \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ - actor_rollout_ref.rollout.multi_stage_wake_up=True \ - actor_rollout_ref.rollout.n=16 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ - actor_rollout_ref.ref.fsdp_config.param_offload=True \ - actor_rollout_ref.rollout.over_sample_rate=$OVER_SAMPLE_RATE \ - algorithm.use_kl_in_reward=False \ - trainer.critic_warmup=0 \ - trainer.logger='["console","wandb"]' \ - trainer.project_name='over_sample_ratio_benchmark' \ - trainer.experiment_name=$EXPERIMENT_NAME \ - trainer.n_gpus_per_node=8 \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=20 \ - trainer.val_before_train=True \ - data.train_files=$HOME/data/gsm8k/train.parquet \ - data.val_files=$HOME/data/gsm8k/test.parquet \ - actor_rollout_ref.rollout.multi_turn.tool_config_path="$PROJECT_DIR/examples/sglang_multiturn/config/tool_config/gsm8k_tool_config.yaml" \ - trainer.total_epochs=15 \ - actor_rollout_ref.rollout.update_weights_bucket_megabytes=512 - diff --git a/examples/sglang_multiturn/run_qwen3-4b_dpo_multiturn_benchmark.sh b/examples/sglang_multiturn/run_qwen3-4b_dpo_multiturn_benchmark.sh new file mode 100644 index 00000000000..f414748bf4e --- /dev/null +++ b/examples/sglang_multiturn/run_qwen3-4b_dpo_multiturn_benchmark.sh @@ -0,0 +1,92 @@ +# run on 8xH100 +# make sure your current working directory is the root of the project + +set -x + +ulimit -n 65535 + +PROJECT_DIR="$(pwd)" +CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" + +function now() { + date '+%d-%H-%M' +} + +OVER_SAMPLE_RATE=$1 + + +PROJECT_DIR="$(pwd)" +CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" + +hf download \ + BytedTsinghua-SIA/DAPO-Math-17k \ + --repo-type dataset \ + --local-dir $HOME/data/BytedTsinghua-SIA/DAPO-Math-17k + + +hf download \ + Maxwell-Jia/AIME_2024 \ + --repo-type dataset \ + --local-dir $HOME/data/Maxwell-Jia/AIME_2024 + + +EXPERIMENT_NAME="qwen3-4b_$(now)_$OVER_SAMPLE_RATE" + + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + algorithm.use_kl_in_reward=False \ + algorithm.kl_ctrl.kl_coef=0.0 \ + data.train_files=$HOME/data/BytedTsinghua-SIA/DAPO-Math-17k \ + data.val_files=$HOME/data/Maxwell-Jia/AIME_2024 \ + data.return_raw_chat=True \ + data.train_batch_size=32 \ + data.max_prompt_length=2048 \ + data.max_response_length=16384 \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + data.custom_cls.path=$PROJECT_DIR/recipe/retool/retool.py \ + data.custom_cls.name=CustomRLHFDataset \ + custom_reward_function.path=$PROJECT_DIR/recipe/retool/retool.py \ + custom_reward_function.name=compute_score \ + actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.kl_loss_coef=0.0 \ + actor_rollout_ref.actor.clip_ratio_low=0.2 \ + actor_rollout_ref.actor.clip_ratio_high=0.28 \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=32 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=32768 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.rollout.name=sglang \ + actor_rollout_ref.rollout.mode=async \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \ + actor_rollout_ref.rollout.update_weights_bucket_megabytes=512 \ + actor_rollout_ref.rollout.multi_stage_wake_up=True \ + actor_rollout_ref.rollout.multi_turn.enable=True \ + actor_rollout_ref.rollout.multi_turn.max_user_turns=16 \ + actor_rollout_ref.rollout.multi_turn.max_assistant_turns=16 \ + actor_rollout_ref.rollout.multi_turn.tool_config_path=$PROJECT_DIR/recipe/retool/sandbox_fusion_tool_config.yaml \ + actor_rollout_ref.rollout.multi_turn.format=hermes \ + actor_rollout_ref.rollout.n=8 \ + actor_rollout_ref.rollout.over_sample_rate=$OVER_SAMPLE_RATE \ + actor_rollout_ref.rollout.val_kwargs.top_p=0.6 \ + actor_rollout_ref.rollout.val_kwargs.temperature=1.0 \ + actor_rollout_ref.rollout.val_kwargs.n=30 \ + trainer.logger=['console','wandb'] \ + trainer.project_name=sglang-dapo-multiturn \ + trainer.experiment_name=$EXPERIMENT_NAME \ + trainer.n_gpus_per_node=4 \ + trainer.log_val_generations=20 \ + trainer.val_before_train=True \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=20 \ + trainer.total_epochs=15 \ + $@ \ No newline at end of file diff --git a/examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh b/examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh index a1f765f3bef..80e3b8120b9 100644 --- a/examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh +++ b/examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh @@ -51,6 +51,7 @@ python3 -m verl.trainer.main_ppo \ actor_rollout_ref.rollout.mode=async \ actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \ + actor_rollout_ref.rollout.update_weights_bucket_megabytes=512 \ actor_rollout_ref.rollout.multi_stage_wake_up=True \ actor_rollout_ref.rollout.multi_turn.enable=True \ actor_rollout_ref.rollout.multi_turn.max_user_turns=16 \ diff --git a/over_sample_enhanced.sh b/over_sample_enhanced.sh index 16ba58e962d..0f063de856d 100644 --- a/over_sample_enhanced.sh +++ b/over_sample_enhanced.sh @@ -28,12 +28,12 @@ while true; do # 启动训练（后台运行） echo "启动训练..." # 使用引号保护参数，确保正确传递 - bash examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_benchmark.sh "$rate" & + bash examples/sglang_multiturn/run_qwen3-4b_dpo_multiturn_benchmark.sh "$rate" & TRAIN_PID=$! # 等待训练完成或超时 - echo "训练将在60秒后自动终止..." - for i in {1..2700}; do + echo "训练将在45分钟后自动终止..." + for i in {1..600}; do # 检查进程是否还在运行 if ! kill -0 $TRAIN_PID 2>/dev/null; then echo "训练进程已结束" @@ -50,9 +50,14 @@ while true; do echo "清理 sglang 进程..." pkill -f sglang - # 等待30秒 + # 额外清理：确保没有残留的Python进程 + echo "清理可能的Python训练进程..." + pkill -f "python.*train" 2>/dev/null + pkill -f "python.*run_qwen" 2>/dev/null + + # 等待180秒 echo "等待180秒进行下一组实验..." - sleep 180 + sleep 60 echo "实验 OVER_SAMPLE_RATE = $rate 完成" echo "==========================================" From b66327e2eae0c75133aac7f69639e73f8e1b84b2 Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Wed, 6 Aug 2025 21:30:44 -0700 Subject: [PATCH 40/54] fix hydra --- .../sglang_multiturn/run_qwen3-4b_dpo_multiturn_benchmark.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/sglang_multiturn/run_qwen3-4b_dpo_multiturn_benchmark.sh b/examples/sglang_multiturn/run_qwen3-4b_dpo_multiturn_benchmark.sh index f414748bf4e..9a33aacddb9 100644 --- a/examples/sglang_multiturn/run_qwen3-4b_dpo_multiturn_benchmark.sh +++ b/examples/sglang_multiturn/run_qwen3-4b_dpo_multiturn_benchmark.sh @@ -88,5 +88,4 @@ python3 -m verl.trainer.main_ppo \ trainer.nnodes=1 \ trainer.save_freq=-1 \ trainer.test_freq=20 \ - trainer.total_epochs=15 \ - $@ \ No newline at end of file + trainer.total_epochs=15 \ No newline at end of file From 23c2957fe19904a0640671f6b0a830e7838beb43 Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Wed, 6 Aug 2025 21:35:51 -0700 Subject: [PATCH 41/54] use 8 gpu --- .../sglang_multiturn/run_qwen3-4b_dpo_multiturn_benchmark.sh | 2 +- examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/sglang_multiturn/run_qwen3-4b_dpo_multiturn_benchmark.sh b/examples/sglang_multiturn/run_qwen3-4b_dpo_multiturn_benchmark.sh index 9a33aacddb9..2a2ce6f7237 100644 --- a/examples/sglang_multiturn/run_qwen3-4b_dpo_multiturn_benchmark.sh +++ b/examples/sglang_multiturn/run_qwen3-4b_dpo_multiturn_benchmark.sh @@ -82,7 +82,7 @@ python3 -m verl.trainer.main_ppo \ trainer.logger=['console','wandb'] \ trainer.project_name=sglang-dapo-multiturn \ trainer.experiment_name=$EXPERIMENT_NAME \ - trainer.n_gpus_per_node=4 \ + trainer.n_gpus_per_node=8 \ trainer.log_val_generations=20 \ trainer.val_before_train=True \ trainer.nnodes=1 \ diff --git a/examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh b/examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh index 80e3b8120b9..7de37122449 100644 --- a/examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh +++ b/examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh @@ -65,7 +65,7 @@ python3 -m verl.trainer.main_ppo \ trainer.logger=['console','wandb'] \ trainer.project_name=sglang-dapo-multiturn \ trainer.experiment_name=qwen3-4b_dapo_multiturn \ - trainer.n_gpus_per_node=4 \ + trainer.n_gpus_per_node=8 \ trainer.log_val_generations=20 \ trainer.val_before_train=True \ trainer.nnodes=1 \ From 97d137df83bd23ccaf79496488f5d9b2ca05e980 Mon Sep 17 00:00:00 2001 From: PopSoda2002 Date: Thu, 7 Aug 2025 04:46:30 +0000 Subject: [PATCH 42/54] delete not used variale --- verl/workers/rollout/sglang_rollout/sglang_rollout.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index 31824e6907c..964f2d04bef 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -1130,12 +1130,9 @@ async def monitor_and_cancel(): while completed_count < target_completion: await asyncio.sleep(0.1) - # cancel remaining tasks - cancelled_count = 0 for task in all_tasks: if not task.done(): task.cancel() - cancelled_count += 1 # send abort signal to engine, interrupt all ongoing requests try: From 35b65eb92621a63f0bf1a3023eab8e8ff443d00b Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Wed, 6 Aug 2025 21:50:23 -0700 Subject: [PATCH 43/54] use full set to eval --- over_sample_enhanced.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/over_sample_enhanced.sh b/over_sample_enhanced.sh index 0f063de856d..9c02b913407 100644 --- a/over_sample_enhanced.sh +++ b/over_sample_enhanced.sh @@ -33,7 +33,7 @@ while true; do # 等待训练完成或超时 echo "训练将在45分钟后自动终止..." - for i in {1..600}; do + for i in {1..2700}; do # 检查进程是否还在运行 if ! kill -0 $TRAIN_PID 2>/dev/null; then echo "训练进程已结束" @@ -57,7 +57,7 @@ while true; do # 等待180秒 echo "等待180秒进行下一组实验..." - sleep 60 + sleep 180 echo "实验 OVER_SAMPLE_RATE = $rate 完成" echo "==========================================" From 4c882d18e8b781c411870a3e119ec571909321e9 Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Mon, 11 Aug 2025 11:44:26 -0700 Subject: [PATCH 44/54] finish over sampling --- .../run_qwen3-4b_dpo_multiturn_benchmark.sh | 91 ------------------- over_sample_enhanced.sh | 68 -------------- verl/trainer/ppo/metric_utils.py | 26 +++++- .../rollout/sglang_rollout/sglang_rollout.py | 2 +- 4 files changed, 25 insertions(+), 162 deletions(-) delete mode 100644 examples/sglang_multiturn/run_qwen3-4b_dpo_multiturn_benchmark.sh delete mode 100644 over_sample_enhanced.sh diff --git a/examples/sglang_multiturn/run_qwen3-4b_dpo_multiturn_benchmark.sh b/examples/sglang_multiturn/run_qwen3-4b_dpo_multiturn_benchmark.sh deleted file mode 100644 index 2a2ce6f7237..00000000000 --- a/examples/sglang_multiturn/run_qwen3-4b_dpo_multiturn_benchmark.sh +++ /dev/null @@ -1,91 +0,0 @@ -# run on 8xH100 -# make sure your current working directory is the root of the project - -set -x - -ulimit -n 65535 - -PROJECT_DIR="$(pwd)" -CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" - -function now() { - date '+%d-%H-%M' -} - -OVER_SAMPLE_RATE=$1 - - -PROJECT_DIR="$(pwd)" -CONFIG_PATH="$PROJECT_DIR/examples/sglang_multiturn/config" - -hf download \ - BytedTsinghua-SIA/DAPO-Math-17k \ - --repo-type dataset \ - --local-dir $HOME/data/BytedTsinghua-SIA/DAPO-Math-17k - - -hf download \ - Maxwell-Jia/AIME_2024 \ - --repo-type dataset \ - --local-dir $HOME/data/Maxwell-Jia/AIME_2024 - - -EXPERIMENT_NAME="qwen3-4b_$(now)_$OVER_SAMPLE_RATE" - - -python3 -m verl.trainer.main_ppo \ - algorithm.adv_estimator=grpo \ - algorithm.use_kl_in_reward=False \ - algorithm.kl_ctrl.kl_coef=0.0 \ - data.train_files=$HOME/data/BytedTsinghua-SIA/DAPO-Math-17k \ - data.val_files=$HOME/data/Maxwell-Jia/AIME_2024 \ - data.return_raw_chat=True \ - data.train_batch_size=32 \ - data.max_prompt_length=2048 \ - data.max_response_length=16384 \ - data.filter_overlong_prompts=True \ - data.truncation='error' \ - data.custom_cls.path=$PROJECT_DIR/recipe/retool/retool.py \ - data.custom_cls.name=CustomRLHFDataset \ - custom_reward_function.path=$PROJECT_DIR/recipe/retool/retool.py \ - custom_reward_function.name=compute_score \ - actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507 \ - actor_rollout_ref.model.use_remove_padding=True \ - actor_rollout_ref.model.enable_gradient_checkpointing=True \ - actor_rollout_ref.actor.use_kl_loss=False \ - actor_rollout_ref.actor.kl_loss_coef=0.0 \ - actor_rollout_ref.actor.clip_ratio_low=0.2 \ - actor_rollout_ref.actor.clip_ratio_high=0.28 \ - actor_rollout_ref.actor.clip_ratio_c=10.0 \ - actor_rollout_ref.actor.optim.lr=1e-6 \ - actor_rollout_ref.actor.ppo_mini_batch_size=32 \ - actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=32768 \ - actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ - actor_rollout_ref.rollout.name=sglang \ - actor_rollout_ref.rollout.mode=async \ - actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ - actor_rollout_ref.rollout.gpu_memory_utilization=0.80 \ - actor_rollout_ref.rollout.update_weights_bucket_megabytes=512 \ - actor_rollout_ref.rollout.multi_stage_wake_up=True \ - actor_rollout_ref.rollout.multi_turn.enable=True \ - actor_rollout_ref.rollout.multi_turn.max_user_turns=16 \ - actor_rollout_ref.rollout.multi_turn.max_assistant_turns=16 \ - actor_rollout_ref.rollout.multi_turn.tool_config_path=$PROJECT_DIR/recipe/retool/sandbox_fusion_tool_config.yaml \ - actor_rollout_ref.rollout.multi_turn.format=hermes \ - actor_rollout_ref.rollout.n=8 \ - actor_rollout_ref.rollout.over_sample_rate=$OVER_SAMPLE_RATE \ - actor_rollout_ref.rollout.val_kwargs.top_p=0.6 \ - actor_rollout_ref.rollout.val_kwargs.temperature=1.0 \ - actor_rollout_ref.rollout.val_kwargs.n=30 \ - trainer.logger=['console','wandb'] \ - trainer.project_name=sglang-dapo-multiturn \ - trainer.experiment_name=$EXPERIMENT_NAME \ - trainer.n_gpus_per_node=8 \ - trainer.log_val_generations=20 \ - trainer.val_before_train=True \ - trainer.nnodes=1 \ - trainer.save_freq=-1 \ - trainer.test_freq=20 \ - trainer.total_epochs=15 \ No newline at end of file diff --git a/over_sample_enhanced.sh b/over_sample_enhanced.sh deleted file mode 100644 index 9c02b913407..00000000000 --- a/over_sample_enhanced.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash - -# 简化版实验循环脚本 -# 每个实验运行60秒，间隔30秒 - -cd ~/verl -export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 - -# 检查是否传入了参数 -if [ $# -eq 0 ]; then - # 如果没有传入参数，使用默认值 - rates=(0 0.05 0.1 0.15 0.2 0.25 0.3 0.35 0.4) - echo "使用默认的 OVER_SAMPLE_RATE 数组: ${rates[*]}" -else - # 使用传入的参数作为数组 - rates=("$@") - echo "使用传入的 OVER_SAMPLE_RATE 数组: ${rates[*]}" -fi - -# 无限循环 -while true; do - for rate in "${rates[@]}"; do - echo "==========================================" - echo "开始实验: OVER_SAMPLE_RATE = $rate" - echo "时间: $(date)" - echo "==========================================" - - # 启动训练（后台运行） - echo "启动训练..." - # 使用引号保护参数，确保正确传递 - bash examples/sglang_multiturn/run_qwen3-4b_dpo_multiturn_benchmark.sh "$rate" & - TRAIN_PID=$! - - # 等待训练完成或超时 - echo "训练将在45分钟后自动终止..." - for i in {1..2700}; do - # 检查进程是否还在运行 - if ! kill -0 $TRAIN_PID 2>/dev/null; then - echo "训练进程已结束" - break - fi - sleep 1 - done - - # 终止训练进程 - echo "终止训练进程..." - kill $TRAIN_PID 2>/dev/null - - # 清理所有 sglang 相关进程 - echo "清理 sglang 进程..." - pkill -f sglang - - # 额外清理：确保没有残留的Python进程 - echo "清理可能的Python训练进程..." - pkill -f "python.*train" 2>/dev/null - pkill -f "python.*run_qwen" 2>/dev/null - - # 等待180秒 - echo "等待180秒进行下一组实验..." - sleep 180 - - echo "实验 OVER_SAMPLE_RATE = $rate 完成" - echo "==========================================" - done - - echo "一轮实验完成，开始下一轮..." - echo "==========================================" -done \ No newline at end of file diff --git a/verl/trainer/ppo/metric_utils.py b/verl/trainer/ppo/metric_utils.py index f0bf71912d8..aefea35fb0a 100644 --- a/verl/trainer/ppo/metric_utils.py +++ b/verl/trainer/ppo/metric_utils.py @@ -121,8 +121,6 @@ def compute_data_metrics(batch: DataProto, use_critic: bool = True) -> dict[str, aborted_mask = (response_length == 0).bool() non_aborted_mask = ~aborted_mask - print("over sample rate in metric_utils: ", non_aborted_mask.sum() / len(non_aborted_mask)) - non_aborted_sequence_score = sequence_score[non_aborted_mask] non_aborted_sequence_reward = sequence_reward[non_aborted_mask] @@ -143,6 +141,21 @@ def compute_data_metrics(batch: DataProto, use_critic: bool = True) -> dict[str, return_diff_var = torch.var(valid_returns - valid_values) return_var = torch.var(valid_returns) + # Aborted samples and non-aborted response length statistics + # response_length_non_aborted/*: statistics computed on non-aborted samples only + aborted_ratio = torch.mean(aborted_mask.float()).detach().item() + + non_aborted_response_length = response_length[non_aborted_mask] + if non_aborted_response_length.numel() > 0: + non_aborted_response_length_mean = torch.mean(non_aborted_response_length).detach().item() + non_aborted_response_length_max = torch.max(non_aborted_response_length).detach().item() + non_aborted_response_length_min = torch.min(non_aborted_response_length).detach().item() + non_aborted_response_length_clip_ratio = ( + torch.mean(torch.eq(non_aborted_response_length, max_response_length).float()).detach().item() + ) + else: + raise ValueError("All samples are aborted, this should not happen.") + metrics = { # score "critic/score/mean": score_mean, @@ -179,6 +192,15 @@ def compute_data_metrics(batch: DataProto, use_critic: bool = True) -> dict[str, "response_length/clip_ratio": torch.mean(torch.eq(response_length, max_response_length).float()) .detach() .item(), + # response length (non-aborted only) + # These statistics exclude aborted samples to avoid skew from zeros + "response_length_non_aborted/mean": non_aborted_response_length_mean, + "response_length_non_aborted/max": non_aborted_response_length_max, + "response_length_non_aborted/min": non_aborted_response_length_min, + "response_length_non_aborted/clip_ratio": non_aborted_response_length_clip_ratio, + # aborted ratio + # Fraction of samples whose response length is zero + "response/aborted_ratio": aborted_ratio, # prompt length "prompt_length/mean": torch.mean(prompt_length).detach().item(), "prompt_length/max": torch.max(prompt_length).detach().item(), diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index 61ff21e4718..e21d77260ab 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -1089,6 +1089,7 @@ def _req_level_generate_sequences(self, prompts: DataProto, **kwargs) -> DataPro total_requests = len(req_list) target_completion = int(total_requests * (1 - self.config.over_sample_rate)) # abort when target_completion of requests are completed + completed_count = 0 aborted_requests = [] @@ -1172,7 +1173,6 @@ async def run_with_cancellation(): except asyncio.CancelledError: pass - # run async tasks loop = asyncio.get_event_loop() output_req_list = loop.run_until_complete(run_with_cancellation()) From a794b292435159c12c0d5c801663241f95db329a Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Wed, 13 Aug 2025 18:16:27 -0700 Subject: [PATCH 45/54] fix rollout config --- examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh | 4 ++-- verl/workers/config/rollout.py | 4 ++++ verl/workers/rollout/sglang_rollout/sglang_rollout.py | 3 ++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh index e5ab236a4ad..81613eefcec 100644 --- a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh +++ b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh @@ -40,11 +40,11 @@ python3 -m verl.trainer.main_ppo \ actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ actor_rollout_ref.rollout.name=sglang \ actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ - actor_rollout_ref.rollout.multi_stage_wake_up=True \ + actor_rollout_ref.rollout.multi_stage_wake_up=False \ actor_rollout_ref.rollout.n=16 \ actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ actor_rollout_ref.ref.fsdp_config.param_offload=True \ - actor_rollout_ref.rollout.over_sample_rate=0 \ + actor_rollout_ref.rollout.over_sample_rate=0.1 \ algorithm.use_kl_in_reward=False \ trainer.critic_warmup=0 \ trainer.logger='["console","wandb"]' \ diff --git a/verl/workers/config/rollout.py b/verl/workers/config/rollout.py index 370a61f928c..0b5e21dbfbe 100644 --- a/verl/workers/config/rollout.py +++ b/verl/workers/config/rollout.py @@ -86,6 +86,10 @@ class RolloutConfig(BaseConfig): do_sample: bool = True n: int = 1 + # Early termination threshold for multi-turn rollout in sglang. + # Abort remaining requests when (1 - over_sample_rate) * total_requests are completed. + over_sample_rate: float = 0.0 + prompt_length: int = 512 response_length: int = 512 diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index eaa22016a28..a8975fabaa0 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -463,7 +463,7 @@ def _init_inference_engine(self, trust_remote_code, actor_module, port): trust_remote_code=trust_remote_code, # NOTE(linjunrong): add rank to prevent SGLang generate same port inside PortArgs.init_new # when random.seed is being set during training - port=30000 + rank, + port=302000 + rank, # NOTE(Chenyang): if you want to debug the SGLang engine output # please set the following parameters # Otherwise, it will make the engine run too slow @@ -1088,6 +1088,7 @@ def _req_level_generate_sequences(self, prompts: DataProto, **kwargs) -> DataPro # add progress monitoring and abort function total_requests = len(req_list) target_completion = int(total_requests * (1 - self.config.over_sample_rate)) + print(f"target_completion: {target_completion}") # abort when target_completion of requests are completed completed_count = 0 From cdc2eff5cdb5a99b43d5a97984ef05af2128e461 Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Wed, 13 Aug 2025 18:38:52 -0700 Subject: [PATCH 46/54] do not capture all --- .../rollout/sglang_rollout/sglang_rollout.py | 23 ++----------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index a8975fabaa0..88429fb6212 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -1122,10 +1122,6 @@ async def process_request_with_monitoring(req): logger.info(f"Request {req.request_id} was cancelled, creating padding") aborted_requests.append(req.request_id) return self._create_padding_request(req) - except Exception as e: - logger.error(f"Uncaught exception in process_request_with_monitoring: {e}") - logger.error("This shall not happen, please check the code") - raise e async def monitor_and_cancel(): nonlocal completed_count @@ -1137,10 +1133,7 @@ async def monitor_and_cancel(): task.cancel() # send abort signal to engine, interrupt all ongoing requests - try: - await self._engine.abort_request(abort_all=True) - except Exception as e: - logger.error(f"Failed to send abort signal to engine: {e}") + await self._engine.abort_request(abort_all=True) async def run_with_cancellation(): nonlocal all_tasks @@ -1153,19 +1146,7 @@ async def run_with_cancellation(): try: # wait for all tasks to complete (including cancelled ones) - results = await asyncio.gather(*all_tasks, return_exceptions=True) - - # process results, convert exceptions to padding - output_req_list = [] - for i, result in enumerate(results): - if isinstance(result, Exception): - # if it is an exception (including CancelledError), create padding - logger.warning(f"Task {i} resulted in exception: {result}") - output_req_list.append(self._create_padding_request(req_list[i])) - else: - output_req_list.append(result) - - return output_req_list + return await asyncio.gather(*all_tasks, return_exceptions=True) finally: # cancel monitoring task monitor_task.cancel() From 77e8491d39932fc008827b5582d05ac88e38f03e Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Wed, 13 Aug 2025 18:41:13 -0700 Subject: [PATCH 47/54] clean up validation --- verl/workers/rollout/sglang_rollout/sglang_rollout.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index 88429fb6212..072ea927c2b 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -1096,14 +1096,12 @@ def _req_level_generate_sequences(self, prompts: DataProto, **kwargs) -> DataPro # distinguish training and validation if is_validate: - # validation mode: process all requests without abort - async def process_all_requests(): - return await asyncio.gather( + # Validation mode: process all requests without abort + output_req_list = asyncio.run( + asyncio.gather( *[self._async_rollout_a_request(req, do_sample, is_validate, **kwargs) for req in req_list], ) - - loop = asyncio.get_event_loop() - output_req_list = loop.run_until_complete(process_all_requests()) + ) else: completion_lock = asyncio.Lock() all_tasks = [] From 4fd6b36079489220146ac24b5bc80d41413b8913 Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Wed, 13 Aug 2025 19:11:23 -0700 Subject: [PATCH 48/54] fix conflicts register --- .../rollout/sglang_rollout/sglang_rollout.py | 81 +++++++++++-------- 1 file changed, 46 insertions(+), 35 deletions(-) diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index 072ea927c2b..0cf0af6b9fd 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -25,6 +25,26 @@ from typing import Any, Optional from uuid import uuid4 +# Workaround: avoid duplicate AutoConfig.register conflicts (e.g., 'aimv2') +try: + from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig + + _orig_ac_register = AutoConfig.register + + def _safe_ac_register(model_type, config, exist_ok=False): + return _orig_ac_register(model_type, config, exist_ok=True) + + AutoConfig.register = _safe_ac_register + + _orig_cfg_register = CONFIG_MAPPING.register + + def _safe_cfg_register(key, config, exist_ok=False): + return _orig_cfg_register(key, config, exist_ok=True) + + CONFIG_MAPPING.register = _safe_cfg_register +except Exception: + pass + import numpy as np import sglang.srt.entrypoints.engine import torch @@ -1103,58 +1123,49 @@ def _req_level_generate_sequences(self, prompts: DataProto, **kwargs) -> DataPro ) ) else: - completion_lock = asyncio.Lock() all_tasks = [] - async def process_request_with_monitoring(req): - nonlocal completed_count + async def rollout_a_request_with_cancellation_handler(req): try: result = await self._async_rollout_a_request(req, do_sample, is_validate, **kwargs) - - async with completion_lock: - if completed_count < target_completion: - completed_count += 1 - return result + return result except asyncio.CancelledError: # request is cancelled, return padding logger.info(f"Request {req.request_id} was cancelled, creating padding") aborted_requests.append(req.request_id) return self._create_padding_request(req) - async def monitor_and_cancel(): - nonlocal completed_count - while completed_count < target_completion: - await asyncio.sleep(0.1) - - for task in all_tasks: - if not task.done(): - task.cancel() - - # send abort signal to engine, interrupt all ongoing requests - await self._engine.abort_request(abort_all=True) - async def run_with_cancellation(): nonlocal all_tasks + nonlocal completed_count + all_tasks = [ + asyncio.create_task(rollout_a_request_with_cancellation_handler(req)) for req in req_list + ] - # create all tasks - all_tasks = [asyncio.create_task(process_request_with_monitoring(req)) for req in req_list] - - # start monitoring task - monitor_task = asyncio.create_task(monitor_and_cancel()) - + # Wait for target_completion tasks to complete try: - # wait for all tasks to complete (including cancelled ones) - return await asyncio.gather(*all_tasks, return_exceptions=True) + for completed_task in asyncio.as_completed(all_tasks): + await completed_task + completed_count += 1 + if completed_count >= target_completion: + break finally: - # cancel monitoring task - monitor_task.cancel() + # Cancel remaining tasks + for t in all_tasks: + if not t.done(): + t.cancel() + + # Wait for all tasks to finish (including cancelled ones) + final_results = await asyncio.gather(*all_tasks, return_exceptions=True) + try: - await monitor_task - except asyncio.CancelledError: - pass + await self._engine.abort_request(abort_all=True) + except Exception as e: + logger.error(f"Failed to send abort signal to SGLang engine: {e}") + + return final_results - loop = asyncio.get_event_loop() - output_req_list = loop.run_until_complete(run_with_cancellation()) + output_req_list = asyncio.run(run_with_cancellation()) sorted_output_req_list = sorted(output_req_list, key=lambda x: (x.batch_data_id, x.rollout_offset)) else: From 58f3570ad727631f0824d8038c7f391933a0ec19 Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Wed, 13 Aug 2025 19:43:24 -0700 Subject: [PATCH 49/54] fix future --- .../rollout/sglang_rollout/sglang_rollout.py | 28 ++++--------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index 0cf0af6b9fd..d986a820461 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -25,26 +25,6 @@ from typing import Any, Optional from uuid import uuid4 -# Workaround: avoid duplicate AutoConfig.register conflicts (e.g., 'aimv2') -try: - from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig - - _orig_ac_register = AutoConfig.register - - def _safe_ac_register(model_type, config, exist_ok=False): - return _orig_ac_register(model_type, config, exist_ok=True) - - AutoConfig.register = _safe_ac_register - - _orig_cfg_register = CONFIG_MAPPING.register - - def _safe_cfg_register(key, config, exist_ok=False): - return _orig_cfg_register(key, config, exist_ok=True) - - CONFIG_MAPPING.register = _safe_cfg_register -except Exception: - pass - import numpy as np import sglang.srt.entrypoints.engine import torch @@ -1108,6 +1088,7 @@ def _req_level_generate_sequences(self, prompts: DataProto, **kwargs) -> DataPro # add progress monitoring and abort function total_requests = len(req_list) target_completion = int(total_requests * (1 - self.config.over_sample_rate)) + print(f"total_requests: {total_requests}") print(f"target_completion: {target_completion}") # abort when target_completion of requests are completed @@ -1117,11 +1098,12 @@ def _req_level_generate_sequences(self, prompts: DataProto, **kwargs) -> DataPro # distinguish training and validation if is_validate: # Validation mode: process all requests without abort - output_req_list = asyncio.run( - asyncio.gather( + async def _run_validation_tasks(): + return await asyncio.gather( *[self._async_rollout_a_request(req, do_sample, is_validate, **kwargs) for req in req_list], ) - ) + + output_req_list = asyncio.run(_run_validation_tasks()) else: all_tasks = [] From 23452ed084e3dddd2c658d3aad67872285c0ce9b Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Wed, 13 Aug 2025 19:57:54 -0700 Subject: [PATCH 50/54] use event loop --- verl/workers/rollout/sglang_rollout/sglang_rollout.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index d986a820461..afe2beed97d 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -1098,12 +1098,13 @@ def _req_level_generate_sequences(self, prompts: DataProto, **kwargs) -> DataPro # distinguish training and validation if is_validate: # Validation mode: process all requests without abort - async def _run_validation_tasks(): + async def process_all_requests(): return await asyncio.gather( *[self._async_rollout_a_request(req, do_sample, is_validate, **kwargs) for req in req_list], ) - output_req_list = asyncio.run(_run_validation_tasks()) + loop = asyncio.get_event_loop() + output_req_list = loop.run_until_complete(process_all_requests()) else: all_tasks = [] From 3ce9b1f6d5c06f153ff97c360901358491107245 Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Wed, 13 Aug 2025 20:06:35 -0700 Subject: [PATCH 51/54] get event loop --- verl/workers/rollout/sglang_rollout/sglang_rollout.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index afe2beed97d..5ab912da980 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -1148,7 +1148,8 @@ async def run_with_cancellation(): return final_results - output_req_list = asyncio.run(run_with_cancellation()) + loop = asyncio.get_event_loop() + output_req_list = loop.run_until_complete(run_with_cancellation()) sorted_output_req_list = sorted(output_req_list, key=lambda x: (x.batch_data_id, x.rollout_offset)) else: From d6128a26ecae468bda472c9ed090dad134475ad0 Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Wed, 13 Aug 2025 20:28:06 -0700 Subject: [PATCH 52/54] upd, fix up --- .../run_qwen2.5-3b_gsm8k_multiturn.sh | 4 ++-- .../run_qwen3_4b_dapo_multiturn.sh | 4 ++++ .../rollout/sglang_rollout/sglang_rollout.py | 24 +++++++++---------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh index 81613eefcec..e5ab236a4ad 100644 --- a/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh +++ b/examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn.sh @@ -40,11 +40,11 @@ python3 -m verl.trainer.main_ppo \ actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ actor_rollout_ref.rollout.name=sglang \ actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ - actor_rollout_ref.rollout.multi_stage_wake_up=False \ + actor_rollout_ref.rollout.multi_stage_wake_up=True \ actor_rollout_ref.rollout.n=16 \ actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=32 \ actor_rollout_ref.ref.fsdp_config.param_offload=True \ - actor_rollout_ref.rollout.over_sample_rate=0.1 \ + actor_rollout_ref.rollout.over_sample_rate=0 \ algorithm.use_kl_in_reward=False \ trainer.critic_warmup=0 \ trainer.logger='["console","wandb"]' \ diff --git a/examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh b/examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh index 8f2d7c81acb..53f856cca27 100644 --- a/examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh +++ b/examples/sglang_multiturn/run_qwen3_4b_dapo_multiturn.sh @@ -17,6 +17,10 @@ hf download \ --repo-type dataset \ --local-dir $HOME/data/Maxwell-Jia/AIME_2024 +# Note that this script is using AgentLoop instead of SGLang Multi-Turn +# We are concerned that the reward is not actually converge, since the +# reward of retool is encouraging the model to generate more turns to +# call more tools. The answers are not actually correct. python3 -m verl.trainer.main_ppo \ algorithm.adv_estimator=grpo \ diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index 5ab912da980..ec8ef2821c0 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -463,17 +463,18 @@ def _init_inference_engine(self, trust_remote_code, actor_module, port): trust_remote_code=trust_remote_code, # NOTE(linjunrong): add rank to prevent SGLang generate same port inside PortArgs.init_new # when random.seed is being set during training - port=302000 + rank, - # NOTE(Chenyang): if you want to debug the SGLang engine output - # please set the following parameters - # Otherwise, it will make the engine run too slow - # log_level="INFO", + port=30000 + rank, + # NOTE(Chenyang): turn on log_level to see the decoding speed of SGLang Engine + # log_level="INFO" + # NOTE(Chenyang): turn the following lines to see the input and output of each request # log_requests=True, # log_requests_level=2, + # NOTE(Chenyang): turn on max_running_requests to set the max concurrent running requests # max_running_requests=1, mm_attention_backend="fa3", attention_backend=attention_backend if attention_backend is not None else "fa3", - # In async mode, we want token in token out. + # In async mode for AgentLoop, SGLang support token in token out to avoid the tokenizer + # inconsistency issue. skip_tokenizer_init=self.config.mode == "async", ) else: @@ -1098,13 +1099,12 @@ def _req_level_generate_sequences(self, prompts: DataProto, **kwargs) -> DataPro # distinguish training and validation if is_validate: # Validation mode: process all requests without abort - async def process_all_requests(): - return await asyncio.gather( + loop = asyncio.get_event_loop() + output_req_list = loop.run_until_complete( + asyncio.gather( *[self._async_rollout_a_request(req, do_sample, is_validate, **kwargs) for req in req_list], ) - - loop = asyncio.get_event_loop() - output_req_list = loop.run_until_complete(process_all_requests()) + ) else: all_tasks = [] @@ -1384,7 +1384,7 @@ def _create_padding_request(self, original_req: AsyncRolloutRequest) -> AsyncRol padding_req = AsyncRolloutRequest( batch_data_id=original_req.batch_data_id, rollout_offset=original_req.rollout_offset, - request_id=original_req.request_id + "_padding", + request_id=original_req.request_id, state=AsyncRolloutRequestStateEnum.COMPLETED, messages=original_req.messages, multi_modal_keys=original_req.multi_modal_keys, From ba87964f3042419b242831af812bb9d24f8f05cb Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Wed, 13 Aug 2025 20:29:01 -0700 Subject: [PATCH 53/54] fix up --- .../rollout/sglang_rollout/sglang_rollout.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index ec8ef2821c0..1f35f4288d8 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -1086,16 +1086,6 @@ def _req_level_generate_sequences(self, prompts: DataProto, **kwargs) -> DataPro prompts, ) - # add progress monitoring and abort function - total_requests = len(req_list) - target_completion = int(total_requests * (1 - self.config.over_sample_rate)) - print(f"total_requests: {total_requests}") - print(f"target_completion: {target_completion}") - # abort when target_completion of requests are completed - - completed_count = 0 - aborted_requests = [] - # distinguish training and validation if is_validate: # Validation mode: process all requests without abort @@ -1106,6 +1096,13 @@ def _req_level_generate_sequences(self, prompts: DataProto, **kwargs) -> DataPro ) ) else: + # add progress monitoring and abort function + total_requests = len(req_list) + target_completion = int(total_requests * (1 - self.config.over_sample_rate)) + # abort when target_completion of requests are completed + + completed_count = 0 + aborted_requests = [] all_tasks = [] async def rollout_a_request_with_cancellation_handler(req): From 96c5d75792b181dbb744f3395377608ab2a32ae6 Mon Sep 17 00:00:00 2001 From: zhaochenyang20 Date: Wed, 13 Aug 2025 20:41:19 -0700 Subject: [PATCH 54/54] delete comment --- verl/workers/rollout/sglang_rollout/sglang_rollout.py | 1 - 1 file changed, 1 deletion(-) diff --git a/verl/workers/rollout/sglang_rollout/sglang_rollout.py b/verl/workers/rollout/sglang_rollout/sglang_rollout.py index 1f35f4288d8..383b98ce9d6 100644 --- a/verl/workers/rollout/sglang_rollout/sglang_rollout.py +++ b/verl/workers/rollout/sglang_rollout/sglang_rollout.py @@ -1006,7 +1006,6 @@ async def calc_reward_and_release_fn(name: str, tool: BaseTool): all_rewards = {**tool_reward_scores, **{"user_turn_rewards": user_turn_rewards}} _req.finalize(self.processing_class, all_rewards, finish_reason_type) if self.config.calculate_log_probs: - # 把input_ids输入sglang内生成一遍，并设置max_new_tokens=0，以生成log_probs debug_sampling_params = {**self.sampling_params} debug_sampling_params["max_new_tokens"] = 0 output = await self._engine.async_generate(