From 8cbe175b7e4eef7508fc5ef69536dd4f6bd39c20 Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 04:25:42 +0000
Subject: [PATCH 01/30] fix ppo value offload bugs

---
 slime/backends/megatron_utils/actor.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/slime/backends/megatron_utils/actor.py b/slime/backends/megatron_utils/actor.py
index 16f1748cc5..031b8b4cd4 100644
--- a/slime/backends/megatron_utils/actor.py
+++ b/slime/backends/megatron_utils/actor.py
@@ -356,7 +356,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
         if self.args.debug_rollout_only:
             return None
 
-        if self.args.offload_train:
+        if self.args.offload_train or self.args.use_critic:
             self.wake_up()
 
         with timer("data_preprocess"):
@@ -368,7 +368,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
             self.train_actor(rollout_id, rollout_data, external_data=external_data)
             result = None
 
-        if self.args.offload_train:
+        if self.args.offload_train or self.args.use_critic:
             self.sleep()
 
         return result

From 3ebce5a542936248a8475429cc8a2d97e336dcee Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 04:42:41 +0000
Subject: [PATCH 02/30] fix ppo value offload bugs

---
 slime/backends/megatron_utils/actor.py | 4 ++--
 slime/utils/arguments.py               | 3 +++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/slime/backends/megatron_utils/actor.py b/slime/backends/megatron_utils/actor.py
index 031b8b4cd4..16f1748cc5 100644
--- a/slime/backends/megatron_utils/actor.py
+++ b/slime/backends/megatron_utils/actor.py
@@ -356,7 +356,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
         if self.args.debug_rollout_only:
             return None
 
-        if self.args.offload_train or self.args.use_critic:
+        if self.args.offload_train:
             self.wake_up()
 
         with timer("data_preprocess"):
@@ -368,7 +368,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
             self.train_actor(rollout_id, rollout_data, external_data=external_data)
             result = None
 
-        if self.args.offload_train or self.args.use_critic:
+        if self.args.offload_train:
             self.sleep()
 
         return result
diff --git a/slime/utils/arguments.py b/slime/utils/arguments.py
index af526bf5a5..e8a1730782 100644
--- a/slime/utils/arguments.py
+++ b/slime/utils/arguments.py
@@ -1762,6 +1762,9 @@ def slime_validate_args(args):
     if args.offload_rollout is None:
         args.offload_rollout = False
 
+    if args.use_critic:
+        args.offload_train = True
+
     if args.eval_function_path is None:
         args.eval_function_path = args.rollout_function_path
 

From 250acde161d95091b02802dacc5e7f1ac790726e Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 08:49:35 +0000
Subject: [PATCH 03/30] fix ppo value offload bugs

---
 slime/backends/megatron_utils/actor.py        | 21 +++++++++++++++----
 .../update_weight_from_distributed.py         | 12 ++++++++---
 slime/backends/sglang_utils/sglang_engine.py  |  3 ---
 train.py                                      |  2 +-
 4 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/slime/backends/megatron_utils/actor.py b/slime/backends/megatron_utils/actor.py
index 16f1748cc5..049cf8a50a 100644
--- a/slime/backends/megatron_utils/actor.py
+++ b/slime/backends/megatron_utils/actor.py
@@ -81,7 +81,7 @@ def init(
                 logger.info(f"Set torch_memory_saver.memory_margin_bytes to {x}")
                 torch_memory_saver.memory_margin_bytes = x
 
-        (self.model, self.optimizer, self.opt_param_scheduler, loaded_rollout_id) = initialize_model_and_optimizer(
+        self.model, self.optimizer, self.opt_param_scheduler, loaded_rollout_id = initialize_model_and_optimizer(
             args, role
         )
 
@@ -159,6 +159,13 @@ def sleep(self) -> None:
 
         clear_memory(clear_host_memory=True)
         print_memory("before offload model")
+        if (
+            self.role == "actor"
+            and self.args.use_critic
+            and not self.args.colocate
+            and hasattr(self.weight_updater, "disconnect_rollout_engines")
+        ):
+            self.weight_updater.disconnect_rollout_engines()
         destroy_process_groups()
 
         torch_memory_saver.pause()
@@ -549,10 +556,14 @@ def update_weights(self) -> None:
             self.rollout_manager.get_updatable_engines_and_lock.remote()
         )
 
-        if self.args.offload_train:
+        reconnect_rollout_engines = self.args.offload_train and self.args.use_critic and not self.args.colocate
+
+        if reconnect_rollout_engines:
+            self.wake_up()
+        elif self.args.offload_train:
             reload_process_groups()
 
-        if num_new_engines > 0:
+        if num_new_engines > 0 or reconnect_rollout_engines:
             self.weight_updater.connect_rollout_engines(
                 rollout_engines,
                 rollout_engine_lock,
@@ -587,7 +598,9 @@ def update_weights(self) -> None:
                 else:
                     self.weights_backuper.backup("old_actor")
 
-        if self.args.offload_train:
+        if reconnect_rollout_engines:
+            self.sleep()
+        elif self.args.offload_train:
             destroy_process_groups()
 
     def load_other_checkpoint(self, model_tag: str, path: str) -> None:
diff --git a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
index 6a7182e7bf..6c00450c53 100644
--- a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
+++ b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
@@ -68,9 +68,7 @@ def connect_rollout_engines(
 
         if self._is_pp_src_rank:
             if self._model_update_groups is not None:
-                disconnect_rollout_engines_from_distributed(
-                    self.args, self._group_name, self._model_update_groups, self.rollout_engines
-                )
+                self.disconnect_rollout_engines()
             self._model_update_groups = connect_rollout_engines_from_distributed(
                 self.args,
                 self._group_name,
@@ -78,6 +76,14 @@ def connect_rollout_engines(
                 engine_gpu_counts=engine_gpu_counts,
             )
 
+    def disconnect_rollout_engines(self) -> None:
+        if not getattr(self, "_is_pp_src_rank", False) or self._model_update_groups is None:
+            return
+        disconnect_rollout_engines_from_distributed(
+            self.args, self._group_name, self._model_update_groups, self.rollout_engines
+        )
+        self._model_update_groups = None
+
     @torch.no_grad()
     def update_weights(self) -> None:
         """
diff --git a/slime/backends/sglang_utils/sglang_engine.py b/slime/backends/sglang_utils/sglang_engine.py
index 9da3549455..c28e13d5f9 100644
--- a/slime/backends/sglang_utils/sglang_engine.py
+++ b/slime/backends/sglang_utils/sglang_engine.py
@@ -26,9 +26,6 @@ def get_base_gpu_id(args, rank):
     else:
         num_actor_gpus = 0 if args.debug_rollout_only else args.actor_num_gpus_per_node * args.actor_num_nodes
         start_index = (num_actor_gpus + rank * num_gpus) % args.num_gpus_per_node
-        if args.use_critic:
-            num_critic_gpus = args.critic_num_gpus_per_node * args.critic_num_nodes
-            start_index = (num_actor_gpus + num_critic_gpus + rank * num_gpus) % args.num_gpus_per_node
     return start_index
 
 
diff --git a/train.py b/train.py
index a470cde35f..2404a0bbd1 100644
--- a/train.py
+++ b/train.py
@@ -90,7 +90,7 @@ def save(rollout_id):
         offload_train(actor_trains_this_step)
         if args.offload_rollout:
             ray.get(rollout_manager.onload_weights.remote())
-            actor_model.update_weights()
+        actor_model.update_weights()
 
         if args.offload_rollout:
             ray.get(rollout_manager.onload_kv.remote())

From 9f9c1873703bf0d24eed4536040440b2dc6cf79b Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 09:06:09 +0000
Subject: [PATCH 04/30] fix ppo value offload bugs

---
 .../update_weight/update_weight_from_distributed.py           | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
index 6c00450c53..822b801776 100644
--- a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
+++ b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
@@ -68,7 +68,9 @@ def connect_rollout_engines(
 
         if self._is_pp_src_rank:
             if self._model_update_groups is not None:
-                self.disconnect_rollout_engines()
+                disconnect_rollout_engines_from_distributed(
+                    self.args, self._group_name, self._model_update_groups, self.rollout_engines
+                )
             self._model_update_groups = connect_rollout_engines_from_distributed(
                 self.args,
                 self._group_name,

From 9bd6779d04e15ebba2b6bd1ca0b9a0d4c8f0de10 Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 09:15:53 +0000
Subject: [PATCH 05/30] fix ppo value offload bugs

---
 .github/copilot-instructions.md         |  88 ++++++++++++++
 .github/workflows/pr-test.yml           |   2 +-
 .github/workflows/pr-test.yml.j2        |   1 +
 tests/test_qwen3_4B_ppo_disaggregate.py | 150 ++++++++++++++++++++++++
 4 files changed, 240 insertions(+), 1 deletion(-)
 create mode 100644 .github/copilot-instructions.md
 create mode 100644 tests/test_qwen3_4B_ppo_disaggregate.py

diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
new file mode 100644
index 0000000000..a971894a75
--- /dev/null
+++ b/.github/copilot-instructions.md
@@ -0,0 +1,88 @@
+# Copilot Instructions for slime
+
+## What is slime
+
+slime is an LLM post-training framework for RL scaling. It connects **Megatron** (training) with **SGLang** (rollout/inference) via a **data buffer**, orchestrated by **Ray**. It supports GRPO, PPO, SFT, and on-policy distillation across models like Qwen3, DeepSeek V3, GLM, and Llama 3.
+
+## Build & Install
+
+```bash
+pip install -e . --no-deps          # editable install (no-deps because heavy deps like megatron/sglang are managed separately)
+pip install -r requirements.txt     # install Python dependencies
+```
+
+## Lint & Format
+
+Pre-commit handles all formatting (black, isort, ruff, autoflake):
+
+```bash
+pre-commit run --all-files --show-diff-on-failure --color=always
+```
+
+Toolchain: **black** (line-length 119), **isort** (black profile), **ruff** (E/F/B/UP rules, line-length 320), **autoflake** (remove unused imports).
+
+## Tests
+
+Tests are GPU-dependent end-to-end tests that require multi-GPU hardware, model weights, and datasets. They are not standard unit tests.
+
+```bash
+# Run a single test (requires GPUs; uses gpu_lock_exec to claim N GPUs)
+python tests/ci/gpu_lock_exec.py --count 4 -- python tests/test_qwen2.5_0.5B_gsm8k_short.py
+
+# Run pytest (for any pytest-compatible tests)
+pytest tests/ -m "unit"           # run only unit-marked tests
+pytest tests/test_chunked_gae.py  # run a specific test file
+```
+
+CI is triggered on PRs via GitHub labels (`run-ci-short`, `run-ci-fsdp`, `run-ci-long`). PR test workflows are auto-generated from `pr-test.yml.j2` via `generate_github_workflows.py`.
+
+## Architecture
+
+### Three-module design
+
+1. **Training** (`slime/backends/`): Megatron-based (`megatron_utils/`) or FSDP-based (`fsdp_utils/`) training backends. Handles forward/backward, loss computation, checkpointing, and weight updates.
+2. **Rollout** (`slime/rollout/`): SGLang-based inference engine that generates responses, computes rewards, and applies filters. The `sglang_rollout.py` is the main rollout driver.
+3. **Data Buffer** (`slime/rollout/data_source.py`): Manages prompt datasets and generated samples flowing between rollout and training.
+
+### Orchestration layer
+
+- **Ray** (`slime/ray/`): Manages GPU placement groups, actor groups, and the `RolloutManager`. `placement_group.py` allocates GPUs; `train_actor.py` wraps training workers.
+- **Router** (`slime/router/`): A FastAPI-based request router (`SlimeRouter`) that load-balances across SGLang engine instances with health checking and failure quarantine.
+
+### Plugin system (`slime_plugins/`)
+
+- `models/`: Model-specific adapters for SGLang (e.g., `glm4.py`, `qwen3_next.py`, `deepseek_v32.py`).
+- `mbridge/`: Model-specific adapters for the Megatron training backend.
+- `rollout_buffer/`: Standalone data generation component that can be used independently from training.
+
+### Entry points
+
+- `train.py`: Standard synchronous RL training loop.
+- `train_async.py`: Asynchronous training variant.
+- `scripts/`: Ready-to-run shell scripts for various model configurations.
+
+## Key Conventions
+
+### Arguments are in three categories
+1. **Megatron args**: Standard Megatron flags (e.g., `--tensor-model-parallel-size 2`).
+2. **SGLang args**: Must be prefixed with `--sglang-` (e.g., `--sglang-mem-fraction-static`).
+3. **slime args**: Defined in `slime/utils/arguments.py`.
+
+### Dynamic function loading
+Custom generate functions, reward models, and filters are specified as dotted Python paths (e.g., `my_module.my_reward_fn`) and loaded at runtime via `slime.utils.misc.load_function`. This is the primary extension mechanism.
+
+### Reward models (`slime/rollout/rm_hub/`)
+Built-in reward types: `math`, `dapo`, `deepscaler`, `f1`, `gpqa`, `ifbench`, `remote_rm`, `random`. Custom RMs are loaded via `--custom-rm-path`. All RM functions are async.
+
+### The `Sample` dataclass (`slime/utils/types.py`)
+Central data type flowing through the entire pipeline. Contains prompt, response, reward, loss_mask, tokens, metadata, and status tracking. Samples are grouped as `list[list[Sample]]` (groups of samples per prompt).
+
+### Training backends are swappable
+Megatron (`slime/backends/megatron_utils/`) and FSDP (`slime/backends/fsdp_utils/`) are interchangeable backends. Both implement actor, checkpoint, and weight update interfaces.
+
+### Distributed debugging
+When debugging multi-GPU/multi-node issues, see `.claude/skills/SKILL.md` for patterns around asymmetric keys, startup race conditions, and the modify-sync-restart-verify workflow.
+
+## Contribution Scope
+
+Per CONTRIBUTING.md, the project accepts **bug fixes** and **general-purpose large-scale RL optimizations** with clear benchmarks. Large refactors, abstraction proposals, and features that can't be verified through CI are out of scope.
diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml
index 3c663db0b1..9e0067943f 100644
--- a/.github/workflows/pr-test.yml
+++ b/.github/workflows/pr-test.yml
@@ -201,7 +201,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        info: [{"num_gpus": 8, "test_file": "test_quick_start_glm4_9B.py"}, {"num_gpus": 8, "test_file": "test_glm4.7_30B_A3B_pd_mooncake.py"}, {"num_gpus": 8, "test_file": "test_qwen3_30B_A3B.py", "use_deepep": "1", "use_fp8_rollout": "1"}, {"num_gpus": 8, "test_file": "test_qwen3.6_35B_A3B_pd_mooncake.py", "use_deepep": "1"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_qwen3_30B_A3B_r3.py", "use_deepep": "1", "use_fp8_rollout": "1"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_qwen3_30B_A3B_r3.py"}, {"num_gpus": 8, "test_file": "test_qwen3_4B_ppo.py"}, {"num_gpus": 8, "test_file": "test_qwen3_4B_ppo_train_critic_only.py"}, {"num_gpus": 8, "test_file": "test_moonlight_16B_A3B.py"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_moonlight_16B_A3B_r3.py"}, {"num_gpus": 8, "test_file": "test_mimo_7B_mtp_only_grad.py"}, {"num_gpus": 8, "test_file": "test_qwen2.5_0.5B_debug_rollout_then_train.py"}, {"num_gpus": 8, "test_file": "test_qwen2.5_0.5B_opd_sglang.py"}]
+        info: [{"num_gpus": 8, "test_file": "test_quick_start_glm4_9B.py"}, {"num_gpus": 8, "test_file": "test_glm4.7_30B_A3B_pd_mooncake.py"}, {"num_gpus": 8, "test_file": "test_qwen3_30B_A3B.py", "use_deepep": "1", "use_fp8_rollout": "1"}, {"num_gpus": 8, "test_file": "test_qwen3.6_35B_A3B_pd_mooncake.py", "use_deepep": "1"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_qwen3_30B_A3B_r3.py", "use_deepep": "1", "use_fp8_rollout": "1"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_qwen3_30B_A3B_r3.py"}, {"num_gpus": 8, "test_file": "test_qwen3_4B_ppo.py"}, {"num_gpus": 8, "test_file": "test_qwen3_4B_ppo_train_critic_only.py"}, {"num_gpus": 8, "test_file": "test_moonlight_16B_A3B.py"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_moonlight_16B_A3B_r3.py"}, {"num_gpus": 8, "test_file": "test_mimo_7B_mtp_only_grad.py"}, {"num_gpus": 8, "test_file": "test_qwen2.5_0.5B_debug_rollout_then_train.py"}, {"num_gpus": 8, "test_file": "test_qwen2.5_0.5B_opd_sglang.py"}, {"num_gpus": 8, "test_file": "test_qwen3_4B_ppo_disaggregate.py"}]
     defaults:
       run:
         working-directory: ${{ github.workspace }}
diff --git a/.github/workflows/pr-test.yml.j2 b/.github/workflows/pr-test.yml.j2
index 2314e01861..e627b05b33 100644
--- a/.github/workflows/pr-test.yml.j2
+++ b/.github/workflows/pr-test.yml.j2
@@ -32,6 +32,7 @@
         {'test_file': 'test_mimo_7B_mtp_only_grad.py', 'num_gpus': 8},
         {'test_file': 'test_qwen2.5_0.5B_debug_rollout_then_train.py', 'num_gpus': 8},
         {'test_file': 'test_qwen2.5_0.5B_opd_sglang.py', 'num_gpus': 8},
+        {'test_file': 'test_qwen3_4B_ppo_disaggregate.py', 'num_gpus': 8},
       ],
     },
     'e2e-test-precision': {
diff --git a/tests/test_qwen3_4B_ppo_disaggregate.py b/tests/test_qwen3_4B_ppo_disaggregate.py
new file mode 100644
index 0000000000..52119a5b67
--- /dev/null
+++ b/tests/test_qwen3_4B_ppo_disaggregate.py
@@ -0,0 +1,150 @@
+import os
+import tempfile
+
+import slime.utils.external_utils.command_utils as U
+
+
+ENABLE_EVAL = bool(int(os.environ.get("SLIME_TEST_ENABLE_EVAL", "1")))
+TIGHT_HOST_MEMORY = bool(int(os.environ.get("SLIME_TEST_TIGHT_HOST_MEMORY", "1")))
+
+MODEL_NAME = "Qwen3-4B"
+MODEL_TYPE = "qwen3-4B"
+NUM_GPUS = 8
+
+
+def prepare():
+    U.exec_command("mkdir -p /root/models /root/datasets")
+    U.exec_command("hf download Qwen/Qwen3-4B --local-dir /root/models/Qwen3-4B")
+    U.hf_download_dataset("zhuzilin/dapo-math-17k")
+    U.hf_download_dataset("zhuzilin/aime-2024")
+
+    U.convert_checkpoint(model_name=MODEL_NAME, megatron_model_type=MODEL_TYPE, num_gpus_per_node=NUM_GPUS)
+
+
+def execute():
+    megatron_config = tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False)
+    megatron_config.write(
+        """
+megatron:
+  - name: default
+    role: critic
+    overrides:
+      lr: 1e-5
+  - name: default
+    role: actor
+    overrides:
+      lr: 1e-6
+"""
+    )
+    megatron_config.close()
+
+    ckpt_args = f"--hf-checkpoint /root/models/{MODEL_NAME}/ " f"--ref-load /root/{MODEL_NAME}_torch_dist "
+
+    rollout_args = (
+        "--prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl "
+        "--input-key prompt "
+        "--label-key label "
+        "--apply-chat-template "
+        "--rollout-shuffle "
+        "--rm-type deepscaler "
+        "--num-rollout 3 "
+        "--rollout-batch-size 8 "
+        "--n-samples-per-prompt 4 "
+        "--rollout-max-response-len 8192 "
+        "--rollout-temperature 0.8 "
+        "--global-batch-size 32 "
+        "--balance-data "
+    )
+
+    eval_args = (
+        f"{'--eval-interval 20 ' if ENABLE_EVAL else ''}"
+        "--eval-prompt-data aime24 /root/datasets/aime-2024/aime-2024.jsonl "
+        "--n-samples-per-eval-prompt 1 "
+        "--eval-max-response-len 16384 "
+        "--eval-top-k 1 "
+    )
+
+    perf_args = (
+        "--tensor-model-parallel-size 2 "
+        "--sequence-parallel "
+        "--pipeline-model-parallel-size 1 "
+        "--context-parallel-size 2 "
+        "--recompute-granularity full "
+        "--recompute-method uniform "
+        "--recompute-num-layers 1 "
+        "--use-dynamic-batch-size "
+        f"--max-tokens-per-gpu {2048 if TIGHT_HOST_MEMORY else 16384} "
+    )
+
+    ppo_args = (
+        "--advantage-estimator ppo "
+        f"{'' if TIGHT_HOST_MEMORY else '--use-kl-loss '}"
+        "--kl-loss-coef 0.00 "
+        "--kl-loss-type k1 "
+        "--kl-coef 0.00 "
+        "--entropy-coef 0.00 "
+        "--eps-clip 4e-4 "
+        "--num-critic-only-steps 1 "
+        "--normalize-advantages "
+    )
+
+    optimizer_args = (
+        "--optimizer adam "
+        "--lr 1e-6 "
+        "--lr-decay-style constant "
+        "--weight-decay 0.1 "
+        "--adam-beta1 0.9 "
+        "--adam-beta2 0.98 "
+    )
+
+    sglang_args = (
+        "--rollout-num-gpus-per-engine 2 "
+        "--rollout-num-gpus 4 "
+        "--sglang-mem-fraction-static 0.8 "
+        "--sglang-cuda-graph-max-bs 32 "
+        "--sglang-max-running-requests 512 "
+        "--sglang-enable-metrics "
+    )
+
+    ci_args = "--ci-test "
+
+    misc_args = (
+        # default dropout in megatron is 0.1
+        "--attention-dropout 0.0 "
+        "--hidden-dropout 0.0 "
+        # should be good for model performance
+        "--accumulate-allreduce-grads-in-fp32 "
+        "--attention-softmax-in-fp32 "
+        # need to comment this when using model with MLA
+        "--attention-backend flash "
+        "--actor-num-nodes 1 "
+        "--actor-num-gpus-per-node 4 "
+    )
+
+    train_args = (
+        f"--megatron-config-path {megatron_config.name} "
+        f"{ckpt_args} "
+        f"{rollout_args} "
+        f"{optimizer_args} "
+        f"{ppo_args} "
+        f"{U.get_default_wandb_args(__file__)} "
+        f"{perf_args} "
+        f"{eval_args} "
+        f"{sglang_args} "
+        f"{ci_args} "
+        f"{misc_args} "
+    )
+
+    U.execute_train(
+        train_args=train_args,
+        num_gpus_per_node=NUM_GPUS,
+        megatron_model_type=MODEL_TYPE,
+    )
+
+
+if __name__ == "__main__":
+    # TODO also use typer
+    prepare()
+    for proxy_var in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY"):
+        os.environ.pop(proxy_var, None)
+    execute()

From dcc0aa0be80a21c2f3e1b618d3e7bbb0ddbd7f01 Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 04:25:42 +0000
Subject: [PATCH 06/30] fix ppo value offload bugs

---
 slime/backends/megatron_utils/actor.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/slime/backends/megatron_utils/actor.py b/slime/backends/megatron_utils/actor.py
index 049cf8a50a..15effaa521 100644
--- a/slime/backends/megatron_utils/actor.py
+++ b/slime/backends/megatron_utils/actor.py
@@ -363,7 +363,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
         if self.args.debug_rollout_only:
             return None
 
-        if self.args.offload_train:
+        if self.args.offload_train or self.args.use_critic:
             self.wake_up()
 
         with timer("data_preprocess"):
@@ -375,7 +375,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
             self.train_actor(rollout_id, rollout_data, external_data=external_data)
             result = None
 
-        if self.args.offload_train:
+        if self.args.offload_train or self.args.use_critic:
             self.sleep()
 
         return result

From fbf33bd29ca18d0fe18406a4cdff3bda6a80555c Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 04:42:41 +0000
Subject: [PATCH 07/30] fix ppo value offload bugs

---
 slime/backends/megatron_utils/actor.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/slime/backends/megatron_utils/actor.py b/slime/backends/megatron_utils/actor.py
index 15effaa521..049cf8a50a 100644
--- a/slime/backends/megatron_utils/actor.py
+++ b/slime/backends/megatron_utils/actor.py
@@ -363,7 +363,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
         if self.args.debug_rollout_only:
             return None
 
-        if self.args.offload_train or self.args.use_critic:
+        if self.args.offload_train:
             self.wake_up()
 
         with timer("data_preprocess"):
@@ -375,7 +375,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
             self.train_actor(rollout_id, rollout_data, external_data=external_data)
             result = None
 
-        if self.args.offload_train or self.args.use_critic:
+        if self.args.offload_train:
             self.sleep()
 
         return result

From 631e078fe9cced70f41d8bfaccd8fb62f93572bb Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 08:49:35 +0000
Subject: [PATCH 08/30] fix ppo value offload bugs

---
 .../update_weight/update_weight_from_distributed.py           | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
index 822b801776..6c00450c53 100644
--- a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
+++ b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
@@ -68,9 +68,7 @@ def connect_rollout_engines(
 
         if self._is_pp_src_rank:
             if self._model_update_groups is not None:
-                disconnect_rollout_engines_from_distributed(
-                    self.args, self._group_name, self._model_update_groups, self.rollout_engines
-                )
+                self.disconnect_rollout_engines()
             self._model_update_groups = connect_rollout_engines_from_distributed(
                 self.args,
                 self._group_name,

From 8970ecbbf04e4748fb847fbb5355eb8acd4324c1 Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 09:06:09 +0000
Subject: [PATCH 09/30] fix ppo value offload bugs

---
 .../update_weight/update_weight_from_distributed.py           | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
index 6c00450c53..822b801776 100644
--- a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
+++ b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
@@ -68,7 +68,9 @@ def connect_rollout_engines(
 
         if self._is_pp_src_rank:
             if self._model_update_groups is not None:
-                self.disconnect_rollout_engines()
+                disconnect_rollout_engines_from_distributed(
+                    self.args, self._group_name, self._model_update_groups, self.rollout_engines
+                )
             self._model_update_groups = connect_rollout_engines_from_distributed(
                 self.args,
                 self._group_name,

From edc4363a1231e10293b03f725094548936ea8f24 Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 04:25:42 +0000
Subject: [PATCH 10/30] fix ppo value offload bugs

---
 slime/backends/megatron_utils/actor.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/slime/backends/megatron_utils/actor.py b/slime/backends/megatron_utils/actor.py
index 049cf8a50a..15effaa521 100644
--- a/slime/backends/megatron_utils/actor.py
+++ b/slime/backends/megatron_utils/actor.py
@@ -363,7 +363,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
         if self.args.debug_rollout_only:
             return None
 
-        if self.args.offload_train:
+        if self.args.offload_train or self.args.use_critic:
             self.wake_up()
 
         with timer("data_preprocess"):
@@ -375,7 +375,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
             self.train_actor(rollout_id, rollout_data, external_data=external_data)
             result = None
 
-        if self.args.offload_train:
+        if self.args.offload_train or self.args.use_critic:
             self.sleep()
 
         return result

From d8636c51a5ee8f23998102adab87c69ebc1f7aa6 Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 04:42:41 +0000
Subject: [PATCH 11/30] fix ppo value offload bugs

---
 slime/backends/megatron_utils/actor.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/slime/backends/megatron_utils/actor.py b/slime/backends/megatron_utils/actor.py
index 15effaa521..049cf8a50a 100644
--- a/slime/backends/megatron_utils/actor.py
+++ b/slime/backends/megatron_utils/actor.py
@@ -363,7 +363,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
         if self.args.debug_rollout_only:
             return None
 
-        if self.args.offload_train or self.args.use_critic:
+        if self.args.offload_train:
             self.wake_up()
 
         with timer("data_preprocess"):
@@ -375,7 +375,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
             self.train_actor(rollout_id, rollout_data, external_data=external_data)
             result = None
 
-        if self.args.offload_train or self.args.use_critic:
+        if self.args.offload_train:
             self.sleep()
 
         return result

From 88a3d243e007e99f7cf3f2e362d3be0e64774d39 Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 08:49:35 +0000
Subject: [PATCH 12/30] fix ppo value offload bugs

---
 .../update_weight/update_weight_from_distributed.py           | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
index 822b801776..6c00450c53 100644
--- a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
+++ b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
@@ -68,9 +68,7 @@ def connect_rollout_engines(
 
         if self._is_pp_src_rank:
             if self._model_update_groups is not None:
-                disconnect_rollout_engines_from_distributed(
-                    self.args, self._group_name, self._model_update_groups, self.rollout_engines
-                )
+                self.disconnect_rollout_engines()
             self._model_update_groups = connect_rollout_engines_from_distributed(
                 self.args,
                 self._group_name,

From 811081f3fc18a94f0d16e95d34ce847c7582e0d0 Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 09:06:09 +0000
Subject: [PATCH 13/30] fix ppo value offload bugs

---
 .../update_weight/update_weight_from_distributed.py           | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
index 6c00450c53..822b801776 100644
--- a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
+++ b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
@@ -68,7 +68,9 @@ def connect_rollout_engines(
 
         if self._is_pp_src_rank:
             if self._model_update_groups is not None:
-                self.disconnect_rollout_engines()
+                disconnect_rollout_engines_from_distributed(
+                    self.args, self._group_name, self._model_update_groups, self.rollout_engines
+                )
             self._model_update_groups = connect_rollout_engines_from_distributed(
                 self.args,
                 self._group_name,

From 5123f5622042cf9ad1c730138a0c53b3cb8db210 Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 09:24:09 +0000
Subject: [PATCH 14/30] fix ppo value offload bugs

---
 .github/workflows/pr-test.yml    | 2 +-
 .github/workflows/pr-test.yml.j2 | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml
index 9e0067943f..3c663db0b1 100644
--- a/.github/workflows/pr-test.yml
+++ b/.github/workflows/pr-test.yml
@@ -201,7 +201,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        info: [{"num_gpus": 8, "test_file": "test_quick_start_glm4_9B.py"}, {"num_gpus": 8, "test_file": "test_glm4.7_30B_A3B_pd_mooncake.py"}, {"num_gpus": 8, "test_file": "test_qwen3_30B_A3B.py", "use_deepep": "1", "use_fp8_rollout": "1"}, {"num_gpus": 8, "test_file": "test_qwen3.6_35B_A3B_pd_mooncake.py", "use_deepep": "1"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_qwen3_30B_A3B_r3.py", "use_deepep": "1", "use_fp8_rollout": "1"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_qwen3_30B_A3B_r3.py"}, {"num_gpus": 8, "test_file": "test_qwen3_4B_ppo.py"}, {"num_gpus": 8, "test_file": "test_qwen3_4B_ppo_train_critic_only.py"}, {"num_gpus": 8, "test_file": "test_moonlight_16B_A3B.py"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_moonlight_16B_A3B_r3.py"}, {"num_gpus": 8, "test_file": "test_mimo_7B_mtp_only_grad.py"}, {"num_gpus": 8, "test_file": "test_qwen2.5_0.5B_debug_rollout_then_train.py"}, {"num_gpus": 8, "test_file": "test_qwen2.5_0.5B_opd_sglang.py"}, {"num_gpus": 8, "test_file": "test_qwen3_4B_ppo_disaggregate.py"}]
+        info: [{"num_gpus": 8, "test_file": "test_quick_start_glm4_9B.py"}, {"num_gpus": 8, "test_file": "test_glm4.7_30B_A3B_pd_mooncake.py"}, {"num_gpus": 8, "test_file": "test_qwen3_30B_A3B.py", "use_deepep": "1", "use_fp8_rollout": "1"}, {"num_gpus": 8, "test_file": "test_qwen3.6_35B_A3B_pd_mooncake.py", "use_deepep": "1"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_qwen3_30B_A3B_r3.py", "use_deepep": "1", "use_fp8_rollout": "1"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_qwen3_30B_A3B_r3.py"}, {"num_gpus": 8, "test_file": "test_qwen3_4B_ppo.py"}, {"num_gpus": 8, "test_file": "test_qwen3_4B_ppo_train_critic_only.py"}, {"num_gpus": 8, "test_file": "test_moonlight_16B_A3B.py"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_moonlight_16B_A3B_r3.py"}, {"num_gpus": 8, "test_file": "test_mimo_7B_mtp_only_grad.py"}, {"num_gpus": 8, "test_file": "test_qwen2.5_0.5B_debug_rollout_then_train.py"}, {"num_gpus": 8, "test_file": "test_qwen2.5_0.5B_opd_sglang.py"}]
     defaults:
       run:
         working-directory: ${{ github.workspace }}
diff --git a/.github/workflows/pr-test.yml.j2 b/.github/workflows/pr-test.yml.j2
index e627b05b33..2314e01861 100644
--- a/.github/workflows/pr-test.yml.j2
+++ b/.github/workflows/pr-test.yml.j2
@@ -32,7 +32,6 @@
         {'test_file': 'test_mimo_7B_mtp_only_grad.py', 'num_gpus': 8},
         {'test_file': 'test_qwen2.5_0.5B_debug_rollout_then_train.py', 'num_gpus': 8},
         {'test_file': 'test_qwen2.5_0.5B_opd_sglang.py', 'num_gpus': 8},
-        {'test_file': 'test_qwen3_4B_ppo_disaggregate.py', 'num_gpus': 8},
       ],
     },
     'e2e-test-precision': {

From d89cf24ccfe7fcf1811d1beec9e134e0a38d3139 Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 04:25:42 +0000
Subject: [PATCH 15/30] fix ppo value offload bugs

---
 slime/backends/megatron_utils/actor.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/slime/backends/megatron_utils/actor.py b/slime/backends/megatron_utils/actor.py
index 049cf8a50a..15effaa521 100644
--- a/slime/backends/megatron_utils/actor.py
+++ b/slime/backends/megatron_utils/actor.py
@@ -363,7 +363,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
         if self.args.debug_rollout_only:
             return None
 
-        if self.args.offload_train:
+        if self.args.offload_train or self.args.use_critic:
             self.wake_up()
 
         with timer("data_preprocess"):
@@ -375,7 +375,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
             self.train_actor(rollout_id, rollout_data, external_data=external_data)
             result = None
 
-        if self.args.offload_train:
+        if self.args.offload_train or self.args.use_critic:
             self.sleep()
 
         return result

From 81afe96ee4923cf6ae8af13396ec8b571b113ebd Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 04:42:41 +0000
Subject: [PATCH 16/30] fix ppo value offload bugs

---
 slime/backends/megatron_utils/actor.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/slime/backends/megatron_utils/actor.py b/slime/backends/megatron_utils/actor.py
index 15effaa521..049cf8a50a 100644
--- a/slime/backends/megatron_utils/actor.py
+++ b/slime/backends/megatron_utils/actor.py
@@ -363,7 +363,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
         if self.args.debug_rollout_only:
             return None
 
-        if self.args.offload_train or self.args.use_critic:
+        if self.args.offload_train:
             self.wake_up()
 
         with timer("data_preprocess"):
@@ -375,7 +375,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
             self.train_actor(rollout_id, rollout_data, external_data=external_data)
             result = None
 
-        if self.args.offload_train or self.args.use_critic:
+        if self.args.offload_train:
             self.sleep()
 
         return result

From 3884802f96964fc8fe15cb4ddc595279e0b103d8 Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 08:49:35 +0000
Subject: [PATCH 17/30] fix ppo value offload bugs

---
 .../update_weight/update_weight_from_distributed.py           | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
index 822b801776..6c00450c53 100644
--- a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
+++ b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
@@ -68,9 +68,7 @@ def connect_rollout_engines(
 
         if self._is_pp_src_rank:
             if self._model_update_groups is not None:
-                disconnect_rollout_engines_from_distributed(
-                    self.args, self._group_name, self._model_update_groups, self.rollout_engines
-                )
+                self.disconnect_rollout_engines()
             self._model_update_groups = connect_rollout_engines_from_distributed(
                 self.args,
                 self._group_name,

From 59077df7bb8ec4a612147a092bba384ffddcd647 Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 09:06:09 +0000
Subject: [PATCH 18/30] fix ppo value offload bugs

---
 .../update_weight/update_weight_from_distributed.py           | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
index 6c00450c53..822b801776 100644
--- a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
+++ b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
@@ -68,7 +68,9 @@ def connect_rollout_engines(
 
         if self._is_pp_src_rank:
             if self._model_update_groups is not None:
-                self.disconnect_rollout_engines()
+                disconnect_rollout_engines_from_distributed(
+                    self.args, self._group_name, self._model_update_groups, self.rollout_engines
+                )
             self._model_update_groups = connect_rollout_engines_from_distributed(
                 self.args,
                 self._group_name,

From 179871fe26fc6e9f390a86dde64c5f2c14925e67 Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 09:31:48 +0000
Subject: [PATCH 19/30] fix ppo value offload bugs

---
 .github/workflows/pr-test.yml.j2 | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pr-test.yml.j2 b/.github/workflows/pr-test.yml.j2
index 2314e01861..e627b05b33 100644
--- a/.github/workflows/pr-test.yml.j2
+++ b/.github/workflows/pr-test.yml.j2
@@ -32,6 +32,7 @@
         {'test_file': 'test_mimo_7B_mtp_only_grad.py', 'num_gpus': 8},
         {'test_file': 'test_qwen2.5_0.5B_debug_rollout_then_train.py', 'num_gpus': 8},
         {'test_file': 'test_qwen2.5_0.5B_opd_sglang.py', 'num_gpus': 8},
+        {'test_file': 'test_qwen3_4B_ppo_disaggregate.py', 'num_gpus': 8},
       ],
     },
     'e2e-test-precision': {

From 6cda7ec39a917c048e6ad08b4213b62872a7c9f9 Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 04:25:42 +0000
Subject: [PATCH 20/30] fix ppo value offload bugs

---
 slime/backends/megatron_utils/actor.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/slime/backends/megatron_utils/actor.py b/slime/backends/megatron_utils/actor.py
index 049cf8a50a..15effaa521 100644
--- a/slime/backends/megatron_utils/actor.py
+++ b/slime/backends/megatron_utils/actor.py
@@ -363,7 +363,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
         if self.args.debug_rollout_only:
             return None
 
-        if self.args.offload_train:
+        if self.args.offload_train or self.args.use_critic:
             self.wake_up()
 
         with timer("data_preprocess"):
@@ -375,7 +375,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
             self.train_actor(rollout_id, rollout_data, external_data=external_data)
             result = None
 
-        if self.args.offload_train:
+        if self.args.offload_train or self.args.use_critic:
             self.sleep()
 
         return result

From 99165e9447019f919af8f915a2ae23392566c691 Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 04:42:41 +0000
Subject: [PATCH 21/30] fix ppo value offload bugs

---
 slime/backends/megatron_utils/actor.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/slime/backends/megatron_utils/actor.py b/slime/backends/megatron_utils/actor.py
index 15effaa521..049cf8a50a 100644
--- a/slime/backends/megatron_utils/actor.py
+++ b/slime/backends/megatron_utils/actor.py
@@ -363,7 +363,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
         if self.args.debug_rollout_only:
             return None
 
-        if self.args.offload_train or self.args.use_critic:
+        if self.args.offload_train:
             self.wake_up()
 
         with timer("data_preprocess"):
@@ -375,7 +375,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
             self.train_actor(rollout_id, rollout_data, external_data=external_data)
             result = None
 
-        if self.args.offload_train or self.args.use_critic:
+        if self.args.offload_train:
             self.sleep()
 
         return result

From bafee9a0854c225c207136bc46af7d10213dac29 Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 08:49:35 +0000
Subject: [PATCH 22/30] fix ppo value offload bugs

---
 .../update_weight/update_weight_from_distributed.py           | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
index 822b801776..6c00450c53 100644
--- a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
+++ b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
@@ -68,9 +68,7 @@ def connect_rollout_engines(
 
         if self._is_pp_src_rank:
             if self._model_update_groups is not None:
-                disconnect_rollout_engines_from_distributed(
-                    self.args, self._group_name, self._model_update_groups, self.rollout_engines
-                )
+                self.disconnect_rollout_engines()
             self._model_update_groups = connect_rollout_engines_from_distributed(
                 self.args,
                 self._group_name,

From eba4b86c72f7ade201906d8b5fdc58d4fdc5de9d Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 09:06:09 +0000
Subject: [PATCH 23/30] fix ppo value offload bugs

---
 .../update_weight/update_weight_from_distributed.py           | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
index 6c00450c53..822b801776 100644
--- a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
+++ b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
@@ -68,7 +68,9 @@ def connect_rollout_engines(
 
         if self._is_pp_src_rank:
             if self._model_update_groups is not None:
-                self.disconnect_rollout_engines()
+                disconnect_rollout_engines_from_distributed(
+                    self.args, self._group_name, self._model_update_groups, self.rollout_engines
+                )
             self._model_update_groups = connect_rollout_engines_from_distributed(
                 self.args,
                 self._group_name,

From a7a79bc31a133027193e8030ce0c43219174ecdc Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 04:25:42 +0000
Subject: [PATCH 24/30] fix ppo value offload bugs

---
 slime/backends/megatron_utils/actor.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/slime/backends/megatron_utils/actor.py b/slime/backends/megatron_utils/actor.py
index 049cf8a50a..15effaa521 100644
--- a/slime/backends/megatron_utils/actor.py
+++ b/slime/backends/megatron_utils/actor.py
@@ -363,7 +363,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
         if self.args.debug_rollout_only:
             return None
 
-        if self.args.offload_train:
+        if self.args.offload_train or self.args.use_critic:
             self.wake_up()
 
         with timer("data_preprocess"):
@@ -375,7 +375,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
             self.train_actor(rollout_id, rollout_data, external_data=external_data)
             result = None
 
-        if self.args.offload_train:
+        if self.args.offload_train or self.args.use_critic:
             self.sleep()
 
         return result

From 122a5d3d1a28d301d24fdc3e95c84287333d4fa8 Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 04:42:41 +0000
Subject: [PATCH 25/30] fix ppo value offload bugs

---
 slime/backends/megatron_utils/actor.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/slime/backends/megatron_utils/actor.py b/slime/backends/megatron_utils/actor.py
index 15effaa521..049cf8a50a 100644
--- a/slime/backends/megatron_utils/actor.py
+++ b/slime/backends/megatron_utils/actor.py
@@ -363,7 +363,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
         if self.args.debug_rollout_only:
             return None
 
-        if self.args.offload_train or self.args.use_critic:
+        if self.args.offload_train:
             self.wake_up()
 
         with timer("data_preprocess"):
@@ -375,7 +375,7 @@ def train(self, rollout_id: int, rollout_data_ref: Box, external_data=None):
             self.train_actor(rollout_id, rollout_data, external_data=external_data)
             result = None
 
-        if self.args.offload_train or self.args.use_critic:
+        if self.args.offload_train:
             self.sleep()
 
         return result

From 9f8e12ffe709dc74351e72f66460cd1890d42bc7 Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 08:49:35 +0000
Subject: [PATCH 26/30] fix ppo value offload bugs

---
 .../update_weight/update_weight_from_distributed.py           | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
index 822b801776..6c00450c53 100644
--- a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
+++ b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
@@ -68,9 +68,7 @@ def connect_rollout_engines(
 
         if self._is_pp_src_rank:
             if self._model_update_groups is not None:
-                disconnect_rollout_engines_from_distributed(
-                    self.args, self._group_name, self._model_update_groups, self.rollout_engines
-                )
+                self.disconnect_rollout_engines()
             self._model_update_groups = connect_rollout_engines_from_distributed(
                 self.args,
                 self._group_name,

From 6c1daae679545118768135619b3f8ab32c235b08 Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 09:06:09 +0000
Subject: [PATCH 27/30] fix ppo value offload bugs

---
 .../update_weight/update_weight_from_distributed.py           | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
index 6c00450c53..822b801776 100644
--- a/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
+++ b/slime/backends/megatron_utils/update_weight/update_weight_from_distributed.py
@@ -68,7 +68,9 @@ def connect_rollout_engines(
 
         if self._is_pp_src_rank:
             if self._model_update_groups is not None:
-                self.disconnect_rollout_engines()
+                disconnect_rollout_engines_from_distributed(
+                    self.args, self._group_name, self._model_update_groups, self.rollout_engines
+                )
             self._model_update_groups = connect_rollout_engines_from_distributed(
                 self.args,
                 self._group_name,

From 136da9073c2286284537a743e967fb4846ca1bc1 Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 09:24:09 +0000
Subject: [PATCH 28/30] fix ppo value offload bugs

---
 .github/workflows/pr-test.yml.j2 | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/pr-test.yml.j2 b/.github/workflows/pr-test.yml.j2
index e627b05b33..2314e01861 100644
--- a/.github/workflows/pr-test.yml.j2
+++ b/.github/workflows/pr-test.yml.j2
@@ -32,7 +32,6 @@
         {'test_file': 'test_mimo_7B_mtp_only_grad.py', 'num_gpus': 8},
         {'test_file': 'test_qwen2.5_0.5B_debug_rollout_then_train.py', 'num_gpus': 8},
         {'test_file': 'test_qwen2.5_0.5B_opd_sglang.py', 'num_gpus': 8},
-        {'test_file': 'test_qwen3_4B_ppo_disaggregate.py', 'num_gpus': 8},
       ],
     },
     'e2e-test-precision': {

From 1c6eff744fafb03ea46865d7d4fd312e909c94ab Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Thu, 30 Apr 2026 09:49:32 +0000
Subject: [PATCH 29/30] fix ppo value offload bugs

---
 .github/workflows/pr-test.yml    | 2 +-
 .github/workflows/pr-test.yml.j2 | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml
index 3c663db0b1..fc78dd348d 100644
--- a/.github/workflows/pr-test.yml
+++ b/.github/workflows/pr-test.yml
@@ -201,7 +201,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        info: [{"num_gpus": 8, "test_file": "test_quick_start_glm4_9B.py"}, {"num_gpus": 8, "test_file": "test_glm4.7_30B_A3B_pd_mooncake.py"}, {"num_gpus": 8, "test_file": "test_qwen3_30B_A3B.py", "use_deepep": "1", "use_fp8_rollout": "1"}, {"num_gpus": 8, "test_file": "test_qwen3.6_35B_A3B_pd_mooncake.py", "use_deepep": "1"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_qwen3_30B_A3B_r3.py", "use_deepep": "1", "use_fp8_rollout": "1"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_qwen3_30B_A3B_r3.py"}, {"num_gpus": 8, "test_file": "test_qwen3_4B_ppo.py"}, {"num_gpus": 8, "test_file": "test_qwen3_4B_ppo_train_critic_only.py"}, {"num_gpus": 8, "test_file": "test_moonlight_16B_A3B.py"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_moonlight_16B_A3B_r3.py"}, {"num_gpus": 8, "test_file": "test_mimo_7B_mtp_only_grad.py"}, {"num_gpus": 8, "test_file": "test_qwen2.5_0.5B_debug_rollout_then_train.py"}, {"num_gpus": 8, "test_file": "test_qwen2.5_0.5B_opd_sglang.py"}]
+        info: [{"num_gpus": 8, "test_file": "test_quick_start_glm4_9B.py"}, {"num_gpus": 8, "test_file": "test_glm4.7_30B_A3B_pd_mooncake.py"}, {"num_gpus": 8, "test_file": "test_qwen3_30B_A3B.py", "use_deepep": "1", "use_fp8_rollout": "1"}, {"num_gpus": 8, "test_file": "test_qwen3.6_35B_A3B_pd_mooncake.py", "use_deepep": "1"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_qwen3_30B_A3B_r3.py", "use_deepep": "1", "use_fp8_rollout": "1"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_qwen3_30B_A3B_r3.py"}, {"num_gpus": 8, "test_file": "test_qwen3_4B_ppo.py"}, {"num_gpus": 8, "test_file": "test_qwen3_4B_ppo_disaggregate.py"}, {"num_gpus": 8, "test_file": "test_qwen3_4B_ppo_train_critic_only.py"}, {"num_gpus": 8, "test_file": "test_moonlight_16B_A3B.py"}, {"enable_eval": "0", "num_gpus": 8, "test_file": "test_moonlight_16B_A3B_r3.py"}, {"num_gpus": 8, "test_file": "test_mimo_7B_mtp_only_grad.py"}, {"num_gpus": 8, "test_file": "test_qwen2.5_0.5B_debug_rollout_then_train.py"}, {"num_gpus": 8, "test_file": "test_qwen2.5_0.5B_opd_sglang.py"}]
     defaults:
       run:
         working-directory: ${{ github.workspace }}
diff --git a/.github/workflows/pr-test.yml.j2 b/.github/workflows/pr-test.yml.j2
index 2314e01861..509a3d19d9 100644
--- a/.github/workflows/pr-test.yml.j2
+++ b/.github/workflows/pr-test.yml.j2
@@ -26,6 +26,7 @@
         {'test_file': 'test_qwen3_30B_A3B_r3.py', 'num_gpus': 8, 'use_deepep': '1', 'use_fp8_rollout': '1', 'enable_eval': '0'},
         {'test_file': 'test_qwen3_30B_A3B_r3.py', 'num_gpus': 8, 'enable_eval': '0'},
         {'test_file': 'test_qwen3_4B_ppo.py', 'num_gpus': 8},
+        {'test_file': 'test_qwen3_4B_ppo_disaggregate.py', 'num_gpus': 8},
         {'test_file': 'test_qwen3_4B_ppo_train_critic_only.py', 'num_gpus': 8},
         {'test_file': 'test_moonlight_16B_A3B.py', 'num_gpus': 8},
         {'test_file': 'test_moonlight_16B_A3B_r3.py', 'num_gpus': 8, 'enable_eval': '0'},

From 44c40ef6050147bc850cfc2cb435015a66b5f210 Mon Sep 17 00:00:00 2001
From: lilei <799812479@qq.com>
Date: Wed, 6 May 2026 02:53:57 +0000
Subject: [PATCH 30/30] remove copilot instructions from git

---
 .github/copilot-instructions.md | 88 ---------------------------------
 1 file changed, 88 deletions(-)
 delete mode 100644 .github/copilot-instructions.md

diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
deleted file mode 100644
index a971894a75..0000000000
--- a/.github/copilot-instructions.md
+++ /dev/null
@@ -1,88 +0,0 @@
-# Copilot Instructions for slime
-
-## What is slime
-
-slime is an LLM post-training framework for RL scaling. It connects **Megatron** (training) with **SGLang** (rollout/inference) via a **data buffer**, orchestrated by **Ray**. It supports GRPO, PPO, SFT, and on-policy distillation across models like Qwen3, DeepSeek V3, GLM, and Llama 3.
-
-## Build & Install
-
-```bash
-pip install -e . --no-deps          # editable install (no-deps because heavy deps like megatron/sglang are managed separately)
-pip install -r requirements.txt     # install Python dependencies
-```
-
-## Lint & Format
-
-Pre-commit handles all formatting (black, isort, ruff, autoflake):
-
-```bash
-pre-commit run --all-files --show-diff-on-failure --color=always
-```
-
-Toolchain: **black** (line-length 119), **isort** (black profile), **ruff** (E/F/B/UP rules, line-length 320), **autoflake** (remove unused imports).
-
-## Tests
-
-Tests are GPU-dependent end-to-end tests that require multi-GPU hardware, model weights, and datasets. They are not standard unit tests.
-
-```bash
-# Run a single test (requires GPUs; uses gpu_lock_exec to claim N GPUs)
-python tests/ci/gpu_lock_exec.py --count 4 -- python tests/test_qwen2.5_0.5B_gsm8k_short.py
-
-# Run pytest (for any pytest-compatible tests)
-pytest tests/ -m "unit"           # run only unit-marked tests
-pytest tests/test_chunked_gae.py  # run a specific test file
-```
-
-CI is triggered on PRs via GitHub labels (`run-ci-short`, `run-ci-fsdp`, `run-ci-long`). PR test workflows are auto-generated from `pr-test.yml.j2` via `generate_github_workflows.py`.
-
-## Architecture
-
-### Three-module design
-
-1. **Training** (`slime/backends/`): Megatron-based (`megatron_utils/`) or FSDP-based (`fsdp_utils/`) training backends. Handles forward/backward, loss computation, checkpointing, and weight updates.
-2. **Rollout** (`slime/rollout/`): SGLang-based inference engine that generates responses, computes rewards, and applies filters. The `sglang_rollout.py` is the main rollout driver.
-3. **Data Buffer** (`slime/rollout/data_source.py`): Manages prompt datasets and generated samples flowing between rollout and training.
-
-### Orchestration layer
-
-- **Ray** (`slime/ray/`): Manages GPU placement groups, actor groups, and the `RolloutManager`. `placement_group.py` allocates GPUs; `train_actor.py` wraps training workers.
-- **Router** (`slime/router/`): A FastAPI-based request router (`SlimeRouter`) that load-balances across SGLang engine instances with health checking and failure quarantine.
-
-### Plugin system (`slime_plugins/`)
-
-- `models/`: Model-specific adapters for SGLang (e.g., `glm4.py`, `qwen3_next.py`, `deepseek_v32.py`).
-- `mbridge/`: Model-specific adapters for the Megatron training backend.
-- `rollout_buffer/`: Standalone data generation component that can be used independently from training.
-
-### Entry points
-
-- `train.py`: Standard synchronous RL training loop.
-- `train_async.py`: Asynchronous training variant.
-- `scripts/`: Ready-to-run shell scripts for various model configurations.
-
-## Key Conventions
-
-### Arguments are in three categories
-1. **Megatron args**: Standard Megatron flags (e.g., `--tensor-model-parallel-size 2`).
-2. **SGLang args**: Must be prefixed with `--sglang-` (e.g., `--sglang-mem-fraction-static`).
-3. **slime args**: Defined in `slime/utils/arguments.py`.
-
-### Dynamic function loading
-Custom generate functions, reward models, and filters are specified as dotted Python paths (e.g., `my_module.my_reward_fn`) and loaded at runtime via `slime.utils.misc.load_function`. This is the primary extension mechanism.
-
-### Reward models (`slime/rollout/rm_hub/`)
-Built-in reward types: `math`, `dapo`, `deepscaler`, `f1`, `gpqa`, `ifbench`, `remote_rm`, `random`. Custom RMs are loaded via `--custom-rm-path`. All RM functions are async.
-
-### The `Sample` dataclass (`slime/utils/types.py`)
-Central data type flowing through the entire pipeline. Contains prompt, response, reward, loss_mask, tokens, metadata, and status tracking. Samples are grouped as `list[list[Sample]]` (groups of samples per prompt).
-
-### Training backends are swappable
-Megatron (`slime/backends/megatron_utils/`) and FSDP (`slime/backends/fsdp_utils/`) are interchangeable backends. Both implement actor, checkpoint, and weight update interfaces.
-
-### Distributed debugging
-When debugging multi-GPU/multi-node issues, see `.claude/skills/SKILL.md` for patterns around asymmetric keys, startup race conditions, and the modify-sync-restart-verify workflow.
-
-## Contribution Scope
-
-Per CONTRIBUTING.md, the project accepts **bug fixes** and **general-purpose large-scale RL optimizations** with clear benchmarks. Large refactors, abstraction proposals, and features that can't be verified through CI are out of scope.