Rebase to latest slime

.github/workflows/conda-ci.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: self-hosted
     container:
       image: lmsysorg/sglang:v0.5.0rc0-cu126
-      options: --gpus all --ipc=host --shm-size=16g --ulimit memlock=-1 --ulimit stack=67108864 --memory=0 --memory-swap=0 -v /mnt/nvme0n1/models:/root/models -v /mnt/nvme0n1/datasets:/root/datasets
+      options: --privileged --cap-add SYS_NICE --security-opt seccomp=unconfined --gpus all --ipc=host --shm-size=16g --ulimit memlock=-1 --ulimit stack=67108864 --memory=0 --memory-swap=0 -v /mnt/nvme0n1/models:/root/models -v /mnt/nvme0n1/datasets:/root/datasets
 
     defaults:
       run:

.github/workflows/pr-test.yml.j2

@@ -2,28 +2,31 @@
     'e2e-test-short': {
       'label': 'run-ci-short',
       'tests': [
-        {'test_file': 'test_qwen2.5_0.5B_gsm8k_async_short.py', 'num_gpus': 4},
-        {'test_file': 'test_qwen2.5_0.5B_gsm8k_short.py', 'num_gpus': 4},
-        {'test_file': 'test_qwen2.5_0.5B_sglang_config.py', 'num_gpus': 8},
-        {'test_file': 'test_qwen2.5_0.5B_sglang_config_distributed.py', 'num_gpus': 8},
+        {'test_file': 'test_qwen3.5_0.8B_gsm8k_async_short.py', 'num_gpus': 4},
+        {'test_file': 'test_qwen3.5_0.8B_gsm8k_short.py', 'num_gpus': 4},
+        {'test_file': 'test_qwen2.5_0.5B_ppo_critic_only_short.py', 'num_gpus': 4},
       ],
     },
-    'e2e-test-fsdp': {
-      'label': 'run-ci-fsdp',
+    'e2e-test-sglang-config': {
+      'label': 'run-ci-sglang-config',
       'tests': [
-        {'test_file': 'test_qwen3_4B_fsdp_true_on_policy.py --colocated', 'num_gpus': 4},
-        {'test_file': 'test_qwen3_vl_4B_fsdp.py', 'num_gpus': 8},
-        {'test_file': 'test_qwen3_0.6B_megatron_fsdp_align.py', 'num_gpus': 4},
+        {'test_file': 'test_qwen2.5_0.5B_sglang_config.py', 'num_gpus': 8},
+        {'test_file': 'test_qwen2.5_0.5B_sglang_config_distributed.py', 'num_gpus': 8},
+        {'test_file': 'test_sglang_config_mixed_offload.py', 'num_gpus': 8},
+        {'test_file': 'test_sglang_config_mixed_offload_ft.py', 'num_gpus': 8},
       ],
     },
     'e2e-test-megatron': {
       'label': 'run-ci-megatron',
       'tests': [
         {'test_file': 'test_quick_start_glm4_9B.py', 'num_gpus': 8},
+        {'test_file': 'test_glm4.7_30B_A3B_pd_mooncake.py', 'num_gpus': 8},
         {'test_file': 'test_qwen3_30B_A3B.py', 'num_gpus': 8, 'use_deepep': '1', 'use_fp8_rollout': '1'},
+        {'test_file': 'test_qwen3.6_35B_A3B_pd_mooncake.py', 'num_gpus': 8, 'use_deepep': '1'},
         {'test_file': 'test_qwen3_30B_A3B_r3.py', 'num_gpus': 8, 'use_deepep': '1', 'use_fp8_rollout': '1', 'enable_eval': '0'},
         {'test_file': 'test_qwen3_30B_A3B_r3.py', 'num_gpus': 8, 'enable_eval': '0'},
         {'test_file': 'test_qwen3_4B_ppo.py', 'num_gpus': 8},
+        {'test_file': 'test_qwen3_4B_ppo_disaggregate.py', 'num_gpus': 8},
         {'test_file': 'test_qwen3_4B_ppo_train_critic_only.py', 'num_gpus': 8},
         {'test_file': 'test_moonlight_16B_A3B.py', 'num_gpus': 8},
         {'test_file': 'test_moonlight_16B_A3B_r3.py', 'num_gpus': 8, 'enable_eval': '0'},
@@ -36,21 +39,22 @@
       'label': 'run-ci-precision',
       'tests': [
         {'test_file': 'test_qwen3_0.6B_parallel_check.py', 'num_gpus': 8},
-        {'test_file': 'test_qwen3_0.6B_megatron_fsdp_align.py', 'num_gpus': 4},
       ],
     },
     'e2e-test-ckpt': {
       'label': 'run-ci-ckpt',
       'tests': [
         {'test_file': 'test_qwen3_4B_ckpt.py', 'num_gpus': 8},
-        {'test_file': 'test_qwen3_4B_ckpt.py --async-save', 'num_gpus': 8},
+        {'test_file': 'test_qwen3_4B_ckpt.py', 'test_args': '--async-save', 'num_gpus': 8},
       ],
     },
 
     'e2e-test-plugin-contracts': {
       'label': 'run-ci-plugin-contracts',
       'always': True,
+      'cpu': True,
       'tests': [
+        {'test_file': 'test_megatron_argument_validation.py', 'num_gpus': 0},
         {'test_file': 'plugin_contracts/test_plugin_rollout_contracts.py', 'num_gpus': 0},
         {'test_file': 'plugin_contracts/test_plugin_runtime_hook_contracts.py', 'num_gpus': 0},
         {'test_file': 'plugin_contracts/test_plugin_path_loading_contracts.py', 'num_gpus': 0},
@@ -62,19 +66,18 @@
       'label': 'run-ci-image',
       'image': 'slimerl/slime-test:latest',
       'tests': [
-        {'test_file': 'test_qwen2.5_0.5B_gsm8k_async_short.py', 'num_gpus': 4},
-        {'test_file': 'test_qwen2.5_0.5B_gsm8k_short.py', 'num_gpus': 4},
-        {'test_file': 'test_qwen3_4B_fsdp_true_on_policy.py', 'num_gpus': 2},
-        {'test_file': 'test_qwen3_vl_4B_fsdp.py', 'num_gpus': 8},
+        {'test_file': 'test_qwen3.5_0.8B_gsm8k_async_short.py', 'num_gpus': 4},
+        {'test_file': 'test_qwen3.5_0.8B_gsm8k_short.py', 'num_gpus': 4},
         {'test_file': 'test_quick_start_glm4_9B.py', 'num_gpus': 8},
+        {'test_file': 'test_glm4.7_30B_A3B_pd_mooncake.py', 'num_gpus': 8},
         {'test_file': 'test_qwen3_30B_A3B.py', 'num_gpus': 8},
+        {'test_file': 'test_qwen3.6_35B_A3B_pd_mooncake.py', 'num_gpus': 8, 'use_deepep': '1'},
         {'test_file': 'test_qwen3_4B_ppo.py', 'num_gpus': 8},
         {'test_file': 'test_moonlight_16B_A3B.py', 'num_gpus': 8},
         {'test_file': 'test_mimo_7B_mtp_only_grad.py', 'num_gpus': 8},
         {'test_file': 'test_qwen3_0.6B_parallel_check.py', 'num_gpus': 8},
-        {'test_file': 'test_qwen3_0.6B_megatron_fsdp_align.py', 'num_gpus': 4},
         {'test_file': 'test_qwen3_4B_ckpt.py', 'num_gpus': 8},
-        {'test_file': 'test_qwen3_4B_ckpt.py --async-save', 'num_gpus': 8},
+        {'test_file': 'test_qwen3_4B_ckpt.py', 'test_args': '--async-save', 'num_gpus': 8},
         {'test_file': 'test_qwen2.5_0.5B_debug_rollout_then_train.py', 'num_gpus': 8},
         {'test_file': 'test_qwen2.5_0.5B_opd_sglang.py', 'num_gpus': 8},
       ],
@@ -109,24 +112,11 @@ jobs:
 <% else %>
     if: (github.event_name == 'workflow_dispatch') || (github.event.pull_request && contains(github.event.pull_request.labels.*.name, '<< config.label >>'))
 <% endif %>
+<% if config.get('cpu') %>
+    runs-on: ubuntu-latest
+<% else %>
     runs-on: self-hosted
-    container:
-      image: << config.image if config.image else 'slimerl/slime:latest' >>
-      options: >
-        --gpus all
-        --ipc=host
-        --shm-size=16g
-        --ulimit memlock=-1
-        --ulimit stack=67108864
-        --memory=0
-        --memory-swap=0
-        -e http_proxy=$http_proxy
-        -e https_proxy=$https_proxy
-        -e HTTP_PROXY=$HTTP_PROXY
-        -e HTTPS_PROXY=$HTTPS_PROXY
-        -v /mnt/nvme0n1/slime_ci:/data/slime_ci
-        -v /mnt/nvme0n1/slime_ci/models:/root/models
-        -v /mnt/nvme0n1/slime_ci/datasets:/root/datasets
+<% endif %>
     strategy:
       fail-fast: false
       matrix:
@@ -145,38 +135,101 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+<% if config.get('cpu') %>
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          pip install torch --index-url https://download.pytorch.org/whl/cpu
+          pip install pytest numpy packaging pyyaml omegaconf tqdm httpx pybase64 pylatexenc sympy aiohttp pillow
 
       - name: Install
         shell: bash
-        run: cd $GITHUB_WORKSPACE && pip install -e . --no-deps --break-system-packages
+        run: cd $GITHUB_WORKSPACE && pip install -e . --no-deps
+<% else %>
+<% endif %>
 
       - name: Execute
         shell: bash
         run: |
+<% if config.get('cpu') %>
           TEST_PATH="${{ matrix.info.test_file }}"
           if [[ "$TEST_PATH" != tests/* ]]; then
             TEST_PATH="tests/$TEST_PATH"
           fi
+          TEST_ARGS="${{ matrix.info.test_args || '' }}"
+          if [[ -n "$TEST_ARGS" ]]; then
+            read -r -a TEST_ARGS_ARRAY < <(printf '%s\n' "$TEST_ARGS")
+          else
+            TEST_ARGS_ARRAY=()
+          fi
           if [ "${{ matrix.info.num_gpus }}" = "0" ]; then
-            python "$TEST_PATH"
+            python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}"
           else
-            python tests/ci/gpu_lock_exec.py --count ${{ matrix.info.num_gpus }} -- python "$TEST_PATH"
+            python tests/ci/gpu_lock_exec.py --count ${{ matrix.info.num_gpus }} -- python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}"
           fi
+<% else %>
+          docker run --rm \
+            --privileged \
+            --cap-add SYS_NICE \
+            --security-opt seccomp=unconfined \
+            --network host \
+            --gpus all \
+            --ipc=host \
+            --shm-size=16g \
+            --ulimit memlock=-1 \
+            --ulimit stack=67108864 \
+            --memory=0 \
+            --memory-swap=0 \
+            -e http_proxy \
+            -e https_proxy \
+            -e HTTP_PROXY \
+            -e HTTPS_PROXY \
+            -e GITHUB_COMMIT_NAME \
+            -e WANDB_API_KEY \
+            -e SLIME_TEST_ENABLE_INFINITE_RUN \
+            -e SLIME_TEST_USE_DEEPEP \
+            -e SLIME_TEST_USE_FP8_ROLLOUT \
+            -e SLIME_TEST_ENABLE_EVAL \
+            -e TEST_FILE="${{ matrix.info.test_file }}" \
+            -e TEST_ARGS="${{ matrix.info.test_args || '' }}" \
+            -e NUM_GPUS="${{ matrix.info.num_gpus }}" \
+            -v "$GITHUB_WORKSPACE:$GITHUB_WORKSPACE" \
+            -v /mnt/nvme0n1/slime_ci:/data/slime_ci \
+            -v /mnt/nvme0n1/slime_ci/models:/root/models \
+            -v /mnt/nvme0n1/slime_ci/datasets:/root/datasets \
+            -w "$GITHUB_WORKSPACE" \
+            << config.image if config.image else 'slimerl/slime:latest' >> \
+            bash -lc '
+              set -euo pipefail
+              pip install -e . --no-deps --break-system-packages
+              TEST_PATH="$TEST_FILE"
+              if [[ "$TEST_PATH" != tests/* ]]; then
+                TEST_PATH="tests/$TEST_PATH"
+              fi
+              if [[ -n "$TEST_ARGS" ]]; then
+                read -r -a TEST_ARGS_ARRAY < <(printf "%s\n" "$TEST_ARGS")
+              else
+                TEST_ARGS_ARRAY=()
+              fi
+              if [ "$NUM_GPUS" = "0" ]; then
+                python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}"
+              else
+                python tests/ci/gpu_lock_exec.py --count "$NUM_GPUS" -- python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}"
+              fi
+            '
+<% endif %>
 <% endfor %>
 
   e2e-test-changed-detect:
     if: (github.event_name == 'workflow_dispatch') || (github.event.pull_request && contains(github.event.pull_request.labels.*.name, 'run-ci-changed'))
     runs-on: self-hosted
-    container:
-      image: slimerl/slime:latest
-      options: >
-        --gpus all
-        --ipc=host
-        --shm-size=16g
-        --ulimit memlock=-1
-        --ulimit stack=67108864
-        --memory=0
-        --memory-swap=0
     outputs:
       matrix: ${{ steps.detect.outputs.matrix }}
       has_tests: ${{ steps.detect.outputs.has_tests }}
@@ -217,23 +270,6 @@ jobs:
     needs: e2e-test-changed-detect
     if: needs.e2e-test-changed-detect.outputs.has_tests == 'true'
     runs-on: self-hosted
-    container:
-      image: slimerl/slime:latest
-      options: >
-        --gpus all
-        --ipc=host
-        --shm-size=16g
-        --ulimit memlock=-1
-        --ulimit stack=67108864
-        --memory=0
-        --memory-swap=0
-        -e http_proxy=$http_proxy
-        -e https_proxy=$https_proxy
-        -e HTTP_PROXY=$HTTP_PROXY
-        -e HTTPS_PROXY=$HTTPS_PROXY
-        -v /mnt/nvme0n1/slime_ci:/data/slime_ci
-        -v /mnt/nvme0n1/slime_ci/models:/root/models
-        -v /mnt/nvme0n1/slime_ci/datasets:/root/datasets
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.e2e-test-changed-detect.outputs.matrix) }}
@@ -252,19 +288,55 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Install
-        shell: bash
-        run: cd $GITHUB_WORKSPACE && pip install -e . --no-deps --break-system-packages
-
       - name: Execute
         shell: bash
         run: |
-          TEST_PATH="${{ matrix.info.test_file }}"
-          if [[ "$TEST_PATH" != tests/* ]]; then
-            TEST_PATH="tests/$TEST_PATH"
-          fi
-          if [ "${{ matrix.info.num_gpus }}" = "0" ]; then
-            python "$TEST_PATH"
-          else
-            python tests/ci/gpu_lock_exec.py --count ${{ matrix.info.num_gpus }} -- python "$TEST_PATH"
-          fi
+          docker run --rm \
+            --privileged \
+            --cap-add SYS_NICE \
+            --security-opt seccomp=unconfined \
+            --network host \
+            --gpus all \
+            --ipc=host \
+            --shm-size=16g \
+            --ulimit memlock=-1 \
+            --ulimit stack=67108864 \
+            --memory=0 \
+            --memory-swap=0 \
+            -e http_proxy \
+            -e https_proxy \
+            -e HTTP_PROXY \
+            -e HTTPS_PROXY \
+            -e GITHUB_COMMIT_NAME \
+            -e WANDB_API_KEY \
+            -e SLIME_TEST_ENABLE_INFINITE_RUN \
+            -e SLIME_TEST_USE_DEEPEP \
+            -e SLIME_TEST_USE_FP8_ROLLOUT \
+            -e SLIME_TEST_ENABLE_EVAL \
+            -e TEST_FILE="${{ matrix.info.test_file }}" \
+            -e TEST_ARGS="${{ matrix.info.test_args || '' }}" \
+            -e NUM_GPUS="${{ matrix.info.num_gpus }}" \
+            -v "$GITHUB_WORKSPACE:$GITHUB_WORKSPACE" \
+            -v /mnt/nvme0n1/slime_ci:/data/slime_ci \
+            -v /mnt/nvme0n1/slime_ci/models:/root/models \
+            -v /mnt/nvme0n1/slime_ci/datasets:/root/datasets \
+            -w "$GITHUB_WORKSPACE" \
+            slimerl/slime:latest \
+            bash -lc '
+              set -euo pipefail
+              pip install -e . --no-deps --break-system-packages
+              TEST_PATH="$TEST_FILE"
+              if [[ "$TEST_PATH" != tests/* ]]; then
+                TEST_PATH="tests/$TEST_PATH"
+              fi
+              if [[ -n "$TEST_ARGS" ]]; then
+                read -r -a TEST_ARGS_ARRAY < <(printf "%s\n" "$TEST_ARGS")
+              else
+                TEST_ARGS_ARRAY=()
+              fi
+              if [ "$NUM_GPUS" = "0" ]; then
+                python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}"
+              else
+                python tests/ci/gpu_lock_exec.py --count "$NUM_GPUS" -- python "$TEST_PATH" "${TEST_ARGS_ARRAY[@]}"
+              fi
+            '

CONTRIBUTING.md

@@ -6,7 +6,7 @@ Thank you for your interest in contributing to slime! We deeply appreciate every
 
 ## Collaboration Scope
 
-slime is the RL training infrastructure behind [GLM-4.5 through GLM-5](https://z.ai) and a large number of internal experiments at Z.ai. We open-sourced slime because we believe the training scenarios used internally cover the majority of cutting-edge RL algorithm requirements, and we hope to provide the community with a correct and efficient large-scale RL training infrastructure.
+slime is the RL training infrastructure behind [GLM-4.5 through GLM-5.1](https://z.ai) and a large number of internal experiments at Z.ai. We open-sourced slime because we believe the training scenarios used internally cover the majority of cutting-edge RL algorithm requirements, and we hope to provide the community with a correct and efficient large-scale RL training infrastructure.
 
 Our goal for open-source collaboration is focused on **bug fixes** and **general-purpose large-scale RL optimizations**. We have had several successful collaborations with the community in this area, including:
 

README.md

@@ -10,8 +10,8 @@
 1.  **High-Performance Training**: Supports efficient training in various modes by connecting Megatron with SGLang;
 2.  **Flexible Data Generation**: Enables arbitrary training data generation workflows through custom data generation interfaces and server-based engines.
 
-slime is the RL-framework behind [GLM-5](https://z.ai/blog/glm-5), [GLM-4.7](https://z.ai/blog/glm-4.7), [GLM-4.6](https://z.ai/blog/glm-4.6), [GLM-4.5](https://z.ai/blog/glm-4.5) and apart from models from Z.ai, we also supports the following models:
-- Qwen3 series (Qwen3Next, Qwen3MoE, Qwen3), Qwen2.5 series;
+slime is the RL-framework behind [GLM-5.1](https://z.ai/blog/glm-5.1), [GLM-5](https://z.ai/blog/glm-5), [GLM-4.7](https://z.ai/blog/glm-4.7), [GLM-4.6](https://z.ai/blog/glm-4.6), [GLM-4.5](https://z.ai/blog/glm-4.5) and apart from models from Z.ai, we also supports the following models:
+- Qwen series (Qwen3.6, Qwen3.5, Qwen3Next, Qwen3MoE, Qwen3, Qwen2.5);
 - DeepSeek V3 series (DeepSeek V3, V3.1, DeepSeek R1);
 - Llama 3.
 
@@ -51,6 +51,14 @@ We also provide examples for some use cases not covered in the quick start guide
 
 slime has powered several novel research projects and production systems. Here are some notable examples:
 
+### 🌈 Relax: Asynchronous RL Engine for Omni-Modal Agentic Training
+
+[**Relax**](https://github.com/redai-infra/Relax) (Reinforcement Engine Leveraging Agentic X-modality) is an omni-modal agentic RL framework open-sourced by the RedAI Infra team, built upon the slime infrastructure stack that combines Ray, Megatron-LM, and SGLang. Relax adopts a service-oriented architecture on Ray Serve with Megatron-LM and SGLang as training/inference backends. It uses [TransferQueue](https://github.com/Ascend/TransferQueue) to fully decouple Actor, Rollout, ActorFwd, Reference, and Advantage computation onto independent GPU clusters, and introduces **DCS (Distributed Checkpoint Service)** — an NCCL-broadcast weight-sync engine that streams updated Actor weights to Rollout/ActorFwd/Reference asynchronously and overlaps the transfer with the next training step, enabling fully-async training at configurable staleness. Relax supports end-to-end RL for text, vision, and audio (including Qwen3-Omni) and agentic multi-turn rollouts.
+
+### 🦞 OpenClaw-RL: Train a Personalized Clawbot Simply by Talking to It
+
+[**OpenClaw-RL**](https://github.com/Gen-Verse/OpenClaw-RL) is an RL server for personalized OpenClaw agents. It hosts the OpenClaw model and improves it from prior conversations across deployments, while slime's asynchronous RL infrastructure prevents training from interfering with API serving. It supports two automatic optimization methods: GRPO with binary feedback inferred from subsequent states, and on-policy distillation that extracts hindsight hints from later feedback for the current policy.
+
 ### ⚛️ P1: Mastering Physics Olympiads with Reinforcement Learning
 
 [**P1**](https://prime-rl.github.io/P1/) is a family of open-source physics reasoning models trained entirely through reinforcement learning. P1 leverages slime as the RL post training framework, and introduces a multi-stage RL training algorithm that progressively enhances reasoning ability through adaptive learnability adjustment and stabilization mechanisms. Enpowered by this training paradigm, P1 delivers breakthrough performance in open-source physics reasoning.

README_zh.md

@@ -10,8 +10,8 @@
 1. **高性能训练**：通过连接 Megatron 与 SGLang，支持各种模式的高效训练；
 2. **灵活的数据生成**：通过自定义数据生成接口以及 server based engine，实现任意的数据训练数据生成流程。
 
-slime 是 [GLM-5](https://z.ai/blog/glm-5)、[GLM-4.7](https://z.ai/blog/glm-4.7)、[GLM-4.6](https://z.ai/blog/glm-4.6)、[GLM-4.5](https://z.ai/blog/glm-4.5) 背后的 RL 训练框架，除此之外，slime 还支持:
-- Qwen3 系列 (Qwen3Next, Qwen3MoE, Qwen3), Qwen2.5 系列；
+slime 是 [GLM-5.1](https://z.ai/blog/glm-5.1)、[GLM-5](https://z.ai/blog/glm-5)、[GLM-4.7](https://z.ai/blog/glm-4.7)、[GLM-4.6](https://z.ai/blog/glm-4.6)、[GLM-4.5](https://z.ai/blog/glm-4.5) 背后的 RL 训练框架，除此之外，slime 还支持:
+- Qwen 系列 (Qwen3.6、Qwen3.5、Qwen3Next、Qwen3MoE、Qwen3、Qwen2.5)；
 - DeepSeek V3 系列 (DeepSeek V3, V3.1, DeepSeek R1)；
 - Llama 3。