From c4cd0b6e675c621406d59ff748a30bf185940ba7 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 14:52:32 +0800 Subject: [PATCH 001/105] more --- .github/workflows/pr-test.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/pr-test.yml diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml new file mode 100644 index 00000000000..1d8fd01632c --- /dev/null +++ b/.github/workflows/pr-test.yml @@ -0,0 +1,32 @@ +name: PR Test + +on: + push: + branches: [main] + pull_request: + branches: [main] + types: [synchronize, labeled] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test-disaggregate-glm4-9B: + if: github.event.pull_request.draft == false + runs-on: self-hosted + container: + image: slimerl/slime:latest + options: --gpus all --ipc=host --shm-size=16g --ulimit memlock=-1 --ulimit stack=67108864 --memory=0 --memory-swap=0 -v /data/models:/root/models -v /data/datasets:/root/datasets + + defaults: + run: + working-directory: ${{ github.workspace }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Execute + run: TODO + shell: bash From 6cb02eaac567c99716e60f2f471743466441020a Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 14:54:52 +0800 Subject: [PATCH 002/105] more --- .github/workflows/pr-test.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 1d8fd01632c..2187d9826e6 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -13,6 +13,7 @@ concurrency: jobs: test-disaggregate-glm4-9B: + # TODO may use run-ci label etc if: github.event.pull_request.draft == false runs-on: self-hosted container: @@ -27,6 +28,10 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Install + shell: bash + run: cd $GITHUB_WORKSPACE && pip install -e . + - name: Execute - run: TODO shell: bash + run: TODO From 1ab13dc5238f0c488854d4c17fc6b5e3eb288d98 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 14:55:20 +0800 Subject: [PATCH 003/105] temp rm old --- tests/test-qwen2.5-0.5B-gsm8k-async.sh | 135 ---------------- tests/test-qwen2.5-0.5B-gsm8k.sh | 135 ---------------- tests/test_fsdp_import.py | 9 -- tests/test_quick_start_glm4-9B.sh | 141 ---------------- tests/test_qwen3-0.6B_fsdp_colocated_2xGPU.sh | 116 ------------- tests/test_qwen3-0.6B_fsdp_distributed.sh | 81 ---------- tests/test_qwen3-30B-A3B.sh | 152 ------------------ 7 files changed, 769 deletions(-) delete mode 100644 tests/test-qwen2.5-0.5B-gsm8k-async.sh delete mode 100644 tests/test-qwen2.5-0.5B-gsm8k.sh delete mode 100644 tests/test_fsdp_import.py delete mode 100644 tests/test_quick_start_glm4-9B.sh delete mode 100755 tests/test_qwen3-0.6B_fsdp_colocated_2xGPU.sh delete mode 100644 tests/test_qwen3-0.6B_fsdp_distributed.sh delete mode 100644 tests/test_qwen3-30B-A3B.sh diff --git a/tests/test-qwen2.5-0.5B-gsm8k-async.sh b/tests/test-qwen2.5-0.5B-gsm8k-async.sh deleted file mode 100644 index 8b04ab2c4f4..00000000000 --- a/tests/test-qwen2.5-0.5B-gsm8k-async.sh +++ /dev/null @@ -1,135 +0,0 @@ -#!/bin/bash - -# for rerun the task -pkill -9 sglang -sleep 3 -ray stop --force -pkill -9 ray -pkill -9 python -sleep 3 -pkill -9 ray -pkill -9 python - -set -ex - - -huggingface-cli download --repo-type dataset zhuzilin/gsm8k --local-dir gsm8k - - -# will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 - -SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -source "${SCRIPT_DIR}/../scripts/models/qwen2.5-0.5B.sh" - -CKPT_ARGS=( - --hf-checkpoint /root/Qwen2.5-0.5B-Instruct/ - --ref-load /root/Qwen2.5-0.5B-Instruct_torch_dist/ -) - -ROLLOUT_ARGS=( - --prompt-data gsm8k/train.parquet - --input-key messages - --label-key label - --apply-chat-template - --rollout-shuffle - --rm-type math - --num-rollout 3000 - --rollout-batch-size 32 - --n-samples-per-prompt 8 - --rollout-max-response-len 1024 - --rollout-temperature 0.8 - --rollout-num-gpus 2 - - --over-sampling-batch-size 64 - --dynamic-sampling-filter-path miles.rollout.filter_hub.dynamic_sampling_filters.check_reward_nonzero_std - - --global-batch-size 256 -) - -EVAL_ARGS=( - --eval-interval 20 - --eval-prompt-data gsm8k gsm8k/test.parquet - --n-samples-per-eval-prompt 1 - --eval-max-response-len 1024 - --eval-top-k 1 -) - -PERF_ARGS=( - --tensor-model-parallel-size 1 - --sequence-parallel - --pipeline-model-parallel-size 1 - --context-parallel-size 1 - --expert-model-parallel-size 1 - --expert-tensor-parallel-size 1 - - # --micro-batch-size 1 - --use-dynamic-batch-size - --max-tokens-per-gpu 9216 -) - -GRPO_ARGS=( - --advantage-estimator grpo - --use-kl-loss - --kl-loss-coef 0.00 - --kl-loss-type low_var_kl - --entropy-coef 0.00 - --eps-clip 0.2 - --eps-clip-high 0.28 -) - -OPTIMIZER_ARGS=( - --optimizer adam - --lr 1e-6 - --lr-decay-style constant - --weight-decay 0.1 - --adam-beta1 0.9 - --adam-beta2 0.98 -) - -WANDB_ARGS=( - --use-wandb - --wandb-project miles-test - --wandb-group test-qwen2.5-0.5B-gsm8k -) - -SGLANG_ARGS=( - --rollout-num-gpus-per-engine 1 - --sglang-mem-fraction-static 0.7 -) - -MISC_ARGS=( - # default dropout in megatron is 0.1 - --attention-dropout 0.0 - --hidden-dropout 0.0 - # should be good for model performance - --accumulate-allreduce-grads-in-fp32 - --attention-softmax-in-fp32 - # need to comment this when using model with MLA - --attention-backend flash -) - -# launch the master node of ray in container -ray start --head --node-ip-address ${MASTER_ADDR} --num-gpus 4 --disable-usage-stats - -ray job submit --address="http://127.0.0.1:8265" \ - --runtime-env-json='{ - "env_vars": { - "PYTHONPATH": "/root/Megatron-LM", - "CUDA_DEVICE_MAX_CONNECTIONS": "1" - } - }' \ - -- python3 train_async.py \ - --actor-num-nodes 1 \ - --actor-num-gpus-per-node 2 \ - ${MODEL_ARGS[@]} \ - ${CKPT_ARGS[@]} \ - ${ROLLOUT_ARGS[@]} \ - ${OPTIMIZER_ARGS[@]} \ - ${GRPO_ARGS[@]} \ - ${DISTRIBUTED_ARGS[@]} \ - ${WANDB_ARGS[@]} \ - ${PERF_ARGS[@]} \ - ${EVAL_ARGS[@]} \ - ${SGLANG_ARGS[@]} \ - ${MISC_ARGS[@]} diff --git a/tests/test-qwen2.5-0.5B-gsm8k.sh b/tests/test-qwen2.5-0.5B-gsm8k.sh deleted file mode 100644 index fd598c454fb..00000000000 --- a/tests/test-qwen2.5-0.5B-gsm8k.sh +++ /dev/null @@ -1,135 +0,0 @@ -#!/bin/bash - -# for rerun the task -pkill -9 sglang -sleep 3 -ray stop --force -pkill -9 ray -pkill -9 python -sleep 3 -pkill -9 ray -pkill -9 python - -set -ex - - -huggingface-cli download --repo-type dataset zhuzilin/gsm8k --local-dir gsm8k - - -# will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 - -SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -source "${SCRIPT_DIR}/../scripts/models/qwen2.5-0.5B.sh" - -CKPT_ARGS=( - --hf-checkpoint /root/Qwen2.5-0.5B-Instruct/ - --ref-load /root/Qwen2.5-0.5B-Instruct_torch_dist/ -) - -ROLLOUT_ARGS=( - --prompt-data gsm8k/train.parquet - --input-key messages - --label-key label - --apply-chat-template - --rollout-shuffle - --rm-type math - --num-rollout 3000 - --rollout-batch-size 32 - --n-samples-per-prompt 8 - --rollout-max-response-len 1024 - --rollout-temperature 0.8 - - --over-sampling-batch-size 64 - --dynamic-sampling-filter-path miles.rollout.filter_hub.dynamic_sampling_filters.check_reward_nonzero_std - - --global-batch-size 256 -) - -EVAL_ARGS=( - --eval-interval 20 - --eval-prompt-data gsm8k gsm8k/test.parquet - --n-samples-per-eval-prompt 1 - --eval-max-response-len 1024 - --eval-top-k 1 -) - -PERF_ARGS=( - --tensor-model-parallel-size 1 - --sequence-parallel - --pipeline-model-parallel-size 1 - --context-parallel-size 1 - --expert-model-parallel-size 1 - --expert-tensor-parallel-size 1 - - # --micro-batch-size 1 - --use-dynamic-batch-size - --max-tokens-per-gpu 9216 -) - -GRPO_ARGS=( - --advantage-estimator grpo - --use-kl-loss - --kl-loss-coef 0.00 - --kl-loss-type low_var_kl - --entropy-coef 0.00 - --eps-clip 0.2 - --eps-clip-high 0.28 -) - -OPTIMIZER_ARGS=( - --optimizer adam - --lr 1e-6 - --lr-decay-style constant - --weight-decay 0.1 - --adam-beta1 0.9 - --adam-beta2 0.98 -) - -WANDB_ARGS=( - --use-wandb - --wandb-project miles-test - --wandb-group test-qwen2.5-0.5B-gsm8k -) - -SGLANG_ARGS=( - --rollout-num-gpus-per-engine 1 - --sglang-mem-fraction-static 0.7 -) - -MISC_ARGS=( - # default dropout in megatron is 0.1 - --attention-dropout 0.0 - --hidden-dropout 0.0 - # should be good for model performance - --accumulate-allreduce-grads-in-fp32 - --attention-softmax-in-fp32 - # need to comment this when using model with MLA - --attention-backend flash -) - -# launch the master node of ray in container -ray start --head --node-ip-address ${MASTER_ADDR} --num-gpus 4 --disable-usage-stats - -ray job submit --address="http://127.0.0.1:8265" \ - --runtime-env-json='{ - "env_vars": { - "PYTHONPATH": "/root/Megatron-LM", - "CUDA_DEVICE_MAX_CONNECTIONS": "1" - } - }' \ - -- python3 train.py \ - --actor-num-nodes 1 \ - --actor-num-gpus-per-node 4 \ - --colocate \ - ${MODEL_ARGS[@]} \ - ${CKPT_ARGS[@]} \ - ${ROLLOUT_ARGS[@]} \ - ${OPTIMIZER_ARGS[@]} \ - ${GRPO_ARGS[@]} \ - ${DISTRIBUTED_ARGS[@]} \ - ${WANDB_ARGS[@]} \ - ${PERF_ARGS[@]} \ - ${EVAL_ARGS[@]} \ - ${SGLANG_ARGS[@]} \ - ${MISC_ARGS[@]} diff --git a/tests/test_fsdp_import.py b/tests/test_fsdp_import.py deleted file mode 100644 index 66b6861ed15..00000000000 --- a/tests/test_fsdp_import.py +++ /dev/null @@ -1,9 +0,0 @@ -import pytest - - -def test_fsdp_import(): - try: - from torch.distributed.fsdp import FullyShardedDataParallel as FSDP - except ImportError: - pytest.skip("FSDP not available in this environment") - assert FSDP is not None diff --git a/tests/test_quick_start_glm4-9B.sh b/tests/test_quick_start_glm4-9B.sh deleted file mode 100644 index 2af9e24fc25..00000000000 --- a/tests/test_quick_start_glm4-9B.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash - -set -e - -# will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 - -NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) -if [ "$NVLINK_COUNT" -gt 0 ]; then - HAS_NVLINK=1 -else - HAS_NVLINK=0 -fi -echo "HAS_NVLINK: $HAS_NVLINK (detected $NVLINK_COUNT NVLink references)" - -SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -source "${SCRIPT_DIR}/../scripts/models/glm4-9B.sh" - -CKPT_ARGS=( - --hf-checkpoint /root/models/GLM-Z1-9B-0414/ - --ref-load /root/GLM-Z1-9B-0414_torch_dist - - --fp8-format e4m3 - --fp8-recipe blockwise -) - -ROLLOUT_ARGS=( - --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl - --input-key prompt - --label-key label - --apply-chat-template - --rollout-shuffle - - --rm-type deepscaler - - --num-rollout 3 - --rollout-batch-size 8 - --n-samples-per-prompt 8 - --rollout-max-response-len 8192 - --rollout-temperature 0.8 - - --global-batch-size 32 - --balance-data -) - -EVAL_ARGS=( - --eval-interval 20 - --eval-prompt-data aime24 /root/datasets/aime-2024/aime-2024.jsonl - --n-samples-per-eval-prompt 1 - --eval-max-response-len 16384 - --eval-top-k 1 -) - -PERF_ARGS=( - --tensor-model-parallel-size 2 - --sequence-parallel - --pipeline-model-parallel-size 1 - --context-parallel-size 2 - --expert-model-parallel-size 1 - --expert-tensor-parallel-size 1 - - --recompute-granularity full - --recompute-method uniform - --recompute-num-layers 1 - - --use-dynamic-batch-size - --max-tokens-per-gpu 4608 -) - -GRPO_ARGS=( - --advantage-estimator grpo - --use-kl-loss - --kl-loss-coef 0.00 - --kl-loss-type low_var_kl - --entropy-coef 0.00 - --eps-clip 0.2 - --eps-clip-high 0.28 - - --use-tis - --calculate-per-token-loss -) - -OPTIMIZER_ARGS=( - --optimizer adam - --lr 1e-6 - --lr-decay-style constant - --weight-decay 0.1 - --adam-beta1 0.9 - --adam-beta2 0.98 -) - -SGLANG_ARGS=( - --rollout-num-gpus-per-engine 2 - - --use-miles-router -) - -MISC_ARGS=( - # default dropout in megatron is 0.1 - --attention-dropout 0.0 - --hidden-dropout 0.0 - # should be good for model performance - --accumulate-allreduce-grads-in-fp32 - --attention-softmax-in-fp32 - # need to comment this when using model with MLA - --attention-backend flash -) - -# launch the master node of ray in container -export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} -export no_proxy="127.0.0.1,${MASTER_ADDR}" -ray start --head --node-ip-address ${MASTER_ADDR} --num-gpus 8 --disable-usage-stats - -# Build the runtime environment JSON with proper variable substitution -RUNTIME_ENV_JSON="{ - \"env_vars\": { - \"PYTHONPATH\": \"/root/Megatron-LM/\", - \"CUDA_DEVICE_MAX_CONNECTIONS\": \"1\", - \"NCCL_NVLS_ENABLE\": \"${HAS_NVLINK}\", - \"no_proxy\": \"${no_proxy}\" - } -}" - -ray job submit --address="http://127.0.0.1:8265" \ - --runtime-env-json="${RUNTIME_ENV_JSON}" \ - -- python3 train.py \ - --actor-num-nodes 1 \ - --actor-num-gpus-per-node 4 \ - --rollout-num-gpus 4 \ - ${MODEL_ARGS[@]} \ - ${CKPT_ARGS[@]} \ - ${ROLLOUT_ARGS[@]} \ - ${OPTIMIZER_ARGS[@]} \ - ${GRPO_ARGS[@]} \ - ${DISTRIBUTED_ARGS[@]} \ - ${WANDB_ARGS[@]} \ - ${PERF_ARGS[@]} \ - ${EVAL_ARGS[@]} \ - ${SGLANG_ARGS[@]} \ - ${MISC_ARGS[@]} \ - --ci-test diff --git a/tests/test_qwen3-0.6B_fsdp_colocated_2xGPU.sh b/tests/test_qwen3-0.6B_fsdp_colocated_2xGPU.sh deleted file mode 100755 index edf96f37420..00000000000 --- a/tests/test_qwen3-0.6B_fsdp_colocated_2xGPU.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash - -# FSDP Colocated 2GPU Training Script with Weights & Biases Support -# -# This script runs FSDP training with wandb logging enabled. -# -# Wandb Configuration: -# - Rank and world size are automatically detected from distributed context -# - Only rank 0 will log to wandb to avoid duplicate entries -# - Distributed coordination handled by torch.distributed in FSDP actors -# -# To customize wandb settings: -# 1. Uncomment and set --wandb-team if you're using a team/organization (optional for personal accounts) -# 2. Set your wandb API key if needed (or use 'wandb login' beforehand) -# 3. Modify project name and group as needed -# 4. Change wandb mode to 'offline' for local logging only -# 5. Uncomment --wandb-dir to specify custom log directory - -# for rerun the task -pkill -9 sglang -sleep 3 -ray stop --force -pkill -9 ray -pkill -9 python -sleep 3 -pkill -9 ray -pkill -9 python - -set -ex - -# will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 - -CKPT_ARGS=( - --hf-checkpoint /root/Qwen3-0.6B -) - -ROLLOUT_ARGS=( - --prompt-data /root/dapo-math-17k/dapo-math-17k.jsonl - --input-key prompt - --label-key label - --apply-chat-template - --rollout-shuffle - --rm-type deepscaler - --num-rollout 1000 - --rollout-batch-size 8 - --n-samples-per-prompt 8 - --rollout-max-response-len 4096 - --rollout-temperature 0.8 - - --global-batch-size 64 -) - -GRPO_ARGS=( - --advantage-estimator grpo - #--use-kl-loss - --kl-loss-coef 0.00 - --kl-loss-type low_var_kl - --kl-coef 0.00 - --entropy-coef 0.00 - --eps-clip 0.2 - --eps-clip-high 0.28 -) - -OPTIMIZER_ARGS=( - --optimizer adam - --lr 1e-6 - --lr-decay-style constant - --weight-decay 0.1 - --adam-beta1 0.9 - --adam-beta2 0.98 -) - -SGLANG_ARGS=( - # Set equal to the number of GPUs per node for colocated mode - --rollout-num-gpus-per-engine 2 - --sglang-decode-log-interval 1000 -) - - -WANDB_ARGS=( - --use-wandb - --wandb-project "gsm8k_async_rl" - --wandb-group "fsdp-2gpu-colocated" - --wandb-mode "online" # Change to "offline" for local logging only -) - -FSDP_ARGS=( - # Set to true for FULL_STATE_DICT mode, false for SHARDED_STATE_DICT mode (default) - # --fsdp-full-params # Uncomment this line to enable full params mode - - # Set the bucket size for weight update - --update-weights-bucket-size 512 * 1024 * 1024 # 512MB -) - -# launch the master node of ray in container -ray start --head --node-ip-address 127.0.0.1 --num-gpus 2 --disable-usage-stats - -ray job submit --address="http://127.0.0.1:8265" \ - --runtime-env-json='{ - "env_vars": { - "no_proxy": "localhost,127.0.0.1,0.0.0.0,${MASTER_ADDR}", - "MILES_BACKEND": "fsdp" - } - }' \ - -- python3 train.py \ - --actor-num-nodes 1 \ - --actor-num-gpus-per-node 2 \ - --colocate \ - ${CKPT_ARGS[@]} \ - ${ROLLOUT_ARGS[@]} \ - ${OPTIMIZER_ARGS[@]} \ - ${GRPO_ARGS[@]} \ - ${DISTRIBUTED_ARGS[@]} \ - ${SGLANG_ARGS[@]} \ - ${WANDB_ARGS[@]} \ No newline at end of file diff --git a/tests/test_qwen3-0.6B_fsdp_distributed.sh b/tests/test_qwen3-0.6B_fsdp_distributed.sh deleted file mode 100644 index 0ea41768404..00000000000 --- a/tests/test_qwen3-0.6B_fsdp_distributed.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash - -# for rerun the task -pkill -9 sglang -sleep 3 -ray stop --force -pkill -9 ray -pkill -9 python -sleep 3 -pkill -9 ray -pkill -9 python - -set -ex - -# will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 - -CKPT_ARGS=( - --hf-checkpoint /root/Qwen3-0.6B -) - -ROLLOUT_ARGS=( - --prompt-data /root/dapo-math-17k/dapo-math-17k.jsonl - --input-key prompt - --label-key label - --apply-chat-template - --rollout-shuffle - --rm-type deepscaler - --num-rollout 3000 - --rollout-batch-size 16 - --n-samples-per-prompt 16 - --rollout-max-response-len 8192 - --rollout-temperature 0.8 - - --global-batch-size 128 -) - -GRPO_ARGS=( - --advantage-estimator grpo - #--use-kl-loss - --kl-loss-coef 0.00 - --kl-loss-type low_var_kl - --kl-coef 0.00 - --entropy-coef 0.00 - --eps-clip 0.2 - --eps-clip-high 0.28 -) - -OPTIMIZER_ARGS=( - --optimizer adam - --lr 1e-6 - --lr-decay-style constant - --weight-decay 0.1 - --adam-beta1 0.9 - --adam-beta2 0.98 -) - -SGLANG_ARGS=( - --rollout-num-gpus-per-engine 1 -) - -# launch the master node of ray in container -ray start --head --node-ip-address 127.0.0.1 --num-gpus 4 --disable-usage-stats - -ray job submit --address="http://127.0.0.1:8265" \ - --runtime-env-json='{ - "env_vars": { - "no_proxy": "localhost,127.0.0.1,0.0.0.0,${MASTER_ADDR}", - "MILES_BACKEND": "fsdp" - } - }' \ - -- python3 train.py \ - --actor-num-nodes 1 \ - --actor-num-gpus-per-node 4 \ - --colocate \ - ${CKPT_ARGS[@]} \ - ${ROLLOUT_ARGS[@]} \ - ${OPTIMIZER_ARGS[@]} \ - ${GRPO_ARGS[@]} \ - ${DISTRIBUTED_ARGS[@]} \ - ${SGLANG_ARGS[@]} diff --git a/tests/test_qwen3-30B-A3B.sh b/tests/test_qwen3-30B-A3B.sh deleted file mode 100644 index 125efc82750..00000000000 --- a/tests/test_qwen3-30B-A3B.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/bin/bash - -set -e - -# will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 - -NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) -if [ "$NVLINK_COUNT" -gt 0 ]; then - HAS_NVLINK=1 -else - HAS_NVLINK=0 -fi -echo "HAS_NVLINK: $HAS_NVLINK (detected $NVLINK_COUNT NVLink references)" - -SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -source "${SCRIPT_DIR}/../scripts/models/qwen3-30B-A3B.sh" - -CKPT_ARGS=( - --hf-checkpoint /root/models/Qwen3-30B-A3B-FP8 - --ref-load /root/Qwen3-30B-A3B_torch_dist -) - -ROLLOUT_ARGS=( - --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl - --input-key prompt - --label-key label - --apply-chat-template - --rollout-shuffle - - --rm-type deepscaler - - --num-rollout 3 - --rollout-batch-size 8 - --n-samples-per-prompt 8 - --rollout-max-response-len 8192 - --rollout-temperature 0.8 - - --global-batch-size 32 - --balance-data -) - -EVAL_ARGS=( - --eval-interval 20 - --eval-prompt-data aime24 /root/datasets/aime-2024/aime-2024.jsonl - --n-samples-per-eval-prompt 1 - --eval-max-response-len 16384 - --eval-top-k 1 -) - -PERF_ARGS=( - --tensor-model-parallel-size 4 - --sequence-parallel - --pipeline-model-parallel-size 1 - --context-parallel-size 2 - --expert-model-parallel-size 8 - --expert-tensor-parallel-size 1 - - --recompute-granularity full - --recompute-method uniform - --recompute-num-layers 1 - - --use-dynamic-batch-size - --max-tokens-per-gpu 16384 -) - -GRPO_ARGS=( - --advantage-estimator gspo - --use-kl-loss - --kl-loss-coef 0.00 - --kl-loss-type low_var_kl - --kl-coef 0.00 - --entropy-coef 0.00 - --eps-clip 4e-4 - - --use-tis - --use-routing-replay -) - -OPTIMIZER_ARGS=( - --optimizer adam - --lr 1e-6 - --lr-decay-style constant - --weight-decay 0.1 - --adam-beta1 0.9 - --adam-beta2 0.98 - - --optimizer-cpu-offload - --overlap-cpu-optimizer-d2h-h2d - --use-precision-aware-optimizer -) - -SGLANG_ARGS=( - --rollout-num-gpus-per-engine 8 - --sglang-mem-fraction-static 0.8 - - --sglang-moe-a2a-backend deepep - --sglang-deepep-mode auto - - --sglang-max-running-requests 512 - --sglang-disable-radix-cache -) - -MISC_ARGS=( - # default dropout in megatron is 0.1 - --attention-dropout 0.0 - --hidden-dropout 0.0 - # should be good for model performance - --accumulate-allreduce-grads-in-fp32 - --attention-softmax-in-fp32 - # need to comment this when using model with MLA - --attention-backend flash - - --moe-token-dispatcher-type flex - --moe-enable-deepep -) - -# launch the master node of ray in container -export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} -export no_proxy="127.0.0.1,${MASTER_ADDR}" -ray start --head --node-ip-address ${MASTER_ADDR} --num-gpus 8 --disable-usage-stats - -export CUDA_HOME=${CUDA_HOME:-"/usr/local/cuda"} -# Build the runtime environment JSON with proper variable substitution -RUNTIME_ENV_JSON="{ - \"env_vars\": { - \"PYTHONPATH\": \"/root/Megatron-LM/\", - \"CUDA_DEVICE_MAX_CONNECTIONS\": \"1\", - \"NCCL_NVLS_ENABLE\": \"${HAS_NVLINK}\", - \"no_proxy\": \"${no_proxy}\", - \"CUDA_HOME\": \"${CUDA_HOME}\" - } -}" - -ray job submit --address="http://127.0.0.1:8265" \ - --runtime-env-json="${RUNTIME_ENV_JSON}" \ - -- python3 train.py \ - --actor-num-nodes 1 \ - --actor-num-gpus-per-node 8 \ - --colocate \ - ${MODEL_ARGS[@]} \ - ${CKPT_ARGS[@]} \ - ${ROLLOUT_ARGS[@]} \ - ${OPTIMIZER_ARGS[@]} \ - ${GRPO_ARGS[@]} \ - ${DISTRIBUTED_ARGS[@]} \ - ${WANDB_ARGS[@]} \ - ${PERF_ARGS[@]} \ - ${EVAL_ARGS[@]} \ - ${SGLANG_ARGS[@]} \ - ${MISC_ARGS[@]} \ - --ci-test From 5855b0245f914e54b16ae0a0b3ef444ec1ab6b04 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 14:57:26 +0800 Subject: [PATCH 004/105] more --- .github/workflows/pr-test.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 2187d9826e6..fe4a1d8a661 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -19,6 +19,13 @@ jobs: container: image: slimerl/slime:latest options: --gpus all --ipc=host --shm-size=16g --ulimit memlock=-1 --ulimit stack=67108864 --memory=0 --memory-swap=0 -v /data/models:/root/models -v /data/datasets:/root/datasets + strategy: + fail-fast: false + matrix: + test_name: + - test_quick_start_glm4_9B + - test_qwen3_30B_A3B + # TODO more defaults: run: From eb09a9066a44454b13f122d58ac19422ef34ecd0 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 14:58:45 +0800 Subject: [PATCH 005/105] more --- tests/run_suite.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/run_suite.py diff --git a/tests/run_suite.py b/tests/run_suite.py new file mode 100644 index 00000000000..e69de29bb2d From b0e3b6e8df1f6d4a9f73d00bba729f2effdbea96 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:00:00 +0800 Subject: [PATCH 006/105] more --- .github/workflows/pr-test.yml | 2 +- tests/run_suite.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index fe4a1d8a661..f1dba10ff72 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -41,4 +41,4 @@ jobs: - name: Execute shell: bash - run: TODO + run: python tests/run_suite.py --test-name ${{ matrix.test_name }} diff --git a/tests/run_suite.py b/tests/run_suite.py index e69de29bb2d..09e34c344a2 100644 --- a/tests/run_suite.py +++ b/tests/run_suite.py @@ -0,0 +1,9 @@ +import argparse + +if __name__ == "__main__": + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument("--test-name", type=str) + args = arg_parser.parse_args() + print(f"{args=}") + + TODO From 010a0b1725201881dc23b4100ccc6d55a7f493c6 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:00:52 +0800 Subject: [PATCH 007/105] more --- .github/workflows/pr-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index f1dba10ff72..8190b7ae38d 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -12,7 +12,7 @@ concurrency: cancel-in-progress: true jobs: - test-disaggregate-glm4-9B: + primary_test: # TODO may use run-ci label etc if: github.event.pull_request.draft == false runs-on: self-hosted From 8c00ffa2914a1ee8eb4c42dff8030e77cd4fd940 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:01:03 +0800 Subject: [PATCH 008/105] more --- tests/test_quick_start_glm4_9B.py | 0 tests/test_qwen3_30B_A3B.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/test_quick_start_glm4_9B.py create mode 100644 tests/test_qwen3_30B_A3B.py diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_qwen3_30B_A3B.py b/tests/test_qwen3_30B_A3B.py new file mode 100644 index 00000000000..e69de29bb2d From 9d15f751360a64fddbace8ad3828b9e34484ac02 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:02:01 +0800 Subject: [PATCH 009/105] more --- tests/command_utils.py | 0 tests/test_quick_start_glm4_9B.py | 11 +++++++++++ tests/test_qwen3_30B_A3B.py | 11 +++++++++++ 3 files changed, 22 insertions(+) create mode 100644 tests/command_utils.py diff --git a/tests/command_utils.py b/tests/command_utils.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index e69de29bb2d..014a6fb9153 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -0,0 +1,11 @@ +import command_utils + + +def main(): + TODO + + +if __name__ == '__main__': + import typer + + typer.run(main) diff --git a/tests/test_qwen3_30B_A3B.py b/tests/test_qwen3_30B_A3B.py index e69de29bb2d..014a6fb9153 100644 --- a/tests/test_qwen3_30B_A3B.py +++ b/tests/test_qwen3_30B_A3B.py @@ -0,0 +1,11 @@ +import command_utils + + +def main(): + TODO + + +if __name__ == '__main__': + import typer + + typer.run(main) From 00d12956e33365a32b1350756397941333188a0e Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:02:31 +0800 Subject: [PATCH 010/105] more --- tests/test_quick_start_glm4_9B.py | 4 +--- tests/test_qwen3_30B_A3B.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index 014a6fb9153..e75226474a3 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -6,6 +6,4 @@ def main(): if __name__ == '__main__': - import typer - - typer.run(main) + pass # TODO make the functions executable via typer diff --git a/tests/test_qwen3_30B_A3B.py b/tests/test_qwen3_30B_A3B.py index 014a6fb9153..e75226474a3 100644 --- a/tests/test_qwen3_30B_A3B.py +++ b/tests/test_qwen3_30B_A3B.py @@ -6,6 +6,4 @@ def main(): if __name__ == '__main__': - import typer - - typer.run(main) + pass # TODO make the functions executable via typer From 0f5637b1709692a4769857502b6fba4f2d72465a Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:02:52 +0800 Subject: [PATCH 011/105] more --- tests/test_quick_start_glm4_9B.py | 6 +++++- tests/test_qwen3_30B_A3B.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index e75226474a3..2ab74dab9aa 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -1,7 +1,11 @@ import command_utils -def main(): +def prepare(): + TODO + + +def execute(): TODO diff --git a/tests/test_qwen3_30B_A3B.py b/tests/test_qwen3_30B_A3B.py index e75226474a3..2ab74dab9aa 100644 --- a/tests/test_qwen3_30B_A3B.py +++ b/tests/test_qwen3_30B_A3B.py @@ -1,7 +1,11 @@ import command_utils -def main(): +def prepare(): + TODO + + +def execute(): TODO From 72821a50af853fd4916f0e08c3d9d8eb9d74d847 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:04:23 +0800 Subject: [PATCH 012/105] more --- tests/command_utils.py | 6 ++++++ tests/test_quick_start_glm4_9B.py | 7 +++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index e69de29bb2d..e4abb0b8174 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -0,0 +1,6 @@ +import subprocess + + +def exec_command(cmd: str): + print(f"EXEC: {cmd}", flush=True) + subprocess.run(cmd, shell=True, check=True) diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index 2ab74dab9aa..e656da9f695 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -1,8 +1,11 @@ -import command_utils +import command_utils as U def prepare(): - TODO + U.exec_command("mkdir -p /root/models /root/datasets") + U.exec_command("hf download zai-org/GLM-Z1-9B-0414 --local-dir /root/models/GLM-Z1-9B-0414") + U.exec_command("hf download --repo-type dataset zhuzilin/dapo-math-17k --local-dir /root/datasets/dapo-math-17k") + U.exec_command("hf download --repo-type dataset zhuzilin/aime-2024 --local-dir /root/datasets/aime-2024") def execute(): From a806cbbfc7bde7f3f7b30295f29fdcbe25d59a7d Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:05:30 +0800 Subject: [PATCH 013/105] more --- tests/command_utils.py | 10 ++++++++++ tests/test_quick_start_glm4_9B.py | 2 ++ 2 files changed, 12 insertions(+) diff --git a/tests/command_utils.py b/tests/command_utils.py index e4abb0b8174..c32f68c0095 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -4,3 +4,13 @@ def exec_command(cmd: str): print(f"EXEC: {cmd}", flush=True) subprocess.run(cmd, shell=True, check=True) + + +def convert_checkpoint(): + exec_command( + "source scripts/models/glm4-9B.sh && " + "PYTHONPATH=/root/Megatron-LM torchrun --nproc-per-node 8 tools/convert_hf_to_torch_dist.py " + "${MODEL_ARGS[@]} " + "--hf-checkpoint /root/models/GLM-Z1-9B-0414 " + "--save /root/GLM-Z1-9B-0414_torch_dist" + ) diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index e656da9f695..aaed09478a2 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -7,6 +7,8 @@ def prepare(): U.exec_command("hf download --repo-type dataset zhuzilin/dapo-math-17k --local-dir /root/datasets/dapo-math-17k") U.exec_command("hf download --repo-type dataset zhuzilin/aime-2024 --local-dir /root/datasets/aime-2024") + U.exec_command("hf download --repo-type dataset zhuzilin/aime-2024 --local-dir /root/datasets/aime-2024") + def execute(): TODO From 0fa6c515a432bf3bec965e7c92d0a39e089dcfba Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:06:35 +0800 Subject: [PATCH 014/105] more --- tests/command_utils.py | 8 ++++---- tests/test_quick_start_glm4_9B.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index c32f68c0095..90b4f733201 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -6,11 +6,11 @@ def exec_command(cmd: str): subprocess.run(cmd, shell=True, check=True) -def convert_checkpoint(): +def convert_checkpoint(model_name, model_type): exec_command( - "source scripts/models/glm4-9B.sh && " + f"source scripts/models/{model_type}.sh && " "PYTHONPATH=/root/Megatron-LM torchrun --nproc-per-node 8 tools/convert_hf_to_torch_dist.py " "${MODEL_ARGS[@]} " - "--hf-checkpoint /root/models/GLM-Z1-9B-0414 " - "--save /root/GLM-Z1-9B-0414_torch_dist" + f"--hf-checkpoint /root/models/{model_name} " + f"--save /root/{model_name}_torch_dist" ) diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index aaed09478a2..a9bbb589993 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -7,7 +7,7 @@ def prepare(): U.exec_command("hf download --repo-type dataset zhuzilin/dapo-math-17k --local-dir /root/datasets/dapo-math-17k") U.exec_command("hf download --repo-type dataset zhuzilin/aime-2024 --local-dir /root/datasets/aime-2024") - U.exec_command("hf download --repo-type dataset zhuzilin/aime-2024 --local-dir /root/datasets/aime-2024") + U.convert_checkpoint(model_name="GLM-Z1-9B-0414", model_type="glm4-9B") def execute(): From 21e6704309305d574c45f290752856db850e0b3d Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:09:00 +0800 Subject: [PATCH 015/105] more --- tests/command_utils.py | 14 +++++++++----- tests/test_quick_start_glm4_9B.py | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index 90b4f733201..dc97cb8463f 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -1,11 +1,6 @@ import subprocess -def exec_command(cmd: str): - print(f"EXEC: {cmd}", flush=True) - subprocess.run(cmd, shell=True, check=True) - - def convert_checkpoint(model_name, model_type): exec_command( f"source scripts/models/{model_type}.sh && " @@ -14,3 +9,12 @@ def convert_checkpoint(model_name, model_type): f"--hf-checkpoint /root/models/{model_name} " f"--save /root/{model_name}_torch_dist" ) + + +def launch_train(num_gpus: int, master_addr: str = "127.0.0.1"): + exec_command(f"ray start --head --node-ip-address {master_addr} --num-gpus {num_gpus} --disable-usage-stats") + + +def exec_command(cmd: str): + print(f"EXEC: {cmd}", flush=True) + subprocess.run(cmd, shell=True, check=True) diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index a9bbb589993..c64ccf18f56 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -11,7 +11,7 @@ def prepare(): def execute(): - TODO + U.ray_start() if __name__ == '__main__': From 65674f1c8adfea171e15443c1beb57826e2d9f4f Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:11:34 +0800 Subject: [PATCH 016/105] more --- tests/command_utils.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index dc97cb8463f..05b7319214c 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -11,8 +11,19 @@ def convert_checkpoint(model_name, model_type): ) -def launch_train(num_gpus: int, master_addr: str = "127.0.0.1"): +def ray_start_and_submit( + train_args: str, + num_gpus: int, + master_addr: str = "127.0.0.1", +): exec_command(f"ray start --head --node-ip-address {master_addr} --num-gpus {num_gpus} --disable-usage-stats") + exec_command( + # TODO should this 127.0.0.1 be `master_addr` instead + f'ray job submit --address="http://127.0.0.1:8265" ' + '--runtime-env-json="${RUNTIME_ENV_JSON}"' + '-- python3 train.py ' + f'{train_args}' + ) def exec_command(cmd: str): From 2786a8d35d291a52d7200d18c8cebf40e32b92ce Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:12:53 +0800 Subject: [PATCH 017/105] more --- tests/command_utils.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index 05b7319214c..67c2b782cd0 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -1,3 +1,4 @@ +import json import subprocess @@ -17,10 +18,20 @@ def ray_start_and_submit( master_addr: str = "127.0.0.1", ): exec_command(f"ray start --head --node-ip-address {master_addr} --num-gpus {num_gpus} --disable-usage-stats") + + runtime_env_json = json.dumps({ + "env_vars": { + "PYTHONPATH": "/root/Megatron-LM/", + "CUDA_DEVICE_MAX_CONNECTIONS": "1", + "NCCL_NVLS_ENABLE": has_nvlink, + "no_proxy": f"127.0.0.1,{master_addr}", + } + }) + exec_command( # TODO should this 127.0.0.1 be `master_addr` instead f'ray job submit --address="http://127.0.0.1:8265" ' - '--runtime-env-json="${RUNTIME_ENV_JSON}"' + f'--runtime-env-json="{runtime_env_json}"' '-- python3 train.py ' f'{train_args}' ) From 28fe6bb2fe7a367890f7a1f0b5ef23acf0bc7979 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:14:22 +0800 Subject: [PATCH 018/105] more --- tests/command_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/command_utils.py b/tests/command_utils.py index 67c2b782cd0..66e0b4229d8 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -15,6 +15,7 @@ def convert_checkpoint(model_name, model_type): def ray_start_and_submit( train_args: str, num_gpus: int, + model_type: str, master_addr: str = "127.0.0.1", ): exec_command(f"ray start --head --node-ip-address {master_addr} --num-gpus {num_gpus} --disable-usage-stats") @@ -30,9 +31,11 @@ def ray_start_and_submit( exec_command( # TODO should this 127.0.0.1 be `master_addr` instead + f'source "{script_dir}/../scripts/models/{model_type}.sh" && ' f'ray job submit --address="http://127.0.0.1:8265" ' f'--runtime-env-json="{runtime_env_json}"' '-- python3 train.py ' + "${MODEL_ARGS[@]} " f'{train_args}' ) From 7a949c586865a2f2e82dfdfd2e164e3eed3cb586 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:15:37 +0800 Subject: [PATCH 019/105] more --- tests/command_utils.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index 66e0b4229d8..119d048db5e 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -24,7 +24,7 @@ def ray_start_and_submit( "env_vars": { "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", - "NCCL_NVLS_ENABLE": has_nvlink, + "NCCL_NVLS_ENABLE": check_has_nvlink(), "no_proxy": f"127.0.0.1,{master_addr}", } }) @@ -40,6 +40,13 @@ def ray_start_and_submit( ) -def exec_command(cmd: str): +def check_has_nvlink(): + output = exec_command("nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l", capture_output=True) + return int(output) > 0 + + +def exec_command(cmd: str, capture_output: bool = False): print(f"EXEC: {cmd}", flush=True) - subprocess.run(cmd, shell=True, check=True) + result = subprocess.run(cmd, shell=True, check=True, capture_output=capture_output) + if capture_output: + return result.stdout From 2039b358432de9f130d2957fd4e4cbb9db2e01a5 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:17:14 +0800 Subject: [PATCH 020/105] more --- tests/command_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index 119d048db5e..f4e00e98ffb 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -1,5 +1,9 @@ import json +import os import subprocess +from pathlib import Path + +repo_base_dir = Path(os.path.abspath(__file__)).resolve().parents[1] def convert_checkpoint(model_name, model_type): @@ -31,7 +35,7 @@ def ray_start_and_submit( exec_command( # TODO should this 127.0.0.1 be `master_addr` instead - f'source "{script_dir}/../scripts/models/{model_type}.sh" && ' + f'source "{repo_base_dir}/../scripts/models/{model_type}.sh" && ' f'ray job submit --address="http://127.0.0.1:8265" ' f'--runtime-env-json="{runtime_env_json}"' '-- python3 train.py ' From cc7f9fbd6580dc09b8a246572f22f7beffd08f7a Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:17:36 +0800 Subject: [PATCH 021/105] fmt --- tests/command_utils.py | 20 +++++++++++--------- tests/test_quick_start_glm4_9B.py | 2 +- tests/test_qwen3_30B_A3B.py | 5 +---- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index f4e00e98ffb..5c4264b0529 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -24,23 +24,25 @@ def ray_start_and_submit( ): exec_command(f"ray start --head --node-ip-address {master_addr} --num-gpus {num_gpus} --disable-usage-stats") - runtime_env_json = json.dumps({ - "env_vars": { - "PYTHONPATH": "/root/Megatron-LM/", - "CUDA_DEVICE_MAX_CONNECTIONS": "1", - "NCCL_NVLS_ENABLE": check_has_nvlink(), - "no_proxy": f"127.0.0.1,{master_addr}", + runtime_env_json = json.dumps( + { + "env_vars": { + "PYTHONPATH": "/root/Megatron-LM/", + "CUDA_DEVICE_MAX_CONNECTIONS": "1", + "NCCL_NVLS_ENABLE": check_has_nvlink(), + "no_proxy": f"127.0.0.1,{master_addr}", + } } - }) + ) exec_command( # TODO should this 127.0.0.1 be `master_addr` instead f'source "{repo_base_dir}/../scripts/models/{model_type}.sh" && ' f'ray job submit --address="http://127.0.0.1:8265" ' f'--runtime-env-json="{runtime_env_json}"' - '-- python3 train.py ' + "-- python3 train.py " "${MODEL_ARGS[@]} " - f'{train_args}' + f"{train_args}" ) diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index c64ccf18f56..d4d60ec2f48 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -14,5 +14,5 @@ def execute(): U.ray_start() -if __name__ == '__main__': +if __name__ == "__main__": pass # TODO make the functions executable via typer diff --git a/tests/test_qwen3_30B_A3B.py b/tests/test_qwen3_30B_A3B.py index 2ab74dab9aa..6e2084eb386 100644 --- a/tests/test_qwen3_30B_A3B.py +++ b/tests/test_qwen3_30B_A3B.py @@ -1,6 +1,3 @@ -import command_utils - - def prepare(): TODO @@ -9,5 +6,5 @@ def execute(): TODO -if __name__ == '__main__': +if __name__ == "__main__": pass # TODO make the functions executable via typer From bf2358b959ae019f34663bbdecaefb4c2209dfc8 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:17:44 +0800 Subject: [PATCH 022/105] more --- tests/command_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index 5c4264b0529..ecd79649def 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -8,7 +8,7 @@ def convert_checkpoint(model_name, model_type): exec_command( - f"source scripts/models/{model_type}.sh && " + f"source {repo_base_dir}/scripts/models/{model_type}.sh && " "PYTHONPATH=/root/Megatron-LM torchrun --nproc-per-node 8 tools/convert_hf_to_torch_dist.py " "${MODEL_ARGS[@]} " f"--hf-checkpoint /root/models/{model_name} " @@ -37,7 +37,7 @@ def ray_start_and_submit( exec_command( # TODO should this 127.0.0.1 be `master_addr` instead - f'source "{repo_base_dir}/../scripts/models/{model_type}.sh" && ' + f'source "{repo_base_dir}/scripts/models/{model_type}.sh" && ' f'ray job submit --address="http://127.0.0.1:8265" ' f'--runtime-env-json="{runtime_env_json}"' "-- python3 train.py " From 13c3883f558458b43e552200a388c33e642ab4e4 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:19:03 +0800 Subject: [PATCH 023/105] more --- tests/command_utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index ecd79649def..0c0de0fe868 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -22,7 +22,10 @@ def ray_start_and_submit( model_type: str, master_addr: str = "127.0.0.1", ): - exec_command(f"ray start --head --node-ip-address {master_addr} --num-gpus {num_gpus} --disable-usage-stats") + exec_command( + f"export PYTHONBUFFERED=16 && " + f"ray start --head --node-ip-address {master_addr} --num-gpus {num_gpus} --disable-usage-stats" + ) runtime_env_json = json.dumps( { @@ -36,8 +39,9 @@ def ray_start_and_submit( ) exec_command( - # TODO should this 127.0.0.1 be `master_addr` instead + f"export PYTHONBUFFERED=16 && " f'source "{repo_base_dir}/scripts/models/{model_type}.sh" && ' + # TODO should this 127.0.0.1 be `master_addr` instead f'ray job submit --address="http://127.0.0.1:8265" ' f'--runtime-env-json="{runtime_env_json}"' "-- python3 train.py " From 442a242d14f449cf80227d87679dd613f632e05a Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:19:19 +0800 Subject: [PATCH 024/105] more --- tests/command_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/command_utils.py b/tests/command_utils.py index 0c0de0fe868..30f0d9115d9 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -23,6 +23,7 @@ def ray_start_and_submit( master_addr: str = "127.0.0.1", ): exec_command( + # will prevent ray from buffering stdout/stderr f"export PYTHONBUFFERED=16 && " f"ray start --head --node-ip-address {master_addr} --num-gpus {num_gpus} --disable-usage-stats" ) From e50f08a2e2aee1cef6775dd7a06040f72a9ba2b7 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:20:20 +0800 Subject: [PATCH 025/105] more --- tests/command_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index 30f0d9115d9..ee47e6bdabf 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -16,7 +16,7 @@ def convert_checkpoint(model_name, model_type): ) -def ray_start_and_submit( +def execute_train( train_args: str, num_gpus: int, model_type: str, From 3594b1009c8dd0bdf77260390cd78323ff76e2e7 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:23:40 +0800 Subject: [PATCH 026/105] more --- tests/test_quick_start_glm4_9B.py | 102 +++++++++++++++++++++++++++++- 1 file changed, 101 insertions(+), 1 deletion(-) diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index d4d60ec2f48..6b35af5c50a 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -11,7 +11,107 @@ def prepare(): def execute(): - U.ray_start() + ckpt_args = ( + "--hf-checkpoint /root/models/GLM-Z1-9B-0414/ " + "--ref-load /root/GLM-Z1-9B-0414_torch_dist " + "--fp8-format e4m3 " + "--fp8-recipe blockwise " + ) + + rollout_args = ( + "--prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl " + "--input-key prompt " + "--label-key label " + "--apply-chat-template " + "--rollout-shuffle " + "--rm-type deepscaler " + "--num-rollout 3 " + "--rollout-batch-size 8 " + "--n-samples-per-prompt 8 " + "--rollout-max-response-len 8192 " + "--rollout-temperature 0.8 " + "--global-batch-size 32 " + "--balance-data " + ) + + eval_args = ( + "--eval-interval 20 " + "--eval-prompt-data aime24 /root/datasets/aime-2024/aime-2024.jsonl " + "--n-samples-per-eval-prompt 1 " + "--eval-max-response-len 16384 " + "--eval-top-k 1 " + ) + + perf_args = ( + "--tensor-model-parallel-size 2 " + "--sequence-parallel " + "--pipeline-model-parallel-size 1 " + "--context-parallel-size 2 " + "--expert-model-parallel-size 1 " + "--expert-tensor-parallel-size 1 " + "--recompute-granularity full " + "--recompute-method uniform " + "--recompute-num-layers 1 " + "--use-dynamic-batch-size " + "--max-tokens-per-gpu 4608 " + ) + + grpo_args = ( + "--advantage-estimator grpo " + "--use-kl-loss " + "--kl-loss-coef 0.00 " + "--kl-loss-type low_var_kl " + "--entropy-coef 0.00 " + "--eps-clip 0.2 " + "--eps-clip-high 0.28 " + "--use-tis " + "--calculate-per-token-loss " + ) + + optimizer_args = ( + "--optimizer adam " + "--lr 1e-6 " + "--lr-decay-style constant " + "--weight-decay 0.1 " + "--adam-beta1 0.9 " + "--adam-beta2 0.98 " + ) + + sglang_args = ( + "--rollout-num-gpus-per-engine 2 " + "--use-slime-router " + ) + + misc_args = ( + # default dropout in megatron is 0.1 + "--attention-dropout 0.0 " + "--hidden-dropout 0.0 " + # should be good for model performance + "--accumulate-allreduce-grads-in-fp32 " + "--attention-softmax-in-fp32 " + # need to comment this when using model with MLA + "--attention-backend flash " + "--ci-test " + ) + + train_args = ( + f"{ckpt_args} " + f"{rollout_args} " + f"{optimizer_args} " + f"{grpo_args} " + f"{distributed_args} " + f"{wandb_args} " + f"{perf_args} " + f"{eval_args} " + f"{sglang_args} " + f"{misc_args} " + ) + + U.execute_train( + train_args=train_args, + num_gpus=2, + model_type="glm4-9B", + ) if __name__ == "__main__": From ed251c774fcf999e7c69adf415c5292423fc1f5c Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:25:21 +0800 Subject: [PATCH 027/105] more --- tests/command_utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/command_utils.py b/tests/command_utils.py index ee47e6bdabf..9f0bd8f51f9 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -22,6 +22,18 @@ def execute_train( model_type: str, master_addr: str = "127.0.0.1", ): + exec_command( + "pkill -9 sglang;" + "sleep 3;" + "ray stop --force;" + "pkill -9 ray;" + "pkill -9 python;" + "sleep 3;" + "pkill -9 ray;" + "pkill -9 python;" + "pkill -9 redis;" + ) + exec_command( # will prevent ray from buffering stdout/stderr f"export PYTHONBUFFERED=16 && " From 5f3958bfc47d5be4ff55f0c5a75d14c5dc72000b Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:25:48 +0800 Subject: [PATCH 028/105] more --- tests/command_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index 9f0bd8f51f9..7f1db01b57b 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -56,7 +56,7 @@ def execute_train( f'source "{repo_base_dir}/scripts/models/{model_type}.sh" && ' # TODO should this 127.0.0.1 be `master_addr` instead f'ray job submit --address="http://127.0.0.1:8265" ' - f'--runtime-env-json="{runtime_env_json}"' + f'--runtime-env-json="{runtime_env_json}" ' "-- python3 train.py " "${MODEL_ARGS[@]} " f"{train_args}" From 3819d468138dd926142f22da127a4d0f4100a2b9 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:26:44 +0800 Subject: [PATCH 029/105] more --- tests/test_quick_start_glm4_9B.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index 6b35af5c50a..a162fdec647 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -1,5 +1,7 @@ import command_utils as U +MODEL_NAME = "GLM-Z1-9B-0414" +MODEL_TYPE = "glm4-9B" def prepare(): U.exec_command("mkdir -p /root/models /root/datasets") @@ -7,13 +9,13 @@ def prepare(): U.exec_command("hf download --repo-type dataset zhuzilin/dapo-math-17k --local-dir /root/datasets/dapo-math-17k") U.exec_command("hf download --repo-type dataset zhuzilin/aime-2024 --local-dir /root/datasets/aime-2024") - U.convert_checkpoint(model_name="GLM-Z1-9B-0414", model_type="glm4-9B") + U.convert_checkpoint(model_name=MODEL_NAME, model_type=MODEL_TYPE) def execute(): ckpt_args = ( - "--hf-checkpoint /root/models/GLM-Z1-9B-0414/ " - "--ref-load /root/GLM-Z1-9B-0414_torch_dist " + f"--hf-checkpoint /root/models/{MODEL_NAME}/ " + f"--ref-load /root/{MODEL_TYPE}_torch_dist " "--fp8-format e4m3 " "--fp8-recipe blockwise " ) @@ -110,7 +112,7 @@ def execute(): U.execute_train( train_args=train_args, num_gpus=2, - model_type="glm4-9B", + model_type=MODEL_TYPE, ) From 458f3bd0261087da658a872d351ef793ec2fcdfc Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:27:57 +0800 Subject: [PATCH 030/105] more --- tests/test_qwen3_30B_A3B.py | 115 +++++++++++++++++++++++++++++++++++- 1 file changed, 113 insertions(+), 2 deletions(-) diff --git a/tests/test_qwen3_30B_A3B.py b/tests/test_qwen3_30B_A3B.py index 6e2084eb386..f0770fa1597 100644 --- a/tests/test_qwen3_30B_A3B.py +++ b/tests/test_qwen3_30B_A3B.py @@ -1,9 +1,120 @@ +import command_utils as U + +MODEL_NAME = "Qwen3-30B-A3B" +MODEL_TYPE = "qwen3-30B-A3B" + def prepare(): - TODO + U.exec_command("mkdir -p /root/models /root/datasets") + U.exec_command("hf download Qwen/Qwen3-30B-A3B --local-dir /root/models/Qwen3-30B-A3B") + U.exec_command("hf download Qwen/Qwen3-30B-A3B-FP8 --local-dir /root/models/Qwen3-30B-A3B-FP8") + U.exec_command("hf download --repo-type dataset zhuzilin/dapo-math-17k --local-dir /root/datasets/dapo-math-17k") + U.exec_command("hf download --repo-type dataset zhuzilin/aime-2024 --local-dir /root/datasets/aime-2024") + + U.convert_checkpoint(model_name=MODEL_NAME, model_type=MODEL_TYPE) def execute(): - TODO + ckpt_args = ( + f"--hf-checkpoint /root/models/{MODEL_NAME}-FP8/ " + f"--ref-load /root/{MODEL_TYPE}_torch_dist " + "--fp8-format e4m3 " + "--fp8-recipe blockwise " + ) + + rollout_args = ( + "--prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl " + "--input-key prompt " + "--label-key label " + "--apply-chat-template " + "--rollout-shuffle " + "--rm-type deepscaler " + "--num-rollout 3 " + "--rollout-batch-size 8 " + "--n-samples-per-prompt 8 " + "--rollout-max-response-len 8192 " + "--rollout-temperature 0.8 " + "--global-batch-size 32 " + "--balance-data " + ) + + eval_args = ( + "--eval-interval 20 " + "--eval-prompt-data aime24 /root/datasets/aime-2024/aime-2024.jsonl " + "--n-samples-per-eval-prompt 1 " + "--eval-max-response-len 16384 " + "--eval-top-k 1 " + ) + + perf_args = ( + "--tensor-model-parallel-size 2 " + "--sequence-parallel " + "--pipeline-model-parallel-size 1 " + "--context-parallel-size 2 " + "--expert-model-parallel-size 1 " + "--expert-tensor-parallel-size 1 " + "--recompute-granularity full " + "--recompute-method uniform " + "--recompute-num-layers 1 " + "--use-dynamic-batch-size " + "--max-tokens-per-gpu 4608 " + ) + + grpo_args = ( + "--advantage-estimator grpo " + "--use-kl-loss " + "--kl-loss-coef 0.00 " + "--kl-loss-type low_var_kl " + "--entropy-coef 0.00 " + "--eps-clip 0.2 " + "--eps-clip-high 0.28 " + "--use-tis " + "--calculate-per-token-loss " + ) + + optimizer_args = ( + "--optimizer adam " + "--lr 1e-6 " + "--lr-decay-style constant " + "--weight-decay 0.1 " + "--adam-beta1 0.9 " + "--adam-beta2 0.98 " + ) + + sglang_args = ( + "--rollout-num-gpus-per-engine 2 " + "--use-slime-router " + ) + + misc_args = ( + # default dropout in megatron is 0.1 + "--attention-dropout 0.0 " + "--hidden-dropout 0.0 " + # should be good for model performance + "--accumulate-allreduce-grads-in-fp32 " + "--attention-softmax-in-fp32 " + # need to comment this when using model with MLA + "--attention-backend flash " + "--ci-test " + ) + + train_args = ( + f"{ckpt_args} " + f"{rollout_args} " + f"{optimizer_args} " + f"{grpo_args} " + f"{distributed_args} " + f"{wandb_args} " + f"{perf_args} " + f"{eval_args} " + f"{sglang_args} " + f"{misc_args} " + ) + + U.execute_train( + train_args=train_args, + num_gpus=2, + model_type=MODEL_TYPE, + ) if __name__ == "__main__": From 43f6c15938d10e1f173a13cab2d54214e1d91141 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:29:28 +0800 Subject: [PATCH 031/105] more --- tests/test_quick_start_glm4_9B.py | 1 - tests/test_qwen3_30B_A3B.py | 34 ++++++++++++++++++------------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index a162fdec647..5c4dd3c606a 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -101,7 +101,6 @@ def execute(): f"{rollout_args} " f"{optimizer_args} " f"{grpo_args} " - f"{distributed_args} " f"{wandb_args} " f"{perf_args} " f"{eval_args} " diff --git a/tests/test_qwen3_30B_A3B.py b/tests/test_qwen3_30B_A3B.py index f0770fa1597..15772632a0a 100644 --- a/tests/test_qwen3_30B_A3B.py +++ b/tests/test_qwen3_30B_A3B.py @@ -15,10 +15,8 @@ def prepare(): def execute(): ckpt_args = ( - f"--hf-checkpoint /root/models/{MODEL_NAME}-FP8/ " - f"--ref-load /root/{MODEL_TYPE}_torch_dist " - "--fp8-format e4m3 " - "--fp8-recipe blockwise " + "--hf-checkpoint /root/models/Qwen3-30B-A3B-FP8 " + "--ref-load /root/Qwen3-30B-A3B_torch_dist " ) rollout_args = ( @@ -46,29 +44,29 @@ def execute(): ) perf_args = ( - "--tensor-model-parallel-size 2 " + "--tensor-model-parallel-size 4 " "--sequence-parallel " "--pipeline-model-parallel-size 1 " "--context-parallel-size 2 " - "--expert-model-parallel-size 1 " + "--expert-model-parallel-size 8 " "--expert-tensor-parallel-size 1 " "--recompute-granularity full " "--recompute-method uniform " "--recompute-num-layers 1 " "--use-dynamic-batch-size " - "--max-tokens-per-gpu 4608 " + "--max-tokens-per-gpu 16384 " ) grpo_args = ( - "--advantage-estimator grpo " + "--advantage-estimator gspo " "--use-kl-loss " "--kl-loss-coef 0.00 " "--kl-loss-type low_var_kl " + "--kl-coef 0.00 " "--entropy-coef 0.00 " - "--eps-clip 0.2 " - "--eps-clip-high 0.28 " + "--eps-clip 4e-4 " "--use-tis " - "--calculate-per-token-loss " + "--use-routing-replay " ) optimizer_args = ( @@ -78,11 +76,18 @@ def execute(): "--weight-decay 0.1 " "--adam-beta1 0.9 " "--adam-beta2 0.98 " + "--optimizer-cpu-offload " + "--overlap-cpu-optimizer-d2h-h2d " + "--use-precision-aware-optimizer " ) sglang_args = ( - "--rollout-num-gpus-per-engine 2 " - "--use-slime-router " + "--rollout-num-gpus-per-engine 8 " + "--sglang-mem-fraction-static 0.8 " + "--sglang-moe-a2a-backend deepep " + "--sglang-deepep-mode auto " + "--sglang-max-running-requests 512 " + "--sglang-disable-radix-cache " ) misc_args = ( @@ -94,6 +99,8 @@ def execute(): "--attention-softmax-in-fp32 " # need to comment this when using model with MLA "--attention-backend flash " + "--moe-token-dispatcher-type flex " + "--moe-enable-deepep " "--ci-test " ) @@ -102,7 +109,6 @@ def execute(): f"{rollout_args} " f"{optimizer_args} " f"{grpo_args} " - f"{distributed_args} " f"{wandb_args} " f"{perf_args} " f"{eval_args} " From 9dfe9837175f2a800b47f291060f983709b6d5b8 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:29:48 +0800 Subject: [PATCH 032/105] more --- tests/test_qwen3_30B_A3B.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_qwen3_30B_A3B.py b/tests/test_qwen3_30B_A3B.py index 15772632a0a..6041e333b7b 100644 --- a/tests/test_qwen3_30B_A3B.py +++ b/tests/test_qwen3_30B_A3B.py @@ -15,8 +15,8 @@ def prepare(): def execute(): ckpt_args = ( - "--hf-checkpoint /root/models/Qwen3-30B-A3B-FP8 " - "--ref-load /root/Qwen3-30B-A3B_torch_dist " + f"--hf-checkpoint /root/models/{MODEL_NAME}-FP8 " + f"--ref-load /root/{MODEL_NAME}_torch_dist " ) rollout_args = ( From 91e1efac1d89a7673902be907b8d96a73e3db32b Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:31:32 +0800 Subject: [PATCH 033/105] more --- tests/test_quick_start_glm4_9B.py | 6 +++++- tests/test_qwen3_30B_A3B.py | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index 5c4dd3c606a..5726ac3fd95 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -94,6 +94,10 @@ def execute(): # need to comment this when using model with MLA "--attention-backend flash " "--ci-test " + + "--actor-num-nodes 1 " + "--actor-num-gpus-per-node 4 " + "--rollout-num-gpus 4 " ) train_args = ( @@ -110,7 +114,7 @@ def execute(): U.execute_train( train_args=train_args, - num_gpus=2, + num_gpus=4, model_type=MODEL_TYPE, ) diff --git a/tests/test_qwen3_30B_A3B.py b/tests/test_qwen3_30B_A3B.py index 6041e333b7b..2a34215e5b0 100644 --- a/tests/test_qwen3_30B_A3B.py +++ b/tests/test_qwen3_30B_A3B.py @@ -82,7 +82,7 @@ def execute(): ) sglang_args = ( - "--rollout-num-gpus-per-engine 8 " + "--rollout-num-gpus-per-engine 4 " "--sglang-mem-fraction-static 0.8 " "--sglang-moe-a2a-backend deepep " "--sglang-deepep-mode auto " @@ -101,7 +101,11 @@ def execute(): "--attention-backend flash " "--moe-token-dispatcher-type flex " "--moe-enable-deepep " + "--ci-test " + "--actor-num-nodes 1 " + "--actor-num-gpus-per-node 4 " + "--colocate " ) train_args = ( @@ -118,7 +122,7 @@ def execute(): U.execute_train( train_args=train_args, - num_gpus=2, + num_gpus=4, model_type=MODEL_TYPE, ) From 8c7ae9dae3cd95468725883b29970986605059aa Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:32:24 +0800 Subject: [PATCH 034/105] more --- tests/command_utils.py | 4 ++++ tests/test_quick_start_glm4_9B.py | 2 +- tests/test_qwen3_30B_A3B.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index 7f1db01b57b..4251c6c4835 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -68,6 +68,10 @@ def check_has_nvlink(): return int(output) > 0 +def get_default_wandb_args(): + return TODO + + def exec_command(cmd: str, capture_output: bool = False): print(f"EXEC: {cmd}", flush=True) result = subprocess.run(cmd, shell=True, check=True, capture_output=capture_output) diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index 5726ac3fd95..1cecab4a043 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -105,7 +105,7 @@ def execute(): f"{rollout_args} " f"{optimizer_args} " f"{grpo_args} " - f"{wandb_args} " + f"{U.get_default_wandb_args()} " f"{perf_args} " f"{eval_args} " f"{sglang_args} " diff --git a/tests/test_qwen3_30B_A3B.py b/tests/test_qwen3_30B_A3B.py index 2a34215e5b0..3585fcaeafc 100644 --- a/tests/test_qwen3_30B_A3B.py +++ b/tests/test_qwen3_30B_A3B.py @@ -113,7 +113,7 @@ def execute(): f"{rollout_args} " f"{optimizer_args} " f"{grpo_args} " - f"{wandb_args} " + f"{U.get_default_wandb_args()} " f"{perf_args} " f"{eval_args} " f"{sglang_args} " From 412860effe2257aa096c1eaef403599192720a5a Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:34:40 +0800 Subject: [PATCH 035/105] more --- tests/command_utils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index 4251c6c4835..6ca7c5adfd4 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -69,7 +69,14 @@ def check_has_nvlink(): def get_default_wandb_args(): - return TODO + name = TODO + + return ( + "--use-wandb " + "--wandb-project miles-ci " + f"--wandb-group {name} " + f"--wandb-key {wandb_key} " + ) def exec_command(cmd: str, capture_output: bool = False): From 2c69b1d966831f522914866247f62322139973b2 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 15:59:15 +0800 Subject: [PATCH 036/105] more --- .github/workflows/pr-test.yml | 3 ++- tests/command_utils.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 8190b7ae38d..f4d9e5b0ab0 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -26,10 +26,11 @@ jobs: - test_quick_start_glm4_9B - test_qwen3_30B_A3B # TODO more - defaults: run: working-directory: ${{ github.workspace }} + env: + GITHUB_COMMIT_NAME: ${{ github.sha }}_${{ github.event.pull_request.number || 'non-pr' }} steps: - name: Checkout repository diff --git a/tests/command_utils.py b/tests/command_utils.py index 6ca7c5adfd4..2a3fc0d9347 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -69,7 +69,7 @@ def check_has_nvlink(): def get_default_wandb_args(): - name = TODO + name = os.environ.get("GITHUB_COMMIT_NAME") return ( "--use-wandb " From bfbd3626a8f7f612e409331653dc7f44c0c735dc Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 16:02:26 +0800 Subject: [PATCH 037/105] more --- tests/command_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index 2a3fc0d9347..76730c75160 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -1,5 +1,7 @@ +import datetime import json import os +import random import subprocess from pathlib import Path @@ -69,7 +71,9 @@ def check_has_nvlink(): def get_default_wandb_args(): - name = os.environ.get("GITHUB_COMMIT_NAME") + name = f"{datetime.datetime.now().strftime('%Y%m%d%H%M%S')}-{random.randint(0, 1000000000)}" + if (x := os.environ.get("GITHUB_COMMIT_NAME")) is not None: + name += f"_{x}" return ( "--use-wandb " From 6b6c2e844a82676cad3805aa978f9100548c2723 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 16:03:31 +0800 Subject: [PATCH 038/105] more --- tests/command_utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index 76730c75160..69866a95288 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -71,6 +71,11 @@ def check_has_nvlink(): def get_default_wandb_args(): + wandb_api_key = os.environ.get("WANDB_API_KEY") + if not wandb_api_key: + print("Skip wandb configuration since WANDB_API_KEY is not foudn") + return "" + name = f"{datetime.datetime.now().strftime('%Y%m%d%H%M%S')}-{random.randint(0, 1000000000)}" if (x := os.environ.get("GITHUB_COMMIT_NAME")) is not None: name += f"_{x}" @@ -79,7 +84,7 @@ def get_default_wandb_args(): "--use-wandb " "--wandb-project miles-ci " f"--wandb-group {name} " - f"--wandb-key {wandb_key} " + f"--wandb-key {wandb_api_key} " ) From fe1d63441ac059fedf6dd32d260032888abf6add Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 16:13:21 +0800 Subject: [PATCH 039/105] more --- .github/workflows/pr-test.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index f4d9e5b0ab0..4c8fe56bc76 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -12,20 +12,19 @@ concurrency: cancel-in-progress: true jobs: - primary_test: + e2e-test: # TODO may use run-ci label etc if: github.event.pull_request.draft == false - runs-on: self-hosted + runs-on: ${{ matrix.info.runs_on }} container: image: slimerl/slime:latest options: --gpus all --ipc=host --shm-size=16g --ulimit memlock=-1 --ulimit stack=67108864 --memory=0 --memory-swap=0 -v /data/models:/root/models -v /data/datasets:/root/datasets strategy: fail-fast: false matrix: - test_name: - - test_quick_start_glm4_9B - - test_qwen3_30B_A3B - # TODO more + info: + - {test_name: test_quick_start_glm4_9B, runs_on: 8-h100-runner} + - {test_name: test_qwen3_30B_A3B, runs_on: 8-h100-runner} defaults: run: working-directory: ${{ github.workspace }} From ec158ebc0cb6f9c27f1e456d0bb792893a6262d5 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 16:15:40 +0800 Subject: [PATCH 040/105] more --- .github/workflows/pr-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 4c8fe56bc76..6eb6147a2cc 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -25,6 +25,7 @@ jobs: info: - {test_name: test_quick_start_glm4_9B, runs_on: 8-h100-runner} - {test_name: test_qwen3_30B_A3B, runs_on: 8-h100-runner} + # TODO use deterministic kernel defaults: run: working-directory: ${{ github.workspace }} From c2843e4e364c2def5b1d54941a297eb5eb40c5f0 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 16:16:03 +0800 Subject: [PATCH 041/105] more --- tests/test-qwen2.5-0.5B-gsm8k-async.sh | 135 ++++++++++++++++++ tests/test-qwen2.5-0.5B-gsm8k.sh | 135 ++++++++++++++++++ tests/test_qwen3-0.6B_fsdp_colocated_2xGPU.sh | 116 +++++++++++++++ tests/test_qwen3-0.6B_fsdp_distributed.sh | 81 +++++++++++ 4 files changed, 467 insertions(+) create mode 100644 tests/test-qwen2.5-0.5B-gsm8k-async.sh create mode 100644 tests/test-qwen2.5-0.5B-gsm8k.sh create mode 100755 tests/test_qwen3-0.6B_fsdp_colocated_2xGPU.sh create mode 100644 tests/test_qwen3-0.6B_fsdp_distributed.sh diff --git a/tests/test-qwen2.5-0.5B-gsm8k-async.sh b/tests/test-qwen2.5-0.5B-gsm8k-async.sh new file mode 100644 index 00000000000..4f7fb4f47cd --- /dev/null +++ b/tests/test-qwen2.5-0.5B-gsm8k-async.sh @@ -0,0 +1,135 @@ +#!/bin/bash + +# for rerun the task +pkill -9 sglang +sleep 3 +ray stop --force +pkill -9 ray +pkill -9 python +sleep 3 +pkill -9 ray +pkill -9 python + +set -ex + + +huggingface-cli download --repo-type dataset zhuzilin/gsm8k --local-dir gsm8k + + +# will prevent ray from buffering stdout/stderr +export PYTHONBUFFERED=16 + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +source "${SCRIPT_DIR}/../scripts/models/qwen2.5-0.5B.sh" + +CKPT_ARGS=( + --hf-checkpoint /root/Qwen2.5-0.5B-Instruct/ + --ref-load /root/Qwen2.5-0.5B-Instruct_torch_dist/ +) + +ROLLOUT_ARGS=( + --prompt-data gsm8k/train.parquet + --input-key messages + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type math + --num-rollout 3000 + --rollout-batch-size 32 + --n-samples-per-prompt 8 + --rollout-max-response-len 1024 + --rollout-temperature 0.8 + --rollout-num-gpus 2 + + --over-sampling-batch-size 64 + --dynamic-sampling-filter-path slime.rollout.filter_hub.dynamic_sampling_filters.check_reward_nonzero_std + + --global-batch-size 256 +) + +EVAL_ARGS=( + --eval-interval 20 + --eval-prompt-data gsm8k gsm8k/test.parquet + --n-samples-per-eval-prompt 1 + --eval-max-response-len 1024 + --eval-top-k 1 +) + +PERF_ARGS=( + --tensor-model-parallel-size 1 + --sequence-parallel + --pipeline-model-parallel-size 1 + --context-parallel-size 1 + --expert-model-parallel-size 1 + --expert-tensor-parallel-size 1 + + # --micro-batch-size 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 9216 +) + +GRPO_ARGS=( + --advantage-estimator grpo + --use-kl-loss + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --entropy-coef 0.00 + --eps-clip 0.2 + --eps-clip-high 0.28 +) + +OPTIMIZER_ARGS=( + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 +) + +WANDB_ARGS=( + --use-wandb + --wandb-project slime-test + --wandb-group test-qwen2.5-0.5B-gsm8k +) + +SGLANG_ARGS=( + --rollout-num-gpus-per-engine 1 + --sglang-mem-fraction-static 0.7 +) + +MISC_ARGS=( + # default dropout in megatron is 0.1 + --attention-dropout 0.0 + --hidden-dropout 0.0 + # should be good for model performance + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + # need to comment this when using model with MLA + --attention-backend flash +) + +# launch the master node of ray in container +ray start --head --node-ip-address ${MASTER_ADDR} --num-gpus 4 --disable-usage-stats + +ray job submit --address="http://127.0.0.1:8265" \ + --runtime-env-json='{ + "env_vars": { + "PYTHONPATH": "/root/Megatron-LM", + "CUDA_DEVICE_MAX_CONNECTIONS": "1" + } + }' \ + -- python3 train_async.py \ + --actor-num-nodes 1 \ + --actor-num-gpus-per-node 2 \ + ${MODEL_ARGS[@]} \ + ${CKPT_ARGS[@]} \ + ${ROLLOUT_ARGS[@]} \ + ${OPTIMIZER_ARGS[@]} \ + ${GRPO_ARGS[@]} \ + ${DISTRIBUTED_ARGS[@]} \ + ${WANDB_ARGS[@]} \ + ${PERF_ARGS[@]} \ + ${EVAL_ARGS[@]} \ + ${SGLANG_ARGS[@]} \ + ${MISC_ARGS[@]} diff --git a/tests/test-qwen2.5-0.5B-gsm8k.sh b/tests/test-qwen2.5-0.5B-gsm8k.sh new file mode 100644 index 00000000000..954ce9648db --- /dev/null +++ b/tests/test-qwen2.5-0.5B-gsm8k.sh @@ -0,0 +1,135 @@ +#!/bin/bash + +# for rerun the task +pkill -9 sglang +sleep 3 +ray stop --force +pkill -9 ray +pkill -9 python +sleep 3 +pkill -9 ray +pkill -9 python + +set -ex + + +huggingface-cli download --repo-type dataset zhuzilin/gsm8k --local-dir gsm8k + + +# will prevent ray from buffering stdout/stderr +export PYTHONBUFFERED=16 + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +source "${SCRIPT_DIR}/../scripts/models/qwen2.5-0.5B.sh" + +CKPT_ARGS=( + --hf-checkpoint /root/Qwen2.5-0.5B-Instruct/ + --ref-load /root/Qwen2.5-0.5B-Instruct_torch_dist/ +) + +ROLLOUT_ARGS=( + --prompt-data gsm8k/train.parquet + --input-key messages + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type math + --num-rollout 3000 + --rollout-batch-size 32 + --n-samples-per-prompt 8 + --rollout-max-response-len 1024 + --rollout-temperature 0.8 + + --over-sampling-batch-size 64 + --dynamic-sampling-filter-path slime.rollout.filter_hub.dynamic_sampling_filters.check_reward_nonzero_std + + --global-batch-size 256 +) + +EVAL_ARGS=( + --eval-interval 20 + --eval-prompt-data gsm8k gsm8k/test.parquet + --n-samples-per-eval-prompt 1 + --eval-max-response-len 1024 + --eval-top-k 1 +) + +PERF_ARGS=( + --tensor-model-parallel-size 1 + --sequence-parallel + --pipeline-model-parallel-size 1 + --context-parallel-size 1 + --expert-model-parallel-size 1 + --expert-tensor-parallel-size 1 + + # --micro-batch-size 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 9216 +) + +GRPO_ARGS=( + --advantage-estimator grpo + --use-kl-loss + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --entropy-coef 0.00 + --eps-clip 0.2 + --eps-clip-high 0.28 +) + +OPTIMIZER_ARGS=( + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 +) + +WANDB_ARGS=( + --use-wandb + --wandb-project slime-test + --wandb-group test-qwen2.5-0.5B-gsm8k +) + +SGLANG_ARGS=( + --rollout-num-gpus-per-engine 1 + --sglang-mem-fraction-static 0.7 +) + +MISC_ARGS=( + # default dropout in megatron is 0.1 + --attention-dropout 0.0 + --hidden-dropout 0.0 + # should be good for model performance + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + # need to comment this when using model with MLA + --attention-backend flash +) + +# launch the master node of ray in container +ray start --head --node-ip-address ${MASTER_ADDR} --num-gpus 4 --disable-usage-stats + +ray job submit --address="http://127.0.0.1:8265" \ + --runtime-env-json='{ + "env_vars": { + "PYTHONPATH": "/root/Megatron-LM", + "CUDA_DEVICE_MAX_CONNECTIONS": "1" + } + }' \ + -- python3 train.py \ + --actor-num-nodes 1 \ + --actor-num-gpus-per-node 4 \ + --colocate \ + ${MODEL_ARGS[@]} \ + ${CKPT_ARGS[@]} \ + ${ROLLOUT_ARGS[@]} \ + ${OPTIMIZER_ARGS[@]} \ + ${GRPO_ARGS[@]} \ + ${DISTRIBUTED_ARGS[@]} \ + ${WANDB_ARGS[@]} \ + ${PERF_ARGS[@]} \ + ${EVAL_ARGS[@]} \ + ${SGLANG_ARGS[@]} \ + ${MISC_ARGS[@]} diff --git a/tests/test_qwen3-0.6B_fsdp_colocated_2xGPU.sh b/tests/test_qwen3-0.6B_fsdp_colocated_2xGPU.sh new file mode 100755 index 00000000000..30cc7946579 --- /dev/null +++ b/tests/test_qwen3-0.6B_fsdp_colocated_2xGPU.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +# FSDP Colocated 2GPU Training Script with Weights & Biases Support +# +# This script runs FSDP training with wandb logging enabled. +# +# Wandb Configuration: +# - Rank and world size are automatically detected from distributed context +# - Only rank 0 will log to wandb to avoid duplicate entries +# - Distributed coordination handled by torch.distributed in FSDP actors +# +# To customize wandb settings: +# 1. Uncomment and set --wandb-team if you're using a team/organization (optional for personal accounts) +# 2. Set your wandb API key if needed (or use 'wandb login' beforehand) +# 3. Modify project name and group as needed +# 4. Change wandb mode to 'offline' for local logging only +# 5. Uncomment --wandb-dir to specify custom log directory + +# for rerun the task +pkill -9 sglang +sleep 3 +ray stop --force +pkill -9 ray +pkill -9 python +sleep 3 +pkill -9 ray +pkill -9 python + +set -ex + +# will prevent ray from buffering stdout/stderr +export PYTHONBUFFERED=16 + +CKPT_ARGS=( + --hf-checkpoint /root/Qwen3-0.6B +) + +ROLLOUT_ARGS=( + --prompt-data /root/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 1000 + --rollout-batch-size 8 + --n-samples-per-prompt 8 + --rollout-max-response-len 4096 + --rollout-temperature 0.8 + + --global-batch-size 64 +) + +GRPO_ARGS=( + --advantage-estimator grpo + #--use-kl-loss + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --kl-coef 0.00 + --entropy-coef 0.00 + --eps-clip 0.2 + --eps-clip-high 0.28 +) + +OPTIMIZER_ARGS=( + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 +) + +SGLANG_ARGS=( + # Set equal to the number of GPUs per node for colocated mode + --rollout-num-gpus-per-engine 2 + --sglang-decode-log-interval 1000 +) + + +WANDB_ARGS=( + --use-wandb + --wandb-project "gsm8k_async_rl" + --wandb-group "fsdp-2gpu-colocated" + --wandb-mode "online" # Change to "offline" for local logging only +) + +FSDP_ARGS=( + # Set to true for FULL_STATE_DICT mode, false for SHARDED_STATE_DICT mode (default) + # --fsdp-full-params # Uncomment this line to enable full params mode + + # Set the bucket size for weight update + --update-weights-bucket-size 512 * 1024 * 1024 # 512MB +) + +# launch the master node of ray in container +ray start --head --node-ip-address 127.0.0.1 --num-gpus 2 --disable-usage-stats + +ray job submit --address="http://127.0.0.1:8265" \ + --runtime-env-json='{ + "env_vars": { + "no_proxy": "localhost,127.0.0.1,0.0.0.0,${MASTER_ADDR}", + "SLIME_BACKEND": "fsdp" + } + }' \ + -- python3 train.py \ + --actor-num-nodes 1 \ + --actor-num-gpus-per-node 2 \ + --colocate \ + ${CKPT_ARGS[@]} \ + ${ROLLOUT_ARGS[@]} \ + ${OPTIMIZER_ARGS[@]} \ + ${GRPO_ARGS[@]} \ + ${DISTRIBUTED_ARGS[@]} \ + ${SGLANG_ARGS[@]} \ + ${WANDB_ARGS[@]} \ No newline at end of file diff --git a/tests/test_qwen3-0.6B_fsdp_distributed.sh b/tests/test_qwen3-0.6B_fsdp_distributed.sh new file mode 100644 index 00000000000..fc1dea3816a --- /dev/null +++ b/tests/test_qwen3-0.6B_fsdp_distributed.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +# for rerun the task +pkill -9 sglang +sleep 3 +ray stop --force +pkill -9 ray +pkill -9 python +sleep 3 +pkill -9 ray +pkill -9 python + +set -ex + +# will prevent ray from buffering stdout/stderr +export PYTHONBUFFERED=16 + +CKPT_ARGS=( + --hf-checkpoint /root/Qwen3-0.6B +) + +ROLLOUT_ARGS=( + --prompt-data /root/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 3000 + --rollout-batch-size 16 + --n-samples-per-prompt 16 + --rollout-max-response-len 8192 + --rollout-temperature 0.8 + + --global-batch-size 128 +) + +GRPO_ARGS=( + --advantage-estimator grpo + #--use-kl-loss + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --kl-coef 0.00 + --entropy-coef 0.00 + --eps-clip 0.2 + --eps-clip-high 0.28 +) + +OPTIMIZER_ARGS=( + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 +) + +SGLANG_ARGS=( + --rollout-num-gpus-per-engine 1 +) + +# launch the master node of ray in container +ray start --head --node-ip-address 127.0.0.1 --num-gpus 4 --disable-usage-stats + +ray job submit --address="http://127.0.0.1:8265" \ + --runtime-env-json='{ + "env_vars": { + "no_proxy": "localhost,127.0.0.1,0.0.0.0,${MASTER_ADDR}", + "SLIME_BACKEND": "fsdp" + } + }' \ + -- python3 train.py \ + --actor-num-nodes 1 \ + --actor-num-gpus-per-node 4 \ + --colocate \ + ${CKPT_ARGS[@]} \ + ${ROLLOUT_ARGS[@]} \ + ${OPTIMIZER_ARGS[@]} \ + ${GRPO_ARGS[@]} \ + ${DISTRIBUTED_ARGS[@]} \ + ${SGLANG_ARGS[@]} From b187134baa8973269f903ea186de620f8759d624 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 16:26:57 +0800 Subject: [PATCH 042/105] Revert "more" This reverts commit c2843e4e364c2def5b1d54941a297eb5eb40c5f0. --- tests/test-qwen2.5-0.5B-gsm8k-async.sh | 135 ------------------ tests/test-qwen2.5-0.5B-gsm8k.sh | 135 ------------------ tests/test_qwen3-0.6B_fsdp_colocated_2xGPU.sh | 116 --------------- tests/test_qwen3-0.6B_fsdp_distributed.sh | 81 ----------- 4 files changed, 467 deletions(-) delete mode 100644 tests/test-qwen2.5-0.5B-gsm8k-async.sh delete mode 100644 tests/test-qwen2.5-0.5B-gsm8k.sh delete mode 100755 tests/test_qwen3-0.6B_fsdp_colocated_2xGPU.sh delete mode 100644 tests/test_qwen3-0.6B_fsdp_distributed.sh diff --git a/tests/test-qwen2.5-0.5B-gsm8k-async.sh b/tests/test-qwen2.5-0.5B-gsm8k-async.sh deleted file mode 100644 index 4f7fb4f47cd..00000000000 --- a/tests/test-qwen2.5-0.5B-gsm8k-async.sh +++ /dev/null @@ -1,135 +0,0 @@ -#!/bin/bash - -# for rerun the task -pkill -9 sglang -sleep 3 -ray stop --force -pkill -9 ray -pkill -9 python -sleep 3 -pkill -9 ray -pkill -9 python - -set -ex - - -huggingface-cli download --repo-type dataset zhuzilin/gsm8k --local-dir gsm8k - - -# will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 - -SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -source "${SCRIPT_DIR}/../scripts/models/qwen2.5-0.5B.sh" - -CKPT_ARGS=( - --hf-checkpoint /root/Qwen2.5-0.5B-Instruct/ - --ref-load /root/Qwen2.5-0.5B-Instruct_torch_dist/ -) - -ROLLOUT_ARGS=( - --prompt-data gsm8k/train.parquet - --input-key messages - --label-key label - --apply-chat-template - --rollout-shuffle - --rm-type math - --num-rollout 3000 - --rollout-batch-size 32 - --n-samples-per-prompt 8 - --rollout-max-response-len 1024 - --rollout-temperature 0.8 - --rollout-num-gpus 2 - - --over-sampling-batch-size 64 - --dynamic-sampling-filter-path slime.rollout.filter_hub.dynamic_sampling_filters.check_reward_nonzero_std - - --global-batch-size 256 -) - -EVAL_ARGS=( - --eval-interval 20 - --eval-prompt-data gsm8k gsm8k/test.parquet - --n-samples-per-eval-prompt 1 - --eval-max-response-len 1024 - --eval-top-k 1 -) - -PERF_ARGS=( - --tensor-model-parallel-size 1 - --sequence-parallel - --pipeline-model-parallel-size 1 - --context-parallel-size 1 - --expert-model-parallel-size 1 - --expert-tensor-parallel-size 1 - - # --micro-batch-size 1 - --use-dynamic-batch-size - --max-tokens-per-gpu 9216 -) - -GRPO_ARGS=( - --advantage-estimator grpo - --use-kl-loss - --kl-loss-coef 0.00 - --kl-loss-type low_var_kl - --entropy-coef 0.00 - --eps-clip 0.2 - --eps-clip-high 0.28 -) - -OPTIMIZER_ARGS=( - --optimizer adam - --lr 1e-6 - --lr-decay-style constant - --weight-decay 0.1 - --adam-beta1 0.9 - --adam-beta2 0.98 -) - -WANDB_ARGS=( - --use-wandb - --wandb-project slime-test - --wandb-group test-qwen2.5-0.5B-gsm8k -) - -SGLANG_ARGS=( - --rollout-num-gpus-per-engine 1 - --sglang-mem-fraction-static 0.7 -) - -MISC_ARGS=( - # default dropout in megatron is 0.1 - --attention-dropout 0.0 - --hidden-dropout 0.0 - # should be good for model performance - --accumulate-allreduce-grads-in-fp32 - --attention-softmax-in-fp32 - # need to comment this when using model with MLA - --attention-backend flash -) - -# launch the master node of ray in container -ray start --head --node-ip-address ${MASTER_ADDR} --num-gpus 4 --disable-usage-stats - -ray job submit --address="http://127.0.0.1:8265" \ - --runtime-env-json='{ - "env_vars": { - "PYTHONPATH": "/root/Megatron-LM", - "CUDA_DEVICE_MAX_CONNECTIONS": "1" - } - }' \ - -- python3 train_async.py \ - --actor-num-nodes 1 \ - --actor-num-gpus-per-node 2 \ - ${MODEL_ARGS[@]} \ - ${CKPT_ARGS[@]} \ - ${ROLLOUT_ARGS[@]} \ - ${OPTIMIZER_ARGS[@]} \ - ${GRPO_ARGS[@]} \ - ${DISTRIBUTED_ARGS[@]} \ - ${WANDB_ARGS[@]} \ - ${PERF_ARGS[@]} \ - ${EVAL_ARGS[@]} \ - ${SGLANG_ARGS[@]} \ - ${MISC_ARGS[@]} diff --git a/tests/test-qwen2.5-0.5B-gsm8k.sh b/tests/test-qwen2.5-0.5B-gsm8k.sh deleted file mode 100644 index 954ce9648db..00000000000 --- a/tests/test-qwen2.5-0.5B-gsm8k.sh +++ /dev/null @@ -1,135 +0,0 @@ -#!/bin/bash - -# for rerun the task -pkill -9 sglang -sleep 3 -ray stop --force -pkill -9 ray -pkill -9 python -sleep 3 -pkill -9 ray -pkill -9 python - -set -ex - - -huggingface-cli download --repo-type dataset zhuzilin/gsm8k --local-dir gsm8k - - -# will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 - -SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -source "${SCRIPT_DIR}/../scripts/models/qwen2.5-0.5B.sh" - -CKPT_ARGS=( - --hf-checkpoint /root/Qwen2.5-0.5B-Instruct/ - --ref-load /root/Qwen2.5-0.5B-Instruct_torch_dist/ -) - -ROLLOUT_ARGS=( - --prompt-data gsm8k/train.parquet - --input-key messages - --label-key label - --apply-chat-template - --rollout-shuffle - --rm-type math - --num-rollout 3000 - --rollout-batch-size 32 - --n-samples-per-prompt 8 - --rollout-max-response-len 1024 - --rollout-temperature 0.8 - - --over-sampling-batch-size 64 - --dynamic-sampling-filter-path slime.rollout.filter_hub.dynamic_sampling_filters.check_reward_nonzero_std - - --global-batch-size 256 -) - -EVAL_ARGS=( - --eval-interval 20 - --eval-prompt-data gsm8k gsm8k/test.parquet - --n-samples-per-eval-prompt 1 - --eval-max-response-len 1024 - --eval-top-k 1 -) - -PERF_ARGS=( - --tensor-model-parallel-size 1 - --sequence-parallel - --pipeline-model-parallel-size 1 - --context-parallel-size 1 - --expert-model-parallel-size 1 - --expert-tensor-parallel-size 1 - - # --micro-batch-size 1 - --use-dynamic-batch-size - --max-tokens-per-gpu 9216 -) - -GRPO_ARGS=( - --advantage-estimator grpo - --use-kl-loss - --kl-loss-coef 0.00 - --kl-loss-type low_var_kl - --entropy-coef 0.00 - --eps-clip 0.2 - --eps-clip-high 0.28 -) - -OPTIMIZER_ARGS=( - --optimizer adam - --lr 1e-6 - --lr-decay-style constant - --weight-decay 0.1 - --adam-beta1 0.9 - --adam-beta2 0.98 -) - -WANDB_ARGS=( - --use-wandb - --wandb-project slime-test - --wandb-group test-qwen2.5-0.5B-gsm8k -) - -SGLANG_ARGS=( - --rollout-num-gpus-per-engine 1 - --sglang-mem-fraction-static 0.7 -) - -MISC_ARGS=( - # default dropout in megatron is 0.1 - --attention-dropout 0.0 - --hidden-dropout 0.0 - # should be good for model performance - --accumulate-allreduce-grads-in-fp32 - --attention-softmax-in-fp32 - # need to comment this when using model with MLA - --attention-backend flash -) - -# launch the master node of ray in container -ray start --head --node-ip-address ${MASTER_ADDR} --num-gpus 4 --disable-usage-stats - -ray job submit --address="http://127.0.0.1:8265" \ - --runtime-env-json='{ - "env_vars": { - "PYTHONPATH": "/root/Megatron-LM", - "CUDA_DEVICE_MAX_CONNECTIONS": "1" - } - }' \ - -- python3 train.py \ - --actor-num-nodes 1 \ - --actor-num-gpus-per-node 4 \ - --colocate \ - ${MODEL_ARGS[@]} \ - ${CKPT_ARGS[@]} \ - ${ROLLOUT_ARGS[@]} \ - ${OPTIMIZER_ARGS[@]} \ - ${GRPO_ARGS[@]} \ - ${DISTRIBUTED_ARGS[@]} \ - ${WANDB_ARGS[@]} \ - ${PERF_ARGS[@]} \ - ${EVAL_ARGS[@]} \ - ${SGLANG_ARGS[@]} \ - ${MISC_ARGS[@]} diff --git a/tests/test_qwen3-0.6B_fsdp_colocated_2xGPU.sh b/tests/test_qwen3-0.6B_fsdp_colocated_2xGPU.sh deleted file mode 100755 index 30cc7946579..00000000000 --- a/tests/test_qwen3-0.6B_fsdp_colocated_2xGPU.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash - -# FSDP Colocated 2GPU Training Script with Weights & Biases Support -# -# This script runs FSDP training with wandb logging enabled. -# -# Wandb Configuration: -# - Rank and world size are automatically detected from distributed context -# - Only rank 0 will log to wandb to avoid duplicate entries -# - Distributed coordination handled by torch.distributed in FSDP actors -# -# To customize wandb settings: -# 1. Uncomment and set --wandb-team if you're using a team/organization (optional for personal accounts) -# 2. Set your wandb API key if needed (or use 'wandb login' beforehand) -# 3. Modify project name and group as needed -# 4. Change wandb mode to 'offline' for local logging only -# 5. Uncomment --wandb-dir to specify custom log directory - -# for rerun the task -pkill -9 sglang -sleep 3 -ray stop --force -pkill -9 ray -pkill -9 python -sleep 3 -pkill -9 ray -pkill -9 python - -set -ex - -# will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 - -CKPT_ARGS=( - --hf-checkpoint /root/Qwen3-0.6B -) - -ROLLOUT_ARGS=( - --prompt-data /root/dapo-math-17k/dapo-math-17k.jsonl - --input-key prompt - --label-key label - --apply-chat-template - --rollout-shuffle - --rm-type deepscaler - --num-rollout 1000 - --rollout-batch-size 8 - --n-samples-per-prompt 8 - --rollout-max-response-len 4096 - --rollout-temperature 0.8 - - --global-batch-size 64 -) - -GRPO_ARGS=( - --advantage-estimator grpo - #--use-kl-loss - --kl-loss-coef 0.00 - --kl-loss-type low_var_kl - --kl-coef 0.00 - --entropy-coef 0.00 - --eps-clip 0.2 - --eps-clip-high 0.28 -) - -OPTIMIZER_ARGS=( - --optimizer adam - --lr 1e-6 - --lr-decay-style constant - --weight-decay 0.1 - --adam-beta1 0.9 - --adam-beta2 0.98 -) - -SGLANG_ARGS=( - # Set equal to the number of GPUs per node for colocated mode - --rollout-num-gpus-per-engine 2 - --sglang-decode-log-interval 1000 -) - - -WANDB_ARGS=( - --use-wandb - --wandb-project "gsm8k_async_rl" - --wandb-group "fsdp-2gpu-colocated" - --wandb-mode "online" # Change to "offline" for local logging only -) - -FSDP_ARGS=( - # Set to true for FULL_STATE_DICT mode, false for SHARDED_STATE_DICT mode (default) - # --fsdp-full-params # Uncomment this line to enable full params mode - - # Set the bucket size for weight update - --update-weights-bucket-size 512 * 1024 * 1024 # 512MB -) - -# launch the master node of ray in container -ray start --head --node-ip-address 127.0.0.1 --num-gpus 2 --disable-usage-stats - -ray job submit --address="http://127.0.0.1:8265" \ - --runtime-env-json='{ - "env_vars": { - "no_proxy": "localhost,127.0.0.1,0.0.0.0,${MASTER_ADDR}", - "SLIME_BACKEND": "fsdp" - } - }' \ - -- python3 train.py \ - --actor-num-nodes 1 \ - --actor-num-gpus-per-node 2 \ - --colocate \ - ${CKPT_ARGS[@]} \ - ${ROLLOUT_ARGS[@]} \ - ${OPTIMIZER_ARGS[@]} \ - ${GRPO_ARGS[@]} \ - ${DISTRIBUTED_ARGS[@]} \ - ${SGLANG_ARGS[@]} \ - ${WANDB_ARGS[@]} \ No newline at end of file diff --git a/tests/test_qwen3-0.6B_fsdp_distributed.sh b/tests/test_qwen3-0.6B_fsdp_distributed.sh deleted file mode 100644 index fc1dea3816a..00000000000 --- a/tests/test_qwen3-0.6B_fsdp_distributed.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash - -# for rerun the task -pkill -9 sglang -sleep 3 -ray stop --force -pkill -9 ray -pkill -9 python -sleep 3 -pkill -9 ray -pkill -9 python - -set -ex - -# will prevent ray from buffering stdout/stderr -export PYTHONBUFFERED=16 - -CKPT_ARGS=( - --hf-checkpoint /root/Qwen3-0.6B -) - -ROLLOUT_ARGS=( - --prompt-data /root/dapo-math-17k/dapo-math-17k.jsonl - --input-key prompt - --label-key label - --apply-chat-template - --rollout-shuffle - --rm-type deepscaler - --num-rollout 3000 - --rollout-batch-size 16 - --n-samples-per-prompt 16 - --rollout-max-response-len 8192 - --rollout-temperature 0.8 - - --global-batch-size 128 -) - -GRPO_ARGS=( - --advantage-estimator grpo - #--use-kl-loss - --kl-loss-coef 0.00 - --kl-loss-type low_var_kl - --kl-coef 0.00 - --entropy-coef 0.00 - --eps-clip 0.2 - --eps-clip-high 0.28 -) - -OPTIMIZER_ARGS=( - --optimizer adam - --lr 1e-6 - --lr-decay-style constant - --weight-decay 0.1 - --adam-beta1 0.9 - --adam-beta2 0.98 -) - -SGLANG_ARGS=( - --rollout-num-gpus-per-engine 1 -) - -# launch the master node of ray in container -ray start --head --node-ip-address 127.0.0.1 --num-gpus 4 --disable-usage-stats - -ray job submit --address="http://127.0.0.1:8265" \ - --runtime-env-json='{ - "env_vars": { - "no_proxy": "localhost,127.0.0.1,0.0.0.0,${MASTER_ADDR}", - "SLIME_BACKEND": "fsdp" - } - }' \ - -- python3 train.py \ - --actor-num-nodes 1 \ - --actor-num-gpus-per-node 4 \ - --colocate \ - ${CKPT_ARGS[@]} \ - ${ROLLOUT_ARGS[@]} \ - ${OPTIMIZER_ARGS[@]} \ - ${GRPO_ARGS[@]} \ - ${DISTRIBUTED_ARGS[@]} \ - ${SGLANG_ARGS[@]} From 92d8a4db99557448e04ec895c6e3cbb9cd81981f Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 16:27:48 +0800 Subject: [PATCH 043/105] more --- tests/run_suite.py | 9 --------- tests/test_quick_start_glm4_9B.py | 4 +++- tests/test_qwen3_30B_A3B.py | 4 +++- 3 files changed, 6 insertions(+), 11 deletions(-) delete mode 100644 tests/run_suite.py diff --git a/tests/run_suite.py b/tests/run_suite.py deleted file mode 100644 index 09e34c344a2..00000000000 --- a/tests/run_suite.py +++ /dev/null @@ -1,9 +0,0 @@ -import argparse - -if __name__ == "__main__": - arg_parser = argparse.ArgumentParser() - arg_parser.add_argument("--test-name", type=str) - args = arg_parser.parse_args() - print(f"{args=}") - - TODO diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index 1cecab4a043..9f9be03b9b9 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -120,4 +120,6 @@ def execute(): if __name__ == "__main__": - pass # TODO make the functions executable via typer + # TODO also use typer + prepare() + execute() diff --git a/tests/test_qwen3_30B_A3B.py b/tests/test_qwen3_30B_A3B.py index 3585fcaeafc..ad84d7ef5ff 100644 --- a/tests/test_qwen3_30B_A3B.py +++ b/tests/test_qwen3_30B_A3B.py @@ -128,4 +128,6 @@ def execute(): if __name__ == "__main__": - pass # TODO make the functions executable via typer + # TODO also use typer + prepare() + execute() From 193eb92309932e7c9a2bfae8e9054d177e2ab745 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 16:28:07 +0800 Subject: [PATCH 044/105] more --- .github/workflows/pr-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 6eb6147a2cc..31f882d0978 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -23,8 +23,8 @@ jobs: fail-fast: false matrix: info: - - {test_name: test_quick_start_glm4_9B, runs_on: 8-h100-runner} - - {test_name: test_qwen3_30B_A3B, runs_on: 8-h100-runner} + - {test_file: test_quick_start_glm4_9B.py, runs_on: 8-h100-runner} + - {test_file: test_qwen3_30B_A3B.py, runs_on: 8-h100-runner} # TODO use deterministic kernel defaults: run: @@ -42,4 +42,4 @@ jobs: - name: Execute shell: bash - run: python tests/run_suite.py --test-name ${{ matrix.test_name }} + run: python tests/${{ matrix.test_file }} From 1eb82d8751463b688e6eaf8d4c022d594a8e6c04 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 16:29:19 +0800 Subject: [PATCH 045/105] more --- .github/workflows/pr-test.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 31f882d0978..c92f1514021 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -18,7 +18,16 @@ jobs: runs-on: ${{ matrix.info.runs_on }} container: image: slimerl/slime:latest - options: --gpus all --ipc=host --shm-size=16g --ulimit memlock=-1 --ulimit stack=67108864 --memory=0 --memory-swap=0 -v /data/models:/root/models -v /data/datasets:/root/datasets + options: > + --gpus all + --ipc=host + --shm-size=16g + --ulimit memlock=-1 + --ulimit stack=67108864 + --memory=0 + --memory-swap=0 + -v /data/miles_ci/models:/root/models + -v /data/miles_ci/datasets:/root/datasets strategy: fail-fast: false matrix: From 605d5176f15452bebe08a26eec5c1e57c345eaa1 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 16:59:09 +0800 Subject: [PATCH 046/105] more --- tests/ci/.env.example | 1 + tests/ci/.gitignore | 1 + tests/ci/docker-compose.yml | 19 +++++++++++++++++++ 3 files changed, 21 insertions(+) create mode 100644 tests/ci/.env.example create mode 100644 tests/ci/.gitignore create mode 100644 tests/ci/docker-compose.yml diff --git a/tests/ci/.env.example b/tests/ci/.env.example new file mode 100644 index 00000000000..8a173140442 --- /dev/null +++ b/tests/ci/.env.example @@ -0,0 +1 @@ +GITHUB_RUNNER_TOKEN= \ No newline at end of file diff --git a/tests/ci/.gitignore b/tests/ci/.gitignore new file mode 100644 index 00000000000..2eea525d885 --- /dev/null +++ b/tests/ci/.gitignore @@ -0,0 +1 @@ +.env \ No newline at end of file diff --git a/tests/ci/docker-compose.yml b/tests/ci/docker-compose.yml new file mode 100644 index 00000000000..9a6df06b23e --- /dev/null +++ b/tests/ci/docker-compose.yml @@ -0,0 +1,19 @@ +# NOTE: need to have a `.env` file mimicking `.env.example` + +version: "3.9" + +services: + runner: + image: ghcr.io/actions/actions-runner:2.328.0 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + deploy: + replicas: 4 + restart: always + # ref: https://github.com/actions/runner/issues/367#issuecomment-2007558723 + # ref: https://github.com/actions/runner + entrypoint: > + sh -c " + ./config.sh --url https://github.com/radixark/miles --token ${GITHUB_RUNNER_TOKEN} --unattended && + ./run.sh + " From 305d1261bc460a803a2a6722b565387fd6b81415 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 16:59:37 +0800 Subject: [PATCH 047/105] more --- tests/ci/{ => github_runner}/.env.example | 0 tests/ci/{ => github_runner}/.gitignore | 0 tests/ci/{ => github_runner}/docker-compose.yml | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename tests/ci/{ => github_runner}/.env.example (100%) rename tests/ci/{ => github_runner}/.gitignore (100%) rename tests/ci/{ => github_runner}/docker-compose.yml (100%) diff --git a/tests/ci/.env.example b/tests/ci/github_runner/.env.example similarity index 100% rename from tests/ci/.env.example rename to tests/ci/github_runner/.env.example diff --git a/tests/ci/.gitignore b/tests/ci/github_runner/.gitignore similarity index 100% rename from tests/ci/.gitignore rename to tests/ci/github_runner/.gitignore diff --git a/tests/ci/docker-compose.yml b/tests/ci/github_runner/docker-compose.yml similarity index 100% rename from tests/ci/docker-compose.yml rename to tests/ci/github_runner/docker-compose.yml From 907390ef74ac7293f2b0fc3583c60d08b4e7ab66 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 16:59:56 +0800 Subject: [PATCH 048/105] more --- tests/ci/github_runner/README.md | 0 tests/ci/github_runner/docker-compose.yml | 3 +-- 2 files changed, 1 insertion(+), 2 deletions(-) create mode 100644 tests/ci/github_runner/README.md diff --git a/tests/ci/github_runner/README.md b/tests/ci/github_runner/README.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ci/github_runner/docker-compose.yml b/tests/ci/github_runner/docker-compose.yml index 9a6df06b23e..a1bec844a52 100644 --- a/tests/ci/github_runner/docker-compose.yml +++ b/tests/ci/github_runner/docker-compose.yml @@ -1,5 +1,4 @@ -# NOTE: need to have a `.env` file mimicking `.env.example` - +# Please refer to `README.md` for how to setup this GitHub action runner version: "3.9" services: From ac0b62fbbb7542fdf99d6fe6680ab59d02479ef2 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 17:03:00 +0800 Subject: [PATCH 049/105] more --- tests/ci/github_runner/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/ci/github_runner/README.md b/tests/ci/github_runner/README.md index e69de29bb2d..2ab6ebbc9e9 100644 --- a/tests/ci/github_runner/README.md +++ b/tests/ci/github_runner/README.md @@ -0,0 +1,12 @@ +# How to setup runner + +### Step 1: Env + +Write `.env` mimicking `.env.example`. +The token can be found at https://github.com/radixark/miles/settings/actions/runners/new?arch=x64&os=linux. + +### Step 2: Run + +```shell +(cd /data/tom/primary_synced/miles/tests/ci/github_runner && docker compose up -d) +``` From 731e85909dba8587b019ad83f90a6309153ed223 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 17:04:03 +0800 Subject: [PATCH 050/105] more --- tests/ci/github_runner/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/ci/github_runner/README.md b/tests/ci/github_runner/README.md index 2ab6ebbc9e9..8797590d576 100644 --- a/tests/ci/github_runner/README.md +++ b/tests/ci/github_runner/README.md @@ -10,3 +10,9 @@ The token can be found at https://github.com/radixark/miles/settings/actions/run ```shell (cd /data/tom/primary_synced/miles/tests/ci/github_runner && docker compose up -d) ``` + +### Debugging + +```shell +(cd /data/tom/primary_synced/miles/tests/ci/github_runner && docker compose logs -f) +``` From 0d84d1536ae59c6fca6afcb4cd41af45c2389c3d Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 17:04:51 +0800 Subject: [PATCH 051/105] more --- .github/workflows/pr-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index c92f1514021..01c0fbc8a76 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -15,7 +15,7 @@ jobs: e2e-test: # TODO may use run-ci label etc if: github.event.pull_request.draft == false - runs-on: ${{ matrix.info.runs_on }} + runs-on: self-hosted container: image: slimerl/slime:latest options: > @@ -32,8 +32,8 @@ jobs: fail-fast: false matrix: info: - - {test_file: test_quick_start_glm4_9B.py, runs_on: 8-h100-runner} - - {test_file: test_qwen3_30B_A3B.py, runs_on: 8-h100-runner} + - {test_file: test_quick_start_glm4_9B.py} + - {test_file: test_qwen3_30B_A3B.py} # TODO use deterministic kernel defaults: run: From 579c49e17d78e4a0a99d3cec1927ee3c7aaef6dc Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 17:05:54 +0800 Subject: [PATCH 052/105] more --- tests/ci/github_runner/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ci/github_runner/docker-compose.yml b/tests/ci/github_runner/docker-compose.yml index a1bec844a52..25e40fb93e7 100644 --- a/tests/ci/github_runner/docker-compose.yml +++ b/tests/ci/github_runner/docker-compose.yml @@ -9,6 +9,7 @@ services: deploy: replicas: 4 restart: always + privileged: true # ref: https://github.com/actions/runner/issues/367#issuecomment-2007558723 # ref: https://github.com/actions/runner entrypoint: > From c99e4f5943480a1c7633af1669fc1f8e242800ec Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 17:07:52 +0800 Subject: [PATCH 053/105] more --- tests/ci/github_runner/README.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/ci/github_runner/README.md b/tests/ci/github_runner/README.md index 8797590d576..26e554fb6a9 100644 --- a/tests/ci/github_runner/README.md +++ b/tests/ci/github_runner/README.md @@ -8,11 +8,20 @@ The token can be found at https://github.com/radixark/miles/settings/actions/run ### Step 2: Run ```shell -(cd /data/tom/primary_synced/miles/tests/ci/github_runner && docker compose up -d) +cd /data/tom/primary_synced/miles/tests/ci/github_runner +docker compose up -d ``` ### Debugging +Logs + +```shell +docker compose logs -f +``` + +Exec + ```shell -(cd /data/tom/primary_synced/miles/tests/ci/github_runner && docker compose logs -f) +docker exec -it github_runner-runner-1 /bin/bash ``` From a957135fc9baa575ba5caa87ea3ef6a24168cf75 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 17:08:50 +0800 Subject: [PATCH 054/105] more --- tests/ci/github_runner/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/ci/github_runner/docker-compose.yml b/tests/ci/github_runner/docker-compose.yml index 25e40fb93e7..723de960813 100644 --- a/tests/ci/github_runner/docker-compose.yml +++ b/tests/ci/github_runner/docker-compose.yml @@ -10,6 +10,8 @@ services: replicas: 4 restart: always privileged: true + ipc: host + user: root # ref: https://github.com/actions/runner/issues/367#issuecomment-2007558723 # ref: https://github.com/actions/runner entrypoint: > From 93e21318674ce56eb8a46907d3f79e2e0f74e4ba Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 17:10:06 +0800 Subject: [PATCH 055/105] more --- tests/ci/github_runner/docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/ci/github_runner/docker-compose.yml b/tests/ci/github_runner/docker-compose.yml index 723de960813..381352509d8 100644 --- a/tests/ci/github_runner/docker-compose.yml +++ b/tests/ci/github_runner/docker-compose.yml @@ -9,8 +9,9 @@ services: deploy: replicas: 4 restart: always + environment: + RUNNER_ALLOW_RUNASROOT: "1" privileged: true - ipc: host user: root # ref: https://github.com/actions/runner/issues/367#issuecomment-2007558723 # ref: https://github.com/actions/runner From ed3a7f01358c5ab04baed5a770af97a5ab98db06 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 17:13:42 +0800 Subject: [PATCH 056/105] more --- tests/ci/github_runner/.env.example | 3 ++- tests/ci/github_runner/docker-compose.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/ci/github_runner/.env.example b/tests/ci/github_runner/.env.example index 8a173140442..114a1a80c45 100644 --- a/tests/ci/github_runner/.env.example +++ b/tests/ci/github_runner/.env.example @@ -1 +1,2 @@ -GITHUB_RUNNER_TOKEN= \ No newline at end of file +GITHUB_RUNNER_URL=https://github.com/radixark/miles +GITHUB_RUNNER_TOKEN=paste-your-token-here \ No newline at end of file diff --git a/tests/ci/github_runner/docker-compose.yml b/tests/ci/github_runner/docker-compose.yml index 381352509d8..45732d8563e 100644 --- a/tests/ci/github_runner/docker-compose.yml +++ b/tests/ci/github_runner/docker-compose.yml @@ -17,6 +17,6 @@ services: # ref: https://github.com/actions/runner entrypoint: > sh -c " - ./config.sh --url https://github.com/radixark/miles --token ${GITHUB_RUNNER_TOKEN} --unattended && + ./config.sh --url ${GITHUB_RUNNER_URL} --token ${GITHUB_RUNNER_TOKEN} --unattended && ./run.sh " From 2892f20ace5716e1a91094d9effee02307ca468e Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 17:14:54 +0800 Subject: [PATCH 057/105] more --- .github/workflows/pr-test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 01c0fbc8a76..3e676f459b6 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -42,6 +42,10 @@ jobs: GITHUB_COMMIT_NAME: ${{ github.sha }}_${{ github.event.pull_request.number || 'non-pr' }} steps: + - name: Information + shell: bash + run: whoami + - name: Checkout repository uses: actions/checkout@v4 From 1a1964bfa7543d98bfc20054232dd217f4297edb Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 17:44:44 +0800 Subject: [PATCH 058/105] more --- tests/ci/github_runner/docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/ci/github_runner/docker-compose.yml b/tests/ci/github_runner/docker-compose.yml index 45732d8563e..18b85f0cf95 100644 --- a/tests/ci/github_runner/docker-compose.yml +++ b/tests/ci/github_runner/docker-compose.yml @@ -6,6 +6,7 @@ services: image: ghcr.io/actions/actions-runner:2.328.0 volumes: - /var/run/docker.sock:/var/run/docker.sock + - /data/miles_ci:/data/miles_ci deploy: replicas: 4 restart: always @@ -17,6 +18,6 @@ services: # ref: https://github.com/actions/runner entrypoint: > sh -c " - ./config.sh --url ${GITHUB_RUNNER_URL} --token ${GITHUB_RUNNER_TOKEN} --unattended && + ./config.sh --url ${GITHUB_RUNNER_URL} --token ${GITHUB_RUNNER_TOKEN} --unattended --work /data/miles_ci && ./run.sh " From 5a3210d4d008faf82f3b13c1a667cbd4909b1da1 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 17:52:35 +0800 Subject: [PATCH 059/105] more --- tests/ci/github_runner/docker-compose.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/ci/github_runner/docker-compose.yml b/tests/ci/github_runner/docker-compose.yml index 18b85f0cf95..ac2b5ca8292 100644 --- a/tests/ci/github_runner/docker-compose.yml +++ b/tests/ci/github_runner/docker-compose.yml @@ -18,6 +18,7 @@ services: # ref: https://github.com/actions/runner entrypoint: > sh -c " - ./config.sh --url ${GITHUB_RUNNER_URL} --token ${GITHUB_RUNNER_TOKEN} --unattended --work /data/miles_ci && - ./run.sh + cd /data/miles_ci && + /home/runner/config.sh --url ${GITHUB_RUNNER_URL} --token ${GITHUB_RUNNER_TOKEN} --unattended && + /home/runner/run.sh " From 5828ce1f18bfc09b36e87fb80400f7659bda9ab8 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 17:54:06 +0800 Subject: [PATCH 060/105] more --- tests/ci/github_runner/README.md | 2 ++ tests/ci/github_runner/docker-compose.yml | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/ci/github_runner/README.md b/tests/ci/github_runner/README.md index 26e554fb6a9..8239878ef4c 100644 --- a/tests/ci/github_runner/README.md +++ b/tests/ci/github_runner/README.md @@ -5,6 +5,8 @@ Write `.env` mimicking `.env.example`. The token can be found at https://github.com/radixark/miles/settings/actions/runners/new?arch=x64&os=linux. +WARN: The `GITHUB_RUNNER_TOKEN` changes after a while. + ### Step 2: Run ```shell diff --git a/tests/ci/github_runner/docker-compose.yml b/tests/ci/github_runner/docker-compose.yml index ac2b5ca8292..18b85f0cf95 100644 --- a/tests/ci/github_runner/docker-compose.yml +++ b/tests/ci/github_runner/docker-compose.yml @@ -18,7 +18,6 @@ services: # ref: https://github.com/actions/runner entrypoint: > sh -c " - cd /data/miles_ci && - /home/runner/config.sh --url ${GITHUB_RUNNER_URL} --token ${GITHUB_RUNNER_TOKEN} --unattended && - /home/runner/run.sh + ./config.sh --url ${GITHUB_RUNNER_URL} --token ${GITHUB_RUNNER_TOKEN} --unattended --work /data/miles_ci && + ./run.sh " From f4f5744d5ae47cd3fbf26027b66ef9ee0abc8da9 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 17:57:05 +0800 Subject: [PATCH 061/105] more --- tests/ci/github_runner/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/github_runner/docker-compose.yml b/tests/ci/github_runner/docker-compose.yml index 18b85f0cf95..15e7f7b98b9 100644 --- a/tests/ci/github_runner/docker-compose.yml +++ b/tests/ci/github_runner/docker-compose.yml @@ -18,6 +18,6 @@ services: # ref: https://github.com/actions/runner entrypoint: > sh -c " - ./config.sh --url ${GITHUB_RUNNER_URL} --token ${GITHUB_RUNNER_TOKEN} --unattended --work /data/miles_ci && + ./config.sh --url ${GITHUB_RUNNER_URL} --token ${GITHUB_RUNNER_TOKEN} --unattended --work /data/miles_ci/runner_$(hostname) && ./run.sh " From 03468a3fb3063567ff8bfc71b788e1e288a1ce2d Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 18:02:26 +0800 Subject: [PATCH 062/105] more --- tests/ci/github_runner/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ci/github_runner/docker-compose.yml b/tests/ci/github_runner/docker-compose.yml index 15e7f7b98b9..aaff2f5b0e9 100644 --- a/tests/ci/github_runner/docker-compose.yml +++ b/tests/ci/github_runner/docker-compose.yml @@ -16,6 +16,7 @@ services: user: root # ref: https://github.com/actions/runner/issues/367#issuecomment-2007558723 # ref: https://github.com/actions/runner + # args ref: https://github.com/actions/runner/blob/68ff57dbc4c836d50f46602a8a53301fb9513eb4/src/Runner.Listener/CommandSettings.cs#L53 entrypoint: > sh -c " ./config.sh --url ${GITHUB_RUNNER_URL} --token ${GITHUB_RUNNER_TOKEN} --unattended --work /data/miles_ci/runner_$(hostname) && From df1e7216ca4d998c7b7f390135d1138462adfb62 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 18:03:00 +0800 Subject: [PATCH 063/105] more --- .github/workflows/pr-test.yml | 1 + tests/ci/github_runner/docker-compose.yml | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 3e676f459b6..117f31c150e 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -26,6 +26,7 @@ jobs: --ulimit stack=67108864 --memory=0 --memory-swap=0 + -v /data/miles_ci:/data/miles_ci -v /data/miles_ci/models:/root/models -v /data/miles_ci/datasets:/root/datasets strategy: diff --git a/tests/ci/github_runner/docker-compose.yml b/tests/ci/github_runner/docker-compose.yml index aaff2f5b0e9..657825aeb0b 100644 --- a/tests/ci/github_runner/docker-compose.yml +++ b/tests/ci/github_runner/docker-compose.yml @@ -19,6 +19,7 @@ services: # args ref: https://github.com/actions/runner/blob/68ff57dbc4c836d50f46602a8a53301fb9513eb4/src/Runner.Listener/CommandSettings.cs#L53 entrypoint: > sh -c " - ./config.sh --url ${GITHUB_RUNNER_URL} --token ${GITHUB_RUNNER_TOKEN} --unattended --work /data/miles_ci/runner_$(hostname) && - ./run.sh + cd /data/miles_ci + /home/runner/config.sh --url ${GITHUB_RUNNER_URL} --token ${GITHUB_RUNNER_TOKEN} --unattended --work /data/miles_ci/runner_$(hostname) && + /home/runner/run.sh " From 0478ee11901d6c606104d47b214ec1623fb1937c Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 18:04:19 +0800 Subject: [PATCH 064/105] more --- tests/ci/github_runner/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/github_runner/docker-compose.yml b/tests/ci/github_runner/docker-compose.yml index 657825aeb0b..0d81ccdd078 100644 --- a/tests/ci/github_runner/docker-compose.yml +++ b/tests/ci/github_runner/docker-compose.yml @@ -19,7 +19,7 @@ services: # args ref: https://github.com/actions/runner/blob/68ff57dbc4c836d50f46602a8a53301fb9513eb4/src/Runner.Listener/CommandSettings.cs#L53 entrypoint: > sh -c " - cd /data/miles_ci + cd /data/miles_ci && /home/runner/config.sh --url ${GITHUB_RUNNER_URL} --token ${GITHUB_RUNNER_TOKEN} --unattended --work /data/miles_ci/runner_$(hostname) && /home/runner/run.sh " From 3ef93fe7cf784ffb6525789b7a2d79abc36f14ae Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 18:07:41 +0800 Subject: [PATCH 065/105] more --- tests/ci/github_runner/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/ci/github_runner/docker-compose.yml b/tests/ci/github_runner/docker-compose.yml index 0d81ccdd078..f8a8c5fdc97 100644 --- a/tests/ci/github_runner/docker-compose.yml +++ b/tests/ci/github_runner/docker-compose.yml @@ -7,6 +7,8 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock - /data/miles_ci:/data/miles_ci + # it requires this folder + - /home/runner/externals:/home/runner/externals deploy: replicas: 4 restart: always From 340bde7450a67be487c54d47c676509c66800c6d Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 18:16:03 +0800 Subject: [PATCH 066/105] more --- tests/ci/github_runner/README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/ci/github_runner/README.md b/tests/ci/github_runner/README.md index 8239878ef4c..31b79ba5c90 100644 --- a/tests/ci/github_runner/README.md +++ b/tests/ci/github_runner/README.md @@ -7,7 +7,17 @@ The token can be found at https://github.com/radixark/miles/settings/actions/run WARN: The `GITHUB_RUNNER_TOKEN` changes after a while. -### Step 2: Run +### Step 2: Prepare `/home/runner/externals` + +```shell +docker run --rm -it --privileged --pid=host -v /:/host_root ubuntu /bin/bash -c 'rm -rf /host_root/home/runner/externals && mkdir -p /host_root/home/runner/externals && chmod -R 777 /host_root/home/runner/externals' +docker run -d --name temp-runner ghcr.io/actions/actions-runner:2.328.0 tail -f /dev/null +docker cp temp-runner:/home/runner/externals /home/runner/externals +docker rm -f temp-runner +ls -alh /home/runner/externals +``` + +### Step 3: Run ```shell cd /data/tom/primary_synced/miles/tests/ci/github_runner From 4db96ec2635a27a89018fcfad9a821f1606feb11 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 18:23:33 +0800 Subject: [PATCH 067/105] more --- tests/ci/github_runner/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/github_runner/README.md b/tests/ci/github_runner/README.md index 31b79ba5c90..bb1f7824a2f 100644 --- a/tests/ci/github_runner/README.md +++ b/tests/ci/github_runner/README.md @@ -12,7 +12,7 @@ WARN: The `GITHUB_RUNNER_TOKEN` changes after a while. ```shell docker run --rm -it --privileged --pid=host -v /:/host_root ubuntu /bin/bash -c 'rm -rf /host_root/home/runner/externals && mkdir -p /host_root/home/runner/externals && chmod -R 777 /host_root/home/runner/externals' docker run -d --name temp-runner ghcr.io/actions/actions-runner:2.328.0 tail -f /dev/null -docker cp temp-runner:/home/runner/externals /home/runner/externals +docker cp temp-runner:/home/runner/externals/. /home/runner/externals docker rm -f temp-runner ls -alh /home/runner/externals ``` From f04ca6c2e0e87ddef2fd8d9306ebbfd91dcb01cd Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 18:24:30 +0800 Subject: [PATCH 068/105] more --- .github/workflows/pr-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 117f31c150e..d9277ebf3ca 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -56,4 +56,4 @@ jobs: - name: Execute shell: bash - run: python tests/${{ matrix.test_file }} + run: python tests/${{ matrix.info.test_file }} From c82973cbaf4f8cb270cf2f38573b4fff9d080cf9 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 18:27:00 +0800 Subject: [PATCH 069/105] more --- tests/ci/github_runner/docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/ci/github_runner/docker-compose.yml b/tests/ci/github_runner/docker-compose.yml index f8a8c5fdc97..1d54d7dd17e 100644 --- a/tests/ci/github_runner/docker-compose.yml +++ b/tests/ci/github_runner/docker-compose.yml @@ -10,7 +10,8 @@ services: # it requires this folder - /home/runner/externals:/home/runner/externals deploy: - replicas: 4 + # TODO 4 runner + lock gpu when running + replicas: 1 restart: always environment: RUNNER_ALLOW_RUNASROOT: "1" From 04d687291a9eaf68ceea2041bf45fa8303f8fe26 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 18:28:22 +0800 Subject: [PATCH 070/105] more --- tests/command_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index 69866a95288..68b5cf6409b 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -10,7 +10,7 @@ def convert_checkpoint(model_name, model_type): exec_command( - f"source {repo_base_dir}/scripts/models/{model_type}.sh && " + f". {repo_base_dir}/scripts/models/{model_type}.sh && " "PYTHONPATH=/root/Megatron-LM torchrun --nproc-per-node 8 tools/convert_hf_to_torch_dist.py " "${MODEL_ARGS[@]} " f"--hf-checkpoint /root/models/{model_name} " @@ -55,7 +55,7 @@ def execute_train( exec_command( f"export PYTHONBUFFERED=16 && " - f'source "{repo_base_dir}/scripts/models/{model_type}.sh" && ' + f'. "{repo_base_dir}/scripts/models/{model_type}.sh" && ' # TODO should this 127.0.0.1 be `master_addr` instead f'ray job submit --address="http://127.0.0.1:8265" ' f'--runtime-env-json="{runtime_env_json}" ' From 0a1b8afadd9b5810e44d66a314036b9f324edcc7 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 18:29:47 +0800 Subject: [PATCH 071/105] fmt --- tests/command_utils.py | 7 +------ tests/test_quick_start_glm4_9B.py | 7 ++----- tests/test_qwen3_30B_A3B.py | 7 ++----- 3 files changed, 5 insertions(+), 16 deletions(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index 68b5cf6409b..13387f3da0b 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -80,12 +80,7 @@ def get_default_wandb_args(): if (x := os.environ.get("GITHUB_COMMIT_NAME")) is not None: name += f"_{x}" - return ( - "--use-wandb " - "--wandb-project miles-ci " - f"--wandb-group {name} " - f"--wandb-key {wandb_api_key} " - ) + return "--use-wandb " "--wandb-project miles-ci " f"--wandb-group {name} " f"--wandb-key {wandb_api_key} " def exec_command(cmd: str, capture_output: bool = False): diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index 9f9be03b9b9..35616b8b194 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -3,6 +3,7 @@ MODEL_NAME = "GLM-Z1-9B-0414" MODEL_TYPE = "glm4-9B" + def prepare(): U.exec_command("mkdir -p /root/models /root/datasets") U.exec_command("hf download zai-org/GLM-Z1-9B-0414 --local-dir /root/models/GLM-Z1-9B-0414") @@ -79,10 +80,7 @@ def execute(): "--adam-beta2 0.98 " ) - sglang_args = ( - "--rollout-num-gpus-per-engine 2 " - "--use-slime-router " - ) + sglang_args = "--rollout-num-gpus-per-engine 2 " "--use-slime-router " misc_args = ( # default dropout in megatron is 0.1 @@ -94,7 +92,6 @@ def execute(): # need to comment this when using model with MLA "--attention-backend flash " "--ci-test " - "--actor-num-nodes 1 " "--actor-num-gpus-per-node 4 " "--rollout-num-gpus 4 " diff --git a/tests/test_qwen3_30B_A3B.py b/tests/test_qwen3_30B_A3B.py index ad84d7ef5ff..a1484099db2 100644 --- a/tests/test_qwen3_30B_A3B.py +++ b/tests/test_qwen3_30B_A3B.py @@ -3,6 +3,7 @@ MODEL_NAME = "Qwen3-30B-A3B" MODEL_TYPE = "qwen3-30B-A3B" + def prepare(): U.exec_command("mkdir -p /root/models /root/datasets") U.exec_command("hf download Qwen/Qwen3-30B-A3B --local-dir /root/models/Qwen3-30B-A3B") @@ -14,10 +15,7 @@ def prepare(): def execute(): - ckpt_args = ( - f"--hf-checkpoint /root/models/{MODEL_NAME}-FP8 " - f"--ref-load /root/{MODEL_NAME}_torch_dist " - ) + ckpt_args = f"--hf-checkpoint /root/models/{MODEL_NAME}-FP8 " f"--ref-load /root/{MODEL_NAME}_torch_dist " rollout_args = ( "--prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl " @@ -101,7 +99,6 @@ def execute(): "--attention-backend flash " "--moe-token-dispatcher-type flex " "--moe-enable-deepep " - "--ci-test " "--actor-num-nodes 1 " "--actor-num-gpus-per-node 4 " From 2467f25dd2c2eac41ba9ac818b0a64ac4ca585e4 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 18:36:23 +0800 Subject: [PATCH 072/105] more --- tests/command_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index 13387f3da0b..3e8879a0865 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -10,7 +10,7 @@ def convert_checkpoint(model_name, model_type): exec_command( - f". {repo_base_dir}/scripts/models/{model_type}.sh && " + f"source {repo_base_dir}/scripts/models/{model_type}.sh && " "PYTHONPATH=/root/Megatron-LM torchrun --nproc-per-node 8 tools/convert_hf_to_torch_dist.py " "${MODEL_ARGS[@]} " f"--hf-checkpoint /root/models/{model_name} " @@ -55,7 +55,7 @@ def execute_train( exec_command( f"export PYTHONBUFFERED=16 && " - f'. "{repo_base_dir}/scripts/models/{model_type}.sh" && ' + f'source "{repo_base_dir}/scripts/models/{model_type}.sh" && ' # TODO should this 127.0.0.1 be `master_addr` instead f'ray job submit --address="http://127.0.0.1:8265" ' f'--runtime-env-json="{runtime_env_json}" ' @@ -85,6 +85,6 @@ def get_default_wandb_args(): def exec_command(cmd: str, capture_output: bool = False): print(f"EXEC: {cmd}", flush=True) - result = subprocess.run(cmd, shell=True, check=True, capture_output=capture_output) + result = subprocess.run(["bash", "-c", cmd], shell=False, check=True, capture_output=capture_output) if capture_output: return result.stdout From 2f48468203f445b2541163288659f0451b6054a9 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 18:37:20 +0800 Subject: [PATCH 073/105] more --- .github/workflows/pr-test.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index d9277ebf3ca..4f55b4efaab 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -43,10 +43,6 @@ jobs: GITHUB_COMMIT_NAME: ${{ github.sha }}_${{ github.event.pull_request.number || 'non-pr' }} steps: - - name: Information - shell: bash - run: whoami - - name: Checkout repository uses: actions/checkout@v4 From f8e18969993f7939ea9c4f9a10d7ec83b601f711 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 18:44:00 +0800 Subject: [PATCH 074/105] more --- tests/command_utils.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index 3e8879a0865..48389114b0e 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -9,12 +9,17 @@ def convert_checkpoint(model_name, model_type): + path_dst = f"/root/{model_name}_torch_dist" + if Path(path_dst).exists(): + print(f"convert_checkpoint skip {path_dst} since exists") + return + exec_command( f"source {repo_base_dir}/scripts/models/{model_type}.sh && " "PYTHONPATH=/root/Megatron-LM torchrun --nproc-per-node 8 tools/convert_hf_to_torch_dist.py " "${MODEL_ARGS[@]} " f"--hf-checkpoint /root/models/{model_name} " - f"--save /root/{model_name}_torch_dist" + f"--save {path_dst}" ) @@ -24,17 +29,18 @@ def execute_train( model_type: str, master_addr: str = "127.0.0.1", ): - exec_command( - "pkill -9 sglang;" - "sleep 3;" - "ray stop --force;" - "pkill -9 ray;" - "pkill -9 python;" - "sleep 3;" - "pkill -9 ray;" - "pkill -9 python;" - "pkill -9 redis;" - ) + # cannot be run in CI, o/w kill the parent script + # exec_command( + # "pkill -9 sglang;" + # "sleep 3;" + # "ray stop --force;" + # "pkill -9 ray;" + # "pkill -9 python;" + # "sleep 3;" + # "pkill -9 ray;" + # "pkill -9 python;" + # "pkill -9 redis;" + # ) exec_command( # will prevent ray from buffering stdout/stderr From 2264a71acdf4cbb920d22b84fbc6f2fd1e027571 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 18:46:29 +0800 Subject: [PATCH 075/105] more --- tests/command_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/command_utils.py b/tests/command_utils.py index 48389114b0e..336b234b093 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -9,6 +9,7 @@ def convert_checkpoint(model_name, model_type): + # TODO shall we make it in host-mapped folder and thus can cache it to speedup CI path_dst = f"/root/{model_name}_torch_dist" if Path(path_dst).exists(): print(f"convert_checkpoint skip {path_dst} since exists") From c66eadd8a22becafe6e6df753b2f0dc1c1550ac8 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 19:39:15 +0800 Subject: [PATCH 076/105] more --- tests/command_utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index 336b234b093..37ac89d2916 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -10,17 +10,17 @@ def convert_checkpoint(model_name, model_type): # TODO shall we make it in host-mapped folder and thus can cache it to speedup CI - path_dst = f"/root/{model_name}_torch_dist" - if Path(path_dst).exists(): - print(f"convert_checkpoint skip {path_dst} since exists") - return + # path_dst = f"/root/{model_name}_torch_dist" + # if Path(path_dst).exists(): + # print(f"convert_checkpoint skip {path_dst} since exists") + # return exec_command( f"source {repo_base_dir}/scripts/models/{model_type}.sh && " "PYTHONPATH=/root/Megatron-LM torchrun --nproc-per-node 8 tools/convert_hf_to_torch_dist.py " "${MODEL_ARGS[@]} " f"--hf-checkpoint /root/models/{model_name} " - f"--save {path_dst}" + f"--save /root/{model_name}_torch_dist" ) From 679b570258dd8097d75ba05372a7d60faae09ff4 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 19:39:43 +0800 Subject: [PATCH 077/105] more --- tests/command_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index 37ac89d2916..766956fe859 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -65,7 +65,7 @@ def execute_train( f'source "{repo_base_dir}/scripts/models/{model_type}.sh" && ' # TODO should this 127.0.0.1 be `master_addr` instead f'ray job submit --address="http://127.0.0.1:8265" ' - f'--runtime-env-json="{runtime_env_json}" ' + f"--runtime-env-json='{runtime_env_json}' " "-- python3 train.py " "${MODEL_ARGS[@]} " f"{train_args}" From fb1352af075444c63e6c70443bddb40a976180b5 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 19:40:40 +0800 Subject: [PATCH 078/105] Revert "more" This reverts commit c66eadd8a22becafe6e6df753b2f0dc1c1550ac8. --- tests/command_utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index 766956fe859..31c388e7fcb 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -10,17 +10,17 @@ def convert_checkpoint(model_name, model_type): # TODO shall we make it in host-mapped folder and thus can cache it to speedup CI - # path_dst = f"/root/{model_name}_torch_dist" - # if Path(path_dst).exists(): - # print(f"convert_checkpoint skip {path_dst} since exists") - # return + path_dst = f"/root/{model_name}_torch_dist" + if Path(path_dst).exists(): + print(f"convert_checkpoint skip {path_dst} since exists") + return exec_command( f"source {repo_base_dir}/scripts/models/{model_type}.sh && " "PYTHONPATH=/root/Megatron-LM torchrun --nproc-per-node 8 tools/convert_hf_to_torch_dist.py " "${MODEL_ARGS[@]} " f"--hf-checkpoint /root/models/{model_name} " - f"--save /root/{model_name}_torch_dist" + f"--save {path_dst}" ) From e3790f24a1203d5676dff19e5f9b50a625d3b52b Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 19:46:26 +0800 Subject: [PATCH 079/105] more --- tests/command_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index 31c388e7fcb..d1aff675e63 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -54,7 +54,7 @@ def execute_train( "env_vars": { "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", - "NCCL_NVLS_ENABLE": check_has_nvlink(), + "NCCL_NVLS_ENABLE": "true" if check_has_nvlink() else "false", "no_proxy": f"127.0.0.1,{master_addr}", } } From 8c85ddd529f9cfcae2c9333c709ae16b1348bbcc Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 19:46:43 +0800 Subject: [PATCH 080/105] more --- tests/command_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index d1aff675e63..e7576e59936 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -54,7 +54,7 @@ def execute_train( "env_vars": { "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", - "NCCL_NVLS_ENABLE": "true" if check_has_nvlink() else "false", + "NCCL_NVLS_ENABLE": str(int(check_has_nvlink())), "no_proxy": f"127.0.0.1,{master_addr}", } } From 48cde22cdf404858b42f06c9e86fb2af51f4d206 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 19:48:13 +0800 Subject: [PATCH 081/105] more --- tests/command_utils.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index e7576e59936..86fe2b5a4b4 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -30,18 +30,18 @@ def execute_train( model_type: str, master_addr: str = "127.0.0.1", ): - # cannot be run in CI, o/w kill the parent script - # exec_command( - # "pkill -9 sglang;" - # "sleep 3;" - # "ray stop --force;" - # "pkill -9 ray;" - # "pkill -9 python;" - # "sleep 3;" - # "pkill -9 ray;" - # "pkill -9 python;" - # "pkill -9 redis;" - # ) + exec_command( + "pkill -9 sglang;" + "sleep 3;" + "ray stop --force;" + "pkill -9 ray;" + # cannot be run in CI, o/w kill the parent script + # "pkill -9 python;" + "sleep 3;" + "pkill -9 ray;" + # "pkill -9 python;" + "pkill -9 redis;" + ) exec_command( # will prevent ray from buffering stdout/stderr From c0a92d0735437d03581e722c60df08160cc9b3d6 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 19:48:36 +0800 Subject: [PATCH 082/105] more --- tests/command_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/command_utils.py b/tests/command_utils.py index 86fe2b5a4b4..d2edc738497 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -36,10 +36,13 @@ def execute_train( "ray stop --force;" "pkill -9 ray;" # cannot be run in CI, o/w kill the parent script + # TODO: do we really need this kill? (or can we instead kill miles) # "pkill -9 python;" + "pkill -9 miles;" "sleep 3;" "pkill -9 ray;" # "pkill -9 python;" + "pkill -9 miles;" "pkill -9 redis;" ) From 065690bd20af3fd1b843bd3594acbc45c75f2025 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 19:48:58 +0800 Subject: [PATCH 083/105] more --- tests/command_utils.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index d2edc738497..8990da6d270 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -31,19 +31,19 @@ def execute_train( master_addr: str = "127.0.0.1", ): exec_command( - "pkill -9 sglang;" - "sleep 3;" - "ray stop --force;" - "pkill -9 ray;" + "pkill -9 sglang; " + "sleep 3; " + "ray stop --force; " + "pkill -9 ray; " # cannot be run in CI, o/w kill the parent script # TODO: do we really need this kill? (or can we instead kill miles) - # "pkill -9 python;" - "pkill -9 miles;" - "sleep 3;" - "pkill -9 ray;" - # "pkill -9 python;" - "pkill -9 miles;" - "pkill -9 redis;" + # "pkill -9 python; " + "pkill -9 miles; " + "sleep 3; " + "pkill -9 ray; " + # "pkill -9 python; " + "pkill -9 miles; " + "pkill -9 redis; " ) exec_command( From e9c761a7c5dfc36c89c67c789ad85bf63c4bf9cc Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 19:49:12 +0800 Subject: [PATCH 084/105] typo --- tests/command_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index 8990da6d270..d1d589e03c0 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -83,7 +83,7 @@ def check_has_nvlink(): def get_default_wandb_args(): wandb_api_key = os.environ.get("WANDB_API_KEY") if not wandb_api_key: - print("Skip wandb configuration since WANDB_API_KEY is not foudn") + print("Skip wandb configuration since WANDB_API_KEY is not found") return "" name = f"{datetime.datetime.now().strftime('%Y%m%d%H%M%S')}-{random.randint(0, 1000000000)}" From 79a632ec487bd697e9f98a2b958033928f7c5a46 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 19:49:35 +0800 Subject: [PATCH 085/105] more --- tests/command_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/command_utils.py b/tests/command_utils.py index d1d589e03c0..42b9dfeb234 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -44,6 +44,7 @@ def execute_train( # "pkill -9 python; " "pkill -9 miles; " "pkill -9 redis; " + "true; " ) exec_command( From 678ea84460fd1495b9a8814eb3871b6ae6d3a0e2 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 19:50:34 +0800 Subject: [PATCH 086/105] more --- tests/test_quick_start_glm4_9B.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index 35616b8b194..34c45243f48 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -80,7 +80,7 @@ def execute(): "--adam-beta2 0.98 " ) - sglang_args = "--rollout-num-gpus-per-engine 2 " "--use-slime-router " + sglang_args = "--rollout-num-gpus-per-engine 2 " "--use-miles-router " misc_args = ( # default dropout in megatron is 0.1 From da6ad85f55ba87c893fe035abf8696b05b467f78 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 19:52:37 +0800 Subject: [PATCH 087/105] more --- tests/ci/{github_runner => }/README.md | 8 +++++++- tests/ci/github_runner/docker-compose.yml | 1 + tests/test_quick_start_glm4_9B.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) rename tests/ci/{github_runner => }/README.md (93%) diff --git a/tests/ci/github_runner/README.md b/tests/ci/README.md similarity index 93% rename from tests/ci/github_runner/README.md rename to tests/ci/README.md index bb1f7824a2f..73c7fc10c88 100644 --- a/tests/ci/github_runner/README.md +++ b/tests/ci/README.md @@ -1,4 +1,10 @@ -# How to setup runner +# Doc about CI + +## Configure secrets + + + +## Setup new GitHub runners ### Step 1: Env diff --git a/tests/ci/github_runner/docker-compose.yml b/tests/ci/github_runner/docker-compose.yml index 1d54d7dd17e..abe2c64bc90 100644 --- a/tests/ci/github_runner/docker-compose.yml +++ b/tests/ci/github_runner/docker-compose.yml @@ -20,6 +20,7 @@ services: # ref: https://github.com/actions/runner/issues/367#issuecomment-2007558723 # ref: https://github.com/actions/runner # args ref: https://github.com/actions/runner/blob/68ff57dbc4c836d50f46602a8a53301fb9513eb4/src/Runner.Listener/CommandSettings.cs#L53 + # TODO seems we should not run config.sh repeatedly entrypoint: > sh -c " cd /data/miles_ci && diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index 34c45243f48..abfb0dca0e2 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -16,7 +16,7 @@ def prepare(): def execute(): ckpt_args = ( f"--hf-checkpoint /root/models/{MODEL_NAME}/ " - f"--ref-load /root/{MODEL_TYPE}_torch_dist " + f"--ref-load /root/{MODEL_NAME}_torch_dist " "--fp8-format e4m3 " "--fp8-recipe blockwise " ) From db790151c8f9fd6d3d2c2fefa25d271529349fcc Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 19:54:35 +0800 Subject: [PATCH 088/105] more --- tests/test_quick_start_glm4_9B.py | 2 +- tests/test_qwen3_30B_A3B.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index abfb0dca0e2..a024525fecf 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -111,7 +111,7 @@ def execute(): U.execute_train( train_args=train_args, - num_gpus=4, + num_gpus=8, model_type=MODEL_TYPE, ) diff --git a/tests/test_qwen3_30B_A3B.py b/tests/test_qwen3_30B_A3B.py index a1484099db2..4bf808294c5 100644 --- a/tests/test_qwen3_30B_A3B.py +++ b/tests/test_qwen3_30B_A3B.py @@ -119,7 +119,7 @@ def execute(): U.execute_train( train_args=train_args, - num_gpus=4, + num_gpus=8, model_type=MODEL_TYPE, ) From 2f1234ce0c6a32c6240a61a6fb62e49bf18d5b9e Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 19:54:56 +0800 Subject: [PATCH 089/105] more --- tests/test_qwen3_30B_A3B.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_qwen3_30B_A3B.py b/tests/test_qwen3_30B_A3B.py index 4bf808294c5..2429a639ac7 100644 --- a/tests/test_qwen3_30B_A3B.py +++ b/tests/test_qwen3_30B_A3B.py @@ -80,7 +80,7 @@ def execute(): ) sglang_args = ( - "--rollout-num-gpus-per-engine 4 " + "--rollout-num-gpus-per-engine 8 " "--sglang-mem-fraction-static 0.8 " "--sglang-moe-a2a-backend deepep " "--sglang-deepep-mode auto " @@ -101,7 +101,7 @@ def execute(): "--moe-enable-deepep " "--ci-test " "--actor-num-nodes 1 " - "--actor-num-gpus-per-node 4 " + "--actor-num-gpus-per-node 8 " "--colocate " ) From 3a35c50ba3b801252fe86ee6264bbd01d753534f Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 19:56:12 +0800 Subject: [PATCH 090/105] more --- tests/command_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index 42b9dfeb234..eada1eb170a 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -82,8 +82,7 @@ def check_has_nvlink(): def get_default_wandb_args(): - wandb_api_key = os.environ.get("WANDB_API_KEY") - if not wandb_api_key: + if not os.environ.get("WANDB_API_KEY"): print("Skip wandb configuration since WANDB_API_KEY is not found") return "" @@ -91,7 +90,8 @@ def get_default_wandb_args(): if (x := os.environ.get("GITHUB_COMMIT_NAME")) is not None: name += f"_{x}" - return "--use-wandb " "--wandb-project miles-ci " f"--wandb-group {name} " f"--wandb-key {wandb_api_key} " + # do not put wandb_api_key value here to avoid leaking to logs explicitly + return "--use-wandb " "--wandb-project miles-ci " f"--wandb-group {name} " f"--wandb-key ${{WANDB_API_KEY}} " def exec_command(cmd: str, capture_output: bool = False): From 2f7ccd81c6fcd2a771858e8bc54798de7b0af5d1 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 19:57:44 +0800 Subject: [PATCH 091/105] more --- tests/ci/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/ci/README.md b/tests/ci/README.md index 73c7fc10c88..d6941e7b55e 100644 --- a/tests/ci/README.md +++ b/tests/ci/README.md @@ -1,8 +1,10 @@ # Doc about CI -## Configure secrets +## Configure GitHub secrets +https://github.com/radixark/miles/settings/secrets/actions +* `WANDB_API_KEY`: get from https://wandb.ai/authorize ## Setup new GitHub runners From ddcfc21f4b563bb7879afb151b8551164e6a93f7 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 19:59:51 +0800 Subject: [PATCH 092/105] more --- .github/workflows/pr-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 4f55b4efaab..f1c0dc7e6b8 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -41,6 +41,7 @@ jobs: working-directory: ${{ github.workspace }} env: GITHUB_COMMIT_NAME: ${{ github.sha }}_${{ github.event.pull_request.number || 'non-pr' }} + WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }} steps: - name: Checkout repository From 62c8db2499a7e185ae2eaa1427879fe305176d67 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 20:14:45 +0800 Subject: [PATCH 093/105] more --- tests/test_quick_start_glm4_9B.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index a024525fecf..1955fc1d793 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -1,8 +1,11 @@ +import os + import command_utils as U MODEL_NAME = "GLM-Z1-9B-0414" MODEL_TYPE = "glm4-9B" +ENABLE_FAST_CHECK = bool(int(os.environ.get("MILES_TEST_ENABLE_FAST_CHECK", "0"))) def prepare(): U.exec_command("mkdir -p /root/models /root/datasets") @@ -31,14 +34,14 @@ def execute(): "--num-rollout 3 " "--rollout-batch-size 8 " "--n-samples-per-prompt 8 " - "--rollout-max-response-len 8192 " + f"--rollout-max-response-len {128 if ENABLE_FAST_CHECK else 8192} " "--rollout-temperature 0.8 " "--global-batch-size 32 " "--balance-data " ) eval_args = ( - "--eval-interval 20 " + f"{'' if ENABLE_FAST_CHECK else '--eval-interval 20 '}" "--eval-prompt-data aime24 /root/datasets/aime-2024/aime-2024.jsonl " "--n-samples-per-eval-prompt 1 " "--eval-max-response-len 16384 " From 0b8c86ed9c5a984ec7d9a570931dffbc5e73fb97 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 20:17:14 +0800 Subject: [PATCH 094/105] more --- tests/command_utils.py | 10 ++++++---- tests/test_quick_start_glm4_9B.py | 2 +- tests/test_qwen3_30B_A3B.py | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index eada1eb170a..b85cca43087 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -81,17 +81,19 @@ def check_has_nvlink(): return int(output) > 0 -def get_default_wandb_args(): +def get_default_wandb_args(test_file: str): if not os.environ.get("WANDB_API_KEY"): print("Skip wandb configuration since WANDB_API_KEY is not found") return "" - name = f"{datetime.datetime.now().strftime('%Y%m%d%H%M%S')}-{random.randint(0, 1000000000)}" + test_name = Path(test_file).stem + + run_name = f"{datetime.datetime.now().strftime('%Y%m%d%H%M%S')}-{random.randint(0, 1000000000)}" if (x := os.environ.get("GITHUB_COMMIT_NAME")) is not None: - name += f"_{x}" + run_name += f"_{x}" # do not put wandb_api_key value here to avoid leaking to logs explicitly - return "--use-wandb " "--wandb-project miles-ci " f"--wandb-group {name} " f"--wandb-key ${{WANDB_API_KEY}} " + return "--use-wandb " f"--wandb-project miles-ci-{test_name} " f"--wandb-group {run_name} " f"--wandb-key ${{WANDB_API_KEY}} " def exec_command(cmd: str, capture_output: bool = False): diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index 1955fc1d793..c90e4b48e46 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -105,7 +105,7 @@ def execute(): f"{rollout_args} " f"{optimizer_args} " f"{grpo_args} " - f"{U.get_default_wandb_args()} " + f"{U.get_default_wandb_args(__file__)} " f"{perf_args} " f"{eval_args} " f"{sglang_args} " diff --git a/tests/test_qwen3_30B_A3B.py b/tests/test_qwen3_30B_A3B.py index 2429a639ac7..1c423173b48 100644 --- a/tests/test_qwen3_30B_A3B.py +++ b/tests/test_qwen3_30B_A3B.py @@ -110,7 +110,7 @@ def execute(): f"{rollout_args} " f"{optimizer_args} " f"{grpo_args} " - f"{U.get_default_wandb_args()} " + f"{U.get_default_wandb_args(__file__)} " f"{perf_args} " f"{eval_args} " f"{sglang_args} " From e0a9ec3eb9fef350cc643434760d7cc4de0ad043 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 20:18:38 +0800 Subject: [PATCH 095/105] more --- miles/backends/megatron_utils/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/miles/backends/megatron_utils/model.py b/miles/backends/megatron_utils/model.py index e0b939121ae..0fa9f9145fe 100644 --- a/miles/backends/megatron_utils/model.py +++ b/miles/backends/megatron_utils/model.py @@ -488,9 +488,9 @@ def train(rollout_id, model, optimizer, opt_param_scheduler, data_iterator, num_ if args.ci_test: if step_id == 0 and "train/ppo_kl" in log_dict and "train/pg_clipfrac" in log_dict: - assert log_dict["train/ppo_kl"] == 0.0 and log_dict["train/pg_clipfrac"] == 0.0 + assert log_dict["train/ppo_kl"] == 0.0 and log_dict["train/pg_clipfrac"] == 0.0, f"{loss_dict=}" if accumulated_step_id == 0 and "train/kl_loss" in log_dict: - assert log_dict["train/kl_loss"] == 0.0 + assert log_dict["train/kl_loss"] == 0.0, f"{loss_dict=}" print(f"{role_tag}step {accumulated_step_id}: {log_dict}") # Close out pre-hooks if using distributed optimizer and overlapped param gather. From 535e158d9d9f8068f7a96e02de17ece2de6dd02c Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 20:19:19 +0800 Subject: [PATCH 096/105] fmt --- tests/command_utils.py | 7 ++++++- tests/test_quick_start_glm4_9B.py | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/command_utils.py b/tests/command_utils.py index b85cca43087..f5772ae2cbc 100644 --- a/tests/command_utils.py +++ b/tests/command_utils.py @@ -93,7 +93,12 @@ def get_default_wandb_args(test_file: str): run_name += f"_{x}" # do not put wandb_api_key value here to avoid leaking to logs explicitly - return "--use-wandb " f"--wandb-project miles-ci-{test_name} " f"--wandb-group {run_name} " f"--wandb-key ${{WANDB_API_KEY}} " + return ( + "--use-wandb " + f"--wandb-project miles-ci-{test_name} " + f"--wandb-group {run_name} " + f"--wandb-key ${{WANDB_API_KEY}} " + ) def exec_command(cmd: str, capture_output: bool = False): diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index c90e4b48e46..6925b09de17 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -7,6 +7,7 @@ ENABLE_FAST_CHECK = bool(int(os.environ.get("MILES_TEST_ENABLE_FAST_CHECK", "0"))) + def prepare(): U.exec_command("mkdir -p /root/models /root/datasets") U.exec_command("hf download zai-org/GLM-Z1-9B-0414 --local-dir /root/models/GLM-Z1-9B-0414") From 962c5f1b2b96252886d6b2de61fbbdc91e61995a Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 21:01:27 +0800 Subject: [PATCH 097/105] more --- tests/test_qwen3_30B_A3B.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_qwen3_30B_A3B.py b/tests/test_qwen3_30B_A3B.py index 1c423173b48..a1bd0eaaa1e 100644 --- a/tests/test_qwen3_30B_A3B.py +++ b/tests/test_qwen3_30B_A3B.py @@ -1,9 +1,14 @@ +import os + import command_utils as U MODEL_NAME = "Qwen3-30B-A3B" MODEL_TYPE = "qwen3-30B-A3B" +TIGHT_HOST_MEMORY = bool(int(os.environ.get("MILES_TEST_TIGHT_HOST_MEMORY", "1"))) + + def prepare(): U.exec_command("mkdir -p /root/models /root/datasets") U.exec_command("hf download Qwen/Qwen3-30B-A3B --local-dir /root/models/Qwen3-30B-A3B") @@ -52,7 +57,7 @@ def execute(): "--recompute-method uniform " "--recompute-num-layers 1 " "--use-dynamic-batch-size " - "--max-tokens-per-gpu 16384 " + f"--max-tokens-per-gpu {4096 if TIGHT_HOST_MEMORY else 16384} " ) grpo_args = ( From d97317ef5cda9870d63f6fe7dace9ef6dffd28db Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 21:01:52 +0800 Subject: [PATCH 098/105] more --- tests/test_quick_start_glm4_9B.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index 6925b09de17..30efd362ff9 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -60,7 +60,7 @@ def execute(): "--recompute-method uniform " "--recompute-num-layers 1 " "--use-dynamic-batch-size " - "--max-tokens-per-gpu 4608 " + f"--max-tokens-per-gpu {2048 if TIGHT_HOST_MEMORY else 4608} " ) grpo_args = ( From 8a61d34741736b5c9a642d1803f0687a246b596b Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 21:02:06 +0800 Subject: [PATCH 099/105] more --- tests/test_quick_start_glm4_9B.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index 30efd362ff9..e43590f2ef5 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -6,6 +6,7 @@ MODEL_TYPE = "glm4-9B" ENABLE_FAST_CHECK = bool(int(os.environ.get("MILES_TEST_ENABLE_FAST_CHECK", "0"))) +TIGHT_HOST_MEMORY = bool(int(os.environ.get("MILES_TEST_TIGHT_HOST_MEMORY", "1"))) def prepare(): From 1999b7512de770f966ac5b709f21bda5d5601faa Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 21:27:04 +0800 Subject: [PATCH 100/105] more --- tests/test_quick_start_glm4_9B.py | 6 +++--- tests/test_qwen3_30B_A3B.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_quick_start_glm4_9B.py b/tests/test_quick_start_glm4_9B.py index e43590f2ef5..b8176b7b9f2 100644 --- a/tests/test_quick_start_glm4_9B.py +++ b/tests/test_quick_start_glm4_9B.py @@ -5,7 +5,7 @@ MODEL_NAME = "GLM-Z1-9B-0414" MODEL_TYPE = "glm4-9B" -ENABLE_FAST_CHECK = bool(int(os.environ.get("MILES_TEST_ENABLE_FAST_CHECK", "0"))) +ENABLE_EVAL = bool(int(os.environ.get("MILES_TEST_ENABLE_EVAL", "1"))) TIGHT_HOST_MEMORY = bool(int(os.environ.get("MILES_TEST_TIGHT_HOST_MEMORY", "1"))) @@ -36,14 +36,14 @@ def execute(): "--num-rollout 3 " "--rollout-batch-size 8 " "--n-samples-per-prompt 8 " - f"--rollout-max-response-len {128 if ENABLE_FAST_CHECK else 8192} " + f"--rollout-max-response-len 8192 " "--rollout-temperature 0.8 " "--global-batch-size 32 " "--balance-data " ) eval_args = ( - f"{'' if ENABLE_FAST_CHECK else '--eval-interval 20 '}" + f"{'--eval-interval 20 ' if ENABLE_EVAL else ''}" "--eval-prompt-data aime24 /root/datasets/aime-2024/aime-2024.jsonl " "--n-samples-per-eval-prompt 1 " "--eval-max-response-len 16384 " diff --git a/tests/test_qwen3_30B_A3B.py b/tests/test_qwen3_30B_A3B.py index a1bd0eaaa1e..c863a715ec2 100644 --- a/tests/test_qwen3_30B_A3B.py +++ b/tests/test_qwen3_30B_A3B.py @@ -6,6 +6,7 @@ MODEL_TYPE = "qwen3-30B-A3B" +ENABLE_EVAL = bool(int(os.environ.get("MILES_TEST_ENABLE_EVAL", "1"))) TIGHT_HOST_MEMORY = bool(int(os.environ.get("MILES_TEST_TIGHT_HOST_MEMORY", "1"))) @@ -39,7 +40,7 @@ def execute(): ) eval_args = ( - "--eval-interval 20 " + f"{'--eval-interval 20 ' if ENABLE_EVAL else ''}" "--eval-prompt-data aime24 /root/datasets/aime-2024/aime-2024.jsonl " "--n-samples-per-eval-prompt 1 " "--eval-max-response-len 16384 " From 42f0543fd5e54b700898b9152d08383d03bbc2aa Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 21:38:57 +0800 Subject: [PATCH 101/105] more --- tests/test_qwen3_30B_A3B.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_qwen3_30B_A3B.py b/tests/test_qwen3_30B_A3B.py index c863a715ec2..978d8e2e47b 100644 --- a/tests/test_qwen3_30B_A3B.py +++ b/tests/test_qwen3_30B_A3B.py @@ -58,7 +58,7 @@ def execute(): "--recompute-method uniform " "--recompute-num-layers 1 " "--use-dynamic-batch-size " - f"--max-tokens-per-gpu {4096 if TIGHT_HOST_MEMORY else 16384} " + f"--max-tokens-per-gpu {1024 if TIGHT_HOST_MEMORY else 16384} " ) grpo_args = ( From 99bd7798427198888a8631a4d43675bfa7195f83 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 22:06:51 +0800 Subject: [PATCH 102/105] more --- tests/test_qwen3_30B_A3B.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_qwen3_30B_A3B.py b/tests/test_qwen3_30B_A3B.py index 978d8e2e47b..35f312bd12a 100644 --- a/tests/test_qwen3_30B_A3B.py +++ b/tests/test_qwen3_30B_A3B.py @@ -58,7 +58,7 @@ def execute(): "--recompute-method uniform " "--recompute-num-layers 1 " "--use-dynamic-batch-size " - f"--max-tokens-per-gpu {1024 if TIGHT_HOST_MEMORY else 16384} " + f"--max-tokens-per-gpu {2048 if TIGHT_HOST_MEMORY else 16384} " ) grpo_args = ( From 08be50c55113992c140999e1a54291026e77ca29 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 22:38:03 +0800 Subject: [PATCH 103/105] more --- tests/test_qwen3_30B_A3B.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_qwen3_30B_A3B.py b/tests/test_qwen3_30B_A3B.py index 35f312bd12a..978d8e2e47b 100644 --- a/tests/test_qwen3_30B_A3B.py +++ b/tests/test_qwen3_30B_A3B.py @@ -58,7 +58,7 @@ def execute(): "--recompute-method uniform " "--recompute-num-layers 1 " "--use-dynamic-batch-size " - f"--max-tokens-per-gpu {2048 if TIGHT_HOST_MEMORY else 16384} " + f"--max-tokens-per-gpu {1024 if TIGHT_HOST_MEMORY else 16384} " ) grpo_args = ( From c57f4c8968b2e2f72d29b41d0e31942792b23e39 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 22:40:11 +0800 Subject: [PATCH 104/105] disable use_kl_loss --- tests/test_qwen3_30B_A3B.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_qwen3_30B_A3B.py b/tests/test_qwen3_30B_A3B.py index 978d8e2e47b..e0b93ea6e54 100644 --- a/tests/test_qwen3_30B_A3B.py +++ b/tests/test_qwen3_30B_A3B.py @@ -63,7 +63,7 @@ def execute(): grpo_args = ( "--advantage-estimator gspo " - "--use-kl-loss " + f"{'' if TIGHT_HOST_MEMORY else '--use-kl-loss '}" "--kl-loss-coef 0.00 " "--kl-loss-type low_var_kl " "--kl-coef 0.00 " From 7ed3dd7e2fc6fab858bd0d0ae27e59524039723f Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Thu, 9 Oct 2025 22:50:01 +0800 Subject: [PATCH 105/105] more --- tests/test_qwen3_30B_A3B.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_qwen3_30B_A3B.py b/tests/test_qwen3_30B_A3B.py index e0b93ea6e54..db4f01d41a9 100644 --- a/tests/test_qwen3_30B_A3B.py +++ b/tests/test_qwen3_30B_A3B.py @@ -58,7 +58,7 @@ def execute(): "--recompute-method uniform " "--recompute-num-layers 1 " "--use-dynamic-batch-size " - f"--max-tokens-per-gpu {1024 if TIGHT_HOST_MEMORY else 16384} " + f"--max-tokens-per-gpu {2048 if TIGHT_HOST_MEMORY else 16384} " ) grpo_args = (