Update release v0.1.1

Dockerfile

@@ -1,42 +1,4 @@
-ARG REAL_CPU_BASE_IMAGE
-ARG REAL_GPU_BASE_IMAGE
-
-# >>>>>> CPU image
-FROM ${REAL_CPU_BASE_IMAGE} as cpu
-
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt update
-RUN apt install -y ca-certificates
-RUN sed -i "s@http://.*archive.ubuntu.com@https://mirrors.tuna.tsinghua.edu.cn@g" /etc/apt/sources.list
-RUN sed -i "s@http://.*security.ubuntu.com@https://mirrors.tuna.tsinghua.edu.cn@g" /etc/apt/sources.list
-RUN apt update
-RUN apt install -y net-tools python3-pip pkg-config libopenblas-base libopenmpi-dev git
-
-RUN pip3 install -U pip
-RUN pip3 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
-# Install PyTorch in advance to prevent rebuilding this large Docker layer.
-RUN pip3 install torch==2.3.1
-
-RUN pip3 install deepspeed==0.14.0 megatron==0.6.0
-
-COPY ./requirements.txt /requirements.txt
-RUN pip3 install -r /requirements.txt && rm /requirements.txt
-
-COPY . /realhf
-RUN REAL_NO_EXT=1 pip3 install -e /realhf --no-build-isolation
-WORKDIR /realhf
-
-# >>>>>> Documentation images
-# FROM cpu AS docs-builder
-# RUN pip install -U sphinx sphinx-nefertiti -i https://pypi.tuna.tsinghua.edu.cn/simple
-# RUN sphinx-build -M html /realhf/docs/source/ /realhf/docs/build/
-FROM nginx:alpine AS docs
-COPY ./docs/build/html /usr/share/nginx/html
-EXPOSE 80
-CMD ["nginx", "-g", "daemon off;"]
-
-# >>>>>> GPU image
-FROM ${REAL_GPU_BASE_IMAGE} AS gpu
+FROM nvcr.io/nvidia/pytorch:24.07-py3 AS gpu
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt update
@@ -67,9 +29,9 @@ RUN pip3 install flash-attn==2.4.2 --no-build-isolation
 # Install grouped_gemm for MoE acceleration
 RUN pip3 install git+https://github.com/tgale96/grouped_gemm.git@v0.1.4 --no-build-isolation --no-deps
 
-COPY . /realhf
-RUN REAL_CUDA=1 pip3 install -e /realhf --no-build-isolation
-WORKDIR /realhf
+COPY . /AReaL
+RUN REAL_CUDA=1 pip3 install -e /AReaL --no-build-isolation
+WORKDIR /AReaL
 
 RUN git clone --depth=1 -b v0.6.3.post1 https://github.com/vllm-project/vllm.git /vllm
 RUN apt install kmod ccache -y
@@ -85,4 +47,4 @@ RUN apt-get update && apt-get install -y python3.10-venv
 RUN git clone --depth=1 https://github.com/QwenLM/Qwen2.5-Math /qwen2_5-math && mv /qwen2_5-math/evaluation/latex2sympy /latex2sympy
 RUN python3 -m venv /sympy
 RUN /sympy/bin/pip install /latex2sympy
-RUN /sympy/bin/pip install regex numpy tqdm datasets python_dateutil sympy==1.12 antlr4-python3-runtime==4.11.1 word2number Pebble timeout-decorator prettytable
+RUN /sympy/bin/pip install regex numpy tqdm datasets python_dateutil sympy==1.12 antlr4-python3-runtime==4.11.1 word2number Pebble timeout-decorator prettytable

README.md

@@ -103,20 +103,30 @@ We have listed detailed hardware requirements and the approach to setting up env
 
 *Figure 2. Total time consumption for RL training under varying computational resource settings for 10 epochs.*
 
-## RL Training LRM directly from the Base Model
+## Qwen2.5-7B-Zero RL Training
 
-We start RL training from the base model Qwen2.5-7B with DeepSeek-R1-Zero style training.  The initial training curves and results are presented below. As the training progresses, both the rewards and the response lengths gradually grow. A similar trend is also revealed [Open-Reasoner-Zero](https://github.com/Open-Reasoner-Zero/Open-Reasoner-Zero). **The simultaneous growth of average response lengths and rewards** can be considered as a critical sign of **the emergent deep thinking capabilities of LRM** to solve challenging reasoning problems.
+We start RL training from the base model Qwen2.5-7B with DeepSeek-R1-Zero style training. The initial training curves and results are presented below. We conducted experiments on the data released by [Open-Reasoner-Zero](https://github.com/Open-Reasoner-Zero/Open-Reasoner-Zero). As the training progresses, both the rewards and the response lengths gradually grow. A similar trend is also revealed [Open-Reasoner-Zero](https://github.com/Open-Reasoner-Zero/Open-Reasoner-Zero). **The simultaneous growth of average response lengths and rewards** can be considered as a critical sign of **the emergent deep thinking capabilities of LRM** to solve challenging reasoning problems.
 
 
-![undefined](/assets/7b_zero_curve.png) 
+![undefined](/assets/7b_zero_training_curve.png) 
 *Figure 3. The training curve of Qwen2.5-7B-Zero during RL training*
 
-|          | MATH500  | AIME 2024 | AMC 2023 |
+
+We evaluate the performances of the intermediate checkpoints on the MATH500 and AIME24 datasets in the following figure. It appears that both the accuracy and the generated length continue to show an upward trend.
+
+![undefined](/assets/7b_zero_eval_acc.png)
+*Figure 4. The test accuracy and response length evaluated on the MATH500 and AIME24 datasets.*
+
+We also conduct a experiment on the [DeepScalseR](https://github.com/agentica-project/deepscaler) dataset, which demonstrates a similar training trend. The evaluation results are presented in the following table.
+
+|          | MATH-500  | AIME 2024 | AMC 2023 |
 | -------- | -------- | --------- | -------- |
 | Qwen2.5-7B | 34.0 | 2.0 | 17.8 |
-| Open-Reasoner-Zero-7B Step = 150 | ~77 | ~15 | - |
-| Qwen2.5-7B-Zero    Step = 150 KL = 0 | 73.9 | 16.8 | 51.1 |
-| Qwen2.5-7B-Zero    Step = 150 KL = 0.001 | 76.3 | 13.3 | 53.7 |
+| Open-Reasoner-Zero-7B, Step=150 | ~77 | ~15 | - |
+| Qwen2.5-7B-Zero,    Step=150 Dataset=DeepScaleR | 75.9 | 13.9 | 56.1 |
+| Qwen2.5-7B-Zero,    Step=150 Dataset=ORZ | 77.3 | 13.9 | 57.1 |
+| Qwen2.5-7B-Zero,    Step=200 Dataset=ORZ | 78.0 | 14.5 | 58.7 |
+Table 2. Evaluation on AIME 2024, AMC 2023, and MATH-500. The results are reported using Pass@1 accuracy, which are averaged over 32 samples for each problem and evaluated with a temperature of 1.0.
 
 - **Training:** 
 
@@ -128,7 +138,7 @@ wget https://huggingface.co/datasets/inclusionAI/AReaL-RL-Data/resolve/main/data
 wget https://huggingface.co/datasets/inclusionAI/AReaL-RL-Data/resolve/main/data/id2info.json?download=true
 
 MODEL_PATH=${path_to_Qwen2.5-7B}
-bash ./examples/train_7B_zero_n16_on_ray.sh $MODEL_PATH $DATA_PATH/prompts_for_zero.jsonl $DATA_PATH/id2info.json 16384
+bash ./examples/train_7B_zero_n16_on_ray.sh $MODEL_PATH $DATA_PATH/prompts_for_zero.jsonl $DATA_PATH/id2info.json 24000
 
 ```
 

examples/README.md

@@ -29,18 +29,52 @@ This tutorial provides a Docker image. Below are the tested software versions:
 | | Version |
 |---|:---:|
 | OS | CentOS 7 / Ubuntu 22.04 or any other system that meets the software requirements below |
-| Nvidia Driver | 550.127.08 |
+| NVIDIA Driver | 550.127.08 |
 | CUDA | 12.5 |
 | Git LFS | Refer to: https://docs.github.com/en/repositories/working-with-files/managing-large-files/installing-git-large-file-storage. Mainly used for downloading models, datasets, and AReaL project code. |
 | Docker | 27.5.1 |
 |NVIDIA Container Toolkit|[Installing the NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html)|
 | AReaL Image | `ghcr.io/inclusionai/areal-runtime:v0.1.0`. This image includes AReaL's runtime dependencies and Ray components. |
 
+Since the installation of NVIDIA Drivers and CUDA, as well as the mounting of shared storage, depends on node configurations and system versions, please complete these installations independently. This tutorial does not cover their setup.
+
+For multi-node training, ensure that the shared storage is mounted to the `/storage` directory on every node. All subsequent downloads and resources will be stored in this directory. The AReaL container will also mount this directory to `/storage` within the container, enabling seamless access during training.
+
+
+# One-Click Environment Setup and Training Launch
+
+This section provides a one-click setup script to automatically configure the node environment:
+
+1. Install Docker, Git LFS, and NVIDIA Container Toolkit
+2. Pull the AReaL image on each node
+3. Download AReaL code, models, and datasets
+4. Set up a Ray cluster
+5. [Optional] Launch a training task within the Ray cluster
+
+Please perform the following operations on any chosen node:
+
+```bash
+mkdir -p /storage/codes
+cd /storage/codes/
+git clone https://github.com/inclusionAI/AReaL.git
+cd /storage/codes/AReaL
+
+python ./examples/env/setup_env_and_start_train.py setup --private_key_file /path/to/ssh_key --ssh_port 22 --username root --hostnames NODE_IP_1 NODE_IP_2 NODE_IP_3 NODE_IP_4 --train_param 1.5B_n1
+```
+
+`setup_env_and_start_train.py setup` arguments：
+
+- `private_key_file`: SSH secret key. Using by connecting nodes.
+- `ssh_port`: SSH port
+- `username`: SSH username
+- `hostnames`: IP list. Split with space. Can be 1, 4, or 16 node IPs
+- `train_param`: [Optional] Training parameters used to launch a training task immediately after environment setup. Valid options are: `1.5B_n1`, `1.5B_n4`, `1.5B_n16`, `7B_n4`, `7B_n16`
+
+If the script in this section fails to execute or encounters errors due to environmental discrepancies, you may manually configure the environment and launch training by following the instructions in the subsequent sections of this tutorial.
+
 # Environment Setup
-After preparing the nodes and the OS, follow this section to download the AReaL project code, models, and datasets, then start the Ray cluster.
 
-- For multi-node training, mount the shared storage to the `/storage` directory on each node. All downloaded content will be stored here and later mounted into the AReaL environment image container.
-- Since shared storage is used, downloading only needs to be done on one node.
+Since shared storage is used, downloading only needs to be done on one node.
 
 ## Code and Cluster Configuration
 Clone the AReaL project code to `/storage/codes`:

examples/README_zh.md

@@ -37,19 +37,52 @@
 ||版本说明|
 |---|---|
 |OS|CentOS 7 / Ubuntu 22.04 或其他满足下方软件运行的系统|
-|Nvidia Driver|版本：550.127.08|
+|NVIDIA Driver|版本：550.127.08|
 |CUDA|版本：12.5|
 |Git LFS|参考：[Git LFS 安装指南](https://docs.github.com/en/repositories/working-with-files/managing-large-files/installing-git-large-file-storage) 主要用于下载模型，数据集，AReaL 工程代码|
 |Docker|版本：27.5.1|
 |NVIDIA Container Toolkit|[NVIDIA Container Toolkit 安装指南](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html)|
 |镜像|ghcr.io/inclusionai/areal-runtime:v0.1.0 这个镜像中包含运行依赖和 Ray 的相关组件|
 
 
-# 运行环境配置
-在准备好节点和系统环境之后，按照本节的说明下载 AReaL 工程代码，模型，数据集，然后启动 Ray 集群。
+由于 NVIDIA Driver 和 CUDA 的安装以及共享存储的挂载与节点和系统版本有关，请自行完成安装，本教程不进行介绍。
 
-- 如果是多节点训练，请先将共享存储挂载到每个节点的 `/storage` 目录上，后续下载的内容都将放在这个目录下，并且最后挂载到 AReaL 环境镜像的容器里。
-- 由于使用了共享存储，下载操作只需要在一个节点上进行。
+如果是多节点训练，请先将共享存储挂载到每个节点的 `/storage` 目录上，后续下载的内容都将放在这个目录下，并且 AReaL 容器也会将该目录挂载到容器的 `/storage`，以便训练时访问。
+
+
+# 一键搭建环境并启动训练
+
+本节提供一个一键安装脚本，自动完成节点的环境配置工作：
+1. 安装 Docker，Git LFS，NVIDIA Container Toolkit
+2. 在每个节点上拉取 AReaL 镜像
+3. 下载 AReaL 代码，模型，数据集
+4. 搭建 Ray 集群
+5. 【可选】在 Ray 集群中启动一个训练任务
+
+请选择任意一个节点执行如下操作：
+
+```bash
+mkdir -p /storage/codes
+cd /storage/codes/
+git clone https://github.com/inclusionAI/AReaL.git
+cd /storage/codes/AReaL
+
+python ./examples/env/setup_env_and_start_train.py setup --private_key_file /path/to/ssh_key --ssh_port 22 --username root --hostnames NODE_IP_1 NODE_IP_2 NODE_IP_3 NODE_IP_4 --train_param 1.5B_n1
+```
+
+`setup_env_and_start_train.py setup` 参数说明：
+
+- `private_key_file`：SSH 私钥文件，用于连接节点
+- `ssh_port`：SSH 端口
+- `username`：SSH 用户名
+- `hostnames`：IP 列表，用空格分割。可以是 1/4/16 个节点 IP
+- `train_param`：【可选】训练参数，用于在完成环境搭建后直接启动一个训练任务。可选值为 `1.5B_n1`，`1.5B_n4`，`1.5B_n16`，`7B_n4`，`7B_n16`
+
+如果因为环境差异，无法运行本节中的脚本或运行出现错误，也可以按照本教程后续章节的内容手动完成环境配置和启动训练。
+
+# 环境配置
+
+由于使用了共享存储，下载操作只需要在一个节点上完成。
 
 ## 代码和集群配置
 将 AReaL 项目代码克隆到 `/storage/codes` 中：

examples/env/scripts/download-dataset.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+set -e
+
+mkdir -p /storage/datasets/
+cd /storage/datasets/
+if [ $REGION == 'CN' ]; then
+    wget https://www.modelscope.cn/datasets/inclusionAI/AReaL-RL-Data/resolve/master/data/prompts_for_r1_distilled.jsonl
+    wget https://www.modelscope.cn/datasets/inclusionAI/AReaL-RL-Data/resolve/master/data/prompts_for_zero.jsonl
+    wget https://www.modelscope.cn/datasets/inclusionAI/AReaL-RL-Data/resolve/master/data/id2info.json
+else
+    wget https://huggingface.co/datasets/inclusionAI/AReaL-RL-Data/resolve/main/data/prompts_for_r1_distilled.jsonl?download=true
+    wget https://huggingface.co/datasets/inclusionAI/AReaL-RL-Data/resolve/main/data/prompts_for_zero.jsonl?download=true
+    wget https://huggingface.co/datasets/inclusionAI/AReaL-RL-Data/resolve/main/data/id2info.json?download=true
+fi

examples/env/scripts/download-model.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+set -e
+
+mkdir -p /storage/models
+cd /storage/models
+if [ $REGION == 'CN' ]; then
+    git clone https://www.modelscope.cn/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B.git
+    git clone https://www.modelscope.cn/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B.git
+else
+    GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
+    GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B
+fi