wuxibin89 · Oct 19, 2026 · Oct 14, 2026 · Oct 15, 2026 · Oct 15, 2026 · Oct 15, 2026
diff --git a/.github/workflows/e2e_ppo_grpo_trainer_trtllm.yml b/.github/workflows/e2e_ppo_grpo_trainer_trtllm.yml
@@ -274,34 +274,6 @@ jobs:
       - name: clean up
         run: |
           rm -rf checkpoints
-      - name: Prepare GSM8K dataset (data_preprocess)
-        run: |
-          python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k --local_save_dir ${PWD}/data/gsm8k
-      - name: Running GRPO E2E with FP8 TRT-LLM rollout (Qwen3-0.6B, gsm8k)
-        run: |
-          unset TORCH_CUDA_ARCH_LIST  # Let TRT-LLM CUTLASS DSL auto-detect runtime GPU arch
-          export TRTLLM_ENABLE_PDL=0  # Disable Programmatic Dependent Launch (uses SM 9.0 griddepcontrol)
-          ray stop --force
-          export INFER_TP=2 ACTOR_TP=2 ACTOR_PP=2 ACTOR_VPP=2 ACTOR_EP=1 ACTOR_CP=2 REF_TP=2 REF_PP=2 REF_VPP=2 REF_EP=1 REF_CP=2 GEN_MOE_TP=null GEN_MOE_EP=null
-          export NNODES=1 GPUS_PER_NODE=8 TRTLLM_MOE_BACKEND=TRITON
-          export DATA_DIR=${PWD} DAPO_MATH_TRAIN=${PWD}/data/gsm8k/train.parquet AIME_VAL=${PWD}/data/gsm8k/test.parquet MODEL_PATH=${HOME}/models/Qwen/Qwen3-0.6B
-          INFER_BACKEND=trtllm ROLLOUT_QUANTIZATION=fp8 bash examples/grpo_trainer/run_qwen3_30b_a3b_megatron.sh \
-            reward_model.reward_kwargs.overlong_buffer_cfg.len=258 \
-            reward_model.reward_kwargs.max_resp_len=512 \
-            data.max_prompt_length=128 \
-            data.max_response_length=64 \
-            data.train_batch_size=32 \
-            actor_rollout_ref.rollout.n=1 \
-            actor_rollout_ref.rollout.max_num_seqs=16 \
-            actor_rollout_ref.rollout.max_num_batched_tokens=1024 \
-            actor_rollout_ref.rollout.max_model_len=1024 \
-            actor_rollout_ref.actor.megatron.override_transformer_config.moe_grouped_gemm=False \
-            actor_rollout_ref.actor.megatron.override_transformer_config.moe_permute_fusion=False \
-            trainer.total_training_steps=1 \
-            trainer.logger='["console"]'
-      - name: clean up
-        run: |
-          rm -rf checkpoints
 
   cleanup:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/e2e_ppo_trainer_megatron_sglang.yml b/.github/workflows/e2e_ppo_trainer_megatron_sglang.yml
@@ -203,8 +203,7 @@ jobs:
       - name: Running GSM8K E2E training tests with 3D parallelism on 8 L20 GPUs with FP8 rollout
         run: |
           $UV_RUN ray stop --force
-          export VLLM_USE_V1=1
-          ROLLOUT_QUANTIZATION=fp8 TOTAL_TRAIN_STEPS=2 MODEL_ID=Qwen/Qwen3-0.6B bash tests/special_e2e/run_ppo_trainer_megatron.sh
+          ROLLOUT_QUANTIZATION=FP8_BLOCK TOTAL_TRAIN_STEPS=2 MODEL_ID=Qwen/Qwen3-0.6B bash tests/special_e2e/run_ppo_trainer_megatron.sh
       - name: clean up
         run: |
           rm -rf checkpoints

diff --git a/.github/workflows/e2e_ppo_trainer_megatron_vllm.yml b/.github/workflows/e2e_ppo_trainer_megatron_vllm.yml
@@ -209,7 +209,7 @@ jobs:
         run: |
           $UV_RUN ray stop --force
           export VLLM_USE_V1=1
-          ROLLOUT_QUANTIZATION=fp8 TOTAL_TRAIN_STEPS=2 MODEL_ID=Qwen/Qwen3-0.6B bash tests/special_e2e/run_ppo_trainer_megatron.sh
+          ROLLOUT_QUANTIZATION=FP8_BLOCK TOTAL_TRAIN_STEPS=2 MODEL_ID=Qwen/Qwen3-0.6B bash tests/special_e2e/run_ppo_trainer_megatron.sh
       - name: Running GSM8K E2E PPO training tests on 8 L20 GPUs with Megatron-FSDP (Qwen3)
         run: |
           $UV_RUN ray stop --force

diff --git a/.github/workflows/e2e_ppo_trainer_megatron_vllm_2.yml b/.github/workflows/e2e_ppo_trainer_megatron_vllm_2.yml
@@ -183,7 +183,7 @@ jobs:
           MAX_PROMPT_LENGTH=512 MAX_RESPONSE_LENGTH=512 \
           MODEL_ID=Qwen/Qwen3-30B-A3B-Instruct-2507 USE_MBRIDGE=True \
           COMMON_PP=2 COMMON_VPP=null COMMON_CP=1 COMMON_TP=4 COMMON_EP=4 COMMON_ETP=1 INFER_TP=2 \
-          USE_DIST_CKPT=False ALL_OFFLOAD=True SKIP_SAVE_HF_MODEL=1 ROLLOUT_QUANTIZATION=fp8 bash tests/special_e2e/run_ppo_trainer_megatron.sh
+          USE_DIST_CKPT=False ALL_OFFLOAD=True SKIP_SAVE_HF_MODEL=1 ROLLOUT_QUANTIZATION=FP8_BLOCK bash tests/special_e2e/run_ppo_trainer_megatron.sh
       - name: clean up
         run: |
           rm -rf checkpoints

diff --git a/.github/workflows/e2e_ppo_trainer_megatron_vllm_rocm.yml b/.github/workflows/e2e_ppo_trainer_megatron_vllm_rocm.yml
@@ -253,7 +253,7 @@ jobs:
           MAX_PROMPT_LENGTH=512 MAX_RESPONSE_LENGTH=512 \
           MODEL_ID=Qwen/Qwen3-30B-A3B-Instruct-2507 USE_MBRIDGE=True \
           COMMON_PP=2 COMMON_VPP=null COMMON_CP=1 COMMON_TP=4 COMMON_EP=4 COMMON_ETP=1 INFER_TP=2 \
-          USE_DIST_CKPT=False ALL_OFFLOAD=True SKIP_SAVE_HF_MODEL=1 ROLLOUT_QUANTIZATION=fp8 \
+          USE_DIST_CKPT=False ALL_OFFLOAD=True SKIP_SAVE_HF_MODEL=1 ROLLOUT_QUANTIZATION=FP8_BLOCK \
           bash tests/special_e2e/run_ppo_trainer_megatron.sh \
           global_profiler.tool=null actor_rollout_ref.actor.profiler.tool=null actor_rollout_ref.ref.profiler.tool=null actor_rollout_ref.rollout.profiler.tool=null critic.profiler.tool=null "${VLLM_ROCM_OVERRIDE}"
       - name: clean up

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,3 +1,5 @@
+exclude: ^\.venv[^/]*/
+
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: "v0.12.2"
@@ -58,7 +60,7 @@ repos:
 
       - id: check-naming-conventions
         name: Check naming conventions
-        entry: sh -c 'fail=0; if grep -rIn --exclude-dir=.git --exclude-dir=.github --exclude-dir=venv --exclude-dir=.venv --exclude-dir=__pycache__ --exclude=.pre-commit-config.yaml "veRL" .; then echo "Please use verl instead of veRL"; fail=1; fi; if grep -rIn --exclude-dir=.git --exclude-dir=.github --exclude-dir=venv --exclude-dir=.venv --exclude-dir=__pycache__ --exclude=ascend_sglang_best_practices.rst --exclude=.pre-commit-config.yaml -E "Sglang|sgLang|sglAng|sglaNg|sglanG" .; then echo "Please use SGLang or sglang"; fail=1; fi; exit $fail'
+        entry: sh -c 'fail=0; if grep -rIn --exclude-dir=.git --exclude-dir=.github --exclude-dir=venv --exclude-dir=.venv* --exclude-dir=__pycache__ --exclude=.pre-commit-config.yaml "veRL" .; then echo "Please use verl instead of veRL"; fail=1; fi; if grep -rIn --exclude-dir=.git --exclude-dir=.github --exclude-dir=venv --exclude-dir=.venv* --exclude-dir=__pycache__ --exclude=ascend_sglang_best_practices.rst --exclude=.pre-commit-config.yaml -E "Sglang|sgLang|sglAng|sglaNg|sglanG" .; then echo "Please use SGLang or sglang"; fail=1; fi; exit $fail'
         language: system
         pass_filenames: false
 
@@ -76,6 +78,6 @@ repos:
 
       - id: compileall
         name: Compile all python files
-        entry: sh -c 'PYTHONWARNINGS=error python3 -m compileall -q . -x "(^|[\\/])(\.venv|venv|\.git|verl[\\/]experimental[\\/]vla)([\\/]|$)"'
+        entry: sh -c 'PYTHONWARNINGS=error python3 -m compileall -q . -x "(^|[\\/])(\.venv[^\\/]*|venv|\.git|verl[\\/]experimental[\\/]vla)([\\/]|$)"'
         language: python
         pass_filenames: false
@@ -120,8 +120,8 @@ verl 通过层级化的 YAML 配置文件管理所有参数，涉及到的所有
 | `actor_rollout_ref.rollout.calculate_log_probs` | `false` | 是否在 rollout 阶段计算 log probs |
 | `actor_rollout_ref.rollout.skip_tokenizer_init` | `true` | 是否跳过分词器初始化 |
 | `actor_rollout_ref.rollout.enable_rollout_routing_replay` | `false` | 是否启用 rollout 路由重放 |
-| `actor_rollout_ref.rollout.quantization` | `null` | 量化方式 |
-| `actor_rollout_ref.rollout.quantization_config_file` | `null` | 量化配置文件路径 |
+| `actor_rollout_ref.rollout.quantization.linear` / `.moe` | `null` | compressed-tensors 量化 preset（如 `FP8_BLOCK`） |
+| `actor_rollout_ref.rollout.quantization.config_file` | `null` | compressed-tensors 量化配置文件路径 |
 | `actor_rollout_ref.rollout.layered_summon` | `false` | 是否启用分层召唤（仅 FSDP 方案） |
 
 #### 1.1.5 Rollout 验证采样配置

@@ -37,7 +37,7 @@ Last updated: 03/03/2026.
 | `max_num_batched_tokens`| `actor_rollout_ref.rollout.max_num_batched_tokens` |在一次批处理（batch）中可以处理的最大总Token数|
 | `skip_tokenizer_init`| `actor_rollout_ref.rollout.skip_tokenizer_init` |跳过初始化分词器并将 input_ids 传递到推理请求中|
 | `enable_prefix_caching` | `actor_rollout_ref.rollout.enable_prefix_caching` |用于启用自动前缀缓存 |
-| `quantization`| `actor_rollout_ref.rollout.quantization`，默认为None|`量化方法`|
+| `quantization`| `actor_rollout_ref.rollout.engine_kwargs.vllm.quantization`，默认不设置|`量化方法`|
 
 ### 2. sglang:
 
@@ -74,7 +74,7 @@ verl中通过rollout config管理推理后端参数使能，包含通用参数
 | log_level| 无，默认设置为error |日志记录器的日志级别|
 | skip_tokenizer_init| actor_rollout_ref.rollout.skip_tokenizer_init |跳过初始化分词器并将 input_ids 传递到推理请求中|
 | skip_server_warmup| 无，默认设置为True |跳过预热|
-| quantization| actor_rollout_ref.rollout.quantization，默认为None|量化方法|
+| quantization| actor_rollout_ref.rollout.engine_kwargs.sglang.quantization，默认不设置|量化方法|
 | attention_backend|actor_rollout_ref.rollout.engine_kwargs.sglang.attention_backend|attention内核,NPU应该设置为ascend|
 
 ---

@@ -61,7 +61,7 @@ Last updated: 05/13/2026.
 | SGLang 参数 | verl 对应通用参数 | 功能说明 |
 |:---|:---|:---|
 | `attention_backend` | `actor_rollout_ref.rollout.engine_kwargs.sglang.attention_backend` | **注意力后端选择** — NPU 上应设置为 `ascend` 以调用昇腾优化内核 |
-| `quantization` | `actor_rollout_ref.rollout.quantization` | **量化支持** — 支持模型量化加载与推理 |
+| `quantization` | `actor_rollout_ref.rollout.engine_kwargs.sglang.quantization` | **量化支持** — 支持模型量化加载与推理 |
 
 
 > 更多 SGLang NPU 特性参数请参考 [sglang 社区 NPU 特性支持文档](https://docs.sglang.io/docs/hardware-platforms/ascend-npus/ascend_npu_support_features)
@@ -229,7 +229,7 @@ repatch
 | 模型路径 | `model_path` | `model_path` | `actor_rollout_ref.model.path` |
 | 显存控制 | `gpu_memory_utilization` | `mem_fraction_static` | `actor_rollout_ref.rollout.gpu_memory_utilization` |
 | 图模式 | `enforce_eager` | `disable_cuda_graph` | `actor_rollout_ref.rollout.enforce_eager` |
-| 量化 | `quantization` | `quantization` | `actor_rollout_ref.rollout.quantization` |
+| 量化 | `quantization` | `quantization` | `actor_rollout_ref.rollout.engine_kwargs.{vllm,sglang}.quantization` |
 | 最大序列长度 | `max_model_len` | — | `actor_rollout_ref.rollout.max_model_len` |
 | 最大并发数 | `max_num_seqs` | `max_running_requests` | `actor_rollout_ref.rollout.max_num_seqs` |
 | 分词器 | `skip_tokenizer_init` | `skip_tokenizer_init` | `actor_rollout_ref.rollout.skip_tokenizer_init` |

@@ -139,6 +139,7 @@ verl is fast with:
 
    low_precision/fp8.md
    low_precision/nvfp4_qat.md
+   low_precision/qat.md
 
 .. toctree::
    :maxdepth: 1

@@ -20,72 +20,36 @@ FP8 rollout-only mode keeps training in BF16 and quantizes rollout inference to
 
 ### Implementation
 
-We monkey patch several vLLM functions to enable FP8 rollout for reinforcement learning:
-
-1. **Quantize weights**: Quantize model weights on-the-fly from higher-precision formats to FP8.
-2. **Process weights after loading**: For vLLM, we replace the `vllm.model_executor.layers.quantization.fp8.Fp8LinearMethod.process_weights_after_loading` function to handle weight processing after quantization. For SGLang, this patch is not needed as it natively supports loading quantized weights.
+The rollout engine starts from a compressed-tensors `quantization_config` with dummy weights, and the trainer quantizes every weight sync to that config with the compressed-tensors compressors (`verl/utils/qat/compressed_tensors.py`). On the vLLM side we monkey patch `process_weights_after_loading` of the compressed-tensors FP8 linear/MoE methods so refits are written back into the live parameters (`verl/utils/vllm/vllm_fp8_utils.py`).
 
 ### Support Matrix
 
-- FP8 blockwise quantization for rollout
+- FP8 blockwise quantization for rollout (compressed-tensors preset `FP8_BLOCK`)
   - Used in Deepseek, which is 1x128 quantization for activations and 128x128 quantization for model weights
 - Dense models and MoE models
 - Async rollout interfaces
-- vLLM 0.10.x & vLLM 0.11 & vLLM 0.12 & SGLang 0.5.5
-- FSDP and Megatron training backends
+- vLLM or SGLang rollout (SGLang validated on Hopper only) with the Megatron training backend; TRT-LLM and FSDP are not supported
 
 ### Usage
 
 Enable in config file:
 
 ```yaml
 rollout:
-  quantization: "fp8"
+  quantization:
+    linear: FP8_BLOCK  # dense linear layers (attention, dense MLP, shared experts)
+    moe: FP8_BLOCK     # routed experts
+    ignore: []         # extra "re:" patterns kept in BF16
 ```
 
 Or via command line:
 
 ```bash
-actor_rollout_ref.rollout.quantization=fp8
-```
-
-#### Skipping layers in SGLang FP8 rollout
-
-When using SGLang FP8 rollout, you can skip FP8 weight quantization for
-selected modules. Skipped modules stay in the rollout model dtype instead
-of being converted to FP8. This is useful for layers that are not
-compatible with block-wise FP8 weight quantization, or for modules that
-you prefer to keep in higher precision.
-
-Set `SGLANG_FP8_IGNORED_LAYERS` before starting training:
-
-```bash
-SGLANG_FP8_IGNORED_LAYERS=linear_attn \
-python3 -m verl.trainer.main_ppo \
-  actor_rollout_ref.rollout.name=sglang \
-  actor_rollout_ref.rollout.quantization=fp8 \
-  ...
-```
-
-Multiple entries can be separated by commas:
-
-```bash
-SGLANG_FP8_IGNORED_LAYERS=linear_attn,visual
-```
-
-You can also use the model `quantization_config`:
-
-```json
-{
-  "quantization_config": {
-    "ignored_layers": ["re:.*linear_attn.*"]
-  }
-}
+actor_rollout_ref.rollout.quantization.linear=FP8_BLOCK \
+actor_rollout_ref.rollout.quantization.moe=FP8_BLOCK
 ```
 
-Plain module names, full module paths, and `re:` regex patterns are
-supported. verl applies the same ignored-layer rules when launching
-SGLang and when syncing updated actor weights into the rollout engine.
+Routers, `lm_head`, embeddings, vision towers and MTP layers stay in BF16 by default, as do layers whose weight shape is not a multiple of the 128x128 block. To provide a complete compressed-tensors `quantization_config` instead, set `actor_rollout_ref.rollout.quantization.config_file=/path/to/quantization_config.json` (exclusive with `linear` / `moe` / `ignore`).
 
 ### Experiments and Outcomes
 
@@ -183,8 +147,9 @@ actor_rollout_ref.actor.optim.override_optimizer_config:
   fp8_recipe: "blockwise"
 
 # FP8 rollout inference (vLLM)
-actor_rollout_ref.rollout:
-  quantization: fp8
+actor_rollout_ref.rollout.quantization:
+  linear: FP8_BLOCK
+  moe: FP8_BLOCK
 ```
 
 ### Support Matrix

@@ -1,87 +1,58 @@
-# NVFP4 QAT (Quantization-Aware Training) in verl
+# NVFP4 QAT
 
-Last updated: 04/02/2026
+Last updated: 10/19/2026
+verl supports NVFP4 as **W4A4 routed experts** on Blackwell:
 
-verl supports NVFP4 Quantization-Aware Training (QAT), which applies fake quantization during training so the model learns to tolerate NVFP4 quantization error. At rollout time, weights are packed into real NVFP4 format for vLLM inference. This closes the precision gap between training and inference, preventing KL divergence explosion.
+- trainer: Megatron + ModelOpt QAT
+- rollout: vLLM compressed-tensors
+- kernel: FlashInfer CUTLASS
+- dense linear layers: BF16 or MXFP8; NVFP4 linear is not supported
 
-| Training Backend | Training Precision | Rollout Precision | vLLM Quant Method |
-|---|---|---|---|
-| **FSDP** | BF16 + fake quantization | NVFP4 W4A16 | `compressed-tensors` |
-| **Megatron** | BF16 + fake quantization | NVFP4 W4A16 | `modelopt` |
+W4A16, FSDP NVFP4 and non-CUTLASS NVFP4 backends are not supported by this
+path.
 
-> [!TIP]
-> For ready-to-run scripts, environment setup, and experimental results, see the [QAT recipe](https://github.com/verl-project/verl-recipe/tree/main/qat).
-
----
-
-## Key Configuration
-
-### FSDP Backend
-
-Configured under `actor_rollout_ref.actor.fsdp_config.qat`:
-
-```yaml
-actor_rollout_ref:
-  actor:
-    fsdp_config:
-      qat:
-        enable: true
-        mode: "w4a16"
-        group_size: 16
-        ignore_patterns:
-          - "lm_head"
-          - "embed_tokens"
-          - "re:.*mlp.gate$"
-        quantization_config_path: "recipe/qat/config/nvfp4_w4a16.json"
-```
-
-| Parameter | Description | Default |
-|-----------|-------------|---------|
-| `fsdp_config.qat.enable` | Enable QAT | `False` |
-| `fsdp_config.qat.mode` | Quantization mode | `"w4a16"` |
-| `fsdp_config.qat.group_size` | Quantization group size | `16` |
-| `fsdp_config.qat.ignore_patterns` | Layers to skip. Supports `re:` prefix for regex, otherwise substring match | `["lm_head", "embed_tokens", "re:.*mlp.gate$"]` |
-| `fsdp_config.qat.quantization_config_path` | vLLM quantization config JSON path | Required |
-
-### Megatron Backend
-
-Configured under `actor_rollout_ref.actor.megatron.qat`:
+## Configuration
 
 ```yaml
 actor_rollout_ref:
   actor:
     megatron:
       qat:
         enable: true
-        mode: "w4a16"
-        group_size: 16
-        ignore_patterns:
-          - "lm_head"
-          - "*mlp.gate"
-        quantization_config_path: "recipe/qat/config/nvfp4_w4a16_megatron.json"
+        mode: nvfp4
+  rollout:
+    quantization:
+      linear: null
+      moe: NVFP4
 ```
 
-| Parameter | Description | Default |
-|-----------|-------------|---------|
-| `megatron.qat.enable` | Enable QAT | `False` |
-| `megatron.qat.mode` | Quantization mode | `"w4a16"` |
-| `megatron.qat.group_size` | Quantization group size | `16` |
-| `megatron.qat.ignore_patterns` | Layers to skip. Uses `fnmatch` glob syntax | `["lm_head", "*mlp.gate"]` |
-| `megatron.qat.quantization_config_path` | vLLM quantization config JSON path | Required |
+The weight uses E2M1 values in groups of 16, FP8 E4M3 local scales and one
+tensor-level global divisor. Activations use the same NVFP4 hierarchy with
+dynamic group scales. Before ModelOpt observes an activation, verl exports the
+neutral tensor divisor `1.0`; subsequent syncs use the running QAT value.
 
----
+ModelOpt fake quantization and compressed-tensors export share the same QDQ
+implementation. The exporter also aligns gate/up global scales, as required by
+vLLM's fused MoE loader.
 
-## Support Matrix
+## Validation
 
-- NVFP4 W4A16 (weight-only FP4 quantization)
-- Dense models and MoE models
-- FSDP and Megatron training backends
-- Full quantization and FFN-only quantization strategies
-- Verified on Qwen3-8B-Base and Qwen3-30B-A3B-Base
+Use CUDA Graph mode; eager-mode results are not representative:
 
----
-
-## Notes
+```bash
+.venv-vllm/bin/python scripts/rollout_logprob_diff.py \
+  --model-path /mnt/efs/model/Qwen/Qwen3.5-35B-A3B --ep 8 \
+  --quant-moe NVFP4 --qat-mode nvfp4 \
+  '+actor_rollout_ref.rollout.engine_kwargs.vllm.enable_flashinfer_autotune=false'
+```
 
-- FSDP backend has scalability limitations for very large models. For large-scale training, use the Megatron backend.
-- FSDP uses `re:` prefix regex for `ignore_patterns`, while Megatron uses `fnmatch` glob syntax. The two are not interchangeable.
+On Qwen3.5-35B-A3B with EP=8, CUDA Graph and no R3, the final validation
+measured `k3_kl=0.25778` without QAT and `k3_kl=0.32112` with NVFP4 QAT
+(Pearson correlation `0.991763` and `0.981248`, respectively). FlashInfer's
+fused finalize is nondeterministic; repeated results ranged from `0.219–0.258`
+and `0.263–0.380`. A synthetic single-expert CUTLASS comparison confirmed that
+the exported weights and activation QDQ use the expected NVFP4 format, but the
+fused kernel still differed from dequantize-then-BF16 matmul by mean absolute
+`0.0084` per MoE layer (correlation `0.99855`). This validates the refit
+lifecycle and kernel execution, but not numerical acceptance: the accumulated
+model-level mismatch remains too large for training.