[yewentao256]

Overview

• Keep the MoE intermediate tensor in expert-sorted order so the down projection can use TMA for both activations and weights.
• Fuse SiLU, multiplication, and FP8 quantization into one Triton kernel.

Test

alias bench_prefill="vllm bench serve --model $MODEL  --dataset-name random --host 127.0.0.1 --port 9256 --random-input-len 2048 --random-output-len 1 --request-rate inf --num-prompts 32 --num-warmups 4 --trust-remote-code --temperature 0  --max-concurrency 1"
alias bench_profile="vllm bench serve --model $MODEL  --dataset-name random --host 127.0.0.1 --port 9256 --random-input-len 2 --random-output-len 128 --request-rate inf --num-prompts 2 --num-warmups 1 --profile"
alias lm_eval="lm_eval run --model local-chat-completions --model_args base_url=http://127.0.0.1:9256/v1/chat/completions,model=$MODEL,num_concurrent=1024,seed=256 --tasks aime25 --trust_remote_code --apply_chat_template"
alias glm_serve="c 8 -- vllm serve zai-org/GLM-5.3    --trust-remote-code     -tp 8     --load-format fastsafetensors     --gpu-memory-utilization 0.88     --enable-prefix-caching     --host 0.0.0.0     --port 9256     --max-model-len auto     --max-num-seqs 48     --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":5}'"

Acc

|Tasks |Version|Filter|n-shot|  Metric   |   |Value |   |Stderr|
|------|------:|------|-----:|-----------|---|-----:|---|-----:|
|aime25|      0|none  |     0|exact_match|↑  |0.8333|±  |0.0692|

Also covered in current unit test

Perf

# main
============ Serving Benchmark Result ============
Successful requests:                     32        
Failed requests:                         0         
Maximum request concurrency:             1         
Benchmark duration (s):                  1.30      
Total input tokens:                      65536     
Total generated tokens:                  32        
Request throughput (req/s):              24.61     
Output token throughput (tok/s):         24.61     
Peak output token throughput (tok/s):    24.00     
Peak concurrent requests:                25.00     
Total token throughput (tok/s):          50424.10  
---------------Time to First Token----------------
Mean TTFT (ms):                          40.50     
Median TTFT (ms):                        40.44     
P99 TTFT (ms):                           47.45     
-----Time per Output Token (excl. 1st token)------
Mean TPOT (ms):                          0.00      
Median TPOT (ms):                        0.00      
P99 TPOT (ms):                           0.00      
---------------Inter-token Latency----------------
Mean ITL (ms):                           0.00      
Median ITL (ms):                         0.00      
P99 ITL (ms):                            0.00      
==================================================
============ Serving Benchmark Result ============
Successful requests:                     32        
Failed requests:                         0         
Maximum request concurrency:             1         
Benchmark duration (s):                  1.26      
Total input tokens:                      65536     
Total generated tokens:                  32        
Request throughput (req/s):              25.46     
Output token throughput (tok/s):         25.46     
Peak output token throughput (tok/s):    25.00     
Peak concurrent requests:                26.00     
Total token throughput (tok/s):          52170.86  
---------------Time to First Token----------------
Mean TTFT (ms):                          39.18     
Median TTFT (ms):                        38.48     
P99 TTFT (ms):                           48.57     
-----Time per Output Token (excl. 1st token)------
Mean TPOT (ms):                          0.00      
Median TPOT (ms):                        0.00      
P99 TPOT (ms):                           0.00      
---------------Inter-token Latency----------------
Mean ITL (ms):                           0.00      
Median ITL (ms):                         0.00      
P99 ITL (ms):                            0.00      
==================================================

# Now
============ Serving Benchmark Result ============
Successful requests:                     32        
Failed requests:                         0         
Maximum request concurrency:             1         
Benchmark duration (s):                  1.21      
Total input tokens:                      65536     
Total generated tokens:                  32        
Request throughput (req/s):              26.49     
Output token throughput (tok/s):         26.49     
Peak output token throughput (tok/s):    26.00     
Peak concurrent requests:                27.00     
Total token throughput (tok/s):          54272.47  
---------------Time to First Token----------------
Mean TTFT (ms):                          37.64     
Median TTFT (ms):                        37.74     
P99 TTFT (ms):                           40.06     
-----Time per Output Token (excl. 1st token)------
Mean TPOT (ms):                          0.00      
Median TPOT (ms):                        0.00      
P99 TPOT (ms):                           0.00      
---------------Inter-token Latency----------------
Mean ITL (ms):                           0.00      
Median ITL (ms):                         0.00      
P99 ITL (ms):                            0.00      
==================================================
============ Serving Benchmark Result ============
Successful requests:                     32        
Failed requests:                         0         
Maximum request concurrency:             1         
Benchmark duration (s):                  1.19      
Total input tokens:                      65536     
Total generated tokens:                  32        
Request throughput (req/s):              26.99     
Output token throughput (tok/s):         26.99     
Peak output token throughput (tok/s):    27.00     
Peak concurrent requests:                28.00     
Total token throughput (tok/s):          55305.37  
---------------Time to First Token----------------
Mean TTFT (ms):                          36.94     
Median TTFT (ms):                        37.20     
P99 TTFT (ms):                           39.96     
-----Time per Output Token (excl. 1st token)------
Mean TPOT (ms):                          0.00      
Median TPOT (ms):                        0.00      
P99 TPOT (ms):                           0.00      
---------------Inter-token Latency----------------
Mean ITL (ms):                           0.00      
Median ITL (ms):                         0.00      
P99 ITL (ms):                            0.00      
==================================================

[claude]

Claude Code Review

This repository is configured for manual code reviews. Comment @claude review for a one-time review, or @claude review always to subscribe this PR to a review on every future push.

_{Tip: disable this comment in your organization's Code Review settings.}

[yewentao256]

/ci run

[github-actions]

✅ Triggered Buildkite CI #98729 for commit db10aedc2d4f.

ℹ️ This PR is 1 commit behind upstream main. Merged upstream main into this branch as db10aedc2d4f before starting CI. Pull the branch before pushing more changes.

[yewentao256]

/ci run

[github-actions]

✅ Triggered Buildkite CI #99568 for commit 60bfb07fe56f.

[mgoin]

Why are we using the triton moe for GLM on Hopper? We should try to use a better kernel backend. I assume deepgemm is available and should be preferred, but now we have humming and maybe other backends that could be considered

EDIT: I now remember the TritonOrDeepGemmExperts backend and TP case.. but we should consider evaluating other backends still to improve. I just dislike the triton backend since it requires these explicit configs

[yewentao256]

@mgoin Thanks for the review. It is from benchmark. This is benchmark from main (even without my optimization). We can try exploration some prefill kernel change to DeepGEMM as a follow up, but I don't think we can fully use it to replace Triton by default now.

TP8

M	Triton	DeepGEMM	FlashInfer CUTLASS	Marlin W8A16	DG/T	FI/T	Marlin/T
1	33.9	77.7	92.8	28.4	2.29	2.74	0.84
2	45.0	88.3	100.2	43.4	1.96	2.23	0.97
4	54.3	108.7	110.7	58.7	2.00	2.04	1.08
8	87.3	162.6	148.7	98.3	1.86	1.70	1.13
16	136.8	237.7	205.8	148.3	1.74	1.50	1.08
24	185.2	298.5	265.6	187.5	1.61	1.43	1.01
32	229.5	338.1	324.1	218.9	1.47	1.41	0.95
48	258.6	432.0	404.5	272.7	1.67	1.56	1.05
64	278.9	459.3	433.3	295.1	1.65	1.55	1.06
96	307.7	508.0	581.5	326.8	1.65	1.89	1.06
128	311.1	507.8	593.8	333.3	1.63	1.91	1.07
192	324.2	520.0	614.6	367.4	1.60	1.90	1.13
256	331.7	524.8	634.4	374.2	1.58	1.91	1.13
384	362.3	531.7	696.8	403.4	1.47	1.92	1.11
512	386.1	537.2	748.4	509.9	1.39	1.94	1.32
768	472.5	554.8	854.5	552.4	1.17	1.81	1.17
1024	467.1	565.9	945.5	631.7	1.21	2.02	1.35
1280	488.0	583.3	1055.5	694.7	1.20	2.16	1.42
1536	512.5	599.1	1150.4	870.7	1.17	2.24	1.70
1792	565.2	617.2	1256.9	963.5	1.09	2.22	1.70
1920	589.8	622.8	1304.4	1065.3	1.06	2.21	1.81
2048	638.6	627.5	1352.9	1173.0	0.98	2.12	1.84
2176	710.3	635.4	1411.7	1337.4	0.89	1.99	1.88
2304	758.8	645.4	1466.4	1442.3	0.85	1.93	1.90
2432	795.1	658.6	1519.4	1517.5	0.83	1.91	1.91
2560	810.3	665.3	1564.2	1546.8	0.82	1.93	1.91
2816	860.3	685.2	1669.5	1601.1	0.80	1.94	1.86
3072	863.5	696.9	1766.3	1642.2	0.81	2.05	1.90
3328	887.0	715.0	1884.0	1660.8	0.81	2.12	1.87
3584	939.5	753.1	1997.0	1723.3	0.80	2.13	1.83
4096	1073.9	908.0	2326.9	2031.7	0.85	2.17	1.89

TP4

M	Triton	DeepGEMM	FlashInfer CUTLASS	Marlin W8A16	DG/T	FI/T	Marlin/T
1	37.8	87.9	98.6	40.7	2.32	2.60	1.08
2	55.5	103.0	106.5	57.7	1.86	1.92	1.04
4	88.6	149.2	140.4	96.5	1.68	1.58	1.09
8	141.2	230.1	187.1	149.8	1.63	1.32	1.06
16	256.9	388.6	305.1	267.3	1.51	1.19	1.04
24	322.7	481.2	378.0	329.5	1.49	1.17	1.02
32	378.8	561.7	460.7	386.7	1.48	1.22	1.02
48	474.1	701.6	579.0	485.1	1.48	1.22	1.02
64	510.3	731.7	614.8	514.0	1.43	1.20	1.01
96	557.1	813.4	863.3	575.3	1.46	1.55	1.03
128	577.7	835.4	888.4	603.4	1.45	1.54	1.04
192	594.2	846.6	901.8	657.4	1.42	1.52	1.11
256	607.0	855.7	929.6	653.5	1.41	1.53	1.08
384	640.3	862.7	986.4	683.7	1.35	1.54	1.07
512	655.8	869.6	1018.0	852.6	1.33	1.55	1.30
768	679.4	888.1	1124.8	915.2	1.31	1.66	1.35
1024	763.8	897.6	1218.7	1034.4	1.18	1.60	1.35
1280	791.8	918.0	1331.6	1155.3	1.16	1.68	1.46
1536	828.9	933.3	1429.3	1408.9	1.13	1.72	1.70
1792	871.1	954.8	1545.1	1544.8	1.10	1.77	1.77
1920	926.6	962.5	1598.1	1761.8	1.04	1.72	1.90
2048	1014.5	967.5	1652.2	1966.4	0.95	1.63	1.94
2176	1086.0	976.3	1716.9	2192.2	0.90	1.58	2.02
2304	1195.9	983.9	1770.7	2392.7	0.82	1.48	2.00
2432	1237.5	1006.0	1823.2	2497.9	0.81	1.47	2.02
2560	1287.9	1014.8	1879.6	2589.5	0.79	1.46	2.01
2816	1313.4	1036.3	1995.2	2629.8	0.79	1.52	2.00
3072	1343.1	1045.0	2096.4	2652.2	0.78	1.56	1.97
3328	1374.7	1068.5	2219.9	2696.1	0.78	1.61	1.96
3584	1422.2	1109.3	2330.2	2764.8	0.78	1.64	1.94
4096	1671.3	1243.3	2674.7	3277.6	0.74	1.60	1.96