Repository navigation
docs(cookbook): add AMD MI300X/MI325X/MI355X support for GLM-5.2 #28471
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
zijiexia
merged 12 commits into
sgl-project:main
from
andyluo7:feat/glm-5.2-amd-cookbook
Jun 30, 2026
Merged
Changes from 1 commit
Commits
Show all changes
12 commits
Select commit
Hold shift + click to select a range
29a6648
docs(cookbook): add AMD MI300X/MI325X/MI355X support for GLM-5.2
andyluo7 3530557
docs(cookbook): address GLM-5.2 AMD review — disable MTP, differentia…
andyluo7 1e4cb7e
docs(cookbook): warn GLM-5.2 gfx950 block-FP8 accuracy bug; mark AMD …
andyluo7 fa56770
docs(cookbook): link upstream issue sgl-project/sglang#28685 in gfx95…
andyluo7 1ad78e4
Merge branch 'main' into feat/glm-5.2-amd-cookbook
zijiexia c070fe1
docs(cookbook): cite upstream CK fix ROCm/rocm-libraries#8639 in gfx9…
andyluo7 075566e
Fix description and update accuracy warning in GLM-5.2
zijiexia 05d1c29
docs(cookbook): mark GLM-5.2 gfx950 block-FP8 fixed; pin 0618 image, …
andyluo7 b1ec59f
Merge remote-tracking branch 'origin/main' into feat/glm-5.2-amd-cook…
andyluo7 29c87c7
docs(cookbook): add MI355X FP8 benchmarks; verify all 3 GLM-5.2 AMD s…
andyluo7 db73c2f
docs(cookbook): restore GLM-5.2 AMD supportedHardware + MTP/CP disabl…
zijiexia 6d3772c
Merge branch 'main' of https://github.com/sgl-project/sglang into fea…
zijiexia File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,6 +6,7 @@ export const config = { | |
|
|
||
| supportedHardware: [ | ||
| "h200", "b200", "gb300", "b300", | ||
| "mi355x", "mi325x", "mi300x", | ||
| ], | ||
|
|
||
| // Single released checkpoint — no size/mode split. | ||
|
|
@@ -90,6 +91,9 @@ sgl-eval run aime25 \\ | |
| b200: "lmsysorg/sglang:latest", | ||
| gb300: "lmsysorg/sglang:latest", | ||
| b300: "lmsysorg/sglang:latest", | ||
| mi355x: "lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260616", | ||
| mi325x: "lmsysorg/sglang-rocm:v0.5.13.post1-rocm700-mi30x-20260616", | ||
| mi300x: "lmsysorg/sglang-rocm:v0.5.13.post1-rocm700-mi30x-20260616", | ||
| }, | ||
|
|
||
| github: { | ||
|
|
@@ -106,7 +110,7 @@ sgl-eval run aime25 \\ | |
| knobs: [ | ||
| { id: "tp", label: "TP", values: [null, 4, 8] }, | ||
| { id: "cp", label: "CP (DSA prefill)", values: [null, 1, 2, 4, 8], | ||
| disable: { hw: ["b200", "gb300", "b300"] }, | ||
| disable: { hw: ["b200", "gb300", "b300", "mi355x", "mi325x", "mi300x"] }, | ||
| disableReason: "DSA prefill Context Parallel is verified on Hopper (H200); the Blackwell sm100 DSA-CP FP8 rope kernel is not yet adapted." }, | ||
| { id: "dpAttn", label: "DP-Attention", | ||
| values: [null, false, 4, 8], | ||
|
|
@@ -628,5 +632,299 @@ sgl-eval run aime25 \\ | |
| "--port {{PORT}}", | ||
| ], | ||
| }, | ||
|
|
||
| // ==================================================================== | ||
| // AMD MI300X / MI325X / MI355X (ROCm) — TP8, DSA tilelang backend. | ||
| // No MTP: EAGLE spec-decode kernel does not yet build for gfx950. | ||
| // Mirrors the GLM-5.1 ROCm recipe (model path swapped). | ||
| // ==================================================================== | ||
| { | ||
| match: { hw: "mi355x", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, | ||
| verified: true, | ||
| env: [], | ||
| flags: [ | ||
| "--model-path {{MODEL_NAME}}", | ||
| "--tp 8", | ||
| "--dsa-prefill-backend tilelang", | ||
| "--dsa-decode-backend tilelang", | ||
| "--chunked-prefill-size 131072", | ||
| "--mem-fraction-static 0.80", | ||
| "--watchdog-timeout 1200", | ||
| "--host {{HOST_IP}}", | ||
| "--port {{PORT}}", | ||
| ], | ||
| }, | ||
| { | ||
| match: { hw: "mi355x", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, | ||
| verified: true, | ||
| env: [], | ||
| flags: [ | ||
| "--model-path {{MODEL_NAME}}", | ||
| "--tp 8", | ||
| "--dsa-prefill-backend tilelang", | ||
| "--dsa-decode-backend tilelang", | ||
| "--chunked-prefill-size 131072", | ||
| "--mem-fraction-static 0.80", | ||
| "--watchdog-timeout 1200", | ||
| "--host {{HOST_IP}}", | ||
| "--port {{PORT}}", | ||
| ], | ||
| }, | ||
| { | ||
| match: { hw: "mi355x", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, | ||
| verified: true, | ||
| env: [], | ||
| flags: [ | ||
| "--model-path {{MODEL_NAME}}", | ||
| "--tp 8", | ||
| "--dsa-prefill-backend tilelang", | ||
| "--dsa-decode-backend tilelang", | ||
| "--chunked-prefill-size 131072", | ||
| "--mem-fraction-static 0.80", | ||
| "--watchdog-timeout 1200", | ||
| "--host {{HOST_IP}}", | ||
| "--port {{PORT}}", | ||
| ], | ||
| }, | ||
| { | ||
| match: { hw: "mi355x", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, | ||
| verified: false, | ||
| env: [], | ||
| flags: [ | ||
| "--model-path {{MODEL_NAME}}", | ||
| "--tp 8", | ||
| "--dsa-prefill-backend tilelang", | ||
| "--dsa-decode-backend tilelang", | ||
| "--chunked-prefill-size 131072", | ||
| "--mem-fraction-static 0.80", | ||
| "--watchdog-timeout 1200", | ||
| "--host {{HOST_IP}}", | ||
| "--port {{PORT}}", | ||
| ], | ||
| }, | ||
| { | ||
| match: { hw: "mi355x", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" }, | ||
| verified: false, | ||
| env: [], | ||
| flags: [ | ||
| "--model-path {{MODEL_NAME}}", | ||
| "--tp 8", | ||
| "--dsa-prefill-backend tilelang", | ||
| "--dsa-decode-backend tilelang", | ||
| "--chunked-prefill-size 131072", | ||
| "--mem-fraction-static 0.80", | ||
| "--watchdog-timeout 1200", | ||
| "--host {{HOST_IP}}", | ||
| "--port {{PORT}}", | ||
| ], | ||
| }, | ||
| { | ||
| match: { hw: "mi355x", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, | ||
| verified: false, | ||
| env: [], | ||
| flags: [ | ||
| "--model-path {{MODEL_NAME}}", | ||
| "--tp 8", | ||
| "--dsa-prefill-backend tilelang", | ||
| "--dsa-decode-backend tilelang", | ||
| "--chunked-prefill-size 131072", | ||
| "--mem-fraction-static 0.80", | ||
| "--watchdog-timeout 1200", | ||
| "--host {{HOST_IP}}", | ||
| "--port {{PORT}}", | ||
| ], | ||
| }, | ||
| { | ||
| match: { hw: "mi325x", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, | ||
| verified: false, | ||
| env: [], | ||
| flags: [ | ||
| "--model-path {{MODEL_NAME}}", | ||
| "--tp 8", | ||
| "--dsa-prefill-backend tilelang", | ||
| "--dsa-decode-backend tilelang", | ||
| "--chunked-prefill-size 131072", | ||
| "--mem-fraction-static 0.80", | ||
| "--watchdog-timeout 1200", | ||
| "--host {{HOST_IP}}", | ||
| "--port {{PORT}}", | ||
| ], | ||
| }, | ||
| { | ||
| match: { hw: "mi325x", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, | ||
| verified: false, | ||
| env: [], | ||
| flags: [ | ||
| "--model-path {{MODEL_NAME}}", | ||
| "--tp 8", | ||
| "--dsa-prefill-backend tilelang", | ||
| "--dsa-decode-backend tilelang", | ||
| "--chunked-prefill-size 131072", | ||
| "--mem-fraction-static 0.80", | ||
| "--watchdog-timeout 1200", | ||
| "--host {{HOST_IP}}", | ||
| "--port {{PORT}}", | ||
| ], | ||
| }, | ||
| { | ||
| match: { hw: "mi325x", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, | ||
| verified: false, | ||
| env: [], | ||
| flags: [ | ||
| "--model-path {{MODEL_NAME}}", | ||
| "--tp 8", | ||
| "--dsa-prefill-backend tilelang", | ||
| "--dsa-decode-backend tilelang", | ||
| "--chunked-prefill-size 131072", | ||
| "--mem-fraction-static 0.80", | ||
| "--watchdog-timeout 1200", | ||
| "--host {{HOST_IP}}", | ||
| "--port {{PORT}}", | ||
| ], | ||
| }, | ||
| { | ||
| match: { hw: "mi325x", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, | ||
| verified: false, | ||
| env: [], | ||
| flags: [ | ||
| "--model-path {{MODEL_NAME}}", | ||
| "--tp 8", | ||
| "--dsa-prefill-backend tilelang", | ||
| "--dsa-decode-backend tilelang", | ||
| "--chunked-prefill-size 131072", | ||
| "--mem-fraction-static 0.80", | ||
| "--watchdog-timeout 1200", | ||
| "--host {{HOST_IP}}", | ||
| "--port {{PORT}}", | ||
| ], | ||
| }, | ||
| { | ||
| match: { hw: "mi325x", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" }, | ||
| verified: false, | ||
| env: [], | ||
| flags: [ | ||
| "--model-path {{MODEL_NAME}}", | ||
| "--tp 8", | ||
| "--dsa-prefill-backend tilelang", | ||
| "--dsa-decode-backend tilelang", | ||
| "--chunked-prefill-size 131072", | ||
| "--mem-fraction-static 0.80", | ||
| "--watchdog-timeout 1200", | ||
| "--host {{HOST_IP}}", | ||
| "--port {{PORT}}", | ||
| ], | ||
| }, | ||
| { | ||
| match: { hw: "mi325x", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, | ||
| verified: false, | ||
| env: [], | ||
| flags: [ | ||
| "--model-path {{MODEL_NAME}}", | ||
| "--tp 8", | ||
| "--dsa-prefill-backend tilelang", | ||
| "--dsa-decode-backend tilelang", | ||
| "--chunked-prefill-size 131072", | ||
| "--mem-fraction-static 0.80", | ||
| "--watchdog-timeout 1200", | ||
| "--host {{HOST_IP}}", | ||
| "--port {{PORT}}", | ||
| ], | ||
| }, | ||
| { | ||
| match: { hw: "mi300x", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, | ||
| verified: false, | ||
| env: [], | ||
| flags: [ | ||
| "--model-path {{MODEL_NAME}}", | ||
| "--tp 8", | ||
| "--dsa-prefill-backend tilelang", | ||
| "--dsa-decode-backend tilelang", | ||
| "--chunked-prefill-size 131072", | ||
| "--mem-fraction-static 0.80", | ||
| "--watchdog-timeout 1200", | ||
| "--host {{HOST_IP}}", | ||
| "--port {{PORT}}", | ||
| ], | ||
| }, | ||
| { | ||
| match: { hw: "mi300x", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, | ||
| verified: false, | ||
| env: [], | ||
| flags: [ | ||
| "--model-path {{MODEL_NAME}}", | ||
| "--tp 8", | ||
| "--dsa-prefill-backend tilelang", | ||
| "--dsa-decode-backend tilelang", | ||
| "--chunked-prefill-size 131072", | ||
| "--mem-fraction-static 0.80", | ||
| "--watchdog-timeout 1200", | ||
| "--host {{HOST_IP}}", | ||
| "--port {{PORT}}", | ||
| ], | ||
| }, | ||
| { | ||
| match: { hw: "mi300x", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, | ||
| verified: false, | ||
| env: [], | ||
| flags: [ | ||
| "--model-path {{MODEL_NAME}}", | ||
| "--tp 8", | ||
| "--dsa-prefill-backend tilelang", | ||
| "--dsa-decode-backend tilelang", | ||
| "--chunked-prefill-size 131072", | ||
| "--mem-fraction-static 0.80", | ||
| "--watchdog-timeout 1200", | ||
| "--host {{HOST_IP}}", | ||
| "--port {{PORT}}", | ||
| ], | ||
| }, | ||
| { | ||
| match: { hw: "mi300x", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Does the bf16 fit in single node mi300x? might need multinodes |
||
| verified: false, | ||
| env: [], | ||
| flags: [ | ||
| "--model-path {{MODEL_NAME}}", | ||
| "--tp 8", | ||
| "--dsa-prefill-backend tilelang", | ||
| "--dsa-decode-backend tilelang", | ||
| "--chunked-prefill-size 131072", | ||
| "--mem-fraction-static 0.80", | ||
| "--watchdog-timeout 1200", | ||
| "--host {{HOST_IP}}", | ||
| "--port {{PORT}}", | ||
| ], | ||
| }, | ||
| { | ||
| match: { hw: "mi300x", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" }, | ||
| verified: false, | ||
| env: [], | ||
| flags: [ | ||
| "--model-path {{MODEL_NAME}}", | ||
| "--tp 8", | ||
| "--dsa-prefill-backend tilelang", | ||
| "--dsa-decode-backend tilelang", | ||
| "--chunked-prefill-size 131072", | ||
| "--mem-fraction-static 0.80", | ||
| "--watchdog-timeout 1200", | ||
| "--host {{HOST_IP}}", | ||
| "--port {{PORT}}", | ||
| ], | ||
| }, | ||
| { | ||
| match: { hw: "mi300x", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, | ||
| verified: false, | ||
| env: [], | ||
| flags: [ | ||
| "--model-path {{MODEL_NAME}}", | ||
| "--tp 8", | ||
| "--dsa-prefill-backend tilelang", | ||
| "--dsa-decode-backend tilelang", | ||
| "--chunked-prefill-size 131072", | ||
| "--mem-fraction-static 0.80", | ||
| "--watchdog-timeout 1200", | ||
| "--host {{HOST_IP}}", | ||
| "--port {{PORT}}", | ||
| ], | ||
| }, | ||
| ], | ||
| }; | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Can we add disable reason for AMD?