amd/deepseek_v4 integration 1/N - 0426

.codespellrc

@@ -1,3 +1,3 @@
 [codespell]
-ignore-words-list = ans, als, hel, boostrap, childs, te, vas, hsa, ment, cann, thi, makro, wil, rouge, PRIS, ather, MIS, medias, allready, inout, nd
+ignore-words-list = ans, als, hel, boostrap, childs, te, vas, hsa, ment, cann, thi, makro, wil, rouge, PRIS, ather, MIS, medias, allready, inout, nd, dOut
 skip = *.json, *.jsonl, *.patch, *.txt, *.lock

.github/workflows/release-pypi-nightly.yml

@@ -48,6 +48,11 @@ jobs:
         run: |
           pip install build wheel setuptools setuptools-scm
 
+      # Needed by setuptools-rust to build the bundled native gRPC extension
+      # (rust/sglang-grpc) when `python -m build` builds the sglang wheel.
+      - name: Install protoc
+        run: sudo bash scripts/ci/utils/install_protoc.sh
+
       - name: Build wheel
         id: build
         run: |

.github/workflows/release-whl-kernel.yml

@@ -50,6 +50,15 @@ jobs:
             runner: arm-kernel-build-node
     runs-on: ${{ matrix.runner }}
     steps:
+      # Self-hosted build nodes retain the workspace across jobs. Prior builds
+      # leave root-owned artifacts under sgl-kernel/build/ that actions/checkout
+      # cannot remove, causing EACCES on rmdir. Wipe them via a throwaway root
+      # container before checkout recreates the workspace.
+      - name: Clean workspace (remove root-owned files from prior runs)
+        run: |
+          docker run --rm -v "${{ github.workspace }}:/workspace" alpine:3 \
+            sh -c 'rm -rf /workspace/..?* /workspace/.[!.]* /workspace/*' || true
+
       - uses: actions/checkout@v4
         with:
           submodules: "recursive"
@@ -69,11 +78,42 @@ jobs:
           BUILD_JOBS: 64
           NVCC_THREADS: 8
 
+      # rename_wheels.sh tags the cu129 wheel's METADATA Version with +cu129
+      # (needed elsewhere — e.g. local install discrimination — see PR #23587),
+      # but PyPI rejects PEP 440 local version labels with HTTP 400. Repack a
+      # PyPI-clean copy with the +cu129 segment stripped into dist-pypi/, and
+      # upload that copy. dist/ is left untouched so the +cu129 wheel still
+      # flows through the upload-artifact -> sgl-project/whl index path below.
+      - name: Strip +cu129 local version for PyPI upload
+        working-directory: sgl-kernel
+        run: |
+          set -eux
+          pip install wheel
+          mkdir -p dist-pypi
+          for w in dist/*.whl; do
+            tmp=$(mktemp -d)
+            python3 -m wheel unpack "$w" --dest "$tmp"
+            unpacked=$(find "$tmp" -mindepth 1 -maxdepth 1 -type d | head -1)
+            info=$(find "$unpacked" -maxdepth 1 -type d -name "*.dist-info" | head -1)
+            meta="$info/METADATA"
+            orig=$(grep '^Version:' "$meta" | head -1 | sed 's/^Version:[[:space:]]*//')
+            new=$(echo "$orig" | sed 's/+cu[0-9]\+$//')
+            if [ "$orig" != "$new" ]; then
+              sed -i "s/^Version:.*/Version: ${new}/" "$meta"
+              old_base=$(basename "$info")
+              new_base="${old_base/${orig}/${new}}"
+              mv "$info" "$(dirname "$info")/${new_base}"
+            fi
+            python3 -m wheel pack "$unpacked" --dest-dir dist-pypi
+            rm -rf "$tmp"
+          done
+          ls -lh dist-pypi/
+
       - name: Upload to PyPI
         working-directory: sgl-kernel
         run: |
           pip install twine
-          python3 -m twine upload --skip-existing dist/* -u __token__ -p ${{ secrets.PYPI_TOKEN_SGLANG_KERNEL }}
+          python3 -m twine upload --skip-existing dist-pypi/* -u __token__ -p ${{ secrets.PYPI_TOKEN_SGLANG_KERNEL }}
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
@@ -149,6 +189,15 @@ jobs:
             runner: arm-kernel-build-node
     runs-on: ${{ matrix.runner }}
     steps:
+      # Self-hosted build nodes retain the workspace across jobs. Prior builds
+      # leave root-owned artifacts under sgl-kernel/build/ that actions/checkout
+      # cannot remove, causing EACCES on rmdir. Wipe them via a throwaway root
+      # container before checkout recreates the workspace.
+      - name: Clean workspace (remove root-owned files from prior runs)
+        run: |
+          docker run --rm -v "${{ github.workspace }}:/workspace" alpine:3 \
+            sh -c 'rm -rf /workspace/..?* /workspace/.[!.]* /workspace/*' || true
+
       - uses: actions/checkout@v4
         with:
           submodules: "recursive"
@@ -235,6 +284,15 @@ jobs:
         python-version: ["3.10"]
         rocm-version: ["700", "720"]
     steps:
+      # Self-hosted build nodes retain the workspace across jobs. Prior builds
+      # leave root-owned artifacts under sgl-kernel/build/ that actions/checkout
+      # cannot remove, causing EACCES on rmdir. Wipe them via a throwaway root
+      # container before checkout recreates the workspace.
+      - name: Clean workspace (remove root-owned files from prior runs)
+        run: |
+          docker run --rm -v "${{ github.workspace }}:/workspace" alpine:3 \
+            sh -c 'rm -rf /workspace/..?* /workspace/.[!.]* /workspace/*' || true
+
       - uses: actions/checkout@v4
         with:
           submodules: "recursive"
@@ -370,6 +428,15 @@ jobs:
         python-version: ["3.10"]
         musa-version: ["43"]
     steps:
+      # Self-hosted build nodes retain the workspace across jobs. Prior builds
+      # leave root-owned artifacts under sgl-kernel/build/ that actions/checkout
+      # cannot remove, causing EACCES on rmdir. Wipe them via a throwaway root
+      # container before checkout recreates the workspace.
+      - name: Clean workspace (remove root-owned files from prior runs)
+        run: |
+          docker run --rm -v "${{ github.workspace }}:/workspace" alpine:3 \
+            sh -c 'rm -rf /workspace/..?* /workspace/.[!.]* /workspace/*' || true
+
       - uses: actions/checkout@v4
         with:
           submodules: "recursive"

docker/Dockerfile

@@ -11,7 +11,7 @@ ARG GRACE_BLACKWELL_DEEPEP_BRANCH=gb200_blog_part_2
 ARG HOPPER_SBO_DEEPEP_COMMIT=9f2fc4b3182a51044ae7ecb6610f7c9c3258c4d6
 ARG DEEPEP_COMMIT=9af0e0d0e74f3577af1979c9b9e1ac2cad0104ee
 ARG BUILD_AND_DOWNLOAD_PARALLEL=8
-ARG SGL_KERNEL_VERSION=0.4.1
+ARG SGL_KERNEL_VERSION=0.4.1.post1
 ARG SGL_VERSION
 ARG USE_LATEST_SGLANG=0
 ARG GDRCOPY_VERSION=2.5.1

docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx

@@ -29,13 +29,13 @@ tag: NEW
       <td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><strong><a href="https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash">DeepSeek-V4-Flash</a></strong></td>
       <td style={{padding: "9px 12px", textAlign: "right", backgroundColor: "rgba(255,255,255,0.05)"}}><strong>284B</strong></td>
       <td style={{padding: "9px 12px", textAlign: "right", backgroundColor: "rgba(255,255,255,0.02)"}}>13B</td>
-      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>single-node serving: B200 / GB300 / H200 on 4 GPUs</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>single-node serving: B200 / GB200 / GB300 / H200 on 4 GPUs</td>
     </tr>
     <tr>
       <td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><strong><a href="https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro">DeepSeek-V4-Pro</a></strong></td>
       <td style={{padding: "9px 12px", textAlign: "right", backgroundColor: "rgba(255,255,255,0.05)"}}><strong>1.6T</strong></td>
       <td style={{padding: "9px 12px", textAlign: "right", backgroundColor: "rgba(255,255,255,0.02)"}}>49B</td>
-      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>high-capacity: B200 8 GPU / GB300 4 GPU / H200 16 GPU (2 nodes)</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>high-capacity: B200 8 GPU / GB200 8 GPU (2 nodes) / GB300 4 GPU / H200 16 GPU (2 nodes)</td>
     </tr>
   </tbody>
 </table>
@@ -88,6 +88,10 @@ Please refer to the [official SGLang installation guide](../../../docs/get-start
       <td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>NVIDIA B200</td>
       <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>lmsysorg/sglang:deepseek-v4-blackwell</code></td>
     </tr>
+    <tr>
+      <td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>NVIDIA GB200</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>lmsysorg/sglang:deepseek-v4-grace-blackwell</code></td>
+    </tr>
     <tr>
       <td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>NVIDIA GB300</td>
       <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>lmsysorg/sglang:deepseek-v4-grace-blackwell</code></td>

docs_new/src/snippets/autoregressive/deepseek-v4-deployment.jsx

@@ -4,6 +4,7 @@ export const DeepSeekV4Deployment = () => {
   //
   //   Hardware (quantization determined by GPU generation):
   //     B200  → FP4 weights, Flash TP=4 / Pro TP=8 single-node
+  //     GB200 → FP4 weights, Flash TP=4 / Pro TP=8 2-node
   //     GB300 → FP4 weights, Flash TP=4 / Pro TP=4 single-node
   //     H200  → FP8 weights, Flash TP=4 / Pro TP=16 2-node
   //   Model variant → HF slug:
@@ -27,6 +28,7 @@ export const DeepSeekV4Deployment = () => {
       items: [
         { id: "b200",  label: "B200 (FP4)",  default: true  },
         { id: "b300",  label: "B300 (FP4)",  default: false  },
+        { id: "gb200", label: "GB200 (FP4)", default: false },
         { id: "gb300", label: "GB300 (FP4)", default: false },
         { id: "h200",  label: "H200 (FP8)",  default: false },
       ],
@@ -138,6 +140,8 @@ export const DeepSeekV4Deployment = () => {
     "b200|big":    { slug: "deepseek-ai/DeepSeek-V4-Pro",   tp: 8,  multinode: false },
     "gb300|small": { slug: "deepseek-ai/DeepSeek-V4-Flash", tp: 4,  multinode: false },
     "gb300|big":   { slug: "deepseek-ai/DeepSeek-V4-Pro",   tp: 4,  multinode: false },
+    "gb200|small": { slug: "deepseek-ai/DeepSeek-V4-Flash", tp: 4,  multinode: false },
+    "gb200|big":   { slug: "deepseek-ai/DeepSeek-V4-Pro",   tp: 8,  multinode: true, nnodes: 2 },
     // H200 needs an FP8-only Instruct ckpt (deepseek-ai's Flash/Pro repos ship
     // FP4-mixed weights that Hopper can't run). sgl-project publishes FP8
     // repackagings for both variants.
@@ -150,6 +154,8 @@ export const DeepSeekV4Deployment = () => {
     "b200|big":    { tp: 8,  multinode: false },
     "gb300|small": { tp: 4,  multinode: false },
     "gb300|big":   { tp: 4,  multinode: false },
+    "gb200|small": { tp: 4,  multinode: false },
+    "gb200|big":   { tp: 8,  multinode: true, nnodes: 2 },
     "h200|small":  { tp: 4,  multinode: false },
     "h200|big":    { tp: 16, multinode: true, nnodes: 2 },
   };
@@ -178,17 +184,29 @@ export const DeepSeekV4Deployment = () => {
     "gb300|small|max-throughput",
     "h200|small|cp",
     "h200|small|pd-disagg",
+    "h200|big|low-latency",
+    "h200|big|balanced",
+    "h200|big|max-throughput",
     "h200|big|pd-disagg",
     "gb300|small|cp",
     "gb300|big|cp",
     "gb300|small|pd-disagg",
     "gb300|big|pd-disagg",
+    "gb200|small|low-latency",
+    "gb200|small|balanced",
+    "gb200|small|max-throughput",
+    "gb200|small|cp",
+    "gb200|big|low-latency",
+    "gb200|big|balanced",
+    "gb200|big|max-throughput",
   ]);
   // Recipes whose command is intentionally not yet provided (e.g. blocked by an
   // upstream limitation). Showing a minimal placeholder is friendlier to users
   // than emitting a commented-out invalid command.
   const TBD_RECIPES = new Set([
     "h200|big|cp",
+    "gb200|small|pd-disagg",
+    "gb200|big|pd-disagg",
   ]);
   const TBD_PLACEHOLDER = "# to be provided";
   const BEING_VERIFIED_NOTE =
@@ -242,18 +260,24 @@ export const DeepSeekV4Deployment = () => {
       h200:  ["SGLANG_DSV4_FP4_EXPERTS=0"],   // allinone _ENV_H200
       b200:  [],                              // _ENV_B200 minus NVSHMEM
       gb300: [],                              // _ENV_GB300
+      // GB200 multinode needs NCCL MNNVL for cross-node NVLink communication.
+      gb200: multinode ? ["NCCL_MNNVL_ENABLE=1", "NCCL_CUMEM_ENABLE=1"] : [],
     }[hardware];
 
     // Recipe-specific env (matches allinone exactly, taking size into account).
     const recipeEnv = [];
     if (recipe === "low-latency") {
-      // H200 big low-latency has extra dispatch-token cap (allinone line 233).
+      // Big low-latency dispatch-token cap.
       if (hardware === "h200" && isBig) {
         recipeEnv.push("SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128");
+      } else if (hardware === "gb200" && isBig) {
+        recipeEnv.push("SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256");
       }
     } else if (recipe === "balanced") {
       if (hardware === "h200") {
-        recipeEnv.push("SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256");
+        recipeEnv.push(isBig
+          ? "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128"
+          : "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256");
       } else {
         // Blackwell: small=1024, big=256 (allinone ternary).
         recipeEnv.push(isBig
@@ -262,7 +286,9 @@ export const DeepSeekV4Deployment = () => {
       }
     } else if (recipe === "max-throughput") {
       if (hardware === "h200") {
-        recipeEnv.push("SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256");
+        recipeEnv.push(isBig
+          ? "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128"
+          : "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256");
       } else {
         recipeEnv.push(isBig
           ? "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256"
@@ -293,6 +319,7 @@ export const DeepSeekV4Deployment = () => {
       // allinone:
       //   H200 small: pure TP + MTP_314
       //   H200 big:   DP-attn + DeepEP + MTP_314 + cg=32 max-run=64 + multi-node + mem-frac 0.82
+      //   GB200 big:  pure TP + multinode + flashinfer_mxfp4 + MTP_314 + mem-frac 0.82 (no DP-attn/DeepEP)
       //   Blackwell:  TP + flashinfer_mxfp4 + MTP_314 + chunked-prefill-size 4096 + autotune-fix
       //               Big Blackwell additionally: mem-frac 0.82
       flags.push(`  --tp ${tp}`);
@@ -308,8 +335,8 @@ export const DeepSeekV4Deployment = () => {
         flags.push("  --moe-runner-backend flashinfer_mxfp4");
       }
       if (hardware === "h200" && isBig) {
-        flags.push("  --cuda-graph-max-bs 32");
-        flags.push("  --max-running-requests 64");
+        flags.push("  --cuda-graph-max-bs 8");
+        flags.push("  --max-running-requests 32");
       }
       // MTP 3/4
       flags.push("  --speculative-algo EAGLE");
@@ -320,7 +347,7 @@ export const DeepSeekV4Deployment = () => {
         flags.push("  --chunked-prefill-size 4096");
         flags.push("  --disable-flashinfer-autotune");
       }
-      if (isBig) flags.push("  --mem-fraction-static 0.82");
+      if (isBig) flags.push("  --mem-fraction-static 0.88");
     } else if (recipe === "balanced") {
       // allinone balanced: TP + DP + DP-attn + DeepEP + MTP_112.
       //   H200 small: cg=128 max-run=128  |  H200 big: cg=128 max-run=128 (same)
@@ -335,8 +362,17 @@ export const DeepSeekV4Deployment = () => {
       flags.push("  --speculative-num-steps 1");
       flags.push("  --speculative-eagle-topk 1");
       flags.push("  --speculative-num-draft-tokens 2");
-      if (isBig) flags.push("  --mem-fraction-static 0.82");
-      if (hardware === "h200") {
+      if (hardware === "h200" && isBig) {
+        flags.push("  --mem-fraction-static 0.88");
+      } else if (isBig && hardware === "gb200") {
+        flags.push("  --mem-fraction-static 0.78");
+      } else if (isBig) {
+        flags.push("  --mem-fraction-static 0.82");
+      }
+      if (hardware === "h200" && isBig) {
+        flags.push("  --cuda-graph-max-bs 8");
+        flags.push("  --max-running-requests 32");
+      } else if (hardware === "h200") {
         flags.push("  --cuda-graph-max-bs 128");
         flags.push("  --max-running-requests 128");
       } else if (isBig && hardware === "b200") {
@@ -345,6 +381,9 @@ export const DeepSeekV4Deployment = () => {
       } else if (isBig && hardware === "gb300") {
         flags.push("  --cuda-graph-max-bs 128");
         flags.push("  --max-running-requests 256");
+      } else if (isBig && hardware === "gb200") {
+        flags.push("  --cuda-graph-max-bs 64");
+        flags.push("  --max-running-requests 128");
       }
       // allinone H200 gates DEEPEP_LARGE_SMS_FLAG on !multinode — only H200 big
       // is multi-node; all Blackwell cells get the flag unconditionally.
@@ -359,7 +398,13 @@ export const DeepSeekV4Deployment = () => {
       flags.push("  --enable-dp-attention");
       if (multinode) flags.push(...multiNodeFlags(nnodes));
       flags.push("  --moe-a2a-backend deepep");
-      if (isBig) flags.push("  --mem-fraction-static 0.82");
+      if (hardware === "h200" && isBig) {
+        flags.push("  --mem-fraction-static 0.88");
+      } else if (isBig && hardware === "gb200") {
+        flags.push("  --mem-fraction-static 0.78");
+      } else if (isBig) {
+        flags.push("  --mem-fraction-static 0.82");
+      }
       if (hardware === "h200") {
         flags.push("  --cuda-graph-max-bs 128");
         flags.push("  --max-running-requests 256");
@@ -369,6 +414,9 @@ export const DeepSeekV4Deployment = () => {
       } else if (isBig && hardware === "gb300") {
         flags.push("  --cuda-graph-max-bs 128");
         flags.push("  --max-running-requests 256");
+      } else if (isBig && hardware === "gb200") {
+        flags.push("  --cuda-graph-max-bs 64");
+        flags.push("  --max-running-requests 256");
       }
       if (!multinode) flags.push(DEEPEP_LARGE_SMS_FLAG);
     } else if (recipe === "cp") {
@@ -416,7 +464,18 @@ export const DeepSeekV4Deployment = () => {
     const envAll = [...HW_ENV, ...recipeEnv, ...COMMON_ENV];
     const envBlock = envAll.length ? envAll.join(" \\\n") + " \\\n" : "";
     const base = `${envBlock}sglang serve \\\n${flags.join(" \\\n")}`;
-    const withMultinode = multinode ? prependMultiNodeNote(base, nnodes) : base;
+    // GB200 multinode may need machine-specific NVSHMEM / Gloo env vars;
+    // emit them as commented hints above the env block so users know to check.
+    let cmd = base;
+    if (hardware === "gb200" && multinode) {
+      cmd =
+        `# The following env vars may be needed depending on your cluster:\n` +
+        `#   GLOO_SOCKET_IFNAME=<your-nic>\n` +
+        `#   NVSHMEM_ENABLE_NIC_PE_MAPPING=1\n` +
+        `#   NVSHMEM_HCA_LIST=<your-hca-list>\n` +
+        cmd;
+    }
+    const withMultinode = multinode ? prependMultiNodeNote(cmd, nnodes) : cmd;
     const verifyKey = `${hardware}|${modelSize}|${recipe}`;
     if (TBD_RECIPES.has(verifyKey)) return TBD_PLACEHOLDER;
     return VERIFIED_RECIPES.has(verifyKey)
@@ -447,14 +506,15 @@ export const DeepSeekV4Deployment = () => {
     const specKey = `${hardware}|${modelSize}`;
     const { tp: pdTp, multinode, nnodes } = PD_TP_SPEC[specKey];
     const slug = HW_SIZE_SPEC[specKey].slug;
-    const ibDevice = { h200: "mlx5_0", b200: "mlx5_7", gb300: "" }[hardware];
+    const ibDevice = { h200: "mlx5_0", b200: "mlx5_7", gb300: "", gb200: "" }[hardware];
     const isGB300 = hardware === "gb300";
-    const isBlackwell = hardware === "b200" || isGB300;
+    const isBlackwell = hardware === "b200" || hardware === "gb200" || isGB300;
 
     const HW_ENV = {
       h200:  ["SGLANG_DSV4_FP4_EXPERTS=0"],
       b200:  [],
       gb300: [],
+      gb200: [],
     }[hardware];
     // Whitelist #5: only SGLANG_MOONCAKE_CUSTOM_MEM_POOL kept; MC_FORCE_MNNVL /
     // NCCL_MNNVL_ENABLE / NCCL_CUMEM_ENABLE may also be needed depending on the

python/pyproject.toml

@@ -59,7 +59,7 @@ dependencies = [
   "sentencepiece",
   "setproctitle",
   "flash-attn-4>=4.0.0b9",
-  "sglang-kernel==0.4.1",
+  "sglang-kernel==0.4.1.post1",
   "soundfile==0.13.1",
   "tiktoken",
   "timm==1.0.16",