feat: SGLang DGD recipe for DeepSeek-V4-Flash on B200

components/src/dynamo/frontend/sglang_prepost.py

@@ -174,7 +174,7 @@ def build_tool_call_guided_decoding(
         )
         constraint = parser.get_structure_constraint(
             tool_choice,
-            parallel_tool_calls=parallel_tool_calls,
+            # parallel_tool_calls=parallel_tool_calls,
         )
 
     if isinstance(constraint, tuple) and len(constraint) == 2:

components/src/dynamo/sglang/request_handlers/llm/decode_handler.py

@@ -308,7 +308,7 @@ async def generate(
                 **input_param,
                 sampling_params=sampling_params,
                 stream=True,
-                return_routed_experts=return_routed_experts,
+                # return_routed_experts=return_routed_experts,
                 bootstrap_host=bootstrap_info["bootstrap_host"],
                 bootstrap_port=bootstrap_info["bootstrap_port"],
                 bootstrap_room=bootstrap_info["bootstrap_room"],
@@ -346,7 +346,7 @@ async def generate(
                 video_data=video_data,
                 sampling_params=sampling_params,
                 stream=True,
-                return_routed_experts=return_routed_experts,
+                # return_routed_experts=return_routed_experts,
                 external_trace_header=trace_header,
                 rid=trace_id,
                 data_parallel_rank=dp_rank,

recipes/deepseek-v4-flash/sglang/Dockerfile.dsv4-sglang

@@ -0,0 +1,103 @@
+# DeepSeek-V4-Flash SGLang + Dynamo runtime image
+#
+# Multi-stage build:
+#   1. wheel_builder: builds dynamo runtime wheel from source (includes V4 parsers)
+#   2. final: layers dynamo onto sglang base
+#
+# Build (from repo root):
+#   docker build -f recipes/deepseek-v4-flash/sglang/Dockerfile.dsv4-sglang \
+#     -t nvcr.io/nvidian/dynamo-dev/kprashanth:sglang-dsv4-<hash> .
+
+ARG DSV4_BASE_IMAGE=lmsysorg/sglang:deepseek-v4-blackwell
+ARG DYNAMO_DONOR_IMAGE=nvcr.io/nvidian/dynamo-dev/biswa:vllm-runtime-beaafcba0
+
+# ---------- Stage 1: Build dynamo wheels from source ----------
+FROM quay.io/pypa/manylinux_2_28_x86_64 AS wheel_builder
+
+RUN yum install -y openssl-devel clang-devel && yum clean all
+
+# Install modern protoc (manylinux's protobuf-compiler is too old)
+RUN curl -sL https://github.com/protocolbuffers/protobuf/releases/download/v28.3/protoc-28.3-linux-x86_64.zip \
+    -o /tmp/protoc.zip && unzip -o /tmp/protoc.zip -d /usr/local && rm /tmp/protoc.zip
+
+# Install Rust
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ENV PATH=/root/.cargo/bin:${PATH}
+
+# Install maturin
+ENV PATH=/opt/python/cp312-cp312/bin:${PATH}
+RUN pip install maturin
+
+# Copy dynamo source
+COPY lib /workspace/lib
+COPY Cargo.toml Cargo.lock README.md LICENSE /workspace/
+
+WORKDIR /workspace/lib/bindings/python
+RUN maturin build --release --interpreter /opt/python/cp312-cp312/bin/python3 \
+    -o /opt/dynamo/wheelhouse
+
+# Build the pure-python ai-dynamo package
+COPY pyproject.toml hatch_build.py /workspace/
+COPY deploy /workspace/deploy
+COPY components /workspace/components
+WORKDIR /workspace
+RUN pip wheel --no-deps -w /opt/dynamo/wheelhouse .
+
+# ---------- Stage 2: Donor for infra binaries ----------
+FROM ${DYNAMO_DONOR_IMAGE} AS dynamo-donor
+
+# ---------- Stage 3: Final image ----------
+FROM ${DSV4_BASE_IMAGE}
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# RDMA / IB deps for NIXL UCX transport
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates curl jq \
+    libibverbs1 rdma-core ibverbs-utils libibumad3 libnuma1 librdmacm1 ibverbs-providers \
+    && rm -rf /var/lib/apt/lists/*
+
+# nats-server + etcd from donor
+COPY --from=dynamo-donor /usr/bin/nats-server /usr/bin/nats-server
+COPY --from=dynamo-donor /usr/local/bin/etcd /usr/local/bin/etcd
+ENV PATH=/usr/local/bin/etcd:${PATH}
+
+# UCX from donor
+COPY --from=dynamo-donor /usr/local/ucx /usr/local/ucx
+ENV PATH=/usr/local/ucx/bin:${PATH}
+
+# NIXL from donor
+COPY --from=dynamo-donor /opt/nvidia/nvda_nixl /opt/nvidia/nvda_nixl
+ENV NIXL_PREFIX=/opt/nvidia/nvda_nixl \
+    NIXL_LIB_DIR=/opt/nvidia/nvda_nixl/lib \
+    NIXL_PLUGIN_DIR=/opt/nvidia/nvda_nixl/lib/nixl_plugins
+ENV LD_LIBRARY_PATH=/opt/nvidia/nvda_nixl/lib:/usr/local/ucx/lib:${LD_LIBRARY_PATH}
+
+# NIXL wheel from donor (pre-built, architecture-specific)
+COPY --from=dynamo-donor /opt/dynamo/wheelhouse/nixl*.whl /opt/dynamo/wheelhouse/
+
+# Dynamo wheels from our source build (includes V4 parsers + routed_experts fix)
+COPY --from=wheel_builder /opt/dynamo/wheelhouse/ai_dynamo_runtime*.whl /opt/dynamo/wheelhouse/
+COPY --from=wheel_builder /opt/dynamo/wheelhouse/ai_dynamo-*.whl /opt/dynamo/wheelhouse/
+
+RUN pip install --no-cache-dir --force-reinstall --no-deps \
+    /opt/dynamo/wheelhouse/ai_dynamo_runtime*.whl \
+    /opt/dynamo/wheelhouse/ai_dynamo-*.whl && \
+    pip install --no-cache-dir --no-deps \
+    /opt/dynamo/wheelhouse/nixl*.whl 2>/dev/null || true && \
+    python3 -c "from dynamo._core import get_tool_parser_names; assert 'deepseek_v4' in get_tool_parser_names(), 'V4 parser missing!'; print('V4 parser verified in wheel')"
+
+# Dynamo Python components (frontend, sglang worker, common) — from our source
+COPY components/src/dynamo /workspace/components/src/dynamo
+
+# Fix: sglang repo dir at /workspace/sglang shadows the Python package
+ENV PYTHONPATH=/workspace/sglang/python:/workspace/components/src:${PYTHONPATH}
+
+# DeepGEMM JIT env vars
+ENV SGLANG_JIT_DEEPGEMM_PRECOMPILE=0 \
+    SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1
+
+WORKDIR /workspace
+
+ENTRYPOINT []
+CMD ["bash"]

recipes/deepseek-v4-flash/sglang/sglang-dgd.yaml

@@ -0,0 +1,120 @@
+# DeepSeek-V4-Flash SGLang DynamoGraphDeployment
+# Aggregated mode (no P/D disagg), B200x4 TP4, MXFP4 MoE, EAGLE MTP 3/4
+#
+# Deploy:
+#   kubectl apply -f sglang-dgd.yaml -n <namespace>
+#
+# Test:
+#   kubectl port-forward -n <namespace> svc/sglang-dsv4-flash-frontend 8000:8000
+#   curl http://localhost:8000/v1/models
+#   curl http://localhost:8000/v1/chat/completions -H 'Content-Type: application/json' \
+#     -d '{"model":"deepseek-ai/DeepSeek-V4-Flash","messages":[{"role":"user","content":"Hello"}]}'
+
+apiVersion: nvidia.com/v1alpha1
+kind: DynamoGraphDeployment
+metadata:
+  name: sglang-dsv4-flash
+spec:
+  services:
+    Frontend:
+      componentType: frontend
+      replicas: 1
+      volumeMounts:
+        - name: shared-model-cache
+          mountPoint: /models
+      extraPodSpec:
+        mainContainer:
+          image: nvcr.io/nvidian/dynamo-dev/kprashanth:sglang-dsv4-a91b278ca1
+          imagePullPolicy: Always
+          env:
+            - name: HF_HOME
+              value: /models
+            - name: HF_HUB_OFFLINE
+              value: "1"
+          startupProbe:
+            httpGet:
+              path: /health
+              port: 8000
+            periodSeconds: 10
+            timeoutSeconds: 10
+            failureThreshold: 360
+
+    decode:
+      componentType: worker
+      subComponentType: decode
+      replicas: 1
+      resources:
+        limits:
+          gpu: "4"
+      volumeMounts:
+        - name: shared-model-cache
+          mountPoint: /models
+      sharedMemory:
+        size: 200Gi
+      extraPodSpec:
+        nodeSelector:
+          nvidia.com/gpu.product: NVIDIA-B200
+        tolerations:
+          - key: nvidia.com/gpu
+            operator: Equal
+            value: "true"
+            effect: NoSchedule
+        mainContainer:
+          image: nvcr.io/nvidian/dynamo-dev/kprashanth:sglang-dsv4-a91b278ca1
+          imagePullPolicy: Always
+          workingDir: /workspace
+          command:
+            - python3
+            - -m
+            - dynamo.sglang
+          args:
+            - --model-path
+            - deepseek-ai/DeepSeek-V4-Flash
+            - --served-model-name
+            - deepseek-ai/DeepSeek-V4-Flash
+            - --trust-remote-code
+            - --tp
+            - "4"
+            - --moe-runner-backend
+            - flashinfer_mxfp4
+            - --speculative-algo
+            - EAGLE
+            - --speculative-num-steps
+            - "3"
+            - --speculative-eagle-topk
+            - "1"
+            - --speculative-num-draft-tokens
+            - "4"
+            - --chunked-prefill-size
+            - "4096"
+            - --disable-flashinfer-autotune
+            - --dyn-tool-call-parser
+            - deepseek_v4
+            - --dyn-reasoning-parser
+            - deepseek_v4
+          env:
+            - name: HF_HOME
+              value: /models
+            - name: HF_HUB_OFFLINE
+              value: "1"
+            - name: CUDA_VISIBLE_DEVICES
+              value: "0,1,2,3,4,5,6,7"
+            - name: SGLANG_JIT_DEEPGEMM_PRECOMPILE
+              value: "0"
+            - name: SGLANG_JIT_DEEPGEMM_FAST_WARMUP
+              value: "1"
+            - name: NCCL_CUMEM_ENABLE
+              value: "1"
+            - name: GLOO_SOCKET_IFNAME
+              value: eth0
+          startupProbe:
+            httpGet:
+              path: /health
+              port: 9090
+            periodSeconds: 10
+            timeoutSeconds: 10
+            failureThreshold: 360
+
+  pvcs:
+    - name: shared-model-cache
+      create: false