[NVIDIA][AgentX] Qwen3.8-Flash-Next B300 SGLang / [NVIDIA][AgentX] Qwen3.8-Flash-Next B300 SGLang

inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml

@@ -1,9 +1,18 @@
 base:
   schema: 2
   name: qwen3.8next-fp4-b300-sglang-agentic
+  identity:
+    model:
+      revision: fc694b54fb0174e0913e6adf86691ef85a4ead47
+    container: {}
   resources:
     gpu_type: b300
     gpus_per_node: 8
+  slurm:
+    time_limit: '6:00:00'
+  health_check:
+    max_attempts: 1440
+    interval_seconds: 5
   frontend:
     type: sglang
     enable_multiple_frontends: false
@@ -17,94 +26,185 @@ base:
       nodes: 1
       workers: 1
       gpus: 1
+      env:
+        PYTHONNOUSERSITE: '1'
+        PYTHONUNBUFFERED: '1'
+        NO_COLOR: '1'
+        # Outlast AIPerf's 300 s client connection pool.
+        SGLANG_TIMEOUT_KEEP_ALIVE: '1800'
+        # Explicitly disable the prohibited draft precision conversion path.
+        SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '0'
+        # Samples verify tokens on the top-k support before temperature and top-p; same distribution.
+        SGLANG_Q35MF_EDGE_SPARSE_VERIFY_SAMPLING: '1'
       args:
-        served-model-name: RadixArk/Qwen3.8-Flash-Next-NVFP4
+        served-model-name: nvidia/Qwen3.8-Flash-Next-NVFP4
+        revision: fc694b54fb0174e0913e6adf86691ef85a4ead47
+        speculative-draft-model-revision: fc694b54fb0174e0913e6adf86691ef85a4ead47
         trust-remote-code: true
+        quantization: modelopt_mixed
+        context-length: 262144
         tensor-parallel-size: 1
+        pipeline-parallel-size: 1
         data-parallel-size: 1
         expert-parallel-size: 1
-        # Quantization is read from the checkpoint; the hybrid GDN
-        # linear-attention layers take their own backends.
+        attention-backend: trtllm_mha
         linear-attn-prefill-backend: flashinfer
         linear-attn-decode-backend: flashinfer
-        # SM100+ flashinfer linear-attn decode requires a bf16 SSM state.
+        linear-attn-verify-backend: triton
         mamba-ssm-dtype: bfloat16
+        mamba-radix-cache-strategy: extra_buffer_lazy
+        mamba-track-interval: 256
+        max-mamba-cache-size: 480
+        mem-fraction-static: 0.88
+        max-running-requests: 96
+        page-size: 64
+        chunked-prefill-size: 16384
+        max-prefill-tokens: 16384
+        moe-runner-backend: flashinfer_trtllm
+        moe-a2a-backend: none
+        disable-shared-experts-fusion: true
         speculative-algorithm: NEXTN
         speculative-num-steps: 3
         speculative-eagle-topk: 1
         speculative-num-draft-tokens: 4
-        reasoning-parser: auto
-        mem-fraction-static: 0.8
-        stream-interval: 50
-        scheduler-recv-interval: 10
-        tokenizer-path: RadixArk/Qwen3.8-Flash-Next-NVFP4
+        speculative-moe-runner-backend: flashinfer_trtllm
+        # The native connector injects model-specific golden AL for throughput;
+        # quality evaluations use real verification. No manual AL override.
+        enable-linear-replayssm-spec: true
+        cuda-graph-backend-prefill: breakable
+        cuda-graph-backend-decode: full
+        cuda-graph-max-bs-decode: 96
+        cuda-graph-bs-decode: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96]
+        kv-cache-dtype: bfloat16
+        speculative-draft-kv-cache-dtype: bfloat16
+        enable-hierarchical-cache: true
+        # One combined packed target/draft KV + Mamba host-pool budget: 298 of the
+        # 299 GB generated allowance, keeping 1 GB of page-alignment headroom.
+        hicache-size: 298
+        hicache-io-backend: kernel
+        hicache-mem-layout: page_first
+        hicache-write-policy: write_back
+        ple-offload-embedding: true
+        ple-offload-backend: pinned
         enable-metrics: true
         enable-cache-report: true
-      env:
-        TORCH_CUDA_ARCH_LIST: '10.0'
-        PYTHONNOUSERSITE: '1'
-        NCCL_NVLS_ENABLE: '1'
-        SGL_ENABLE_JIT_DEEPGEMM: 'false'
-        SGLANG_ENABLE_FLASHINFER_GEMM: 'true'
-        # Outlast AIPerf's 300 s client connection pool.
-        SGLANG_TIMEOUT_KEEP_ALIVE: '1800'
-  # The checkpoint loads from the shared HF cache on Lustre, which took over
-  # 30 minutes on a cold node.
-  health_check:
-    max_attempts: 1440
-    interval_seconds: 5
+        reasoning-parser: auto
+        tokenizer-worker-num: 1
+        scheduler-recv-interval: 1
+        decode-log-interval: 10
+        stream-interval: 20
+        watchdog-timeout: 1800
+        random-seed: 35
   benchmark:
     env:
       AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'
+      # Native single-node binding verifies these against the generated point.
+      TOTAL_CPU_DRAM_GB: '299'
+      KV_OFFLOADING: dram
 
-# One variant per point: admission is 2x CONC and the graph batch is CONC. NEXTN
-# resets an unset admission limit to 48, so it stays explicit.
-override_c1:
+override_tp4_c1:
   roles:
     agg:
+      gpus: 4
       args:
+        <<: &gpu_only_args
+          # null drops the base setting, so SGLang's default applies.
+          enable-hierarchical-cache: null
+          hicache-size: null
+          hicache-io-backend: null
+          hicache-mem-layout: null
+          hicache-write-policy: null
+          mamba-radix-cache-strategy: null
+          mamba-track-interval: null
+          max-mamba-cache-size: null
+          # Dense small sizes: decode batches up to 16 pad by at most one request.
+          cuda-graph-bs-decode: [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96]
+          tokenizer-worker-num: 6
+        tensor-parallel-size: 4
+        expert-parallel-size: 4
+        flashinfer-allreduce-fusion-backend: trtllm
         max-running-requests: 2
-        cuda-graph-max-bs-decode: 1
   benchmark:
     env:
+      <<: &gpu_only_client
+        KV_OFFLOADING: none
+        TOTAL_CPU_DRAM_GB: '0'
       CONC: '1'
 
-override_c4:
+override_c2:
   roles:
     agg:
-      args:
-        max-running-requests: 8
-        cuda-graph-max-bs-decode: 4
+      env:
+        # Graphs the speculative post-verify step (sampling, acceptance, Mamba commit); the rest are its fast paths.
+        SGLANG_ENABLE_EAGLE_POST_VERIFY_CUDA_GRAPH: '1'
+        SGLANG_Q35MF_MKH_N3B: '1'
+        SGLANG_Q35MF_MKH_N3C: '1'
+        SGLANG_Q35MF_BK_VEC_COPY: '1'
+        SGLANG_Q35MF_EDGE_SHM_SYNC: '1'
+        SGLANG_Q35MF_EDGE_PACKED_VERIFY_BCAST: '1'
+        SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH: '1'
+      args: *gpu_only_args
   benchmark:
     env:
-      CONC: '4'
+      <<: *gpu_only_client
+      CONC: '2'
 
-override_c8:
+override_c12:
   roles:
     agg:
-      args:
-        max-running-requests: 16
-        cuda-graph-max-bs-decode: 8
+      args: &gpu_fp8_kv_args
+        <<: *gpu_only_args
+        kv-cache-dtype: fp8_e4m3
+        speculative-draft-kv-cache-dtype: fp8_e4m3
+        # Context length: no decode-time Mamba checkpoints; the AgentX replay never re-sends model output.
+        mamba-track-interval: 262144
+        cuda-graph-bs-decode: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96]
   benchmark:
     env:
-      CONC: '8'
+      <<: *gpu_only_client
+      CONC: '12'
 
-override_c12:
+override_c24:
   roles:
     agg:
       args:
-        max-running-requests: 24
-        cuda-graph-max-bs-decode: 12
+        <<: *gpu_fp8_kv_args
+        max-mamba-cache-size: 960
+        # Every size to 32, so decode batches at this concurrency never pad.
+        cuda-graph-bs-decode: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 40, 48, 56, 64, 72, 80, 88, 96]
   benchmark:
     env:
-      CONC: '12'
+      <<: *gpu_only_client
+      CONC: '24'
+
+override_c48:
+  roles:
+    agg:
+      args: &hicache_fp8_kv_args
+        kv-cache-dtype: fp8_e4m3
+        speculative-draft-kv-cache-dtype: fp8_e4m3
+        # Context length: no decode-time Mamba checkpoints; the AgentX replay never re-sends model output.
+        mamba-track-interval: 262144
+        max-mamba-cache-size: 720
+        cuda-graph-bs-decode: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 88, 96]
+  benchmark:
+    env:
+      CONC: '48'
 
-override_c16:
+override_c80:
   roles:
     agg:
       args:
-        max-running-requests: 32
-        cuda-graph-max-bs-decode: 16
+        <<: *hicache_fp8_kv_args
+        max-mamba-cache-size: 640
+        max-running-requests: 128
+        cuda-graph-max-bs-decode: 128
+        # Every size to 80, so decode batches at this concurrency never pad.
+        cuda-graph-bs-decode: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128]
+        # Cache only the prompt on finish; the AgentX replay never re-sends model output.
+        strip-thinking-cache: true
+        # At most two Mamba states per conversation path; stale interior states skip host write-back.
+        mamba-max-states-per-path: 2
   benchmark:
     env:
-      CONC: '16'
+      CONC: '80'

inferencex-e2e/configs/nvidia-master.yaml

@@ -4822,12 +4822,9 @@ qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto:
           ep: 1
           dp-attn: false
 
-# Qwen3.8-Flash-Next NVFP4 AgentX on B300 via SGLang with native NEXTN MTP.
-# Day-zero recipe; mirrors the B200 arm. TP1 per the cookbook's verified
-# single-node command: 126 GiB of NVFP4 weights fit on one B300.
 qwen3.8next-fp4-b300-sglang-agentic-mtp:
-  image: lmsysorg/sglang:v0.5.21-cu130@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469
-  model: RadixArk/Qwen3.8-Flash-Next-NVFP4
+  image: lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90
+  model: nvidia/Qwen3.8-Flash-Next-NVFP4
   model-prefix: qwen3.8next
   runner: cluster:b300-dsxe
   precision: fp4
@@ -4836,9 +4833,12 @@ qwen3.8next-fp4-b300-sglang-agentic-mtp:
   srt-recipe-dir: qwen3.8next/sglang/b300-fp4-mtp
   scenarios:
     agentic-coding:
+    # HiCache rows take the 3 TB policy share per GPU (299 GB); the recipe's 298 GB keeps alignment headroom.
     - dram-utilization: 0.8
       search-space:
-      - { tp: 1, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16], srt-recipe: agentic.yaml }
+      - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: agentic.yaml }
+      - { tp: 1, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [2, 12, 24], srt-recipe: agentic.yaml }
+      - { tp: 1, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48, 80], srt-recipe: agentic.yaml }
 
 qwen3.5-fp8-h100-sglang-agentic:
   image: lmsysorg/sglang:v0.5.12-cu130

inferencex-e2e/perf-changelog.yaml

@@ -11703,3 +11703,12 @@
     - "Disagg 3 × DEP2 prefill and 1 × DEP16 decode at c1228 (override_3p1d_dep2_dep16_c1228) on nightly-dev-cu13-20261028-586355a8, with the prefill host-DRAM HiCache base recipe and the retired key's c1228 decode sizing: max-running-requests 384, decode CUDA graph batch 24, disaggregation-decode-extra-slots 36."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4679
   append-only: true
+
+- config-keys:
+    - qwen3.8next-fp4-b300-sglang-agentic-mtp
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Update Qwen3.8-Flash-Next FP4 B300 SGLang AgentX to nvidia/Qwen3.8-Flash-Next-NVFP4 on lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8 (digest-pinned)."
+    - "Update the search space to TP4/EP4 at concurrency 1, TP1/EP1 at concurrency 2, 12 and 24, and TP1/EP1 with HiCache at concurrency 48 and 80."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX