diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/gb300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/gb300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..b27a420d2f --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/gb300-fp4-mtp/agentic.yaml @@ -0,0 +1,73 @@ +base: + schema: 2 + name: glm5.3flash-fp4-gb300-sglang-agentic-mtp + identity: + model: + revision: 1d00738646cbb0c6d0a9eb403dac5f366f1473a7 + container: {} + frameworks: + sglang: 0.5.22 + resources: + gpu_type: gb300 + gpus_per_node: 4 + slurm: + time_limit: '4:00:00' + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + args: + served-model-name: nvidia/GLM-5.3-Flash-NVFP4 + tensor-parallel-size: 4 + expert-parallel-size: 1 + quantization: modelopt_fp4 + context-length: 1048576 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: EAGLE + speculative-num-steps: 5 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 6 + reasoning-parser: auto + tool-call-parser: auto + watchdog-timeout: 1800 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + HF_HUB_OFFLINE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + health_check: + max_attempts: 1440 + interval_seconds: 5 + benchmark: + env: + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +override_gpu_kv: + benchmark: + env: + KV_OFFLOADING: none + +override_hicache: + roles: + agg: + args: + enable-hierarchical-cache: true + # The 0.8 default sizes the host pool at about 1.0x the device KV pool on GB300. + hicache-host-memory-fraction: 0.90 + benchmark: + env: + KV_OFFLOADING: dram diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/gb300-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/gb300-fp8-mtp/agentic.yaml new file mode 100644 index 0000000000..17f27639d4 --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3flash/sglang/gb300-fp8-mtp/agentic.yaml @@ -0,0 +1,72 @@ +base: + schema: 2 + name: glm5.3flash-fp8-gb300-sglang-agentic-mtp + identity: + model: + revision: 46a050bec13002ad29ee795c16e61cf4fa3c4a9c + container: {} + frameworks: + sglang: 0.5.22 + resources: + gpu_type: gb300 + gpus_per_node: 4 + slurm: + time_limit: '4:00:00' + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + args: + served-model-name: zai-org/GLM-5.3-Flash + tensor-parallel-size: 4 + expert-parallel-size: 1 + context-length: 1048576 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: EAGLE + speculative-num-steps: 5 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 6 + reasoning-parser: auto + tool-call-parser: auto + watchdog-timeout: 1800 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + HF_HUB_OFFLINE: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + health_check: + max_attempts: 1440 + interval_seconds: 5 + benchmark: + env: + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +override_gpu_kv: + benchmark: + env: + KV_OFFLOADING: none + +override_hicache: + roles: + agg: + args: + enable-hierarchical-cache: true + # The 0.8 default sizes the host pool at about 1.0x the device KV pool on GB300. + hicache-host-memory-fraction: 0.90 + benchmark: + env: + KV_OFFLOADING: dram diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 9082960a32..a31011febd 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5712,6 +5712,41 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: tp: 8 ep: 8 dp-attn: true + +glm5.3flash-fp8-gb300-sglang-agentic-mtp: + image: lmsysorg/sglang:v0.5.22-cu130@sha256:e3a4f06001a29542fc4a8638536d9ffc5eb9c9942b08603c09b19a56eae4a1ae + model: zai-org/GLM-5.3-Flash + model-prefix: glm5.3flash + runner: cluster:gb300-nv + precision: fp8 + framework: sglang + multinode: false + srt-recipe-dir: glm5.3flash/sglang/gb300-fp8-mtp + scenarios: + agentic-coding: + # HiCache auto-sizes at fraction 0.90; run 42026173061 measured 708.7-722.9 GB per node (4 ranks) of 964.7 GB. + - dram-utilization: 0.7495 + search-space: + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 24, 32], srt-recipe: "agentic.yaml:override_gpu_kv" } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [40, 44, 48], srt-recipe: "agentic.yaml:override_hicache" } + +glm5.3flash-fp4-gb300-sglang-agentic-mtp: + image: lmsysorg/sglang:v0.5.22-cu130@sha256:e3a4f06001a29542fc4a8638536d9ffc5eb9c9942b08603c09b19a56eae4a1ae + model: nvidia/GLM-5.3-Flash-NVFP4 + model-prefix: glm5.3flash + runner: cluster:gb300-nv + precision: fp4 + framework: sglang + multinode: false + srt-recipe-dir: glm5.3flash/sglang/gb300-fp4-mtp + scenarios: + agentic-coding: + # HiCache auto-sizes at fraction 0.90; run 42026173061 measured 708.7-722.9 GB per node (4 ranks) of 964.7 GB. + - dram-utilization: 0.7495 + search-space: + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 24, 32], srt-recipe: "agentic.yaml:override_gpu_kv" } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [40, 44, 48], srt-recipe: "agentic.yaml:override_hicache" } + # H100 AgentX arm for DeepSeek-V4.1-Flash. H100 is not in the upstream hardware # table (h200, gb200, gb300, mi350x are). It needs its own script, not the # shared symlink: the sparse attention indexer's diff --git a/inferencex-e2e/configs/runners.yaml b/inferencex-e2e/configs/runners.yaml index 278a52160a..0ec0a45bce 100644 --- a/inferencex-e2e/configs/runners.yaml +++ b/inferencex-e2e/configs/runners.yaml @@ -636,6 +636,8 @@ clusters: DeepSeek-R1-0528: {root: scratch, dir: DeepSeek-R1-0528} DeepSeek-R1-0528-NVFP4-v2: {root: scratch, dir: DeepSeek-R1-0528-NVFP4-v2} GLM-5.2-NVFP4: {root: scratch, dir: GLM-5.2-NVFP4} + GLM-5.3-Flash: {root: data-models, dir: GLM-5.3-Flash} + GLM-5.3-Flash-NVFP4: {root: data-models, dir: GLM-5.3-Flash-NVFP4} Kimi-K3.1-Pro: {root: scratch, dir: Kimi-K3.1-Pro} MiniMax-M3.1-Flash: {root: data-models, dir: MiniMax-M3.1-Flash} Muse-Spark-Open-816B-A42B-NVFP4: {root: data-models, dir: Muse-Spark-Open-816B-A42B-NVFP4} diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index e26e2a2c46..115c736df7 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12205,3 +12205,14 @@ - "Retune the Muse Spark Open GB200 curve and set SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION=0 at every concurrency; run TP4/EP4 at concurrency 1 without KV offload and at concurrency 4, 7, 10, 14, 16, 28, 36, 40 with HiCache, dropping concurrency 2, 6, 8, 20, 24 and 32." - "Use prefill-decode interval 64-256 and stream interval 20 at every HiCache point, shortest-prefill-first except at concurrency 7, request admission 2-6 at concurrency 4-16, 16K chunked prefill at concurrency 28-40, and at concurrency 40 mem-fraction-static 0.93, a 160 GB HiCache pool and CUDA graphs up to 48." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + +- config-keys: + - glm5.3flash-fp8-gb300-sglang-agentic-mtp + - glm5.3flash-fp4-gb300-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add native SGLang GLM-5.3-Flash AgentX on one GB300 tray (TP4/EP1) with lmsysorg/sglang:v0.5.22-cu130, FP8 (zai-org/GLM-5.3-Flash) and NVFP4 (nvidia/GLM-5.3-Flash-NVFP4), embedded EAGLE MTP 5/1/6." + - "SGLang cookbook GB300 recipe: TRT-LLM DSA prefill and decode, FP8 E4M3 KV cache, flashinfer_trtllm MoE, auto reasoning and tool-call parsers; NVFP4 adds modelopt_fp4 quantization." + - "GPU-resident KV at concurrency 1, 4, 8, 16, 24 and 32; auto-sized HiCache host tier (--hicache-host-memory-fraction 0.90, recorded as 723 GB per node, the measured maximum) at concurrency 40, 44 and 48." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4715