diff --git a/benchmarks/single_node/agentic/glm5.2_fp8_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp8_b200_sglang_mtp.sh new file mode 100755 index 0000000000..66f03c6bbe --- /dev/null +++ b/benchmarks/single_node/agentic/glm5.2_fp8_b200_sglang_mtp.sh @@ -0,0 +1,387 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# Agentic trace replay benchmark for GLM-5.2 FP8 on B200 using SGLang with +# EAGLE/MTP speculative decoding. First GLM-5.2 FP8 AgentX recipe on B200; it +# is spec-decode only, per the AgentX policy that agentic recipes are run and +# published with speculative decoding enabled rather than as an STP/MTP A/B +# (MODELS.md: GLM-5.2 agentic non-MTP is deprecated after 2026-08-03). +# +# Port of the validated agentic/glm5.2_fp4_b200_sglang_mtp.sh. The FP8 deltas +# are the blocks marked "FP8:" below -- the checkpoint (zai-org/GLM-5.2-FP8, +# ~756 GB of block-quantized e4m3 weights against ~465 GB for GLM-5.2-NVFP4), +# --quantization fp8 in place of modelopt_fp4, and the memory notes that follow +# from the larger resident weights. Everything else is the NVFP4 B200 script +# unchanged so the two precision curves stay comparable. +# +# Server flags follow the SGLang cookbook GLM-5.x single-node recipes +# (https://docs.sglang.io/cookbook/autoregressive/GLM/GLM-5.2; the published +# GLM-5.1-FP8 cookbook entry uses the same EAGLE shape with quantization: fp8): +# DP_ATTENTION=false -> low-latency arm (TP8, fp8 KV, cutedsl bf16 GEMM) +# DP_ATTENTION=true -> high-throughput DEP arm (TP8 + DP8 attention-DP + +# EP_SIZE expert-parallel MoE via --ep-size) +# Only the low-latency arm is wired into the master config for this MTP recipe +# (see the entry comment on glm5.2-fp8-b200-sglang-agentic-mtp); the DEP branch +# is kept intact so the throughput arm can be added without re-deriving it. +# +# Required env vars: +# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, +# EP_SIZE, DP_ATTENTION +# +# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +# B200: runners/launch_b200-nscale-compat.sh resolves the checkpoint to a +# cluster-local path and then rewrites MODEL to that path, so `hf download +# "$MODEL"` cannot work on this runner. Keep the HF repo id separate for the +# day-zero case where GLM-5.2-FP8 has not been staged yet. +# FP8: the upstream zai-org release; the golden AL below was measured on it. +HF_MODEL_ID="${HF_MODEL_ID:-zai-org/GLM-5.2-FP8}" + +# A non-empty directory is NOT a staged checkpoint. The NVFP4 sibling found +# /lustre/fsw/gharunners/models/GLM-5.2-NVFP4 holding config.json, +# generation_config.json, hf_quant_config.json, chat_template.jinja, README.md +# and .quant_summary.txt and NOTHING else -- an aborted or metadata-only pull. +# An `ls -A` emptiness guard accepts that, so all five cells of run 30729467646 +# skipped the download and went straight to serve; SGLang read config.json +# fine, then died in AutoTokenizer.from_pretrained with "Couldn't instantiate +# the backend tokenizer" because neither the tokenizer files nor a single +# weight shard were on disk. Check for a COMPLETE checkpoint instead: the +# tokenizer, the shard index, and every shard the index names. +checkpoint_is_complete() { + local dir="$1" + [[ -d "$dir" ]] || return 1 + [[ -f "$dir/tokenizer_config.json" ]] || return 1 + [[ -f "$dir/tokenizer.json" || -f "$dir/tokenizer.model" ]] || return 1 + [[ -f "$dir/model.safetensors.index.json" ]] || return 1 + CKPT_DIR="$dir" python3 - <<'PYEOF' +import json, os, sys +d = os.environ["CKPT_DIR"] +with open(os.path.join(d, "model.safetensors.index.json")) as fh: + shards = sorted(set(json.load(fh)["weight_map"].values())) +missing = [s for s in shards if not os.path.isfile(os.path.join(d, s))] +if missing: + print(f"{len(missing)}/{len(shards)} shards missing, e.g. {missing[:3]}", file=sys.stderr) + sys.exit(1) +PYEOF +} + +if [[ -n "${MODEL_PATH:-}" ]]; then + if ! checkpoint_is_complete "$MODEL_PATH"; then + # Every concurrency of this sweep runs as its own allocation against + # the same shared path, so serialize: one cell pulls the ~756 GB + # checkpoint (141 shards) and the rest wait on it rather than five + # racing writers. `hf download` resumes into a partially-populated + # --local-dir, so a metadata-only stub is fine to download on top of. + mkdir -p "$MODEL_PATH" + MODEL_DOWNLOAD_LOCK="${MODEL_PATH%/}.download.lock" + echo "Checkpoint at $MODEL_PATH is incomplete; acquiring $MODEL_DOWNLOAD_LOCK" + exec 9>"$MODEL_DOWNLOAD_LOCK" + flock -w "${MODEL_DOWNLOAD_LOCK_TIMEOUT:-21600}" 9 || { + echo "Error: timed out waiting for another cell to stage $MODEL_PATH" >&2 + exit 1 + } + if checkpoint_is_complete "$MODEL_PATH"; then + echo "Another cell staged $MODEL_PATH while we waited" + else + hf download "$HF_MODEL_ID" --local-dir "$MODEL_PATH" + fi + flock -u 9 + exec 9>&- + checkpoint_is_complete "$MODEL_PATH" || { + echo "Error: $MODEL_PATH is still incomplete after hf download $HF_MODEL_ID." >&2 + exit 1 + } + fi +else + hf download "$HF_MODEL_ID" + export MODEL_PATH="$HF_MODEL_ID" +fi +nvidia-smi + +resolve_trace_source +install_agentic_deps + +SERVER_LOG="$RESULT_DIR/server.log" +mkdir -p "$RESULT_DIR" + +CACHE_ARGS=() +if require_agentic_kv_offload_backend hicache; then + # HiCache extends RadixAttention: prefixes evicted from the HBM KV pool + # spill to a pinned host pool instead of being recomputed. On the + # 1M-context agentic corpus the live working set outgrows HBM past + # conc 8 (TP8) / 64 (DP8) and the radix hit rate collapses to <0.1 + # against a ~0.97 theoretical ceiling, so every turn re-prefills its + # whole history; the host tier restores those hits at C2C bandwidth. + # GLM-5.2 is DSA/MLA-family (attention_backend=dsa): every TP rank holds + # complete per-token KV. The ratio 0.75 sizes the host pool off the HBM KV + # pool, which is smaller here than on NVFP4 (see MEM_FRACTION_STATIC), so + # ratio mode yields a proportionally smaller pinned tier at c1-c8. Use the + # same 169 GB/rank absolute target pool at c12/c16 as the NVFP4 sibling: + # that figure is bounded by host DRAM, which the precision does not change. + # + # cluster:b200-nscale advertises 2,063,920 MiB and this config exposes 80%, + # giving the benchmark 1,731 GB. A 169 GB/rank packed target+MTP pool plus + # the coupled 38.73 GB/rank DSA indexer uses about 1,662 GB across TP8. + # Keep the 270 GB ceiling so deployments with more usable host DRAM can + # explicitly override the default. + DEFAULT_HICACHE_RATIO=0.75 + DEFAULT_HICACHE_SIZE=0 + case "$CONC" in + 12|16) DEFAULT_HICACHE_SIZE=169 ;; + esac + MAX_HICACHE_SIZE=270 + HICACHE_SIZE="${HICACHE_SIZE:-$DEFAULT_HICACHE_SIZE}" + if ! [[ "$HICACHE_SIZE" =~ ^[0-9]+$ ]]; then + echo "Error: HICACHE_SIZE must be a non-negative integer, got $HICACHE_SIZE" >&2 + exit 1 + fi + if awk -v s="$HICACHE_SIZE" -v cap="$MAX_HICACHE_SIZE" 'BEGIN { exit !(s > cap) }'; then + echo "Error: HICACHE_SIZE=$HICACHE_SIZE exceeds configured limit $MAX_HICACHE_SIZE" >&2 + exit 1 + fi + HICACHE_RATIO="${HICACHE_RATIO:-$DEFAULT_HICACHE_RATIO}" + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_back}" + HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" + CACHE_ARGS=( + --enable-hierarchical-cache + --hicache-write-policy "$HICACHE_WRITE_POLICY" + --hicache-io-backend "$HICACHE_IO_BACKEND" + --hicache-mem-layout "$HICACHE_MEM_LAYOUT" + ) + if awk -v s="$HICACHE_SIZE" 'BEGIN { exit !(s > 0) }'; then + echo "HiCache CPU tier: target_size=$HICACHE_SIZE GB, total_capacity=${TOTAL_CPU_DRAM_GB} GB, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT" + CACHE_ARGS+=(--hicache-size "$HICACHE_SIZE") + else + if awk -v r="$HICACHE_RATIO" -v cap="$DEFAULT_HICACHE_RATIO" 'BEGIN { exit !(r > cap) }'; then + echo "Error: HICACHE_RATIO=$HICACHE_RATIO exceeds configured limit $DEFAULT_HICACHE_RATIO" >&2 + exit 1 + fi + echo "HiCache CPU tier: ratio=$HICACHE_RATIO, total_capacity=${TOTAL_CPU_DRAM_GB} GB, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT" + CACHE_ARGS+=(--hicache-ratio "$HICACHE_RATIO") + fi +fi + +# With attention-DP, front the DP ranks with sglang-router using consistent +# hashing on the AIPerf correlation id so multi-turn sessions stay on the DP +# rank that holds their radix-cache prefix. +USE_SGLANG_ROUTER=false +SGLANG_BACKEND_PORT="$PORT" +ROUTER_LOG="$RESULT_DIR/router.log" +if [ "$DP_ATTENTION" = "true" ]; then + USE_SGLANG_ROUTER=true + export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true + SGLANG_BACKEND_PORT=$((PORT + 1)) + SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000)) +fi + +# MTP: GLM-5.2 ships its own nextn head (num_nextn_predict_layers=1), so EAGLE +# runs off the checkpoint with no external draft model. num-steps 3 / +# eagle-topk 1 / num-draft-tokens 4 is 3 speculative tokens per verification +# step -- the same shape the NVFP4 B200/B300 siblings and the GLM-5.1-FP8 +# cookbook speculative-mtp entry use, and the draft length whose golden AL is +# pinned below. +SPEC_ARGS=( + --speculative-algorithm EAGLE + --speculative-num-steps 3 + --speculative-eagle-topk 1 + --speculative-num-draft-tokens 4 +) + +PARALLEL_ARGS=(--tp "$TP" --ep-size "$EP_SIZE") +CHUNKED_PREFILL_SIZE=8192 +if [ "$DP_ATTENTION" = "true" ]; then + # chunked-prefill-size is a whole-engine budget split across DP ranks: + # the cookbook HT cell's 8192 becomes 1,024 tokens/rank/step under dp8, + # which starves prefill on the 1M-context agentic corpus (observed: a + # conc-256 warmup could not drain within AIPerf's 1800s grace period + # while KV usage sat at ~0.01). Use the cookbook's own dp8 lever from + # the B200 cells (32768 = ~4096/rank). + CHUNKED_PREFILL_SIZE=32768 + PARALLEL_ARGS+=( + --dp "$TP" + --enable-dp-attention + --tokenizer-worker-num "$TP" + --dist-init-addr "127.0.0.1:$((PORT + 2000))" + ) + # Carried over from the NVFP4 sibling, where the draft MoE is bf16 + # (hf_quant_config excludes model.layers.78*) and inheriting the target + # model's FlashInfer all-to-all dies at init with "Pre-permute function + # for flashinfer to triton is not registered". FP8: GLM-5.2-FP8 quantizes + # the nextn experts like every other layer (modules_to_not_convert lists + # only layer-78 norms and biases), so the draft MoE takes the FP8 runner + # and this pin is likely unnecessary here; it is kept so the DEP arm, if + # wired, starts from the configuration that is known to boot. Only + # relevant once expert parallelism puts an a2a in the MoE path -- the + # plain-TP arm below has none. + SPEC_ARGS+=( + --speculative-moe-a2a-backend none + --speculative-moe-runner-backend triton + ) +else + # Cookbook low-latency levers; the DP-attention cell omits them. + PARALLEL_ARGS+=( + --kv-cache-dtype fp8_e4m3 + --bf16-gemm-backend cutedsl + --max-prefill-tokens 8192 + ) +fi + +# AgentX concurrency counts live session trees, not individual requests. +# Allow subagent fan-out to exceed CONC without clipping request bursts. +MAX_RUNNING_REQUESTS=$((2 * CONC)) +GRAPH_ARGS=() +if [ "$DP_ATTENTION" != "true" ]; then + # Cookbook low-latency captures graphs up to its request cap; the + # DP-attention cell leaves the CUDA-graph batch list at SGLang defaults. + # --cuda-graph-max-bs counts requests, not verification tokens: SGLang's + # spec-decode graph runner scales each captured batch by + # --speculative-num-draft-tokens itself. + CUDA_GRAPH_MAX_BS=$MAX_RUNNING_REQUESTS + [ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 + GRAPH_ARGS=(--cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS") +fi + +# B200: 180 GB HBM3e per GPU. 0.83 leaves ~31 GB of non-static headroom for +# the EAGLE draft head's verification-batch activations, the extra CUDA-graph +# capture at 4 draft tokens, and GLM-5.2's DSA indexer temporaries; that +# headroom is a fraction of the card and does not depend on the checkpoint. +# FP8: what does change is the KV pool inside the static share. The ~756 GB +# checkpoint is ~94.5 GB/GPU across TP8 (NVFP4: ~58 GB/GPU), so the fp8 KV +# pool is roughly 55 GB/GPU here against ~91 GB/GPU on NVFP4. HiCache absorbs +# the difference as host spill; if c12/c16 show HBM pressure, raise the +# fraction here (overridable) before touching the concurrency grid. +MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.83}" + +export PYTHONNOUSERSITE=1 +export TORCH_CUDA_ARCH_LIST=10.0 +# Each concurrency in a full sweep is a separate Slurm allocation, while the +# Nscale home directory is shared. Keep SGLang's FlashInfer autotune, Triton, +# Inductor, and CUDA JIT caches allocation-local so concurrent cells cannot +# overwrite the same per-rank runtime-cache files. Non-Slurm launchers can +# provide an explicit SGLANG_CACHE_DIR override. +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + export SGLANG_CACHE_DIR="${SGLANG_CACHE_DIR:-/tmp/sglang-cache-${SLURM_JOB_ID}}" +fi +# Agentic warmup dispatches hundreds of large prompts at once; allow up to +# 15 minutes of TCP progress before AIPerf declares a connection dead. +export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 +# AIPerf pins one pooled keep-alive connection per session (client-side +# keep-alive 300s) while uvicorn's default SGLANG_TIMEOUT_KEEP_ALIVE is 5s; +# inter-turn idle gaps (capped at 10s) can reuse a socket exactly as the +# server closes it -> ECONNRESET -> terminal warmup failure. Outlast the +# client pool so the race cannot occur. +export SGLANG_TIMEOUT_KEEP_ALIVE=900 + +# AgentX pins acceptance to the committed golden AL so submissions are compared +# on system performance at a fixed acceptance target rather than on draft-head +# quality (golden_al_distribution/README.md). 2.99 is the GLM-5.2 curve at +# num_speculative_tokens=3, thinking_on +# (golden_al_distribution/glm5.2_mtp.yaml, SPEED-Bench coding, run 28058352479). +# FP8: that curve was measured on this very checkpoint (glm-5.2-fp8), so no +# cross-precision assumption is involved here. +# +# SGLANG_SIMULATE_ACC_TOKEN_MODE only exists from SGLang v0.5.16. An older +# image would silently honor ACC_LEN/ACC_METHOD and ignore the token-mode half +# of the contract. +# +# EVAL_ONLY leaves simulated acceptance off: it commits drafted tokens +# regardless of the target logits, so generated text is wrong and the eval +# would score ~0. +if [ "${EVAL_ONLY:-false}" != "true" ]; then + export SGLANG_SIMULATE_ACC_LEN=2.99 + export SGLANG_SIMULATE_ACC_METHOD=match-expected + export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token +fi + +SGLANG_CMD=( + python3 -m sglang.launch_server + --model-path "$MODEL_PATH" + --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$SGLANG_BACKEND_PORT" + --trust-remote-code + "${PARALLEL_ARGS[@]}" + # FP8: zai-org/GLM-5.2-FP8 ships quantization_config quant_method=fp8 with + # 128x128 weight blocks and dynamic e4m3 activations; pass the method + # explicitly as the Qwen3.5 FP8 B200/B300 SGLang siblings do. + --quantization fp8 + # GLM-5.2 emits the GLM-4.7-style // format; + # the glm47 parser is required for structured message.tool_calls (glm45 + # leaves calls as raw text). Without it the SWE-bench mini-swe-agent eval + # dies with RepeatedFormatError ("No tool calls found in the response") on + # every instance and scores 0. Reasoning parser keeps hybrid-thinking + # output in reasoning_content instead of polluting content. Neither flag + # affects trace-replay throughput (pre-canned replay discards live + # responses). + --tool-call-parser glm47 + --reasoning-parser glm45 + --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --max-running-requests "$MAX_RUNNING_REQUESTS" + "${SPEC_ARGS[@]}" + "${GRAPH_ARGS[@]}" + "${CACHE_ARGS[@]}" + --watchdog-timeout 1800 + --enable-metrics +) + +printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" +printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" + +{ + echo "=== SGLANG_SIMULATE_ACC_* env vars at launch (empty => real verification) ===" + env | grep -E '^SGLANG_SIMULATE_ACC_' | sort || true + echo "============================================================================" +} | tee "$SERVER_LOG" + +echo "Starting SGLang server for B200..." +"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [ "$USE_SGLANG_ROUTER" = "true" ]; then + echo "Starting SGLang router on port $PORT for $TP DP ranks..." + python3 -m sglang_router.launch_router \ + --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \ + --policy consistent_hashing \ + --request-id-headers x-correlation-id \ + --dp-aware \ + --host 0.0.0.0 \ + --port "$PORT" \ + --prometheus-host 127.0.0.1 \ + --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" \ + --connect-timeout-secs 900 \ + --request-timeout-secs 14400 \ + --disable-health-check \ + --disable-retries > "$ROUTER_LOG" 2>&1 & + ROUTER_PID=$! + echo "Router PID: $ROUTER_PID" + wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" +fi + +if [ "${EVAL_ONLY}" = "true" ]; then + # GLM-5.2's chat template defaults to reasoning_effort=Max when the + # client passes no chat_template_kwargs (mini-swe-agent doesn't), and the + # heavy thinking burns the default 75-step budget: on the 23-instance + # slice, 12/23 trajectories exited LimitsExceeded unsubmitted while 10 of + # the 11 that submitted resolved. Double the step budget for this recipe; + # other recipes keep the shared 75 default. + export SWEBENCH_AGENT_STEP_LIMIT=150 + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0a6c86df6d..5a475c22a6 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9030,6 +9030,36 @@ glm5.2-fp4-b200-sglang-agentic-mtp: search-space: - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16] } +# First GLM-5.2 FP8 AgentX recipe on B200, the FP8 precision sibling of +# glm5.2-fp4-b200-sglang-agentic-mtp. Same spec-decode-only shape per the AgentX +# policy (MODELS.md): SGLang EAGLE off GLM-5.2's built-in nextn head (num-steps +# 3, eagle-topk 1, 4 draft tokens = 3 speculative tokens) with acceptance pinned +# to the golden AL 2.99 (golden_al_distribution/glm5.2_mtp.yaml, thinking_on, +# K=3) through SGLANG_SIMULATE_ACC_*. That curve was measured on glm-5.2-fp8, +# i.e. on this checkpoint. Pinned to the 2026-09-08 cu13 dev nightly (the first that carries sgl-project/sglang#38318, the guard for the EAGLE DSA fp8 read-door crash seen on the 2026-09-07 build), the same +# tag the B200 Qwen3.5 FP8/FP4 SGLang AgentX recipes moved to that day. +# +# Same single arm and concurrency grid as the NVFP4 B200 and B300 siblings: +# cookbook low-latency TP8 with HiCache host-DRAM offload, conc +# [1, 4, 8, 12, 16] (steps of at least 2, hard stop at 16). TP8-only for +# memory as well as comparability -- the ~756 GB block-FP8 checkpoint needs +# ~94.5 GB/GPU across 8 B200s and does not fit below 8; the fp8 KV pool that +# remains inside --mem-fraction-static 0.83 is ~55 GB/GPU against ~91 GB/GPU +# on NVFP4, which HiCache absorbs as host spill. +glm5.2-fp8-b200-sglang-agentic-mtp: + image: lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b + model: zai-org/GLM-5.2-FP8 + model-prefix: glm5.2 + runner: cluster:b200-nscale + precision: fp8 + framework: sglang + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16] } + glm5.2-fp4-gb200-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:v0.5.17-cu130 model: nvidia/GLM-5.2-NVFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6b249ea15e..53b3fe7e88 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6921,3 +6921,18 @@ - "Pick up the latest automatic ROCm DeepSeek-V4 optimizations, including fused mHC post/pre plus RMSNorm, gfx950 C4A top-k dispatch, fused C4 compressor GEMMs, fused SWA q/kv RMSNorm plus q FP8 quantization, and medium-batch cooperative top-k tuning." - "Keep the existing VLLM_ROCM_USE_AITER=1, VLLM_ROCM_USE_AITER_MOE=1, and --moe-backend aiter settings, and explicitly add VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 plus VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 to both STP and MTP paths. The current checkpoint's shared-expert path does not satisfy the latest vLLM fusion conditions, so that fusion flag self-disables while preserving recipe parity." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2792 + +- config-keys: + - glm5.2-fp8-b200-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add the first GLM-5.2 FP8 AgentX (agentic-coding) recipe on B200: zai-org/GLM-5.2-FP8 on single-node SGLang with EAGLE off the checkpoint's built-in nextn head (num-steps 3, eagle-topk 1, 4 draft tokens = 3 speculative tokens), spec-decoding=mtp only per the AgentX policy, routed to benchmarks/single_node/agentic/glm5.2_fp8_b200_sglang_mtp.sh." + - "Port of the validated glm5.2_fp4_b200_sglang_mtp.sh with two functional deltas: HF_MODEL_ID zai-org/GLM-5.2-FP8 and --quantization fp8 (the checkpoint ships quant_method=fp8 with 128x128 weight blocks and dynamic e4m3 activations) in place of modelopt_fp4. Serve flags are otherwise identical: TP8 low-latency arm with --kv-cache-dtype fp8_e4m3, --bf16-gemm-backend cutedsl, --max-prefill-tokens 8192, chunked-prefill 8192, --mem-fraction-static 0.83, glm47 tool-call and glm45 reasoning parsers, cuda-graph-max-bs min(2*CONC, 64), HiCache write_back/direct/page_first_direct with ratio 0.75 at c1-c8 and a 169 GB/rank target pool at c12/c16." + - "Throughput runs pin SGLang simulated acceptance to the committed golden AL 2.99 (golden_al_distribution/glm5.2_mtp.yaml, thinking_on, K=3, run 28058352479), which was measured on the FP8 checkpoint itself; EVAL_ONLY runs keep real target verification." + - "Image lmsysorg/sglang:nightly-dev-cu13-20260907-30705c00 (2026-09-07 cu13 dev nightly, digest sha256:19b8fa1223cc339c1eae7a5b703f1a8c2543b5b119155bf3d7efaef18f77f007, tag commit sgl-project/sglang@30705c00; Docker Hub last pushed 2026-09-07T01:43:42Z). The same tag the B200 Qwen3.5 FP8/FP4 SGLang AgentX recipes moved to in #2861/#2862; SGLANG_SIMULATE_ACC_TOKEN_MODE requires v0.5.16 or newer." + - "Memory: the ~756 GB block-FP8 checkpoint (141 shards) is ~94.5 GB/GPU at TP8 on 180 GB B200s (NVFP4: ~58 GB/GPU), leaving an fp8 KV pool of roughly 55 GB/GPU inside the 0.83 static share against ~91 GB/GPU on NVFP4. TP8-only for memory and comparability; HiCache absorbs the smaller HBM pool as host spill. MEM_FRACTION_STATIC and HICACHE_* stay env-overridable for on-node tuning." + - "Search space mirrors glm5.2-fp4-b200-sglang-agentic-mtp exactly so the two precision curves are comparable: one TP8 + HiCache arm at conc [1, 4, 8, 12, 16], dram-utilization 0.80." + - "Adds glm5.2/fp8 routing to runners/launch_b200-nscale-compat.sh (MODEL_PATH default /scratch/models/GLM-5.2-FP8, SRT_SLURM_MODEL_PREFIX glm5.2-fp8), which previously hard-failed with 'Unsupported model prefix/precision' for this model. The script's completeness-checked, flock-serialized hf download stages the checkpoint on the day-zero run if it is not already on the cluster." + - "Re-pin from lmsysorg/sglang:nightly-dev-cu13-20260907-30705c00 to lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b (2026-09-08 cu13 dev nightly, digest sha256:9a352a35c973a2357372e85f3bcb5388b6b3c46c1329165987260f3b089647dc; Docker Hub last pushed 2026-09-08T01:40:59Z, tag commit sgl-project/sglang@20ca564b). The 2026-09-07 build carries an unguarded kv_index_translator.translate_dcp_read_ids call on the DSA fp8 KV read path that the EAGLE draft backend never binds, so GLM-5.2 MTP runs crash intermittently with AttributeError (observed on the MI355X FP8 sibling in run 34173459478 after 74 minutes of serving). sgl-project/sglang#38318 (merged 2026-09-07T20:03Z) adds the None guard and is six commits behind 20ca564b." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2863 diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh index 15b4013aac..d72ee46c61 100644 --- a/runners/launch_b200-nscale-compat.sh +++ b/runners/launch_b200-nscale-compat.sh @@ -55,6 +55,9 @@ elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" ]]; then elif [[ $MODEL_PREFIX == "glm5.2" && $PRECISION == "fp4" ]]; then export MODEL_PATH="${MODEL_PATH:-/scratch/models/GLM-5.2-NVFP4}" export SRT_SLURM_MODEL_PREFIX="glm5.2-fp4" +elif [[ $MODEL_PREFIX == "glm5.2" && $PRECISION == "fp8" ]]; then + export MODEL_PATH="${MODEL_PATH:-/scratch/models/GLM-5.2-FP8}" + export SRT_SLURM_MODEL_PREFIX="glm5.2-fp8" elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "int4" ]]; then export MODEL_PATH="/scratch/models/Kimi-K2.5" export SRT_SLURM_MODEL_PREFIX="kimik2.5"