diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang.sh index 297878af26..4321644d8c 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang.sh @@ -56,6 +56,9 @@ install_agentic_deps SERVER_LOG="$RESULT_DIR/server.log" mkdir -p "$RESULT_DIR" +export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 +export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 + CACHE_ARGS=() if require_agentic_kv_offload_backend hicache; then # DeepSeek V4 HiCache currently rejects --hicache-size and supports @@ -72,7 +75,6 @@ if require_agentic_kv_offload_backend hicache; then HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" - export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 CACHE_ARGS=( --enable-hierarchical-cache --hicache-ratio "$HICACHE_RATIO" diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh index 3600382acb..5599ca4717 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh @@ -56,6 +56,9 @@ install_agentic_deps SERVER_LOG="$RESULT_DIR/server.log" mkdir -p "$RESULT_DIR" +export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 +export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 + CACHE_ARGS=() if require_agentic_kv_offload_backend hicache; then # DeepSeek V4 HiCache currently rejects --hicache-size and supports @@ -77,7 +80,6 @@ if require_agentic_kv_offload_backend hicache; then HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_back}" HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" - export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 CACHE_ARGS=( --enable-hierarchical-cache --hicache-ratio "$HICACHE_RATIO" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a049d6e9ac..10a90a6d85 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -13767,7 +13767,7 @@ minimaxm3-fp8-h200-vllm-agentic: - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } dsv4-fp4-b200-sglang-agentic-hicache: - image: lmsysorg/sglang:v0.5.13-cu130 + image: lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:b200-dgxc @@ -13789,7 +13789,7 @@ dsv4-fp4-b200-sglang-agentic-hicache: # covers the middle/high-interactivity range omitted by the one-decode DEP # throughput curves below. Each engine start carries at most four concurrencies. dsv4-fp4-b300-sglang-agentic-hicache: - image: lmsysorg/sglang:v0.5.13-cu130 + image: lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:b300-nv diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0cf684364e..cd19882fc8 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4643,3 +4643,10 @@ - "Recipes sourced from NVIDIA/srt-slurm branch sa-submission-q2-2026 (gb300_nvfp4 MTP recipes)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1799 +- config-keys: + - dsv4-fp4-b200-sglang-agentic-hicache + - dsv4-fp4-b300-sglang-agentic-hicache + description: + - "Update SGLang image from lmsysorg/sglang:v0.5.13-cu130 to lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233" + - "Enable SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 unconditionally (previously hicache-only) and SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 to free out-of-window SWA KV slots during chunked prefill, relieving SWA pool pressure and restoring prefix-cache hit rate on multi-turn agentic workloads" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2112 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 8c7aa76bb2..58051e6405 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -430,24 +430,6 @@ else salloc --partition=$SLURM_PARTITION --account=$SLURM_ACCOUNT --gres=gpu:$GPU_COUNT --exclusive --mem=0 --time="$SALLOC_TIME_LIMIT" --no-shell --job-name="$RUNNER_NAME" JOB_ID=$(squeue --name="$RUNNER_NAME" -u "$USER" -h -o %A | head -n1) - # DSv4 is also staged on the compute nodes' local RAID. Loading the 806 GB - # checkpoint independently from Lustre on every TP rank leaves the loader - # threads blocked in Lustre I/O for hours. Select the local copy only after - # Slurm assigns a node, and retain the shared-Lustre path as a fallback for - # nodes whose local staging is incomplete. - if [[ "$MODEL_PREFIX" == "dsv4" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "sglang" ]]; then - LOCAL_MODEL_PATH=/raid/models/DeepSeek-V4-Pro-NVFP4 - if srun --jobid="$JOB_ID" bash -c \ - 'test -f "$1/config.json" && test -f "$1/model.safetensors.index.json" && test "$(find "$1" -maxdepth 1 -name "model-*.safetensors" | wc -l)" -eq 64' \ - _ "$LOCAL_MODEL_PATH"; then - export MODEL_PATH="$LOCAL_MODEL_PATH" - export MODEL="$MODEL_PATH" - echo "Using node-local DSv4 checkpoint: $MODEL_PATH" - else - echo "Node-local DSv4 checkpoint unavailable; using shared checkpoint: $MODEL_PATH" - fi - fi - # Use flock to serialize concurrent imports to the same squash file # Override ENROOT_CACHE_PATH to avoid permission issues with system-wide cache on worker nodes srun --jobid=$JOB_ID bash -c "