diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro_sglang.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro_sglang.sh new file mode 100755 index 0000000000..f3e92ad622 --- /dev/null +++ b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro_sglang.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash + +# Qwen3.5-397B-A17B NVFP4 on four RTX PRO 6000 Blackwell GPUs. +# SM120 has no trtllm-gen kernels, so the routed experts and the NVFP4 GEMMs +# run on FlashInfer CUTLASS and attention runs on FlashInfer. The node is +# PCIe-only, so collectives use plain NCCL rather than the custom all-reduce. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars \ + MODEL \ + TP \ + EP_SIZE \ + CONC \ + ISL \ + OSL \ + RANDOM_RANGE_RATIO \ + RESULT_FILENAME + +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# copy. Either way, SERVE_MODEL is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi + SERVE_MODEL="$MODEL_PATH" +else + hf download "$MODEL" + SERVE_MODEL="$MODEL" +fi + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +nvidia-smi + +export SGLANG_ENABLE_JIT_DEEPGEMM=false +export PYTHONUNBUFFERED=1 + +SERVER_LOG=/workspace/server.log + +# 96 GiB per GPU leaves far less headroom than the B300 recipe assumes. The +# weights take ~56 GiB per rank and the prefill/decode CUDA graphs another +# ~7 GiB, so the static fraction has to stay low enough that a prefill chunk's +# activations still fit: at 0.8 the KV pool grew to 2.2M tokens (30x what +# concurrency 64 needs) and the first 8k prefill OOM'd. 0.7 still leaves ~1M +# KV tokens, and a 2-request prefill chunk keeps the activation peak bounded. +MEM_FRAC_STATIC="${MEM_FRAC_STATIC:-0.7}" +CHUNKED_PREFILL_SIZE=$((ISL * 2)) +MAX_PREFILL_TOKENS=$((ISL * 2)) +MAX_RUNNING_REQUESTS=128 +CONTEXT_LENGTH=$((ISL + OSL + 20)) + +# Default: recv every ~10 requests; if CONC >= 16, relax to ~30 requests between scheduler recv polls. +if [[ $CONC -ge 16 ]]; then + SCHEDULER_RECV_INTERVAL=30 +else + SCHEDULER_RECV_INTERVAL=10 +fi + +if [[ "$EVAL_ONLY" == "true" ]]; then + setup_eval_context + CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN" +fi + +echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL" + +start_gpu_monitor + +set -x +PYTHONNOUSERSITE=1 python3 -m sglang.launch_server \ + --model-path "$SERVE_MODEL" \ + --served-model-name "$MODEL" \ + --host 0.0.0.0 \ + --port "$PORT" \ + --trust-remote-code \ + --tensor-parallel-size "$TP" \ + --data-parallel-size 1 \ + --ep-size "$EP_SIZE" \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder \ + --quantization modelopt_fp4 \ + --fp4-gemm-backend flashinfer_cutlass \ + --moe-runner-backend flashinfer_cutlass \ + --attention-backend flashinfer \ + --kv-cache-dtype fp8_e4m3 \ + --mamba-ssm-dtype bfloat16 \ + --mamba-scheduler-strategy no_buffer \ + --disable-custom-all-reduce \ + --disable-radix-cache \ + --mem-fraction-static "$MEM_FRAC_STATIC" \ + --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" \ + --max-prefill-tokens "$MAX_PREFILL_TOKENS" \ + --context-length "$CONTEXT_LENGTH" \ + --cuda-graph-max-bs-decode "$CONC" \ + --max-running-requests "$MAX_RUNNING_REQUESTS" \ + --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" \ + --stream-interval 20 > "$SERVER_LOG" 2>&1 & + +SERVER_PID=$! + +wait_for_server_ready \ + --port "$PORT" \ + --server-log "$SERVER_LOG" \ + --server-pid "$SERVER_PID" + +run_benchmark_serving \ + --model "$MODEL" \ + --port "$PORT" \ + --backend vllm \ + --input-len "$ISL" \ + --output-len "$OSL" \ + --random-range-ratio "$RANDOM_RANGE_RATIO" \ + --num-prompts "$((CONC * 10))" \ + --max-concurrency "$CONC" \ + --result-filename "$RESULT_FILENAME" \ + --result-dir /workspace/ \ + --trust-remote-code + +if [[ "$RUN_EVAL" == "true" ]]; then + run_eval --framework lm-eval --port "$PORT" + append_lm_eval_summary +fi + +stop_gpu_monitor +set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro_sglang_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro_sglang_mtp.sh new file mode 100755 index 0000000000..2692860015 --- /dev/null +++ b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_rtx6000pro_sglang_mtp.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash + +# Qwen3.5-397B-A17B NVFP4 on four RTX PRO 6000 Blackwell GPUs, with the +# built-in MTP draft head driven through SGLang's EAGLE speculative path. +# SM120 has no trtllm-gen kernels, so the routed experts and the NVFP4 GEMMs +# run on FlashInfer CUTLASS and attention runs on FlashInfer. The node is +# PCIe-only, so collectives use plain NCCL rather than the custom all-reduce. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars \ + MODEL \ + TP \ + EP_SIZE \ + CONC \ + ISL \ + OSL \ + RANDOM_RANGE_RATIO \ + RESULT_FILENAME + +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# copy. Either way, SERVE_MODEL is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi + SERVE_MODEL="$MODEL_PATH" +else + hf download "$MODEL" + SERVE_MODEL="$MODEL" +fi + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +nvidia-smi + +export SGLANG_ENABLE_JIT_DEEPGEMM=false +export PYTHONUNBUFFERED=1 + +SERVER_LOG=/workspace/server.log + +# SGLang holds back total * (1 - mem-fraction-static) as slack and gives the +# rest to the KV and Mamba pools, so the MTP draft head (Qwen3_5ForCausalLMMTP, +# 4.05 GiB per rank on top of the 56.12 GiB target model) has to be paid for by +# raising the fraction, not lowering it. 0.65 and 0.75 both left the pools +# empty on this 96 GiB SKU once the draft's own Mamba state was accounted for. +# 0.85 came up but left only 4.56 GiB free per rank, too thin for a 16k prefill +# chunk, so 0.80 trades surplus KV (1.7M tokens, ~3x what concurrency 64 needs) +# for activation headroom. +MEM_FRAC_STATIC="${MEM_FRAC_STATIC:-0.80}" +CHUNKED_PREFILL_SIZE=$((ISL * 2)) +MAX_PREFILL_TOKENS=$((ISL * 2)) +# The client never opens more than CONC connections, so sizing the Mamba state +# pool for 128 requests (as the B300 recipe does) just strands memory that the +# draft head needs here. +MAX_RUNNING_REQUESTS="$CONC" +CONTEXT_LENGTH=$((ISL + OSL + 20)) + +# Default: recv every ~10 requests; if CONC >= 16, relax to ~30 requests between scheduler recv polls. +if [[ $CONC -ge 16 ]]; then + SCHEDULER_RECV_INTERVAL=30 +else + SCHEDULER_RECV_INTERVAL=10 +fi + +if [[ "$EVAL_ONLY" == "true" ]]; then + setup_eval_context + CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN" +fi + +echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL" + +start_gpu_monitor + +set -x +PYTHONNOUSERSITE=1 python3 -m sglang.launch_server \ + --model-path "$SERVE_MODEL" \ + --served-model-name "$MODEL" \ + --host 0.0.0.0 \ + --port "$PORT" \ + --trust-remote-code \ + --tensor-parallel-size "$TP" \ + --data-parallel-size 1 \ + --ep-size "$EP_SIZE" \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder \ + --quantization modelopt_fp4 \ + --fp4-gemm-backend flashinfer_cutlass \ + --moe-runner-backend flashinfer_cutlass \ + --attention-backend flashinfer \ + --kv-cache-dtype fp8_e4m3 \ + --mamba-ssm-dtype bfloat16 \ + --mamba-scheduler-strategy no_buffer \ + --disable-custom-all-reduce \ + --disable-radix-cache \ + --mem-fraction-static "$MEM_FRAC_STATIC" \ + --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" \ + --max-prefill-tokens "$MAX_PREFILL_TOKENS" \ + --context-length "$CONTEXT_LENGTH" \ + --cuda-graph-max-bs-decode "$CONC" \ + --max-running-requests "$MAX_RUNNING_REQUESTS" \ + --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" \ + --stream-interval 20 \ + --speculative-algorithm EAGLE \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 > "$SERVER_LOG" 2>&1 & + +SERVER_PID=$! + +wait_for_server_ready \ + --port "$PORT" \ + --server-log "$SERVER_LOG" \ + --server-pid "$SERVER_PID" + +# EAGLE-style spec decoding is trained against chat-formatted inputs, so the +# benchmark must send chat prompts or the acceptance rate silently collapses. +run_benchmark_serving \ + --model "$MODEL" \ + --port "$PORT" \ + --backend vllm \ + --input-len "$ISL" \ + --output-len "$OSL" \ + --random-range-ratio "$RANDOM_RANGE_RATIO" \ + --num-prompts "$((CONC * 10))" \ + --max-concurrency "$CONC" \ + --result-filename "$RESULT_FILENAME" \ + --result-dir /workspace/ \ + --use-chat-template \ + --trust-remote-code + +if [[ "$RUN_EVAL" == "true" ]]; then + run_eval --framework lm-eval --port "$PORT" + append_lm_eval_summary +fi + +stop_gpu_monitor +set +x diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 8ee037c8c2..9fc14fd4e8 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1343,6 +1343,42 @@ qwen3.5-fp4-b300-sglang: - { tp: 4, ep: 1, conc-start: 4, conc-end: 128 } - { tp: 2, ep: 2, conc-start: 4, conc-end: 128 } +# Qwen3.5-397B-A17B NVFP4 single-node SGLang sweep using 4 of 8 RTX PRO +# 6000 Blackwell GPUs. Both arms use ordinary NCCL collectives on PCIe. +qwen3.5-fp4-rtx6000pro-sglang: + image: lmsysorg/sglang:v0.5.16-cu130 + model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-prefix: qwen3.5 + runner: rtx6000pro-lat + precision: fp4 + framework: sglang + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 4, conc-list: [1, 4, 16, 64] } + - { tp: 4, ep: 4, conc-list: [1, 4, 16, 64] } + +# Same sweep with the built-in MTP draft head driven through SGLang's EAGLE +# speculative path. +qwen3.5-fp4-rtx6000pro-sglang-mtp: + image: lmsysorg/sglang:v0.5.16-cu130 + model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-prefix: qwen3.5 + runner: rtx6000pro-lat + precision: fp4 + framework: sglang + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 4, conc-list: [1, 4, 16, 64], spec-decoding: mtp } + - { tp: 4, ep: 4, conc-list: [1, 4, 16, 64], spec-decoding: mtp } + qwen3.5-fp4-b300-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130 model: nvidia/Qwen3.5-397B-A17B-NVFP4 diff --git a/configs/runners.yaml b/configs/runners.yaml index 69788865ad..5e992eb4e5 100644 --- a/configs/runners.yaml +++ b/configs/runners.yaml @@ -164,6 +164,10 @@ labels: - gb300-nv_0 - gb300-nv_1 - gb300-nv_2 + rtx6000pro: + - rtx6000pro-lat_00 + rtx6000pro-lat: + - rtx6000pro-lat_00 cluster:h100-cw: - h100-cw_00 - h100-cw_01 @@ -251,6 +255,8 @@ labels: - gb300-nv_0 - gb300-nv_1 - gb300-nv_2 + cluster:rtx6000pro-lat: + - rtx6000pro-lat_00 cluster:mi300x-amds: - mi300x-amds_00 - mi300x-amds_01 @@ -318,6 +324,9 @@ hardware: cluster:gb300-nv: available-cpu-dram-mib: 860_160 gpus-per-node: 4 + cluster:rtx6000pro-lat: + available-cpu-dram-mib: 1_500_000 + gpus-per-node: 8 cluster:mi300x-amds: available-cpu-dram-mib: 2_321_924 gpus-per-node: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c1736ae420..8f5831507e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5311,3 +5311,31 @@ - "Search space mirrors the non-MTP entry's KV arms -- TP8 GPU-resident and TP8 host-DRAM offload -- so the spec-decoding delta is readable at equal concurrency, but stops at conc 16 rather than 24. The non-MTP bring-up sweep (run 30326393603) showed the GPU-resident arm already thrashing at conc >= 16 (prefix cache hit rate 2.7%, TTFT p50 86-191s) because GPU KV holds only ~3.1 max-length requests, so conc 24 would spend a full job per arm re-measuring that regime; conc 16 still exercises the DRAM tier meaningfully (62% external prefix cache hit rate). TP8-only for the same memory reason: a ~1.5 TB MXFP4 checkpoint needs ~188 GB/GPU across 8 B300s and does not fit below 8 GPUs." - "Sets VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1, which the upstream recipe requires on both its blackwell and nvidia paths and which this repo had never set. It defaults to 0 and is threaded into LatentMoERunner as runner_args={\"enable_k3_latent_moe_tail_fusion\": ...} at vllm/models/kimi_k3/nvidia/model.py:549 -- the same runner whose shared-experts output buffer asserted (fused_moe/runner/shared_experts.py:165, all 8 TP ranks at once) when the wider flag alignment was attempted, so every K3 run here so far has been exercising a MoE tail path upstream does not use. Enabled on its own, ahead of re-attempting gpu-memory-utilization 0.95, max-num-seqs 32, --no-enable-flashinfer-autotune or VLLM_USE_V2_MODEL_RUNNER=1, to isolate its effect." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2418 + +- config-keys: + - qwen3.5-fp4-rtx6000pro-sglang + scenario-type: + - fixed-seq-len + description: + - "Add Qwen3.5-397B-A17B NVFP4 single-node SGLang benchmarking using four GPUs per job on the 8x RTX PRO 6000 Blackwell Latitude runner" + - "Sweep 8k/1k at concurrency 1, 4, 16, and 64 for both TP4 and TEP4 (TP4 attention with EP4 MoE, without DP attention)" + - "Pin SGLang v0.5.16-cu130, the first tag whose auto backend resolution knows that trtllm-gen MoE is SM100-only and routes modelopt_fp4 experts to FlashInfer CUTLASS on SM120" + - "Use FlashInfer CUTLASS for both the NVFP4 GEMMs and the routed experts and the FlashInfer attention backend on SM120, with FP8 KV cache, bf16 Mamba SSM state, the no_buffer Mamba scheduler strategy, radix cache disabled, and decode CUDA graph capture capped to each tested concurrency" + - "Use standard NCCL collectives on the PCIe topology (custom all-reduce disabled explicitly) and disable IB/RoCE probing to avoid the node's NCCL 2.28.9 bnxt_re initialization crash while preserving local CUDA P2P/SHM" + - "Deviate from the B300 SGLang recipe where 96 GiB per GPU forces it: mem-fraction-static 0.7 with a 2-request prefill chunk (0.8 with a 32k chunk sized the KV pool at 2.2M tokens and OOM'd on the first 8k prefill), and a single tokenizer worker (the B300 recipe's six each pin a ~0.7 GiB CUDA context on GPU 0)" + - "Teach the RTX PRO 6000 launcher to prefer a framework-tagged benchmark script so vLLM and SGLang recipes for the same model can coexist" + - "Validated on the node through the real launcher at all four corners of the sweep, every request successful: TEP4 concurrency 64 640/640 at 6198 tok/s, TP4 concurrency 64 640/640 at 5999 tok/s, TEP4 concurrency 1 10/10 at 672 tok/s, TP4 concurrency 1 10/10 at 658 tok/s" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2312 + +- config-keys: + - qwen3.5-fp4-rtx6000pro-sglang-mtp + scenario-type: + - fixed-seq-len + description: + - "Add an MTP arm to the Qwen3.5-397B-A17B NVFP4 RTX PRO 6000 SGLang sweep, driving the checkpoint's built-in MTP draft head through SGLang's EAGLE path with 3 steps, top-k 1, and 4 draft tokens" + - "Sweep the same 8k/1k grid as the non-MTP config: concurrency 1, 4, 16, and 64 for both TP4 and TEP4" + - "Raise mem-fraction-static to 0.80 for this arm: the fraction bounds weights plus the KV and Mamba pools, so the 4.05 GiB per-rank draft head has to be paid for by raising it, not lowering it (0.65 and 0.75 both left the pools empty on this 96 GiB SKU, and 0.85 came up with only 4.56 GiB free per rank)" + - "Size max-running-requests to the tested concurrency instead of a fixed 128, since the client never opens more connections and the oversized Mamba state pool is what starved the draft head" + - "Pass --use-chat-template to the benchmark, as required for EAGLE-style speculative decoding" + - "Validated on the node through the real launcher, every request successful: TEP4 concurrency 64 640/640 at 6895 tok/s (11% above the non-MTP arm's 6198 tok/s, mean TTFT 27.2s to 9.6s), and TEP4 concurrency 1 10/10 at 1233 tok/s (83% above the non-MTP arm's 672 tok/s, mean TPOT 12.6ms to 6.5ms)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2312 diff --git a/runners/launch_rtx6000pro-lat.sh b/runners/launch_rtx6000pro-lat.sh new file mode 100755 index 0000000000..6ad56e7092 --- /dev/null +++ b/runners/launch_rtx6000pro-lat.sh @@ -0,0 +1,144 @@ +#!/usr/bin/bash +set -euo pipefail + +# This runner executes directly on the single RTX PRO 6000 GPU node. Docker +# therefore owns a separate image cache from the node's RKE2/containerd cache. +HF_HUB_CACHE_MOUNT="${HF_HUB_CACHE_MOUNT:-/var/lib/inferencex/hf-hub-cache}" +export HF_HUB_CACHE="${HF_HUB_CACHE:-/mnt/hf_hub_cache/}" +PORT="${PORT:-8888}" + +# NCCL 2.28.9 segfaults while probing this node's bnxt_re RDMA devices. +# Disable that RDMA path by default while preserving local CUDA P2P/SHM and +# allowing an explicit caller override. +export NCCL_IB_DISABLE="${NCCL_IB_DISABLE:-1}" + +: "${GITHUB_WORKSPACE:?GITHUB_WORKSPACE must be set}" +: "${IMAGE:?IMAGE must be set}" +: "${EXP_NAME:?EXP_NAME must be set}" +: "${PRECISION:?PRECISION must be set}" + +mkdir -p "$HF_HUB_CACHE_MOUNT" + +export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}" +if [[ ! "$GPU_COUNT" =~ ^[1-9][0-9]*$ ]]; then + echo "GPU_COUNT must be a positive integer, got: $GPU_COUNT" >&2 + exit 1 +fi + +export CUDA_VISIBLE_DEVICES +CUDA_VISIBLE_DEVICES="$(seq -s, 0 "$((GPU_COUNT - 1))")" + +# Some Slurm/enroot configs spell registry paths as nvcr.io#namespace/image. +# Docker requires the normal slash form. +DOCKER_IMAGE="${IMAGE//#//}" + +SPEC_SUFFIX="" +if [[ "${SPEC_DECODING:-}" == "mtp" ]]; then + SPEC_SUFFIX="_mtp" +fi + +export SCENARIO_SUBDIR="${SCENARIO_SUBDIR:-fixed_seq_len/}" +SCENARIO_SUBDIR="${SCENARIO_SUBDIR#/}" +SCENARIO_SUBDIR="${SCENARIO_SUBDIR%/}/" +# Prefer a framework-tagged script (e.g. qwen3.5_fp4_rtx6000pro_sglang.sh) so +# models with multiple inference engines can coexist; fall back to the name +# without an engine suffix for scripts that haven't been retagged. +BENCH_BASE="benchmarks/single_node/${SCENARIO_SUBDIR}${EXP_NAME%%_*}_${PRECISION}_rtx6000pro" +BENCH_SCRIPT="${BENCH_BASE}_${FRAMEWORK:-}${SPEC_SUFFIX}.sh" +if [[ ! -f "$GITHUB_WORKSPACE/$BENCH_SCRIPT" ]]; then + BENCH_SCRIPT="${BENCH_BASE}${SPEC_SUFFIX}.sh" +fi + +if [[ ! -f "$GITHUB_WORKSPACE/$BENCH_SCRIPT" ]]; then + echo "Benchmark script not found: $GITHUB_WORKSPACE/$BENCH_SCRIPT" >&2 + exit 1 +fi + +server_name="bmk-server-${RUNNER_NAME:-rtx6000pro-lat}" +server_name="${server_name//[^a-zA-Z0-9_.-]/-}" + +cleanup() { + docker rm -f "$server_name" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +# Clear a container left behind by a cancelled or interrupted workflow. +cleanup + +docker run \ + --rm \ + --pull=missing \ + --name="$server_name" \ + --runtime=nvidia \ + --gpus="$GPU_COUNT" \ + --network=host \ + --ipc=host \ + --privileged \ + --shm-size=32g \ + --ulimit memlock=-1 \ + --ulimit stack=67108864 \ + --security-opt seccomp=unconfined \ + --cap-add=SYS_PTRACE \ + --volume "$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE" \ + --volume "$GITHUB_WORKSPACE:/workspace/" \ + --workdir=/workspace/ \ + --env HF_TOKEN \ + --env HF_HUB_CACHE \ + --env MODEL \ + --env MODEL_PREFIX \ + --env MODEL_PATH \ + --env TP \ + --env PP_SIZE \ + --env DCP_SIZE \ + --env PCP_SIZE \ + --env EP_SIZE \ + --env DP_SIZE \ + --env DP_ATTENTION \ + --env GPU_COUNT \ + --env CONC \ + --env MAX_MODEL_LEN \ + --env ISL \ + --env OSL \ + --env FRAMEWORK \ + --env PRECISION \ + --env DISAGG \ + --env SPEC_DECODING \ + --env NUM_SPEC_TOKENS \ + --env RUN_EVAL \ + --env EVAL_ONLY \ + --env EVAL_LIMIT \ + --env EVAL_MAX_MODEL_LEN \ + --env RUNNER_TYPE \ + --env RUNNER_NAME \ + --env RESULT_FILENAME \ + --env RESULT_DIR \ + --env RANDOM_RANGE_RATIO \ + --env GPU_MEM_UTIL \ + --env AIPERF_FAILED_REQUEST_THRESHOLD \ + --env KV_OFFLOADING \ + --env KV_OFFLOAD_BACKEND \ + --env KV_OFFLOAD_BACKEND_METADATA \ + --env ROUTER_METADATA \ + --env KV_P2P_TRANSFER \ + --env TOTAL_CPU_DRAM_GB \ + --env DURATION \ + --env SCENARIO_TYPE \ + --env SCENARIO_SUBDIR \ + --env IS_AGENTIC \ + --env SWEBENCH_GEN_MODE \ + --env SWEBENCH_USE_MODAL \ + --env MODAL_TOKEN_ID \ + --env MODAL_TOKEN_SECRET \ + --env PROFILE \ + --env SGLANG_TORCH_PROFILER_DIR \ + --env VLLM_TORCH_PROFILER_DIR \ + --env VLLM_RPC_TIMEOUT \ + --env PYTHONDONTWRITEBYTECODE \ + --env PYTHONPYCACHEPREFIX=/tmp/pycache/ \ + --env PORT="$PORT" \ + --env CUDA_DEVICE_ORDER=PCI_BUS_ID \ + --env CUDA_VISIBLE_DEVICES \ + --env NCCL_IB_DISABLE \ + --entrypoint=/bin/bash \ + "$DOCKER_IMAGE" \ + "$BENCH_SCRIPT"