diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index f2e3252866..7ff2b96576 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -1706,25 +1706,6 @@ dsr1-fp4-mi355x-sglang-disagg-mtp: - "DECODE_MTP_SIZE=1" -dsv4-fp8-mi355x-sglang: - image: rocm/sgl-dev:deepseek-v4-mi35x - model: sgl-project/DeepSeek-V4-Pro-FP8 - model-prefix: dsv4 - runner: mi355x - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - # DSv4-Pro FP4 on MI355X via SGLang. Uses a rocm720 mi35x image built off the # amd/deepseek_v4 branch in sgl-project/sglang; the SHA is encoded in the # image tag, so bumping sglang is just an image tag bump here. Sweeps diff --git a/benchmarks/single_node/dsv4_fp8_mi355x.sh b/benchmarks/single_node/dsv4_fp8_mi355x.sh deleted file mode 100755 index 7ee626e87b..0000000000 --- a/benchmarks/single_node/dsv4_fp8_mi355x.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Transformers in the container doesn't recognize the `deepseek_v4` model_type. -# PR #23608's fallback in hf_transformers_utils.get_config tries to handle this -# by writing a patched config to /tmp, but in practice isn't catching the error -# in this image. Patch the cached config.json directly instead: set model_type -# to `deepseek_v3` so AutoConfig.from_pretrained succeeds, and keep -# architectures=['DeepseekV4ForCausalLM'] so SGLang dispatches to its native -# DSv4 model class (python/sglang/srt/models/deepseek_v4.py). -python3 << PYEOF -import json -from huggingface_hub import hf_hub_download -path = hf_hub_download(repo_id="$MODEL", filename="config.json") -with open(path) as f: - config = json.load(f) -if config.get("model_type") == "deepseek_v4": - config["model_type"] = "deepseek_v3" - with open(path, "w") as f: - json.dump(config, f, indent=2) - print(f"Patched {path}: model_type deepseek_v4 -> deepseek_v3") -else: - print(f"No patch needed: model_type is {config.get('model_type')!r}") -PYEOF - -# DSv4-specific SGLang env vars (from sgl-project/sglang#23608) -export SGLANG_OPT_USE_FUSED_COMPRESS=false -export SGLANG_OPT_USE_OLD_COMPRESSOR=true -export SGLANG_OPT_USE_TILELANG_SWA_PREPARE=false -export SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK=false -export SGLANG_OPT_USE_FUSED_HASH_TOPK=false -export SGLANG_HACK_FLASHMLA_BACKEND=torch -export SGLANG_OPT_DEEPGEMM_HC_PRENORM=false -export SGLANG_OPT_USE_TILELANG_MHC_PRE=false -export SGLANG_OPT_USE_TILELANG_MHC_POST=false -export SGLANG_ENABLE_THINKING=1 -export SGLANG_USE_AITER=1 -export SGLANG_USE_ROCM700A=1 -export SGLANG_TOPK_TRANSFORM_512_TORCH=1 -export SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 -export SGLANG_DSV4_FP4_EXPERTS=false -export SGLANG_OPT_DPSK_V4_RADIX=0 -export SGLANG_OPT_USE_OVERLAP_STORE_CACHE=false -export SGLANG_OPT_USE_FUSED_STORE_CACHE=false -export SGLANG_FORCE_TRITON_MOE_FP8=1 - -SERVER_LOG=/workspace/server.log -PORT=${PORT:-8888} - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -python3 -m sglang.launch_server \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --dp $TP \ - --enable-dp-attention \ - --trust-remote-code \ - --disable-radix-cache \ - --attention-backend compressed \ - --max-running-request 256 \ - --page-size 256 \ - --chunked-prefill-size 8192 \ - --disable-shared-experts-fusion \ - --disable-cuda-graph \ - --tool-call-parser deepseekv4 \ - --reasoning-parser deepseek-v4 \ - --watchdog-timeout 1800 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x