diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 4277e40c13..85047d8fb3 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -209,6 +209,25 @@ jobs: done fi + - name: Clear root-owned benchmark leftovers (pre-checkout) + run: | + set -uo pipefail + case "${GITHUB_WORKSPACE:-}" in + */actions-runner/_work/*) ;; + *) echo "Refusing to clean unexpected workspace '${GITHUB_WORKSPACE:-}'"; exit 0 ;; + esac + for d in benchmark_logs benchmark_artifacts; do + target="$GITHUB_WORKSPACE/$d" + [ -e "$target" ] || continue + if rm -rf "$target" 2>/dev/null; then + echo "removed $target" + elif sudo rm -rf "$target"; then + echo "removed $target (sudo)" + else + echo "WARNING: could not remove $target" >&2 + fi + done + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: token: ${{ secrets.REPO_PAT }} @@ -307,16 +326,19 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: multinode_server_logs_${{ env.RESULT_FILENAME }} - # multinode launchers package server logs into this tarball. - path: multinode_server_logs.tar.gz + path: | + multinode_server_logs.tar.gz + benchmark_artifacts/slurm_job-*.out + benchmark_artifacts/slurm_job-*.err if-no-files-found: ignore - name: Upload agentic aggregated result - if: ${{ !inputs.eval-only && inputs.scenario-type == 'agentic-coding' }} + if: ${{ always() && !inputs.eval-only && inputs.scenario-type == 'agentic-coding' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: bmk_agentic_${{ env.RESULT_FILENAME }} path: ${{ env.RESULT_FILENAME }}_conc*.json + if-no-files-found: ignore - name: Upload agentic raw results if: ${{ always() && inputs.scenario-type == 'agentic-coding' }} diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index d54e2e7357..b3cfe895d7 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -242,6 +242,7 @@ jobs: conc: ${{ matrix.config.conc[0] }} kv-offloading: ${{ matrix.config.kv-offloading }} kv-offload-backend: ${{ matrix.config.kv-offload-backend }} + total-cpu-dram-gb: ${{ matrix.config['total-cpu-dram-gb'] }} duration: ${{ inputs.duration-override != '' && inputs.duration-override || matrix.config.duration }} run-eval: false scenario-type: agentic-coding diff --git a/benchmarks/multi_node/agentic/kimik3_fp4_mi355x_vllm-disagg.sh b/benchmarks/multi_node/agentic/kimik3_fp4_mi355x_vllm-disagg.sh new file mode 100644 index 0000000000..9e86168c08 --- /dev/null +++ b/benchmarks/multi_node/agentic/kimik3_fp4_mi355x_vllm-disagg.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash + +# Agentic trace-replay recipe for a disaggregated vLLM server on MI355X +# (Kimi-K3 MXFP4, DSpark draft, 1P1D and 2P1D TP8). +# +# CI sibling of agentic/dsv4_fp4_mi355x_sglang-disagg.sh: driven entirely by +# environment variables and submits a SLURM job via amd_utils/submit.sh. Two +# things differ from that recipe because the engine differs: +# +# * the KV offload tier is vLLM's own SimpleCPUOffloadConnector, not HiCache, +# so none of the HICACHE_*/MC_* tunables apply. server_vllm.sh composes it +# with MoRIIO under MultiConnector on the prefill side (see +# build_kv_transfer_configs there). +# * the MoRI queue sizing is NOT set here. env.sh pins Kimi-K3 defaults +# (MORI_IO_SQ_BACKOFF_TIMEOUT_US=500000, QP_MAX_SEND_WR=8192, CQE=16384, +# SGE=2, TC_DISABLE=0) and job.slurm -e's them into the engine container; +# setting them here as well would create a second source of truth. +# +# The serve body (TP, cudagraph capture, fp8 KV, DSpark speculative-config, 1M +# context) lives in amd_utils/models_vllm.yaml under Kimi-K3. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/../../benchmark_lib.sh" + +# ISL/OSL are deliberately absent from this list. Agentic matrix entries carry no +# isl/osl (generate_sweep_configs.py builds them from the trace corpus instead), +# and they reach submit.sh only as positional filler that the agentic path never +# reads -- BENCH_INPUT_LEN/BENCH_OUTPUT_LEN are consumed by bench.sh, not by +# trace_replay.sh. Requiring them would abort the run over two unused numbers. +check_env_vars \ + CONC_LIST \ + IMAGE \ + SPEC_DECODING \ + MODEL_PATH \ + PREFILL_NUM_WORKERS \ + PREFILL_TP \ + PREFILL_EP \ + PREFILL_DP_ATTN \ + DECODE_NUM_WORKERS \ + DECODE_TP \ + DECODE_EP \ + DECODE_DP_ATTN \ + PREFILL_NODES \ + DECODE_NODES \ + RANDOM_RANGE_RATIO \ + DURATION \ + KV_OFFLOADING \ + IS_AGENTIC \ + FRAMEWORK + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +set -x + +# Use upstreamed multi_node scripts (no external clone needed) +cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 + +export ISL="${ISL:-0}" +export OSL="${OSL:-0}" + +# ── SLURM / image ── +# 12 h rather than the fixed-seq-len 8 h: a 3600 s replay sits behind ~2.8 TB of +# weight loading on both roles plus cudagraph capture, and benchmark-multinode-tmpl.yml +# already allows 780 min of wall clock for agentic-coding jobs. +export TIME_LIMIT="${TIME_LIMIT:-12:00:00}" +export MODEL_PATH=$MODEL_PATH +export MODEL_NAME=$MODEL_NAME +export CONTAINER_IMAGE=$IMAGE + +# ── MoRIIO write + wait_all ── +# job.slurm derives a node-local image from CONTAINER_IMAGE with the exact +# ROCm/mori commit SGLang MI35x pins (#341 / IOEngine.wait_all). The live server +# only verifies the baked capability; it never installs a wheel at startup. +export MORI_WAITALL_BUILD="${MORI_WAITALL_BUILD:-1}" +export MORI_WAITALL_COMMIT="${MORI_WAITALL_COMMIT:-f7e6ac6863c53821bc7afb91a578cc6ce38fcad0}" +export MORIIO_READ_MODE="${MORIIO_READ_MODE:-false}" + +# job.slurm's fallback pin is routinely garbage-collected from Docker Hub +# (only ~16 nightlies are retained), so pin the tag this arm was validated +# against. Keep it equal to the router version recorded in amd-master.yaml. +export VLLM_ROUTER_IMAGE="${VLLM_ROUTER_IMAGE:-vllm/vllm-router:nightly-20260809-d2ba586}" + +# ── Identity / result naming ── +export MODEL_PREFIX="${MODEL_PREFIX:-kimik3}" +export PRECISION="${PRECISION:-fp4}" +export RESULT_FILENAME="${RESULT_FILENAME:-${RUNNER_NAME:-kimik3-fp4-disagg-agentic}}" + +# ── Agentic benchmark params ── +export DURATION="${DURATION:-3600}" +# K3's native context. Kept equal to the --max-model-len in models_vllm.yaml so +# the client's --max-context-length matches what the servers actually serve. +export MAX_MODEL_LEN="${MAX_MODEL_LEN:-1048576}" + +# ── KV cache offloading ── +# KV_OFFLOADING=none | dram (from YAML). The only backend this arm implements is +# vLLM's built-in SimpleCPUOffloadConnector ("vllm-simple"); the capacity itself +# comes from the matrix as TOTAL_CPU_DRAM_GB, sized from the PREFILL worker's +# per-node GPU footprint because only prefill offloads to CPU DRAM today. +# +# check_env_vars above already rejects KV_OFFLOADING=dram without a backend and +# without a positive TOTAL_CPU_DRAM_GB (benchmark_lib.sh), so the only thing left +# to assert here is that the backend is one this arm can actually serve. +export KV_OFFLOADING="${KV_OFFLOADING:-none}" +if [[ "$KV_OFFLOADING" != "none" ]]; then + export KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-vllm-simple}" + if [[ "$KV_OFFLOAD_BACKEND" != "vllm-simple" ]]; then + echo "ERROR: KV_OFFLOAD_BACKEND=$KV_OFFLOAD_BACKEND unsupported on this arm (vllm-simple only)" >&2 + exit 1 + fi +fi + +# ── Server metrics ── +# vLLM exposes Prometheus /metrics on every engine port unconditionally (no +# --enable-metrics equivalent), so the aiperf server-metrics scrape always has +# somewhere to point. server_vllm.sh builds the per-worker URL list. +export ENABLE_METRICS="${ENABLE_METRICS:-1}" + +# ── MTP ── +# Labels the result JSON only: the draft is configured by --speculative-config in +# models_vllm.yaml (k=2 DSpark), and server_vllm.sh never reads this. +export DECODE_MTP_SIZE="${DECODE_MTP_SIZE:-2}" + +# Derive EP/DP enable flags from the topology inputs. +if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then +export PREFILL_ENABLE_EP=false +else +export PREFILL_ENABLE_EP=true +fi + +if [[ "$PREFILL_DP_ATTN" == "true" ]]; then +export PREFILL_ENABLE_DP=true +else +export PREFILL_ENABLE_DP=false +fi + +if [[ "${DECODE_EP:-1}" -eq 1 ]]; then +export DECODE_ENABLE_EP=false +else +export DECODE_ENABLE_EP=true +fi + +if [[ "$DECODE_DP_ATTN" == "true" ]]; then +export DECODE_ENABLE_DP=true +else +export DECODE_ENABLE_DP=false +fi + +# Launch the job. CONC_LIST is space-delimited in YAML; submit.sh wants 'x'. +JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ + $PREFILL_NUM_WORKERS \ + $DECODE_NODES \ + $DECODE_NUM_WORKERS \ + $ISL $OSL "${CONC_LIST// /x}" inf \ + ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ + ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ + ${PREFILL_TP} ${DECODE_TP} \ + ${RANDOM_RANGE_RATIO}) + +if [[ $? -ne 0 ]]; then + echo "Failed to submit job" >&2 + exit 1 +fi + +echo "$JOB_ID" diff --git a/benchmarks/multi_node/amd_utils/apply_k3_moriio_patches.sh b/benchmarks/multi_node/amd_utils/apply_k3_moriio_patches.sh new file mode 100644 index 0000000000..e8b8fc481e --- /dev/null +++ b/benchmarks/multi_node/amd_utils/apply_k3_moriio_patches.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Apply #51052 Kimi-K3 MoRIIO hybrid transfer into the engine container. +# +# This is deliberately a unified diff rather than a full-file overlay. An +# overlay replaces whole modules, so it also pins every unrelated symbol in +# those files to whichever vLLM it was captured from: a cb810-era snapshot +# silently downgraded vllm/v1/core/kv_cache_utils.py and the newer engine died +# with "cannot import name 'update_kv_cache_capacity'". The diff only touches +# what #51052 changes, and fails loudly when it no longer fits the image. +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PATCH_FILE="${K3_MORIIO_PATCH:-$HERE/patches/k3_moriio_51052.patch}" +ROOT="$(python3 -c 'import importlib.util as u, os; print(os.path.dirname(os.path.dirname(u.find_spec("vllm").origin)))')" + +if [[ -z "$ROOT" || ! -d "$ROOT/vllm" ]]; then + echo "[k3-moriio] ERROR: could not resolve vLLM root (ROOT='$ROOT')" >&2 + exit 1 +fi + +MORIIO_DIR="$ROOT/vllm/distributed/kv_transfer/kv_connector/v1/moriio" +if grep -RqsE '_draft_only_layers|as_attn_mamba' "$MORIIO_DIR" \ + && python3 -c 'from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import MoRIIOConnector' 2>/dev/null; then + echo "[k3-moriio] #51052 already applied (markers + import ok)" + exit 0 +fi + +if [[ ! -f "$PATCH_FILE" ]]; then + echo "[k3-moriio] ERROR: patch not found at $PATCH_FILE" >&2 + exit 1 +fi + +if grep -q $'\r' "$PATCH_FILE"; then + tmp=$(mktemp) + tr -d '\r' < "$PATCH_FILE" > "$tmp" + PATCH_FILE="$tmp" + trap 'rm -f "$tmp"' EXIT +fi + +if (cd "$ROOT" && git apply -p1 "$PATCH_FILE" 2>/dev/null); then + echo "[k3-moriio] applied #51052 with git apply" +elif patch -p1 -d "$ROOT" --forward --no-backup-if-mismatch < "$PATCH_FILE"; then + echo "[k3-moriio] applied #51052 with patch" +else + echo "[k3-moriio] ERROR: #51052 does not apply to this image." >&2 + echo "[k3-moriio] Regenerate patches/k3_moriio_51052.patch against it; do not" >&2 + echo "[k3-moriio] substitute a full-file overlay, which downgrades unrelated modules." >&2 + find "$ROOT" -name '*.rej' -newermt '-5 minutes' -print >&2 2>/dev/null || true + exit 1 +fi + +python3 - <<'PY' +from vllm.distributed.kv_transfer.kv_connector.v1.moriio import moriio_common as c +from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import ( + MoRIIOConnector, +) +from vllm.v1.core import kv_cache_utils + +assert hasattr(c, "as_attn_mamba"), "as_attn_mamba missing after patch" +assert MoRIIOConnector is not None +# The engine imports this from kv_cache_utils; a stale replacement of that +# module is exactly the failure mode this script now refuses to create. +assert hasattr(kv_cache_utils, "update_kv_cache_capacity"), ( + "kv_cache_utils lost update_kv_cache_capacity" +) +print("[k3-moriio] #51052 applied; IMPORT_OK") +PY diff --git a/benchmarks/multi_node/amd_utils/ensure_mori_wait_all.sh b/benchmarks/multi_node/amd_utils/ensure_mori_wait_all.sh new file mode 100644 index 0000000000..8f11c24464 --- /dev/null +++ b/benchmarks/multi_node/amd_utils/ensure_mori_wait_all.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Verify that job.slurm selected the source-built mori #341 image. Serving must +# never mutate site-packages or silently fall back to the Python polling path. +set -euo pipefail + +python3 - <<'PY' +import mori +from mori.io import IOEngine, StatusCode + +version = getattr(mori, "__version__", "?") +wait_all = hasattr(IOEngine, "wait_all") +print("[k3-mori-waitall] mori=", version, "at", mori.__file__) +print("[k3-mori-waitall] IOEngine.wait_all=", wait_all) +if not wait_all: + raise RuntimeError( + "IOEngine.wait_all missing: job.slurm must prepare the pinned " + "ROCm/mori#341 derived image before serving" + ) +assert StatusCode is not None +PY diff --git a/benchmarks/multi_node/amd_utils/env.sh b/benchmarks/multi_node/amd_utils/env.sh index a05182bf16..683cb4875e 100755 --- a/benchmarks/multi_node/amd_utils/env.sh +++ b/benchmarks/multi_node/amd_utils/env.sh @@ -50,11 +50,17 @@ export NCCL_IB_HCA=${NCCL_IB_HCA:-$IBDEVICES} # ============================================================================= # Shared by the vLLM MoRIIOConnector and the SGLang/MoRI KV-transfer path. -export MORI_IO_SQ_BACKOFF_TIMEOUT_US=50000 -export MORI_IO_QP_MAX_SEND_WR=16384 -export MORI_IO_QP_MAX_CQE=32768 -export MORI_IO_QP_MAX_SGE=2 -export MORI_IO_TC_DISABLE=0 +if [[ "${MODEL_NAME:-}" == "Kimi-K3" ]]; then + export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}" + export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-8192}" + export MORI_IO_QP_MAX_CQE="${MORI_IO_QP_MAX_CQE:-16384}" +else + export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-50000}" + export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-16384}" + export MORI_IO_QP_MAX_CQE="${MORI_IO_QP_MAX_CQE:-32768}" +fi +export MORI_IO_QP_MAX_SGE="${MORI_IO_QP_MAX_SGE:-2}" +export MORI_IO_TC_DISABLE="${MORI_IO_TC_DISABLE:-0}" # QoS/DSCP configuration # Priority order: 1) Set by runner, 2) Detect via nicctl, 3) Detect from hostname @@ -180,6 +186,15 @@ $1 == "DSCP" && $2 == ":" && $NF == p { set +x echo "[INFO] IBDEVICES=$IBDEVICES UCX_NET_DEVICES=$UCX_NET_DEVICES NCCL_SOCKET_IFNAME=$NCCL_SOCKET_IFNAME UCX_IB_GID_INDEX=$UCX_IB_GID_INDEX UCX_IB_TRAFFIC_CLASS=${UCX_IB_TRAFFIC_CLASS:-unset}" + if [[ "$MODEL_NAME" == "Kimi-K3" ]]; then + export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}" + export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-8192}" + export MORI_IO_QP_MAX_CQE="${MORI_IO_QP_MAX_CQE:-16384}" + export MORI_IO_QP_MAX_SGE="${MORI_IO_QP_MAX_SGE:-2}" + export MORI_IO_TC_DISABLE="${MORI_IO_TC_DISABLE:-0}" + echo "[INFO] Kimi-K3 MoRI IO: SQ_BACKOFF=${MORI_IO_SQ_BACKOFF_TIMEOUT_US}us SEND_WR=${MORI_IO_QP_MAX_SEND_WR} CQE=${MORI_IO_QP_MAX_CQE} SGE=${MORI_IO_QP_MAX_SGE} TC_DISABLE=${MORI_IO_TC_DISABLE}" + fi + else # ========================================================================= # SGLang-specific environment diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 895f1ac2fb..7ee6b6280c 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -42,13 +42,26 @@ if [[ -z "${DOCKER_IMAGE_NAME:-}" ]]; then fi MODEL_NAME="${MODEL_NAME:-None}" -if ! grep -q "^${MODEL_NAME}:" "$MODELS_YAML"; then - echo "Error: Model '$MODEL_NAME' not found in $MODELS_YAML" - echo "Available models:" - grep -E '^[A-Za-z]' "$MODELS_YAML" | sed 's/:.*$//' | sed 's/^/ - /' - exit 1 +# Resolve the models.yaml entry the same way server_sglang.sh does: agentic runs +# (IS_AGENTIC) use the '-AgentX' recipe, non-agentic disaggregated runs use +# '-DI'. Fall back to the bare model name if the variant key is absent. +# MODEL_NAME itself is left unchanged so env.sh/server_sglang.sh still see the base name. +if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then + MODEL_YAML_KEY="${MODEL_NAME}-AgentX" +else + MODEL_YAML_KEY="${MODEL_NAME}-DI" +fi +if ! grep -q "^${MODEL_YAML_KEY}:" "$MODELS_YAML"; then + if grep -q "^${MODEL_NAME}:" "$MODELS_YAML"; then + MODEL_YAML_KEY="$MODEL_NAME" + else + echo "Error: Model '$MODEL_YAML_KEY' (nor bare '$MODEL_NAME') not found in $MODELS_YAML" + echo "Available models:" + grep -E '^[A-Za-z]' "$MODELS_YAML" | sed 's/:.*$//' | sed 's/^/ - /' + exit 1 + fi fi -echo "Model found: $MODEL_NAME" +echo "Model found: $MODEL_YAML_KEY (MODEL_NAME=$MODEL_NAME, IS_AGENTIC=${IS_AGENTIC:-0})" RUN_FILE="server.sh" echo "Runfile set: $RUN_FILE" @@ -57,29 +70,6 @@ echo "Runfile set: $RUN_FILE" # $(pwd) is amd_utils/ (the sbatch submit dir); go up 3 levels to reach the repo root. export DI_REPO_DIR=$(cd "$(pwd)/../../.." && pwd) -# ── In-tree sglang patches: auto-apply for known-affected images ────── -# sglang v0.5.12.post1 ships a known-broken MoRI PD-disaggregation -# backend that crashes hybrid-attention models (GLM-5, Qwen3.5-MoE, -# anything with state_types: List[StateType]) at startup. We carry an -# in-tree overlay of mori/conn.py that fixes the wire format + the -# legacy state_type fallback (see patches/README.md for the bug -# analysis and patch detail). -# -# Auto-applied when the image tag contains "v0.5.12.post1", unless the -# caller sets MORI_CONN_PATCH=skip. The overlay is appended to -# ${EXTRA_DOCKER_MOUNTS:-} so callers can still inject other mounts. -# Dedup guard avoids double-mounting if EXTRA_DOCKER_MOUNTS already -# contains the target path (docker rejects duplicate destinations). -_MORI_PATCH_FILE="$DI_REPO_DIR/benchmarks/multi_node/amd_utils/patches/mori_conn.py" -_MORI_PATCH_TARGET="/sgl-workspace/sglang/python/sglang/srt/disaggregation/mori/conn.py" -if [[ "${MORI_CONN_PATCH:-auto}" != "skip" ]] \ - && [[ -f "$_MORI_PATCH_FILE" ]] \ - && [[ "${DOCKER_IMAGE_NAME:-}" == *"v0.5.12.post1"* ]] \ - && [[ "${EXTRA_DOCKER_MOUNTS:-}" != *"$_MORI_PATCH_TARGET"* ]]; then - EXTRA_DOCKER_MOUNTS="${EXTRA_DOCKER_MOUNTS:-} -v ${_MORI_PATCH_FILE}:${_MORI_PATCH_TARGET}:ro" - export EXTRA_DOCKER_MOUNTS - echo "[job.slurm] auto-applied MoRI conn.py overlay: ${_MORI_PATCH_FILE}" -fi xP="${xP:-1}" yD="${yD:-1}" @@ -106,6 +96,10 @@ DECODE_MTP_SIZE=${DECODE_MTP_SIZE:-0} ROUTER_TYPE="${ROUTER_TYPE:-vllm-router}" ROUTER_PORT="${ROUTER_PORT:-30000}" PROXY_PING_PORT="${PROXY_PING_PORT:-36367}" +# At info the router logs several CONSISTENT_HASH_DEBUG lines per request, which +# dominated the artifact. Raise to info when routing decisions are the thing +# under investigation. +ROUTER_LOG_LEVEL="${ROUTER_LOG_LEVEL:-warn}" # ============================================================================= # Model Path Resolution @@ -202,11 +196,41 @@ else fi } - if check_model_path "$MODEL_DIR/$MODEL_NAME" "$MODEL_DIR"; then + # Extract hf_dir from models.yaml (same as vllm-disagg path above) + SGL_DISK_DIR_NAME=$(awk '/^'"$MODEL_NAME"':/{found=1; next} + found && /^[^ ]/{exit} + found && /hf_dir:/{gsub(/[" ]/, "", $2); print $2; exit}' "$MODELS_YAML") + SGL_DISK_DIR_NAME="${SGL_DISK_DIR_NAME:-$MODEL_NAME}" + + # Prefer the caller-supplied MODEL_PATH (recipe scripts set this explicitly); + # fall back to MODEL_DIR/hf_dir then MODEL_DIR/MODEL_NAME. + if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL_DIR" ]]; then + # Caller already resolved the path (e.g. MODEL_PATH=/it-share/hf_cache/models--...) + # Use it directly if it exists on all nodes, otherwise try subdirectory combos. + if check_model_path "$MODEL_PATH" "MODEL_PATH (caller-supplied)"; then + echo "Selected MODEL_PATH: $MODEL_PATH (caller-supplied, available on all nodes)" + elif check_model_path "$MODEL_PATH/$SGL_DISK_DIR_NAME" "$MODEL_PATH/$SGL_DISK_DIR_NAME"; then + MODEL_PATH="$MODEL_PATH/$SGL_DISK_DIR_NAME" + echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" + elif check_model_path "$MODEL_PATH/$MODEL_NAME" "$MODEL_PATH/$MODEL_NAME"; then + MODEL_PATH="$MODEL_PATH/$MODEL_NAME" + echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" + else + echo "FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in:" + echo " - $MODEL_PATH" + echo " - $MODEL_PATH/$SGL_DISK_DIR_NAME" + echo " - $MODEL_PATH/$MODEL_NAME" + exit 1 + fi + elif check_model_path "$MODEL_DIR/$SGL_DISK_DIR_NAME" "$MODEL_DIR/$SGL_DISK_DIR_NAME"; then + MODEL_PATH="$MODEL_DIR/$SGL_DISK_DIR_NAME" + echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" + elif check_model_path "$MODEL_DIR/$MODEL_NAME" "$MODEL_DIR"; then MODEL_PATH="$MODEL_DIR/$MODEL_NAME" echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" else echo "FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in:" + echo " - $MODEL_DIR/$SGL_DISK_DIR_NAME" echo " - $MODEL_DIR/$MODEL_NAME" exit 1 fi @@ -310,6 +334,12 @@ export RUNNER_TYPE="${RUNNER_TYPE:-}" export RESULT_FILENAME="${RESULT_FILENAME:-}" export SPEC_DECODING="${SPEC_DECODING:-}" export IS_MULTINODE="${IS_MULTINODE:-false}" +export SWEBENCH_USE_MODAL="${SWEBENCH_USE_MODAL:-false}" +export MODAL_TOKEN_ID="${MODAL_TOKEN_ID:-}" +export MODAL_TOKEN_SECRET="${MODAL_TOKEN_SECRET:-}" +export HF_TOKEN="${HF_TOKEN:-}" +export SCENARIO_TYPE="${SCENARIO_TYPE:-}" +export EVAL_LIMIT="${EVAL_LIMIT:-}" SANITIZED_USER=$(echo "$USER_NAME" | tr -c 'a-zA-Z0-9_.-' '_') export DOCKER_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_${MODEL_NAME}_${SLURM_JOB_ID}" @@ -317,14 +347,67 @@ export DOCKER_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_${MODEL_NAME}_${S # vLLM external router container. # NOTE: vllm/vllm-router only retains ~16 recent nightlies on Docker Hub; older # dated tags are garbage-collected (manifest unknown) -VLLM_ROUTER_IMAGE="${VLLM_ROUTER_IMAGE:-vllm/vllm-router:nightly-20260617-e667ebb}" +VLLM_ROUTER_IMAGE="${VLLM_ROUTER_IMAGE:-vllm/vllm-router:nightly-20260716-1fbcde7}" ROUTER_CONT_NAME="router_vllm_${SANITIZED_USER}_${SLURM_JOB_ID}" +# Separate agentic benchmark-client container (see CLIENT_IMAGE handling below). +CLIENT_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_client_${SLURM_JOB_ID}" export RUN_FILE_FULL="$WS_PATH/${RUN_FILE}" SELECTED_NODELIST_SRUN=$(echo "$SELECTED_NODES" | paste -sd,) +# ============================================================================= +# RDMA QoS / DCQCN Pre-flight Check +# ============================================================================= +# Gate the run on NIC QoS (PFC/DSCP) and DCQCN config on every node before +# any container/GPU time is spent. Runs on the bare host (nicctl is a host +# tool). See rdma_check.sh for details. +RDMA_CHECK_SCRIPT="$(pwd)/helpers/rdma_check.sh" +if [[ "${SKIP_RDMA_CHECK:-0}" == "1" ]]; then + echo "[INFO] SKIP_RDMA_CHECK=1 set; skipping RDMA QoS/DCQCN pre-flight check" +elif [[ -f "$RDMA_CHECK_SCRIPT" ]]; then + echo "Checking RDMA QoS/DCQCN configuration on all $NUM_NODES allocated node(s)..." + srun --nodelist="$SELECTED_NODELIST_SRUN" --ntasks=$NUM_NODES bash "$RDMA_CHECK_SCRIPT" + RDMA_CHECK_RC=$? + if [[ $RDMA_CHECK_RC -ne 0 ]]; then + echo "FATAL: RDMA QoS/DCQCN pre-flight check failed on one or more nodes (see [FAIL] lines above)." + echo " Set SKIP_RDMA_CHECK=1 to bypass (not recommended -- MoRI cross-node transfers would run unprotected by PFC/DCQCN)." + exit 1 + fi + echo "RDMA QoS/DCQCN pre-flight check passed on all $NUM_NODES node(s)" +else + echo "[WARN] $RDMA_CHECK_SCRIPT not found; skipping RDMA QoS/DCQCN pre-flight check" +fi + +collect_server_logs() { + local shared_dir="${BENCHMARK_LOGS_DIR}/server_logs/slurm_job-${SLURM_JOB_ID}" + mkdir -p "$shared_dir/nodes" + echo "[logs] collecting per-node server logs into $shared_dir/nodes" + srun --nodelist="$SELECTED_NODELIST_SRUN" --nodes="$NUM_NODES" \ + --ntasks="$NUM_NODES" --ntasks-per-node=1 bash -c ' + host=$(hostname -s) + src="/tmp/slurm_job-'"$SLURM_JOB_ID"'" + dst="'"$shared_dir"'/nodes/$host" + mkdir -p "$dst" + if [[ -d "$src" ]]; then + cp -r "$src"/. "$dst"/ 2>/dev/null || true + fi + eval "$DOCKER_CMD_DETECT" + $DOCKER_CMD logs "'"$DOCKER_CONT_NAME"'" >"$dst/container.log" 2>&1 || true + $DOCKER_CMD logs "'"$ROUTER_CONT_NAME"'" >"$dst/router_container.log" 2>&1 || true + printf "host=%s\njob=%s\n" "$host" "'"$SLURM_JOB_ID"'" >"$dst/metadata.txt" + ' 2>/dev/null || echo "[logs] WARN: per-node log collection was incomplete" +} + cleanup() { - echo "[${SLURM_JOB_ID}] termination received on $(hostname); cleaning up..." + echo "[${SLURM_JOB_ID}] termination received on $(hostname); cleaning up container + stale logs..." + collect_server_logs || true + # Backstop: on scancel/timeout/step-hang the foreground `exec docker run` + # client is killed before --rm can fire, so the container (and its GPU/ports/ + # :30000) leaks and the step never returns. Force-remove THIS job's container + # on every allocated node. Scoped to $DOCKER_CONT_NAME so it never touches + # other users' containers. (Ported from InferenceY 51ebfa88.) + srun --nodelist="$SELECTED_NODELIST_SRUN" \ + bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' 2>/dev/null || true' 2>/dev/null || true rm -rf ${SLURM_SUBMIT_DIR}/logs 2>/dev/null || true echo "[${SLURM_JOB_ID}] cleanup done." } @@ -342,6 +425,58 @@ srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c ' echo "NFS cache refreshed on $(hostname)" ' +# Kimi-K3 MoRIIO write mode requires IOEngine.wait_all from ROCm/mori #341. +# Build it once per node into a derived image before serving, so package +# installation stays out of the live engine container. +# +# Prefill and decode must agree on the transfer direction, so the read/write +# decision is made here once for the whole job and propagated to every +# container via MORIIO_READ_MODE. If the derived image cannot be produced on +# every node, the job falls back to READ mode on the base image rather than +# serving a mismatched pair or failing outright. +if [[ "$ENGINE" == "vllm-disagg" && "$MODEL_NAME" == "Kimi-K3" && + "${MORI_WAITALL_BUILD:-0}" == "1" && "${DRY_RUN:-0}" != "1" ]]; then + MORI_PIN="${MORI_WAITALL_COMMIT:-f7e6ac6863c53821bc7afb91a578cc6ce38fcad0}" + MORI_SOURCE_DIR="${MORI_WAITALL_SOURCE_DIR:-${BENCHMARK_LOGS_DIR}/mori-${MORI_PIN:0:8}}" + # No registry prefix: Docker resolves "localhost/..." as a registry host and + # tries to pull the derived image over HTTP instead of using the local one. + MORI_DERIVED_IMAGE="${MORI_WAITALL_IMAGE:-k3-moriio-${SLURM_JOB_ID}:mori341}" + MORI_WAITALL_OK=1 + + if [[ ! -d "$MORI_SOURCE_DIR/.git" || + "$(git -C "$MORI_SOURCE_DIR" rev-parse HEAD 2>/dev/null || true)" != "$MORI_PIN" ]]; then + rm -rf "$MORI_SOURCE_DIR" + git clone --filter=blob:none https://github.com/ROCm/mori.git "$MORI_SOURCE_DIR" \ + || MORI_WAITALL_OK=0 + if [[ "$MORI_WAITALL_OK" == "1" ]]; then + git -C "$MORI_SOURCE_DIR" checkout -q "$MORI_PIN" || MORI_WAITALL_OK=0 + git -C "$MORI_SOURCE_DIR" submodule update --init --recursive --depth 1 \ + || MORI_WAITALL_OK=0 + fi + fi + + if [[ "$MORI_WAITALL_OK" == "1" ]]; then + echo "[mori-image] source HEAD=$(git -C "$MORI_SOURCE_DIR" rev-parse HEAD)" + srun --nodelist="$SELECTED_NODELIST_SRUN" --nodes="$NUM_NODES" \ + --ntasks="$NUM_NODES" --ntasks-per-node=1 \ + bash "$DI_REPO_DIR/benchmarks/multi_node/amd_utils/prepare_mori_waitall_image.sh" \ + "$DOCKER_IMAGE_NAME" "$MORI_DERIVED_IMAGE" "$MORI_SOURCE_DIR" \ + || MORI_WAITALL_OK=0 + fi + + if [[ "$MORI_WAITALL_OK" == "1" ]]; then + DOCKER_IMAGE_NAME="$MORI_DERIVED_IMAGE" + echo "[mori-image] serving from $DOCKER_IMAGE_NAME (wait_all available)" + elif [[ "${MORI_WAITALL_REQUIRED:-0}" == "1" ]]; then + echo "FATAL: could not prepare the wait_all mori image and MORI_WAITALL_REQUIRED=1" >&2 + exit 1 + else + export MORIIO_READ_MODE=true + echo "[mori-image] WARNING: wait_all image unavailable on at least one node;" \ + "falling back to MoRIIO READ mode on $DOCKER_IMAGE_NAME" + fi +fi + # ============================================================================= # Build engine-specific Docker environment variables # ============================================================================= @@ -366,7 +501,6 @@ DOCKER_ENV_COMMON=( -e BENCH_MAX_CONCURRENCY=\$BENCH_MAX_CONCURRENCY -e BENCH_REQUEST_RATE=\$BENCH_REQUEST_RATE -e TQDM_MININTERVAL=\$TQDM_MININTERVAL - -e DRY_RUN=\$DRY_RUN -e BENCHMARK_LOGS_DIR=/benchmark_logs -e ENGINE=\$ENGINE -e WS_PATH=${WS_PATH} @@ -379,14 +513,67 @@ DOCKER_ENV_COMMON=( -e RUNNER_TYPE=\$RUNNER_TYPE -e RESULT_FILENAME=\$RESULT_FILENAME -e SPEC_DECODING=\$SPEC_DECODING + # DISAGG was never forwarded into the container at all (not even under a + # different name), so process_agentic_result.py's env_bool("DISAGG") always + # defaulted to false in the result JSON regardless of the actual topology. + -e DISAGG=\${DISAGG:-false} -e PREFILL_TP_SIZE=\$PREFILL_TP_SIZE + # PREFILL_TP/DECODE_TP/*_NUM_WORKERS (below, undecorated -- distinct from the + # *_SIZE vars server_sglang.sh uses for launch args) are what + # process_agentic_result.py's _gpu_shape() reads for multinode runs. Without + # these the container never sees them (only *_SIZE was passed), so agentic + # result JSONs silently recorded tp=0 / prefill_tp=0 / prefill_num_workers=0 + # for every multinode run. + -e PREFILL_TP=\$PREFILL_TP + -e PREFILL_NUM_WORKERS=\$PREFILL_NUM_WORKERS -e PREFILL_ENABLE_EP=\$PREFILL_ENABLE_EP -e PREFILL_ENABLE_DP=\$PREFILL_ENABLE_DP + # process_agentic_result.py reads these (distinct from the ENABLE_* launch + # booleans above) for agg_bmk.json's parallelism metadata; without them it + # defaulted to ep=1/dp_attention=false even for DEP8 arms. + -e PREFILL_EP=\${PREFILL_EP:-1} + -e PREFILL_DP_ATTN=\${PREFILL_DP_ATTN:-false} + -e PREFILL_CONTEXT_LENGTH=\${PREFILL_CONTEXT_LENGTH:-} + -e PREFILL_CHUNKED_PREFILL_SIZE=\${PREFILL_CHUNKED_PREFILL_SIZE:-} + -e DISABLE_CUSTOM_ALL_REDUCE=\${DISABLE_CUSTOM_ALL_REDUCE:-} + -e MAX_MODEL_LEN=\${MAX_MODEL_LEN:-} + -e DURATION=\${DURATION:-1800} + -e IS_AGENTIC=\${IS_AGENTIC:-0} + -e KV_OFFLOADING=\${KV_OFFLOADING:-none} + -e KV_OFFLOAD_BACKEND=\${KV_OFFLOAD_BACKEND:-} + -e KV_OFFLOAD_BACKEND_METADATA=\"\${KV_OFFLOAD_BACKEND_METADATA:-}\" + -e TOTAL_CPU_DRAM_GB=\${TOTAL_CPU_DRAM_GB:-} + -e ENABLE_METRICS=\${ENABLE_METRICS:-0} + -e PREFILL_ROUTER_POLICY=\${PREFILL_ROUTER_POLICY:-random} + -e DECODE_ROUTER_POLICY=\${DECODE_ROUTER_POLICY:-random} + -e IBDEVICES=\${IBDEVICES:-} + -e MORI_IO_SQ_BACKOFF_TIMEOUT_US=\${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-} + -e MORI_IO_QP_MAX_SEND_WR=\${MORI_IO_QP_MAX_SEND_WR:-} + -e MORI_IO_QP_MAX_CQE=\${MORI_IO_QP_MAX_CQE:-} + -e MORI_IO_QP_MAX_SGE=\${MORI_IO_QP_MAX_SGE:-} + -e MORI_IO_TC_DISABLE=\${MORI_IO_TC_DISABLE:-} + -e MORI_RDMA_TC=\${MORI_RDMA_TC:-} + -e MORI_IO_TC=\${MORI_IO_TC:-} + -e MORIIO_READ_MODE=\${MORIIO_READ_MODE:-} -e DECODE_TP_SIZE=\$DECODE_TP_SIZE + -e DECODE_TP=\$DECODE_TP + -e DECODE_NUM_WORKERS=\$DECODE_NUM_WORKERS -e DECODE_ENABLE_EP=\$DECODE_ENABLE_EP -e DECODE_ENABLE_DP=\$DECODE_ENABLE_DP + # See PREFILL_EP/PREFILL_DP_ATTN comment above -- decode counterpart. + -e DECODE_EP=\${DECODE_EP:-1} + -e DECODE_DP_ATTN=\${DECODE_DP_ATTN:-false} -e DECODE_MTP_SIZE=\$DECODE_MTP_SIZE -e IS_MULTINODE=\$IS_MULTINODE + -e DRY_RUN=\${DRY_RUN:-0} + # SWE-bench agentic eval runs inside this container and needs Modal/HF + # credentials to launch sandboxes and download datasets. + -e SWEBENCH_USE_MODAL=\${SWEBENCH_USE_MODAL:-false} + -e MODAL_TOKEN_ID=\${MODAL_TOKEN_ID:-} + -e MODAL_TOKEN_SECRET=\${MODAL_TOKEN_SECRET:-} + -e HF_TOKEN=\${HF_TOKEN:-} + -e SCENARIO_TYPE=\${SCENARIO_TYPE:-} + -e \"EVAL_LIMIT=\${EVAL_LIMIT:-}\" ) # Engine-specific env vars @@ -419,7 +606,6 @@ elif [[ "$ENGINE" == "atom-disagg" ]]; then -e MAX_MODEL_LEN=${MAX_MODEL_LEN:-} -e MAX_NUM_BATCHED_TOKENS=${MAX_NUM_BATCHED_TOKENS:-} -e EXTRA_SERVER_ARGS=\${EXTRA_SERVER_ARGS:-} - -e IBDEVICES=${IBDEVICES:-} ) else DOCKER_ENV_ENGINE=( @@ -427,9 +613,60 @@ else ) fi +# HiCache / Mooncake settings are delivered via a bind-mounted config file rather +# than a long list of docker -e flags. Write it once to the shared benchmark-logs +# dir (already a host path, visible on every node) and mount it read-only at +# /config/hicache_mc.env, where env.sh sources it before applying its defaults. +# Empty values are preserved so env.sh's "${VAR:-default}" fallbacks still apply. +HICACHE_MC_CONFIG="${BENCHMARK_LOGS_DIR}/hicache_mc_${SLURM_JOB_ID}.env" +cat > "$HICACHE_MC_CONFIG" < $HICACHE_MC_CONFIG" + # Engine-specific container filter for pre-clean CONT_FILTER="name=^container_${ENGINE}_" +# ============================================================================= +# Optional: separate benchmark-client image (agentic runs) — node-0 sibling +# ============================================================================= +# When CLIENT_IMAGE is set, node 0 runs the aiperf trace replay in its own +# sibling container built from CLIENT_IMAGE (which ships a pre-baked aiperf + +# deps), instead of rebuilding the aiperf venv inside the server container every +# run. Give the server container access to the host docker socket + CLI and the +# host paths the sibling needs for its bind mounts. These fragments are expanded +# at submit time and injected into the server `docker run` below; empty (no-op) +# when CLIENT_IMAGE is unset, so the in-container aiperf path is unchanged. +CLIENT_DOCKER_MOUNTS="" +CLIENT_DOCKER_ENV="" +if [[ -n "${CLIENT_IMAGE:-}" ]]; then + HOST_DOCKER_BIN="$(command -v docker || echo /usr/bin/docker)" + CLIENT_DOCKER_MOUNTS="-v /var/run/docker.sock:/var/run/docker.sock -v ${HOST_DOCKER_BIN}:/usr/bin/docker" + CLIENT_DOCKER_ENV="-e CLIENT_IMAGE=${CLIENT_IMAGE} -e HOST_REPO_DIR=${DI_REPO_DIR} -e HOST_MODEL_DIR=${MODEL_DIR} -e HOST_BENCH_LOGS=${BENCHMARK_LOGS_DIR} -e CLIENT_CONT_NAME=${CLIENT_CONT_NAME}" + echo "[client] node-0 sibling benchmark-client image enabled: ${CLIENT_IMAGE}" + # Best-effort pre-pull on all nodes so node 0's sibling launch doesn't stall. + srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD pull '"$CLIENT_IMAGE"' >/dev/null 2>&1 || true' 2>/dev/null || true +fi + srun \ --nodelist="$SELECTED_NODELIST_SRUN" \ --kill-on-bad-exit=1 \ @@ -444,9 +681,9 @@ echo \"Rank \$SLURM_PROCID on \$(hostname)\" eval \"\$DOCKER_CMD_DETECT\" echo \"[docker-detect] rank \$SLURM_PROCID: DOCKER_CMD=\$DOCKER_CMD\" -# Enable out-of-tree RDMA library mounts for atom-disagg (mooncake requires host RDMA stack) +# Enable out-of-tree RDMA library mounts for MoRIIO/Mooncake host RDMA stacks. RDMA_MOUNTS=() -if [[ "$ENGINE" == "atom-disagg" ]]; then +if [[ "$ENGINE" == "atom-disagg" || "$ENGINE" == "vllm-disagg" ]]; then # When the container base OS differs from the host (e.g. Ubuntu 24.04 image # on a 22.04 host), the container's bundled libibverbs/libionic may be @@ -520,11 +757,24 @@ if [[ \${#RDMA_MOUNTS[@]} -gt 0 ]]; then else echo \"[rdma] no out-of-tree RDMA mounts needed\" fi -fi # end: if ENGINE == atom-disagg +fi # end: if ENGINE == atom-disagg || vllm-disagg -# Pre-clean (idempotent) +# Pre-clean (idempotent): stop then force-remove so GPU VRAM is released before +# the sanity gate. stop-only left containers in Created/Exited state on some nodes. \$DOCKER_CMD ps -aq --filter \"$CONT_FILTER\" | xargs -r \$DOCKER_CMD rm -f || true -\$DOCKER_CMD ps -aq | xargs -r \$DOCKER_CMD stop || true +\$DOCKER_CMD ps -aq | xargs -r \$DOCKER_CMD stop -t 15 || true +\$DOCKER_CMD ps -aq | xargs -r \$DOCKER_CMD rm -f || true +sleep 2 + +# GPU sanity gate: containers are stopped, so any remaining VRAM use is a bare +# (non-containerized) process hogging the GPU -- fail fast (and name it) +# instead of OOMing in model load ~30 min later. set -e + --kill-on-bad-exit +# tears down the whole job on non-zero exit. +if [[ \"${SKIP_GPU_SANITY:-0}\" == \"1\" ]]; then + echo \"[INFO] SKIP_GPU_SANITY=1 set; skipping GPU sanity pre-flight check\" +else + bash \"$DI_REPO_DIR/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh\" +fi # Start vLLM external router container on node 0 if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \"\$SLURM_PROCID\" == \"0\" ]]; then @@ -532,6 +782,7 @@ if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \ \$DOCKER_CMD run -d \ --name \"$ROUTER_CONT_NAME\" \ --network host \ + --ulimit nofile=1048576:1048576 \ -v /tmp:/run_logs \ \"$VLLM_ROUTER_IMAGE\" \ bash -lc \"mkdir -p /run_logs/slurm_job-${SLURM_JOB_ID} && exec vllm-router \ @@ -543,7 +794,7 @@ if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \ --policy consistent_hash \ --prefill-policy consistent_hash \ --decode-policy consistent_hash \ - --log-level info 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/vllm_router_\$(hostname).log \" + --log-level ${ROUTER_LOG_LEVEL} 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/vllm_router_\$(hostname).log \" fi # Skip exec on vllm-disagg rank 0 so we can stop the router after the main @@ -572,6 +823,7 @@ fi --device=/dev/infiniband/uverbs7 \ --ulimit memlock=-1 \ --ulimit stack=67108864 \ + --ulimit nofile=1048576:1048576 \ --network host \ --ipc host \ --group-add video \ @@ -586,10 +838,13 @@ fi -v /tmp:/run_logs \ -v ${BENCHMARK_LOGS_DIR}:/benchmark_logs \ -v ${DI_REPO_DIR}:${DOCKER_MOUNT_PATH} \ + -v ${HICACHE_MC_CONFIG}:/config/hicache_mc.env:ro \ ${EXTRA_DOCKER_MOUNTS:-} \ + ${CLIENT_DOCKER_MOUNTS} \ \${RDMA_MOUNTS[@]+"\${RDMA_MOUNTS[@]}"} \ ${DOCKER_ENV_COMMON[*]} \ ${DOCKER_ENV_ENGINE[*]} \ + ${CLIENT_DOCKER_ENV} \ --name \"$DOCKER_CONT_NAME\" \ --entrypoint \"\" \ \"$DOCKER_IMAGE_NAME\" bash -lc ' @@ -604,9 +859,11 @@ echo \"[rank 0] Main container exited (rc=\$DOCKER_EXIT_CODE). Stopping vllm-rou \$DOCKER_CMD rm -f \"$ROUTER_CONT_NAME\" 2>/dev/null || true exit \$DOCKER_EXIT_CODE " +SRUN_RC=$? +collect_server_logs || true if [[ "${KEEP_CONTAINERS}" != "1" ]]; then - srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' 2>/dev/null || true' + srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' '"$CLIENT_CONT_NAME"' 2>/dev/null || true' # Clean up vLLM external router container on node 0 if [[ "$ENGINE" == "vllm-disagg" && "$ROUTER_TYPE" == "vllm-router" ]]; then @@ -615,3 +872,5 @@ if [[ "${KEEP_CONTAINERS}" != "1" ]]; then ' fi fi + +exit "$SRUN_RC" diff --git a/benchmarks/multi_node/amd_utils/models_vllm.yaml b/benchmarks/multi_node/amd_utils/models_vllm.yaml index 13456c2dcb..8466beb196 100644 --- a/benchmarks/multi_node/amd_utils/models_vllm.yaml +++ b/benchmarks/multi_node/amd_utils/models_vllm.yaml @@ -30,6 +30,15 @@ Kimi-K2.5-MXFP4: env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_PAGED_ATTN=0 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_USE_AITER_TRITON_SILU_MUL=0 VLLM_ENGINE_READY_TIMEOUT_S=3600" hf_dir: "models--amd--Kimi-K2.5-MXFP4" +# Kimi-K3 MXFP4 with DSpark speculative decoding. Synthetic AL matches the +# single-node AMD throughput arm; eval runs switch rejection_sample_method to +# "block" in server_vllm.sh. +Kimi-K3: + prefill_flags: "--tensor-parallel-size 8 --moe-backend auto --load-format fastsafetensors --gpu-memory-utilization 0.9 --language-model-only --max-num-seqs 32 --enable-auto-tool-choice --tool-call-parser kimi_k3 --reasoning-parser kimi_k3 --max-model-len 1048576 --enable-prefix-caching --kv-cache-dtype fp8 --no-async-scheduling --attention-config '{\"mla_prefill_backend\":\"ROCM_AITER_FA\"}' --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"max_cudagraph_capture_size\":96,\"custom_ops\":[\"+fused_rms_norm_gated\"],\"cudagraph_capture_sizes\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96]}' --speculative-config '{\"model\":\"/models/Inferact-Kimi-K3-DSpark\",\"num_speculative_tokens\":2,\"method\":\"dspark\",\"attention_backend\":\"TRITON_MLA\",\"kv_cache_dtype\":\"auto\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"synthetic\",\"synthetic_acceptance_length\":2.51}' --enable-prompt-tokens-details" + decode_flags: "--tensor-parallel-size 8 --moe-backend auto --load-format fastsafetensors --gpu-memory-utilization 0.9 --language-model-only --max-num-seqs 32 --enable-auto-tool-choice --tool-call-parser kimi_k3 --reasoning-parser kimi_k3 --max-model-len 1048576 --enable-prefix-caching --kv-cache-dtype fp8 --no-async-scheduling --attention-config '{\"mla_prefill_backend\":\"ROCM_AITER_FA\"}' --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"max_cudagraph_capture_size\":96,\"custom_ops\":[\"+fused_rms_norm_gated\"],\"cudagraph_capture_sizes\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96]}' --speculative-config '{\"model\":\"/models/Inferact-Kimi-K3-DSpark\",\"num_speculative_tokens\":2,\"method\":\"dspark\",\"attention_backend\":\"TRITON_MLA\",\"kv_cache_dtype\":\"auto\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"synthetic\",\"synthetic_acceptance_length\":2.51}' --enable-prompt-tokens-details" + env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4=1 AITER_BF16_FP8_MOE_BOUND=0 VLLM_ROCM_AITER_MLA_ASM_PADDING=asm VLLM_USE_BREAKABLE_CUDAGRAPH=0 SAFETENSORS_FAST_GPU=1 VLLM_SSM_CONV_STATE_LAYOUT=DS PYTHONHASHSEED=42 HSA_NO_SCRATCH_RECLAIM=0 VLLM_ENGINE_READY_TIMEOUT_S=7200 VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1200" + hf_dir: "Kimi-K3" + MiniMax-M2.5: # AITER fused-MoE kernel fmoe_bf16_blockscaleFp8_g1u1_vs_silu_32x384 for gfx950 writes OOB when run with MiniMax's shapes at M=8K(=num batched tokens), crashing vllm during AITER warmup. # Set token budget to 4k to avoid using that shape, instead of disabling AITER_MOE. diff --git a/benchmarks/multi_node/amd_utils/mori_build_inside.sh b/benchmarks/multi_node/amd_utils/mori_build_inside.sh new file mode 100644 index 0000000000..44fad4d788 --- /dev/null +++ b/benchmarks/multi_node/amd_utils/mori_build_inside.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Build ROCm/mori #341 (IOEngine.wait_all) into the current engine container. +# This runs only while preparing a derived image; serving never compiles mori. +set -euo pipefail + +SRC=${SRC:-/mori-src} +MORI_VERSION=${MORI_VERSION:-1.0.1+mori341.f7e6ac68} + +echo "[mori-build] staging source off shared storage" +rm -rf /tmp/mori +cp -a "$SRC" /tmp/mori +cd /tmp/mori +git config --global --add safe.directory '*' || true +echo "[mori-build] HEAD=$(git rev-parse HEAD 2>/dev/null || echo unknown)" + +# Match SGLang's BUILD_UMBP=ON build instead of disabling a subsystem. +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get install -y libpci-dev libgrpc++-dev protobuf-compiler-grpc + +# Probe with the compiler rather than testing a hard-coded path. libpci-dev +# installs pci/pci.h under the multiarch prefix (/usr/include/x86_64-linux-gnu), +# which is on the default include path but absent from /usr/include/pci, so a +# path test reports the dependency missing on an image where it is fine. +check_header() { + local header=$1 compiler=$2 src + src=$(mktemp "/tmp/mori_dep_XXXXXX.${3}") + printf '#include <%s>\nint main(void){return 0;}\n' "$header" >"$src" + if "$compiler" -fsyntax-only "$src" 2>/dev/null; then + echo "[mori-build] $header is on the include path" + rm -f "$src" + return 0 + fi + rm -f "$src" + echo "[mori-build] FATAL: $compiler cannot find <$header> after apt-get install" >&2 + return 1 +} + +check_header pci/pci.h cc c +check_header grpcpp/grpcpp.h c++ cc + +echo "[mori-build] installing build requirements" +python3 -m pip install --break-system-packages -r requirements-build.txt +export SETUPTOOLS_SCM_PRETEND_VERSION="$MORI_VERSION" +export SETUPTOOLS_SCM_PRETEND_VERSION_FOR_AMD_MORI="$MORI_VERSION" +export MORI_GPU_ARCHS=${MORI_GPU_ARCHS:-gfx950} +export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL:-$(nproc)}" + +echo "[mori-build] building arch=$MORI_GPU_ARCHS jobs=$CMAKE_BUILD_PARALLEL_LEVEL" +python3 -m pip install --break-system-packages --no-build-isolation \ + --force-reinstall . + +cd / +python3 - <<'PY' +import mori +import mori.io as io + +print("[mori-build] mori=", getattr(mori, "__version__", "?"), mori.__file__) +print("[mori-build] wait_all=", hasattr(io.IOEngine, "wait_all")) +assert hasattr(io.IOEngine, "wait_all") +assert hasattr(io, "StatusCode") +PY +echo "MORI_BUILD_OK" diff --git a/benchmarks/multi_node/amd_utils/patches/k3_moriio_51052.patch b/benchmarks/multi_node/amd_utils/patches/k3_moriio_51052.patch new file mode 100644 index 0000000000..561a6587b6 Binary files /dev/null and b/benchmarks/multi_node/amd_utils/patches/k3_moriio_51052.patch differ diff --git a/benchmarks/multi_node/amd_utils/prepare_mori_waitall_image.sh b/benchmarks/multi_node/amd_utils/prepare_mori_waitall_image.sh new file mode 100644 index 0000000000..15169a4271 --- /dev/null +++ b/benchmarks/multi_node/amd_utils/prepare_mori_waitall_image.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Prepare a node-local engine image with ROCm/mori #341 baked in. +set -euo pipefail + +BASE_IMAGE=${1:?base image required} +OUT_IMAGE=${2:?output image required} +MORI_SOURCE=${3:?mori source directory required} +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +CNAME="mori341_build_${SLURM_JOB_ID:-manual}_$(hostname -s)" + +if docker ps >/dev/null 2>&1; then + CTR=(docker) +elif podman ps >/dev/null 2>&1; then + CTR=(podman) +else + CTR=(sudo docker) +fi + +cleanup() { + "${CTR[@]}" rm -f "$CNAME" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +if "${CTR[@]}" image inspect "$OUT_IMAGE" >/dev/null 2>&1 && + "${CTR[@]}" run --rm -i --entrypoint python3 "$OUT_IMAGE" - <<'PY' +from mori.io import IOEngine +assert hasattr(IOEngine, "wait_all") +PY +then + echo "[mori-image] reuse $OUT_IMAGE on $(hostname -s)" + exit 0 +fi + +test -s "$MORI_SOURCE/src/io/engine.cpp" +test -s "$HERE/mori_build_inside.sh" +cleanup + +echo "[mori-image] build $OUT_IMAGE from $BASE_IMAGE on $(hostname -s)" +"${CTR[@]}" run -d --name "$CNAME" --network host \ + -v "$MORI_SOURCE":/mori-src:ro \ + -v "$HERE/mori_build_inside.sh":/tmp/mori_build_inside.sh:ro \ + --entrypoint bash "$BASE_IMAGE" -c 'sleep infinity' >/dev/null + +# Keep the compile transcript with the run: this build happens on the compute +# node, so without a copy on shared storage a failure leaves only whatever the +# srun stream happened to interleave. +BUILD_LOG="${BENCHMARK_LOGS_DIR:-/tmp}/server_logs/slurm_job-${SLURM_JOB_ID:-manual}/mori_build_$(hostname -s).log" +mkdir -p "$(dirname "$BUILD_LOG")" +if ! "${CTR[@]}" exec "$CNAME" bash /tmp/mori_build_inside.sh 2>&1 | tee "$BUILD_LOG"; then + echo "[mori-image] FAILED to build $OUT_IMAGE on $(hostname -s); see $BUILD_LOG" >&2 + tail -40 "$BUILD_LOG" >&2 || true + exit 1 +fi +"${CTR[@]}" commit \ + --change 'LABEL mori.source=ROCm/mori@f7e6ac6863c53821bc7afb91a578cc6ce38fcad0' \ + --change 'LABEL mori.waitall=true' \ + "$CNAME" "$OUT_IMAGE" >/dev/null + +"${CTR[@]}" run --rm -i --entrypoint python3 "$OUT_IMAGE" - <<'PY' +import mori +from mori.io import IOEngine +print("[mori-image] version=", getattr(mori, "__version__", "?")) +print("[mori-image] wait_all=", hasattr(IOEngine, "wait_all")) +assert hasattr(IOEngine, "wait_all") +PY +echo "MORI_IMAGE_DONE $OUT_IMAGE on $(hostname -s)" diff --git a/benchmarks/multi_node/amd_utils/server_vllm.sh b/benchmarks/multi_node/amd_utils/server_vllm.sh index f19ce8560b..c6fde47d9c 100755 --- a/benchmarks/multi_node/amd_utils/server_vllm.sh +++ b/benchmarks/multi_node/amd_utils/server_vllm.sh @@ -145,10 +145,12 @@ pf = bash_escape(m.get('prefill_flags', '--tensor-parallel-size 8')) df = bash_escape(m.get('decode_flags', '--tensor-parallel-size 8')) ev = bash_escape(m.get('env', '')) dev = bash_escape(m.get('decode_env', '')) +pev = bash_escape(m.get('prefill_env', '')) print(f'PREFILL_SERVER_CONFIG=\"{pf}\"') print(f'DECODE_SERVER_CONFIG=\"{df}\"') print(f'MODEL_ENVS=\"{ev}\"') print(f'DECODE_MODEL_ENVS=\"{dev}\"') +print(f'PREFILL_MODEL_ENVS=\"{pev}\"') ")" echo "Loaded model configuration for: $MODEL_NAME" @@ -168,6 +170,27 @@ if [[ -n "${DECODE_TP_SIZE:-}" ]]; then DECODE_SERVER_CONFIG+=" --tensor-parallel-size ${DECODE_TP_SIZE}" fi fi +# Throughput arms pin a synthetic acceptance length, which commits drafted +# tokens without consulting the target's logits: fast, but the generated text is +# wrong. An accuracy run has to verify for real. Without this, GSM8K scored +# 0.3518 against the 0.9 gate while the same server measured throughput fine. +# The single-node siblings (kimik3_fp4_mi355x_mtp.sh, dsv4_fp4_b300_vllm_mtp.sh) +# make the same split; the multi-node path only documented it. +# vLLM rejects synthetic_acceptance_length unless the method is 'synthetic', +# so drop that key in the same rewrite. +if [[ "${EVAL_ONLY:-false}" == "true" || "${RUN_EVAL:-false}" == "true" ]]; then + _real_verify() { + printf '%s' "$1" | + sed -E 's/\\?"rejection_sample_method\\?"[[:space:]]*:[[:space:]]*\\?"synthetic\\?"/\\"rejection_sample_method\\": \\"block\\"/g' | + sed -E 's/,[[:space:]]*\\?"synthetic_acceptance_length\\?"[[:space:]]*:[[:space:]]*[0-9.]+//g' + } + if echo "$PREFILL_SERVER_CONFIG" | grep -q 'rejection_sample_method'; then + PREFILL_SERVER_CONFIG=$(_real_verify "$PREFILL_SERVER_CONFIG") + DECODE_SERVER_CONFIG=$(_real_verify "$DECODE_SERVER_CONFIG") + echo "[eval] speculative decoding switched to real block verification" + fi +fi + if [[ "${PREFILL_ENABLE_EP:-false}" == "true" ]] && ! echo "$PREFILL_SERVER_CONFIG" | grep -q -- '--enable-expert-parallel'; then PREFILL_SERVER_CONFIG+=" --enable-expert-parallel" fi @@ -181,6 +204,18 @@ if [[ "${DECODE_ENABLE_DP:-false}" == "true" ]] && ! echo "$DECODE_SERVER_CONFIG DECODE_SERVER_CONFIG+=" --enable-dp-attention" fi +# Health and metrics are polled every few seconds for the whole run, and each +# poll wrote an access-log line: that spam was the bulk of the engine logs +# shipped in CI artifacts. Suppressing the access log for those endpoints does +# not affect metrics collection, and keeps real request lines. +QUIET_ENDPOINTS="/health,/metrics,/ping,/load" +for _cfg in PREFILL DECODE; do + _v="${_cfg}_SERVER_CONFIG" + if ! echo "${!_v}" | grep -q -- '--disable-access-log-for-endpoints'; then + printf -v "$_v" '%s --disable-access-log-for-endpoints %s' "${!_v}" "$QUIET_ENDPOINTS" + fi +done + echo "PREFILL_SERVER_CONFIG (after TP/EP/DP): $PREFILL_SERVER_CONFIG" echo "DECODE_SERVER_CONFIG (after TP/EP/DP): $DECODE_SERVER_CONFIG" @@ -220,6 +255,50 @@ echo "Decode node IPs: ${DECODE_ARGS}" # MoRI-IO proxy ZMQ registration port (must match vllm-router --vllm-discovery-address) PROXY_PING_PORT="${PROXY_PING_PORT:-36367}" +# Compose MoRIIO with the optional CPU KV tier on prefill only. The matrix's DRAM +# budget is sized from the prefill worker, while decode always pulls over MoRIIO. +build_kv_transfer_configs() { + local moriio_prefill moriio_decode + # MORIIO_READ_MODE=true|false (default true). Write path needs a mori build + # that exposes IOEngine.wait_all for the #51052 batch barrier. + local read_mode="${MORIIO_READ_MODE:-true}" + moriio_prefill="{\"kv_connector\": \"MoRIIOConnector\", \"kv_role\": \"kv_producer\", \"kv_connector_extra_config\": {\"proxy_ip\": \"${NODE0_ADDR}\", \"proxy_ping_port\": \"${PROXY_PING_PORT}\", \"http_port\": \"${SERVER_PORT}\", \"read_mode\": ${read_mode}}}" + moriio_decode="{\"kv_connector\": \"MoRIIOConnector\", \"kv_role\": \"kv_consumer\", \"kv_connector_extra_config\": {\"proxy_ip\": \"${NODE0_ADDR}\", \"proxy_ping_port\": \"${PROXY_PING_PORT}\", \"http_port\": \"${SERVER_PORT}\", \"read_mode\": ${read_mode}}}" + KV_TRANSFER_PREFILL="'${moriio_prefill}'" + KV_TRANSFER_DECODE="'${moriio_decode}'" + echo "[INFO] MoRIIO read_mode=${read_mode}" + + if [[ "${KV_OFFLOADING:-none}" == "dram" && "${KV_OFFLOAD_BACKEND:-}" == "vllm-simple" ]]; then + if [[ -z "${TOTAL_CPU_DRAM_GB:-}" || "${TOTAL_CPU_DRAM_GB}" == "0" ]]; then + echo "ERROR: kv-offloading=dram with vllm-simple requires TOTAL_CPU_DRAM_GB." >&2 + exit 1 + fi + local per_rank simple + per_rank=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / PREFILL_TP_SIZE )) + simple="{\"kv_connector\": \"SimpleCPUOffloadConnector\", \"kv_role\": \"kv_both\", \"kv_connector_extra_config\": {\"cpu_bytes_to_use_per_rank\": ${per_rank}, \"lazy_offload\": false}}" + KV_TRANSFER_PREFILL="'{\"kv_connector\": \"MultiConnector\", \"kv_role\": \"kv_both\", \"kv_connector_extra_config\": {\"connectors\": [${moriio_prefill}, ${simple}]}}'" + fi +} + +apply_model_patches() { + if [[ "${MODEL_NAME}" == "Kimi-K3" ]]; then + local here + here="$(dirname "${BASH_SOURCE[0]}")" + bash "${here}/../../single_node/agentic/apply_k3_container_patches.sh" \ + || { echo "ERROR: apply_k3_container_patches.sh failed" >&2; exit 1; } + bash "${here}/apply_k3_moriio_patches.sh" \ + || { echo "ERROR: apply_k3_moriio_patches.sh failed" >&2; exit 1; } + # Only the write path needs the #341 batch barrier. job.slurm already + # decided read vs write for the whole job, so a missing wait_all here + # means the derived image did not reach this container and the two + # roles would disagree on the transfer direction. + if [[ "${MORIIO_READ_MODE:-true}" != "true" ]]; then + bash "${here}/ensure_mori_wait_all.sh" \ + || { echo "ERROR: write mode requested but IOEngine.wait_all is missing" >&2; exit 1; } + fi + fi +} + # vLLM runtime environment (static vars moved to env.sh; these depend on per-node state) setup_vllm_env() { export VLLM_NIXL_SIDE_CHANNEL_HOST=${rdma_ip} @@ -251,15 +330,22 @@ if [ "$NODE_RANK" -eq 0 ]; then setup_vllm_env + for env_pair in ${PREFILL_MODEL_ENVS}; do + export "$env_pair" + echo "[PREFILL_ENV] $env_pair" + done + # Router is started as an external container by job.slurm (VLLM_ROUTER_IMAGE) echo "Using external vllm-router container (started by job.slurm on this node)" + apply_model_patches + build_kv_transfer_configs SERVED_MODEL="${MODEL_NAME}" PREFILL_CMD="vllm serve ${MODEL_PATH} \ --served-model-name ${SERVED_MODEL} \ --port $SERVER_PORT \ --trust-remote-code \ - --kv-transfer-config '{\"kv_connector\": \"MoRIIOConnector\", \"kv_role\": \"kv_producer\", \"kv_connector_extra_config\": {\"proxy_ip\": \"${NODE0_ADDR}\", \"proxy_ping_port\": \"${PROXY_PING_PORT}\", \"http_port\": \"${SERVER_PORT}\", \"read_mode\": true}}' \ + --kv-transfer-config ${KV_TRANSFER_PREFILL} \ ${PREFILL_SERVER_CONFIG}" if [[ "$DRY_RUN" -eq 1 ]]; then @@ -305,10 +391,26 @@ if [ "$NODE_RANK" -eq 0 ]; then cd $WS_PATH export ROUTER_PORT=$ROUTER_PORT - BENCH_CMD="bash $WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \ - $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \ - ${BENCH_OUTPUT_LEN} \"${BENCH_MAX_CONCURRENCY}\" ${BENCH_REQUEST_RATE} \ - ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}" + if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then + METRICS_URLS=() + for _ip in ${PREFILL_ARGS} ${DECODE_ARGS}; do + METRICS_URLS+=("http://${_ip}:${SERVER_PORT}/metrics") + done + if [[ "${#METRICS_URLS[@]}" -gt 0 ]]; then + AIPERF_SERVER_METRICS_URLS=$(IFS=,; echo "${METRICS_URLS[*]}") + export AIPERF_SERVER_METRICS_URLS + echo "AIPERF_SERVER_METRICS_URLS=${AIPERF_SERVER_METRICS_URLS}" + fi + BENCH_CMD="bash $WS_PATH/trace_replay.sh \ + $MODEL_DIR $MODEL_NAME \"${BENCH_MAX_CONCURRENCY}\" /run_logs/slurm_job-${SLURM_JOB_ID}" + echo "Benchmark runner: trace_replay.sh (agentic)" + else + BENCH_CMD="bash $WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \ + $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \ + ${BENCH_OUTPUT_LEN} \"${BENCH_MAX_CONCURRENCY}\" ${BENCH_REQUEST_RATE} \ + ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}" + echo "Benchmark runner: bench.sh (fixed-seq-len)" + fi if [[ "${EVAL_ONLY:-false}" == "true" ]]; then echo "EVAL_ONLY mode: skipping throughput benchmark" @@ -399,6 +501,11 @@ if [ "$NODE_RANK" -eq 0 ]; then if [[ "$DRY_RUN" -eq 0 ]]; then cp -r /run_logs/slurm_job-${SLURM_JOB_ID} "$LOGS_OUTPUT/" + # This container is root and the destination is the bind-mounted CI + # workspace, so leave the copy world-writable. Otherwise a cancelled run + # strands root-owned files that actions/checkout cannot clean, and the + # runner fails every later job with EACCES. + chmod -R a+rwX "$LOGS_OUTPUT/slurm_job-${SLURM_JOB_ID}" 2>/dev/null || true echo "Copied results to $LOGS_OUTPUT/slurm_job-${SLURM_JOB_ID}" fi @@ -420,12 +527,19 @@ elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$xP" ]; then setup_vllm_env + for env_pair in ${PREFILL_MODEL_ENVS}; do + export "$env_pair" + echo "[PREFILL_ENV] $env_pair" + done + + apply_model_patches + build_kv_transfer_configs SERVED_MODEL="${MODEL_NAME}" PREFILL_CMD="vllm serve ${MODEL_PATH} \ --served-model-name ${SERVED_MODEL} \ --port $SERVER_PORT \ --trust-remote-code \ - --kv-transfer-config '{\"kv_connector\": \"MoRIIOConnector\", \"kv_role\": \"kv_producer\", \"kv_connector_extra_config\": {\"proxy_ip\": \"${NODE0_ADDR}\", \"proxy_ping_port\": \"${PROXY_PING_PORT}\", \"http_port\": \"${SERVER_PORT}\", \"read_mode\": true}}' \ + --kv-transfer-config ${KV_TRANSFER_PREFILL} \ ${PREFILL_SERVER_CONFIG}" if [[ "$DRY_RUN" -eq 1 ]]; then @@ -476,12 +590,14 @@ else echo "[DECODE_ENV] $env_pair" done + apply_model_patches + build_kv_transfer_configs SERVED_MODEL="${MODEL_NAME}" DECODE_CMD="vllm serve ${MODEL_PATH} \ --served-model-name ${SERVED_MODEL} \ --port $SERVER_PORT \ --trust-remote-code \ - --kv-transfer-config '{\"kv_connector\": \"MoRIIOConnector\", \"kv_role\": \"kv_consumer\", \"kv_connector_extra_config\": {\"proxy_ip\": \"${NODE0_ADDR}\", \"proxy_ping_port\": \"${PROXY_PING_PORT}\", \"http_port\": \"${SERVER_PORT}\", \"read_mode\": true}}' \ + --kv-transfer-config ${KV_TRANSFER_DECODE} \ ${DECODE_SERVER_CONFIG}" if [[ "$DRY_RUN" -eq 1 ]]; then diff --git a/benchmarks/multi_node/amd_utils/trace_replay.sh b/benchmarks/multi_node/amd_utils/trace_replay.sh new file mode 100644 index 0000000000..f4e9f84c23 --- /dev/null +++ b/benchmarks/multi_node/amd_utils/trace_replay.sh @@ -0,0 +1,166 @@ +#!/bin/bash +# Dual-Engine Disaggregated Benchmark Runner +# +# ENGINE=sglang (default): SGLang benchmark +# ENGINE=vllm: vLLM benchmark +# +# Produces JSON result files via benchmark_serving.py so that the CI pipeline +# can collect and process results. +# +# Usage: bash bench.sh \ +# \ +# + +ENGINE="${ENGINE:-sglang-disagg}" + +model_path=$1 +model_name=$2 +concurrency_list=${3:-"1"} +MODEL_PATH="${MODEL_PATH:-${model_path}/${model_name}}" +# vllm-disagg uses --served-model-name MODEL_NAME; sglang defaults to MODEL_PATH +if [[ "$ENGINE" == "vllm-disagg" ]]; then + MODEL="${MODEL_NAME:-${MODEL_PATH}}" +else + MODEL="${MODEL_PATH}" +fi +log_path=${4:-/run_logs} + +# Split BENCH_MAX_CONCURRENCY (x-delimited, e.g. "8x16x32") into an array. +# Falls back to 1 if unset so the loop always runs at least once. +IFS='x' read -r -a chosen_concurrencies <<< "${concurrency_list}" + + +ROUTER_PORT="${ROUTER_PORT:-30000}" + +export TRANSFORMERS_VERBOSITY=error +export TOKENIZERS_PARALLELISM=false + +# echo "Config ${chosen_isl}; ${chosen_osl}; ${chosen_concurrencies[0]}; ${chosen_req_rate}" + +RESULT_DIR="${RESULT_DIR:-${log_path}/agentic}" +mkdir -p "$RESULT_DIR" + +source "$(dirname "$0")/../../benchmark_lib.sh" + +# clear_kv_caches — wipe all KV cache tiers on every backend worker before a +# concurrency point, so each conc is measured cold (no prefix reuse bleeding in +# from the previous conc). Mirrors mori-scheduler/scripts/benchmark/lib/ +# clear_caches.sh, but the worker base URLs are already resolved by +# server_sglang.sh (SERVER_FLUSH_URLS_CSV) so no SSH/IP lookup is needed. +# +# Tiers (SGLang server APIs), hit on EACH worker directly (the router does not +# fan /flush_cache out): +# L1 (GPU radix) + L2 (host hicache): POST /flush_cache — NO-OP while any +# request is in flight, so we drain-retry until "Cache flushed" or +# FLUSH_DRAIN_TIMEOUT (default 120s) elapses. +# L3 (umbp / mooncake store): POST /hicache/storage-backend/clear +# — HTTP != 200 when L3 is off, tolerated. +# Best-effort: logs WARN, never hard-fails the sweep. +clear_kv_caches() { + local drain_tmo="${FLUSH_DRAIN_TIMEOUT:-120}" + local urls_csv="${SERVER_FLUSH_URLS_CSV:-}" + if [[ -z "$urls_csv" ]]; then + echo "[clear_caches] WARN: SERVER_FLUSH_URLS_CSV unset; skipping cache flush" >&2 + return 0 + fi + local -a urls + IFS=',' read -r -a urls <<< "$urls_csv" + local url start ok resp code + for url in "${urls[@]}"; do + [[ -n "$url" ]] || continue + # L1 + L2: drain-retry until flushed (no-op while requests in flight). + start=$(date +%s); ok=0; resp="" + while :; do + resp=$(curl -sf -m 10 -X POST "${url}/flush_cache" 2>/dev/null || true) + echo "$resp" | grep -qi "Cache flushed" && { ok=1; break; } + (( $(date +%s) - start >= drain_tmo )) && break + sleep 3 + done + if [[ "$ok" == 1 ]]; then + echo "[clear_caches] ${url}: L1+L2 flushed" + else + echo "[clear_caches] WARN ${url}: L1+L2 flush NOT confirmed after ${drain_tmo}s (resp='${resp:0:80}')" >&2 + fi + # L3: storage-backend clear (umbp / mooncake). 200 when a backend is attached. + code=$(curl -s -m 60 -o /dev/null -w '%{http_code}' -X POST "${url}/hicache/storage-backend/clear" 2>/dev/null || echo 000) + if [[ "$code" == 200 ]]; then + echo "[clear_caches] ${url}: L3 store cleared" + else + echo "[clear_caches] ${url}: L3 clear http=${code} (no storage backend / L3 off — ok)" + fi + done +} + +# REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" + +PORT="${ROUTER_PORT}" +MODEL="${MODEL:-${BENCH_MODEL}}" +DURATION="${DURATION:-1800}" +export MODEL DURATION MAX_MODEL_LEN +RESULT_DIR="${RESULT_DIR:-${profile_folder}}" +# Base name for the per-conc aggregate written by the existing +# utils.agentic.aggregation.process_agentic_result module. +# The workflow guard / upload steps expect a "${RESULT_FILENAME}_conc.json" +# file per concurrency, so each concurrency below is always suffixed with +# _conc (matching agentic_srt.sh on the gb200 path). +RESULT_FILENAME_BASE="${RESULT_FILENAME:-agentic_bench}" + +mkdir -p "$RESULT_DIR" + +if [ "$PREFILL_ENABLE_DP" = "true" ]; then + set -x + export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true + set +x +fi + +resolve_trace_source +install_agentic_deps + +ANY_FAILED=0 +for max_concurrency in "${chosen_concurrencies[@]}"; do + + echo "==========================================" + echo "Agentic trace replay: conc=$max_concurrency" + echo "==========================================" + + # Clear all KV cache tiers on every backend before this conc point so it is + # measured cold (no prefix reuse from the previous conc). Default on; set + # CLEAR_CACHE_BETWEEN_CONC=0 to disable. Best-effort — never fails the run. + if [[ "${CLEAR_CACHE_BETWEEN_CONC:-1}" == "1" ]]; then + echo "conc=$max_concurrency: clearing L1/L2/L3 on all backends (no server restart)" + clear_kv_caches || echo "WARNING: cache clear had issues for conc=$max_concurrency" >&2 + fi + + # Mirror agentic_srt.sh (the srtctl/gb200 path): every concurrency writes + # its artifacts into a conc_/ subdir of RESULT_DIR. The CI matrix explodes + # agentic runs to one concurrency per job, but benchmark-multinode-tmpl.yml + # still expects the per-conc nesting (LOGS/agentic/conc_*/...) and the + # _conc result-file suffix, so we always nest to keep the layout identical + # across runners and avoid overwriting earlier runs in local multi-conc sweeps. + CONC_RESULT_DIR="$RESULT_DIR/conc_${max_concurrency}" + mkdir -p "$CONC_RESULT_DIR" + + CONC="$max_concurrency" + USERS="$max_concurrency" + export CONC USERS + build_replay_cmd "$CONC_RESULT_DIR" + + # Per-conc result name consumed by write_agentic_result_json. Always suffix + # with _conc so the file matches + # the workflow guard's "${RESULT_FILENAME}_conc*.json" glob (and the agg / + # checkpoint upload steps) for both single-conc CI runs and multi-conc sweeps. + export RESULT_FILENAME="${RESULT_FILENAME_BASE}_conc${max_concurrency}" + if ! run_agentic_replay_and_write_outputs "$CONC_RESULT_DIR"; then + echo "WARNING: agentic trace replay for conc=$max_concurrency failed (replay or validation) after writing available results" >&2 + ANY_FAILED=1 + fi + + echo "-----------------------------------------" + +done + +export RESULT_FILENAME="$RESULT_FILENAME_BASE" + +if [ "$ANY_FAILED" -ne 0 ]; then + echo "WARNING: at least one conc had a non-zero exit; per-conc result files were still written when possible." >&2 +fi diff --git a/benchmarks/single_node/agentic/apply_k3_container_patches.sh b/benchmarks/single_node/agentic/apply_k3_container_patches.sh new file mode 100644 index 0000000000..bcc7324823 --- /dev/null +++ b/benchmarks/single_node/agentic/apply_k3_container_patches.sh @@ -0,0 +1,264 @@ +#!/usr/bin/env bash +# ============================================================================= +# Kimi-K3 / MI355X (gfx950) in-container patches — all three in one place. +# +# Everything here patches files inside the running container only +# (site-packages). Nothing outside the container is touched. Each patch is +# idempotent, verifies its own anchor, backs up to .orig, and no-ops if +# the image already ships the fix. A failed anchor aborts that patch cleanly +# rather than corrupting the file, so a future image with different sources +# degrades to "unpatched", never to "broken". +# +# [1] aiter pybind11 internals mismatch -> unblocks ROCM_AITER_FA prefill +# [2] TritonMLA cudagraph support -> FULL cudagraphs for DSpark (5.52x TPOT) +# [3] KV block-pool negative-count clamp -> stops the mid-run engine crash +# +# Env: +# SKIP_KIMI_PATCHES=1 skip everything +# PYTHON=... interpreter to use (default python3) +# ============================================================================= +set -euo pipefail +PY=${PYTHON:-python3} + +if [ "${SKIP_KIMI_PATCHES:-0}" = "1" ]; then + echo "[kimi-patches] SKIP_KIMI_PATCHES=1, doing nothing." + exit 0 +fi + +# Locate an installed module's file, or empty string if unimportable. +_modfile() { + $PY - "$1" <<'EOF' +import importlib, os, sys +try: + print(os.path.abspath(importlib.import_module(sys.argv[1]).__file__)) +except Exception: + print("") +EOF +} + +# _patch <<'PYEOF' ... old/new python ... PYEOF +# The heredoc body must define OLD and NEW strings. +_patch() { + local target="$1" marker="$2" label="$3" + if [ -z "$target" ] || [ ! -f "$target" ]; then + echo "[$label] target not found; skipping." + return 0 + fi + if grep -q "$marker" "$target"; then + echo "[$label] already patched." + return 0 + fi + cp -n "$target" "$target.orig" 2>/dev/null || true + if $PY - "$target" "$label"; then + return 0 + else + echo "[$label] patch failed; left unchanged." >&2 + return 0 + fi +} + +# ----------------------------------------------------------------------------- +# [1] aiter: JIT modules must use torch's bundled pybind11 +# ----------------------------------------------------------------------------- +# aiter/jit/utils/cpp_extension.py appends the STANDALONE pybind11 include via +# -I, which outranks the -isystem path carrying torch's bundled copy. The 117 +# prebuilt aiter .so are built against torch's (PYBIND11_INTERNALS_VERSION 11); +# the standalone package here is version 12. pybind11 keeps a SEPARATE type +# registry per internals id, so a JIT-built module cannot see aiter_tensor_t +# registered by the prebuilt core and the first call dies during warmup with +# TypeError: fmha_fwd_bf16_opus_fwd(): incompatible function arguments +# even though arity and types match exactly. +patch_aiter_pybind11() { + local label="aiter-pybind11" + local target; target=$(_modfile aiter.jit.utils.cpp_extension) + if [ -z "$target" ] || [ ! -f "$target" ]; then + echo "[$label] aiter not present; skipping."; return 0 + fi + + # Only act if the two pybind11s actually disagree. + local need; need=$($PY - <<'EOF' +import os, re +try: + import torch, pybind11 +except Exception: + print("no"); raise SystemExit +def ver(p): + f = os.path.join(p, "pybind11", "detail", "internals.h") + if not os.path.isfile(f): return None + m = re.search(r"define\s+PYBIND11_INTERNALS_VERSION\s+(\d+)", open(f).read()) + return int(m.group(1)) if m else None +t = ver(os.path.join(os.path.dirname(torch.__file__), "include")) +s = ver(pybind11.get_include()) +print("yes" if (t is not None and s is not None and t != s) else "no") +EOF +) + if [ "$need" != "yes" ]; then + echo "[$label] pybind11 internals already agree; nothing to do."; return 0 + fi + + if grep -q "_use_torch_pybind11" "$target"; then + echo "[$label] already patched." + else + cp -n "$target" "$target.orig" 2>/dev/null || true + $PY - "$target" <<'EOF' || echo "[aiter-pybind11] patch failed; unchanged." >&2 +import sys, io +p = sys.argv[1] +src = io.open(p, encoding="utf-8").read() +old = " extra_include_paths.append(pybind11.get_include())\n" +new = ( + " # PATCHED: prefer torch's bundled pybind11 so JIT modules land in the\n" + " # same pybind11 type registry as the prebuilt .so files.\n" + " _use_torch_pybind11 = False\n" + " if not torch_exclude:\n" + " _use_torch_pybind11 = os.path.isdir(\n" + " os.path.join(TORCH_INCLUDE_ROOT, \"pybind11\")\n" + " )\n" + " if not _use_torch_pybind11:\n" + " extra_include_paths.append(pybind11.get_include())\n" +) +if src.count(old) != 1: + sys.stderr.write("[aiter-pybind11] anchor missing or not unique; aborting.\n") + sys.exit(1) +io.open(p, "w", encoding="utf-8").write(src.replace(old, new)) +print("[aiter-pybind11] patched", p) +EOF + fi + + # Drop JIT artifacts built against the wrong pybind11 so they rebuild. + # aiter honours AITER_JIT_DIR and falls back to ~/.aiter when dist-packages + # is read-only, so ask aiter rather than deriving the path from $target. + local jitdir + jitdir=$($PY -c 'from aiter.jit.core import get_user_jit_dir; print(get_user_jit_dir())' 2>/dev/null || true) + [ -n "$jitdir" ] && [ -d "$jitdir" ] || jitdir=$(dirname "$(dirname "$target")") + shopt -s nullglob + for so in "$jitdir"/*.so; do + if grep -qa "__pybind11_internals_v12" "$so" 2>/dev/null; then + rm -f "$so"; rm -rf "$jitdir/build/$(basename "${so%.so}")" + echo "[$label] removed stale v12 module: $(basename "$so")" + fi + done + shopt -u nullglob +} + +# ----------------------------------------------------------------------------- +# [2] vLLM: let DSpark spec-decode keep FULL cudagraphs +# ----------------------------------------------------------------------------- +# TritonMLAMetadataBuilder._cudagraph_support = UNIFORM_SINGLE_TOKEN_DECODE caps +# min_cg_support below UNIFORM_BATCH, so config/compilation.py downgrades +# FULL_AND_PIECEWISE -> PIECEWISE under spec-decode. dflash/speculator.py then +# gives the DSpark drafter CUDAGraphMode.NONE -- fully eager -- and logs nothing. +# TRITON_MLA cannot be swapped out: it is the only ROCm MLA backend with +# supports_non_causal_multi_token_decode=True, which DSpark requires +# (ROCM_AITER_MLA fails with "non-causal attention not supported"). +# The builder already calls _init_reorder_batch_threshold(1, +# supports_spec_as_decode=True) "so full-cudagraph capture admits it", so +# UNIFORM_BATCH is the self-consistent value. +# MEASURED 8x MI355X single stream, 600-token gens: +# before 14.05 tok/s ITL 71.16 ms -> after 77.65 tok/s ITL 12.88 ms (5.52x) +patch_triton_mla_cudagraph() { + local label="triton-mla-cudagraph" + local target; target=$(_modfile vllm.v1.attention.backends.mla.triton_mla) + if [ -z "$target" ] || [ ! -f "$target" ]; then + echo "[$label] target not found; skipping."; return 0 + fi + if grep -q "AttentionCGSupport.UNIFORM_BATCH" "$target"; then + echo "[$label] already patched."; return 0 + fi + cp -n "$target" "$target.orig" 2>/dev/null || true + $PY - "$target" <<'EOF' || echo "[triton-mla-cudagraph] patch failed; unchanged." >&2 +import sys, io +p = sys.argv[1] +src = io.open(p, encoding="utf-8").read() +old = """ _cudagraph_support: ClassVar[AttentionCGSupport] = ( + AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE + )""" +new = """ # PATCHED: UNIFORM_SINGLE_TOKEN_DECODE forced a PIECEWISE downgrade under + # spec-decode, which silently made the DSpark drafter fully eager. + _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH""" +if src.count(old) != 1: + sys.stderr.write("[triton-mla-cudagraph] anchor missing or not unique; aborting.\n") + sys.exit(1) +io.open(p, "w", encoding="utf-8").write(src.replace(old, new)) +print("[triton-mla-cudagraph] patched", p) +EOF +} + +# ----------------------------------------------------------------------------- +# [3] vLLM: clamp the negative block count that corrupts the KV free list +# ----------------------------------------------------------------------------- +# single_type_kv_cache_manager.py, in allocate_external_computed_blocks(), is the +# ONLY unguarded get_new_blocks() call site in that file (siblings clamp or +# early-return). When len(req_blocks) exceeds the block count implied by +# num_total_computed_tokens the argument goes NEGATIVE, and a negative count is +# silently destructive rather than rejected: +# * block_pool.get_new_blocks only rejects num_blocks > free +# * popleft_n passes its own assert num_free_blocks >= n +# * it runs num_free_blocks -= n -> an INCREASE +# * range(n) iterates zero times, so the linked list is untouched +# num_free_blocks is then inflated relative to the real free list; a later +# legitimate pop walks past the tail and the engine dies mid-run on +# kv_cache_utils.py assert curr_block is not None +# block_pool.py assert block.ref_cnt == 0 +# Load-dependent: c10 died at 3612 s, c12 at 487 s, c16 at 354 s. On the EXTERNAL +# block path, so it needs --kv-transfer-config to appear. NOTE +# --no-async-scheduling was tested and does NOT help (c12 died at 490 s). +patch_kv_blockpool() { + local label="kv-blockpool" + local target; target=$(_modfile vllm.v1.core.single_type_kv_cache_manager) + if [ -z "$target" ] || [ ! -f "$target" ]; then + echo "[$label] target not found; skipping."; return 0 + fi + # NB: the marker must be unique to OUR patch. "num_new_blocks = max(" is NOT + # -- stock already has it at three other call sites (lines ~208/1511/1601), + # so using it silently skipped the patch on a clean image. + if grep -q "KIMI-PATCH-KV-BLOCKPOOL" "$target"; then + echo "[$label] already patched."; return 0 + fi + cp -n "$target" "$target.orig" 2>/dev/null || true + $PY - "$target" <<'EOF' || echo "[kv-blockpool] patch failed; unchanged." >&2 +import sys, io +p = sys.argv[1] +src = io.open(p, encoding="utf-8").read() +old = """ req_blocks = self.req_to_blocks[request_id] + allocated_blocks = self.block_pool.get_new_blocks( + cdiv(num_total_computed_tokens, self.block_size) - len(req_blocks) + )""" +new = """ req_blocks = self.req_to_blocks[request_id] + # KIMI-PATCH-KV-BLOCKPOOL: clamp to >= 0; a negative count silently + # inflates FreeKVCacheBlockQueue.num_free_blocks and corrupts the free list. + num_new_blocks = max( + 0, cdiv(num_total_computed_tokens, self.block_size) - len(req_blocks) + ) + allocated_blocks = self.block_pool.get_new_blocks(num_new_blocks)""" +if src.count(old) != 1: + sys.stderr.write("[kv-blockpool] anchor missing or not unique; aborting.\n") + sys.exit(1) +io.open(p, "w", encoding="utf-8").write(src.replace(old, new)) +print("[kv-blockpool] patched", p) +EOF +} + +# Per-patch switches, so a single patch can be isolated without disabling the +# others. Note patch [1] is load-bearing: without it ROCM_AITER_FA prefill dies +# at warmup with the fmha_fwd_bf16_opus TypeError, so skipping it does not give +# a clean baseline -- it gives a different crash. +# SKIP_PATCH_AITER=1 skip [1] aiter pybind11 +# SKIP_PATCH_CUDAGRAPH=1 skip [2] TritonMLA UNIFORM_BATCH <- the HIP-999 suspect +# SKIP_PATCH_BLOCKPOOL=1 skip [3] KV block-pool clamp +echo "[kimi-patches] applying in-container patches..." +if [ "${SKIP_PATCH_AITER:-0}" = "1" ]; then + echo "[aiter-pybind11] SKIPPED via SKIP_PATCH_AITER=1" +else + patch_aiter_pybind11 || true +fi +if [ "${SKIP_PATCH_CUDAGRAPH:-0}" = "1" ]; then + echo "[triton-mla-cudagraph] SKIPPED via SKIP_PATCH_CUDAGRAPH=1" +else + patch_triton_mla_cudagraph || true +fi +if [ "${SKIP_PATCH_BLOCKPOOL:-0}" = "1" ]; then + echo "[kv-blockpool] SKIPPED via SKIP_PATCH_BLOCKPOOL=1" +else + patch_kv_blockpool || true +fi +echo "[kimi-patches] done." diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 9851e8d6d0..1a36fa3ea1 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -2030,6 +2030,62 @@ qwen3.5-fp8-mi355x-sglang-agentic-hicache: # CONC ranges mirror dsv4-fp4-b200-vllm-agentic for cross-hardware # comparability. Offload sweep is none-only (SGLang has no equivalent of # vLLM's SimpleCPUOffloadConnector path that we exercise on b200). + +# Kimi-K3 MXFP4 1P1D AgentX: DSpark speculative decoding, MoRIIO KV transfer, +# and optional SimpleCPUOffloadConnector DRAM tier on prefill. +kimik3-fp4-mi355x-vllm-disagg-agentic-mtp: + image: vllm/vllm-openai-rocm:nightly-ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:mi355x-amds + precision: fp4 + framework: vllm-disagg + router: { name: vllm-router, version: "nightly-20260809-d2ba586" } + kv-p2p-transfer: mori + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.50 + search-space: + - spec-decoding: mtp + kv-offloading: none + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=2" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [8, 12, 16] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=2" + dsv4-fp4-mi355x-vllm-agentic: image: vllm/vllm-openai-rocm:v0.22.0 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 336edc9a61..d96fc880b3 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4589,3 +4589,11 @@ - "Image: lmsysorg/sglang:v0.5.12" - "14 topologies across 1k/1k and 8k/1k: prefill TP8 STP + decode wide-EP (DEP16/DEP32 high-throughput) and per-node TP8 low-latency, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1895 + +- config-keys: + - kimik3-fp4-mi355x-vllm-disagg-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add Kimi-K3 MI355X vLLM 1P1D disaggregated AgentX with DSpark, MoRIIO KV transfer, and optional DRAM offload." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2602 diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 8cb92b7a16..486e86c8aa 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -58,8 +58,17 @@ if [[ "$IS_MULTINODE" == "true" ]]; then cleanup_and_save_logs() { if [[ -n "${GITHUB_ACTIONS:-}" && -n "${JOB_ID:-}" ]]; then local art_dir="$GITHUB_WORKSPACE/benchmark_artifacts" + local job_logs_dir="$BENCHMARK_LOGS_DIR/logs/slurm_job-${JOB_ID}" + local server_logs_dir="$BENCHMARK_LOGS_DIR/server_logs/slurm_job-${JOB_ID}" mkdir -p "$art_dir" cp -r "$BENCHMARK_LOGS_DIR"/slurm_job-${JOB_ID}.{out,err} "$art_dir/" 2>/dev/null || true + if [[ -d "$server_logs_dir" && ! -f "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" ]]; then + tar czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" \ + -C "$server_logs_dir" . 2>/dev/null || true + elif [[ -d "$job_logs_dir" && ! -f "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" ]]; then + tar czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" \ + -C "$job_logs_dir" . 2>/dev/null || true + fi fi # Print .err inline so failures are visible in CI output local err_file="$BENCHMARK_LOGS_DIR/slurm_job-${JOB_ID:-unknown}.err" @@ -78,7 +87,15 @@ if [[ "$IS_MULTINODE" == "true" ]]; then SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_mi355x_${FRAMEWORK}.sh" if [[ "$FRAMEWORK" == "sglang-disagg" ]] || [[ "$FRAMEWORK" == "vllm-disagg" ]] || [[ "$FRAMEWORK" == "atom-disagg" ]]; then - BENCHMARK_SUBDIR="multi_node" + # Agentic recipes live under multi_node/agentic/ and export the + # HiCache tunables (page-size, io-backend, ...); fixed-seq-len recipes + # live at the multi_node/ root. Honor SCENARIO_SUBDIR so agentic-coding + # configs pick the agentic recipe instead of the root one. + if [[ "${SCENARIO_SUBDIR}" == "agentic/" ]]; then + BENCHMARK_SUBDIR="multi_node/agentic" + else + BENCHMARK_SUBDIR="multi_node" + fi else BENCHMARK_SUBDIR="single_node/fixed_seq_len" fi @@ -126,7 +143,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # search for "FRAMEWORK_DIFF_IF_STATEMENT #3" for this if-statement # Find the latest log directory that contains the data - if [[ "${EVAL_ONLY:-false}" != "true" ]]; then + if [[ "${EVAL_ONLY:-false}" != "true" && "${IS_AGENTIC:-0}" != "1" ]]; then cat > collect_latest_results.py <<'PY' import os, sys job_dir, isl, osl, nexp, framework = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4]), sys.argv[5] @@ -172,8 +189,18 @@ PY shopt -s nullglob for eval_file in "$EVAL_DIR"/*; do [ -f "$eval_file" ] || continue - cp "$eval_file" "$GITHUB_WORKSPACE/" - echo "Copied eval artifact: $(basename "$eval_file")" + eval_dest="$GITHUB_WORKSPACE/$(basename "$eval_file")" + rm -f "$eval_dest" + # Eval artifacts are created as root inside the container; sudo + # is required to overwrite any stale root-owned files in the + # workspace from prior runs on this runner. + if sudo cp "$eval_file" "$eval_dest"; then + sudo chown "$(id -u):$(id -g)" "$eval_dest" 2>/dev/null || true + echo "Copied eval artifact: $(basename "$eval_file")" + else + echo "ERROR: failed to copy eval artifact: $(basename "$eval_file")" >&2 + exit 1 + fi done shopt -u nullglob else @@ -181,6 +208,48 @@ PY fi fi + # Stage agentic raw artifacts + server logs for the CI upload steps. + # server_sglang.sh copies /run_logs/slurm_job- to + # $BENCHMARK_LOGS_DIR/logs/slurm_job- on shared storage, and + # trace_replay.sh writes each concurrency's aiperf artifacts under + # agentic/conc_/ (mirroring agentic_srt.sh). benchmark-multinode-tmpl.yml + # uploads them from $GITHUB_WORKSPACE/LOGS/agentic/conc_*/... plus a + # multinode_server_logs.tar.gz, so preserve the conc_/ nesting here + # before the logs dir is removed below. The agg result JSON is already + # written straight to the mounted workspace by the existing agentic + # aggregation module. + if [[ "${IS_AGENTIC:-0}" == "1" ]]; then + JOB_LOGS_DIR="$BENCHMARK_LOGS_DIR/logs/slurm_job-${JOB_ID}" + if [ -d "$JOB_LOGS_DIR" ]; then + # trace_replay.sh always nests artifacts under agentic/conc_/. + # Copy the whole agentic/ tree so the conc_/ subdirs are + # preserved for the LOGS/agentic/conc_*/... upload globs. + AGENTIC_SRC="$JOB_LOGS_DIR/agentic" + if [ -d "$AGENTIC_SRC" ] && find "$AGENTIC_SRC" -mindepth 1 -maxdepth 1 -type d -name 'conc_*' -print -quit 2>/dev/null | grep -q .; then + echo "Staging agentic raw artifacts from $AGENTIC_SRC" + mkdir -p "$GITHUB_WORKSPACE/LOGS/agentic" + cp -r "$AGENTIC_SRC"/. "$GITHUB_WORKSPACE/LOGS/agentic/" + # Container artifacts arrive root-owned; chown/chmod so git clean + # and later jobs (possibly a different runner user) can remove LOGS/. + sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE/LOGS" 2>/dev/null || true + chmod -R a+rwX "$GITHUB_WORKSPACE/LOGS" 2>/dev/null || true + ls -laR "$GITHUB_WORKSPACE/LOGS/agentic" + else + echo "WARNING: no agentic conc_*/ artifacts found under $JOB_LOGS_DIR/agentic" + fi + # Server/router/prefill/decode logs for the multinode_server_logs_* artifact. + SERVER_LOGS_DIR="$BENCHMARK_LOGS_DIR/server_logs/slurm_job-${JOB_ID}" + [[ -d "$SERVER_LOGS_DIR" ]] || SERVER_LOGS_DIR="$JOB_LOGS_DIR" + if tar czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" -C "$SERVER_LOGS_DIR" . 2>/dev/null; then + echo "Created multinode_server_logs.tar.gz" + else + echo "WARNING: failed to create multinode_server_logs.tar.gz" + fi + else + echo "WARNING: agentic staging skipped; $JOB_LOGS_DIR not found" + fi + fi + echo "All result files processed" # Use sync scancel to ensure nfs file handle is released in time set +x @@ -268,6 +337,7 @@ else --container-remap-root \ --no-container-entrypoint --export=ALL,AIPERF_DATASET_MMAP_CACHE_DIR=/aiperf_mmap_cache \ bash "$BENCHMARK_SCRIPT" + benchmark_rc=$? scancel $JOB_ID @@ -275,4 +345,6 @@ else echo "gpucore files exist. not good" rm -f gpucore.* fi + + exit "$benchmark_rc" fi